├── pydep ├── __init__.py ├── testdata │ ├── ex0 │ │ ├── requirements.txt │ │ └── setup.py │ ├── ex_nosetuppy │ │ └── requirements.txt │ ├── ex_nothing │ │ └── explain.txt │ ├── ex_requirementstxtvcs │ │ └── requirements.txt │ ├── ex_norequirementstxt │ │ └── setup.py │ ├── ex_weirdsetuppy │ │ └── setup.py │ └── ex_setuppy_prints │ │ └── setup.py ├── util.py ├── vcs.py ├── vcs_test.py ├── setup_py.py ├── req_test.py └── req.py ├── .gitignore ├── setup.py ├── LICENSE ├── README.md └── pydep-run.py /pydep/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | venv/ 3 | 4 | -------------------------------------------------------------------------------- /pydep/testdata/ex0/requirements.txt: -------------------------------------------------------------------------------- 1 | dep1 2 | dep2==0.0 3 | dep3[blah]>=0.0.4 4 | -------------------------------------------------------------------------------- /pydep/testdata/ex_nosetuppy/requirements.txt: -------------------------------------------------------------------------------- 1 | dep1 2 | dep2==0.0 3 | dep3[blah]>=0.0.4 4 | -------------------------------------------------------------------------------- /pydep/testdata/ex_nothing/explain.txt: -------------------------------------------------------------------------------- 1 | this represents a repository that has no requirements.txt or setup.py 2 | -------------------------------------------------------------------------------- /pydep/testdata/ex_requirementstxtvcs/requirements.txt: -------------------------------------------------------------------------------- 1 | git+https://github.com/foo/bar 2 | hg+https://code.google.com/p/foo 3 | foo 4 | -------------------------------------------------------------------------------- /pydep/testdata/ex_norequirementstxt/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='ex0', 5 | version='0.1.2', 6 | url='http://github.com/fake/fake', 7 | packages=['ex0'], 8 | install_requires=[ 9 | 'dep1', 10 | 'dep2==0.0', 11 | 'dep3[blah]>=0.0.4', 12 | ], 13 | ) 14 | -------------------------------------------------------------------------------- /pydep/testdata/ex0/setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | A package representative of Python dependency management best practices. 3 | """ 4 | 5 | from setuptools import setup 6 | 7 | setup( 8 | name='ex0', 9 | version='0.1.2', 10 | url='http://github.com/fake/fake', 11 | packages=['ex0'], 12 | install_requires=[ 13 | 'dep1', 14 | 'dep2==0.0', 15 | 'dep3[blah]>=0.0.4', 16 | ], 17 | ) 18 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='pydep', 5 | version='0.0', 6 | url='http://github.com/sourcegraph/pydep', 7 | packages=['pydep'], 8 | scripts=['pydep-run.py'], 9 | author='Beyang Liu', 10 | description='A simple module that will print the dependencies of a python project' 11 | 'Usage: python -m pydep ', 12 | zip_safe=False, 13 | install_requires=[ 14 | "pip==7.1.2", 15 | ], 16 | ) 17 | -------------------------------------------------------------------------------- /pydep/util.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import stat 4 | import sys 5 | 6 | 7 | def rmtree(dir): 8 | """ 9 | Recursivly removes files in a given dir. 10 | On Windows tries to remove readonly files (i.e. contents of .git) 11 | """ 12 | if sys.platform == 'win32' or sys.platform == 'cygwin' or sys.platform == 'msys': 13 | shutil.rmtree(dir, onerror =__fix_read_only) 14 | else: 15 | shutil.rmtree(dir) 16 | 17 | def __fix_read_only(fn, path, excinfo): 18 | os.chmod(path, stat.S_IWRITE) 19 | fn(path) 20 | -------------------------------------------------------------------------------- /pydep/testdata/ex_weirdsetuppy/setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | This setup.py executes in a `if __name__ == '__main__':` clause 3 | (this is why we use `runpy.run_path` instead of `import` in setup_py.py) 4 | 5 | """ 6 | 7 | from setuptools import setup 8 | 9 | if __name__ == '__main__': 10 | setup( 11 | name='ex0', 12 | version='0.1.2', 13 | url='http://github.com/fake/fake', 14 | packages=['ex0'], 15 | install_requires=[ 16 | 'dep1', 17 | 'dep2==0.0', 18 | 'dep3[blah]>=0.0.4', 19 | ], 20 | ) 21 | -------------------------------------------------------------------------------- /pydep/testdata/ex_setuppy_prints/setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | A package representative of Python dependency management best practices. 3 | """ 4 | 5 | from setuptools import setup 6 | import subprocess 7 | 8 | print('THIS SHOULD NOT APPEAR IN OUTPUT (but will in test)') 9 | subprocess.call(['echo', 'THIS SHOULD NOT APPEAR IN OUTPUT EITHER (but will in test)']) 10 | 11 | setup( 12 | name='ex0', 13 | version='0.1.2', 14 | url='http://github.com/fake/fake', 15 | packages=['ex0'], 16 | install_requires=[ 17 | 'dep1', 18 | 'dep2==0.0', 19 | 'dep3[blah]>=0.0.4', 20 | ], 21 | ) 22 | -------------------------------------------------------------------------------- /pydep/vcs.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | repo_url_patterns = [ 4 | r'(?:git\+)?((?:https?|git)\://github.com/(?:[^/#]+)/(?:[^/#]+))(?:/.*)?', 5 | r'(?:git\+|hg\+)?((?:https?|git|hg)\://bitbucket.org/(?:[^/#]+)/(?:[^/#]+))(?:/.*)?', 6 | r'(?:git\+|hg\+)?((?:https?|git|hg)\://code.google.com/p/(?:[^/#]+))(?:/.*)?', 7 | ] 8 | 9 | 10 | def parse_repo_url(url): 11 | """Returns the canonical repository clone URL from a string that contains it""" 12 | for pattern in repo_url_patterns: 13 | match = re.match(pattern, url) 14 | if match is not None: 15 | return match.group(1) 16 | return None 17 | 18 | def parse_repo_url_and_revision(url): 19 | """Returns the canonical repository clone URL and revision from a string that contains it""" 20 | full_url = parse_repo_url(url) 21 | if full_url is None: 22 | return url, '' # fall back to returning the full URL 23 | components = full_url.split('@') 24 | if len(components) == 2: 25 | return components[0], components[1] 26 | elif len(components) == 1: 27 | return components[0], '' 28 | return full_url, '' # fall back to returning the full URL 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Sourcegraph, Inc. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /pydep/vcs_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from pydep.vcs import parse_repo_url, parse_repo_url_and_revision 3 | 4 | 5 | class TestVCS(unittest.TestCase): 6 | def test_parse_repo_url(self): 7 | testcases = [ 8 | ('http://github.com/foo/bar', 'http://github.com/foo/bar'), 9 | ('https://github.com/foo/bar/baz', 'https://github.com/foo/bar'), 10 | ('git+https://github.com/foo/bar/baz', 'https://github.com/foo/bar'), 11 | ('git+git://github.com/foo/bar.git#egg=foo', 'git://github.com/foo/bar.git'), 12 | 13 | ('https://code.google.com/p/bar', 'https://code.google.com/p/bar'), 14 | ('https://code.google.com/p/bar/blah/blah', 'https://code.google.com/p/bar'), 15 | ('git+https://code.google.com/p/bar/blah/blah', 'https://code.google.com/p/bar'), 16 | ('git+git://code.google.com/p/bar.git#egg=foo', 'git://code.google.com/p/bar.git'), 17 | 18 | ('https://bitbucket.org/foo/bar', 'https://bitbucket.org/foo/bar'), 19 | ('https://bitbucket.org/foo/bar/baz', 'https://bitbucket.org/foo/bar'), 20 | ('hg+https://bitbucket.org/foo/bar', 'https://bitbucket.org/foo/bar'), 21 | ('git+git://bitbucket.org/foo/bar', 'git://bitbucket.org/foo/bar'), 22 | 23 | ('https://google.com', None), 24 | 25 | ('file:///tmp/', None), 26 | ] 27 | 28 | for testcase in testcases: 29 | self.assertEqual(testcase[1], parse_repo_url(testcase[0])) 30 | 31 | def test_parse_repo_url_and_revision(self): 32 | testcases = [ 33 | ('http://github.com/foo/bar', 'http://github.com/foo/bar', ''), 34 | ('http://github.com/foo/bar@12345', 'http://github.com/foo/bar', '12345'), 35 | ('file:///tmp/', 'file:///tmp/', ''), 36 | ] 37 | 38 | for testcase in testcases: 39 | (url, rev) = parse_repo_url_and_revision(testcase[0]) 40 | self.assertEqual(testcase[1], url) 41 | self.assertEqual(testcase[2], rev) 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | pydep 2 | ===== 3 | 4 | `pydep` is a simple module / command line tool that will print the dependencies of a python project.
5 | `pydep` is still under active development and should be considered "alpha"-quality software. 6 | 7 | Install 8 | ----- 9 | ``` 10 | pip install pydep # install latest release 11 | pip install git+git://github.com/sourcegraph/pydep # install from dev master 12 | ``` 13 | 14 | Usage 15 | ----- 16 | 17 | ``` 18 | pydep-run.py -h # print out options 19 | pydep-run.py dep # run pydep on directory 20 | pydep-run.py demo # print some info out that demonstrates capabilities 21 | ``` 22 | 23 | For example, 24 | ``` 25 | > pip install pydep 26 | > git clone https://github.com/mitsuhiko/flask 27 | > cd flask 28 | > pydep-run.py dep . 29 | [{"resolved": true, "project_name": "Werkzeug", "unsafe_name": "Werkzeug", "key": "werkzeug", "modules": null, "packages": ["werkzeug", "werkzeug.debug", "werkzeug.contrib", "werkzeug.testsuite", "werkzeug.testsuite.contrib"], 30 | "type": "setuptools", "specs": [[">=", "0.7"]], "repo_url": "git://github.com/mitsuhiko/werkzeug", "extras": []}, 31 | {"resolved": true, "project_name": "Jinja2", "unsafe_name": "Jinja2", "key": "jinja2", "modules": null, "packages": 32 | ["jinja2", "jinja2.testsuite", "jinja2.testsuite.res"], "type": "setuptools", "specs": [[">=", "2.4"]], "repo_url": 33 | "git://github.com/mitsuhiko/jinja2", "extras": []}, {"resolved": true, "project_name": "itsdangerous", "unsafe_name": 34 | "itsdangerous", "key": "itsdangerous", "modules": ["itsdangerous"], "packages": null, "type": "setuptools", "specs": 35 | [[">=", "0.21"]], "repo_url": "http://github.com/mitsuhiko/itsdangerous", "extras": []}, {"resolved": true, 36 | "project_name": "click", "unsafe_name": "click", "key": "click", "modules": null, "packages": ["click"], "type": 37 | "setuptools", "specs": [[">=", "0.6"]], "repo_url": "http://github.com/mitsuhiko/click", "extras": []}] 38 | ``` 39 | 40 | Additional requirements 41 | ----- 42 | - pip 7.0+ 43 | - curl, unzip, gunzip, tar 44 | 45 | Tests 46 | ----- 47 | Install nose (`pip install nose`). Then, `nosetests -s` from the root repository directory. 48 | 49 | Contributing 50 | ------------ 51 | Make a pull request! 52 | -------------------------------------------------------------------------------- /pydep/setup_py.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import runpy 4 | from os import path 5 | 6 | 7 | PY2 = sys.version_info[0] == 2 8 | 9 | 10 | def setup_dirs(container_dir): 11 | """ 12 | Given a directory that may contain Python packages (defined by setup.py 13 | files), returns a list of package root directories (i.e., directories that 14 | contain a setup.py) 15 | """ 16 | rootdirs = [] 17 | for dirpath, dirnames, filenames in os.walk(container_dir): 18 | for filename in filenames: 19 | if filename == 'setup.py': 20 | rootdirs.append(dirpath) 21 | break 22 | 23 | # Clean up unwanted subdirectories before next walk. 24 | newnames = [] 25 | for dirname in dirnames: 26 | if dirname.startswith(".") or dirname.startswith("virtualenv_") or dirname == "testdata": 27 | continue 28 | newnames.append(dirname) 29 | dirnames[:] = newnames 30 | 31 | return rootdirs 32 | 33 | 34 | def setup_info_dir(rootdir): 35 | """ 36 | Returns (metadata, error_string) tuple. error_string is None if no error. 37 | """ 38 | setupfile = path.join(rootdir, 'setup.py') 39 | if not path.exists(setupfile): 40 | return None, setupfile + ' does not exist' 41 | return setup_info(setupfile), None 42 | 43 | 44 | def setup_info(setupfile): 45 | """Returns metadata for a PyPI package by running its setupfile""" 46 | setup_dict = {} 47 | 48 | def setup_replacement(**kw): 49 | iter = kw.iteritems() if PY2 else kw.items() 50 | for k, v in iter: 51 | setup_dict[k] = v 52 | 53 | setuptools_mod = __import__('setuptools') 54 | import distutils.core # for some reason, __import__('distutils.core') doesn't work 55 | 56 | # Mod setup() 57 | old_setuptools_setup = setuptools_mod.setup 58 | setuptools_mod.setup = setup_replacement 59 | old_distutils_setup = distutils.core.setup 60 | distutils.core.setup = setup_replacement 61 | # Mod sys.path (changing sys.path is necessary in addition to changing the working dir, 62 | # because of Python's import resolution order) 63 | old_sys_path = list(sys.path) 64 | sys.path.insert(0, path.dirname(setupfile)) 65 | # Change working dir (necessary because some setup.py files read relative paths from the filesystem) 66 | old_wd = os.getcwd() 67 | os.chdir(path.dirname(setupfile)) 68 | # Redirect stdout to stderr (*including for subprocesses*) 69 | old_sys_stdout = sys.stdout # redirects in python process 70 | sys.stdout = sys.stderr 71 | old_stdout = os.dup(1) # redirects in subprocesses 72 | stderr_dup = os.dup(2) 73 | os.dup2(stderr_dup, 1) 74 | 75 | try: 76 | runpy.run_path(path.basename(setupfile), run_name='__main__') 77 | finally: 78 | # Restore stdout 79 | os.dup2(old_stdout, 1) # restores for subprocesses 80 | os.close(stderr_dup) 81 | sys.stdout = old_sys_stdout # restores for python process 82 | # Restore working dir 83 | os.chdir(old_wd) 84 | # Restore sys.path 85 | sys.path = old_sys_path 86 | # Restore setup() 87 | distutils.core.setup = old_distutils_setup 88 | setuptools_mod.setup = old_setuptools_setup 89 | 90 | return setup_dict 91 | -------------------------------------------------------------------------------- /pydep-run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import json 5 | import argparse 6 | import pydep.req 7 | import pydep.setup_py 8 | import pydep.util 9 | 10 | from os import path, devnull 11 | import subprocess 12 | import tempfile 13 | import shutil 14 | 15 | 16 | def main(): 17 | # Parse args 18 | argparser = argparse.ArgumentParser( 19 | description='pydep is simple command line tool that tells you about package dependency metadata in Python') 20 | subparsers = argparser.add_subparsers() 21 | 22 | dep_parser = subparsers.add_parser('dep', help='print the dependencies of a python project in JSON') 23 | dep_parser.add_argument('--raw', action='store_true', 24 | help='If true, pydep will not try to resolve dependencies to VCS URLs') 25 | dep_parser.add_argument('dir', help='path to root directory of project code') 26 | dep_parser.set_defaults(func=dep) 27 | 28 | info_parser = subparsers.add_parser('info', help='print metadata of a python project in JSON') 29 | info_parser.add_argument('dir', help='path to root directory of project code') 30 | info_parser.set_defaults(func=info) 31 | 32 | list_info_parser = subparsers.add_parser('list', help='print metadata of all python projects in a directory') 33 | list_info_parser.add_argument('dir', help='path to containing directory') 34 | list_info_parser.set_defaults(func=list_info) 35 | 36 | smoke_parser = subparsers.add_parser('demo', 37 | help='run pydep against some popular repositories, printing out dependency' 38 | ' information about each') 39 | smoke_parser.set_defaults(func=smoke_test) 40 | 41 | args = argparser.parse_args() 42 | args.func(args) 43 | 44 | 45 | # 46 | # Sub-commands 47 | # 48 | 49 | 50 | def list_info(args): 51 | """Subcommand to print out metadata of all packages contained in a directory""" 52 | container_dir = args.dir 53 | setup_dirs = pydep.setup_py.setup_dirs(container_dir) 54 | setup_infos = [] 55 | for setup_dir in setup_dirs: 56 | setup_dict, err = pydep.setup_py.setup_info_dir(setup_dir) 57 | if err is not None: 58 | sys.stderr.write('failed due to error: %s\n' % err) 59 | sys.exit(1) 60 | setup_infos.append( 61 | setup_dict_to_json_serializable_dict(setup_dict, rootdir=path.relpath(setup_dir, container_dir))) 62 | print(json.dumps(setup_infos)) 63 | 64 | 65 | def info(args): 66 | """Subcommand to print out metadata of package""" 67 | setup_dict, err = pydep.setup_py.setup_info_dir(args.dir) 68 | if err is not None: 69 | sys.stderr.write('failed due to error: %s\n' % err) 70 | sys.exit(1) 71 | print(json.dumps(setup_dict_to_json_serializable_dict(setup_dict))) 72 | 73 | 74 | def dep(args): 75 | """Subcommand to print out dependencies of project""" 76 | reqs, err = pydep.req.requirements(args.dir, not args.raw) 77 | if err is not None: 78 | sys.stderr.write('failed due to error: %s\n' % err) 79 | sys.exit(1) 80 | print(json.dumps(reqs)) 81 | 82 | 83 | def smoke_test(args): 84 | """Test subcommand that runs pydep on a few popular repositories and prints the results.""" 85 | testcases = [ 86 | ('Flask', 'https://github.com/mitsuhiko/flask.git'), 87 | ('Graphite, a webapp that depends on Django', 'https://github.com/graphite-project/graphite-web'), 88 | # TODO: update smoke_test to call setup_dirs/list instead of assuming setup.py exists at the repository root 89 | # ('Node', 'https://github.com/joyent/node.git'), 90 | ] 91 | tmpdir = None 92 | try: 93 | tmpdir = tempfile.mkdtemp() 94 | for title, cloneURL in testcases: 95 | print('Downloading and processing %s...' % title) 96 | subdir = path.splitext(path.basename(cloneURL))[0] 97 | dir_ = path.join(tmpdir, subdir) 98 | with open(devnull , 'w') as handle: 99 | subprocess.call(['git', 'clone', cloneURL, dir_], stdout=handle, stderr=handle) 100 | 101 | print('') 102 | reqs, err = pydep.req.requirements(dir_, True) 103 | if err is None: 104 | print('Here is some info about the dependencies of %s' % title) 105 | if len(reqs) == 0: 106 | print('(There were no dependencies found for %s)' % title) 107 | else: 108 | print(json.dumps(reqs, indent=2)) 109 | else: 110 | print('failed with error: %s' % err) 111 | 112 | print('') 113 | setup_dict, err = pydep.setup_py.setup_info_dir(dir_) 114 | if err is None: 115 | print('Here is the metadata for %s' % title) 116 | print(json.dumps(setup_dict_to_json_serializable_dict(setup_dict), indent=2)) 117 | else: 118 | print('failed with error: %s' % err) 119 | 120 | except Exception as e: 121 | print('failed with exception %s' % str(e)) 122 | finally: 123 | if tmpdir: 124 | pydep.util.rmtree(tmpdir) 125 | 126 | # 127 | # Helpers 128 | # 129 | 130 | 131 | def setup_dict_to_json_serializable_dict(d, **kw): 132 | return { 133 | 'rootdir': kw['rootdir'] if 'rootdir' in kw else None, 134 | 'project_name': d['name'] if 'name' in d else None, 135 | 'version': d['version'] if 'version' in d else None, 136 | 'repo_url': d['url'] if 'url' in d else None, 137 | 'packages': d['packages'] if 'packages' in d else None, 138 | 'modules': d['py_modules'] if 'py_modules' in d else None, 139 | 'scripts': d['scripts'] if 'scripts' in d else None, 140 | 'author': d['author'] if 'author' in d else None, 141 | 'description': d['description'] if 'description' in d else None, 142 | } 143 | 144 | 145 | if __name__ == '__main__': 146 | main() 147 | -------------------------------------------------------------------------------- /pydep/req_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from pip import req as req 3 | 4 | from pydep.req import * 5 | 6 | testdatadir = path.join(path.dirname(__file__), 'testdata') 7 | 8 | 9 | class TestRequirements(unittest.TestCase): 10 | def setUp(self): 11 | self.maxDiff = None 12 | 13 | def test_requirements(self): 14 | """ 15 | Tests the requirements() function (the main external function of this package). 16 | The purpose of this test is to test that the correct dependency extraction method is applied, NOT to 17 | test the correctness of the individual dependency extraction methods. 18 | """ 19 | expected0 = [ 20 | {'extras': (), 21 | 'key': 'dep2', 22 | 'modules': None, 23 | 'packages': None, 24 | 'project_name': 'dep2', 25 | 'repo_url': None, 26 | 'resolved': False, 27 | 'specs': [('==', '0.0')], 28 | 'type': 'setuptools', 29 | 'unsafe_name': 'dep2'}, 30 | {'extras': (), 31 | 'key': 'dep1', 32 | 'modules': None, 33 | 'packages': None, 34 | 'project_name': 'dep1', 35 | 'repo_url': None, 36 | 'resolved': False, 37 | 'specs': [], 38 | 'type': 'setuptools', 39 | 'unsafe_name': 'dep1'}, 40 | {'extras': ('blah',), 41 | 'key': 'dep3', 42 | 'modules': None, 43 | 'packages': None, 44 | 'project_name': 'dep3', 45 | 'repo_url': None, 46 | 'resolved': False, 47 | 'specs': [('>=', '0.0.4')], 48 | 'type': 'setuptools', 49 | 'unsafe_name': 'dep3'} 50 | ] 51 | testcases = [ 52 | ('ex0', expected0), 53 | ('ex_nosetuppy', expected0), 54 | ('ex_norequirementstxt', expected0), 55 | ('ex_weirdsetuppy', expected0), 56 | ('ex_setuppy_prints', expected0), 57 | ('ex_nothing', '<>'), 58 | ('ex_requirementstxtvcs', [ 59 | {'key': 'https://code.google.com/p/foo', 60 | 'repo_url': 'https://code.google.com/p/foo', 61 | 'type': 'vcs', 62 | 'modules': None, 'extras': None, 'packages': None, 'project_name': None, 'resolved': False, 63 | 'specs': None, 'unsafe_name': None}, 64 | {'key': 'https://github.com/foo/bar', 65 | 'repo_url': 'https://github.com/foo/bar', 66 | 'type': 'vcs', 'extras': None, 'modules': None, 'packages': None, 'project_name': None, 67 | 'resolved': False, 'specs': None, 'unsafe_name': None}, 68 | {'extras': (), 69 | 'key': 'foo', 70 | 'project_name': 'foo', 71 | 'type': 'setuptools', 72 | 'unsafe_name': 'foo', 73 | 'modules': None, 74 | 'packages': None, 75 | 'repo_url': None, 76 | 'resolved': False, 77 | 'specs': []} 78 | ]), 79 | ] 80 | for testcase in testcases: 81 | dir_, exp = testcase[0], testcase[1] 82 | rootdir = path.join(testdatadir, dir_) 83 | reqs, err = requirements(rootdir, resolve=False) 84 | if err is not None: 85 | if exp != '<>': 86 | print('unexpected error: ', err) 87 | self.assertEqual(exp, '<>') 88 | else: 89 | self.assertListEqual(sorted(exp, key=lambda x: x['key']), sorted(reqs, key=lambda x: x['key'])) 90 | 91 | def test_SetupToolsRequirement(self): 92 | testcases = [ 93 | ("foo==0.0.0", { 94 | 'extras': (), 95 | 'key': 'foo', 96 | 'modules': None, 97 | 'packages': None, 98 | 'project_name': 'foo', 99 | 'repo_url': None, 100 | 'resolved': False, 101 | 'specs': [('==', '0.0.0')], 102 | 'type': 'setuptools', 103 | 'unsafe_name': 'foo' 104 | }), 105 | ("foo[bar]>=0.1b", { 106 | 'extras': ('bar',), 107 | 'key': 'foo', 108 | 'modules': None, 109 | 'packages': None, 110 | 'project_name': 'foo', 111 | 'repo_url': None, 112 | 'resolved': False, 113 | 'specs': [('>=', '0.1b')], 114 | 'type': 'setuptools', 115 | 'unsafe_name': 'foo' 116 | }), 117 | ] 118 | for testcase in testcases: 119 | req_str, exp_dict = testcase[0], testcase[1] 120 | st_req = SetupToolsRequirement(pr.Requirement.parse(req_str)) 121 | self.assertDictEqual(exp_dict, st_req.to_dict()) 122 | 123 | def test_PipVCSInstallRequirement(self): 124 | requirements_str = """ 125 | git+https://github.com/foo/bar 126 | git+https://code.google.com/p/foo 127 | git+git://code.google.com/p/foo#egg=bar 128 | """ 129 | expected = [ 130 | { 131 | 'type': 'vcs', 132 | 'key': 'https://github.com/foo/bar', 133 | 'repo_url': 'https://github.com/foo/bar', 134 | 'unsafe_name': None, 'extras': None, 'modules': None, 'packages': None, 'project_name': None, 135 | 'resolved': False, 'specs': None, 136 | }, 137 | { 138 | 'type': 'vcs', 139 | 'key': 'https://code.google.com/p/foo', 140 | 'repo_url': 'https://code.google.com/p/foo', 141 | 'unsafe_name': None, 'extras': None, 'modules': None, 'packages': None, 'project_name': None, 142 | 'resolved': False, 'specs': None, 143 | }, 144 | { 145 | 'type': 'vcs', 146 | 'key': 'bar(git://code.google.com/p/foo)', 147 | 'repo_url': 'git://code.google.com/p/foo', 148 | 'unsafe_name': 'bar', 149 | 'project_name': 'bar', 150 | 'specs': [], 'extras': (), 'modules': None, 'packages': None, 'resolved': False, 151 | }, 152 | ] 153 | 154 | handle, requirements_file = tempfile.mkstemp() 155 | with open(requirements_file, 'w') as f: 156 | f.write(requirements_str) 157 | 158 | pip_reqs = req.parse_requirements(requirements_file, session=pip.download.PipSession()) 159 | reqs = [PipURLInstallRequirement(r).to_dict() for r in pip_reqs] 160 | os.fdopen(handle).close() 161 | os.remove(requirements_file) 162 | 163 | self.assertListEqual(expected, reqs) 164 | -------------------------------------------------------------------------------- /pydep/req.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains classes for dealing with pip and setuptools requirements classes 3 | """ 4 | 5 | import sys 6 | import pkg_resources as pr 7 | import pip 8 | import tempfile 9 | import shutil 10 | import subprocess 11 | import os 12 | import re 13 | from glob import glob 14 | from os import path 15 | 16 | from pydep import setup_py 17 | from pydep.util import rmtree 18 | from pydep.vcs import parse_repo_url, parse_repo_url_and_revision 19 | 20 | 21 | def requirements(rootdir, resolve=True): 22 | """Accepts the root directory of a PyPI package and returns its requirements. If 23 | both *requirements.txt and setup.py exist, it combines the dependencies 24 | defined in both, giving precedence to those defined in requirements.txt. 25 | Returns (requirements, error_string) tuple. error_string is None if no 26 | error. 27 | 28 | """ 29 | reqs = {} 30 | reqstxt_reqs, reqstxt_err = requirements_from_requirements_txt(rootdir) 31 | if reqstxt_err is None: 32 | for r in reqstxt_reqs: 33 | reqs[r.to_dict()['key']] = r 34 | setuppy_reqs, setuppy_err = requirements_from_setup_py(rootdir) 35 | if setuppy_err is None: 36 | for r in setuppy_reqs: 37 | if r.to_dict()['key'] not in reqs: 38 | reqs[r.to_dict()['key']] = r 39 | if reqstxt_err is not None and setuppy_err is not None: 40 | return None, 'could not get requirements due to 2 errors: %s, %s' % (reqstxt_err, setuppy_err) 41 | 42 | reqs = list(reqs.values()) 43 | 44 | if resolve: 45 | for req in reqs: 46 | err = req.resolve() 47 | if err is not None: 48 | sys.stderr.write('error resolving requirement %s: %s\n' % (str(req), err)) 49 | 50 | return [r.to_dict() for r in reqs], None 51 | 52 | 53 | def requirements_from_setup_py(rootdir): 54 | """ 55 | Accepts the root directory of a PyPI package and returns its requirements extracted from its setup.py 56 | Returns [], {'', None} 57 | """ 58 | setup_dict, err = setup_py.setup_info_dir(rootdir) 59 | if err is not None: 60 | return None, err 61 | 62 | reqs = [] 63 | if 'install_requires' in setup_dict: 64 | req_strs = setup_dict['install_requires'] 65 | for req_str in req_strs: 66 | reqs.append(SetupToolsRequirement(pr.Requirement.parse(req_str))) 67 | return reqs, None 68 | 69 | 70 | REQUIREMENTS_FILE_GLOB = '*requirements*.txt' 71 | 72 | 73 | def requirements_from_requirements_txt(rootdir): 74 | req_files = glob(path.join(rootdir, REQUIREMENTS_FILE_GLOB)) 75 | if len(req_files) == 0: 76 | return None, 'no requirements file found' 77 | 78 | all_reqs = {} 79 | for f in req_files: 80 | for install_req in pip.req.parse_requirements(f, session=pip.download.PipSession()): 81 | if install_req.link is not None: 82 | req = PipURLInstallRequirement(install_req) 83 | else: 84 | req = SetupToolsRequirement(install_req.req) 85 | all_reqs[str(req)] = req 86 | 87 | return all_reqs.values(), None 88 | 89 | 90 | class SetupToolsRequirement(object): 91 | """ 92 | This represents a standard python requirement as defined by setuptools (e.g., "mypkg>=0.0.1"). 93 | The constructor takes a pkg_resources.Requirement. 94 | """ 95 | def __init__(self, req): 96 | self.req = req 97 | self.metadata = None 98 | 99 | def __str__(self): 100 | return self.req.__str__() 101 | 102 | def to_dict(self): 103 | repo_url, py_modules, packages = None, None, None 104 | if self.metadata is not None: 105 | if 'url' in self.metadata: 106 | repo_url = parse_repo_url(self.metadata['url']) 107 | if repo_url is None and 'download_url' in self.metadata: 108 | repo_url = parse_repo_url(self.metadata['download_url']) 109 | py_modules = self.metadata['py_modules'] if 'py_modules' in self.metadata else None 110 | packages = self.metadata['packages'] if 'packages' in self.metadata else None 111 | 112 | return { 113 | 'type': 'setuptools', 114 | 'resolved': (self.metadata is not None), 115 | 'project_name': self.req.project_name, 116 | 'unsafe_name': self.req.unsafe_name, 117 | 'key': self.req.key, 118 | 'specs': self.req.specs, 119 | 'extras': self.req.extras, 120 | 'repo_url': repo_url, 121 | 'packages': packages, 122 | 'modules': py_modules, 123 | } 124 | 125 | def resolve(self): 126 | """ 127 | Downloads this requirement from PyPI and returns metadata from its setup.py. 128 | Returns an error string or None if no error. 129 | """ 130 | tmp_dir = tempfile.mkdtemp() 131 | with open(os.devnull, 'w') as devnull: 132 | try: 133 | cmd = ['install', '--quiet', 134 | '--download', tmp_dir, 135 | '--build', tmp_dir, 136 | '--no-clean', '--no-deps', 137 | '--no-binary', ':all:', str(self.req)] 138 | pip.main(cmd) 139 | except Exception as e: 140 | rmtree(tmp_dir) 141 | return 'error downloading requirement: {}'.format(str(e)) 142 | 143 | project_dir = path.join(tmp_dir, self.req.project_name) 144 | setup_dict, err = setup_py.setup_info_dir(project_dir) 145 | if err is not None: 146 | return None, err 147 | rmtree(tmp_dir) 148 | 149 | self.metadata = setup_dict 150 | return None 151 | 152 | 153 | class PipURLInstallRequirement(object): 154 | """ 155 | This represents a URL requirement as seen by pip (e.g., 'git+git://github.com/foo/bar/'). 156 | Such a requirement is not a valid requirement by setuptools standards. (In a setup.py, 157 | you would add the name/version of the requirement to install_requires as with PyPI packages, 158 | and then add the URL link to dependency_links. 159 | Also included archive files such as *.zip, *.tar, *.zip.gz, or *.tar.gz) 160 | The constructor takes a pip.req.InstallRequirement. 161 | """ 162 | _archive_regex = re.compile('^(http|https)://[^/]+/.+\.(zip|tar)(\.gz|)$', re.IGNORECASE) 163 | 164 | def __init__(self, install_req): 165 | self._install_req = install_req 166 | if install_req.link is None: 167 | raise 'No URL found in install_req: %s' % str(install_req) 168 | self.url, self.revision = parse_repo_url_and_revision(install_req.link.url) 169 | self.metadata = None 170 | self.vcs = None 171 | self.type = 'vcs' 172 | if install_req.link.url.find('+') >= 0: 173 | self.vcs = install_req.link.url[:install_req.link.url.find('+')] 174 | elif self._archive_regex.match(install_req.link.url) is not None: 175 | self.type = 'archive' 176 | self.setuptools_req = install_req.req # may be None 177 | 178 | def __str__(self): 179 | return self.url.__str__() 180 | 181 | def to_dict(self): 182 | project_name, unsafe_name, specs, extras, key = None, None, None, None, self.url 183 | if self.setuptools_req is not None: 184 | r = self.setuptools_req 185 | project_name, unsafe_name, specs, extras = r.project_name, r.unsafe_name, r.specs, r.extras 186 | key = '%s(%s)' % (r.key, self.url) 187 | 188 | py_modules, packages = None, None 189 | if self.metadata is not None: 190 | py_modules = self.metadata['py_modules'] if 'py_modules' in self.metadata else None 191 | packages = self.metadata['packages'] if 'packages' in self.metadata else None 192 | if project_name is None and 'name' in self.metadata: 193 | project_name = self.metadata['name'] 194 | 195 | return { 196 | 'type': self.type, 197 | 'resolved': (self.metadata is not None), 198 | 'project_name': project_name, 199 | 'unsafe_name': unsafe_name, 200 | 'key': key, 201 | 'specs': specs, 202 | 'extras': extras, 203 | 'repo_url': self.url, 204 | 'packages': packages, 205 | 'modules': py_modules, 206 | } 207 | 208 | def resolve(self): 209 | """ 210 | Downloads this requirement from the VCS repository or archive file and returns metadata from its setup.py. 211 | Returns an error string or None if no error. 212 | """ 213 | tmpdir = tempfile.mkdtemp() 214 | with open(os.devnull, 'w') as devnull: 215 | # Because of a bug in pip when dealing with VCS URLs, we can't use pip to download the repository 216 | if self.vcs == 'git': 217 | subprocess.call(['git', 'clone', '--depth=1', str(self.url), tmpdir], stdout=devnull, stderr=devnull) 218 | elif self.vcs == 'hg': 219 | subprocess.call(['hg', 'clone', str(self.url), tmpdir], stdout=devnull, stderr=devnull) 220 | elif self.vcs is None and self.type == 'archive': 221 | install_url = self._install_req.url 222 | tmparchive = tempfile.mkstemp()[1] 223 | subprocess.call(['curl', '-L', install_url, '-o', tmparchive], stdout=devnull, stderr=devnull) 224 | if install_url.endswith(".gz"): 225 | subprocess.call(['gunzip', '-c', tmparchive], stdout=devnull, stderr=devnull) 226 | install_url = install_url[0:-3] 227 | if install_url.endswith(".tar"): 228 | subprocess.call(['tar', '-xvf', tmparchive, '-C', tmpdir], stdout=devnull, stderr=devnull) 229 | elif install_url.endswith(".zip"): 230 | subprocess.call(['unzip', '-j', '-o', tmparchive, '-d', tmpdir], stdout=devnull, stderr=devnull) 231 | else: 232 | return 'cannot resolve requirement {} (from {}) with unrecognized VCS: {}'.format( 233 | str(self), 234 | str(self._install_req), 235 | self.vcs 236 | ) 237 | setup_dict, err = setup_py.setup_info_dir(tmpdir) 238 | if err is not None: 239 | return None, err 240 | rmtree(tmpdir) 241 | self.metadata = setup_dict 242 | return None 243 | --------------------------------------------------------------------------------