├── BUILD ├── tools └── typing │ ├── BUILD │ ├── mypy_version.txt │ └── mypy.ini ├── .bazelignore ├── extract_wheels ├── lib │ ├── __init__.py │ ├── BUILD │ ├── requirements_test.py │ ├── requirements.py │ ├── purelib.py │ ├── namespace_pkgs_test.py │ ├── namespace_pkgs.py │ ├── wheel.py │ └── bazel.py ├── __main__.py ├── BUILD └── __init__.py ├── example ├── requirements.txt ├── main.py ├── BUILD └── WORKSPACE ├── .gitattributes ├── .bazelrc ├── .github └── workflows │ └── continuous-integration.yml ├── WORKSPACE ├── repositories.bzl ├── .gitignore ├── defs.bzl ├── README.md └── LICENSE /BUILD: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tools/typing/BUILD: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.bazelignore: -------------------------------------------------------------------------------- 1 | example/ 2 | -------------------------------------------------------------------------------- /extract_wheels/lib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /example/requirements.txt: -------------------------------------------------------------------------------- 1 | boto3 2 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | example/* linguist-vendored 2 | -------------------------------------------------------------------------------- /tools/typing/mypy_version.txt: -------------------------------------------------------------------------------- 1 | mypy==0.780 2 | -------------------------------------------------------------------------------- /example/main.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | 3 | if __name__ == "__main__": 4 | pass 5 | -------------------------------------------------------------------------------- /.bazelrc: -------------------------------------------------------------------------------- 1 | build --aspects @mypy_integration//:mypy.bzl%mypy_aspect 2 | build --output_groups=+mypy 3 | -------------------------------------------------------------------------------- /extract_wheels/__main__.py: -------------------------------------------------------------------------------- 1 | """Main entry point.""" 2 | import extract_wheels 3 | 4 | if __name__ == "__main__": 5 | extract_wheels.main() 6 | -------------------------------------------------------------------------------- /extract_wheels/BUILD: -------------------------------------------------------------------------------- 1 | load("//:repositories.bzl", "all_requirements") 2 | 3 | py_binary( 4 | name = "extract_wheels", 5 | srcs = ["__init__.py", "__main__.py"], 6 | main = "__main__.py", 7 | deps = ["//extract_wheels/lib"], 8 | ) 9 | -------------------------------------------------------------------------------- /.github/workflows/continuous-integration.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | # Checks-out the repository under $GITHUB_WORKSPACE, so the job can access it 15 | - uses: actions/checkout@v2 16 | 17 | - name: Setup Bazel 18 | uses: abhinavsingh/setup-bazel@v3 19 | with: 20 | # Bazel version to install e.g. 1.2.1, 2.0.0, ... 21 | version: 2.0.0 # optional, default is 2.0.0 22 | 23 | - name: Run tests 24 | run: bazel test //... 25 | -------------------------------------------------------------------------------- /tools/typing/mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | # This should be the oldest supported release of Python 3 | # https://devguide.python.org/#status-of-python-branches 4 | python_version = 3.5 5 | 6 | # Third-Party packages without Stub files 7 | # https://mypy.readthedocs.io/en/latest/stubs.html 8 | [mypy-pkginfo.*] 9 | ignore_missing_imports = True 10 | 11 | [mypy-extract_wheels.*] 12 | check_untyped_defs = True 13 | disallow_incomplete_defs = True 14 | disallow_untyped_calls = True 15 | disallow_untyped_decorators = True 16 | disallow_untyped_defs = True 17 | no_implicit_optional = True 18 | strict_equality = True 19 | strict_optional = True 20 | warn_no_return = True 21 | warn_redundant_casts = True 22 | warn_return_any = True 23 | warn_unreachable = True 24 | warn_unused_ignores = True 25 | -------------------------------------------------------------------------------- /extract_wheels/lib/BUILD: -------------------------------------------------------------------------------- 1 | load("//:repositories.bzl", "requirement") 2 | 3 | py_library( 4 | name = "lib", 5 | visibility = ["//extract_wheels:__subpackages__"], 6 | srcs = [ 7 | "bazel.py", 8 | "namespace_pkgs.py", 9 | "purelib.py", 10 | "requirements.py", 11 | "wheel.py", 12 | ], 13 | deps = [ 14 | requirement("pkginfo"), 15 | requirement("setuptools"), 16 | ], 17 | ) 18 | 19 | py_test( 20 | name = "namespace_pkgs_test", 21 | size = "small", 22 | srcs = [ 23 | "namespace_pkgs_test.py", 24 | ], 25 | tags = ["unit"], 26 | deps = [ 27 | ":lib", 28 | ], 29 | ) 30 | 31 | py_test( 32 | name = "requirements_test", 33 | size = "small", 34 | srcs = [ 35 | "requirements_test.py", 36 | ], 37 | tags = ["unit"], 38 | deps = [ 39 | ":lib", 40 | ], 41 | ) 42 | -------------------------------------------------------------------------------- /example/BUILD: -------------------------------------------------------------------------------- 1 | load("@pip//:requirements.bzl", "requirement") 2 | 3 | # Toolchain setup, this is optional. 4 | # Demonstrate that we can use the same python interpreter for the toolchain and executing pip in pip install (see WORKSPACE). 5 | # 6 | #load("@rules_python//python:defs.bzl", "py_runtime_pair") 7 | # 8 | #py_runtime( 9 | # name = "python3_runtime", 10 | # files = ["@python_interpreter//:files"], 11 | # interpreter = "@python_interpreter//:python_bin", 12 | # python_version = "PY3", 13 | # visibility = ["//visibility:public"], 14 | #) 15 | # 16 | #py_runtime_pair( 17 | # name = "my_py_runtime_pair", 18 | # py2_runtime = None, 19 | # py3_runtime = ":python3_runtime", 20 | #) 21 | # 22 | #toolchain( 23 | # name = "my_py_toolchain", 24 | # toolchain = ":my_py_runtime_pair", 25 | # toolchain_type = "@bazel_tools//tools/python:toolchain_type", 26 | #) 27 | # End of toolchain setup. 28 | 29 | py_binary( 30 | name = "main", 31 | srcs = ["main.py"], 32 | deps = [ 33 | requirement("boto3"), 34 | ], 35 | ) 36 | -------------------------------------------------------------------------------- /extract_wheels/lib/requirements_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from extract_wheels.lib import requirements 4 | 5 | 6 | class TestRequirementExtrasParsing(unittest.TestCase): 7 | def test_parses_requirement_for_extra(self) -> None: 8 | cases = [ 9 | ("name[foo]", ("name", frozenset(["foo"]))), 10 | ("name[ Foo123 ]", ("name", frozenset(["Foo123"]))), 11 | (" name1[ foo ] ", ("name1", frozenset(["foo"]))), 12 | ( 13 | "name [fred,bar] @ http://foo.com ; python_version=='2.7'", 14 | ("name", frozenset(["fred", "bar"])), 15 | ), 16 | ( 17 | "name[quux, strange];python_version<'2.7' and platform_version=='2'", 18 | ("name", frozenset(["quux", "strange"])), 19 | ), 20 | ("name; (os_name=='a' or os_name=='b') and os_name=='c'", (None, None),), 21 | ("name@http://foo.com", (None, None),), 22 | ] 23 | 24 | for case, expected in cases: 25 | with self.subTest(): 26 | self.assertTupleEqual( 27 | requirements._parse_requirement_for_extra(case), expected 28 | ) 29 | 30 | 31 | if __name__ == "__main__": 32 | unittest.main() 33 | -------------------------------------------------------------------------------- /WORKSPACE: -------------------------------------------------------------------------------- 1 | workspace(name = "rules_python_external") 2 | 3 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 4 | 5 | http_archive( 6 | name = "rules_python", 7 | sha256 = "d2865e2ce23ee217aaa408ddaa024ca472114a6f250b46159d27de05530c75e3", 8 | strip_prefix = "rules_python-7b222cfdb4e59b9fd2a609e1fbb233e94fdcde7c", 9 | url = "https://github.com/bazelbuild/rules_python/archive/7b222cfdb4e59b9fd2a609e1fbb233e94fdcde7c.tar.gz", 10 | ) 11 | 12 | load("@rules_python//python:repositories.bzl", "py_repositories") 13 | py_repositories() 14 | 15 | load("//:repositories.bzl", "rules_python_external_dependencies") 16 | rules_python_external_dependencies() 17 | 18 | mypy_integration_version = "0.0.7" # latest @ Feb 10th 2020 19 | 20 | http_archive( 21 | name = "mypy_integration", 22 | sha256 = "bf7ecd386740328f96c343dca095a63b93df7f86f8d3e1e2e6ff46e400880077", # for 0.0.7 23 | strip_prefix = "bazel-mypy-integration-{version}".format(version = mypy_integration_version), 24 | url = "https://github.com/thundergolfer/bazel-mypy-integration/archive/{version}.zip".format( 25 | version = mypy_integration_version 26 | ), 27 | ) 28 | 29 | load( 30 | "@mypy_integration//repositories:repositories.bzl", 31 | mypy_integration_repositories = "repositories", 32 | ) 33 | 34 | mypy_integration_repositories() 35 | 36 | load("@mypy_integration//:config.bzl", "mypy_configuration") 37 | mypy_configuration("//tools/typing:mypy.ini") 38 | 39 | load("@mypy_integration//repositories:deps.bzl", mypy_integration_deps = "deps") 40 | mypy_integration_deps("//tools/typing:mypy_version.txt") 41 | 42 | load("@mypy_integration//repositories:pip_repositories.bzl", "pip_deps") 43 | pip_deps() 44 | -------------------------------------------------------------------------------- /extract_wheels/lib/requirements.py: -------------------------------------------------------------------------------- 1 | import re 2 | from typing import Dict, Set, Tuple, Optional 3 | 4 | 5 | def parse_extras(requirements_path: str) -> Dict[str, Set[str]]: 6 | """Parse over the requirements.txt file to find extras requested. 7 | 8 | Args: 9 | requirements_path: The filepath for the requirements.txt file to parse. 10 | 11 | Returns: 12 | A dictionary mapping the requirement name to a set of extras requested. 13 | """ 14 | 15 | extras_requested = {} 16 | with open(requirements_path, "r") as requirements: 17 | # Merge all backslash line continuations so we parse each requirement as a single line. 18 | for line in requirements.read().replace("\\\n", "").split("\n"): 19 | requirement, extras = _parse_requirement_for_extra(line) 20 | if requirement and extras: 21 | extras_requested[requirement] = extras 22 | 23 | return extras_requested 24 | 25 | 26 | def _parse_requirement_for_extra( 27 | requirement: str, 28 | ) -> Tuple[Optional[str], Optional[Set[str]]]: 29 | """Given a requirement string, returns the requirement name and set of extras, if extras specified. 30 | Else, returns (None, None) 31 | """ 32 | 33 | # https://www.python.org/dev/peps/pep-0508/#grammar 34 | extras_pattern = re.compile( 35 | r"^\s*([0-9A-Za-z][0-9A-Za-z_.\-]*)\s*\[\s*([0-9A-Za-z][0-9A-Za-z_.\-]*(?:\s*,\s*[0-9A-Za-z][0-9A-Za-z_.\-]*)*)\s*\]" 36 | ) 37 | 38 | matches = extras_pattern.match(requirement) 39 | if matches: 40 | return ( 41 | matches.group(1), 42 | {extra.strip() for extra in matches.group(2).split(",")}, 43 | ) 44 | 45 | return None, None 46 | -------------------------------------------------------------------------------- /repositories.bzl: -------------------------------------------------------------------------------- 1 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 2 | load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") 3 | 4 | _RULE_DEPS = [ 5 | ( 6 | "pypi__pip", 7 | "https://files.pythonhosted.org/packages/00/b6/9cfa56b4081ad13874b0c6f96af8ce16cfbc1cb06bedf8e9164ce5551ec1/pip-19.3.1-py2.py3-none-any.whl", 8 | "6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7", 9 | ), 10 | ( 11 | "pypi__pkginfo", 12 | "https://files.pythonhosted.org/packages/e6/d5/451b913307b478c49eb29084916639dc53a88489b993530fed0a66bab8b9/pkginfo-1.5.0.1-py2.py3-none-any.whl", 13 | "a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32", 14 | ), 15 | ( 16 | "pypi__setuptools", 17 | "https://files.pythonhosted.org/packages/54/28/c45d8b54c1339f9644b87663945e54a8503cfef59cf0f65b3ff5dd17cf64/setuptools-42.0.2-py2.py3-none-any.whl", 18 | "c8abd0f3574bc23afd2f6fd2c415ba7d9e097c8a99b845473b0d957ba1e2dac6", 19 | ), 20 | ( 21 | "pypi__wheel", 22 | "https://files.pythonhosted.org/packages/00/83/b4a77d044e78ad1a45610eb88f745be2fd2c6d658f9798a15e384b7d57c9/wheel-0.33.6-py2.py3-none-any.whl", 23 | "f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28", 24 | ), 25 | ] 26 | 27 | _GENERIC_WHEEL = """\ 28 | package(default_visibility = ["//visibility:public"]) 29 | 30 | load("@rules_python//python:defs.bzl", "py_library") 31 | 32 | py_library( 33 | name = "lib", 34 | srcs = glob(["**/*.py"]), 35 | data = glob(["**/*"], exclude=["**/*.py", "**/* *", "BUILD", "WORKSPACE"]), 36 | # This makes this directory a top-level in the python import 37 | # search path for anything that depends on this. 38 | imports = ["."], 39 | ) 40 | """ 41 | 42 | # Collate all the repository names so they can be easily consumed 43 | all_requirements = [name for (name, _, _) in _RULE_DEPS] 44 | 45 | def requirement(pkg): 46 | return "@pypi__"+ pkg + "//:lib" 47 | 48 | def rules_python_external_dependencies(): 49 | for (name, url, sha256) in _RULE_DEPS: 50 | maybe( 51 | http_archive, 52 | name, 53 | url=url, 54 | sha256=sha256, 55 | type="zip", 56 | build_file_content=_GENERIC_WHEEL, 57 | ) 58 | -------------------------------------------------------------------------------- /extract_wheels/lib/purelib.py: -------------------------------------------------------------------------------- 1 | """Functions to make purelibs Bazel compatible""" 2 | import pathlib 3 | import shutil 4 | 5 | from extract_wheels.lib import wheel 6 | 7 | 8 | def spread_purelib_into_root(wheel_dir: str) -> None: 9 | """Unpacks purelib directories into the root. 10 | 11 | Args: 12 | wheel_dir: The root of the extracted wheel directory. 13 | """ 14 | dist_info = wheel.get_dist_info(wheel_dir) 15 | wheel_metadata_file_path = pathlib.Path(dist_info, "WHEEL") 16 | wheel_metadata_dict = wheel.parse_wheel_meta_file(str(wheel_metadata_file_path)) 17 | 18 | if "Root-Is-Purelib" not in wheel_metadata_dict: 19 | raise ValueError( 20 | "Invalid WHEEL file '%s'. Expected key 'Root-Is-Purelib'." 21 | % wheel_metadata_file_path 22 | ) 23 | root_is_purelib = wheel_metadata_dict["Root-Is-Purelib"] 24 | 25 | if root_is_purelib.lower() == "true": 26 | # The Python package code is in the root of the Wheel, so no need to 'spread' anything. 27 | return 28 | 29 | dot_data_dir = wheel.get_dot_data_directory(wheel_dir) 30 | # 'Root-Is-Purelib: false' is no guarantee a .date directory exists with 31 | # package code in it. eg. the 'markupsafe' package. 32 | if not dot_data_dir: 33 | return 34 | 35 | for child in pathlib.Path(dot_data_dir).iterdir(): 36 | # TODO(Jonathon): Should all other potential folders get ignored? eg. 'platlib' 37 | if str(child).endswith("purelib"): 38 | _spread_purelib(child, wheel_dir) 39 | 40 | 41 | def _spread_purelib(purelib_dir: pathlib.Path, root_dir: str) -> None: 42 | """Recursively moves all sibling directories of the purelib to the root. 43 | 44 | Args: 45 | purelib_dir: The directory of the purelib. 46 | root_dir: The directory to move files into. 47 | """ 48 | for grandchild in purelib_dir.iterdir(): 49 | # Some purelib Wheels, like Tensorflow 2.0.0, have directories 50 | # split between the root and the purelib directory. In this case 51 | # we should leave the purelib 'sibling' alone. 52 | # See: https://github.com/dillon-giacoppo/rules_python_external/issues/8 53 | if not pathlib.Path(root_dir, grandchild.name).exists(): 54 | shutil.move( 55 | src=str(grandchild), dst=root_dir, 56 | ) 57 | -------------------------------------------------------------------------------- /extract_wheels/lib/namespace_pkgs_test.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import shutil 3 | import tempfile 4 | from typing import Optional 5 | import unittest 6 | 7 | from extract_wheels.lib import namespace_pkgs 8 | 9 | 10 | class TempDir: 11 | def __init__(self) -> None: 12 | self.dir = tempfile.mkdtemp() 13 | 14 | def root(self) -> str: 15 | return self.dir 16 | 17 | def add_dir(self, rel_path: str) -> None: 18 | d = pathlib.Path(self.dir, rel_path) 19 | d.mkdir(parents=True) 20 | 21 | def add_file(self, rel_path: str, contents: Optional[str] = None) -> None: 22 | f = pathlib.Path(self.dir, rel_path) 23 | f.parent.mkdir(parents=True, exist_ok=True) 24 | if contents: 25 | with open(str(f), "w") as writeable_f: 26 | writeable_f.write(contents) 27 | else: 28 | f.touch() 29 | 30 | def remove(self) -> None: 31 | shutil.rmtree(self.dir) 32 | 33 | 34 | class TestImplicitNamespacePackages(unittest.TestCase): 35 | def test_finds_correct_namespace_packages(self) -> None: 36 | directory = TempDir() 37 | directory.add_file("foo/bar/biz.py") 38 | directory.add_file("foo/bee/boo.py") 39 | directory.add_file("foo/buu/__init__.py") 40 | directory.add_file("foo/buu/bii.py") 41 | 42 | expected = { 43 | directory.root() + "/foo", 44 | directory.root() + "/foo/bar", 45 | directory.root() + "/foo/bee", 46 | } 47 | actual = namespace_pkgs.implicit_namespace_packages(directory.root()) 48 | self.assertEqual(actual, expected) 49 | 50 | def test_ignores_empty_directories(self) -> None: 51 | directory = TempDir() 52 | directory.add_file("foo/bar/biz.py") 53 | directory.add_dir("foo/cat") 54 | 55 | expected = { 56 | directory.root() + "/foo", 57 | directory.root() + "/foo/bar", 58 | } 59 | actual = namespace_pkgs.implicit_namespace_packages(directory.root()) 60 | self.assertEqual(actual, expected) 61 | 62 | def test_empty_case(self) -> None: 63 | directory = TempDir() 64 | directory.add_file("foo/__init__.py") 65 | directory.add_file("foo/bar/__init__.py") 66 | directory.add_file("foo/bar/biz.py") 67 | 68 | actual = namespace_pkgs.implicit_namespace_packages(directory.root()) 69 | self.assertEqual(actual, set()) 70 | 71 | 72 | if __name__ == "__main__": 73 | unittest.main() 74 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Intellij 2 | .ijwb/ 3 | .idea/ 4 | 5 | # Bazel 6 | bazel-* 7 | 8 | # Byte-compiled / optimized / DLL files 9 | __pycache__/ 10 | *.py[cod] 11 | *$py.class 12 | 13 | # C extensions 14 | *.so 15 | 16 | # Distribution / packaging 17 | .Python 18 | build/ 19 | develop-eggs/ 20 | dist/ 21 | downloads/ 22 | eggs/ 23 | .eggs/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | pip-wheel-metadata/ 30 | share/python-wheels/ 31 | *.egg-info/ 32 | .installed.cfg 33 | *.egg 34 | MANIFEST 35 | 36 | # PyInstaller 37 | # Usually these files are written by a python script from a template 38 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 39 | *.manifest 40 | *.spec 41 | 42 | # Installer logs 43 | pip-log.txt 44 | pip-delete-this-directory.txt 45 | 46 | # Unit test / coverage reports 47 | htmlcov/ 48 | .tox/ 49 | .nox/ 50 | .coverage 51 | .coverage.* 52 | .cache 53 | nosetests.xml 54 | coverage.xml 55 | *.cover 56 | *.py,cover 57 | .hypothesis/ 58 | .pytest_cache/ 59 | 60 | # Translations 61 | *.mo 62 | *.pot 63 | 64 | # Django stuff: 65 | *.log 66 | local_settings.py 67 | db.sqlite3 68 | db.sqlite3-journal 69 | 70 | # Flask stuff: 71 | instance/ 72 | .webassets-cache 73 | 74 | # Scrapy stuff: 75 | .scrapy 76 | 77 | # Sphinx documentation 78 | docs/_build/ 79 | 80 | # PyBuilder 81 | target/ 82 | 83 | # Jupyter Notebook 84 | .ipynb_checkpoints 85 | 86 | # IPython 87 | profile_default/ 88 | ipython_config.py 89 | 90 | # pyenv 91 | .python-version 92 | 93 | # pipenv 94 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 95 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 96 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 97 | # install all needed dependencies. 98 | #Pipfile.lock 99 | 100 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 101 | __pypackages__/ 102 | 103 | # Celery stuff 104 | celerybeat-schedule 105 | celerybeat.pid 106 | 107 | # SageMath parsed files 108 | *.sage.py 109 | 110 | # Environments 111 | .env 112 | .venv 113 | env/ 114 | venv/ 115 | ENV/ 116 | env.bak/ 117 | venv.bak/ 118 | 119 | # Spyder project settings 120 | .spyderproject 121 | .spyproject 122 | 123 | # Rope project settings 124 | .ropeproject 125 | 126 | # mkdocs documentation 127 | /site 128 | 129 | # mypy 130 | .mypy_cache/ 131 | .dmypy.json 132 | dmypy.json 133 | 134 | # Pyre type checker 135 | .pyre/ 136 | -------------------------------------------------------------------------------- /extract_wheels/lib/namespace_pkgs.py: -------------------------------------------------------------------------------- 1 | """Utility functions to discover python package types""" 2 | import os 3 | import textwrap 4 | from typing import Set, List, Optional 5 | 6 | from extract_wheels.lib import wheel 7 | 8 | 9 | def implicit_namespace_packages( 10 | directory: str, ignored_dirnames: Optional[List[str]] = None 11 | ) -> Set[str]: 12 | """Discovers namespace packages implemented using the 'native namespace packages' method. 13 | 14 | AKA 'implicit namespace packages', which has been supported since Python 3.3. 15 | See: https://packaging.python.org/guides/packaging-namespace-packages/#native-namespace-packages 16 | 17 | Args: 18 | directory: The root directory to recursively find packages in. 19 | ignored_dirnames: A list of directories to exclude from the search 20 | 21 | Returns: 22 | The set of directories found under root to be packages using the native namespace method. 23 | """ 24 | namespace_pkg_dirs = set() 25 | for dirpath, dirnames, filenames in os.walk(directory, topdown=True): 26 | # We are only interested in dirs with no __init__.py file 27 | if "__init__.py" in filenames: 28 | dirnames[:] = [] # Remove dirnames from search 29 | continue 30 | 31 | for ignored_dir in ignored_dirnames or []: 32 | if ignored_dir in dirnames: 33 | dirnames.remove(ignored_dir) 34 | 35 | non_empty_directory = dirnames or filenames 36 | if ( 37 | non_empty_directory 38 | and 39 | # The root of the directory should never be an implicit namespace 40 | dirpath != directory 41 | ): 42 | namespace_pkg_dirs.add(dirpath) 43 | 44 | return namespace_pkg_dirs 45 | 46 | 47 | def add_pkgutil_style_namespace_pkg_init(dir_path: str) -> None: 48 | """Adds 'pkgutil-style namespace packages' init file to the given directory 49 | 50 | See: https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages 51 | 52 | Args: 53 | dir_path: The directory to create an __init__.py for. 54 | 55 | Raises: 56 | ValueError: If the directory already contains an __init__.py file 57 | """ 58 | ns_pkg_init_filepath = os.path.join(dir_path, "__init__.py") 59 | 60 | if os.path.isfile(ns_pkg_init_filepath): 61 | raise ValueError("%s already contains an __init__.py file." % dir_path) 62 | 63 | with open(ns_pkg_init_filepath, "w") as ns_pkg_init_f: 64 | # See https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages 65 | ns_pkg_init_f.write( 66 | textwrap.dedent( 67 | """\ 68 | # __path__ manipulation added by rules_python_external to support namespace pkgs. 69 | __path__ = __import__('pkgutil').extend_path(__path__, __name__) 70 | """ 71 | ) 72 | ) 73 | -------------------------------------------------------------------------------- /example/WORKSPACE: -------------------------------------------------------------------------------- 1 | workspace(name = "example_repo") 2 | 3 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 4 | 5 | http_archive( 6 | name = "rules_python", 7 | url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.2/rules_python-0.0.2.tar.gz", 8 | strip_prefix = "rules_python-0.0.2", 9 | sha256 = "b5668cde8bb6e3515057ef465a35ad712214962f0b3a314e551204266c7be90c", 10 | ) 11 | 12 | load("@rules_python//python:repositories.bzl", "py_repositories") 13 | 14 | py_repositories() 15 | 16 | local_repository( 17 | name = "rules_python_external", 18 | path = "../", 19 | ) 20 | 21 | load("@rules_python_external//:repositories.bzl", "rules_python_external_dependencies") 22 | 23 | rules_python_external_dependencies() 24 | 25 | load("@rules_python_external//:defs.bzl", "pip_install") 26 | 27 | pip_install( 28 | # (Optional) You can provide extra parameters to pip. 29 | # Here, make pip output verbose (this is usable with `quiet = False`). 30 | #extra_pip_args = ["-v"], 31 | 32 | # (Optional) You can exclude custom elements in the data section of the generated BUILD files for pip packages. 33 | # Exclude directories with spaces in their names in this example (avoids build errors if there are such directories). 34 | #pip_data_exclude = ["**/* */**"], 35 | 36 | # (Optional) You can provide a python_interpreter (path) or a python_interpreter_target (a Bazel target, that 37 | # acts as an executable). The latter can be anything that could be used as Python interpreter. E.g.: 38 | # 1. Python interpreter that you compile in the build file (as above in @python_interpreter). 39 | # 2. Pre-compiled python interpreter included with http_archive 40 | # 3. Wrapper script, like in the autodetecting python toolchain. 41 | #python_interpreter_target = "@python_interpreter//:python_bin", 42 | 43 | # (Optional) You can set quiet to False if you want to see pip output. 44 | #quiet = False, 45 | 46 | # Uses the default repository name "pip" 47 | requirements = "//:requirements.txt", 48 | ) 49 | 50 | # You could optionally use an in-build, compiled python interpreter as a toolchain, 51 | # and also use it to execute pip. 52 | # 53 | # Special logic for building python interpreter with OpenSSL from homebrew. 54 | # See https://devguide.python.org/setup/#macos-and-os-x 55 | #_py_configure = """ 56 | #if [[ "$OSTYPE" == "darwin"* ]]; then 57 | # ./configure --prefix=$(pwd)/bazel_install --with-openssl=$(brew --prefix openssl) 58 | #else 59 | # ./configure --prefix=$(pwd)/bazel_install 60 | #fi 61 | #""" 62 | # 63 | # NOTE: you need to have the SSL headers installed to build with openssl support (and use HTTPS). 64 | # E.g. on Ubuntu: `sudo apt install libssl-dev` 65 | #http_archive( 66 | # name = "python_interpreter", 67 | # build_file_content = """ 68 | #exports_files(["python_bin"]) 69 | #filegroup( 70 | # name = "files", 71 | # srcs = glob(["bazel_install/**"], exclude = ["**/* *"]), 72 | # visibility = ["//visibility:public"], 73 | #) 74 | #""", 75 | # patch_cmds = [ 76 | # "mkdir $(pwd)/bazel_install", 77 | # _py_configure, 78 | # "make", 79 | # "make install", 80 | # "ln -s bazel_install/bin/python3 python_bin", 81 | # ], 82 | # sha256 = "dfab5ec723c218082fe3d5d7ae17ecbdebffa9a1aea4d64aa3a2ecdd2e795864", 83 | # strip_prefix = "Python-3.8.3", 84 | # urls = ["https://www.python.org/ftp/python/3.8.3/Python-3.8.3.tar.xz"], 85 | #) 86 | 87 | # Optional: 88 | # Register the toolchain with the same python interpreter we used for pip in pip_install(). 89 | #register_toolchains("//:my_py_toolchain") 90 | # End of in-build Python interpreter setup. 91 | -------------------------------------------------------------------------------- /extract_wheels/__init__.py: -------------------------------------------------------------------------------- 1 | """extract_wheels 2 | 3 | extract_wheels resolves and fetches artifacts transitively from the Python Package Index (PyPI) based on a 4 | requirements.txt. It generates the required BUILD files to consume these packages as Python libraries. 5 | 6 | Under the hood, it depends on the `pip wheel` command to do resolution, download, and compilation into wheels. 7 | """ 8 | import argparse 9 | import glob 10 | import os 11 | import subprocess 12 | import sys 13 | import json 14 | 15 | from extract_wheels.lib import bazel, requirements 16 | 17 | 18 | def configure_reproducible_wheels() -> None: 19 | """Modifies the environment to make wheel building reproducible. 20 | 21 | Wheels created from sdists are not reproducible by default. We can however workaround this by 22 | patching in some configuration with environment variables. 23 | """ 24 | 25 | # wheel, by default, enables debug symbols in GCC. This incidentally captures the build path in the .so file 26 | # We can override this behavior by disabling debug symbols entirely. 27 | # https://github.com/pypa/pip/issues/6505 28 | if "CFLAGS" in os.environ: 29 | os.environ["CFLAGS"] += " -g0" 30 | else: 31 | os.environ["CFLAGS"] = "-g0" 32 | 33 | # set SOURCE_DATE_EPOCH to 1980 so that we can use python wheels 34 | # https://github.com/NixOS/nixpkgs/blob/master/doc/languages-frameworks/python.section.md#python-setuppy-bdist_wheel-cannot-create-whl 35 | if "SOURCE_DATE_EPOCH" not in os.environ: 36 | os.environ["SOURCE_DATE_EPOCH"] = "315532800" 37 | 38 | # Python wheel metadata files can be unstable. 39 | # See https://bitbucket.org/pypa/wheel/pull-requests/74/make-the-output-of-metadata-files/diff 40 | if "PYTHONHASHSEED" not in os.environ: 41 | os.environ["PYTHONHASHSEED"] = "0" 42 | 43 | 44 | def main() -> None: 45 | """Main program. 46 | 47 | Exits zero on successful program termination, non-zero otherwise. 48 | """ 49 | 50 | configure_reproducible_wheels() 51 | 52 | parser = argparse.ArgumentParser( 53 | description="Resolve and fetch artifacts transitively from PyPI" 54 | ) 55 | parser.add_argument( 56 | "--requirements", 57 | action="store", 58 | required=True, 59 | help="Path to requirements.txt from where to install dependencies", 60 | ) 61 | parser.add_argument( 62 | "--repo", 63 | action="store", 64 | required=True, 65 | help="The external repo name to install dependencies. In the format '@{REPO_NAME}'", 66 | ) 67 | parser.add_argument( 68 | "--extra_pip_args", action="store", help="Extra arguments to pass down to pip.", 69 | ) 70 | parser.add_argument( 71 | "--pip_data_exclude", 72 | action="store", 73 | help="Additional data exclusion parameters to add to the pip packages BUILD file.", 74 | ) 75 | parser.add_argument( 76 | "--enable_implicit_namespace_pkgs", 77 | action="store_true", 78 | help="Disables conversion of implicit namespace packages into pkg-util style packages.", 79 | ) 80 | args = parser.parse_args() 81 | 82 | pip_args = [sys.executable, "-m", "pip", "wheel", "-r", args.requirements] 83 | if args.extra_pip_args: 84 | pip_args += json.loads(args.extra_pip_args)["args"] 85 | 86 | # Assumes any errors are logged by pip so do nothing. This command will fail if pip fails 87 | subprocess.run(pip_args, check=True) 88 | 89 | extras = requirements.parse_extras(args.requirements) 90 | 91 | if args.pip_data_exclude: 92 | pip_data_exclude = json.loads(args.pip_data_exclude)["exclude"] 93 | else: 94 | pip_data_exclude = [] 95 | 96 | targets = [ 97 | '"%s%s"' 98 | % ( 99 | args.repo, 100 | bazel.extract_wheel( 101 | whl, extras, pip_data_exclude, args.enable_implicit_namespace_pkgs 102 | ), 103 | ) 104 | for whl in glob.glob("*.whl") 105 | ] 106 | 107 | with open("requirements.bzl", "w") as requirement_file: 108 | requirement_file.write( 109 | bazel.generate_requirements_file_contents(args.repo, targets) 110 | ) 111 | -------------------------------------------------------------------------------- /defs.bzl: -------------------------------------------------------------------------------- 1 | load("//:repositories.bzl", "all_requirements") 2 | 3 | DEFAULT_REPOSITORY_NAME = "pip" 4 | 5 | def _pip_repository_impl(rctx): 6 | python_interpreter = rctx.attr.python_interpreter 7 | if rctx.attr.python_interpreter_target != None: 8 | target = rctx.attr.python_interpreter_target 9 | python_interpreter = rctx.path(target) 10 | else: 11 | if "/" not in python_interpreter: 12 | python_interpreter = rctx.which(python_interpreter) 13 | if not python_interpreter: 14 | fail("python interpreter not found") 15 | 16 | rctx.file("BUILD", "") 17 | 18 | # Get the root directory of these rules 19 | rules_root = rctx.path(Label("//:BUILD")).dirname 20 | thirdparty_roots = [ 21 | # Includes all the external dependencies from repositories.bzl 22 | rctx.path(Label("@" + repo + "//:BUILD.bazel")).dirname 23 | for repo in all_requirements 24 | ] 25 | separator = ":" if not "windows" in rctx.os.name.lower() else ";" 26 | pypath = separator.join([str(p) for p in [rules_root] + thirdparty_roots]) 27 | 28 | args = [ 29 | python_interpreter, 30 | "-m", 31 | "extract_wheels", 32 | "--requirements", 33 | rctx.path(rctx.attr.requirements), 34 | "--repo", 35 | "@%s" % rctx.attr.name, 36 | ] 37 | 38 | if rctx.attr.extra_pip_args: 39 | args += [ 40 | "--extra_pip_args", 41 | struct(args = rctx.attr.extra_pip_args).to_json(), 42 | ] 43 | 44 | if rctx.attr.pip_data_exclude: 45 | args += [ 46 | "--pip_data_exclude", 47 | struct(exclude = rctx.attr.pip_data_exclude).to_json(), 48 | ] 49 | 50 | if rctx.attr.enable_implicit_namespace_pkgs: 51 | args += [ 52 | "--enable_implicit_namespace_pkgs" 53 | ] 54 | 55 | result = rctx.execute( 56 | args, 57 | environment = { 58 | # Manually construct the PYTHONPATH since we cannot use the toolchain here 59 | "PYTHONPATH": pypath, 60 | }, 61 | timeout = rctx.attr.timeout, 62 | quiet = rctx.attr.quiet, 63 | ) 64 | if result.return_code: 65 | fail("rules_python_external failed: %s (%s)" % (result.stdout, result.stderr)) 66 | 67 | return 68 | 69 | pip_repository = repository_rule( 70 | attrs = { 71 | "requirements": attr.label(allow_single_file = True, mandatory = True), 72 | "wheel_env": attr.string_dict(), 73 | "python_interpreter": attr.string(default = "python3"), 74 | "python_interpreter_target": attr.label(allow_single_file = True, doc = """ 75 | If you are using a custom python interpreter built by another repository rule, 76 | use this attribute to specify its BUILD target. This allows pip_repository to invoke 77 | pip using the same interpreter as your toolchain. If set, takes precedence over 78 | python_interpreter. 79 | """), 80 | # 600 is documented as default here: https://docs.bazel.build/versions/master/skylark/lib/repository_ctx.html#execute 81 | "timeout": attr.int(default = 600), 82 | "quiet": attr.bool(default = True), 83 | "extra_pip_args": attr.string_list( 84 | doc = "Extra arguments to pass on to pip. Must not contain spaces.", 85 | ), 86 | "pip_data_exclude": attr.string_list( 87 | doc = "Additional data exclusion parameters to add to the pip packages BUILD file.", 88 | ), 89 | "enable_implicit_namespace_pkgs": attr.bool( 90 | default = False, 91 | doc = """ 92 | If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary 93 | and py_test targets must specify either `legacy_create_init=False` or the global Bazel option 94 | `--incompatible_default_to_explicit_init_py` to prevent `__init__.py` being automatically generated in every directory. 95 | 96 | This option is required to support some packages which cannot handle the conversion to pkg-util style. 97 | """, 98 | ), 99 | }, 100 | implementation = _pip_repository_impl, 101 | ) 102 | 103 | def pip_install(requirements, name = DEFAULT_REPOSITORY_NAME, **kwargs): 104 | pip_repository( 105 | name = name, 106 | requirements = requirements, 107 | **kwargs 108 | ) 109 | -------------------------------------------------------------------------------- /extract_wheels/lib/wheel.py: -------------------------------------------------------------------------------- 1 | """Utility class to inspect an extracted wheel directory""" 2 | import glob 3 | import os 4 | import stat 5 | import zipfile 6 | from typing import Dict, Optional, Set 7 | 8 | import pkg_resources 9 | import pkginfo 10 | 11 | 12 | def current_umask() -> int: 13 | """Get the current umask which involves having to set it temporarily.""" 14 | mask = os.umask(0) 15 | os.umask(mask) 16 | return mask 17 | 18 | 19 | def set_extracted_file_to_default_mode_plus_executable(path: str) -> None: 20 | """ 21 | Make file present at path have execute for user/group/world 22 | (chmod +x) is no-op on windows per python docs 23 | """ 24 | os.chmod(path, (0o777 & ~current_umask() | 0o111)) 25 | 26 | 27 | class Wheel: 28 | """Representation of the compressed .whl file""" 29 | 30 | def __init__(self, path: str): 31 | self._path = path 32 | 33 | @property 34 | def path(self) -> str: 35 | return self._path 36 | 37 | @property 38 | def name(self) -> str: 39 | return str(self.metadata.name) 40 | 41 | @property 42 | def metadata(self) -> pkginfo.Wheel: 43 | return pkginfo.get_metadata(self.path) 44 | 45 | def dependencies(self, extras_requested: Optional[Set[str]] = None) -> Set[str]: 46 | dependency_set = set() 47 | 48 | for wheel_req in self.metadata.requires_dist: 49 | req = pkg_resources.Requirement(wheel_req) # type: ignore 50 | 51 | if req.marker is None or any( 52 | req.marker.evaluate({"extra": extra}) 53 | for extra in extras_requested or [""] 54 | ): 55 | dependency_set.add(req.name) # type: ignore 56 | 57 | return dependency_set 58 | 59 | def unzip(self, directory: str) -> None: 60 | with zipfile.ZipFile(self.path, "r") as whl: 61 | whl.extractall(directory) 62 | # The following logic is borrowed from Pip: 63 | # https://github.com/pypa/pip/blob/cc48c07b64f338ac5e347d90f6cb4efc22ed0d0b/src/pip/_internal/utils/unpacking.py#L240 64 | for info in whl.infolist(): 65 | name = info.filename 66 | # Do not attempt to modify directories. 67 | if name.endswith("/") or name.endswith("\\"): 68 | continue 69 | mode = info.external_attr >> 16 70 | # if mode and regular file and any execute permissions for 71 | # user/group/world? 72 | if mode and stat.S_ISREG(mode) and mode & 0o111: 73 | name = os.path.join(directory, name) 74 | set_extracted_file_to_default_mode_plus_executable(name) 75 | 76 | 77 | def get_dist_info(wheel_dir: str) -> str: 78 | """"Returns the relative path to the dist-info directory if it exists. 79 | 80 | Args: 81 | wheel_dir: The root of the extracted wheel directory. 82 | 83 | Returns: 84 | Relative path to the dist-info directory if it exists, else, None. 85 | """ 86 | dist_info_dirs = glob.glob(os.path.join(wheel_dir, "*.dist-info")) 87 | if not dist_info_dirs: 88 | raise ValueError( 89 | "No *.dist-info directory found. %s is not a valid Wheel." % wheel_dir 90 | ) 91 | 92 | if len(dist_info_dirs) > 1: 93 | raise ValueError( 94 | "Found more than 1 *.dist-info directory. %s is not a valid Wheel." 95 | % wheel_dir 96 | ) 97 | 98 | return dist_info_dirs[0] 99 | 100 | 101 | def get_dot_data_directory(wheel_dir: str) -> Optional[str]: 102 | """Returns the relative path to the data directory if it exists. 103 | 104 | See: https://www.python.org/dev/peps/pep-0491/#the-data-directory 105 | 106 | Args: 107 | wheel_dir: The root of the extracted wheel directory. 108 | 109 | Returns: 110 | Relative path to the data directory if it exists, else, None. 111 | """ 112 | 113 | dot_data_dirs = glob.glob(os.path.join(wheel_dir, "*.data")) 114 | if not dot_data_dirs: 115 | return None 116 | 117 | if len(dot_data_dirs) > 1: 118 | raise ValueError( 119 | "Found more than 1 *.data directory. %s is not a valid Wheel." % wheel_dir 120 | ) 121 | 122 | return dot_data_dirs[0] 123 | 124 | 125 | def parse_wheel_meta_file(wheel_dir: str) -> Dict[str, str]: 126 | """Parses the given WHEEL file into a dictionary. 127 | 128 | Args: 129 | wheel_dir: The file path of the WHEEL metadata file in dist-info. 130 | 131 | Returns: 132 | The WHEEL file mapped into a dictionary. 133 | """ 134 | contents = {} 135 | with open(wheel_dir, "r") as wheel_file: 136 | for line in wheel_file: 137 | cleaned = line.strip() 138 | if not cleaned: 139 | continue 140 | try: 141 | key, value = cleaned.split(":", maxsplit=1) 142 | contents[key] = value.strip() 143 | except ValueError: 144 | raise RuntimeError( 145 | "Encounted invalid line in WHEEL file: '%s'" % cleaned 146 | ) 147 | return contents 148 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 🚨 Deprecated 2 | **This repository has been merged into [bazelbuild/rules_python@v0.1.0](https://github.com/bazelbuild/rules_python/releases/tag/0.1.0) and is no longer maintained.** 3 | 4 | `rules_python_external` is now the canonical way to manage Python dependencies for rules_python. For instructions on how to use, see https://github.com/bazelbuild/rules_python#using-the-packaging-rules. 5 | 6 | --- 7 | # rules_python_external ![](https://github.com/dillon-giacoppo/rules_python_external/workflows/CI/badge.svg) 8 | 9 | Bazel rules to transitively fetch and install Python dependencies from a requirements.txt file. 10 | 11 | ## Features 12 | 13 | The rules address most of the top packaging issues in [`bazelbuild/rules_python`](https://github.com/bazelbuild/rules_python). This means the rules support common packages such 14 | as [`tensorflow`](https://pypi.org/project/tensorflow/) and [`google.cloud`](https://github.com/googleapis/google-cloud-python) natively. 15 | 16 | * Transitive dependency resolution: 17 | [#35](https://github.com/bazelbuild/rules_python/issues/35), 18 | [#102](https://github.com/bazelbuild/rules_python/issues/102) 19 | * Minimal runtime dependencies: 20 | [#184](https://github.com/bazelbuild/rules_python/issues/184) 21 | * Support for [spreading purelibs](https://www.python.org/dev/peps/pep-0491/#installing-a-wheel-distribution-1-0-py32-none-any-whl): 22 | [#71](https://github.com/bazelbuild/rules_python/issues/71) 23 | * Support for [namespace packages](https://packaging.python.org/guides/packaging-namespace-packages/): 24 | [#14](https://github.com/bazelbuild/rules_python/issues/14), 25 | [#55](https://github.com/bazelbuild/rules_python/issues/55), 26 | [#65](https://github.com/bazelbuild/rules_python/issues/65), 27 | [#93](https://github.com/bazelbuild/rules_python/issues/93), 28 | [#189](https://github.com/bazelbuild/rules_python/issues/189) 29 | * Fetches pip packages only for building Python targets: 30 | [#96](https://github.com/bazelbuild/rules_python/issues/96) 31 | * Reproducible builds: 32 | [#154](https://github.com/bazelbuild/rules_python/issues/154), 33 | [#176](https://github.com/bazelbuild/rules_python/issues/176) 34 | 35 | ## Usage 36 | 37 | #### Prerequisites 38 | 39 | The rules support Python >= 3.5 (the oldest [maintained release](https://devguide.python.org/#status-of-python-branches)). 40 | 41 | #### Setup `WORKSPACE` 42 | 43 | ```python 44 | rules_python_external_version = "{COMMIT_SHA}" 45 | 46 | http_archive( 47 | name = "rules_python_external", 48 | sha256 = "", # Fill in with correct sha256 of your COMMIT_SHA version 49 | strip_prefix = "rules_python_external-{version}".format(version = rules_python_external_version), 50 | url = "https://github.com/dillon-giacoppo/rules_python_external/archive/v{version}.zip".format(version = rules_python_external_version), 51 | ) 52 | 53 | # Install the rule dependencies 54 | load("@rules_python_external//:repositories.bzl", "rules_python_external_dependencies") 55 | rules_python_external_dependencies() 56 | 57 | load("@rules_python_external//:defs.bzl", "pip_install") 58 | pip_install( 59 | name = "py_deps", 60 | requirements = "//:requirements.txt", 61 | # (Optional) You can provide a python interpreter (by path): 62 | python_interpreter = "/usr/bin/python3.8", 63 | # (Optional) Alternatively you can provide an in-build python interpreter, that is available as a Bazel target. 64 | # This overrides `python_interpreter`. 65 | # Note: You need to set up the interpreter target beforehand (not shown here). Please see the `example` folder for further details. 66 | #python_interpreter_target = "@python_interpreter//:python_bin", 67 | ) 68 | ``` 69 | 70 | #### Example `BUILD` file. 71 | 72 | ```python 73 | load("@py_deps//:requirements.bzl", "requirement") 74 | 75 | py_binary( 76 | name = "main", 77 | srcs = ["main.py"], 78 | deps = [ 79 | requirement("boto3"), 80 | ], 81 | ) 82 | ``` 83 | 84 | Note that above you do not need to add transitively required packages to `deps = [ ... ]` 85 | 86 | #### Setup `requirements.txt` 87 | 88 | While `rules_python_external` **does not** require a _transitively-closed_ `requirements.txt` file, it is recommended. 89 | But if you want to just have top-level packages listed, that also will work. 90 | 91 | Transitively-closed requirements specs are very tedious to produce and maintain manually. To automate the process we 92 | recommend [`pip-compile` from `jazzband/pip-tools`](https://github.com/jazzband/pip-tools#example-usage-for-pip-compile). 93 | 94 | For example, `pip-compile` takes a `requirements.in` like this: 95 | 96 | ``` 97 | boto3~=1.9.227 98 | botocore~=1.12.247 99 | click~=7.0 100 | ``` 101 | 102 | `pip-compile` 'compiles' it so you get a transitively-closed `requirements.txt` like this, which should be passed to 103 | `pip_install` below: 104 | 105 | ``` 106 | boto3==1.9.253 107 | botocore==1.12.253 108 | click==7.0 109 | docutils==0.15.2 # via botocore 110 | jmespath==0.9.4 # via boto3, botocore 111 | python-dateutil==2.8.1 # via botocore 112 | s3transfer==0.2.1 # via boto3 113 | six==1.14.0 # via python-dateutil 114 | urllib3==1.25.8 # via botocore 115 | ``` 116 | 117 | ### Demo 118 | 119 | You can find a demo in the [example/](./example) directory. 120 | 121 | ## Development 122 | 123 | ### Testing 124 | 125 | `bazel test //...` 126 | 127 | ## Adopters 128 | 129 | Here's a (non-exhaustive) list of companies that use `rules_python_external` in production. Don't see yours? [You can add it in a PR](https://github.com/dillon-giacoppo/rules_python_external/edit/master/README.md)! 130 | 131 | * [Canva](https://www.canva.com/) 132 | -------------------------------------------------------------------------------- /extract_wheels/lib/bazel.py: -------------------------------------------------------------------------------- 1 | """Utility functions to manipulate Bazel files""" 2 | import os 3 | import textwrap 4 | import json 5 | from typing import Iterable, List, Dict, Set 6 | 7 | from extract_wheels.lib import namespace_pkgs, wheel, purelib 8 | 9 | 10 | def generate_build_file_contents( 11 | name: str, dependencies: List[str], pip_data_exclude: List[str] 12 | ) -> str: 13 | """Generate a BUILD file for an unzipped Wheel 14 | 15 | Args: 16 | name: the target name of the py_library 17 | dependencies: a list of Bazel labels pointing to dependencies of the library 18 | 19 | Returns: 20 | A complete BUILD file as a string 21 | 22 | We allow for empty Python sources as for Wheels containing only compiled C code 23 | there may be no Python sources whatsoever (e.g. packages written in Cython: like `pymssql`). 24 | """ 25 | 26 | data_exclude = ["**/*.py", "**/* *", "BUILD", "WORKSPACE"] + pip_data_exclude 27 | 28 | return textwrap.dedent( 29 | """\ 30 | package(default_visibility = ["//visibility:public"]) 31 | 32 | load("@rules_python//python:defs.bzl", "py_library") 33 | 34 | py_library( 35 | name = "{name}", 36 | srcs = glob(["**/*.py"], allow_empty = True), 37 | data = glob(["**/*"], exclude={data_exclude}), 38 | # This makes this directory a top-level in the python import 39 | # search path for anything that depends on this. 40 | imports = ["."], 41 | deps = [{dependencies}], 42 | ) 43 | """.format( 44 | name=name, 45 | dependencies=",".join(dependencies), 46 | data_exclude=json.dumps(data_exclude), 47 | ) 48 | ) 49 | 50 | 51 | def generate_requirements_file_contents(repo_name: str, targets: Iterable[str]) -> str: 52 | """Generate a requirements.bzl file for a given pip repository 53 | 54 | The file allows converting the PyPI name to a bazel label. Additionally, it adds a function which can glob all the 55 | installed dependencies. This is provided for legacy reasons and can be considered deprecated. 56 | 57 | Args: 58 | repo_name: the name of the pip repository 59 | targets: a list of Bazel labels pointing to all the generated targets 60 | 61 | Returns: 62 | A complete requirements.bzl file as a string 63 | """ 64 | 65 | return textwrap.dedent( 66 | """\ 67 | # Deprecated. This will be removed in a future release 68 | all_requirements = [{requirement_labels}] 69 | 70 | def requirement(name): 71 | name_key = name.replace("-", "_").replace(".", "_").lower() 72 | return "{repo}//pypi__" + name_key 73 | """.format( 74 | repo=repo_name, requirement_labels=",".join(sorted(targets)) 75 | ) 76 | ) 77 | 78 | 79 | def sanitise_name(name: str) -> str: 80 | """Sanitises the name to be compatible with Bazel labels. 81 | 82 | There are certain requirements around Bazel labels that we need to consider. From the Bazel docs: 83 | 84 | Package names must be composed entirely of characters drawn from the set A-Z, a–z, 0–9, '/', '-', '.', and '_', 85 | and cannot start with a slash. 86 | 87 | Due to restrictions on Bazel labels we also cannot allow hyphens. See 88 | https://github.com/bazelbuild/bazel/issues/6841 89 | 90 | Further, rules-python automatically adds the repository root to the PYTHONPATH, meaning a package that has the same 91 | name as a module is picked up. We workaround this by prefixing with `pypi__`. Alternatively we could require 92 | `--noexperimental_python_import_all_repositories` be set, however this breaks rules_docker. 93 | See: https://github.com/bazelbuild/bazel/issues/2636 94 | """ 95 | 96 | return "pypi__" + name.replace("-", "_").replace(".", "_").lower() 97 | 98 | 99 | def setup_namespace_pkg_compatibility(wheel_dir: str) -> None: 100 | """Converts native namespace packages to pkgutil-style packages 101 | 102 | Namespace packages can be created in one of three ways. They are detailed here: 103 | https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package 104 | 105 | 'pkgutil-style namespace packages' (2) and 'pkg_resources-style namespace packages' (3) works in Bazel, but 106 | 'native namespace packages' (1) do not. 107 | 108 | We ensure compatibility with Bazel of method 1 by converting them into method 2. 109 | 110 | Args: 111 | wheel_dir: the directory of the wheel to convert 112 | """ 113 | 114 | namespace_pkg_dirs = namespace_pkgs.implicit_namespace_packages( 115 | wheel_dir, ignored_dirnames=["%s/bin" % wheel_dir,], 116 | ) 117 | 118 | for ns_pkg_dir in namespace_pkg_dirs: 119 | namespace_pkgs.add_pkgutil_style_namespace_pkg_init(ns_pkg_dir) 120 | 121 | 122 | def extract_wheel( 123 | wheel_file: str, 124 | extras: Dict[str, Set[str]], 125 | pip_data_exclude: List[str], 126 | enable_implicit_namespace_pkgs: bool, 127 | ) -> str: 128 | """Extracts wheel into given directory and creates a py_library target. 129 | 130 | Args: 131 | wheel_file: the filepath of the .whl 132 | extras: a list of extras to add as dependencies for the installed wheel 133 | pip_data_exclude: list of file patterns to exclude from the generated data section of the py_library 134 | enable_implicit_namespace_pkgs: if true, disables conversion of implicit namespace packages and will unzip as-is 135 | 136 | Returns: 137 | The Bazel label for the extracted wheel, in the form '//path/to/wheel'. 138 | """ 139 | 140 | whl = wheel.Wheel(wheel_file) 141 | directory = sanitise_name(whl.name) 142 | 143 | os.mkdir(directory) 144 | whl.unzip(directory) 145 | 146 | # Note: Order of operations matters here 147 | purelib.spread_purelib_into_root(directory) 148 | 149 | if not enable_implicit_namespace_pkgs: 150 | setup_namespace_pkg_compatibility(directory) 151 | 152 | extras_requested = extras[whl.name] if whl.name in extras else set() 153 | 154 | sanitised_dependencies = [ 155 | '"//%s"' % sanitise_name(d) for d in sorted(whl.dependencies(extras_requested)) 156 | ] 157 | 158 | with open(os.path.join(directory, "BUILD"), "w") as build_file: 159 | contents = generate_build_file_contents( 160 | sanitise_name(whl.name), sanitised_dependencies, pip_data_exclude, 161 | ) 162 | build_file.write(contents) 163 | 164 | os.remove(whl.path) 165 | 166 | return "//%s" % directory 167 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------