├── .github └── workflows │ └── tests.yml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.rst ├── examples ├── absolute-import-rewrite │ ├── hello │ │ ├── .gitignore │ │ ├── hello │ │ │ ├── __init__.py │ │ │ └── messages.py │ │ └── setup.py │ ├── main.py │ └── vendorize.toml ├── indented-absolute-simple-import-rewrite │ ├── hello │ │ ├── .gitignore │ │ ├── hello │ │ │ ├── __init__.py │ │ │ └── messages.py │ │ └── setup.py │ ├── main.py │ └── vendorize.toml ├── isolated-module-pyproject │ ├── main.py │ └── pyproject.toml ├── isolated-module │ ├── main.py │ └── vendorize.toml ├── local-module │ ├── hello │ │ ├── .gitignore │ │ ├── hello.py │ │ └── setup.py │ ├── main.py │ └── vendorize.toml ├── multiple-dependencies-with-rewrite │ ├── hello │ │ ├── .gitignore │ │ ├── hello │ │ │ ├── __init__.py │ │ │ └── hello.py │ │ └── setup.py │ ├── main.py │ ├── vendorize.toml │ └── world │ │ ├── .gitignore │ │ ├── setup.py │ │ └── world │ │ ├── __init__.py │ │ └── world.py └── multiple-dependencies │ ├── hello │ ├── .gitignore │ ├── hello.py │ └── setup.py │ ├── main.py │ ├── vendorize.toml │ └── world │ ├── .gitignore │ ├── setup.py │ └── world.py ├── makefile ├── setup.py ├── test-requirements.txt ├── tests ├── cli_tests.py ├── import_rewrite_tests.py └── python_source_tests.py ├── tox.ini ├── vendorize.toml └── vendorize ├── __init__.py ├── _vendor ├── __init__.py └── pytoml │ ├── __init__.py │ ├── core.py │ ├── parser.py │ └── writer.py ├── cli.py ├── files.py ├── import_rewrite.py ├── python_source.py └── setuptools_build.py /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | runs-on: "ubuntu-20.04" 8 | 9 | strategy: 10 | matrix: 11 | python-version: ["3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy3.9"] 12 | 13 | steps: 14 | 15 | - uses: actions/checkout@v4 16 | 17 | - name: Use Python ${{ matrix.python-version }} 18 | uses: actions/setup-python@v5 19 | with: 20 | python-version: ${{ matrix.python-version }} 21 | 22 | - run: pip install tox 23 | 24 | - run: tox -e py 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | /_virtualenv 3 | *.dist-info 4 | *.egg-info 5 | /.tox 6 | /MANIFEST 7 | /examples/**/_vendor 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015, Michael Williamson 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 17 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 19 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 20 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 21 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 22 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE README.rst 2 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | python-vendorize 2 | ================ 3 | 4 | ``python-vendorize`` allows pure-Python dependencies to be vendorized: 5 | that is, the Python source of the dependency is copied into your own package. 6 | Best used for small, pure-Python dependencies to avoid version conflicts 7 | when other packages require a different version of the same dependency. 8 | 9 | Dependencies you want vendorizing should be specified in ``vendorize.toml``. 10 | ``target`` should be a string containing the path where vendorized dependencies should be placed, 11 | relative to the directory that ``vendorize.toml`` is in. 12 | ``packages`` should be a list of strings containing the dependencies. 13 | Each of these strings can be anything that ``pip`` would understand, 14 | such as a package name, a package name with version constraints or an URL. 15 | Dependencies can then be vendorized using ``python-vendorize``. 16 | 17 | For instance, suppose I want to vendorize ``six`` so it can be used from the package ``hello``. 18 | The directory structure would be something like: 19 | 20 | :: 21 | 22 | - hello 23 | - __init__.py 24 | - setup.py 25 | - vendorize.toml 26 | 27 | ``vendorize.toml`` might look something like: 28 | 29 | :: 30 | 31 | target = "hello/_vendor" 32 | packages = [ 33 | "six", 34 | ] 35 | 36 | I can then run ``python-vendorize`` in the same directory as ``vendorize.toml``. 37 | The directory structure would then be something like: 38 | 39 | :: 40 | 41 | - hello 42 | - _vendor 43 | - six.dist-info 44 | - ... 45 | - __init__.py 46 | - six.py 47 | - __init__.py 48 | - setup.py 49 | - vendorize.toml 50 | 51 | In ``hello/__init__.py``, ``six`` can be imported from ``_vendor``: 52 | 53 | .. code:: python 54 | 55 | from ._vendor import six 56 | 57 | The configuration can also be stored in ``pyproject.toml`` instead of ``vendorize.toml``. 58 | When using ``pyproject.toml``, the configuration should be stored in ``[tool.vendorize]``. 59 | For instance: 60 | 61 | :: 62 | 63 | [tool.vendorize] 64 | target = "hello/_vendor" 65 | packages = [ 66 | "six", 67 | ] 68 | 69 | Installation 70 | ~~~~~~~~~~~~ 71 | 72 | :: 73 | 74 | pip install vendorize 75 | -------------------------------------------------------------------------------- /examples/absolute-import-rewrite/hello/.gitignore: -------------------------------------------------------------------------------- 1 | /build 2 | -------------------------------------------------------------------------------- /examples/absolute-import-rewrite/hello/hello/__init__.py: -------------------------------------------------------------------------------- 1 | from hello.messages import message 2 | 3 | def hello(): 4 | print(message) 5 | -------------------------------------------------------------------------------- /examples/absolute-import-rewrite/hello/hello/messages.py: -------------------------------------------------------------------------------- 1 | message = "hello" 2 | -------------------------------------------------------------------------------- /examples/absolute-import-rewrite/hello/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="hello", 5 | packages=["hello"]) 6 | -------------------------------------------------------------------------------- /examples/absolute-import-rewrite/main.py: -------------------------------------------------------------------------------- 1 | from _vendor.hello import hello 2 | 3 | hello() 4 | -------------------------------------------------------------------------------- /examples/absolute-import-rewrite/vendorize.toml: -------------------------------------------------------------------------------- 1 | target = "_vendor" 2 | packages = [ 3 | "./hello", 4 | ] 5 | -------------------------------------------------------------------------------- /examples/indented-absolute-simple-import-rewrite/hello/.gitignore: -------------------------------------------------------------------------------- 1 | /build 2 | -------------------------------------------------------------------------------- /examples/indented-absolute-simple-import-rewrite/hello/hello/__init__.py: -------------------------------------------------------------------------------- 1 | try: 2 | import hello.messages 3 | except: 4 | # We include an except clause to catch indentation errors 5 | # e.g. a second line being emitted with no indentation. 6 | raise 7 | 8 | 9 | def print_hello(): 10 | print(hello.messages.message) 11 | -------------------------------------------------------------------------------- /examples/indented-absolute-simple-import-rewrite/hello/hello/messages.py: -------------------------------------------------------------------------------- 1 | message = "hello" 2 | -------------------------------------------------------------------------------- /examples/indented-absolute-simple-import-rewrite/hello/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="hello", 5 | packages=["hello"]) 6 | -------------------------------------------------------------------------------- /examples/indented-absolute-simple-import-rewrite/main.py: -------------------------------------------------------------------------------- 1 | from _vendor.hello import print_hello 2 | 3 | print_hello() 4 | -------------------------------------------------------------------------------- /examples/indented-absolute-simple-import-rewrite/vendorize.toml: -------------------------------------------------------------------------------- 1 | target = "_vendor" 2 | packages = [ 3 | "./hello", 4 | ] 5 | -------------------------------------------------------------------------------- /examples/isolated-module-pyproject/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from _vendor import six 4 | 5 | for key, value in six.iteritems({"one": 1}): 6 | print((key, value)) 7 | -------------------------------------------------------------------------------- /examples/isolated-module-pyproject/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.vendorize] 2 | target = "_vendor" 3 | packages = [ 4 | "six==1.10", 5 | ] 6 | -------------------------------------------------------------------------------- /examples/isolated-module/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from _vendor import six 4 | 5 | for key, value in six.iteritems({"one": 1}): 6 | print((key, value)) 7 | -------------------------------------------------------------------------------- /examples/isolated-module/vendorize.toml: -------------------------------------------------------------------------------- 1 | target = "_vendor" 2 | packages = [ 3 | "six==1.10", 4 | ] 5 | -------------------------------------------------------------------------------- /examples/local-module/hello/.gitignore: -------------------------------------------------------------------------------- 1 | /build 2 | -------------------------------------------------------------------------------- /examples/local-module/hello/hello.py: -------------------------------------------------------------------------------- 1 | def hello(): 2 | print("hello") 3 | -------------------------------------------------------------------------------- /examples/local-module/hello/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="hello", 5 | py_modules=["hello"]) 6 | -------------------------------------------------------------------------------- /examples/local-module/main.py: -------------------------------------------------------------------------------- 1 | from _vendor.hello import hello 2 | 3 | hello() 4 | -------------------------------------------------------------------------------- /examples/local-module/vendorize.toml: -------------------------------------------------------------------------------- 1 | target = "_vendor" 2 | packages = [ 3 | "./hello", 4 | ] 5 | -------------------------------------------------------------------------------- /examples/multiple-dependencies-with-rewrite/hello/.gitignore: -------------------------------------------------------------------------------- 1 | /build 2 | -------------------------------------------------------------------------------- /examples/multiple-dependencies-with-rewrite/hello/hello/__init__.py: -------------------------------------------------------------------------------- 1 | from hello.hello import hello 2 | -------------------------------------------------------------------------------- /examples/multiple-dependencies-with-rewrite/hello/hello/hello.py: -------------------------------------------------------------------------------- 1 | def hello(): 2 | print("hello") 3 | -------------------------------------------------------------------------------- /examples/multiple-dependencies-with-rewrite/hello/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="hello", 5 | packages=["hello"], 6 | ) 7 | -------------------------------------------------------------------------------- /examples/multiple-dependencies-with-rewrite/main.py: -------------------------------------------------------------------------------- 1 | from _vendor.hello import hello 2 | from _vendor.world import world 3 | 4 | hello() 5 | world() 6 | -------------------------------------------------------------------------------- /examples/multiple-dependencies-with-rewrite/vendorize.toml: -------------------------------------------------------------------------------- 1 | target = "_vendor" 2 | packages = [ 3 | "./hello", 4 | "./world", 5 | ] 6 | -------------------------------------------------------------------------------- /examples/multiple-dependencies-with-rewrite/world/.gitignore: -------------------------------------------------------------------------------- 1 | /build 2 | -------------------------------------------------------------------------------- /examples/multiple-dependencies-with-rewrite/world/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="world", 5 | packages=["world"], 6 | ) 7 | -------------------------------------------------------------------------------- /examples/multiple-dependencies-with-rewrite/world/world/__init__.py: -------------------------------------------------------------------------------- 1 | from world.world import world 2 | -------------------------------------------------------------------------------- /examples/multiple-dependencies-with-rewrite/world/world/world.py: -------------------------------------------------------------------------------- 1 | def world(): 2 | print("world") 3 | -------------------------------------------------------------------------------- /examples/multiple-dependencies/hello/.gitignore: -------------------------------------------------------------------------------- 1 | /build 2 | -------------------------------------------------------------------------------- /examples/multiple-dependencies/hello/hello.py: -------------------------------------------------------------------------------- 1 | def hello(): 2 | print("hello") 3 | -------------------------------------------------------------------------------- /examples/multiple-dependencies/hello/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="hello", 5 | py_modules=["hello"], 6 | ) 7 | -------------------------------------------------------------------------------- /examples/multiple-dependencies/main.py: -------------------------------------------------------------------------------- 1 | from _vendor.hello import hello 2 | from _vendor.world import world 3 | 4 | hello() 5 | world() 6 | -------------------------------------------------------------------------------- /examples/multiple-dependencies/vendorize.toml: -------------------------------------------------------------------------------- 1 | target = "_vendor" 2 | packages = [ 3 | "./hello", 4 | "./world", 5 | ] 6 | -------------------------------------------------------------------------------- /examples/multiple-dependencies/world/.gitignore: -------------------------------------------------------------------------------- 1 | /build 2 | -------------------------------------------------------------------------------- /examples/multiple-dependencies/world/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="world", 5 | py_modules=["world"], 6 | ) 7 | -------------------------------------------------------------------------------- /examples/multiple-dependencies/world/world.py: -------------------------------------------------------------------------------- 1 | def world(): 2 | print("world") 3 | -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test 2 | 3 | test: 4 | sh -c '. _virtualenv/bin/activate; py.test tests' 5 | 6 | .PHONY: test-all 7 | 8 | test-all: 9 | tox 10 | 11 | .PHONY: upload 12 | 13 | upload: build-dist 14 | _virtualenv/bin/twine upload dist/* 15 | make clean 16 | 17 | .PHONY: build-dist 18 | 19 | build-dist: clean 20 | _virtualenv/bin/pyproject-build 21 | 22 | .PHONY: clean 23 | 24 | clean: 25 | rm -f MANIFEST 26 | rm -rf build dist 27 | 28 | .PHONY: bootstrap 29 | 30 | bootstrap: _virtualenv 31 | _virtualenv/bin/pip install -e . 32 | ifneq ($(wildcard test-requirements.txt),) 33 | _virtualenv/bin/pip install -r test-requirements.txt 34 | endif 35 | make clean 36 | 37 | _virtualenv: 38 | python3 -m venv _virtualenv 39 | _virtualenv/bin/pip install --upgrade pip 40 | _virtualenv/bin/pip install --upgrade setuptools 41 | _virtualenv/bin/pip install --upgrade wheel 42 | _virtualenv/bin/pip install --upgrade build twine 43 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | 2 | #!/usr/bin/env python 3 | 4 | import os 5 | import sys 6 | from setuptools import setup, find_packages 7 | 8 | def read(fname): 9 | return open(os.path.join(os.path.dirname(__file__), fname)).read() 10 | 11 | 12 | setup( 13 | name='vendorize', 14 | version='0.3.0', 15 | description='Vendorize packages from PyPI', 16 | long_description=read("README.rst"), 17 | author='Michael Williamson', 18 | author_email='mike@zwobble.org', 19 | url='http://github.com/mwilliamson/python-vendorize', 20 | packages=find_packages(exclude=["tests", "tests.*", "examples", "examples.*"]), 21 | entry_points={ 22 | "console_scripts": [ 23 | "python-vendorize=vendorize.cli:main" 24 | ] 25 | }, 26 | keywords="vendor vendorize", 27 | license="BSD-2-Clause", 28 | python_requires=">=3.5", 29 | ) 30 | 31 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | spur.local>=0.3.19,<0.4 3 | tempman>=0.1.3,<0.2 4 | -------------------------------------------------------------------------------- /tests/cli_tests.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import shutil 4 | import contextlib 5 | import io 6 | 7 | from spur import LocalShell 8 | import tempman 9 | 10 | _local = LocalShell() 11 | 12 | def test_vendorizing_single_module_with_no_dependencies_grabs_one_module_file(): 13 | with _vendorize_example("isolated-module") as project_path: 14 | result = _local.run(["python", os.path.join(project_path, "main.py")]) 15 | assert b"('one', 1)" == result.output.strip() 16 | 17 | def test_can_vendorize_local_modules_from_relative_paths(): 18 | with _vendorize_example("local-module") as project_path: 19 | result = _local.run(["python", os.path.join(project_path, "main.py")]) 20 | assert b"hello\n" == result.output 21 | 22 | def test_absolute_paths_in_same_distribution_are_rewritten_to_be_relative(): 23 | with _vendorize_example("absolute-import-rewrite") as project_path: 24 | result = _local.run(["python", os.path.join(project_path, "main.py")]) 25 | assert b"hello\n" == result.output 26 | 27 | def test_can_rewrite_indented_absolute_simple_imports(): 28 | with _vendorize_example("indented-absolute-simple-import-rewrite") as project_path: 29 | result = _local.run(["python", os.path.join(project_path, "main.py")]) 30 | assert b"hello\n" == result.output 31 | 32 | def test_can_vendorize_multiple_dependencies(): 33 | with _vendorize_example("multiple-dependencies") as project_path: 34 | result = _local.run(["python", os.path.join(project_path, "main.py")]) 35 | assert b"hello\nworld\n" == result.output 36 | 37 | def test_can_vendorize_multiple_dependencies_that_require_import_rewriting(): 38 | with _vendorize_example("multiple-dependencies-with-rewrite") as project_path: 39 | result = _local.run(["python", os.path.join(project_path, "main.py")]) 40 | assert b"hello\nworld\n" == result.output 41 | 42 | def test_can_vendorize_with_pyproject_toml(): 43 | with _vendorize_example("isolated-module-pyproject") as project_path: 44 | result = _local.run(["python", os.path.join(project_path, "main.py")]) 45 | assert b"('one', 1)" == result.output.strip() 46 | 47 | @contextlib.contextmanager 48 | def _vendorize_example(example_name): 49 | path = os.path.join(os.path.dirname(__file__), "../examples", example_name) 50 | _clean_project(path) 51 | 52 | _local.run( 53 | ["python-vendorize"], 54 | cwd=path, 55 | encoding="utf-8", 56 | ) 57 | yield path 58 | 59 | 60 | def _clean_project(path): 61 | vendor_path = os.path.join(path, "_vendor") 62 | if os.path.exists(vendor_path): 63 | shutil.rmtree(vendor_path) 64 | -------------------------------------------------------------------------------- /tests/import_rewrite_tests.py: -------------------------------------------------------------------------------- 1 | from vendorize.import_rewrite import rewrite_imports_in_module 2 | 3 | 4 | def test_module_without_imports_is_unchanged(): 5 | result = rewrite_imports_in_module( 6 | "print(42)", 7 | top_level_names=[], 8 | depth=0, 9 | ) 10 | 11 | assert "print(42)" == result 12 | 13 | 14 | def test_absolute_from_import_is_unchanged_if_not_in_set_of_names_to_change(): 15 | result = rewrite_imports_in_module( 16 | "from b import c", 17 | top_level_names=["a"], 18 | depth=0, 19 | ) 20 | 21 | assert "from b import c" == result 22 | 23 | 24 | def test_absolute_from_import_is_rewritten_to_relative_import(): 25 | result = rewrite_imports_in_module( 26 | "from a import b", 27 | top_level_names=["a"], 28 | depth=0, 29 | ) 30 | 31 | assert "from .a import b" == result 32 | 33 | 34 | def test_absolute_from_import_is_rewritten_to_relative_import_according_to_depth(): 35 | result = rewrite_imports_in_module( 36 | "from a import b", 37 | top_level_names=["a"], 38 | depth=2, 39 | ) 40 | 41 | assert "from ...a import b" == result 42 | 43 | 44 | def test_absolute_from_import_is_rewritten_to_relative_import_according_to_depth(): 45 | result = rewrite_imports_in_module( 46 | "from a import b", 47 | top_level_names=["a"], 48 | depth=2, 49 | ) 50 | 51 | assert "from ...a import b" == result 52 | 53 | 54 | def test_relative_from_import_is_ignored(): 55 | result = rewrite_imports_in_module( 56 | "from . import b", 57 | top_level_names=["a"], 58 | depth=2, 59 | ) 60 | 61 | assert "from . import b" == result 62 | 63 | 64 | def test_absolute_simple_import_of_top_level_module_is_rewritten_to_relative_import(): 65 | result = rewrite_imports_in_module( 66 | "import a", 67 | top_level_names=["a"], 68 | depth=2, 69 | ) 70 | 71 | assert "from ... import a" == result 72 | 73 | 74 | def test_absolute_simple_aliased_import_of_top_level_module_is_rewritten_to_relative_import(): 75 | result = rewrite_imports_in_module( 76 | "import a as b", 77 | top_level_names=["a"], 78 | depth=2, 79 | ) 80 | 81 | assert "from ... import a as b" == result 82 | 83 | 84 | def test_absolute_simple_aliased_import_of_submodule_is_rewritten_to_relative_import(): 85 | result = rewrite_imports_in_module( 86 | "import a.b as c", 87 | top_level_names=["a"], 88 | depth=2, 89 | ) 90 | 91 | assert "from ...a import b as c" == result 92 | 93 | 94 | def test_absolute_simple_import_of_submodule_is_rewritten_to_relative_import(): 95 | result = rewrite_imports_in_module( 96 | "import a.b", 97 | top_level_names=["a"], 98 | depth=2, 99 | ) 100 | 101 | assert "from ... import a;from ...a import b as ___vendorize__0" == result 102 | 103 | 104 | def test_absolute_simple_import_of_nested_submodule_is_rewritten_to_relative_import(): 105 | result = rewrite_imports_in_module( 106 | "import a.b.c.d", 107 | top_level_names=["a"], 108 | depth=2, 109 | ) 110 | 111 | assert "from ... import a;from ...a.b.c import d as ___vendorize__0" == result 112 | 113 | 114 | def test_can_have_single_import_statement_that_uses_both_rewritten_and_unrewritten_imports(): 115 | result = rewrite_imports_in_module( 116 | "import a, b", 117 | top_level_names=["a"], 118 | depth=2, 119 | ) 120 | 121 | assert "from ... import a;import b" == result 122 | -------------------------------------------------------------------------------- /tests/python_source_tests.py: -------------------------------------------------------------------------------- 1 | import io 2 | 3 | from vendorize.python_source import find_encoding 4 | 5 | 6 | def test_empty_file_has_encoding_of_utf_8(): 7 | result = find_encoding(io.BytesIO(b"")) 8 | 9 | assert "utf-8" == result 10 | 11 | 12 | def test_encoding_can_be_read_from_first_line(): 13 | result = find_encoding(io.BytesIO(b"# encoding=latin-1")) 14 | 15 | assert "latin-1" == result 16 | 17 | 18 | def test_encoding_can_be_read_from_second_line(): 19 | result = find_encoding(io.BytesIO(b"#!/usr/bin/env python\n# encoding=latin-1")) 20 | 21 | assert "latin-1" == result 22 | 23 | 24 | def test_encoding_cannot_be_read_after_third_line(): 25 | result = find_encoding(io.BytesIO(b"\n\n# encoding=latin-1")) 26 | 27 | assert "utf-8" == result 28 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py35,pypy 3 | [testenv] 4 | changedir = {envtmpdir} 5 | deps=-r{toxinidir}/test-requirements.txt 6 | commands= 7 | py.test {toxinidir}/tests 8 | 9 | [pytest] 10 | python_files = *_tests.py 11 | -------------------------------------------------------------------------------- /vendorize.toml: -------------------------------------------------------------------------------- 1 | target = "vendorize/_vendor" 2 | packages = [ 3 | "pytoml==0.1.11", 4 | ] 5 | -------------------------------------------------------------------------------- /vendorize/__init__.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import io 3 | import os 4 | import pathlib 5 | import subprocess 6 | import sys 7 | 8 | from . import python_source 9 | from ._vendor import pytoml as toml 10 | from .files import mkdir_p, ensure_file_exists 11 | from .import_rewrite import rewrite_imports_in_module 12 | 13 | 14 | def vendorize_directory(path): 15 | config = _read_directory_config(path) 16 | 17 | return vendorize_requirements(config=config, directory_path=path) 18 | 19 | 20 | def _read_directory_config(path): 21 | try: 22 | with open(os.path.join(path, "vendorize.toml")) as fileobj: 23 | return toml.load(fileobj) 24 | except FileNotFoundError: 25 | pass 26 | 27 | try: 28 | with open(os.path.join(path, "pyproject.toml")) as fileobj: 29 | pyproject = toml.load(fileobj) 30 | return pyproject["tool"]["vendorize"] 31 | except FileNotFoundError: 32 | raise RuntimeError("Could not find vendorize config") 33 | 34 | 35 | def vendorize_requirements(config, directory_path): 36 | target_directory = os.path.join(directory_path, config["target"]) 37 | ensure_file_exists(os.path.join(target_directory, "__init__.py")) 38 | _download_requirements( 39 | cwd=directory_path or None, 40 | requirements=config["packages"], 41 | target_directory=target_directory, 42 | ) 43 | top_level_names = _read_top_level_names(target_directory) 44 | _rewrite_imports(target_directory, top_level_names) 45 | 46 | 47 | def vendorize_requirement(cwd, requirement, target_directory): 48 | _download_requirements(cwd=cwd, requirements=[requirement], target_directory=target_directory) 49 | top_level_names = _read_top_level_names(target_directory) 50 | _rewrite_imports(target_directory, top_level_names) 51 | 52 | 53 | def _download_requirements(cwd, requirements, target_directory): 54 | mkdir_p(target_directory) 55 | subprocess.check_call( 56 | [sys.executable, "-m", "pip", "install", "--no-dependencies", "--target", target_directory] + requirements, 57 | cwd=cwd) 58 | 59 | def _read_top_level_names(target_directory): 60 | top_level_names = set() 61 | 62 | for name in os.listdir(target_directory): 63 | if name.endswith(".dist-info"): 64 | path = os.path.join(target_directory, name) 65 | top_level_names.update(_read_top_level_names_from_dist_info(path)) 66 | 67 | elif name.endswith(".egg-info"): 68 | path = os.path.join(target_directory, name) 69 | top_level_names.update(_read_top_level_names_from_egg_info(path)) 70 | 71 | return top_level_names 72 | 73 | 74 | def _read_top_level_names_from_dist_info(dist_info_path): 75 | path = os.path.join(dist_info_path, "RECORD") 76 | 77 | if not os.path.exists(path): 78 | return 79 | 80 | py_extension = ".py" 81 | 82 | with open(path, "rt", encoding="utf-8") as fileobj: 83 | for line in csv.reader(fileobj): 84 | if len(line) > 0: 85 | record_path = line[0] 86 | if record_path.endswith(py_extension): 87 | top_part = pathlib.Path(record_path).parts[0] 88 | if top_part.endswith(py_extension): 89 | yield top_part[:-len(py_extension)] 90 | else: 91 | yield top_part 92 | 93 | 94 | def _read_top_level_names_from_egg_info(egg_info_path): 95 | path = os.path.join(egg_info_path, "top_level.txt") 96 | if os.path.exists(path): 97 | with open(path) as top_level_file: 98 | return list(filter(None, map(lambda line: line.strip(), top_level_file))) 99 | 100 | 101 | def _rewrite_imports(target_directory, top_level_names): 102 | for top_level_name in top_level_names: 103 | module_path = os.path.join(target_directory, top_level_name + ".py") 104 | if os.path.exists(module_path): 105 | _rewrite_imports_in_module(module_path, top_level_names, depth=0) 106 | 107 | package_path = os.path.join(target_directory, top_level_name) 108 | if os.path.exists(package_path): 109 | _rewrite_imports_in_package(package_path, top_level_names, depth=1) 110 | 111 | def _rewrite_imports_in_package(package_path, top_level_names, depth): 112 | for name in os.listdir(package_path): 113 | child_path = os.path.join(package_path, name) 114 | if name.endswith(".py"): 115 | _rewrite_imports_in_module(child_path, top_level_names, depth=depth) 116 | 117 | if os.path.isdir(child_path): 118 | _rewrite_imports_in_package(child_path, top_level_names, depth=depth + 1) 119 | 120 | 121 | def _rewrite_imports_in_module(module_path, top_level_names, depth): 122 | with io.open(module_path, "rb") as source_file: 123 | encoding = python_source.find_encoding(source_file) 124 | 125 | with io.open(module_path, "r", encoding=encoding, newline='') as source_file: 126 | source = source_file.read() 127 | 128 | rewritten_source = rewrite_imports_in_module(source, top_level_names, depth) 129 | 130 | with io.open(module_path, "w", encoding=encoding, newline='') as source_file: 131 | source_file.write(rewritten_source) 132 | 133 | pyc_path = os.path.splitext(module_path)[0] + ".pyc" 134 | if os.path.exists(pyc_path): 135 | os.unlink(pyc_path) 136 | -------------------------------------------------------------------------------- /vendorize/_vendor/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mwilliamson/python-vendorize/04d1ae6c636666ea78447134c37e41c082f15190/vendorize/_vendor/__init__.py -------------------------------------------------------------------------------- /vendorize/_vendor/pytoml/__init__.py: -------------------------------------------------------------------------------- 1 | from .core import TomlError 2 | from .parser import load, loads 3 | from .writer import dump, dumps 4 | -------------------------------------------------------------------------------- /vendorize/_vendor/pytoml/core.py: -------------------------------------------------------------------------------- 1 | class TomlError(RuntimeError): 2 | def __init__(self, message, line, col, filename): 3 | RuntimeError.__init__(self, message, line, col, filename) 4 | self.message = message 5 | self.line = line 6 | self.col = col 7 | self.filename = filename 8 | 9 | def __str__(self): 10 | return '{}({}, {}): {}'.format(self.filename, self.line, self.col, self.message) 11 | 12 | def __repr__(self): 13 | return 'TomlError({!r}, {!r}, {!r}, {!r})'.format(self.message, self.line, self.col, self.filename) 14 | -------------------------------------------------------------------------------- /vendorize/_vendor/pytoml/parser.py: -------------------------------------------------------------------------------- 1 | import string, re, sys, datetime 2 | from .core import TomlError 3 | 4 | if sys.version_info[0] == 2: 5 | _chr = unichr 6 | else: 7 | _chr = chr 8 | 9 | def load(fin, translate=lambda t, x, v: v): 10 | return loads(fin.read(), translate=translate, filename=fin.name) 11 | 12 | def loads(s, filename='', translate=lambda t, x, v: v): 13 | if isinstance(s, bytes): 14 | s = s.decode('utf-8') 15 | 16 | s = s.replace('\r\n', '\n') 17 | 18 | root = {} 19 | tables = {} 20 | scope = root 21 | 22 | src = _Source(s, filename=filename) 23 | ast = _p_toml(src) 24 | 25 | def error(msg): 26 | raise TomlError(msg, pos[0], pos[1], filename) 27 | 28 | def process_value(v): 29 | kind, text, value, pos = v 30 | if kind == 'str' and value.startswith('\n'): 31 | value = value[1:] 32 | if kind == 'array': 33 | if value and any(k != value[0][0] for k, t, v, p in value[1:]): 34 | error('array-type-mismatch') 35 | value = [process_value(item) for item in value] 36 | elif kind == 'table': 37 | value = dict([(k, process_value(value[k])) for k in value]) 38 | return translate(kind, text, value) 39 | 40 | for kind, value, pos in ast: 41 | if kind == 'kv': 42 | k, v = value 43 | if k in scope: 44 | error('duplicate_keys. Key "{0}" was used more than once.'.format(k)) 45 | scope[k] = process_value(v) 46 | else: 47 | is_table_array = (kind == 'table_array') 48 | cur = tables 49 | for name in value[:-1]: 50 | if isinstance(cur.get(name), list): 51 | d, cur = cur[name][-1] 52 | else: 53 | d, cur = cur.setdefault(name, (None, {})) 54 | 55 | scope = {} 56 | name = value[-1] 57 | if name not in cur: 58 | if is_table_array: 59 | cur[name] = [(scope, {})] 60 | else: 61 | cur[name] = (scope, {}) 62 | elif isinstance(cur[name], list): 63 | if not is_table_array: 64 | error('table_type_mismatch') 65 | cur[name].append((scope, {})) 66 | else: 67 | if is_table_array: 68 | error('table_type_mismatch') 69 | old_scope, next_table = cur[name] 70 | if old_scope is not None: 71 | error('duplicate_tables') 72 | cur[name] = (scope, next_table) 73 | 74 | def merge_tables(scope, tables): 75 | if scope is None: 76 | scope = {} 77 | for k in tables: 78 | if k in scope: 79 | error('key_table_conflict') 80 | v = tables[k] 81 | if isinstance(v, list): 82 | scope[k] = [merge_tables(sc, tbl) for sc, tbl in v] 83 | else: 84 | scope[k] = merge_tables(v[0], v[1]) 85 | return scope 86 | 87 | return merge_tables(root, tables) 88 | 89 | class _Source: 90 | def __init__(self, s, filename=None): 91 | self.s = s 92 | self._pos = (1, 1) 93 | self._last = None 94 | self._filename = filename 95 | self.backtrack_stack = [] 96 | 97 | def last(self): 98 | return self._last 99 | 100 | def pos(self): 101 | return self._pos 102 | 103 | def fail(self): 104 | return self._expect(None) 105 | 106 | def consume_dot(self): 107 | if self.s: 108 | self._last = self.s[0] 109 | self.s = self[1:] 110 | self._advance(self._last) 111 | return self._last 112 | return None 113 | 114 | def expect_dot(self): 115 | return self._expect(self.consume_dot()) 116 | 117 | def consume_eof(self): 118 | if not self.s: 119 | self._last = '' 120 | return True 121 | return False 122 | 123 | def expect_eof(self): 124 | return self._expect(self.consume_eof()) 125 | 126 | def consume(self, s): 127 | if self.s.startswith(s): 128 | self.s = self.s[len(s):] 129 | self._last = s 130 | self._advance(s) 131 | return True 132 | return False 133 | 134 | def expect(self, s): 135 | return self._expect(self.consume(s)) 136 | 137 | def consume_re(self, re): 138 | m = re.match(self.s) 139 | if m: 140 | self.s = self.s[len(m.group(0)):] 141 | self._last = m 142 | self._advance(m.group(0)) 143 | return m 144 | return None 145 | 146 | def expect_re(self, re): 147 | return self._expect(self.consume_re(re)) 148 | 149 | def __enter__(self): 150 | self.backtrack_stack.append((self.s, self._pos)) 151 | 152 | def __exit__(self, type, value, traceback): 153 | if type is None: 154 | self.backtrack_stack.pop() 155 | else: 156 | self.s, self._pos = self.backtrack_stack.pop() 157 | return type == TomlError 158 | 159 | def commit(self): 160 | self.backtrack_stack[-1] = (self.s, self._pos) 161 | 162 | def _expect(self, r): 163 | if not r: 164 | raise TomlError('msg', self._pos[0], self._pos[1], self._filename) 165 | return r 166 | 167 | def _advance(self, s): 168 | suffix_pos = s.rfind('\n') 169 | if suffix_pos == -1: 170 | self._pos = (self._pos[0], self._pos[1] + len(s)) 171 | else: 172 | self._pos = (self._pos[0] + s.count('\n'), len(s) - suffix_pos) 173 | 174 | _ews_re = re.compile(r'(?:[ \t]|#[^\n]*\n|#[^\n]*\Z|\n)*') 175 | def _p_ews(s): 176 | s.expect_re(_ews_re) 177 | 178 | _ws_re = re.compile(r'[ \t]*') 179 | def _p_ws(s): 180 | s.expect_re(_ws_re) 181 | 182 | _escapes = { 'b': '\b', 'n': '\n', 'r': '\r', 't': '\t', '"': '"', '\'': '\'', 183 | '\\': '\\', '/': '/', 'f': '\f' } 184 | 185 | _basicstr_re = re.compile(r'[^"\\\000-\037]*') 186 | _short_uni_re = re.compile(r'u([0-9a-fA-F]{4})') 187 | _long_uni_re = re.compile(r'U([0-9a-fA-F]{8})') 188 | _escapes_re = re.compile('[bnrt"\'\\\\/f]') 189 | _newline_esc_re = re.compile('\n[ \t\n]*') 190 | def _p_basicstr_content(s, content=_basicstr_re): 191 | res = [] 192 | while True: 193 | res.append(s.expect_re(content).group(0)) 194 | if not s.consume('\\'): 195 | break 196 | if s.consume_re(_newline_esc_re): 197 | pass 198 | elif s.consume_re(_short_uni_re) or s.consume_re(_long_uni_re): 199 | res.append(_chr(int(s.last().group(1), 16))) 200 | else: 201 | s.expect_re(_escapes_re) 202 | res.append(_escapes[s.last().group(0)]) 203 | return ''.join(res) 204 | 205 | _key_re = re.compile(r'[0-9a-zA-Z-_]+') 206 | def _p_key(s): 207 | with s: 208 | s.expect('"') 209 | r = _p_basicstr_content(s, _basicstr_re) 210 | s.expect('"') 211 | return r 212 | return s.expect_re(_key_re).group(0) 213 | 214 | _float_re = re.compile(r'[+-]?(?:0|[1-9](?:_?\d)*)(?:\.\d(?:_?\d)*)?(?:[eE][+-]?(?:\d(?:_?\d)*))?') 215 | _datetime_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))') 216 | 217 | _basicstr_ml_re = re.compile(r'(?:(?:|"|"")[^"\\\000-\011\013-\037])*') 218 | _litstr_re = re.compile(r"[^'\000-\037]*") 219 | _litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\011\013-\037]))*") 220 | def _p_value(s): 221 | pos = s.pos() 222 | 223 | if s.consume('true'): 224 | return 'bool', s.last(), True, pos 225 | if s.consume('false'): 226 | return 'bool', s.last(), False, pos 227 | 228 | if s.consume('"'): 229 | if s.consume('""'): 230 | r = _p_basicstr_content(s, _basicstr_ml_re) 231 | s.expect('"""') 232 | else: 233 | r = _p_basicstr_content(s, _basicstr_re) 234 | s.expect('"') 235 | return 'str', r, r, pos 236 | 237 | if s.consume('\''): 238 | if s.consume('\'\''): 239 | r = s.expect_re(_litstr_ml_re).group(0) 240 | s.expect('\'\'\'') 241 | else: 242 | r = s.expect_re(_litstr_re).group(0) 243 | s.expect('\'') 244 | return 'str', r, r, pos 245 | 246 | if s.consume_re(_datetime_re): 247 | m = s.last() 248 | s0 = m.group(0) 249 | r = map(int, m.groups()[:6]) 250 | if m.group(7): 251 | micro = float(m.group(7)) 252 | else: 253 | micro = 0 254 | 255 | if m.group(8): 256 | g = int(m.group(8), 10) * 60 + int(m.group(9), 10) 257 | tz = _TimeZone(datetime.timedelta(0, g * 60)) 258 | else: 259 | tz = _TimeZone(datetime.timedelta(0, 0)) 260 | 261 | y, m, d, H, M, S = r 262 | dt = datetime.datetime(y, m, d, H, M, S, int(micro * 1000000), tz) 263 | return 'datetime', s0, dt, pos 264 | 265 | if s.consume_re(_float_re): 266 | m = s.last().group(0) 267 | r = m.replace('_','') 268 | if '.' in m or 'e' in m or 'E' in m: 269 | return 'float', m, float(r), pos 270 | else: 271 | return 'int', m, int(r, 10), pos 272 | 273 | if s.consume('['): 274 | items = [] 275 | with s: 276 | while True: 277 | _p_ews(s) 278 | items.append(_p_value(s)) 279 | s.commit() 280 | _p_ews(s) 281 | s.expect(',') 282 | s.commit() 283 | _p_ews(s) 284 | s.expect(']') 285 | return 'array', None, items, pos 286 | 287 | if s.consume('{'): 288 | _p_ws(s) 289 | items = {} 290 | if not s.consume('}'): 291 | k = _p_key(s) 292 | _p_ws(s) 293 | s.expect('=') 294 | _p_ws(s) 295 | items[k] = _p_value(s) 296 | _p_ws(s) 297 | while s.consume(','): 298 | _p_ws(s) 299 | k = _p_key(s) 300 | _p_ws(s) 301 | s.expect('=') 302 | _p_ws(s) 303 | items[k] = _p_value(s) 304 | _p_ws(s) 305 | s.expect('}') 306 | return 'table', None, items, pos 307 | 308 | s.fail() 309 | 310 | def _p_stmt(s): 311 | pos = s.pos() 312 | if s.consume( '['): 313 | is_array = s.consume('[') 314 | _p_ws(s) 315 | keys = [_p_key(s)] 316 | _p_ws(s) 317 | while s.consume('.'): 318 | _p_ws(s) 319 | keys.append(_p_key(s)) 320 | _p_ws(s) 321 | s.expect(']') 322 | if is_array: 323 | s.expect(']') 324 | return 'table_array' if is_array else 'table', keys, pos 325 | 326 | key = _p_key(s) 327 | _p_ws(s) 328 | s.expect('=') 329 | _p_ws(s) 330 | value = _p_value(s) 331 | return 'kv', (key, value), pos 332 | 333 | _stmtsep_re = re.compile(r'(?:[ \t]*(?:#[^\n]*)?\n)+[ \t]*') 334 | def _p_toml(s): 335 | stmts = [] 336 | _p_ews(s) 337 | with s: 338 | stmts.append(_p_stmt(s)) 339 | while True: 340 | s.commit() 341 | s.expect_re(_stmtsep_re) 342 | stmts.append(_p_stmt(s)) 343 | _p_ews(s) 344 | s.expect_eof() 345 | return stmts 346 | 347 | class _TimeZone(datetime.tzinfo): 348 | def __init__(self, offset): 349 | self._offset = offset 350 | 351 | def utcoffset(self, dt): 352 | return self._offset 353 | 354 | def dst(self, dt): 355 | return None 356 | 357 | def tzname(self, dt): 358 | m = self._offset.total_seconds() // 60 359 | if m < 0: 360 | res = '-' 361 | m = -m 362 | else: 363 | res = '+' 364 | h = m // 60 365 | m = m - h * 60 366 | return '{}{:.02}{:.02}'.format(res, h, m) 367 | -------------------------------------------------------------------------------- /vendorize/_vendor/pytoml/writer.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | import io, datetime, sys 3 | 4 | if sys.version_info[0] == 3: 5 | long = int 6 | unicode = str 7 | 8 | 9 | def dumps(obj, sort_keys=False): 10 | fout = io.StringIO() 11 | dump(fout, obj, sort_keys=sort_keys) 12 | return fout.getvalue() 13 | 14 | 15 | _escapes = {'\n': 'n', '\r': 'r', '\\': '\\', '\t': 't', '\b': 'b', '\f': 'f', '"': '"'} 16 | 17 | 18 | def _escape_string(s): 19 | res = [] 20 | start = 0 21 | 22 | def flush(): 23 | if start != i: 24 | res.append(s[start:i]) 25 | return i + 1 26 | 27 | i = 0 28 | while i < len(s): 29 | c = s[i] 30 | if c in '"\\\n\r\t\b\f': 31 | start = flush() 32 | res.append('\\' + _escapes[c]) 33 | elif ord(c) < 0x20: 34 | start = flush() 35 | res.append('\\u%04x' % ord(c)) 36 | i += 1 37 | 38 | flush() 39 | return '"' + ''.join(res) + '"' 40 | 41 | 42 | def _escape_id(s): 43 | if any(not c.isalnum() and c not in '-_' for c in s): 44 | return _escape_string(s) 45 | return s 46 | 47 | 48 | def _format_list(v): 49 | return '[{0}]'.format(', '.join(_format_value(obj) for obj in v)) 50 | 51 | # Formula from: 52 | # https://docs.python.org/2/library/datetime.html#datetime.timedelta.total_seconds 53 | # Once support for py26 is dropped, this can be replaced by td.total_seconds() 54 | def _total_seconds(td): 55 | return ((td.microseconds 56 | + (td.seconds + td.days * 24 * 3600) * 10**6) / 10.0**6) 57 | 58 | def _format_value(v): 59 | if isinstance(v, bool): 60 | return 'true' if v else 'false' 61 | if isinstance(v, int) or isinstance(v, long): 62 | return unicode(v) 63 | if isinstance(v, float): 64 | return '{0:.17f}'.format(v) 65 | elif isinstance(v, unicode) or isinstance(v, bytes): 66 | return _escape_string(v) 67 | elif isinstance(v, datetime.datetime): 68 | offs = v.utcoffset() 69 | offs = _total_seconds(offs) // 60 if offs is not None else 0 70 | 71 | if offs == 0: 72 | suffix = 'Z' 73 | else: 74 | if offs > 0: 75 | suffix = '+' 76 | else: 77 | suffix = '-' 78 | offs = -offs 79 | suffix = '{0}{1:.02}{2:.02}'.format(suffix, offs // 60, offs % 60) 80 | 81 | if v.microsecond: 82 | return v.strftime('%Y-%m-%dT%H:%M:%S.%f') + suffix 83 | else: 84 | return v.strftime('%Y-%m-%dT%H:%M:%S') + suffix 85 | elif isinstance(v, list): 86 | return _format_list(v) 87 | else: 88 | raise RuntimeError(v) 89 | 90 | 91 | def dump(fout, obj, sort_keys=False): 92 | tables = [((), obj, False)] 93 | 94 | while tables: 95 | name, table, is_array = tables.pop() 96 | if name: 97 | section_name = '.'.join(_escape_id(c) for c in name) 98 | if is_array: 99 | fout.write('[[{0}]]\n'.format(section_name)) 100 | else: 101 | fout.write('[{0}]\n'.format(section_name)) 102 | 103 | table_keys = sorted(table.keys()) if sort_keys else table.keys() 104 | new_tables = [] 105 | for k in table_keys: 106 | v = table[k] 107 | if isinstance(v, dict): 108 | new_tables.append((name + (k,), v, False)) 109 | elif isinstance(v, list) and v and all(isinstance(o, dict) for o in v): 110 | new_tables.extend((name + (k,), d, True) for d in v) 111 | elif v is None: 112 | # based on mojombo's comment: https://github.com/toml-lang/toml/issues/146#issuecomment-25019344 113 | fout.write( 114 | '#{} = null # To use: uncomment and replace null with value\n'.format(_escape_id(k))) 115 | else: 116 | fout.write('{0} = {1}\n'.format(_escape_id(k), _format_value(v))) 117 | 118 | tables.extend(reversed(new_tables)) 119 | 120 | if tables: 121 | fout.write('\n') 122 | -------------------------------------------------------------------------------- /vendorize/cli.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from . import vendorize_directory, vendorize_requirement, vendorize_requirements 4 | 5 | 6 | def main(): 7 | if len(sys.argv) < 2: 8 | vendorize_directory(".") 9 | elif len(sys.argv) < 3: 10 | vendorize_requirements(path=sys.argv[1]) 11 | else: 12 | requirement = sys.argv[1] 13 | target_directory = sys.argv[2] 14 | vendorize_requirement(cwd=".", requirement=requirement, target_directory=target_directory) 15 | 16 | 17 | if __name__ == "__main__": 18 | main() 19 | -------------------------------------------------------------------------------- /vendorize/files.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | def mkdir_p(path): 5 | if not os.path.exists(path): 6 | os.makedirs(path) 7 | 8 | 9 | def ensure_file_exists(path): 10 | if not os.path.exists(path): 11 | mkdir_p(os.path.dirname(path)) 12 | open(path, "w").close() 13 | -------------------------------------------------------------------------------- /vendorize/import_rewrite.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import tokenize 3 | import io 4 | import collections 5 | import itertools 6 | 7 | 8 | def rewrite_imports_in_module(source, top_level_names, depth): 9 | source_lines = source.splitlines(True) 10 | line_endings = list(_find_line_endings(source)) 11 | 12 | def _find_line_ending(position): 13 | for line_ending in line_endings: 14 | if line_ending >= position: 15 | return line_ending 16 | 17 | raise Exception("Could not find line ending") 18 | 19 | def _should_rewrite_import(name): 20 | return name.split(".")[0] in top_level_names 21 | 22 | def _generate_simple_import_replacement(node): 23 | temp_index = itertools.count() 24 | 25 | replacement = [] 26 | 27 | for name in node.names: 28 | if _should_rewrite_import(name.name): 29 | parts = name.name.split(".") 30 | if name.asname is None: 31 | replacement.append("from ." + ("." * depth) + " import " + parts[0]) 32 | variable_name = "___vendorize__{0}".format(next(temp_index)) 33 | if len(parts) > 1: 34 | replacement.append( 35 | "from ." + ("." * depth) + ".".join(parts[:-1]) + 36 | " import " + parts[-1] + 37 | " as " + variable_name 38 | ) 39 | else: 40 | replacement.append( 41 | "from ." + ("." * depth) + ".".join(parts[:-1]) + 42 | " import " + parts[-1] + 43 | " as " + name.asname) 44 | else: 45 | statement = "import " + name.name 46 | if name.asname is not None: 47 | statement += " as " + name.asname 48 | replacement.append(statement) 49 | 50 | _, line_ending_col_offset = _find_line_ending((node.lineno, node.col_offset)) 51 | return _Replacement( 52 | _Location(node.lineno, node.col_offset), 53 | # TODO: handle multi-line statements 54 | line_ending_col_offset - node.col_offset, 55 | ";".join(replacement)) 56 | 57 | def _generate_import_from_replacement(node): 58 | line = source_lines[node.lineno - 1] 59 | col_offset = node.col_offset 60 | from_keyword = "from" 61 | assert line[col_offset:col_offset + len(from_keyword)] == from_keyword 62 | col_offset += len(from_keyword) 63 | while line[col_offset].isspace(): 64 | col_offset += 1 65 | return _Replacement( 66 | _Location(node.lineno, col_offset), 67 | 0, 68 | "." + ("." * depth)) 69 | 70 | replacements = [] 71 | 72 | class ImportVisitor(ast.NodeVisitor): 73 | def visit_Import(self, node): 74 | if any(_should_rewrite_import(name.name) for name in node.names): 75 | replacements.append(_generate_simple_import_replacement(node)) 76 | 77 | def visit_ImportFrom(self, node): 78 | if not node.level and _should_rewrite_import(node.module): 79 | replacements.append(_generate_import_from_replacement(node)) 80 | 81 | python_ast = ast.parse(source) 82 | ImportVisitor().visit(python_ast) 83 | return _replace_strings(source, replacements) 84 | 85 | 86 | def _find_line_endings(source): 87 | token_stream = tokenize.generate_tokens(io.StringIO(source + "\n").readline) 88 | for token_type, token_str, start, end, line in token_stream: 89 | if token_type == tokenize.NEWLINE: 90 | yield start 91 | 92 | 93 | _Location = collections.namedtuple("_Location", ["lineno", "col_offset"]) 94 | 95 | _Replacement = collections.namedtuple("_Replacement", [ 96 | "location", 97 | "length", 98 | "value" 99 | ]) 100 | 101 | def _replace_strings(source, replacements): 102 | lines = source.splitlines(True) 103 | 104 | replacements = sorted(replacements, key=lambda replacement: replacement.location, reverse=True) 105 | 106 | for replacement in replacements: 107 | line_index = replacement.location.lineno - 1 108 | col_offset = replacement.location.col_offset 109 | lines[line_index] = _str_replace(lines[line_index], replacement.length, col_offset, replacement.value) 110 | 111 | return "".join(lines) 112 | 113 | 114 | def _str_replace(original, length, index, to_insert): 115 | return original[:index] + to_insert + original[index + length:] 116 | 117 | 118 | -------------------------------------------------------------------------------- /vendorize/python_source.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | 4 | def find_encoding(fileobj): 5 | for line_index in range(0, 2): 6 | line = fileobj.readline() 7 | # From PEP 263 8 | match = re.match(br"^[ \t\f]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)", line) 9 | if match is not None: 10 | return match.group(1).decode("ascii") 11 | 12 | return "utf-8" 13 | -------------------------------------------------------------------------------- /vendorize/setuptools_build.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2007-2014 The pip developers (see AUTHORS.txt file) 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining 4 | # a copy of this software and associated documentation files (the 5 | # "Software"), to deal in the Software without restriction, including 6 | # without limitation the rights to use, copy, modify, merge, publish, 7 | # distribute, sublicense, and/or sell copies of the Software, and to 8 | # permit persons to whom the Software is furnished to do so, subject to 9 | # the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be 12 | # included in all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | 22 | 23 | # Shim to wrap setup.py invocation with setuptools 24 | SETUPTOOLS_SHIM = ( 25 | "import setuptools, tokenize;__file__=%r;" 26 | "exec(compile(getattr(tokenize, 'open', open)(__file__).read()" 27 | ".replace('\\r\\n', '\\n'), __file__, 'exec'))" 28 | ) 29 | --------------------------------------------------------------------------------