├── .gitignore ├── .travis.yml ├── BUILD.bazel ├── LICENSE ├── README.md ├── WORKSPACE ├── pazel ├── BUILD ├── __init__.py ├── app.py ├── bazel_rules.py ├── generate_rule.py ├── helpers.py ├── import_inference_rules.py ├── output_build.py ├── parse_build.py ├── parse_imports.py ├── pazel_extensions.py └── tests │ ├── BUILD │ ├── __init__.py │ ├── test_bazel_rules.py │ ├── test_generate_rule.py │ ├── test_helpers.py │ ├── test_parse_imports.py │ └── test_pazel_extensions.py ├── sample_app ├── .pazelrc ├── BUILD ├── WORKSPACE ├── __init__.py ├── custom_rules.bzl ├── external │ └── requirements-pip.txt ├── foo │ ├── BUILD │ ├── __init__.py │ ├── bar1.py │ ├── bar2.py │ ├── bar3.py │ ├── bar4.py │ ├── bar5.py │ ├── bar6.py │ └── foo.py ├── tests │ ├── BUILD │ ├── __init__.py │ ├── test_bar1.py │ ├── test_bar2.py │ ├── test_bar3.py │ ├── test_data │ │ └── dummy │ └── test_doctest.py └── xyz │ ├── BUILD │ └── abc1.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | matrix: 3 | include: 4 | - os: linux 5 | python: 2.7 6 | - os: linux 7 | python: 3.6 8 | - os: osx 9 | language: generic 10 | script: 11 | - python setup.py test -------------------------------------------------------------------------------- /BUILD.bazel: -------------------------------------------------------------------------------- 1 | # This file is called BUILD.bazel instead of BUILD to avoid name collision with 'build' directory 2 | # generated by 'python setup.py install'. 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Tuomas Rintamäki 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pazel - generate Bazel BUILD files for Python 2 | 3 | [![Build Status](https://travis-ci.org/tuomasr/pazel.svg?branch=master)](https://travis-ci.org/tuomasr/pazel) 4 | 5 | ## Requirements 6 | 7 | ### pazel 8 | No requirements. Tested on Python 2.7 and 3.6 on Ubuntu 16.04 and macOS High Sierra. 9 | 10 | ### Bazel 11 | Tested on Bazel 0.11.1. All recent versions are expected to work. 12 | 13 | ## Installation 14 | 15 | ``` 16 | > git clone https://github.com/tuomasr/pazel.git 17 | > cd pazel 18 | > python setup.py install 19 | ``` 20 | 21 | ## Usage 22 | 23 | NOTE: `pazel` overwrites any existing BUILD files. Please use version control or have backups of 24 | your current BUILD files before using `pazel`. 25 | 26 | ### Default usage with Bazel 27 | 28 | The following example generates all BUILD files for the sample Python project in `sample_app`. 29 | Start from the `pazel` root directory to which the repository was cloned. 30 | 31 | ``` 32 | > bazel run //pazel:app -- /sample_app -r /sample_app 33 | -c /sample_app/.pazelrc 34 | Generated BUILD files for /sample_app. 35 | ``` 36 | 37 | ### Default usage without Bazel 38 | 39 | Start from the `pazel` root directory. 40 | 41 | ``` 42 | > cd sample_app 43 | > pazel 44 | Generated BUILD files for /sample_app. 45 | ``` 46 | 47 | ### Testing the generated BUILD files 48 | 49 | Now, we can build, test, and run the sample project by running the following invocations in the 50 | `sample_app` directory, respectively. 51 | 52 | ``` 53 | > bazel build 54 | > bazel test ... 55 | > bazel run foo:bar3 56 | ``` 57 | 58 | ### Command-line options 59 | 60 | `pazel -h` shows a summary of the command-line options. Each of them is explained below. 61 | 62 | By default, BUILD files are generated recursively for the current working directory. 63 | Use `pazel ` to generate BUILD file(s) recursively for another directory 64 | or for a single Python file. 65 | 66 | All imports are assumed to be relative to the current working directory. For example, 67 | `sample_app/foo/bar2.py` imports from `sample_app/foo/bar1.py` using `from foo.bar1 import sample`. 68 | Use `pazel -r ` to override the path to which the imports are relative. 69 | 70 | By default, `pazel` adds rules to install all external Python packages. If your environment has 71 | pre-installed packages for which these rules are not required, then use `pazel -p`. 72 | 73 | `pazel` config file `.pazelrc` is read from the current working directory. Use 74 | `pazel -c ` to specify an alternative path. 75 | 76 | ### Ignoring rules in existing BUILD files 77 | 78 | The tag `# pazel-ignore` causes `pazel` to ignore the rule that immediately follows the tag in an 79 | existing BUILD file. In particular, the tag can be used to skip custom rules that `pazel` does not 80 | handle. `pazel` places the ignored rules at the bottom of the BUILD file. See `sample_app/foo/BUILD` 81 | for an example using the tag. 82 | 83 | 84 | ### Customizing and extending pazel 85 | 86 | `pazel` can be programmed using a `.pazelrc` Python file, which is read from the current 87 | working directory or provided explicitly with `pazel -c `. 88 | 89 | The user can define variables `HEADER` and `FOOTER` to add custom header and footer to 90 | all BUILD files, respectively. See `sample_app/.pazelrc` and `sample_app/BUILD` for an example that 91 | adds the same `visibility` to all BUILD files. 92 | 93 | If some pip package has different install name than import name, then the user 94 | should define `EXTRA_IMPORT_NAME_TO_PIP_NAME` dictionary accordingly. `sample_app/.pazelrc` has 95 | `{'yaml': 'pyyaml'}` as an example. In addition, the user can specify local packages and their 96 | corresponding Bazel dependencies using the `EXTRA_LOCAL_IMPORT_NAME_TO_DEP` dictionary. 97 | 98 | The user can add support for custom Bazel rules by defining a new class implementing the `BazelRule` 99 | interface in `pazel/bazel_rules.py` and by adding that class to `EXTRA_BAZEL_RULES` list in 100 | `.pazelrc`. `sample_app/.pazelrc` defines a custom `PyDoctestRule` class that identifies all 101 | doctests and generates custom `py_doctest` Bazel rules for them as defined in 102 | `sample_app/custom_rules.bzl`. 103 | 104 | In addition, the user can implement custom rules for mapping Python imports to Bazel dependencies 105 | that are not natively supported. That is achieved by defining a new class implementing the 106 | `InferenceImportRule` interface in `pazel/import_inference_rules.py` and by adding the class to 107 | `EXTRA_IMPORT_INFERENCE_RULES` list in `.pazelrc`. `sample_app/.pazelrc` defines a custom 108 | `LocalImportAllInferenceRule` class that generates the correct Bazel dependencies for 109 | `from X import *` type of imports where `X` is a local package. 110 | 111 | 112 | ## BUILD file formatting 113 | 114 | `pazel` generates BUILD files that are nearly compatible with 115 | [Buildifier](https://github.com/bazelbuild/buildtools/tree/master/buildifier). Buildifier can be 116 | applied on `pazel`-generated BUILD files to remove the remaining differences, if needed. -------------------------------------------------------------------------------- /WORKSPACE: -------------------------------------------------------------------------------- 1 | workspace(name = "pazel") 2 | -------------------------------------------------------------------------------- /pazel/BUILD: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | py_library( 4 | name = "__init__", 5 | srcs = ["__init__.py"], 6 | deps = [], 7 | ) 8 | 9 | py_binary( 10 | name = "app", 11 | srcs = ["app.py"], 12 | deps = [ 13 | ":generate_rule", 14 | ":helpers", 15 | ":output_build", 16 | ":parse_build", 17 | ":pazel_extensions", 18 | ], 19 | ) 20 | 21 | py_library( 22 | name = "bazel_rules", 23 | srcs = ["bazel_rules.py"], 24 | deps = [], 25 | ) 26 | 27 | py_library( 28 | name = "generate_rule", 29 | srcs = ["generate_rule.py"], 30 | deps = [ 31 | ":bazel_rules", 32 | ":parse_build", 33 | ":parse_imports", 34 | ], 35 | ) 36 | 37 | py_library( 38 | name = "helpers", 39 | srcs = ["helpers.py"], 40 | deps = [], 41 | ) 42 | 43 | py_library( 44 | name = "output_build", 45 | srcs = ["output_build.py"], 46 | deps = [], 47 | ) 48 | 49 | py_library( 50 | name = "parse_build", 51 | srcs = ["parse_build.py"], 52 | deps = [":helpers"], 53 | ) 54 | 55 | py_library( 56 | name = "parse_imports", 57 | srcs = ["parse_imports.py"], 58 | deps = [":helpers"], 59 | ) 60 | 61 | py_library( 62 | name = "pazel_extensions", 63 | srcs = ["pazel_extensions.py"], 64 | deps = [], 65 | ) 66 | -------------------------------------------------------------------------------- /pazel/__init__.py: -------------------------------------------------------------------------------- 1 | """pazel - Generate Bazel BUILD files for Python.""" 2 | -------------------------------------------------------------------------------- /pazel/app.py: -------------------------------------------------------------------------------- 1 | """Entrypoint for generating Bazel BUILD files for a Python project.""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import argparse 8 | import os 9 | 10 | from pazel.generate_rule import parse_script_and_generate_rule 11 | from pazel.helpers import get_build_file_path 12 | from pazel.helpers import is_ignored 13 | from pazel.helpers import is_python_file 14 | from pazel.output_build import output_build_file 15 | from pazel.parse_build import get_ignored_rules 16 | from pazel.pazel_extensions import parse_pazel_extensions 17 | 18 | 19 | def app(input_path, project_root, contains_pre_installed_packages, pazelrc_path): 20 | """Generate BUILD file(s) for a Python script or a directory of Python scripts. 21 | 22 | Args: 23 | input_path (str): Path to a Python file or to a directory containing Python file(s) for 24 | which BUILD files are generated. 25 | project_root (str): Imports in the Python files are relative to this path. 26 | contains_pre_installed_packages (bool): Whether the environment is allowed to contain 27 | pre-installed packages or whether only the Python standard library is available. 28 | pazelrc_path (str): Path to .pazelrc config file for customizing pazel. 29 | 30 | Raises: 31 | RuntimeError: input_path does is not a directory or a Python file. 32 | """ 33 | # Parse user-defined extensions to pazel. 34 | output_extension, custom_bazel_rules, custom_import_inference_rules, import_name_to_pip_name, \ 35 | local_import_name_to_dep, requirement_load = parse_pazel_extensions(pazelrc_path) 36 | 37 | # Handle directories. 38 | if os.path.isdir(input_path): 39 | # Traverse the directory recursively. 40 | for dirpath, _, filenames in os.walk(input_path): 41 | build_source = '' 42 | 43 | # Parse ignored rules in an existing BUILD file, if any. 44 | build_file_path = get_build_file_path(dirpath) 45 | ignored_rules = get_ignored_rules(build_file_path) 46 | 47 | for filename in sorted(filenames): 48 | path = os.path.join(dirpath, filename) 49 | 50 | # If a Python file is met and it is not in the list of ignored rules, 51 | # generate a Bazel rule for it. 52 | if is_python_file(path) and not is_ignored(path, ignored_rules): 53 | new_rule = parse_script_and_generate_rule(path, project_root, 54 | contains_pre_installed_packages, 55 | custom_bazel_rules, 56 | custom_import_inference_rules, 57 | import_name_to_pip_name, 58 | local_import_name_to_dep) 59 | 60 | # Add the new rule and a newline between it and any previous rules. 61 | if new_rule: 62 | if build_source: 63 | build_source += 2*'\n' 64 | 65 | build_source += new_rule 66 | 67 | # If Python files were found, output the BUILD file. 68 | if build_source != '' or ignored_rules: 69 | output_build_file(build_source, ignored_rules, output_extension, custom_bazel_rules, 70 | build_file_path, requirement_load) 71 | # Handle single Python file. 72 | elif is_python_file(input_path): 73 | build_source = '' 74 | 75 | # Parse ignored rules in an existing BUILD file, if any. 76 | build_file_path = get_build_file_path(input_path) 77 | ignored_rules = get_ignored_rules(build_file_path) 78 | 79 | # Check that the script is not in the list of ignored rules. 80 | if not is_ignored(input_path, ignored_rules): 81 | build_source = parse_script_and_generate_rule(input_path, project_root, 82 | contains_pre_installed_packages, 83 | custom_bazel_rules, 84 | custom_import_inference_rules, 85 | import_name_to_pip_name, 86 | local_import_name_to_dep) 87 | 88 | # If Python files were found, output the BUILD file. 89 | if build_source != '' or ignored_rules: 90 | output_build_file(build_source, ignored_rules, output_extension, custom_bazel_rules, 91 | build_file_path, requirement_load) 92 | else: 93 | raise RuntimeError("Invalid input path %s." % input_path) 94 | 95 | 96 | def main(): 97 | """Parse command-line flags and generate the BUILD files accordingly.""" 98 | parser = argparse.ArgumentParser(description='Generate Bazel BUILD files for a Python project.') 99 | 100 | working_directory = os.getcwd() 101 | default_pazelrc_path = os.path.join(working_directory, '.pazelrc') 102 | 103 | parser.add_argument('input_path', nargs='?', type=str, default=working_directory, 104 | help='Target Python file or directory of Python files.' 105 | ' Defaults to the current working directory.') 106 | parser.add_argument('-r', '--project-root', type=str, default=working_directory, 107 | help='Project root directory. Imports are relative to this path.' 108 | ' Defaults to the current working directory.') 109 | parser.add_argument('-p', '--pre-installed-packages', action='store_true', 110 | help='Target will be run in an environment with packages pre-installed.' 111 | ' Affects which packages are listed as pip-installable.') 112 | parser.add_argument('-c', '--pazelrc', type=str, default=default_pazelrc_path, 113 | help='Path to .pazelrc file.') 114 | 115 | args = parser.parse_args() 116 | 117 | # If the user specified custom .pazelrc file, then check that it exists. 118 | custom_pazelrc_path = args.pazelrc != default_pazelrc_path 119 | 120 | if custom_pazelrc_path: 121 | assert os.path.isfile(args.pazelrc), ".pazelrc file %s not found." % args.pazelrc 122 | 123 | app(args.input_path, args.project_root, args.pre_installed_packages, args.pazelrc) 124 | print('Generated BUILD files for %s.' % args.input_path) 125 | 126 | 127 | if __name__ == "__main__": 128 | main() 129 | -------------------------------------------------------------------------------- /pazel/bazel_rules.py: -------------------------------------------------------------------------------- 1 | """Classes for identifying Bazel rule type of a script and generating new rules to BUILD files.""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import os 8 | import re 9 | 10 | # These templates will be filled and used to generate BUILD files. 11 | # Note that both 'data' and 'deps' can be empty in which case they are left out from the rules. 12 | PY_BINARY_TEMPLATE = """py_binary( 13 | name = "{name}", 14 | srcs = ["{name}.py"], 15 | {data} 16 | {deps} 17 | )""" 18 | 19 | PY_LIBRARY_TEMPLATE = """py_library( 20 | name = "{name}", 21 | srcs = ["{name}.py"], 22 | {data} 23 | {deps} 24 | )""" 25 | 26 | PY_TEST_TEMPLATE = """py_test( 27 | name = "{name}", 28 | srcs = ["{name}.py"], 29 | size = "{size}", 30 | {data} 31 | {deps} 32 | )""" 33 | 34 | 35 | class BazelRule(object): 36 | """Base class defining the interface for parsing Bazel rules. 37 | 38 | pazel-native rule classes as well as custom rule classes need to implement this interface. 39 | """ 40 | 41 | # Required class variables. 42 | is_test_rule = None 43 | template = None 44 | rule_identifier = None 45 | 46 | @staticmethod 47 | def applies_to(script_name, script_source): 48 | """Check whether this rule applies to a given script. 49 | 50 | Args: 51 | script_name (str): Name of a Python script without the .py suffix. 52 | script_source (str): Source code of the script. 53 | 54 | Returns: 55 | applies (bool): Whether this Bazel rule can be used to represent the script. 56 | """ 57 | raise NotImplementedError() 58 | 59 | @staticmethod 60 | def find_existing(build_source, script_filename): 61 | """Find existing rule for a given script. 62 | 63 | Args: 64 | build_source (str): Source code of an existing BUILD file. 65 | script_filename (str): Name of a Python script. 66 | 67 | Returns: 68 | match (MatchObject or None): Match found in the BUILD file or None if no matches. 69 | """ 70 | # 'srcs' should contain the script filename. 71 | pattern = 'srcs\s*=\s*\["' + script_filename + '"\]' 72 | match = re.search(pattern, build_source) 73 | 74 | return match 75 | 76 | @staticmethod 77 | def get_load_statement(): 78 | """If the rule requires a special 'load' statement, return it, otherwise return None.""" 79 | return None 80 | 81 | 82 | class PyBinaryRule(BazelRule): 83 | """Class for representing Bazel-native py_binary.""" 84 | 85 | # Required class variables. 86 | is_test_rule = False # Is this a test rule? 87 | template = PY_BINARY_TEMPLATE # Filled version of this will be written to the BUILD file. 88 | rule_identifier = 'py_binary' # The name of the rule. 89 | 90 | @staticmethod 91 | def applies_to(script_name, script_source): 92 | """Check whether this rule applies to a given script. 93 | 94 | Args: 95 | script_name (str): Name of a Python script without the .py suffix. 96 | script_source (str): Source code of the script. 97 | 98 | Returns: 99 | applies (bool): Whether this Bazel rule can be used to represent the script. 100 | """ 101 | # Check if there is indentation level 0 code that launches a function. 102 | entrypoints = re.findall('\nif\s*__name__\s*==\s*["\']__main__["\']\s*:', script_source) 103 | entrypoints += re.findall('\n\S+\([\S+]?\)', script_source) 104 | 105 | # Rule out tests using unittest. 106 | is_test = PyTestRule.applies_to(script_name, script_source) 107 | 108 | applies = len(entrypoints) > 0 and not is_test 109 | 110 | return applies 111 | 112 | 113 | class PyLibraryRule(BazelRule): 114 | """Class for representing Bazel-native py_library.""" 115 | 116 | # Required class variables. 117 | is_test_rule = False # Is this a test rule? 118 | template = PY_LIBRARY_TEMPLATE # Filled version of this will be written to the BUILD file. 119 | rule_identifier = 'py_library' # The name of the rule. 120 | 121 | @staticmethod 122 | def applies_to(script_name, script_source): 123 | """Check whether this rule applies to a given script. 124 | 125 | Args: 126 | script_name (str): Name of a Python script without the .py suffix. 127 | script_source (str): Source code of the script. 128 | 129 | Returns: 130 | applies (bool): Whether this Bazel rule can be used to represent the script. 131 | """ 132 | is_test = PyTestRule.applies_to(script_name, script_source) 133 | is_binary = PyBinaryRule.applies_to(script_name, script_source) 134 | 135 | applies = not (is_test or is_binary) 136 | 137 | return applies 138 | 139 | 140 | class PyTestRule(BazelRule): 141 | """Class for representing Bazel-native py_test.""" 142 | 143 | # Required class variables. 144 | is_test_rule = True # Is this a test rule? 145 | template = PY_TEST_TEMPLATE # Filled version of this will be written to the BUILD file. 146 | rule_identifier = 'py_test' # The name of the rule. 147 | 148 | @staticmethod 149 | def applies_to(script_name, script_source): 150 | """Check whether this rule applies to a given script. 151 | 152 | Args: 153 | script_name (str): Name of a Python script without the .py suffix. 154 | script_source (str): Source code of the script. 155 | 156 | Returns: 157 | applies (bool): Whether this Bazel rule can be used to represent the script. 158 | """ 159 | imports_unittest = len(re.findall('import unittest', script_source)) > 0 or \ 160 | len(re.findall('from unittest', script_source)) > 0 161 | uses_unittest = len(re.findall('unittest.TestCase', script_source)) > 0 or \ 162 | len(re.findall('TestCase', script_source)) > 0 163 | test_filename = script_name.startswith('test_') or script_name.endswith('_test') 164 | 165 | applies = test_filename and imports_unittest and uses_unittest 166 | 167 | return applies 168 | 169 | 170 | def get_native_bazel_rules(): 171 | """Return a copy of the pazel-native classes implementing BazelRule.""" 172 | return [PyBinaryRule, PyLibraryRule, PyTestRule] # No custom classes here. 173 | 174 | 175 | def infer_bazel_rule_type(script_path, script_source, custom_rules): 176 | """Infer the Bazel rule type given the path to the script and its source code. 177 | 178 | Args: 179 | script_path (str): Path to a Python script. 180 | script_source (str): Source code of the Python script. 181 | custom_rules (list of BazelRule classes): User-defined classes implementing BazelRule. 182 | 183 | Returns: 184 | bazel_rule_type (BazelRule): Rule object representing the type of the Python script. 185 | 186 | Raises: 187 | RuntimeError: If zero or more than one Bazel rule is found for the current script. 188 | """ 189 | script_name = os.path.basename(script_path).replace('.py', '') 190 | 191 | bazel_rule_types = [] 192 | 193 | native_rules = get_native_bazel_rules() 194 | registered_rules = native_rules + custom_rules 195 | 196 | for bazel_rule in registered_rules: 197 | if bazel_rule.applies_to(script_name, script_source): 198 | bazel_rule_types.append(bazel_rule) 199 | 200 | if not bazel_rule_types: 201 | raise RuntimeError("No suitable Bazel rule type found for %s." % script_path) 202 | elif len(bazel_rule_types) > 1: 203 | # If the script is recognized by pazel native rule(s) and one exactly custom rule, then use 204 | # the custom rule. This is because the pazel native rules may generate false positives. 205 | is_custom = [rule not in native_rules for rule in bazel_rule_types] 206 | one_custom = sum(is_custom) == 1 207 | 208 | if one_custom: 209 | custom_bazel_rule_idx = is_custom.index(True) 210 | return bazel_rule_types[custom_bazel_rule_idx] 211 | else: 212 | raise RuntimeError("Multiple Bazel rule types (%s) found for %s." 213 | % (bazel_rule_types, script_path)) 214 | 215 | return bazel_rule_types[0] 216 | -------------------------------------------------------------------------------- /pazel/generate_rule.py: -------------------------------------------------------------------------------- 1 | """Generate Bazel rule for a single Python file.""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import os 8 | 9 | from pazel.bazel_rules import infer_bazel_rule_type 10 | from pazel.parse_build import find_existing_data_deps 11 | from pazel.parse_build import find_existing_test_size 12 | from pazel.parse_imports import get_imports 13 | from pazel.parse_imports import infer_import_type 14 | 15 | 16 | def _walk_modules(current_dir, modules): 17 | """Walk modules in different directories. 18 | 19 | This function is similar to os.walk but it returns module names in a sorted order. 20 | 21 | Args: 22 | current_dir (str): Current directory that contains one or many modules. 23 | modules (list of str): List of modules in current_dir or in its subdirectories. 24 | 25 | Yields: 26 | sorted list of module names in a directory. 27 | """ 28 | modules_in_current_dir, remaining_modules, subdirs = [], [], [] 29 | 30 | for module in modules: 31 | # Map e.g. "foo.abc" and "foo.abc.xyz" to "abc" and "abc.xyz", respectively. 32 | remaining_module_name = module.replace(current_dir, '').split('.') 33 | 34 | is_in_current_dir = len(remaining_module_name) == 1 35 | 36 | if is_in_current_dir: 37 | modules_in_current_dir.append(module) 38 | else: 39 | remaining_modules.append(module) 40 | subdirs.append(remaining_module_name[0]) 41 | 42 | # Modules in the current directory precede modules in subdirectories. 43 | yield sorted(modules_in_current_dir) 44 | 45 | # Recurse modules in subdirectories. 46 | sorted_subdirs = sorted(list(set(subdirs))) 47 | 48 | for subdir in sorted_subdirs: 49 | next_dir = current_dir + subdir + '.' if current_dir else subdir + '.' 50 | # Consider only modules in the current subdirectory. 51 | remaining_modules_in_next_dir = [module for module in remaining_modules if 52 | next_dir in module] 53 | 54 | for x in _walk_modules(next_dir, remaining_modules_in_next_dir): 55 | yield x 56 | 57 | 58 | def sort_module_names(module_names): 59 | """Sort modules alphabetically but so that modules in a directory precede modules in subdirs. 60 | 61 | For example, modules ["xyz", "abc", "foo.bar1"] are sorted to ["abc", "xyz", "foo.bar1"]. 62 | 63 | Args: 64 | module_names (list of str): List of module names in dotted notation ("foo.bar.xyz"). 65 | 66 | Returns: 67 | sorted_modules_names (list of str): List of sorted module names. 68 | """ 69 | sorted_module_names = [] 70 | 71 | for module_name in _walk_modules('', module_names): 72 | sorted_module_names += module_name 73 | 74 | return sorted_module_names 75 | 76 | 77 | def generate_rule(script_path, template, package_names, module_names, data_deps, test_size, 78 | import_name_to_pip_name, local_import_name_to_dep): 79 | """Generate a Bazel Python rule given the type of the Python file and imports in it. 80 | 81 | Args: 82 | script_path (str): Path to a Python script. 83 | template (str): Template for writing a Bazel rule. To be filled with name, srcs, deps, etc. 84 | package_names (set of str): Set of imported packages names in dotted notation (pkg1.pkg2). 85 | module_names (set of str): Set of imported module names in dotted notation (pkg.module) 86 | data_deps (str): Data dependencies parsed from an existing BUILD file. 87 | test_size (str): Test size parsed from an existing BUILD file. 88 | import_name_to_pip_name (dict): Mapping from Python package import name to its pip name. 89 | local_import_name_to_dep (dict): Mapping from local package import name to its Bazel 90 | dependency. 91 | 92 | Returns: 93 | rule (str): Bazel rule generated for the current Python script. 94 | """ 95 | script_name = os.path.basename(script_path).replace('.py', '') 96 | deps = '' 97 | tab = ' ' 98 | 99 | num_deps = len(module_names) + len(package_names) 100 | multiple_deps = num_deps > 1 101 | 102 | # Formatting with one dependency: 103 | # deps = ["//my_dep1/foo:abc"] 104 | # Formatting with multiple dependencies: 105 | # deps = [ 106 | # "//my_dep1/foo:abc", 107 | # "//my_dep2/bar:xyz", 108 | # ] 109 | 110 | if multiple_deps: 111 | deps += '\n' 112 | 113 | for module_name in sort_module_names(list(module_names)): 114 | # Import from the same directory as the script resides. 115 | if '.' not in module_name: 116 | module_name = ':' + module_name 117 | else: 118 | # Format the dotted module name to the Bazel format with slashes. 119 | module_name = '//' + module_name.replace('.', '/') 120 | 121 | # Replace the last slash with :. 122 | last_slash_idx = module_name.rfind('/') 123 | module_name = module_name[:last_slash_idx] + ':' + module_name[last_slash_idx + 1:] 124 | 125 | if multiple_deps: 126 | deps += 2*tab 127 | 128 | deps += '\"' + module_name + '\"' 129 | 130 | if multiple_deps: 131 | deps += ',\n' 132 | 133 | # Even if a submodule of a local or external package is required, install the whole package. 134 | package_names = set([p.split('.')[0] for p in package_names]) 135 | 136 | # Split packages to local and external. 137 | local_packages = [p for p in package_names if p in local_import_name_to_dep] 138 | external_packages = [p for p in package_names if p not in local_import_name_to_dep] 139 | 140 | for package_set in (local_packages, external_packages): # List local packages first. 141 | for package_name in sorted(list(package_set)): 142 | if multiple_deps: 143 | deps += 2*tab 144 | 145 | if package_name in local_import_name_to_dep: # Local package. 146 | package_name = local_import_name_to_dep[package_name] 147 | deps += '\"' + package_name + '\"' 148 | else: # External/pip installable package. 149 | package_name = import_name_to_pip_name.get(package_name, package_name) 150 | package_name = 'requirement(\"%s\")' % package_name 151 | deps += package_name 152 | 153 | if multiple_deps: 154 | deps += ',\n' 155 | 156 | if multiple_deps: 157 | deps += tab 158 | 159 | if deps: 160 | deps = 'deps = [{deps}],'.format(deps=deps) 161 | 162 | data = data_deps + ',' if data_deps is not None else '' 163 | size = test_size if test_size is not None else 'small' # If size not given, assume small. 164 | 165 | rule = template.format(name=script_name, deps=deps, data=data, size=size) 166 | # If e.g. 'data' is missing, then remove blank lines. 167 | rule = "\n".join([s for s in rule.splitlines() if s.strip()]) 168 | 169 | return rule 170 | 171 | 172 | def parse_script_and_generate_rule(script_path, project_root, contains_pre_installed_packages, 173 | custom_bazel_rules, custom_import_inference_rules, 174 | import_name_to_pip_name, local_import_name_to_dep): 175 | """Generate Bazel Python rule for a Python script. 176 | 177 | Args: 178 | script_path (str): Path to a Python file for which the Bazel rule is generated. 179 | project_root (str): Imports in the Python script are assumed to be relative to this path. 180 | contains_pre_installed_packages (bool): Environment contains pre-installed packages (true) 181 | or only the standard library (false). 182 | custom_bazel_rules (list of BazelRule classes): Custom rule classes implementing BazelRule. 183 | custom_import_inference_rules (list of ImportInferenceRule classes): Custom rule classes 184 | implementing ImportInferenceRule. 185 | import_name_to_pip_name (dict): Mapping from Python package import name to its pip name. 186 | local_import_name_to_dep (dict): Mapping from local package import name to its Bazel 187 | dependency. 188 | 189 | Returns: 190 | rule (str): Bazel rule generated for the Python script. 191 | """ 192 | with open(script_path, 'r') as script_file: 193 | script_source = script_file.read() 194 | 195 | # Get all imports in the script. 196 | package_names, from_imports = get_imports(script_source) 197 | all_imports = package_names + from_imports 198 | 199 | # Infer the import type: Is a package, module, or an object being imported. 200 | package_names, module_names = infer_import_type(all_imports, project_root, 201 | contains_pre_installed_packages, 202 | custom_import_inference_rules) 203 | 204 | # Infer the Bazel rule type for the script. 205 | bazel_rule_type = infer_bazel_rule_type(script_path, script_source, custom_bazel_rules) 206 | 207 | # Data dependencies or test size cannot be inferred from the script source code currently. 208 | # Use information in any existing BUILD files. 209 | data_deps = find_existing_data_deps(script_path, bazel_rule_type) 210 | test_size = find_existing_test_size(script_path, bazel_rule_type) 211 | 212 | # Generate the Bazel Python rule based on the gathered information. 213 | rule = generate_rule(script_path, bazel_rule_type.template, package_names, module_names, 214 | data_deps, test_size, import_name_to_pip_name, local_import_name_to_dep) 215 | 216 | return rule 217 | -------------------------------------------------------------------------------- /pazel/helpers.py: -------------------------------------------------------------------------------- 1 | """Helper functions for generating Bazel BUILD files for a Python project.""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import ast 8 | import importlib 9 | import os 10 | import platform 11 | import sys 12 | import traceback 13 | 14 | 15 | def contains_python_file(directory): 16 | """Check if the given directory contains at least one .py/.pyc file. 17 | 18 | Args: 19 | directory (str): 20 | 21 | Returns: 22 | contains_py (bool): Whether the directory contains at least one Python file. 23 | """ 24 | files = os.listdir(directory) 25 | contains_py = any(f.endswith('.py') or f.endswith('.pyc') for f in files) 26 | 27 | return contains_py 28 | 29 | 30 | def get_build_file_path(path): 31 | """Get path to a BUILD file next to a given path. 32 | 33 | Args: 34 | path (str): Path to a file or directory. 35 | 36 | Returns 37 | build_file_path (str): Path to the BUILD in the given directory or in the directory 38 | containing the given file. 39 | """ 40 | if os.path.isdir(path): 41 | directory = path 42 | else: 43 | directory = os.path.dirpath(path) 44 | 45 | build_file_path = os.path.join(directory, 'BUILD') 46 | 47 | return build_file_path 48 | 49 | 50 | def is_ignored(script_path, ignored_rules): 51 | """Check whether the given script is in ignored rules. 52 | 53 | Args: 54 | script_path (str): Path to a Python script. 55 | ignored_rules (list of str): Ignored Bazel rules. 56 | 57 | Returns: 58 | ignored (bool): Whether the script should be ignored. 59 | """ 60 | ignored = False 61 | 62 | script_file_name = os.path.basename(script_path) 63 | 64 | for ignored_rule in ignored_rules: 65 | # Parse the rule to an AST node. 66 | try: 67 | node = ast.parse(ignored_rule) 68 | except SyntaxError: 69 | raise SyntaxError("Invalid syntax in an ignored rule %s." % ignored_rule) 70 | 71 | assert len(node.body) == 1, "Unsupported rule type %s." % ignored_rule 72 | 73 | # Check keyword arguments in the rule. If the 'srcs' argument contains the script file name, 74 | # then the script should be ignored. 75 | func_call = node.body[0].value 76 | 77 | for keyword in func_call.keywords: 78 | if keyword.arg == 'srcs': 79 | elements = keyword.value.elts 80 | 81 | assert len(elements) == 1, \ 82 | "Multiple source files not supported in %s." % ignored_rule 83 | 84 | if elements[0].s == script_file_name: 85 | ignored = True 86 | break 87 | 88 | # The script file name may also given as a positional argument. 89 | for positional in func_call.args: 90 | if positional.s == script_file_name: 91 | ignored = True 92 | break 93 | 94 | return ignored 95 | 96 | 97 | def is_python_file(path): 98 | """Check whether the file in the given path is a Python file. 99 | 100 | Args: 101 | path (str): Path to a file. 102 | 103 | Returns: 104 | valid (bool): The file in path is a Python file. 105 | """ 106 | valid = False 107 | 108 | if os.path.isfile(path) and path.endswith('.py'): 109 | valid = True 110 | 111 | return valid 112 | 113 | 114 | def _is_in_stdlib(module, some_object): 115 | """Check if a given module is part of the Python standard library.""" 116 | # Clear PYTHONPATH temporarily and try importing the given module. 117 | original_sys_path = sys.path 118 | lib_path = os.path.dirname(traceback.__file__) 119 | sys.path = [lib_path] 120 | 121 | # On Mac, some extra library paths are required. 122 | if 'darwin' in platform.system().lower(): 123 | for path in original_sys_path: 124 | if 'site-packages' not in path: 125 | sys.path.append(path) 126 | 127 | in_stdlib = False 128 | 129 | try: 130 | module = importlib.import_module(module) 131 | 132 | if some_object: 133 | getattr(module, some_object) 134 | 135 | in_stdlib = True 136 | except (ImportError, AttributeError): 137 | pass 138 | 139 | sys.path = original_sys_path 140 | 141 | return in_stdlib 142 | 143 | 144 | def is_installed(module, some_object=None, contains_pre_installed_packages=False): 145 | """Check if a given module is installed and whether some_object is found in it. 146 | 147 | Args: 148 | module (str): Name of a module. 149 | some_object (str): Name of some object in the module. Can be None. 150 | contains_pre_installed_packages (bool): Whether the environment contains external packages. 151 | 152 | Returns: 153 | installed (bool): The module is installed in the current environment. 154 | """ 155 | installed = False 156 | 157 | # If the application runs inside e.g. a virtualenv that already contains some requirements, 158 | # then try importing the module. If it fails, then the module is not yet installed. 159 | if contains_pre_installed_packages: 160 | try: 161 | module = importlib.import_module(module) 162 | 163 | if some_object: 164 | getattr(module, some_object) 165 | 166 | installed = True 167 | except (ImportError, AttributeError): 168 | installed = False 169 | else: # If we have a clean install, then check if the module is in the standard library. 170 | installed = _is_in_stdlib(module, some_object) 171 | 172 | return installed 173 | 174 | 175 | def parse_enclosed_expression(source, start, opening_token): 176 | """Parse an expression enclosed by a token and its counterpart. 177 | 178 | Args: 179 | source (str): Source code of a Bazel BUILD file, for example. 180 | start (int): Index at which an expression starts. 181 | opening_token (str): A character '(' or '[' that opens an expression. 182 | 183 | Returns: 184 | expression (str): The whole expression that contains the opening and closing tokens. 185 | 186 | Raises: 187 | NotImplementedError: If parsing is not implemented for the given opening token. 188 | """ 189 | if opening_token == '(': 190 | closing_token = ')' 191 | elif opening_token == '[': 192 | closing_token = ']' 193 | else: 194 | raise NotImplementedError("No closing token defined for %s." % opening_token) 195 | 196 | start2 = source.find(opening_token, start) 197 | assert start2 > start, "Could not locate the opening token %s." % opening_token 198 | open_tokens = 0 199 | end = None 200 | 201 | for end_idx, char in enumerate(source[start2:], start2): 202 | if char == opening_token: 203 | open_tokens += 1 204 | elif char == closing_token: 205 | open_tokens -= 1 206 | 207 | if open_tokens == 0: 208 | end = end_idx + 1 209 | break 210 | 211 | assert end, "Could not locate the closing token %s." % closing_token 212 | 213 | expression = source[start:end] 214 | 215 | return expression 216 | -------------------------------------------------------------------------------- /pazel/import_inference_rules.py: -------------------------------------------------------------------------------- 1 | """Interface for defining custom classes for inferring import type.""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | 8 | class ImportInferenceRule(object): 9 | """Base class defining the interface for custom import inference classes. 10 | 11 | Custom classes define how a Python import is mapped to Bazel dependencies. 12 | """ 13 | 14 | @staticmethod 15 | def holds(project_root, base, unknown): 16 | """If this import inference rule holds, then return imported packages and/or modules. 17 | 18 | Args: 19 | project_root (str): Local imports are assumed to be relative to this path. 20 | base (str): Name of a package or a module. 21 | unknown (str): Can package, module, function or any other object. 22 | 23 | Returns: 24 | packages (list of str or None): Imported package names. None if no packages are imported 25 | or if the rule does not match the import. 26 | modules (list of str or None): Imported module names. None if no modules are imported or 27 | if the rule does not match the import. 28 | """ 29 | raise NotImplementedError() 30 | -------------------------------------------------------------------------------- /pazel/output_build.py: -------------------------------------------------------------------------------- 1 | """Output a BUILD file.""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import re 8 | 9 | 10 | def _append_newline(source): 11 | """Add newline to a string if it does not end with a newline.""" 12 | return source if source.endswith('\n') else source + '\n' 13 | 14 | 15 | def output_build_file(build_source, ignored_rules, output_extension, custom_bazel_rules, 16 | build_file_path, requirement_load): 17 | """Output a BUILD file. 18 | 19 | Args: 20 | build_source (str): The contents of the BUILD file to output. 21 | ignored_rules (list of str): Rules the user wants to keep as is. 22 | output_extension (OutputExtension): User-defined header and footer. 23 | custom_bazel_rules (list of BazelRule classes): User-defined BazelRule classes. 24 | build_file_path (str): Path to the BUILD file in which build_source is written. 25 | requirement_load (str): Statement for loading the 'requirement' rule. 26 | """ 27 | header = '' 28 | 29 | if output_extension.header: 30 | header += _append_newline(output_extension.header) 31 | 32 | # Categorize ignored rules to 'load' statements and other remaining rules. 33 | ignored_load_statements = [] 34 | remaining_ignored_rules = [] 35 | 36 | if ignored_rules: 37 | for ignored_rule in ignored_rules: 38 | if 'load(' in ignored_rule: 39 | ignored_load_statements.append(ignored_rule) 40 | else: 41 | remaining_ignored_rules.append(ignored_rule) 42 | 43 | # If the BUILD file contains external packages, add the 'load' statement for installing them. 44 | # Check that this statement is not in the ignored 'load' statements. 45 | ignored_source = '\n'.join(remaining_ignored_rules) 46 | 47 | if any(['requirement("' in source for source in (build_source, ignored_source)]): 48 | in_ignored_load_statements = any(['requirement("' in statement for statement in 49 | ignored_load_statements]) 50 | 51 | if not in_ignored_load_statements: 52 | header += _append_newline(requirement_load) 53 | 54 | # If the BUILD source contains custom Bazel rules, then add the load statements for them unless 55 | # the load statements are already in the ignored load statements. 56 | for custom_rule in custom_bazel_rules: 57 | rule_identifier = custom_rule.rule_identifier 58 | in_ignored_load_statements = any([rule_identifier in statement for statement in 59 | ignored_load_statements]) 60 | 61 | if rule_identifier in build_source and not in_ignored_load_statements: 62 | header += _append_newline(custom_rule.get_load_statement()) 63 | 64 | # Add ignored load statements right after the header. 65 | for ignored_load in ignored_load_statements: 66 | header += _append_newline(ignored_load) 67 | 68 | # If a header exists, add a newline between it and the rules. 69 | if header: 70 | header += '\n' 71 | 72 | output = header + build_source 73 | 74 | # Add other ignored rules than load statements to the bottom, separated by newlines. 75 | if remaining_ignored_rules: 76 | output = output.rstrip() 77 | output += '\n' + '\n'.join(remaining_ignored_rules) 78 | 79 | # Add the footer, separated by a newline. 80 | if output_extension.footer: 81 | output += 2*'\n' + _append_newline(output_extension.footer) 82 | 83 | with open(build_file_path, 'w') as build_file: 84 | output = _append_newline(output) 85 | 86 | # Remove possible duplicate newlines (the user may have added such accidentally). 87 | output = re.sub('\n\n\n*', '\n\n', output) 88 | 89 | build_file.write(output) 90 | -------------------------------------------------------------------------------- /pazel/parse_build.py: -------------------------------------------------------------------------------- 1 | """Parse existing BUILD files.""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import os 8 | import re 9 | 10 | from pazel.helpers import parse_enclosed_expression 11 | 12 | 13 | def find_existing_rule(build_file_path, script_filename, bazel_rule_type): 14 | """Find Bazel rule for a given Python script in a BUILD file. 15 | 16 | Args: 17 | build_file_path (str): Path to an existing BUILD file that may contain a rule for a given 18 | Python script. 19 | script_filename (str): File name of the Python script. 20 | bazel_rule_type (Rule class): pazel-native or a custom class implementing BazelRule. 21 | 22 | Returns: 23 | rule (str): Existing Bazel rule for the Python script. If there is no rule, then None. 24 | """ 25 | # Read the existing BUILD file if there is one. 26 | try: 27 | with open(build_file_path, 'r') as build_file: 28 | build_source = build_file.read() 29 | except IOError: 30 | return None 31 | 32 | # Find existing rules for the current script. 33 | match = bazel_rule_type.find_existing(build_source, script_filename) 34 | 35 | if match is None: 36 | return None 37 | 38 | # Find the start of the rule. 39 | rule_identifier = bazel_rule_type.rule_identifier 40 | start = match.start() 41 | 42 | # If the match is not the beginning of the rule, then go backwards to the start of the rule. 43 | if build_source[start:start + len(rule_identifier)] != rule_identifier: 44 | start = build_source.rfind(bazel_rule_type.rule_identifier, 0, start) 45 | 46 | assert start != -1, "The start of the Bazel Python rule for %s not located." % script_filename 47 | 48 | # Find the rule by matching opening and closing parentheses. 49 | rule = parse_enclosed_expression(build_source, start, '(') 50 | 51 | return rule 52 | 53 | 54 | def find_existing_test_size(script_path, bazel_rule_type): 55 | """Check if the existing Bazel rule for a Python test contains test size. 56 | 57 | Args: 58 | script_path (str): Path to a Python file that is a test. 59 | bazel_rule_type (Rule class): pazel-native or a custom class implementing BazelRule. 60 | 61 | Returns: 62 | test_size (str): Size of the test (small, medium, etc.) if found in the existing BUILD file. 63 | If not found, then None is returned. 64 | """ 65 | if not bazel_rule_type.is_test_rule: 66 | return None 67 | 68 | script_dir = os.path.dirname(script_path) 69 | script_filename = os.path.basename(script_path) 70 | build_file_path = os.path.join(script_dir, 'BUILD') 71 | 72 | rule = find_existing_rule(build_file_path, script_filename, bazel_rule_type) 73 | 74 | # No existing Bazel rules for the given Python file. 75 | if rule is None: 76 | return None 77 | 78 | # Search for the test size. 79 | matches = re.findall('size\s*=\s*\"(small|medium|large|enormous)\"', rule) 80 | 81 | num_matches = len(matches) 82 | 83 | if num_matches > 0: 84 | assert num_matches == 1, "Found multiple test size matches in %s." % rule 85 | return matches[0] 86 | 87 | return None 88 | 89 | 90 | def find_existing_data_deps(script_path, bazel_rule_type): 91 | """Check if the existing Bazel Python rule in a BUILD file contains data dependencies. 92 | 93 | Args: 94 | script_path (str): Path to a Python script. 95 | bazel_rule_type (Rule class): pazel-native or a custom class implementing BazelRule. 96 | 97 | Returns: 98 | data (str): Data dependencies in the existing rule for the Python script. 99 | """ 100 | script_dir = os.path.dirname(script_path) 101 | script_filename = os.path.basename(script_path) 102 | build_file_path = os.path.join(script_dir, 'BUILD') 103 | 104 | rule = find_existing_rule(build_file_path, script_filename, bazel_rule_type) 105 | 106 | # No matches, no data deps. 107 | if rule is None: 108 | return None 109 | 110 | # Search for data deps. 111 | data = None 112 | 113 | # Data deps are a list. 114 | match = re.search('data\s*=\s*\[', rule) 115 | 116 | if match: 117 | data = parse_enclosed_expression(rule, match.start(), '[') 118 | 119 | # Data deps defined by a call to 'glob'. 120 | match = re.search('data\s*=\s*glob\(', rule) 121 | 122 | if match: 123 | data = parse_enclosed_expression(rule, match.start(), '(') 124 | 125 | return data 126 | 127 | 128 | def get_ignored_rules(build_file_path): 129 | """Check if an existing BUILD file contains rule that should be ignored. 130 | 131 | Args: 132 | build_file_path (str): Path to an existing BUILD file. 133 | 134 | Returns: 135 | ignored_rules (list of str): Ignored Bazel rule(s). Empty list if no ignored rules were 136 | found or if the Bazel BUILD does not exist. 137 | """ 138 | try: 139 | with open(build_file_path, 'r') as build_file: 140 | build_source = build_file.read() 141 | except IOError: 142 | return [] 143 | 144 | ignored_rules = [] 145 | 146 | # pazel ignores rules following the tag "# pazel-ignore". Spaces are ignored within the tag but 147 | # the line must start with #. 148 | for match in re.finditer('\n#\s+pazel-ignore\s+', build_source): 149 | start = match.start() 150 | 151 | rule = parse_enclosed_expression(build_source, start, '(') 152 | ignored_rules.append(rule) 153 | 154 | return ignored_rules 155 | -------------------------------------------------------------------------------- /pazel/parse_imports.py: -------------------------------------------------------------------------------- 1 | """Parse imports in Python files and infer what is being imported from which package.""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import ast 8 | import os 9 | 10 | from pazel.helpers import contains_python_file 11 | from pazel.helpers import is_installed 12 | 13 | 14 | def get_imports(script_source): 15 | """Parse imported packages and objects imported from packages. 16 | 17 | Args: 18 | script_source (str): The source code of a Python script. 19 | 20 | Returns: 21 | packages (list of tuple): List of (package name, None) tuples. 22 | from_imports (list of tuple): List of (package/module name, some object) tuples. Note that 23 | some object can be a function, object, module, or package. 24 | """ 25 | packages = [] 26 | from_imports = [] 27 | ast_of_source = ast.parse(script_source) 28 | 29 | for node in ast_of_source.body: 30 | # Parse expressions of the form "from X import Y". 31 | if isinstance(node, ast.ImportFrom): 32 | module = node.module 33 | 34 | for name in node.names: 35 | from_imports.append((module, name.name)) 36 | # Parse expressions of the form "import X". 37 | elif isinstance(node, ast.Import): 38 | for package in node.names: 39 | packages.append((package.name, None)) 40 | 41 | return packages, from_imports 42 | 43 | 44 | def infer_import_type(all_imports, project_root, contains_pre_installed_packages, custom_rules): 45 | """Infer what is being imported. 46 | 47 | Given a list of tuples (package/module, some object) infer whether the first element is a 48 | package or a module and whether it is installed. Also, infer the type of the second element. 49 | 50 | Args: 51 | all_imports (list of tuple): All imports in a Python script. 52 | project_root (str): Local imports are assumed to be relative to this path. 53 | contains_pre_installed_packages (bool): Whether the environment contains external packages. 54 | 55 | Returns: 56 | packages: Set of package names that are imported. 57 | modules: Set of module names that are imported. 58 | """ 59 | modules = [] 60 | packages = [] 61 | 62 | # Base is package/module and the type of unknown is inferred below. 63 | for base, unknown in all_imports: 64 | # Early exit if base is in the installed modules of the current environment. 65 | if is_installed(base, unknown, contains_pre_installed_packages): 66 | continue 67 | 68 | # Prioritize custom inference rules used for parsing imports that pazel does not support. 69 | # These custom rules define how a Python import is mapped to Bazel dependencies. 70 | custom_rule_matches = False 71 | 72 | for inference_rule in custom_rules: 73 | new_packages, new_modules = inference_rule.holds(project_root, base, unknown) 74 | 75 | # If the rule holds, then add to the list of packages and/or modules. 76 | if new_packages is not None: 77 | packages.extend(new_packages) 78 | 79 | if new_modules is not None: 80 | modules.extend(new_modules) 81 | 82 | if new_packages is not None or new_modules is not None: 83 | custom_rule_matches = True # Only allow one match for custom rules. 84 | break 85 | 86 | # One custom rule matched, continue to the next import. 87 | if custom_rule_matches: 88 | continue 89 | 90 | # Then, assume that 'base' is a module and 'unknown' is function, variable or any 91 | # other object in that module. 92 | module_path = os.path.join(project_root, base.replace('.', '/') + '.py') 93 | if os.path.exists(module_path): 94 | modules.append(base) 95 | continue 96 | 97 | # Check if 'unknown' is actually a package or a module. 98 | dotted_path = base + '.%s' % unknown 99 | package_path = os.path.join(project_root, dotted_path.replace('.', '/')) 100 | module_path = os.path.join(project_root, dotted_path.replace('.', '/') + '.py') 101 | 102 | unknown_is_package = os.path.isdir(package_path) and contains_python_file(package_path) 103 | unknown_is_module = os.path.isfile(module_path) 104 | 105 | if unknown_is_package: 106 | # Assume that for package //foo, there exists rule //foo:foo. 107 | # TODO: Relax this assumption. 108 | dotted_path += '.%s' % unknown 109 | modules.append(dotted_path) 110 | continue 111 | 112 | if unknown_is_module: 113 | modules.append(dotted_path) 114 | continue 115 | 116 | # Check if 'base' is a package and 'unknown' is part of its "public" interface 117 | # as declared in __all__ of the __init__.py file. 118 | package_path = os.path.join(project_root, base.replace('.', '/')) 119 | if os.path.isdir(package_path) and _in_public_interface(package_path, unknown): 120 | modules.append(base + '.__init__') 121 | continue 122 | 123 | # Finally, assume that base is either a pip installable or a local package. 124 | packages.append(base) 125 | 126 | return set(packages), set(modules) 127 | 128 | 129 | def _in_public_interface(package_path, unknown): 130 | """Check if 'unknown' is part of the public interface of a package. 131 | 132 | Args: 133 | package_path (str): Path to a Python package. 134 | unknown (str): Some object in the package. 135 | 136 | Returns: 137 | public (bool): Whether 'unknown' if part of the public interface. 138 | """ 139 | public = False 140 | init_path = os.path.join(package_path, '__init__.py') 141 | 142 | # Try parsing the __init__.py file of the package. 143 | try: 144 | with open(init_path, 'r') as init_file: 145 | init_source = init_file.read() 146 | except IOError: 147 | return public 148 | 149 | try: 150 | top_node = ast.parse(init_source) 151 | except SyntaxError: 152 | return public 153 | 154 | for node in top_node.body: 155 | # Check assigning to __all__. 156 | if isinstance(node, ast.Assign): 157 | # The number of variables on the left side should be 1. 158 | if len(node.targets) == 1: 159 | left_side = node.targets[0].id 160 | 161 | if left_side == '__all__': 162 | for element in node.value.elts: 163 | if element.s == unknown: 164 | return True 165 | 166 | return False 167 | -------------------------------------------------------------------------------- /pazel/pazel_extensions.py: -------------------------------------------------------------------------------- 1 | """Handle user-defined pazel extensions.""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import ast 8 | import imp 9 | 10 | 11 | class OutputExtension(object): 12 | """A class representing pazel extension to outputting BUILD files.""" 13 | 14 | def __init__(self, header, footer): 15 | """Instantiate. 16 | 17 | Args: 18 | header (str): Header for BUILD files. 19 | footer (str): Footer for BUILD files. 20 | """ 21 | self.header = header 22 | self.footer = footer 23 | 24 | 25 | def parse_pazel_extensions(pazelrc_path): 26 | """Parse pazel extensions from a .pazelrc file. 27 | 28 | Parses user-defined header and footer as well as updates the list of registered rule classes 29 | that pazel uses to generate Bazel rules for Python scripts. See the main README.md for 30 | instructions for programming pazel. 31 | 32 | Args: 33 | pazelrc_path (str): Path to .pazelrc config file for customizing pazel. 34 | 35 | Returns: 36 | output_extension (OutputExtension): Object containing user-defined header and footer. 37 | custom_bazel_rules (list of BazelRule classes): Custom BazelRule classes. 38 | custom_import_inference_rules (list of ImportInferenceRule classes): Custom classes 39 | for inferring import types. 40 | import_name_to_pip_name (dict): Mapping from Python package import name to its pip name. 41 | local_import_name_to_dep (dict): Mapping from local package import name to its Bazel 42 | dependency. 43 | requirement_load (str): Statement for loading the 'requirement' rule for installing pip 44 | packages. 45 | 46 | Raises: 47 | SyntaxError: If the .pazelrc contains invalid Python syntax. 48 | """ 49 | # Try parsing the .pazelrc to check that it contains valid Python syntax. 50 | try: 51 | with open(pazelrc_path, 'r') as pazelrc_file: 52 | pazelrc_source = pazelrc_file.read() 53 | ast.parse(pazelrc_source) 54 | 55 | pazelrc = imp.load_source('pazelrc', pazelrc_path) 56 | except IOError: 57 | # The file does not exist. Use a dummy pazelrc that contains nothing. 58 | pazelrc = dict() 59 | except SyntaxError: 60 | raise SyntaxError("Invalid syntax in %s. Run the file with an interpreter." % pazelrc_path) 61 | 62 | # Read user-defined header and footer. 63 | header = getattr(pazelrc, 'HEADER', '') 64 | footer = getattr(pazelrc, 'FOOTER', '') 65 | 66 | assert isinstance(header, str), "HEADER must be a string." 67 | assert isinstance(footer, str), "FOOTER must be a string." 68 | 69 | output_extension = OutputExtension(header, footer) 70 | 71 | # Read user-defined BazelRule classes. 72 | custom_bazel_rules = getattr(pazelrc, 'EXTRA_BAZEL_RULES', []) 73 | assert isinstance(custom_bazel_rules, list), "EXTRA_BAZEL_RULES must be a list." 74 | 75 | # Read user-defined ImportInferenceRule classes. 76 | custom_import_inference_rules = getattr(pazelrc, 'EXTRA_IMPORT_INFERENCE_RULES', []) 77 | assert isinstance(custom_import_inference_rules, list), \ 78 | "EXTRA_IMPORT_INFERENCE_RULES must be a list." 79 | 80 | # Read user-defined mapping from package import names to pip package names. 81 | import_name_to_pip_name = getattr(pazelrc, 'EXTRA_IMPORT_NAME_TO_PIP_NAME', dict()) 82 | assert isinstance(import_name_to_pip_name, dict), \ 83 | "EXTRA_IMPORT_NAME_TO_PIP_NAME must be a dictionary." 84 | 85 | # Read user-defined mapping from local import names to their Bazel dependencies. 86 | local_import_name_to_dep = getattr(pazelrc, 'EXTRA_LOCAL_IMPORT_NAME_TO_DEP', dict()) 87 | assert isinstance(local_import_name_to_dep, dict), \ 88 | "EXTRA_LOCAL_IMPORT_NAME_TO_DEP must be a dictionary." 89 | 90 | default_requirement_load = 'load("@my_deps//:requirements.bzl", "requirement")' 91 | requirement_load = getattr(pazelrc, 'REQUIREMENT', default_requirement_load) 92 | 93 | assert isinstance(requirement_load, str), "REQUIREMENT must be a string." 94 | 95 | return output_extension, custom_bazel_rules, custom_import_inference_rules, \ 96 | import_name_to_pip_name, local_import_name_to_dep, requirement_load 97 | -------------------------------------------------------------------------------- /pazel/tests/BUILD: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | py_library( 4 | name = "__init__", 5 | srcs = ["__init__.py"], 6 | deps = [], 7 | ) 8 | 9 | py_test( 10 | name = "test_bazel_rules", 11 | srcs = ["test_bazel_rules.py"], 12 | size = "small", 13 | deps = ["//pazel:bazel_rules"], 14 | ) 15 | 16 | py_test( 17 | name = "test_generate_rule", 18 | srcs = ["test_generate_rule.py"], 19 | size = "small", 20 | deps = ["//pazel:generate_rule"], 21 | ) 22 | 23 | py_test( 24 | name = "test_helpers", 25 | srcs = ["test_helpers.py"], 26 | size = "small", 27 | deps = ["//pazel:helpers"], 28 | ) 29 | 30 | py_test( 31 | name = "test_parse_imports", 32 | srcs = ["test_parse_imports.py"], 33 | size = "small", 34 | deps = ["//pazel:parse_imports"], 35 | ) 36 | 37 | py_test( 38 | name = "test_pazel_extensions", 39 | srcs = ["test_pazel_extensions.py"], 40 | size = "small", 41 | deps = ["//pazel:pazel_extensions"], 42 | ) 43 | -------------------------------------------------------------------------------- /pazel/tests/__init__.py: -------------------------------------------------------------------------------- 1 | """pazel tests.""" 2 | -------------------------------------------------------------------------------- /pazel/tests/test_bazel_rules.py: -------------------------------------------------------------------------------- 1 | """Test identifying Bazel rule type of a script and generating new rules to BUILD files.""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import unittest 8 | 9 | from pazel.bazel_rules import BazelRule 10 | from pazel.bazel_rules import get_native_bazel_rules 11 | from pazel.bazel_rules import infer_bazel_rule_type 12 | from pazel.bazel_rules import PyBinaryRule 13 | from pazel.bazel_rules import PY_BINARY_TEMPLATE 14 | from pazel.bazel_rules import PyLibraryRule 15 | from pazel.bazel_rules import PY_LIBRARY_TEMPLATE 16 | from pazel.bazel_rules import PyTestRule 17 | from pazel.bazel_rules import PY_TEST_TEMPLATE 18 | 19 | 20 | class TestTemplates(unittest.TestCase): 21 | """Test filling different Bazel rule templates.""" 22 | 23 | @staticmethod 24 | def _format_generated(generated): 25 | return "\n".join([g for g in generated.splitlines() if g.strip()]) 26 | 27 | def test_py_binary_template(self): 28 | """Test PY_BINARY_TEMPLATE.""" 29 | name = 'my' 30 | data = 'data = ["something"],' 31 | deps = "deps = ['//foo:bar']" 32 | 33 | expected = """py_binary( 34 | name = "my", 35 | srcs = ["my.py"], 36 | data = ["something"], 37 | deps = ['//foo:bar'] 38 | )""" 39 | 40 | # Generate the rule and strip empty lines. 41 | generated = PY_BINARY_TEMPLATE.format(name=name, data=data, deps=deps) 42 | generated = self._format_generated(generated) 43 | 44 | self.assertEqual(generated, expected) 45 | 46 | def test_py_library_template(self): 47 | """Test PY_LIBRARY_TEMPLATE.""" 48 | name = 'my' 49 | data = '' # Empty 'data' should be stripped away. 50 | deps = "deps = ['//foo:bar']" 51 | 52 | expected = """py_library( 53 | name = "my", 54 | srcs = ["my.py"], 55 | deps = ['//foo:bar'] 56 | )""" 57 | 58 | # Generate the rule and strip empty lines. 59 | generated = PY_LIBRARY_TEMPLATE.format(name=name, data=data, deps=deps) 60 | generated = self._format_generated(generated) 61 | 62 | self.assertEqual(generated, expected) 63 | 64 | def test_py_test_template(self): 65 | """Test PY_TEST_TEMPLATE.""" 66 | name = 'my_test' 67 | size = 'medium' 68 | data = 'data = ["something"],' 69 | deps = "deps = ['//foo:bar']" 70 | 71 | expected = """py_test( 72 | name = "my_test", 73 | srcs = ["my_test.py"], 74 | size = "medium", 75 | data = ["something"], 76 | deps = ['//foo:bar'] 77 | )""" 78 | 79 | # Generate the rule and strip empty lines. 80 | generated = PY_TEST_TEMPLATE.format(name=name, size=size, data=data, deps=deps) 81 | generated = self._format_generated(generated) 82 | 83 | self.assertEqual(generated, expected) 84 | 85 | 86 | # Define a few different script sources. 87 | # pylint: disable=invalid-name 88 | script_name = 'my.py' 89 | test_script_name = 'test_my.py' 90 | 91 | module_source = """ 92 | import time 93 | 94 | def myfunction(): 95 | pass 96 | """ 97 | 98 | binary_with_main_source = """ 99 | def main(): 100 | pass 101 | 102 | if __name__ == "__main__": 103 | main() 104 | """ 105 | 106 | binary_without_main_source = """ 107 | def myfunction(): 108 | pass 109 | 110 | myfunction() 111 | """ 112 | 113 | test_source = """ 114 | import unittest 115 | 116 | class TestPyBinaryRule(unittest.TestCase): 117 | pass 118 | 119 | if __name__ == '__main__': 120 | unittest.main() 121 | """ 122 | 123 | 124 | class TestBazelRule(unittest.TestCase): 125 | """Test BazelRule base class.""" 126 | 127 | def test_applies_to(self): 128 | """Test BazelRule.applies_to.""" 129 | with self.assertRaises(NotImplementedError): 130 | BazelRule.applies_to('script_name', 'script_source') 131 | 132 | def test_find_existing(self): 133 | """Test finding an existing Bazel rule in a BUILD source.""" 134 | build_source = """ 135 | py_test( 136 | name = "test_bazel_rules", 137 | srcs = ["test_bazel_rules.py"], 138 | size = "small", 139 | deps = ["//pazel:bazel_rules"], 140 | )""" 141 | self.assertIsNotNone(BazelRule.find_existing(build_source, 'test_bazel_rules.py')) 142 | self.assertIsNone(BazelRule.find_existing(build_source, 'missing.py')) 143 | 144 | def test_get_load_statement(self): 145 | """Test BazelRule.get_load_statement.""" 146 | self.assertIsNone(BazelRule.get_load_statement()) 147 | 148 | 149 | class TestPyBinaryRule(unittest.TestCase): 150 | """Test PyBinaryRule.""" 151 | 152 | def test_class_variables(self): 153 | """Test class variables of the PyBinaryRule class.""" 154 | self.assertEqual(PyBinaryRule.is_test_rule, False) 155 | self.assertEqual(PyBinaryRule.template, PY_BINARY_TEMPLATE) 156 | self.assertEqual(PyBinaryRule.rule_identifier, 'py_binary') 157 | 158 | def test_applies_to(self): 159 | """Test PyBinaryRule.applies_to for different script sources.""" 160 | # A script containing __main__ should generate a py_binary rule. 161 | self.assertEqual(PyBinaryRule.applies_to(script_name, binary_with_main_source), True) 162 | 163 | # Ditto for a level 0 function call. 164 | self.assertEqual(PyBinaryRule.applies_to(script_name, binary_without_main_source), True) 165 | 166 | # Modules should not generate a py_binary rule. 167 | self.assertEqual(PyBinaryRule.applies_to(script_name, module_source), False) 168 | 169 | # Tests should not generate a py_binary rule even though they contain __main__. 170 | self.assertEqual(PyBinaryRule.applies_to(test_script_name, test_source), False) 171 | 172 | 173 | class TestPyLibraryRule(unittest.TestCase): 174 | """Test PyLibraryRule.""" 175 | 176 | def test_class_variables(self): 177 | """Test class variables of the PyLibraryRule class.""" 178 | self.assertEqual(PyLibraryRule.is_test_rule, False) 179 | self.assertEqual(PyLibraryRule.template, PY_LIBRARY_TEMPLATE) 180 | self.assertEqual(PyLibraryRule.rule_identifier, 'py_library') 181 | 182 | def test_applies_to(self): 183 | """Test PyLibraryRule.applies_to for different script sources.""" 184 | self.assertEqual(PyLibraryRule.applies_to(script_name, binary_with_main_source), False) 185 | self.assertEqual(PyLibraryRule.applies_to(script_name, binary_without_main_source), False) 186 | self.assertEqual(PyLibraryRule.applies_to(script_name, module_source), True) 187 | self.assertEqual(PyLibraryRule.applies_to(test_script_name, test_source), False) 188 | 189 | 190 | class TestPyTestRule(unittest.TestCase): 191 | """Test PyTestRule.""" 192 | 193 | def test_class_variables(self): 194 | """Test class variables of the PyTestRule class.""" 195 | self.assertEqual(PyTestRule.is_test_rule, True) 196 | self.assertEqual(PyTestRule.template, PY_TEST_TEMPLATE) 197 | self.assertEqual(PyTestRule.rule_identifier, 'py_test') 198 | 199 | def test_applies_to(self): 200 | """Test PyTestRule.applies_to for different script sources.""" 201 | self.assertEqual(PyTestRule.applies_to(script_name, binary_with_main_source), False) 202 | self.assertEqual(PyTestRule.applies_to(script_name, binary_without_main_source), False) 203 | self.assertEqual(PyTestRule.applies_to(script_name, module_source), False) 204 | self.assertEqual(PyTestRule.applies_to(test_script_name, test_source), True) 205 | 206 | 207 | class TestNativeBazelRules(unittest.TestCase): 208 | """Test getting native Bazel rule classes.""" 209 | 210 | def test_get_native_bazel_rules(self): 211 | """Test getting the list of native Bazel rules.""" 212 | native_rules = set([PyBinaryRule, PyLibraryRule, PyTestRule]) 213 | self.assertEqual(set(get_native_bazel_rules()), native_rules) 214 | 215 | 216 | class TestBazelRuleInference(unittest.TestCase): 217 | """Test inferring the Bazel rule type of a script.""" 218 | 219 | def test_infer_bazel_rule_type(self): 220 | """Test inferring the Bazel rule type.""" 221 | custom_rules = [] 222 | 223 | self.assertEqual(infer_bazel_rule_type(script_name, binary_with_main_source, custom_rules), 224 | PyBinaryRule) 225 | self.assertEqual(infer_bazel_rule_type(script_name, binary_without_main_source, 226 | custom_rules), PyBinaryRule) 227 | self.assertEqual(infer_bazel_rule_type(script_name, module_source, custom_rules), 228 | PyLibraryRule) 229 | self.assertEqual(infer_bazel_rule_type(test_script_name, test_source, custom_rules), 230 | PyTestRule) 231 | 232 | 233 | if __name__ == '__main__': 234 | unittest.main() 235 | -------------------------------------------------------------------------------- /pazel/tests/test_generate_rule.py: -------------------------------------------------------------------------------- 1 | """Test generating Bazel rules.""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import unittest 8 | 9 | from pazel.generate_rule import sort_module_names 10 | 11 | 12 | class TestGenerateRule(unittest.TestCase): 13 | """Test generating Bazel rules.""" 14 | 15 | def test_sort_module_names(self): 16 | """Test sort_module_names.""" 17 | modules = ["abc", "xyz", "foo.bar1", "foo.bar2", "foo.abc.abc", "foo.cba.cba"] 18 | sorted_modules = sort_module_names(modules) 19 | 20 | expected_sorted_modules = ["abc", "xyz", "foo.bar1", "foo.bar2", "foo.abc.abc", 21 | "foo.cba.cba"] 22 | 23 | self.assertEqual(sorted_modules, expected_sorted_modules) 24 | 25 | 26 | if __name__ == '__main__': 27 | unittest.main() 28 | -------------------------------------------------------------------------------- /pazel/tests/test_helpers.py: -------------------------------------------------------------------------------- 1 | """Test helper functions.""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import unittest 8 | 9 | from pazel.helpers import parse_enclosed_expression 10 | 11 | 12 | class TestHelpers(unittest.TestCase): 13 | """Test helper functions.""" 14 | 15 | def test_parse_enclosed_expression(self): 16 | """Test parse_enclosed_expression.""" 17 | expected_expression = """py_library( 18 | name = "foo", 19 | srcs = ["foo.py"], 20 | deps = [ 21 | "//bar", 22 | requirement("pyyaml"), 23 | ], 24 | )""" 25 | 26 | source = """some text 27 | more text 28 | 29 | {expression} 30 | 31 | more text 32 | end 33 | """.format(expression=expected_expression) 34 | 35 | start = source.find('py_library') # Find the index at which the expression starts. 36 | 37 | expression = parse_enclosed_expression(source, start, '(') 38 | 39 | self.assertEqual(expression, expected_expression) 40 | 41 | 42 | if __name__ == '__main__': 43 | unittest.main() 44 | -------------------------------------------------------------------------------- /pazel/tests/test_parse_imports.py: -------------------------------------------------------------------------------- 1 | """Test parsing imports from a Python file.""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import unittest 8 | 9 | from pazel.parse_imports import get_imports 10 | 11 | 12 | class TestParseImports(unittest.TestCase): 13 | """Test helper functions.""" 14 | 15 | def test_get_imports(self): 16 | """Test parse_enclosed_expression.""" 17 | script_source = """ 18 | import ast 19 | from ast import parse 20 | from foo import bar as abc 21 | from asd import \ 22 | wasd 23 | """ 24 | 25 | packages, from_imports = get_imports(script_source) 26 | 27 | expected_packages = [('ast', None)] 28 | expected_from_imports = [('ast', 'parse'), ('foo', 'bar'), ('asd', 'wasd')] 29 | 30 | self.assertEqual(packages, expected_packages) 31 | self.assertEqual(from_imports, expected_from_imports) 32 | 33 | 34 | if __name__ == '__main__': 35 | unittest.main() 36 | -------------------------------------------------------------------------------- /pazel/tests/test_pazel_extensions.py: -------------------------------------------------------------------------------- 1 | """Test parsing user-defined extensions to pazel.""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import unittest 8 | 9 | from pazel.pazel_extensions import parse_pazel_extensions 10 | 11 | 12 | class TestParseImports(unittest.TestCase): 13 | """Test parsing user-defined extensions to pazel.""" 14 | 15 | def test_invalid_pazelrc_file(self): 16 | """Test that parse_pazel_extensions returns defaults for an invalid pazelrc file.""" 17 | pazelrc_path = 'fail' 18 | 19 | output_extension, custom_bazel_rules, custom_import_inference_rules, \ 20 | import_name_to_pip_name, local_import_name_to_dep, requirement_load \ 21 | = parse_pazel_extensions(pazelrc_path) 22 | 23 | self.assertEqual(output_extension.header, '') 24 | self.assertEqual(output_extension.footer, '') 25 | self.assertEqual(custom_bazel_rules, []) 26 | self.assertEqual(custom_import_inference_rules, []) 27 | self.assertEqual(import_name_to_pip_name, dict()) 28 | self.assertEqual(local_import_name_to_dep, dict()) 29 | self.assertEqual(requirement_load, 'load("@my_deps//:requirements.bzl", "requirement")') 30 | 31 | 32 | if __name__ == '__main__': 33 | unittest.main() 34 | -------------------------------------------------------------------------------- /sample_app/.pazelrc: -------------------------------------------------------------------------------- 1 | """Define pazel extensions for this directory and its subdirectories.""" 2 | 3 | from __future__ import absolute_import 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import os 8 | import re 9 | 10 | from pazel.bazel_rules import BazelRule 11 | 12 | 13 | HEADER = """package(default_visibility = ["//visibility:public"]) 14 | 15 | """ 16 | 17 | FOOTER = """# My footer""" 18 | 19 | # Template will be filled and used to generate BUILD files. 20 | PY_DOCTEST_TEMPLATE = """py_doctest( 21 | "{name}", 22 | "{name}.py", 23 | deps = [{deps}], 24 | {data} 25 | )""" 26 | 27 | class PyDoctestRule(BazelRule): 28 | """Class for representing custom Bazel rule py_doctest as defined in custom_rules.bzl. 29 | 30 | Note that custom rules need to implement the interface defined by BazelRule. 31 | """ 32 | 33 | # Required class variables. 34 | is_test_rule = True # Is this a test rule? 35 | template = PY_DOCTEST_TEMPLATE # Filled version of this will be written to the BUILD file. 36 | rule_identifier = 'py_doctest' # The name of the rule. 37 | 38 | @staticmethod 39 | def applies_to(script_name, script_source): 40 | """Check whether py_doctest rule should be used for the given script. 41 | 42 | Args: 43 | script_name (str): Name of a Python script without the .py suffix. 44 | script_source (str): Source code of the script. 45 | 46 | Returns: 47 | applies (bool): Whether py_doctest should be used to represent the script. 48 | """ 49 | imports_doctest = re.findall('import doctest', script_source) 50 | 51 | return imports_doctest 52 | 53 | @staticmethod 54 | def get_load_statement(): 55 | """Return the load statement required for using this rule.""" 56 | return 'load("//:custom_rules.bzl", "py_doctest")' 57 | 58 | 59 | class LocalImportAllInferenceRule(object): 60 | """Import inference rule for "from some_local_package import *" type of imports. 61 | 62 | The rule is not recursive so only modules in the first level will be imported. 63 | """ 64 | 65 | @staticmethod 66 | def holds(project_root, base, unknown): 67 | """Check if base is a local package and unknown is '*'. 68 | 69 | Args: 70 | project_root (str): Local imports are assumed to be relative to this path. 71 | base (str): Name of a package or a module. 72 | unknown (str): Can package, module, function or any other object. 73 | 74 | Returns: 75 | packages (None): The rule applies only to modules. 76 | modules (list of str or None): The imported modules. None if the rule does not match. 77 | """ 78 | packages = None 79 | modules = None 80 | 81 | # Check if 'base' is a local package. 82 | package_path = os.path.join(project_root, base.replace('.', '/')) 83 | base_is_package = os.path.isdir(package_path) 84 | 85 | if base_is_package and unknown == '*': 86 | python_filenames = [f.replace('.py', '') for f in os.listdir(package_path) 87 | if f.endswith('.py')] 88 | 89 | modules = [] 90 | 91 | for python_filename in python_filenames: 92 | dotted_path = base + '.%s' % python_filename 93 | modules.append(dotted_path) 94 | 95 | return packages, modules 96 | 97 | 98 | # Add custom classes implementing BazelRule to this list so that pazel registers them. 99 | EXTRA_BAZEL_RULES = [PyDoctestRule] 100 | 101 | # Add custom import inference classes implementing ImportInferenceRule to this list. 102 | EXTRA_IMPORT_INFERENCE_RULES = [LocalImportAllInferenceRule] 103 | 104 | # Map import name to pip install name, if they differ. 105 | EXTRA_IMPORT_NAME_TO_PIP_NAME = {'yaml': 'pyyaml'} 106 | 107 | # Map local package import name to its Bazel dependency. 108 | EXTRA_LOCAL_IMPORT_NAME_TO_DEP = {'my_dummy_package': '//my_dummy_package'} 109 | 110 | # Change 'REQUIREMENT' to override the default load statement of the Bazel rule for 111 | # installing pip packages. See https://github.com/bazelbuild/rules_python 112 | REQUIREMENT = """load("@my_deps//:requirements.bzl", "requirement")""" 113 | -------------------------------------------------------------------------------- /sample_app/BUILD: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | py_library( 4 | name = "__init__", 5 | srcs = ["__init__.py"], 6 | ) 7 | 8 | # My footer 9 | -------------------------------------------------------------------------------- /sample_app/WORKSPACE: -------------------------------------------------------------------------------- 1 | workspace(name = "sample_app") 2 | 3 | git_repository( 4 | name = "io_bazel_rules_python", 5 | remote = "https://github.com/bazelbuild/rules_python.git", 6 | # Latest commit as of 7 April 2018. 7 | commit = "b25495c47eb7446729a2ed6b1643f573afa47d99", 8 | ) 9 | 10 | # Only needed for PIP support: 11 | load("@io_bazel_rules_python//python:pip.bzl", "pip_repositories") 12 | 13 | pip_repositories() 14 | 15 | load("@io_bazel_rules_python//python:pip.bzl", "pip_import") 16 | 17 | # This rule translates the specified requirements.txt into 18 | # @my_deps//:requirements.bzl, which itself exposes a pip_install method. 19 | pip_import( 20 | name = "my_deps", 21 | requirements = "requirements-pip.txt", 22 | ) 23 | 24 | # Load the pip_install symbol for my_deps, and create the dependencies' 25 | # repositories. 26 | load("@my_deps//:requirements.bzl", "pip_install") 27 | pip_install() -------------------------------------------------------------------------------- /sample_app/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuomasr/pazel/7109fe565aa50d15ec6de1b6f0bae5ac06a28a3a/sample_app/__init__.py -------------------------------------------------------------------------------- /sample_app/custom_rules.bzl: -------------------------------------------------------------------------------- 1 | def custom_rule(filename, src): 2 | native.py_library( 3 | name = filename, 4 | srcs = [src] 5 | ) 6 | 7 | def py_doctest(filename, src, deps=None, data=None): 8 | if deps == None: 9 | deps = [] 10 | 11 | if data == None: 12 | data = [] 13 | 14 | native.py_binary( 15 | name = filename, 16 | srcs = [src], 17 | deps = deps, 18 | data = data, 19 | args = ['-v'] 20 | ) -------------------------------------------------------------------------------- /sample_app/external/requirements-pip.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | pyyaml 3 | requests -------------------------------------------------------------------------------- /sample_app/foo/BUILD: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | load("@my_deps//:requirements.bzl", "requirement") 4 | 5 | # pazel-ignore 6 | load("//:custom_rules.bzl", "custom_rule") 7 | 8 | py_library( 9 | name = "__init__", 10 | srcs = ["__init__.py"], 11 | deps = ["//foo:bar1"], 12 | ) 13 | 14 | py_library( 15 | name = "bar1", 16 | srcs = ["bar1.py"], 17 | deps = [requirement("numpy")], 18 | ) 19 | 20 | py_library( 21 | name = "bar2", 22 | srcs = ["bar2.py"], 23 | deps = [ 24 | "//foo:bar1", 25 | requirement("pyyaml"), 26 | ], 27 | ) 28 | 29 | py_binary( 30 | name = "bar3", 31 | srcs = ["bar3.py"], 32 | ) 33 | 34 | py_library( 35 | name = "bar6", 36 | srcs = ["bar6.py"], 37 | deps = ["//xyz:abc1"], 38 | ) 39 | 40 | py_library( 41 | name = "foo", 42 | srcs = ["foo.py"], 43 | ) 44 | 45 | # pazel-ignore 46 | py_test( # pazel would mark bar4.py as a library but it remains as a test because it is ignored. 47 | name = "bar4", 48 | srcs = ["bar4.py"], 49 | deps = [], 50 | ) 51 | 52 | # pazel-ignore 53 | custom_rule("bar5", # pazel recognizes positional arguments, too. 54 | "bar5.py" 55 | ) 56 | 57 | # My footer 58 | -------------------------------------------------------------------------------- /sample_app/foo/__init__.py: -------------------------------------------------------------------------------- 1 | from foo.bar1 import sample 2 | 3 | __all__ = ('sample', ) 4 | -------------------------------------------------------------------------------- /sample_app/foo/bar1.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def sample(): 4 | return 1. 5 | -------------------------------------------------------------------------------- /sample_app/foo/bar2.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | 3 | from foo.bar1 import sample 4 | 5 | def increment_sample(): 6 | print(yaml) 7 | return sample() + 1. -------------------------------------------------------------------------------- /sample_app/foo/bar3.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import time 4 | 5 | 6 | def main(): 7 | print("Hello pazel.") 8 | 9 | 10 | main() -------------------------------------------------------------------------------- /sample_app/foo/bar4.py: -------------------------------------------------------------------------------- 1 | """Ignored file.""" -------------------------------------------------------------------------------- /sample_app/foo/bar5.py: -------------------------------------------------------------------------------- 1 | """Ignored file.""" -------------------------------------------------------------------------------- /sample_app/foo/bar6.py: -------------------------------------------------------------------------------- 1 | from xyz import * 2 | -------------------------------------------------------------------------------- /sample_app/foo/foo.py: -------------------------------------------------------------------------------- 1 | # Dummy file to showcase pazel formatting. -------------------------------------------------------------------------------- /sample_app/tests/BUILD: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | load("@my_deps//:requirements.bzl", "requirement") 4 | load("//:custom_rules.bzl", "py_doctest") 5 | 6 | py_library( 7 | name = "__init__", 8 | srcs = ["__init__.py"], 9 | ) 10 | 11 | py_test( 12 | name = "test_bar1", 13 | srcs = ["test_bar1.py"], 14 | size = "medium", 15 | data = ["test_data/dummy"], 16 | deps = [ 17 | "//foo:bar1", 18 | requirement("requests"), 19 | ], 20 | ) 21 | 22 | py_test( 23 | name = "test_bar2", 24 | srcs = ["test_bar2.py"], 25 | size = "large", 26 | deps = ["//foo:bar2"], 27 | ) 28 | 29 | py_test( 30 | name = "test_bar3", 31 | srcs = ["test_bar3.py"], 32 | size = "small", 33 | data = glob(["test_data/*.png"]), 34 | ) 35 | 36 | py_doctest( 37 | "test_doctest", 38 | "test_doctest.py", 39 | deps = [], 40 | ) 41 | 42 | # My footer 43 | -------------------------------------------------------------------------------- /sample_app/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuomasr/pazel/7109fe565aa50d15ec6de1b6f0bae5ac06a28a3a/sample_app/tests/__init__.py -------------------------------------------------------------------------------- /sample_app/tests/test_bar1.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import requests 4 | 5 | from foo.bar1 import sample 6 | 7 | 8 | class Bar1Test(unittest.TestCase): 9 | 10 | def test_sample(self): 11 | value = sample() 12 | self.assertEqual(value, 1.) 13 | 14 | 15 | if __name__ == '__main__': 16 | unittest.main() -------------------------------------------------------------------------------- /sample_app/tests/test_bar2.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from foo.bar2 import increment_sample 4 | 5 | 6 | class Bar2Test(unittest.TestCase): 7 | 8 | def test_increment_sample(self): 9 | value = increment_sample() 10 | self.assertEqual(value, 2.) 11 | 12 | 13 | if __name__ == '__main__': 14 | unittest.main() -------------------------------------------------------------------------------- /sample_app/tests/test_bar3.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | class Bar3Test(unittest.TestCase): 4 | 5 | def test_main(self): 6 | pass 7 | 8 | 9 | if __name__ == '__main__': 10 | unittest.main() -------------------------------------------------------------------------------- /sample_app/tests/test_data/dummy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuomasr/pazel/7109fe565aa50d15ec6de1b6f0bae5ac06a28a3a/sample_app/tests/test_data/dummy -------------------------------------------------------------------------------- /sample_app/tests/test_doctest.py: -------------------------------------------------------------------------------- 1 | def square(x): 2 | """Return the square of x. 3 | 4 | >>> square(2) 5 | 4 6 | >>> square(-2) 7 | 4 8 | """ 9 | 10 | return x * x 11 | 12 | if __name__ == '__main__': 13 | import doctest 14 | doctest.testmod() -------------------------------------------------------------------------------- /sample_app/xyz/BUILD: -------------------------------------------------------------------------------- 1 | package(default_visibility = ["//visibility:public"]) 2 | 3 | py_binary( 4 | name = "abc1", 5 | srcs = ["abc1.py"], 6 | deps = [ 7 | "//foo:__init__", 8 | "//foo:foo", 9 | ], 10 | ) 11 | 12 | # My footer 13 | -------------------------------------------------------------------------------- /sample_app/xyz/abc1.py: -------------------------------------------------------------------------------- 1 | from foo import sample # Import from foo's public interface. 2 | from foo import foo # Import a module with the same name as the package. 3 | 4 | 5 | def main(): 6 | print(sample()) 7 | 8 | 9 | main() 10 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """Entrypoint for starting with pazel. 2 | 3 | Run: 4 | 5 | `python setup.py install` to install pazel. 6 | `python setup.py develop` to develop pazel. 7 | `python setup.py test` to run pazel tests. 8 | """ 9 | 10 | from setuptools import setup, find_packages 11 | 12 | DESCRIPTION = "Generate Bazel BUILD files for a Python project." 13 | 14 | setup( 15 | name='pazel', 16 | version='0.1.0', 17 | description=DESCRIPTION, 18 | packages=find_packages(exclude=('sample_app', 'sample_app.foo', 'sample_app.tests')), 19 | entry_points={ 20 | 'console_scripts': ['pazel = pazel.app:main'] 21 | }, 22 | test_suite='pazel.tests' 23 | ) 24 | --------------------------------------------------------------------------------