├── .gitignore ├── .pre-commit-config.yaml ├── Dockerfile ├── LICENSE ├── README.md ├── action.yml ├── entrypoint.py ├── pyproject.toml ├── screenshot1.png ├── screenshot2.png └── src ├── __init__.py ├── annotationutils.py ├── changeutils.py ├── graph_generator ├── dataflowpass.py ├── extract_graphs.py ├── graphgenerator.py ├── graphgenutils.py ├── type_lattice_generator.py └── typeparsing │ ├── __init__.py │ ├── aliasreplacement.py │ ├── erasure.py │ ├── inheritancerewrite.py │ ├── nodes.py │ ├── pruneannotations.py │ ├── rewriterules │ ├── __init__.py │ ├── removegenericwithany.py │ ├── removerecursivegenerics.py │ ├── removestandalones.py │ ├── removeunionwithanys.py │ └── rewriterule.py │ ├── rewriterulevisitor.py │ └── visitor.py └── metadata └── typingRules.json /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # Pycharm 107 | .idea 108 | 109 | # VSCode 110 | .vscode 111 | 112 | data/ 113 | *.pkl.gz 114 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/psf/black 3 | rev: stable 4 | hooks: 5 | - id: black 6 | language_version: python3.6 7 | - repo: https://github.com/pre-commit/pre-commit-hooks 8 | rev: v2.3.0 9 | hooks: 10 | - id: end-of-file-fixer 11 | - id: trailing-whitespace 12 | - id: mixed-line-ending 13 | - id: check-ast 14 | - id: check-case-conflict 15 | - id: debug-statements 16 | - id: requirements-txt-fixer 17 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-buster 2 | 3 | ENV PYTHONUNBUFFERED=1 4 | 5 | RUN apt update && apt -y upgrade 6 | RUN apt install -y python3-numpy python3-pip python3-requests 7 | RUN pip3 install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html 8 | RUN pip3 install torch-scatter==2.0.4+cpu -f https://pytorch-geometric.com/whl/torch-1.5.0.html 9 | RUN pip3 install dpu-utils typed-ast ptgnn 10 | 11 | ENV PYTHONPATH=/usr/src/ 12 | ADD https://github.com/typilus/typilus-action/releases/download/v0.1/typilus20200507.pkl.gz /usr/src/model.pkl.gz 13 | COPY src /usr/src 14 | COPY entrypoint.py /usr/src/entrypoint.py 15 | 16 | ENTRYPOINT ["python", "/usr/src/entrypoint.py"] 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | All rights reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Typilus: Suggest Python Type Annotations 2 | 3 | A GitHub action that suggests type annotations for Python using machine learning. 4 | 5 | This action makes suggestions within each pull request as 6 | [suggested edits](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/incorporating-feedback-in-your-pull-request#applying-a-suggested-change). 7 | You can then directly apply these suggestions to your code or ignore them. 8 | 9 | | ![Sample Suggestion](/screenshot1.png) | ![Sample Suggestion](/screenshot2.png) | 10 | | -- | -- | 11 | 12 | ***What are Python type annotations?*** 13 | Introduced in Python 3.5, [type hints](https://www.python.org/dev/peps/pep-0484/) 14 | (more traditionally called type annotations) allow users 15 | to annotate their code with the expected types. These annotations are 16 | optionally checked by external tools, such as [mypy](http://www.mypy-lang.org/) and [pyright](https://github.com/Microsoft/pyright), 17 | to prevent type errors; they also facilitate code comprehension and navigation. 18 | The [`typing`](https://docs.python.org/3/library/typing.html) module 19 | provides the core types. 20 | 21 | ***Why use machine learning?*** 22 | Given the dynamic nature of Python, type inference is challenging, 23 | especially over partial contexts. To tackle this challenge, we use a graph neural 24 | network model that predicts types by probabilistically reasoning over 25 | a program’s structure, names, and patterns. This allows us to make 26 | suggestions with only a partial context, at the cost of suggesting some false 27 | positives. 28 | 29 | 30 | ### Install Action in your Repository 31 | 32 | To use the GitHub action, create a workflow file. For example, 33 | ```yaml 34 | name: Typilus Type Annotation Suggestions 35 | 36 | # Controls when the action will run. Triggers the workflow on push or pull request 37 | # events but only for the master branch 38 | on: 39 | pull_request: 40 | branches: [ master ] 41 | 42 | jobs: 43 | suggest: 44 | # The type of runner that the job will run on 45 | runs-on: ubuntu-latest 46 | 47 | steps: 48 | # Checks-out your repository under $GITHUB_WORKSPACE, so that typilus can access it. 49 | - uses: actions/checkout@v2 50 | - uses: typilus/typilus-action@master 51 | env: 52 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 53 | MODEL_PATH: path/to/model.pkl.gz # Optional: provide the path of a custom model instead of the pre-trained model. 54 | SUGGESTION_CONFIDENCE_THRESHOLD: 0.8 # Configure this to limit the confidence of suggestions on un-annotated locations. A float in [0, 1]. Default 0.8 55 | DISAGREEMENT_CONFIDENCE_THRESHOLD: 0.95 # Configure this to limit the confidence of suggestions on annotated locations. A float in [0, 1]. Default 0.95 56 | ``` 57 | The action uses the `GITHUB_TOKEN` to retrieve the diff of the pull request 58 | and to post comments on the analyzed pull request. 59 | 60 | 61 | 62 | #### Technical Details & Internals 63 | This GitHub action is a reimplementation of the Graph2Class model of 64 | [Allamanis _et al._ PLDI 2020](https://arxiv.org/abs/2004.10657) using the 65 | [`ptgnn`](https://github.com/microsoft/ptgnn/) library. Internally, it 66 | uses a Graph Neural Network to predict likely type annotations for Python 67 | code. 68 | 69 | This action uses a pre-trained neural network that has been trained on 70 | a corpus of open-source repositories that use Python's type annotations. 71 | At this point we do _not_ support online adaptation of the model to each project. 72 | 73 | 74 | ##### Training your own model 75 | You may wish to train your own model and use it in this action. To 76 | do so, please follow the steps in [`ptgnn`](https://github.com/microsoft/ptgnn/). 77 | Then provide a path to the model in your GitHub action configuration, through the 78 | `MODEL_PATH` environment variable. 79 | 80 | 81 | ## Contributing 82 | We welcome external contributions and ideas. Please look at the issues in the repository 83 | for ideas and improvements. 84 | -------------------------------------------------------------------------------- /action.yml: -------------------------------------------------------------------------------- 1 | # action.yml 2 | name: 'Typilus: Suggest Python Type Annotations' 3 | description: 'Suggest Likely Python Type Annotations' 4 | branding: 5 | icon: box 6 | color: gray-dark 7 | inputs: 8 | path: 9 | description: 'File or directory to run Typilus on.' 10 | required: false 11 | default: '.' 12 | outputs: 13 | output: 14 | description: 'Makes suggeted edits to the pull requests for adding Python Type annotations.' 15 | runs: 16 | using: 'docker' 17 | image: 'Dockerfile' 18 | args: 19 | - ${{ inputs.path }} 20 | -------------------------------------------------------------------------------- /entrypoint.py: -------------------------------------------------------------------------------- 1 | #!/bin/python 2 | import os 3 | from glob import iglob 4 | import json 5 | import sys 6 | from tempfile import TemporaryDirectory 7 | from typing import Tuple, List 8 | 9 | import requests 10 | from dpu_utils.utils import load_jsonl_gz 11 | from ptgnn.implementations.typilus.graph2class import Graph2Class 12 | 13 | from changeutils import get_changed_files 14 | from annotationutils import ( 15 | annotate_line, 16 | find_annotation_line, 17 | group_suggestions, 18 | annotation_rewrite, 19 | ) 20 | from graph_generator.extract_graphs import extract_graphs, Monitoring 21 | 22 | 23 | class TypeSuggestion: 24 | def __init__( 25 | self, 26 | filepath: str, 27 | name: str, 28 | file_location: Tuple[int, int], 29 | suggestion: str, 30 | symbol_kind: str, 31 | confidence: float, 32 | annotation_lineno: int = 0, 33 | is_disagreement: bool = False, 34 | ): 35 | self.filepath = filepath 36 | self.name = name 37 | self.file_location = file_location 38 | self.suggestion = suggestion 39 | self.symbol_kind = symbol_kind 40 | self.confidence = confidence 41 | self.annotation_lineno = annotation_lineno 42 | self.is_disagreement = is_disagreement 43 | 44 | def __repr__(self) -> str: 45 | return ( 46 | f"Suggestion@{self.filepath}:{self.file_location} " 47 | f"Symbol Name: `{self.name}` Suggestion `{self.suggestion}` " 48 | f"Confidence: {self.confidence:.2%}" 49 | ) 50 | 51 | 52 | assert ( 53 | os.environ["GITHUB_EVENT_NAME"] == "pull_request" 54 | ), "This action runs only on pull request events." 55 | github_token = os.environ["GITHUB_TOKEN"] 56 | debug = False 57 | 58 | with open(os.environ["GITHUB_EVENT_PATH"]) as f: 59 | event_data = json.load(f) 60 | if debug: 61 | print("Event data:") 62 | print(json.dumps(event_data, indent=4)) 63 | 64 | repo_path = "." # TODO: Is this always true? 65 | 66 | if debug: 67 | print("ENV Variables") 68 | for env_name, env_value in os.environ.items(): 69 | print(f"{env_name} --> {env_value}") 70 | 71 | diff_rq = requests.get( 72 | event_data["pull_request"]["url"], 73 | headers={ 74 | "authorization": f"Bearer {github_token}", 75 | "Accept": "application/vnd.github.v3.diff", 76 | }, 77 | ) 78 | print("Diff GET Status Code: ", diff_rq.status_code) 79 | 80 | 81 | changed_files = get_changed_files(diff_rq.text) 82 | if len(changed_files) == 0: 83 | print("No relevant changes found.") 84 | sys.exit(0) 85 | 86 | 87 | monitoring = Monitoring() 88 | suggestion_confidence_threshold = float(os.getenv("SUGGESTION_CONFIDENCE_THRESHOLD", 0.5)) 89 | diagreement_confidence_threshold = float(os.getenv("DISAGREEMENT_CONFIDENCE_THRESHOLD", 0.95)) 90 | 91 | if debug: 92 | print( 93 | f"Confidence thresholds {suggestion_confidence_threshold:.2f} and {diagreement_confidence_threshold:.2f}." 94 | ) 95 | 96 | 97 | with TemporaryDirectory() as out_dir: 98 | typing_rules_path = os.path.join(os.path.dirname(__file__), "metadata", "typingRules.json") 99 | extract_graphs( 100 | repo_path, typing_rules_path, files_to_extract=set(changed_files), target_folder=out_dir, 101 | ) 102 | 103 | def data_iter(): 104 | for datafile_path in iglob(os.path.join(out_dir, "*.jsonl.gz")): 105 | print(f"Looking into {datafile_path}...") 106 | for graph in load_jsonl_gz(datafile_path): 107 | yield graph 108 | 109 | model_path = os.getenv("MODEL_PATH", "/usr/src/model.pkl.gz") 110 | model, nn = Graph2Class.restore_model(model_path, "cpu") 111 | 112 | type_suggestions: List[TypeSuggestion] = [] 113 | for graph, predictions in model.predict(data_iter(), nn, "cpu"): 114 | # predictions has the type: Dict[int, Tuple[str, float]] 115 | filepath = graph["filename"] 116 | 117 | if debug: 118 | print("Predictions:", predictions) 119 | print("SuperNodes:", graph["supernodes"]) 120 | 121 | for supernode_idx, (predicted_type, predicted_prob) in predictions.items(): 122 | supernode_data = graph["supernodes"][str(supernode_idx)] 123 | if supernode_data["type"] == "variable": 124 | continue # Do not suggest annotations on variables for now. 125 | lineno, colno = supernode_data["location"] 126 | suggestion = TypeSuggestion( 127 | filepath, 128 | supernode_data["name"], 129 | (lineno, colno), 130 | annotation_rewrite(predicted_type), 131 | supernode_data["type"], 132 | predicted_prob, 133 | is_disagreement=supernode_data["annotation"] != "??" 134 | and supernode_data["annotation"] != predicted_type, 135 | ) 136 | 137 | print("Suggestion: ", suggestion) 138 | 139 | if lineno not in changed_files[filepath]: 140 | continue 141 | elif suggestion.name == "%UNK%": 142 | continue 143 | 144 | if ( 145 | supernode_data["annotation"] == "??" 146 | and suggestion.confidence > suggestion_confidence_threshold 147 | ): 148 | type_suggestions.append(suggestion) 149 | elif ( 150 | suggestion.is_disagreement 151 | # and suggestion.confidence > diagreement_confidence_threshold 152 | ): 153 | pass # TODO: Disabled for now: type_suggestions.append(suggestion) 154 | 155 | # Add PR comments 156 | if debug: 157 | print("# Suggestions:", len(type_suggestions)) 158 | for suggestion in type_suggestions: 159 | print(suggestion) 160 | 161 | comment_url = event_data["pull_request"]["review_comments_url"] 162 | commit_id = event_data["pull_request"]["head"]["sha"] 163 | 164 | for suggestion in type_suggestions: 165 | if suggestion.symbol_kind == "class-or-function": 166 | suggestion.annotation_lineno = find_annotation_line( 167 | suggestion.filepath[1:], suggestion.file_location, suggestion.name 168 | ) 169 | else: # when the underlying symbol is a parameter 170 | suggestion.annotation_lineno = suggestion.file_location[0] 171 | 172 | # Group type suggestions by (filepath + lineno) 173 | grouped_suggestions = group_suggestions(type_suggestions) 174 | 175 | def bucket_confidences(confidence: float) -> str: 176 | if confidence >= 0.95: 177 | return ":fire:" 178 | if confidence >= 0.85: 179 | return ":bell:" 180 | if confidence >= 0.7: 181 | return ":confused:" 182 | return ":question:" 183 | 184 | def report_confidence(suggestions): 185 | suggestions = sorted(suggestions, key=lambda s: -s.confidence) 186 | return "".join( 187 | f"| `{s.name}` | `{s.suggestion}` | {s.confidence:.1%} {bucket_confidences(s.confidence)} | \n" 188 | for s in suggestions 189 | ) 190 | 191 | for same_line_suggestions in grouped_suggestions: 192 | suggestion = same_line_suggestions[0] 193 | path = suggestion.filepath[1:] # No slash in the beginning 194 | annotation_lineno = suggestion.annotation_lineno 195 | with open(path) as file: 196 | target_line = file.readlines()[annotation_lineno - 1] 197 | data = { 198 | "path": path, 199 | "line": annotation_lineno, 200 | "side": "RIGHT", 201 | "commit_id": commit_id, 202 | "body": "The following type annotation(s) might be useful:\n ```suggestion\n" 203 | f"{annotate_line(target_line, same_line_suggestions)}```\n" 204 | f"### :chart_with_upwards_trend: Prediction Stats\n" 205 | f"| Symbol | Annotation | Confidence |\n" 206 | f"| -- | -- | --: |\n" 207 | f"{report_confidence(same_line_suggestions)}", 208 | } 209 | headers = { 210 | "authorization": f"Bearer {github_token}", 211 | "Accept": "application/vnd.github.v3.raw+json", 212 | } 213 | r = requests.post(comment_url, data=json.dumps(data), headers=headers) 214 | if debug: 215 | print("URL: ", comment_url) 216 | print(f"Data: {data}. Status Code: {r.status_code}. Text: {r.text}") 217 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 100 3 | include = '\.pyi?$' 4 | exclude = ''' 5 | /( 6 | \.git 7 | | \.mypy_cache 8 | | \.tox 9 | | \.venv 10 | | _build 11 | | buck-out 12 | | build 13 | | dist 14 | )/ 15 | ''' 16 | -------------------------------------------------------------------------------- /screenshot1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typilus/typilus-action/00a57f1b62f8811a751ad854f0e44d45ea159376/screenshot1.png -------------------------------------------------------------------------------- /screenshot2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typilus/typilus-action/00a57f1b62f8811a751ad854f0e44d45ea159376/screenshot2.png -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typilus/typilus-action/00a57f1b62f8811a751ad854f0e44d45ea159376/src/__init__.py -------------------------------------------------------------------------------- /src/annotationutils.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import Tuple 3 | import re 4 | from itertools import groupby 5 | 6 | 7 | def find_suggestion_for_return(suggestions): 8 | for s in suggestions: 9 | if s.symbol_kind == "class-or-function": 10 | return s 11 | else: 12 | return None 13 | 14 | 15 | def annotate_line(line, suggestions): 16 | para_suggestions = sorted( 17 | (s for s in suggestions if s.symbol_kind == "parameter"), key=lambda x: x.file_location[1] 18 | ) 19 | annotated_line = annotate_parameters(line, para_suggestions) 20 | 21 | ret_suggestion = find_suggestion_for_return(suggestions) 22 | if ret_suggestion is not None: 23 | annotated_line = annotate_return(annotated_line, ret_suggestion) 24 | 25 | return annotated_line 26 | 27 | 28 | def insert_at(original, inserted, idx): 29 | return original[:idx] + inserted + original[idx:] 30 | 31 | 32 | def annotate_parameters(line, suggestions): 33 | """ 34 | Annotate the parameters of a function on a particular line 35 | """ 36 | annotated_line = " " + line 37 | length_increase = 0 38 | for s in suggestions: 39 | assert line[s.file_location[1] :].startswith(s.name) 40 | insertion_position = s.file_location[1] + len(s.name) + 1 + length_increase 41 | annotated_line = insert_at(annotated_line, f": {s.suggestion}", insertion_position) 42 | length_increase += len(s.suggestion) + 2 43 | return annotated_line 44 | 45 | 46 | def annotate_return(line, suggestion): 47 | """ 48 | Annotate the return of a function 49 | """ 50 | assert line.rstrip().endswith(":") 51 | return line.rstrip()[:-1] + f" -> {suggestion.suggestion}" + ":\n" 52 | 53 | 54 | def find_annotation_line(filepath, location, func_name): 55 | with open(filepath) as f: 56 | lines = f.readlines() 57 | 58 | assert func_name in lines[location[0] - 1] 59 | 60 | # Assume that the function's return is *not* already annotated. 61 | func_def_end = re.compile(r"\)\s*:$") 62 | 63 | annotation_lineno = location[0] 64 | while annotation_lineno <= len(lines): 65 | if func_def_end.search(lines[annotation_lineno - 1].rstrip()) is not None: 66 | break 67 | annotation_lineno += 1 68 | else: 69 | raise Exception("Cannot find the closing brace for the parameter list.") 70 | 71 | return annotation_lineno 72 | 73 | 74 | def group_suggestions(suggestions): 75 | def key(s): 76 | return s.filepath + str(s.annotation_lineno) 77 | 78 | sorted_suggestions = sorted(suggestions, key=key) 79 | return [list(it) for k, it in groupby(sorted_suggestions, key)] 80 | 81 | 82 | ALIASES = {"typing.Text": "str"} 83 | 84 | 85 | def annotation_rewrite(annotation: str) -> str: 86 | for k, v in ALIASES.items(): 87 | annotation = annotation.replace(k, v) 88 | return annotation 89 | -------------------------------------------------------------------------------- /src/changeutils.py: -------------------------------------------------------------------------------- 1 | import re 2 | from typing import Dict, Set, List, NamedTuple 3 | 4 | 5 | HUNK_MATCH = re.compile("^@@ -\d+,\d+ \+(\d+),\d+ @@") 6 | 7 | 8 | def get_line_ranges_of_interest(diff_lines: List[str]) -> Set[int]: 9 | lines_of_interest = set() 10 | current_line = 0 11 | for line in diff_lines: 12 | hunk_start_match = HUNK_MATCH.match(line) 13 | if hunk_start_match: 14 | current_line = int(hunk_start_match.group(1)) 15 | elif line.startswith("+"): 16 | lines_of_interest.add(current_line) 17 | current_line += 1 18 | elif not line.startswith("-"): 19 | current_line += 1 20 | elif line.startswith("\\"): 21 | assert False, "When does this happen?" 22 | 23 | return lines_of_interest 24 | 25 | 26 | def get_changed_files(diff: str, suffix=".py") -> Dict[str, Set[int]]: 27 | per_file_diff = diff.split("diff --git ") 28 | changed_files: Dict[str, Set[int]] = {} 29 | for file_diff in per_file_diff: 30 | if len(file_diff) == 0: 31 | continue 32 | file_diff_lines = file_diff.splitlines() 33 | 34 | if file_diff_lines[1].startswith("deleted"): 35 | continue 36 | elif file_diff_lines[1].startswith("new file"): 37 | assert file_diff_lines[2].startswith("index") 38 | assert file_diff_lines[3].startswith("---") 39 | assert file_diff_lines[4].startswith("+++ b/") 40 | target_filepath = file_diff_lines[4][len("+++ b") :] 41 | remaining_lines = file_diff_lines[5:] 42 | elif file_diff_lines[1].startswith("index"): 43 | assert file_diff_lines[2].startswith("--- a/") 44 | assert file_diff_lines[3].startswith("+++ b/") 45 | target_filepath = file_diff_lines[3][len("+++ b") :] 46 | remaining_lines = file_diff_lines[4:] 47 | elif file_diff_lines[1].startswith("similarity"): 48 | assert file_diff_lines[2].startswith("rename") 49 | assert file_diff_lines[3].startswith("rename") 50 | assert file_diff_lines[4].startswith("index") 51 | assert file_diff_lines[5].startswith("--- a/") 52 | assert file_diff_lines[6].startswith("+++ b/") 53 | target_filepath = file_diff_lines[6][len("+++ b") :] 54 | remaining_lines = file_diff_lines[7:] 55 | else: 56 | raise Exception(file_diff) 57 | 58 | if target_filepath.endswith(suffix): 59 | assert target_filepath not in changed_files 60 | changed_files[target_filepath] = get_line_ranges_of_interest(remaining_lines) 61 | 62 | return changed_files 63 | -------------------------------------------------------------------------------- /src/graph_generator/dataflowpass.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | from itertools import chain 3 | from symtable import SymbolTable 4 | from typing import Any, Dict, Optional, Set, Union, List 5 | 6 | 7 | from typed_ast.ast3 import ( 8 | Mod, 9 | Compare, 10 | Lambda, 11 | arg, 12 | Global, 13 | Nonlocal, 14 | arguments, 15 | Name, 16 | comprehension, 17 | withitem, 18 | Assign, 19 | AnnAssign, 20 | AugAssign, 21 | FormattedValue, 22 | Attribute, 23 | dump, 24 | ) 25 | from typed_ast.ast3 import ( 26 | NodeVisitor, 27 | AST, 28 | FunctionDef, 29 | AsyncFunctionDef, 30 | Return, 31 | Subscript, 32 | Starred, 33 | Delete, 34 | Break, 35 | Continue, 36 | If, 37 | For, 38 | AsyncFor, 39 | While, 40 | Try, 41 | Assert, 42 | With, 43 | AsyncWith, 44 | Raise, 45 | IfExp, 46 | ) 47 | 48 | from .graphgenutils import EdgeType 49 | 50 | 51 | class DataflowPass(NodeVisitor): 52 | def __init__(self, ast_graph_generator): 53 | self.__graph_generator = ast_graph_generator 54 | 55 | # Last Use 56 | self.__last_use: Dict[Any, Set[Any]] = defaultdict(set) 57 | 58 | self.__break_uses: Dict[Any, Set[Any]] = defaultdict(set) 59 | self.__continue_uses: Dict[Any, Set[Any]] = defaultdict(set) 60 | self.__return_uses: Dict[Any, Set[Any]] = defaultdict(set) 61 | 62 | def __visit_variable_like(self, name: Union[str, AST], parent_node: Optional[AST]): 63 | if isinstance(name, Name): 64 | self.visit(name) 65 | return 66 | 67 | if isinstance(name, AST): 68 | node = name 69 | else: 70 | # We need to find the relevant node in the graph 71 | assert parent_node is not None 72 | candidate_children = self.__graph_generator._get_edge_targets( 73 | parent_node, EdgeType.CHILD 74 | ) 75 | for child in candidate_children: 76 | if str(child) == name: 77 | node = child 78 | break 79 | else: 80 | assert False 81 | 82 | # Find relevant symbol (OCCURRENCE_OF) 83 | candidate_symbols = self.__graph_generator._get_edge_targets(node, EdgeType.OCCURRENCE_OF) 84 | if len(candidate_symbols) == 0: 85 | return 86 | assert len(candidate_symbols) == 1 87 | symbol = next(iter(candidate_symbols)) 88 | self.__record_next_use(symbol, node) 89 | 90 | def visit_Name(self, node: Name): 91 | self.__visit_variable_like(node.id, node) 92 | 93 | def __visit_statement_block(self, stmts: List): 94 | for statement in stmts: 95 | self.visit(statement) 96 | 97 | def visit_FunctionDef(self, node: FunctionDef) -> None: 98 | self.__visit_function(node, is_async=False) 99 | 100 | def visit_AsyncFunctionDef(self, node: AsyncFunctionDef) -> None: 101 | self.__visit_function(node, is_async=True) 102 | 103 | def __visit_function(self, node: Union[FunctionDef, AsyncFunctionDef], is_async: bool): 104 | outer_return_uses = self.__return_uses 105 | self.__return_uses = defaultdict(set) 106 | 107 | before_function_uses = self.__clone_last_uses() 108 | self.visit(node.args) 109 | self.__visit_statement_block(node.body) 110 | 111 | # Merge used variables in a dummy return value *only* if they have been accessed within the function. 112 | self.__last_use = self.__merge_uses(self.__last_use, self.__return_uses) 113 | for symbol, last_uses in self.__last_use.items(): 114 | if before_function_uses[symbol] == last_uses: 115 | continue 116 | 117 | self.__return_uses = outer_return_uses 118 | 119 | # endregion 120 | 121 | # region ControlFlow 122 | 123 | def __record_next_use(self, symbol, node) -> None: 124 | for last_node_used in self.__last_use[symbol]: 125 | self.__graph_generator._add_edge(last_node_used, node, EdgeType.NEXT_USE) 126 | self.__last_use[symbol] = {node} 127 | 128 | def __merge_uses( 129 | self, use_set1: Dict[Any, Set[Any]], use_set2: Dict[Any, Set[Any]] 130 | ) -> Dict[Any, Set[Any]]: 131 | merged = defaultdict(set) 132 | for symbol, last_used_nodes in chain(use_set1.items(), use_set2.items()): 133 | merged[symbol] |= last_used_nodes 134 | return merged 135 | 136 | def __clone_last_uses(self) -> Dict[Any, Set[Any]]: 137 | cloned = defaultdict(set) 138 | for symbol, last_used_nodes in self.__last_use.items(): 139 | cloned[symbol] = set(last_used_nodes) 140 | return cloned 141 | 142 | def __loop_back_after( 143 | self, 144 | last_uses_at_end_of_loop: Dict[Any, Set[Any]], 145 | last_uses_just_before_looping_point: Dict[Any, Set[Any]], 146 | ) -> None: 147 | for (symbol, last_use_before_looping_point,) in last_uses_just_before_looping_point.items(): 148 | first_use_after_looping_point = set( 149 | chain( 150 | *( 151 | self.__graph_generator._get_edge_targets(node, EdgeType.NEXT_USE) 152 | for node in last_use_before_looping_point 153 | ) 154 | ) 155 | ) 156 | 157 | for from_node in last_uses_at_end_of_loop[symbol]: 158 | for to_node in first_use_after_looping_point: 159 | self.__graph_generator._add_edge(from_node, to_node, EdgeType.NEXT_USE) 160 | 161 | def visit_Break(self, node: Break): 162 | self.__break_uses = self.__merge_uses(self.__break_uses, self.__last_use) 163 | self.__last_use = defaultdict(set) 164 | 165 | def visit_Continue(self, node: Continue): 166 | self.__continue_uses = self.__merge_uses(self.__last_use, self.__continue_uses) 167 | self.__last_use = defaultdict(set) 168 | 169 | def visit_For(self, node: For): 170 | self.__visit_for(node, False) 171 | 172 | def visit_AsyncFor(self, node: AsyncFor): 173 | self.__visit_for(node, True) 174 | 175 | def __visit_for(self, node, is_async: bool): 176 | self.visit(node.iter) 177 | last_use_before_loop = self.__clone_last_uses() 178 | 179 | outer_break, outer_continue = self.__break_uses, self.__continue_uses 180 | 181 | self.visit(node.target) 182 | self.__visit_statement_block(node.body) 183 | self.__loop_back_after( 184 | self.__merge_uses(self.__last_use, self.__continue_uses), last_use_before_loop, 185 | ) 186 | self.__last_use = self.__merge_uses( 187 | self.__last_use, self.__merge_uses(last_use_before_loop, self.__continue_uses), 188 | ) 189 | 190 | if node.orelse is not None: 191 | last_use_after_loop = self.__clone_last_uses() 192 | self.__visit_statement_block(node.orelse) 193 | self.__last_use = self.__merge_uses(self.__last_use, last_use_after_loop) 194 | 195 | self.__last_use = self.__merge_uses( 196 | self.__last_use, self.__break_uses 197 | ) # Break doesn't go through the else 198 | 199 | self.__break_uses, self.__continue_uses = outer_break, outer_continue 200 | 201 | def visit_If(self, node: If): 202 | self.visit(node.test) 203 | 204 | last_uses_before_body = self.__clone_last_uses() 205 | self.__visit_statement_block(node.body) 206 | 207 | if node.orelse is None: 208 | self.__last_use = self.__merge_uses(self.__last_use, last_uses_before_body) 209 | return 210 | 211 | last_uses_after_then_body = self.__last_use 212 | self.__last_use = last_uses_before_body 213 | self.__visit_statement_block(node.orelse) 214 | self.__last_use = self.__merge_uses(self.__last_use, last_uses_after_then_body) 215 | 216 | def visit_IfExp(self, node: IfExp): 217 | self.visit(node.test) 218 | 219 | last_uses_before_body = self.__clone_last_uses() 220 | self.visit(node.body) 221 | last_uses_after_body = self.__last_use 222 | 223 | self.__last_use = last_uses_before_body 224 | self.visit(node.orelse) 225 | self.__last_use = self.__merge_uses(self.__last_use, last_uses_after_body) 226 | 227 | def visit_Raise(self, node: Raise): 228 | if node.exc is not None: 229 | self.visit(node.exc) 230 | if node.cause is not None: 231 | self.visit(node.cause) 232 | self.__visit_return_like() 233 | 234 | def visit_Return(self, node: Return): 235 | if node.value is not None: 236 | self.visit(node.value) 237 | self.__visit_return_like() 238 | 239 | def __visit_return_like(self): 240 | self.__return_uses = self.__merge_uses(self.__return_uses, self.__last_use) 241 | self.__last_use = defaultdict(set) 242 | 243 | def visit_Try(self, node: Try): 244 | # Heuristic: each handler is an if-like statement 245 | self.__visit_statement_block(node.body) 246 | 247 | before_exec_handlers = self.__clone_last_uses() 248 | after_exec_handlers = self.__clone_last_uses() 249 | for i, exc_handler in enumerate(node.handlers): 250 | self.visit(exc_handler) 251 | after_exec_handlers = self.__merge_uses(after_exec_handlers, self.__last_use) 252 | self.__last_use = before_exec_handlers 253 | before_exec_handlers = self.__clone_last_uses() 254 | 255 | if node.orelse: 256 | before_uses = self.__clone_last_uses() 257 | self.__visit_statement_block(node.orelse) 258 | self.__last_use = self.__merge_uses(before_uses, self.__last_use) 259 | if node.finalbody: 260 | self.__visit_statement_block(node.finalbody) 261 | 262 | def visit_ExceptHandler(self, node): 263 | if node.type: 264 | self.visit(node.type) 265 | if node.name: 266 | self.__visit_variable_like(node.name, node) 267 | self.__visit_statement_block(node.body) 268 | 269 | def visit_While(self, node: While): 270 | last_use_before_loop = self.__clone_last_uses() 271 | self.visit(node.test) 272 | last_use_after_loop_test = self.__clone_last_uses() 273 | 274 | outer_break, outer_continue = self.__break_uses, self.__continue_uses 275 | 276 | self.__visit_statement_block(node.body) 277 | self.__loop_back_after( 278 | self.__merge_uses(self.__last_use, self.__continue_uses), last_use_before_loop, 279 | ) 280 | 281 | self.__last_use = self.__merge_uses( 282 | self.__last_use, self.__merge_uses(last_use_after_loop_test, self.__continue_uses), 283 | ) 284 | if node.orelse is not None: 285 | last_use_before_branch = self.__clone_last_uses() 286 | self.__visit_statement_block(node.orelse) 287 | self.__last_use = self.__merge_uses(last_use_before_branch, self.__last_use) 288 | 289 | self.__last_use = self.__merge_uses(self.__break_uses, self.__last_use) 290 | 291 | self.__break_uses, self.__continue_uses = outer_break, outer_continue 292 | 293 | def visit_With(self, node: With): 294 | self.__visit_with(node) 295 | 296 | def visit_AsyncWith(self, node: AsyncWith): 297 | self.__visit_with(node) 298 | 299 | def __visit_with(self, node: Union[With, AsyncWith]): 300 | for i, w_item in enumerate(node.items): 301 | self.visit(w_item) 302 | self.__visit_statement_block(node.body) 303 | 304 | def visit_withitem(self, node: withitem): 305 | self.visit(node.context_expr) 306 | if node.optional_vars is not None: 307 | self.visit(node.optional_vars) 308 | 309 | # endregion 310 | 311 | def visit_ClassDef(self, node): 312 | for decorator in node.decorator_list: 313 | self.visit(decorator) 314 | 315 | last_uses_before = self.__clone_last_uses() 316 | last_uses_after = self.__clone_last_uses() 317 | for statement in node.body: 318 | self.__last_use = last_uses_before 319 | last_uses_before = self.__clone_last_uses() 320 | self.visit(statement) 321 | last_uses_after = self.__merge_uses(self.__last_use, last_uses_after) 322 | self.__last_use = last_uses_after 323 | 324 | def visit_Assign(self, node: Assign): 325 | self.visit(node.value) 326 | 327 | for target in node.targets: 328 | if isinstance(target, Attribute) or isinstance(target, Name): 329 | self.__visit_variable_like(target, node) 330 | else: 331 | self.visit(target) 332 | 333 | def visit_AugAssign(self, node: AugAssign): 334 | self.visit(node.value) 335 | if isinstance(node.target, Name) or isinstance(node.target, Attribute): 336 | self.__visit_variable_like(node.target, node) 337 | else: 338 | self.visit(node.target) 339 | 340 | def visit_AnnAssign(self, node: AnnAssign): 341 | if node.value is not None: 342 | self.visit(node.value) 343 | self.__visit_variable_like(node.target, node) 344 | 345 | def visit_Call(self, node): 346 | self.visit(node.func) 347 | for arg in node.args: 348 | self.visit(arg) 349 | 350 | for arg in node.keywords: 351 | self.visit(arg) 352 | 353 | def visit_Lambda(self, node: Lambda): 354 | self.visit(node.args) 355 | self.visit(node.body) 356 | 357 | def visit_arg(self, node: arg): 358 | self.__visit_variable_like(node.arg, node) 359 | 360 | def visit_arguments(self, node: arguments): 361 | defaults = [None] * (len(node.args) - len(node.defaults)) + node.defaults 362 | 363 | for argument, default in zip(node.args, defaults): 364 | self.visit(argument) 365 | if default is not None: 366 | self.visit(default) 367 | 368 | if node.vararg is not None: 369 | self.visit(node.vararg) 370 | 371 | if node.kwarg is not None: 372 | self.visit(node.kwarg) 373 | 374 | if len(node.kwonlyargs) > 0: 375 | defaults = [None] * (len(node.kwonlyargs) - len(node.kw_defaults)) + node.kw_defaults 376 | for argument, default in zip(node.kwonlyargs, defaults): 377 | self.visit(argument) 378 | if default is not None: 379 | self.visit(default) 380 | 381 | def visit_keyword(self, node): 382 | self.visit(node.value) 383 | 384 | # region Comprehensions 385 | def visit_comprehension(self, node: comprehension): 386 | self.visit(node.iter) 387 | 388 | before_comp = self.__clone_last_uses() 389 | for if_ in node.ifs: 390 | self.visit(if_) 391 | 392 | self.visit(node.target) 393 | self.__last_use = self.__merge_uses(before_comp, self.__last_use) 394 | 395 | def visit_ListComp(self, node): 396 | self.visit(node.elt) 397 | for generator in node.generators: 398 | self.visit(generator) 399 | 400 | def visit_GeneratorExp(self, node): 401 | self.visit(node.elt) 402 | for generator in node.generators: 403 | self.visit(generator) 404 | 405 | def visit_SetComp(self, node): 406 | self.visit(node.elt) 407 | for generator in node.generators: 408 | self.visit(generator) 409 | 410 | def visit_DictComp(self, node): 411 | self.visit(node.key) 412 | self.visit(node.value) 413 | 414 | for generator in node.generators: 415 | self.visit(generator) 416 | 417 | # endregion 418 | 419 | def visit_Attribute(self, node: Attribute): 420 | self.__visit_variable_like(node, None) 421 | self.visit(node.value) 422 | 423 | def visit_Assert(self, node: Assert): 424 | self.visit(node.test) 425 | if node.msg is not None: 426 | self.visit(node.msg) 427 | 428 | def visit_BinOp(self, node): 429 | self.visit(node.left) 430 | self.visit(node.right) 431 | 432 | def visit_BoolOp(self, node): 433 | for idx, value in enumerate(node.values): 434 | self.visit(value) 435 | 436 | def visit_Compare(self, node: Compare): 437 | self.visit(node.left) 438 | for i, (op, right) in enumerate(zip(node.ops, node.comparators)): 439 | self.visit(right) 440 | 441 | def visit_Delete(self, node: Delete): 442 | for i, target in enumerate(node.targets): 443 | self.visit(target) 444 | 445 | def visit_Global(self, node: Global): 446 | for name in node.names: 447 | self.__visit_variable_like(name, node) 448 | 449 | def visit_Nonlocal(self, node: Nonlocal): 450 | for name in node.names: 451 | self.__visit_variable_like(name, node) 452 | 453 | def visit_Slice(self, node): 454 | if node.lower is not None: 455 | self.visit(node.lower) 456 | if node.upper is not None: 457 | self.visit(node.upper) 458 | if node.step is not None: 459 | self.visit(node.step) 460 | 461 | def visit_Subscript(self, node: Subscript): 462 | self.visit(node.value) 463 | self.visit(node.slice) 464 | 465 | def visit_Starred(self, node: Starred): 466 | self.visit(node.value) 467 | 468 | def visit_UnaryOp(self, node): 469 | self.visit(node.operand) 470 | 471 | # endregion 472 | 473 | # region Data Structure Constructors 474 | def visit_Dict(self, node): 475 | for idx, (key, value) in enumerate(zip(node.keys, node.values)): 476 | if key is not None: 477 | self.visit(key) 478 | self.visit(value) 479 | 480 | def visit_FormattedValue(self, node: FormattedValue): 481 | self.visit(node.value) 482 | if node.format_spec is not None: 483 | self.visit(node.format_spec) 484 | 485 | def visit_List(self, node): 486 | self.__sequence_datastruct_visit(node) 487 | 488 | def visit_Set(self, node): 489 | self.__sequence_datastruct_visit(node) 490 | 491 | def visit_Tuple(self, node): 492 | self.__sequence_datastruct_visit(node) 493 | 494 | def __sequence_datastruct_visit(self, node): 495 | for idx, element in enumerate(node.elts): 496 | self.visit(element) 497 | 498 | # endregion 499 | -------------------------------------------------------------------------------- /src/graph_generator/extract_graphs.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple, List, Optional, Set, Iterator 2 | from dpu_utils.utils import save_jsonl_gz, run_and_debug, ChunkWriter 3 | import traceback 4 | import os 5 | from glob import iglob 6 | 7 | from docopt import docopt 8 | import time 9 | 10 | from .graphgenerator import AstGraphGenerator 11 | from .type_lattice_generator import TypeLatticeGenerator 12 | from .typeparsing import FaultyAnnotation 13 | 14 | 15 | class Monitoring: 16 | def __init__(self): 17 | self.count = 0 # type: int 18 | self.errors = [] 19 | self.file = "" # type: str 20 | self.current_repo = "" 21 | self.empty_files = [] 22 | 23 | def increment_count(self) -> None: 24 | self.count += 1 25 | 26 | def found_error(self, err, trace) -> None: 27 | self.errors.append([self.file, err, trace]) 28 | 29 | def enter_file(self, filename: str) -> None: 30 | self.file = filename 31 | 32 | def enter_repo(self, repo_name: str) -> None: 33 | self.current_repo = repo_name 34 | 35 | 36 | def build_graph( 37 | source_code, monitoring: Monitoring, type_lattice: TypeLatticeGenerator 38 | ) -> Tuple[Optional[List], Optional[List]]: 39 | """ 40 | Parses the code of a file into a custom abstract syntax tree. 41 | """ 42 | try: 43 | visitor = AstGraphGenerator(source_code, type_lattice) 44 | return visitor.build() 45 | except FaultyAnnotation as e: 46 | print("Faulty Annotation: ", e) 47 | print("at file: ", monitoring.file) 48 | except SyntaxError as e: 49 | monitoring.found_error(e, traceback.format_exc()) 50 | except Exception as e: 51 | print(traceback.format_exc()) 52 | monitoring.found_error(e, traceback.format_exc()) 53 | 54 | 55 | def explore_files( 56 | root_dir: str, 57 | files_to_extract: Set[str], 58 | monitoring: Monitoring, 59 | type_lattice: TypeLatticeGenerator, 60 | ) -> Iterator[Tuple]: 61 | """ 62 | Walks through the root_dir and process each file. 63 | """ 64 | for file_path in iglob(os.path.join(root_dir, "**", "*.py"), recursive=True): 65 | if not os.path.isfile(file_path): 66 | continue 67 | with open(file_path, encoding="utf-8", errors="ignore") as f: 68 | monitoring.increment_count() 69 | monitoring.enter_file(file_path) 70 | 71 | # import pdb; pdb.set_trace() 72 | if file_path[len(root_dir) :] not in files_to_extract: 73 | continue 74 | 75 | graph = build_graph(f.read(), monitoring, type_lattice) 76 | if graph is None or len(graph["supernodes"]) == 0: 77 | continue 78 | graph["filename"] = file_path[len(root_dir) :] 79 | yield graph 80 | type_lattice.build_graph() 81 | 82 | 83 | def extract_graphs(root_dir, typing_rules_path, files_to_extract: Set[str], target_folder): 84 | start_time = time.time() 85 | print("Traversing folders ...") 86 | monitoring = Monitoring() 87 | type_lattice = TypeLatticeGenerator(typing_rules_path) 88 | 89 | # Extract graphs 90 | outputs = explore_files(root_dir, files_to_extract, monitoring, type_lattice) 91 | 92 | # Save results 93 | with ChunkWriter( 94 | out_folder=target_folder, 95 | file_prefix="all-graphs", 96 | max_chunk_size=5000, 97 | file_suffix=".jsonl.gz", 98 | ) as writer: 99 | for graph in outputs: 100 | writer.add(graph) 101 | 102 | print("Building and saving the type graph...") 103 | type_lattice.build_graph() 104 | save_jsonl_gz( 105 | [type_lattice.return_json()], os.path.join(target_folder, "_type_lattice.json.gz"), 106 | ) 107 | 108 | print("Done.") 109 | print( 110 | "Generated %d graphs out of %d snippets" 111 | % (monitoring.count - len(monitoring.errors), monitoring.count) 112 | ) 113 | 114 | with open(os.path.join(target_folder, "logs_graph_generator.txt"), "w") as f: 115 | for item in monitoring.errors: 116 | try: 117 | f.write("%s\n" % item) 118 | except: 119 | pass 120 | 121 | print("\nGraph Execution in: ", time.time() - start_time, " seconds") 122 | 123 | 124 | if __name__ == "__main__": 125 | args = docopt(__doc__) 126 | run_and_debug(lambda: main(args), args["--debug"]) 127 | -------------------------------------------------------------------------------- /src/graph_generator/graphgenerator.py: -------------------------------------------------------------------------------- 1 | import keyword 2 | import logging 3 | import re 4 | from collections import defaultdict 5 | from symtable import symtable, Symbol 6 | from typing import Any, Dict, Optional, Set, Union, List, FrozenSet 7 | 8 | from dpu_utils.codeutils import split_identifier_into_parts 9 | from dpu_utils.utils import run_and_debug 10 | from typed_ast.ast3 import ( 11 | Add, 12 | Sub, 13 | Mult, 14 | Div, 15 | FloorDiv, 16 | Mod, 17 | LShift, 18 | RShift, 19 | BitOr, 20 | BitAnd, 21 | BitXor, 22 | Pow, 23 | MatMult, 24 | ExtSlice, 25 | Index, 26 | Compare, 27 | Await, 28 | Lambda, 29 | arg, 30 | Global, 31 | Nonlocal, 32 | arguments, 33 | Name, 34 | comprehension, 35 | alias, 36 | withitem, 37 | JoinedStr, 38 | Assign, 39 | AnnAssign, 40 | AugAssign, 41 | FormattedValue, 42 | TypeIgnore, 43 | Attribute, 44 | Module, 45 | ImportFrom, 46 | ) 47 | from typed_ast.ast3 import And, Or 48 | from typed_ast.ast3 import Eq, Gt, GtE, In, Is, IsNot, Lt, LtE, NotEq, NotIn 49 | from typed_ast.ast3 import Invert, Not, UAdd, USub 50 | from typed_ast.ast3 import ( 51 | NodeVisitor, 52 | parse, 53 | AST, 54 | FunctionDef, 55 | AsyncFunctionDef, 56 | Return, 57 | Yield, 58 | Subscript, 59 | Str, 60 | YieldFrom, 61 | Starred, 62 | Delete, 63 | Break, 64 | Continue, 65 | If, 66 | For, 67 | AsyncFor, 68 | While, 69 | Try, 70 | Assert, 71 | With, 72 | AsyncWith, 73 | Raise, 74 | IfExp, 75 | Call, 76 | ) 77 | 78 | from .dataflowpass import DataflowPass 79 | from .graphgenutils import EdgeType, TokenNode, StrSymbol, SymbolInformation 80 | from .type_lattice_generator import TypeLatticeGenerator 81 | from .typeparsing import ( 82 | parse_type_annotation_node, 83 | parse_type_comment, 84 | TypeAnnotationNode, 85 | ) 86 | 87 | 88 | class AstGraphGenerator(NodeVisitor): 89 | def __init__(self, source: str, type_graph: TypeLatticeGenerator): 90 | self.__type_graph = type_graph 91 | self.__node_to_id: Dict[Any, int] = {} 92 | self.__id_to_node: List[Any] = [] 93 | 94 | self.__symbol_to_supernode_id: Dict[Symbol, int] = {} 95 | 96 | self.__edges: Dict[EdgeType, Dict[int, Set[int]]] = {e: defaultdict(set) for e in EdgeType} 97 | 98 | self.__ast = parse(source) 99 | self.__scope_symtable = [symtable(source, "file.py", "exec")] 100 | 101 | self.__imported_symbols = {} # type: Dict[TypeAnnotationNode, TypeAnnotationNode] 102 | 103 | # For the CHILD edges 104 | self.__current_parent_node: Optional[AST] = None 105 | 106 | # For the NEXT_TOKEN edges 107 | self.__backbone_sequence: List[TokenNode] = [] 108 | self.__prev_token_node: Optional[TokenNode] = None 109 | 110 | # For the RETURNS_TO edge 111 | self.__return_scope: Optional[AST] = None 112 | 113 | # For the OCCURRENCE_OF and Supernodes 114 | self.__variable_like_symbols: Dict[Any, SymbolInformation] = {} 115 | 116 | # Last Lexical Use 117 | self.__last_lexical_use: Dict[Any, Any] = {} 118 | 119 | # region Constants 120 | INDENT = "" 121 | DEDENT = "" 122 | NLINE = "" 123 | 124 | BOOLOP_SYMBOLS = {And: "and", Or: "or"} 125 | 126 | BINOP_SYMBOLS = { 127 | Add: "+", 128 | Sub: "-", 129 | Mult: "*", 130 | Div: "/", 131 | FloorDiv: "//", 132 | Mod: "%", 133 | LShift: "<<", 134 | RShift: ">>", 135 | BitOr: "|", 136 | BitAnd: "&", 137 | BitXor: "^", 138 | Pow: "**", 139 | MatMult: "*.", 140 | } 141 | 142 | CMPOP_SYMBOLS = { 143 | Eq: "==", 144 | Gt: ">", 145 | GtE: ">=", 146 | In: "in", 147 | Is: "is", 148 | IsNot: "is not", 149 | Lt: "<", 150 | LtE: "<=", 151 | NotEq: "!=", 152 | NotIn: "not in", 153 | } 154 | 155 | UNARYOP_SYMBOLS = {Invert: "~", Not: "not", UAdd: "+", USub: "-"} 156 | 157 | IDENTIFER_REGEX = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*") 158 | 159 | BUILT_IN_METHODS_TO_KEEP = frozenset({"__getitem__", "__setitem__", "__enter__", "__call__"}) 160 | 161 | # endregion 162 | 163 | def build(self): 164 | self.visit(self.__ast) 165 | self.__add_subtoken_of_edges() 166 | 167 | dataflow = DataflowPass(self) 168 | dataflow.visit(self.__ast) 169 | 170 | def parse_symbol_info(sinfo: SymbolInformation) -> Dict[str, Any]: 171 | has_annotation = any(s is not None for s in sinfo.annotatable_locations.values()) 172 | 173 | if has_annotation: 174 | first_annotatable_location = min( 175 | k for k, v in sinfo.annotatable_locations.items() if v is not None 176 | ) 177 | annotation_str = str(sinfo.annotatable_locations[first_annotatable_location]) 178 | else: 179 | first_annotatable_location = min(k for k, v in sinfo.annotatable_locations.items()) 180 | annotation_str = None 181 | 182 | return { 183 | "name": sinfo.name, 184 | "annotation": None if not has_annotation else annotation_str, 185 | "location": first_annotatable_location, 186 | "type": sinfo.symbol_type, 187 | } 188 | 189 | def is_annotation_worthy(sinfo: SymbolInformation) -> bool: 190 | if sinfo.name == "self": 191 | return False # "self" by convention is not annotated 192 | elif ( 193 | sinfo.name.startswith("__") 194 | and sinfo.name.endswith("__") 195 | and sinfo.name not in self.BUILT_IN_METHODS_TO_KEEP 196 | ): 197 | return False # Build in methods have fixed conventions 198 | elif any(v == "None" for k, v in sinfo.annotatable_locations.items()): 199 | return False # 'None' is deterministically computable 200 | return True 201 | 202 | return { 203 | "nodes": [self.node_to_label(n) for n in self.__id_to_node], 204 | "edges": { 205 | e.name: {f: list(t) for f, t in v.items() if len(t) > 0} 206 | for e, v in self.__edges.items() 207 | if len(v) > 0 208 | }, 209 | "token-sequence": [self.__node_to_id[t] for t in self.__backbone_sequence], 210 | "supernodes": { 211 | self.__node_to_id[node]: parse_symbol_info(symbol_info) 212 | for node, symbol_info in self.__variable_like_symbols.items() 213 | if len(symbol_info.annotatable_locations) > 0 and is_annotation_worthy(symbol_info) 214 | }, 215 | } 216 | 217 | def __add_subtoken_of_edges(self): 218 | def is_identifier_node(n): 219 | if not isinstance(n, str) and not isinstance(n, TokenNode): 220 | return False 221 | if not self.IDENTIFER_REGEX.fullmatch(str(n)): 222 | return False 223 | if keyword.iskeyword(str(n)): 224 | return False 225 | if n == self.INDENT or n == self.DEDENT or n == self.NLINE: 226 | return False 227 | return True 228 | 229 | all_identifier_like_nodes: Set[TokenNode] = { 230 | n for n in self.__node_to_id if is_identifier_node(n) 231 | } 232 | subtoken_nodes: Dict[str, TokenNode] = {} 233 | 234 | for node in all_identifier_like_nodes: 235 | for subtoken in split_identifier_into_parts(str(node)): 236 | if subtoken == "_": 237 | continue 238 | subtoken_dummy_node = subtoken_nodes.get(subtoken) 239 | if subtoken_dummy_node is None: 240 | subtoken_dummy_node = TokenNode(subtoken) 241 | subtoken_nodes[subtoken] = subtoken_dummy_node 242 | self._add_edge(subtoken_dummy_node, node, EdgeType.SUBTOKEN_OF) 243 | 244 | def __node_id(self, node: Union[AST, TokenNode]) -> int: 245 | assert not isinstance(node, int), "Node should be an object not its int id" 246 | idx = self.__node_to_id.get(node) 247 | if idx is None: 248 | idx = len(self.__node_to_id) 249 | assert len(self.__id_to_node) == len(self.__node_to_id) 250 | self.__node_to_id[node] = idx 251 | self.__id_to_node.append(node) 252 | return idx 253 | 254 | def _get_node(self, node_id: int): 255 | return self.__id_to_node[node_id] 256 | 257 | def _add_edge( 258 | self, from_node: Union[AST, TokenNode], to_node: Union[AST, TokenNode], edge_type: EdgeType, 259 | ) -> None: 260 | from_node_idx = self.__node_id(from_node) 261 | to_node_idx = self.__node_id(to_node) 262 | self.__edges[edge_type][from_node_idx].add(to_node_idx) 263 | 264 | def _get_edge_targets(self, from_node, edge_type: EdgeType) -> FrozenSet: 265 | from_node_idx = self.__node_id(from_node) 266 | return frozenset(self._get_node(n) for n in self.__edges[edge_type][from_node_idx]) 267 | 268 | def visit(self, node: AST): 269 | """Visit a node adding the Child edge.""" 270 | if self.__current_parent_node is not None: 271 | assert self.__current_parent_node in self.__node_to_id or isinstance( 272 | self.__current_parent_node, Module 273 | ), self.__current_parent_node 274 | self._add_edge(self.__current_parent_node, node, EdgeType.CHILD) 275 | parent = self.__current_parent_node 276 | self.__current_parent_node = node 277 | try: 278 | method = "visit_" + node.__class__.__name__ 279 | visitor = getattr(self, method, self.generic_visit) 280 | if visitor == self.generic_visit: 281 | logging.warning("Unvisited AST type: %s", node.__class__.__name__) 282 | return visitor(node) 283 | finally: 284 | self.__current_parent_node = parent 285 | 286 | def add_terminal(self, token_node: TokenNode): 287 | self._add_edge(self.__current_parent_node, token_node, EdgeType.CHILD) 288 | if self.__prev_token_node is not None: 289 | self._add_edge(self.__prev_token_node, token_node, EdgeType.NEXT) 290 | self.__backbone_sequence.append(token_node) 291 | self.__prev_token_node = token_node 292 | 293 | def __visit_statement_block(self, stmts: List): 294 | self.add_terminal(TokenNode(self.INDENT)) # Skip ":" since it is implied 295 | for i, statement in enumerate(stmts): 296 | self.visit(statement) 297 | if i < len(stmts) - 1: 298 | self.add_terminal(TokenNode(self.NLINE)) 299 | if i > 0: 300 | self._add_edge(stmts[i - 1], statement, edge_type=EdgeType.NEXT) 301 | 302 | self.add_terminal(TokenNode(self.DEDENT)) 303 | 304 | def visit_Name_annotatable( 305 | self, 306 | node: Name, 307 | lineno: int, 308 | col_offset: int, 309 | can_annotate_here: Optional[bool], 310 | type_annotation: TypeAnnotationNode = None, 311 | ): 312 | self._add_edge(self.__current_parent_node, node, EdgeType.CHILD) 313 | parent = self.__current_parent_node 314 | self.__current_parent_node = node 315 | try: 316 | return self.__visit_variable_like( 317 | node.id, 318 | node.lineno, 319 | node.col_offset, 320 | can_annotate_here=can_annotate_here, 321 | type_annotation=type_annotation, 322 | ) 323 | finally: 324 | self.__current_parent_node = parent 325 | 326 | def __visit_variable_like( 327 | self, 328 | name: Union[str, AST], 329 | lineno: int, 330 | col_offset: int, 331 | can_annotate_here: Optional[bool], 332 | type_annotation: TypeAnnotationNode = None, 333 | ): 334 | if isinstance(name, Name): 335 | # Transfer any annotation to the name directly. 336 | self.visit_Name_annotatable( 337 | name, lineno, col_offset, can_annotate_here, type_annotation 338 | ) 339 | return 340 | name, node, symbol, symbol_type = self.__get_symbol_for_name(name, lineno, col_offset) 341 | 342 | if type_annotation is not None: 343 | type_annotation = self.__type_graph.canonicalize_annotation( 344 | type_annotation, self.__imported_symbols 345 | ) 346 | 347 | if symbol is not None: 348 | self._add_edge(node, symbol, edge_type=EdgeType.OCCURRENCE_OF) 349 | symbol_info = self.__variable_like_symbols.get(symbol) 350 | if symbol_info is None: 351 | symbol_info = SymbolInformation.create(name, symbol_type) 352 | self.__variable_like_symbols[symbol] = symbol_info 353 | symbol_info.locations.append((lineno, col_offset)) 354 | if can_annotate_here: 355 | symbol_info.annotatable_locations[(lineno, col_offset)] = type_annotation 356 | 357 | # Last lexical use 358 | last_lexical_use_node = self.__last_lexical_use.get(symbol) 359 | if last_lexical_use_node is not None: 360 | self._add_edge(last_lexical_use_node, node, EdgeType.LAST_LEXICAL_USE) 361 | self.__last_lexical_use[symbol] = node 362 | 363 | if type_annotation is not None: 364 | self.__type_graph.add_type(type_annotation, self.__imported_symbols) 365 | 366 | def __get_symbol_for_name(self, name, lineno, col_offset): 367 | if isinstance(name, str): 368 | node = TokenNode(name, lineno, col_offset) 369 | self.add_terminal(node) 370 | 371 | if ( 372 | self.__scope_symtable[-1].get_type() == "class" 373 | and name.startswith("__") 374 | and not name.endswith("__") 375 | ): 376 | name = "_" + self.__scope_symtable[-1].get_name() + name 377 | 378 | current_idx = len(self.__scope_symtable) - 1 379 | while current_idx >= 0: 380 | try: 381 | symbol = self.__scope_symtable[current_idx].lookup(name) 382 | break 383 | except KeyError: 384 | current_idx -= 1 385 | else: 386 | logging.warning(f'Symbol "{name}"@{lineno}:{col_offset} Not Found!') 387 | symbol = None 388 | else: 389 | node = name 390 | assert isinstance(node, Attribute) 391 | # Heuristic: create symbols only for attributes of the form X.Y and X.Y.Z 392 | self.visit(node.value) 393 | self.add_terminal(TokenNode(".", node.lineno, node.col_offset)) 394 | self.add_terminal(TokenNode(node.attr, node.lineno, node.col_offset)) 395 | if isinstance(node.value, Name): 396 | name = f"{node.value.id}.{node.attr}" 397 | symbol = StrSymbol(name) 398 | elif isinstance(node.value, Attribute) and isinstance(node.value.value, Name): 399 | name = f"{node.value.value.id}.{node.value.attr}.{node.attr}" 400 | symbol = StrSymbol(name) 401 | else: 402 | symbol = None 403 | if isinstance(symbol, StrSymbol): 404 | symbol_type = "variable" 405 | elif isinstance(symbol, Symbol): 406 | if symbol.is_namespace(): 407 | symbol_type = "class-or-function" 408 | elif symbol.is_parameter(): 409 | symbol_type = "parameter" 410 | elif symbol.is_imported(): 411 | symbol_type = "imported" 412 | else: 413 | symbol_type = "variable" 414 | else: 415 | symbol_type = None 416 | return name, node, symbol, symbol_type 417 | 418 | def visit_Name(self, node: Name): 419 | self.__visit_variable_like(node.id, node.lineno, node.col_offset, can_annotate_here=None) 420 | 421 | def __enter_child_symbol_table( 422 | self, symtable_type: str, name: str, lineno: Optional[int] = None 423 | ): 424 | for child_symtable in self.__scope_symtable[-1].get_children(): 425 | if ( 426 | child_symtable.get_type() == symtable_type 427 | and child_symtable.get_name() == name 428 | and (lineno is None or child_symtable.get_lineno() == lineno) 429 | ): 430 | self.__scope_symtable.append(child_symtable) 431 | break 432 | else: 433 | raise ValueError( 434 | f"Symbol Table for {name} of type {symtable_type} at {lineno} not found" 435 | ) 436 | 437 | # region Function Parsing 438 | 439 | def visit_FunctionDef(self, node: FunctionDef) -> None: 440 | self.__visit_function(node, is_async=False) 441 | 442 | def visit_AsyncFunctionDef(self, node: AsyncFunctionDef) -> None: 443 | self.__visit_function(node, is_async=True) 444 | 445 | def __visit_function(self, node: Union[FunctionDef, AsyncFunctionDef], is_async: bool): 446 | for decorator in node.decorator_list: 447 | self.add_terminal(TokenNode("@")) 448 | self.visit(decorator) 449 | 450 | if is_async: 451 | self.add_terminal(TokenNode("async")) 452 | self.add_terminal(TokenNode("def")) 453 | 454 | t = None 455 | if node.returns is not None: 456 | t = parse_type_annotation_node(node.returns) 457 | elif node.type_comment is not None and "->" in node.type_comment: 458 | # TODO: Add support for argument types 459 | t = node.type_comment 460 | t = t.split("->")[-1].strip() 461 | t = parse_type_comment(t) 462 | 463 | symbol_name = node.name 464 | self.__visit_variable_like( 465 | symbol_name, node.lineno, node.col_offset, can_annotate_here=True, type_annotation=t, 466 | ) 467 | 468 | old_return_scope = self.__return_scope 469 | self.__enter_child_symbol_table("function", node.name, node.lineno) 470 | try: 471 | 472 | self.add_terminal(TokenNode("(")) 473 | self.visit(node.args) 474 | self.add_terminal(TokenNode(")")) 475 | 476 | self.__return_scope = node 477 | self.__visit_statement_block(node.body) 478 | finally: 479 | self.__return_scope = old_return_scope 480 | self.__scope_symtable.pop() 481 | 482 | def visit_Yield(self, node: Yield): 483 | self._add_edge(node, self.__return_scope, EdgeType.RETURNS_TO) 484 | self.add_terminal(TokenNode("yield")) 485 | if node.value is not None: 486 | self.visit(node.value) 487 | 488 | def visit_YieldFrom(self, node: YieldFrom): 489 | self._add_edge(node, self.__return_scope, EdgeType.RETURNS_TO) 490 | self.add_terminal(TokenNode("yield")) 491 | self.add_terminal(TokenNode("from")) 492 | self.visit(node.value) 493 | 494 | # endregion 495 | 496 | # region ControlFlow 497 | def visit_Break(self, node: Break): 498 | self.add_terminal(TokenNode("break")) 499 | 500 | def visit_Continue(self, node: Continue): 501 | self.add_terminal(TokenNode("continue")) 502 | 503 | def visit_For(self, node: For): 504 | self.__visit_for(node, False) 505 | 506 | def visit_AsyncFor(self, node: AsyncFor): 507 | self.__visit_for(node, True) 508 | 509 | def __visit_for(self, node, is_async: bool): 510 | if is_async: 511 | self.add_terminal(TokenNode("async")) 512 | self.add_terminal(TokenNode("for")) 513 | self.visit(node.target) 514 | self.add_terminal(TokenNode("in")) 515 | self.visit(node.iter) 516 | self._add_edge(node.target, node.iter, EdgeType.COMPUTED_FROM) 517 | 518 | self.__visit_statement_block(node.body) 519 | 520 | if node.orelse is not None: 521 | self.add_terminal(TokenNode("else")) 522 | self.__visit_statement_block(node.orelse) 523 | 524 | def visit_If(self, node: If): 525 | self.add_terminal(TokenNode("if")) 526 | self.visit(node.test) 527 | self.__visit_statement_block(node.body) 528 | 529 | if node.orelse is None: 530 | return 531 | 532 | self.add_terminal(TokenNode("else")) 533 | self.__visit_statement_block(node.orelse) 534 | 535 | def visit_IfExp(self, node: IfExp): 536 | self.visit(node.body) 537 | self.add_terminal(TokenNode("if")) 538 | self.visit(node.test) 539 | self.add_terminal(TokenNode("else")) 540 | self.visit(node.orelse) 541 | 542 | def visit_Raise(self, node: Raise): 543 | self._add_edge(node, self.__return_scope, EdgeType.RETURNS_TO) 544 | self.add_terminal(TokenNode("raise")) 545 | if node.exc is not None: 546 | self.visit(node.exc) 547 | if node.cause is not None: 548 | self.add_terminal(TokenNode("from")) 549 | self.visit(node.cause) 550 | 551 | def visit_Return(self, node: Return): 552 | self._add_edge(node, self.__return_scope, EdgeType.RETURNS_TO) 553 | self.add_terminal(TokenNode("return")) 554 | if node.value is not None: 555 | self.visit(node.value) 556 | 557 | def visit_Try(self, node: Try): 558 | self.add_terminal(TokenNode("try")) 559 | self.__visit_statement_block(node.body) 560 | for i, exc_handler in enumerate(node.handlers): 561 | self.visit(exc_handler) 562 | if i > 0: 563 | self._add_edge(node.handlers[i - 1], exc_handler, EdgeType.NEXT) 564 | 565 | if node.orelse: 566 | self.add_terminal(TokenNode("else")) 567 | self.__visit_statement_block(node.orelse) 568 | if node.finalbody: 569 | self.add_terminal(TokenNode("finally")) 570 | self.__visit_statement_block(node.finalbody) 571 | 572 | def visit_ExceptHandler(self, node): 573 | self.add_terminal(TokenNode("except")) 574 | if node.type: 575 | self.visit(node.type) 576 | if node.name: 577 | self.__visit_variable_like( 578 | node.name, node.lineno, node.col_offset, can_annotate_here=False 579 | ) 580 | self.__visit_statement_block(node.body) 581 | 582 | def visit_While(self, node: While): 583 | self.add_terminal(TokenNode("while")) 584 | self.visit(node.test) 585 | self.__visit_statement_block(node.body) 586 | 587 | if node.orelse is None: 588 | return 589 | self.add_terminal(TokenNode("else")) 590 | self.__visit_statement_block(node.orelse) 591 | 592 | def visit_With(self, node: With): 593 | self.__visit_with(node, False) 594 | 595 | def visit_AsyncWith(self, node: AsyncWith): 596 | self.__visit_with(node, True) 597 | 598 | def __visit_with(self, node: Union[With, AsyncWith], is_asyc: bool): 599 | # TODO: There is a type comment here! node.type_comment 600 | if is_asyc: 601 | self.add_terminal(TokenNode("async")) 602 | self.add_terminal(TokenNode("with")) 603 | for i, w_item in enumerate(node.items): 604 | self.visit(w_item) 605 | if i < len(node.items) - 1: 606 | self.add_terminal(TokenNode(",")) 607 | self.__visit_statement_block(node.body) 608 | 609 | def visit_withitem(self, node: withitem): 610 | self.visit(node.context_expr) 611 | if node.optional_vars is not None: 612 | self.add_terminal(TokenNode("as")) 613 | self.visit(node.optional_vars) 614 | 615 | # endregion 616 | 617 | # region ClassDef 618 | def visit_ClassDef(self, node): 619 | # Add class inheritance (if any) 620 | self.__type_graph.add_class( 621 | node.name, 622 | [ 623 | self.__type_graph.canonicalize_annotation( 624 | parse_type_annotation_node(parent), self.__imported_symbols 625 | ) 626 | for parent in node.bases 627 | ], 628 | ) 629 | if len(node.bases) == 0: 630 | self.__type_graph.add_class(node.name, [parse_type_annotation_node("object")]) 631 | 632 | for decorator in node.decorator_list: 633 | self.add_terminal(TokenNode("@")) 634 | self.visit(decorator) 635 | 636 | self.add_terminal(TokenNode("class")) 637 | self.add_terminal(TokenNode(node.name, node.lineno, node.col_offset)) 638 | if len(node.bases) > 0: 639 | self.add_terminal(TokenNode("(")) 640 | for i, base in enumerate(node.bases): 641 | self.visit(base) 642 | if i < len(node.bases) - 1: 643 | self.add_terminal(TokenNode(",")) 644 | 645 | self.add_terminal(TokenNode(")")) 646 | 647 | self.__enter_child_symbol_table("class", node.name, node.lineno) 648 | try: 649 | self.__visit_statement_block(node.body) 650 | finally: 651 | self.__scope_symtable.pop() 652 | 653 | # endregion 654 | 655 | def visit_Assign(self, node: Assign): 656 | if ( 657 | hasattr(node, "value") 658 | and hasattr(node.value, "func") 659 | and hasattr(node.value.func, "id") 660 | and node.value.func.id == "NewType" 661 | and hasattr(node, "value") 662 | and hasattr(node.value, "args") 663 | and len(node.value.args) == 2 664 | ): 665 | self.__type_graph.add_type_alias( 666 | parse_type_annotation_node(node.value.args[0]), 667 | parse_type_annotation_node(node.value.args[1]), 668 | ) 669 | 670 | # TODO: Type aliases are of the form Vector=List[float] how do we parse these? 671 | 672 | if node.type_comment is not None and len(node.targets) != 1: 673 | assert False 674 | 675 | for i, target in enumerate(node.targets): 676 | if isinstance(target, Attribute) or isinstance(target, Name): 677 | self.__visit_variable_like( 678 | target, 679 | target.lineno, 680 | target.col_offset, 681 | can_annotate_here=True, 682 | type_annotation=parse_type_comment(node.type_comment) 683 | if node.type_comment is not None 684 | else None, 685 | ) 686 | else: 687 | self.visit(target) 688 | if i > 0: 689 | self._add_edge(node.targets[i - 1], target, EdgeType.NEXT) 690 | 691 | self._add_edge(target, node.value, EdgeType.COMPUTED_FROM) 692 | if i < len(node.targets) - 1: 693 | self.add_terminal(TokenNode(",")) 694 | 695 | self.add_terminal(TokenNode("=")) 696 | self.visit(node.value) 697 | 698 | def visit_AugAssign(self, node: AugAssign): 699 | if isinstance(node.target, Name) or isinstance(node.target, Attribute): 700 | self.__visit_variable_like( 701 | node.target, node.lineno, node.col_offset, can_annotate_here=False 702 | ) 703 | else: 704 | self.visit(node.target) 705 | self._add_edge(node.target, node.value, EdgeType.COMPUTED_FROM) 706 | 707 | self.add_terminal(TokenNode(self.BINOP_SYMBOLS[type(node.op)] + "=")) 708 | self.visit(node.value) 709 | 710 | def visit_AnnAssign(self, node: AnnAssign): 711 | self.__visit_variable_like( 712 | node.target, 713 | node.target.lineno, 714 | node.target.col_offset, 715 | can_annotate_here=True, 716 | type_annotation=parse_type_annotation_node(node.annotation), 717 | ) 718 | 719 | if node.value is not None: 720 | self.add_terminal(TokenNode("=")) 721 | self.visit(node.value) 722 | 723 | self._add_edge(node.target, node.value, EdgeType.COMPUTED_FROM) 724 | 725 | # Resolve imports. Since 99.9% of imports are global, don't account for scoping for simplicity. 726 | def visit_Import(self, node): 727 | self.generic_visit(node) 728 | 729 | def visit_ImportFrom(self, node: ImportFrom): 730 | for alias in node.names: 731 | if node.module is not None: 732 | name = parse_type_annotation_node(node.module + "." + alias.name) 733 | else: 734 | name = parse_type_annotation_node(alias.name) 735 | if alias.asname: 736 | self.__imported_symbols[parse_type_annotation_node(alias.asname)] = name 737 | elif node.module is not None: 738 | self.__imported_symbols[parse_type_annotation_node(alias.name)] = name 739 | 740 | # region Ignored 741 | def visit_Module(self, node): 742 | self.generic_visit(node) 743 | 744 | def visit_Load(self, node): 745 | self.generic_visit(node) 746 | 747 | def visit_Store(self, node): 748 | self.generic_visit(node) 749 | 750 | def visit_TypeIgnore(self, node: TypeIgnore): 751 | pass 752 | 753 | # endregion 754 | 755 | def visit_Call(self, node: Call): 756 | self.visit(node.func) 757 | self.add_terminal(TokenNode("(")) 758 | num_args = len(node.args) + len(node.keywords) 759 | num_args_added = 0 760 | for arg in node.args: 761 | self.visit(arg) 762 | num_args_added += 1 763 | if num_args_added < num_args: 764 | self.add_terminal(TokenNode(",")) 765 | 766 | for arg in node.keywords: 767 | self.visit(arg) 768 | num_args_added += 1 769 | if num_args_added < num_args: 770 | self.add_terminal(TokenNode(",")) 771 | 772 | self.add_terminal(TokenNode(")")) 773 | 774 | def visit_Lambda(self, node: Lambda): 775 | self.add_terminal(TokenNode("lambda")) 776 | 777 | try: 778 | self.__enter_child_symbol_table("function", "lambda", node.lineno) 779 | self.visit(node.args) 780 | self.add_terminal(TokenNode(":")) 781 | self.visit(node.body) 782 | self.__scope_symtable.pop() 783 | except ValueError: 784 | pass # In the rare case of nexted lambdas symtable acts odd... 785 | 786 | def visit_arg(self, node: arg): 787 | type_annotation = None 788 | if node.annotation is not None: 789 | type_annotation = parse_type_annotation_node(node.annotation) 790 | elif node.type_comment is not None: 791 | type_annotation = parse_type_comment(node.type_comment) 792 | 793 | self.__visit_variable_like( 794 | node.arg, 795 | node.lineno, 796 | node.col_offset, 797 | can_annotate_here=True, 798 | type_annotation=type_annotation, 799 | ) 800 | 801 | def visit_arguments(self, node: arguments): 802 | defaults = [None] * (len(node.args) - len(node.defaults)) + node.defaults 803 | for i, (argument, default) in enumerate(zip(node.args, defaults)): 804 | self.visit(argument) 805 | if default is not None: 806 | self.add_terminal(TokenNode("=")) 807 | inner_symtable = self.__scope_symtable.pop() 808 | self.visit(default) 809 | self.__scope_symtable.append(inner_symtable) 810 | self._add_edge(argument, default, EdgeType.COMPUTED_FROM) 811 | self.add_terminal(TokenNode(",")) 812 | if i > 0: 813 | self._add_edge(node.args[i - 1], argument, EdgeType.NEXT) 814 | 815 | if node.vararg is not None: 816 | self.add_terminal(TokenNode("*")) 817 | self.visit(node.vararg) 818 | self.add_terminal(TokenNode(",")) 819 | 820 | if node.kwarg is not None: 821 | self.add_terminal(TokenNode("**")) 822 | self.visit(node.kwarg) 823 | 824 | if len(node.kwonlyargs) > 0: 825 | self.add_terminal(TokenNode("*")) 826 | self.add_terminal(TokenNode(",")) 827 | defaults = [None] * (len(node.kwonlyargs) - len(node.kw_defaults)) + node.kw_defaults 828 | for argument, default in zip(node.kwonlyargs, defaults): 829 | self.visit(argument) 830 | if default is not None: 831 | self.add_terminal(TokenNode("=")) 832 | inner_symtable = self.__scope_symtable.pop() 833 | self.visit(default) 834 | self.__scope_symtable.append(inner_symtable) 835 | self._add_edge(argument, default, EdgeType.COMPUTED_FROM) 836 | self.add_terminal(TokenNode(",")) 837 | 838 | def visit_keyword(self, node): 839 | if node.arg is not None: 840 | self.add_terminal(TokenNode(node.arg)) 841 | self.add_terminal(TokenNode("=")) 842 | self.visit(node.value) 843 | 844 | # region Comprehensions 845 | def visit_comprehension(self, node: comprehension): 846 | if node.is_async: 847 | self.add_terminal(TokenNode("async")) 848 | self.add_terminal(TokenNode("for")) 849 | self.visit(node.target) 850 | 851 | self.add_terminal(TokenNode("in")) 852 | inner_symtable = self.__scope_symtable.pop() 853 | self.visit(node.iter) 854 | self.__scope_symtable.append(inner_symtable) 855 | 856 | for if_ in node.ifs: 857 | self.add_terminal(TokenNode("if")) 858 | self.visit(if_) 859 | 860 | def visit_ListComp(self, node): 861 | self.__enter_child_symbol_table("function", "listcomp", node.lineno) 862 | try: 863 | self.add_terminal(TokenNode("[")) 864 | self.visit(node.elt) 865 | for i, generator in enumerate(node.generators): 866 | if i > 0: 867 | # When we have multiple generators, then the symbol table of the iter is in the listcomp symboltable. 868 | # Reasonable, but I don't see any other 869 | self.__scope_symtable.append(self.__scope_symtable[-1]) 870 | self.visit(generator) 871 | if i > 0: 872 | self.__scope_symtable.pop() 873 | self.add_terminal(TokenNode("]")) 874 | finally: 875 | self.__scope_symtable.pop() 876 | 877 | def visit_GeneratorExp(self, node): 878 | self.__enter_child_symbol_table("function", "genexpr", node.lineno) 879 | try: 880 | self.add_terminal(TokenNode("(")) 881 | self.visit(node.elt) 882 | for i, generator in enumerate(node.generators): 883 | if i > 0: 884 | # When we have multiple generators, then the symbol table of the iter is in the genexpr symboltable. 885 | # Reasonable, but I don't see any other 886 | self.__scope_symtable.append(self.__scope_symtable[-1]) 887 | self.visit(generator) 888 | if i > 0: 889 | self.__scope_symtable.pop() 890 | self.add_terminal(TokenNode(")")) 891 | finally: 892 | self.__scope_symtable.pop() 893 | 894 | def visit_SetComp(self, node): 895 | self.__enter_child_symbol_table("function", "setcomp", node.lineno) 896 | try: 897 | self.add_terminal(TokenNode("{")) 898 | self.visit(node.elt) 899 | for i, generator in enumerate(node.generators): 900 | if i > 0: 901 | # When we have multiple generators, then the symbol table of the iter is in the setcomp symboltable. 902 | # Reasonable, but I don't see any other 903 | self.__scope_symtable.append(self.__scope_symtable[-1]) 904 | self.visit(generator) 905 | if i > 0: 906 | self.__scope_symtable.pop() 907 | self.add_terminal(TokenNode("}")) 908 | finally: 909 | self.__scope_symtable.pop() 910 | 911 | def visit_DictComp(self, node): 912 | self.__enter_child_symbol_table("function", "dictcomp", node.lineno) 913 | try: 914 | self.add_terminal(TokenNode("{")) 915 | self.visit(node.key) 916 | self.add_terminal(TokenNode(":")) 917 | self.visit(node.value) 918 | 919 | for i, generator in enumerate(node.generators): 920 | if i > 0: 921 | # When we have multiple generators, then the symbol table of the iter is in the dictcomp symboltable. 922 | # Reasonable, but I don't see any other 923 | self.__scope_symtable.append(self.__scope_symtable[-1]) 924 | self.visit(generator) 925 | if i > 0: 926 | self.__scope_symtable.pop() 927 | self.add_terminal(TokenNode("}")) 928 | finally: 929 | self.__scope_symtable.pop() 930 | 931 | # endregion 932 | 933 | # region Simple Expressions 934 | def visit_alias(self, node: alias): 935 | if node.asname is not None: 936 | self.__imported_symbols[ 937 | parse_type_annotation_node(node.asname) 938 | ] = parse_type_annotation_node(node.name) 939 | 940 | def visit_Attribute(self, node: Attribute): 941 | self.__visit_variable_like(node, node.lineno, node.col_offset, False) 942 | # self.visit(node.value) 943 | 944 | def visit_Assert(self, node: Assert): 945 | self.add_terminal(TokenNode("assert")) 946 | self.visit(node.test) 947 | if node.msg is not None: 948 | self.add_terminal(TokenNode(",")) 949 | self.visit(node.msg) 950 | 951 | def visit_Await(self, node: Await): 952 | self.add_terminal(TokenNode("await")) 953 | self.visit(node.value) 954 | 955 | def visit_BinOp(self, node): 956 | self.visit(node.left) 957 | self.add_terminal(TokenNode(self.BINOP_SYMBOLS[type(node.op)])) 958 | self.visit(node.right) 959 | 960 | def visit_BoolOp(self, node): 961 | for idx, value in enumerate(node.values): 962 | self.visit(value) 963 | if idx < len(node.values) - 1: 964 | self.add_terminal(TokenNode(self.BOOLOP_SYMBOLS[type(node.op)])) 965 | 966 | def visit_Compare(self, node: Compare): 967 | self.visit(node.left) 968 | for i, (op, right) in enumerate(zip(node.ops, node.comparators)): 969 | self.add_terminal(TokenNode(self.CMPOP_SYMBOLS[type(op)])) 970 | self.visit(right) 971 | 972 | def visit_Delete(self, node: Delete): 973 | self.add_terminal(TokenNode("del")) 974 | for i, target in enumerate(node.targets): 975 | self.visit(target) 976 | if i < len(node.targets) - 1: 977 | self.add_terminal(TokenNode(",")) 978 | 979 | def visit_Ellipsis(self, node): 980 | self.add_terminal(TokenNode("...")) 981 | 982 | def visit_ExtSlice(self, node: ExtSlice): 983 | for i, value in enumerate(node.dims): 984 | self.visit(value) 985 | if i < len(node.dims) - 1: 986 | self.add_terminal(TokenNode(",")) 987 | 988 | def visit_Expr(self, node): 989 | self.visit(node.value) 990 | 991 | def visit_Global(self, node: Global): 992 | self.add_terminal(TokenNode("global")) 993 | for name in node.names: 994 | self.__visit_variable_like(name, node.lineno, node.col_offset, can_annotate_here=False) 995 | 996 | def visit_Index(self, node: Index): 997 | self.visit(node.value) 998 | 999 | def visit_Nonlocal(self, node: Nonlocal): 1000 | self.add_terminal(TokenNode("nonlocal")) 1001 | for name in node.names: 1002 | self.__visit_variable_like(name, node.lineno, node.col_offset, can_annotate_here=False) 1003 | 1004 | def visit_Pass(self, node): 1005 | self.add_terminal(TokenNode("pass")) 1006 | 1007 | def visit_Slice(self, node): 1008 | self.add_terminal(TokenNode("[")) 1009 | if node.lower is not None: 1010 | self.visit(node.lower) 1011 | self.add_terminal(TokenNode(":")) 1012 | if node.upper is not None: 1013 | self.visit(node.upper) 1014 | if node.step is not None: 1015 | self.add_terminal(TokenNode(":")) 1016 | self.visit(node.step) 1017 | self.add_terminal(TokenNode("]")) 1018 | 1019 | def visit_Subscript(self, node: Subscript): 1020 | self.visit(node.value) 1021 | self.add_terminal(TokenNode("[")) 1022 | self.visit(node.slice) 1023 | self.add_terminal(TokenNode("]")) 1024 | 1025 | def visit_Starred(self, node: Starred): 1026 | self.add_terminal(TokenNode("*")) 1027 | self.visit(node.value) 1028 | 1029 | def visit_UnaryOp(self, node): 1030 | op = self.UNARYOP_SYMBOLS[type(node.op)] 1031 | self.add_terminal(TokenNode(op)) 1032 | self.visit(node.operand) 1033 | 1034 | # endregion 1035 | 1036 | # region Data Structure Constructors 1037 | def visit_Dict(self, node): 1038 | self.add_terminal(TokenNode("{")) 1039 | for idx, (key, value) in enumerate(zip(node.keys, node.values)): 1040 | if key is None: 1041 | self.add_terminal(TokenNode("None")) 1042 | else: 1043 | self.visit(key) 1044 | self.add_terminal(TokenNode(":")) 1045 | self.visit(value) 1046 | if idx < len(node.keys) - 1: 1047 | self.add_terminal(TokenNode(",")) 1048 | self.add_terminal(TokenNode("}")) 1049 | 1050 | def visit_FormattedValue(self, node: FormattedValue): 1051 | self.add_terminal(TokenNode(str('f"'))) 1052 | self.visit(node.value) 1053 | if node.format_spec is not None: 1054 | self.add_terminal(TokenNode(str(":"))) 1055 | self.visit(node.format_spec) 1056 | self.add_terminal(TokenNode(str('"'))) 1057 | 1058 | def visit_List(self, node): 1059 | self.__sequence_datastruct_visit(node, "[", "]") 1060 | 1061 | def visit_Set(self, node): 1062 | self.__sequence_datastruct_visit(node, "{", "}") 1063 | 1064 | def visit_Tuple(self, node): 1065 | self.__sequence_datastruct_visit(node, "(", ")") 1066 | 1067 | def __sequence_datastruct_visit(self, node, open_brace: str, close_brace: str): 1068 | self.add_terminal(TokenNode(open_brace)) 1069 | for idx, element in enumerate(node.elts): 1070 | self.visit(element) 1071 | self.add_terminal( 1072 | TokenNode(",") 1073 | ) # Always add , this is always correct and useful for len one tuples. 1074 | self.add_terminal(TokenNode(close_brace)) 1075 | 1076 | # endregion 1077 | 1078 | # region literals and constructor-likes 1079 | def visit_Bytes(self, node): 1080 | self.add_terminal(TokenNode(repr(node.s))) 1081 | 1082 | def visit_JoinedStr(self, node: JoinedStr): 1083 | for v in node.values: 1084 | self.visit(v) 1085 | 1086 | def visit_NameConstant(self, node): 1087 | self.add_terminal(TokenNode(str(node.value))) 1088 | 1089 | def visit_Num(self, node): 1090 | self.add_terminal(TokenNode(repr(node.n))) 1091 | 1092 | def visit_Str(self, node: Str): 1093 | self.add_terminal( 1094 | TokenNode('"' + node.s + '"') 1095 | ) # Approximate quote addition, but should be good enough. 1096 | 1097 | # endregion 1098 | 1099 | # region Visualization 1100 | def node_to_label(self, node: Any) -> str: 1101 | if isinstance(node, str): 1102 | return node.replace("\n", "").replace('"', "") 1103 | elif isinstance(node, TokenNode): 1104 | return node.token.replace("\n", "").replace('"', "") 1105 | elif isinstance(node, AST): 1106 | return node.__class__.__name__ 1107 | elif isinstance(node, Symbol): 1108 | return node.get_name() 1109 | elif isinstance(node, StrSymbol): 1110 | return node.name 1111 | elif node is None: 1112 | return "None" 1113 | else: 1114 | raise Exception("Unrecognized node type %s" % type(node)) 1115 | 1116 | def to_dot( 1117 | self, 1118 | filename: str, 1119 | initial_comment: str = "", 1120 | draw_only_edge_types: Optional[Set[EdgeType]] = None, 1121 | ) -> None: 1122 | nodes_to_be_drawn = set() 1123 | 1124 | for edge_type, edges in self.__edges.items(): 1125 | if draw_only_edge_types is not None and edge_type not in draw_only_edge_types: 1126 | continue 1127 | for from_idx, to_idxs in edges.items(): 1128 | nodes_to_be_drawn.add(from_idx) 1129 | for to_idx in to_idxs: 1130 | nodes_to_be_drawn.add(to_idx) 1131 | 1132 | with open(filename, "w") as f: 1133 | if len(initial_comment) > 0: 1134 | f.write("#" + initial_comment) 1135 | f.write("\n") 1136 | f.write("digraph program {\n") 1137 | for node, node_idx in self.__node_to_id.items(): 1138 | if node_idx not in nodes_to_be_drawn: 1139 | continue 1140 | node_lbl = self.node_to_label(node) 1141 | if len(node_lbl) > 15: 1142 | node_lbl = node_lbl[:15] 1143 | if hasattr(node, "lineno"): 1144 | node_lbl += ( 1145 | f":L{node.lineno}:{node.col_offset if hasattr(node, 'col_offset') else -1}" 1146 | ) 1147 | f.write(f'\t node{node_idx}[shape="rectangle", label="{node_lbl}"];\n') 1148 | 1149 | for edge_type, edges in self.__edges.items(): 1150 | if draw_only_edge_types is not None and edge_type not in draw_only_edge_types: 1151 | continue 1152 | for from_idx, to_idxs in edges.items(): 1153 | for to_idx in to_idxs: 1154 | f.write(f'\tnode{from_idx} -> node{to_idx} [label="{edge_type.name}"];\n') 1155 | f.write("}\n") # graph 1156 | 1157 | # endregion 1158 | -------------------------------------------------------------------------------- /src/graph_generator/graphgenutils.py: -------------------------------------------------------------------------------- 1 | import typing 2 | from enum import Enum, auto 3 | from typing import Optional, NamedTuple, List, Dict 4 | 5 | from .typeparsing import TypeAnnotationNode 6 | 7 | 8 | class EdgeType(Enum): 9 | CHILD = auto() 10 | NEXT = auto() 11 | LAST_LEXICAL_USE = auto() 12 | NEXT_USE = auto() 13 | COMPUTED_FROM = auto() 14 | RETURNS_TO = auto() 15 | OCCURRENCE_OF = auto() 16 | SUBTOKEN_OF = auto() 17 | 18 | 19 | class TokenNode: 20 | """A wrapper around token nodes, such that an object-identity is used for comparing nodes.""" 21 | 22 | def __init__(self, token: str, lineno: Optional[int] = None, col_offset: Optional[int] = None): 23 | assert isinstance(token, str) 24 | self.token = token 25 | self.lineno = lineno 26 | self.col_offset = col_offset 27 | 28 | def __str__(self): 29 | return self.token 30 | 31 | 32 | class StrSymbol: 33 | def __init__(self, name: str): 34 | self.name = name 35 | 36 | def __hash__(self): 37 | return hash(self.name) 38 | 39 | def __eq__(self, other): 40 | if not isinstance(other, StrSymbol): 41 | return False 42 | return self.name == other.name 43 | 44 | def __str__(self): 45 | return "Symbol: " + self.name 46 | 47 | 48 | class SymbolInformation(NamedTuple): 49 | name: str 50 | locations: List[typing.Tuple[int, int]] 51 | annotatable_locations: Dict[typing.Tuple[int, int], Optional[TypeAnnotationNode]] 52 | symbol_type: str 53 | 54 | @classmethod 55 | def create(cls, name: str, symbol_type: str) -> "SymbolInformation": 56 | return SymbolInformation(name, [], {}, symbol_type) 57 | -------------------------------------------------------------------------------- /src/graph_generator/type_lattice_generator.py: -------------------------------------------------------------------------------- 1 | from typing import Any, List, Dict, FrozenSet, Set, Tuple, Optional 2 | from collections import defaultdict, ChainMap 3 | from itertools import chain 4 | from functools import lru_cache 5 | import json 6 | 7 | from .typeparsing import ( 8 | TypeAnnotationNode, 9 | NameAnnotationNode, 10 | parse_type_annotation_node, 11 | ) 12 | from .typeparsing import RewriteRuleVisitor 13 | from .typeparsing import DirectInheritanceRewriting 14 | from .typeparsing import EraseOnceTypeRemoval 15 | from .typeparsing import PruneAnnotationVisitor 16 | from .typeparsing import AliasReplacementVisitor 17 | from .typeparsing.rewriterules import RemoveStandAlones 18 | from .typeparsing.rewriterules import RemoveRecursiveGenerics 19 | from .typeparsing.rewriterules import RemoveUnionWithAnys 20 | from .typeparsing.rewriterules import RemoveGenericWithAnys 21 | 22 | 23 | class TypeLatticeGenerator: 24 | 25 | ANY_TYPE = parse_type_annotation_node("typing.Any") 26 | 27 | def __init__(self, typing_rules_path: str, max_depth_size: int = 2, max_list_size: int = 2): 28 | self.__to_process = [] 29 | self.__processed = set() 30 | self.new_type_rules = defaultdict(set) # [new type, ref] 31 | self.module_naming_rules = {} # [short, long.version] 32 | self.__all_types = {self.ANY_TYPE: 0} 33 | self.__ids_to_nodes = [self.ANY_TYPE] 34 | self.__type_reprs = {repr(self.ANY_TYPE): 0} 35 | 36 | # Rewrites 37 | with open(typing_rules_path, "r") as f: 38 | rules = json.load(f) 39 | # alias -> default name 40 | self.__aliases = { 41 | parse_type_annotation_node(k): parse_type_annotation_node(v) 42 | for k, v in rules["aliasing_rules"] 43 | } 44 | for type_annotation in chain(self.__aliases.keys(), self.__aliases.values()): 45 | self.__annotation_to_id(type_annotation) 46 | self.__project_specific_aliases = {} 47 | 48 | self.__max_annotation_depth = max_depth_size 49 | self.__max_depth_pruning_visitor = PruneAnnotationVisitor(self.ANY_TYPE, max_list_size) 50 | 51 | # [specialized type, general type] 52 | self.is_a_edges = defaultdict(set) # type: Dict[int, Set[int]] 53 | self.project_is_a = defaultdict(set) # type: Dict[int, Set[int]] 54 | 55 | # type -> supertypes 56 | for k, v in rules["is_a_relations"]: 57 | self.__add_is_a_relationship( 58 | parse_type_annotation_node(k), parse_type_annotation_node(v) 59 | ) 60 | 61 | for known_type in self.__all_types: 62 | known_type_id = self.__annotation_to_id(known_type) 63 | if known_type != self.ANY_TYPE and len(self.is_a_edges[known_type_id]) == 0: 64 | self.is_a_edges[known_type_id].add(0) 65 | 66 | self.__compute_non_generic_types() 67 | 68 | self.__type_erasure = EraseOnceTypeRemoval() 69 | self.__direct_inheritance_rewriting = DirectInheritanceRewriting( 70 | self.__is_a_relationships, self.__non_generic_types 71 | ) 72 | 73 | self.__rewrites_verbose_annotations = RewriteRuleVisitor( 74 | [ 75 | RemoveUnionWithAnys(), 76 | RemoveStandAlones(), 77 | RemoveRecursiveGenerics(), 78 | RemoveGenericWithAnys(), 79 | ] 80 | ) 81 | assert len(self.__ids_to_nodes) == len(set(repr(r) for r in self.__ids_to_nodes)) 82 | 83 | def create_alias_replacement( 84 | self, imported_symbols: Dict[TypeAnnotationNode, TypeAnnotationNode] 85 | ) -> AliasReplacementVisitor: 86 | return AliasReplacementVisitor( 87 | ChainMap(imported_symbols, self.__project_specific_aliases, self.__aliases) 88 | ) 89 | 90 | def __compute_non_generic_types(self): 91 | # Now get all the annotations that are *not* generics 92 | child_edges = defaultdict(set) 93 | for parent, children in self.is_a_edges.items(): 94 | for child in children: 95 | child_edges[child].add(parent) 96 | 97 | generic_transitive_closure = set() 98 | to_visit = [ 99 | self.__all_types[parse_type_annotation_node("typing.Generic")], 100 | self.__all_types[parse_type_annotation_node("typing.Tuple")], 101 | self.__all_types[parse_type_annotation_node("typing.Callable")], 102 | ] 103 | while len(to_visit) > 0: 104 | next_node = to_visit.pop() 105 | generic_transitive_closure.add(next_node) 106 | to_visit.extend( 107 | (t for t in child_edges[next_node] if t not in generic_transitive_closure) 108 | ) 109 | 110 | non_generic_types = set(self.__all_types.values()) - generic_transitive_closure 111 | 112 | # Add special objects 113 | non_generic_types.add(self.__annotation_to_id(parse_type_annotation_node("abc.ABC"))) 114 | 115 | self.__non_generic_types = frozenset((self.__ids_to_nodes[i] for i in non_generic_types)) 116 | 117 | def __annotation_to_id(self, annotation: TypeAnnotationNode) -> int: 118 | annotation_idx = self.__all_types.get(annotation) 119 | if annotation_idx is None: 120 | if repr(annotation) in self.__type_reprs: 121 | return self.__type_reprs[repr(annotation)] 122 | annotation_idx = len(self.__all_types) 123 | self.__all_types[annotation] = annotation_idx 124 | self.__type_reprs[repr(annotation)] = annotation_idx 125 | self.__ids_to_nodes.append(annotation) 126 | 127 | return annotation_idx 128 | 129 | def __all_reachable_from(self, type_idx: int) -> FrozenSet[int]: 130 | reachable = set() 131 | to_visit = [type_idx] # type: List[int] 132 | while len(to_visit) > 0: 133 | next_type_idx = to_visit.pop() 134 | reachable.add(next_type_idx) 135 | to_visit.extend( 136 | ( 137 | parent_type_idx 138 | for parent_type_idx in self.is_a_edges[next_type_idx] 139 | if parent_type_idx not in reachable 140 | ) 141 | ) 142 | return frozenset(reachable) 143 | 144 | def __add_is_a_relationship( 145 | self, from_type: TypeAnnotationNode, to_type: TypeAnnotationNode 146 | ) -> None: 147 | from_node_idx = self.__annotation_to_id(from_type) 148 | to_node_idx = self.__annotation_to_id(to_type) 149 | 150 | if from_node_idx == to_node_idx: 151 | return 152 | 153 | reachable_from_idx = self.__all_reachable_from(from_node_idx) 154 | if to_node_idx in reachable_from_idx: 155 | # This is already reachable, ignore direct is-a relationship. 156 | return 157 | 158 | all_reachable = self.__all_reachable_from(to_node_idx) 159 | if from_node_idx in all_reachable: 160 | print(f"The {from_node_idx}<->{to_node_idx} would be a circle. Ignoring.") 161 | return 162 | 163 | self.is_a_edges[from_node_idx].add(to_node_idx) 164 | 165 | def __is_a_relationships(self, from_type: TypeAnnotationNode) -> FrozenSet[TypeAnnotationNode]: 166 | from_node_idx = self.__annotation_to_id(from_type) 167 | return frozenset((self.__ids_to_nodes[t] for t in self.is_a_edges[from_node_idx])) 168 | 169 | def add_type( 170 | self, 171 | annotation: TypeAnnotationNode, 172 | imported_symbols: Dict[TypeAnnotationNode, TypeAnnotationNode], 173 | ): 174 | annotation = annotation.accept_visitor(self.create_alias_replacement(imported_symbols)) 175 | if annotation in self.__all_types: 176 | return 177 | pruned = annotation.accept_visitor( 178 | self.__max_depth_pruning_visitor, self.__max_annotation_depth 179 | ) 180 | if pruned != annotation: 181 | self.__add_is_a_relationship(annotation, pruned) 182 | self.__to_process.append(pruned) 183 | else: 184 | self.__to_process.append(annotation) 185 | 186 | @lru_cache(16384) 187 | def __compute_erasures( 188 | self, type_annotation: TypeAnnotationNode 189 | ) -> Tuple[List[TypeAnnotationNode], bool]: 190 | return type_annotation.accept_visitor(self.__type_erasure) 191 | 192 | @lru_cache(16384) 193 | def __rewrite_verbose(self, type_annotation: TypeAnnotationNode) -> TypeAnnotationNode: 194 | return type_annotation.accept_visitor(self.__rewrites_verbose_annotations, None) 195 | 196 | def build_graph(self): 197 | print( 198 | "Building type graph for project... (%s elements to process)" % len(self.__to_process) 199 | ) 200 | i = 0 201 | 202 | while len(self.__to_process) > 0: 203 | next_type = self.__to_process.pop() 204 | 205 | if next_type in self.__processed: 206 | continue 207 | 208 | i += 1 209 | if i > 500: 210 | print( 211 | "Building type graph for project... (%s elements to process)" 212 | % len(self.__to_process) 213 | ) 214 | i = 0 215 | if len(self.__to_process) > 3000: 216 | print(f"Queue quite long. Current element {next_type}") 217 | 218 | all_erasures, erasure_happened = self.__compute_erasures(next_type) 219 | if erasure_happened: 220 | for erased_type in all_erasures: 221 | erased_type = self.__rewrite_verbose(erased_type) 222 | erased_type_has_been_proceesed = erased_type in self.__all_types 223 | 224 | self.__add_is_a_relationship(next_type, erased_type) 225 | if not erased_type_has_been_proceesed: 226 | self.__to_process.append(erased_type) 227 | 228 | all_inherited_types_and_self = next_type.accept_visitor( 229 | self.__direct_inheritance_rewriting 230 | ) 231 | was_rewritten = len(all_inherited_types_and_self) > 1 232 | 233 | if was_rewritten: 234 | if ( 235 | not erasure_happened 236 | or len(self.__to_process) < 5000 237 | or len(all_inherited_types_and_self) < 5 238 | ): 239 | for type_annotation in all_inherited_types_and_self: 240 | type_annotation = type_annotation.accept_visitor( 241 | self.__max_depth_pruning_visitor, self.__max_annotation_depth, 242 | ) 243 | type_annotation = self.__rewrite_verbose(type_annotation) 244 | type_has_been_seen = type_annotation in self.__all_types 245 | 246 | if not type_has_been_seen: 247 | self.__add_is_a_relationship(next_type, type_annotation) 248 | self.__to_process.append(type_annotation) 249 | 250 | if not was_rewritten and not erasure_happened: 251 | # Add a rule to Any 252 | self.__add_is_a_relationship(next_type, self.ANY_TYPE) 253 | self.__processed.add(next_type) 254 | 255 | # Clean up project-specific aliases 256 | self.__project_specific_aliases.clear() 257 | print("Done building type graph") 258 | 259 | def add_class(self, class_name: str, parents: List[TypeAnnotationNode]) -> None: 260 | class_name = NameAnnotationNode(class_name) 261 | for parent in parents: 262 | if parent is None: 263 | continue 264 | assert isinstance(parent, TypeAnnotationNode), (parent, type(parent)) 265 | self.__add_is_a_relationship(class_name, parent) 266 | 267 | def add_type_alias( 268 | self, new_annotation: TypeAnnotationNode, ref_annotation: TypeAnnotationNode 269 | ) -> None: 270 | self.__project_specific_aliases[new_annotation] = ref_annotation 271 | 272 | def canonicalize_annotation( 273 | self, 274 | annotation: TypeAnnotationNode, 275 | local_aliases: Dict[TypeAnnotationNode, TypeAnnotationNode], 276 | ) -> Optional[TypeAnnotationNode]: 277 | if annotation is None: 278 | return None 279 | return annotation.accept_visitor(self.create_alias_replacement(local_aliases)) 280 | 281 | def return_json(self) -> Dict[str, Any]: 282 | edges = [] 283 | for from_type_idx, to_type_idxs in self.is_a_edges.items(): 284 | for to_type_idx in to_type_idxs: 285 | edges.append((from_type_idx, to_type_idx)) 286 | 287 | assert len(self.__ids_to_nodes) == len(set(repr(r) for r in self.__ids_to_nodes)) 288 | return { 289 | "nodes": list((repr(type_annotation) for type_annotation in self.__ids_to_nodes)), 290 | "edges": edges, 291 | } 292 | -------------------------------------------------------------------------------- /src/graph_generator/typeparsing/__init__.py: -------------------------------------------------------------------------------- 1 | from .visitor import TypeAnnotationVisitor 2 | from .nodes import * 3 | 4 | from .aliasreplacement import AliasReplacementVisitor 5 | from .erasure import EraseOnceTypeRemoval 6 | from .inheritancerewrite import DirectInheritanceRewriting 7 | from .pruneannotations import PruneAnnotationVisitor 8 | from .rewriterulevisitor import RewriteRuleVisitor 9 | -------------------------------------------------------------------------------- /src/graph_generator/typeparsing/aliasreplacement.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Tuple 2 | 3 | from .nodes import ( 4 | TypeAnnotationNode, 5 | SubscriptAnnotationNode, 6 | TupleAnnotationNode, 7 | ListAnnotationNode, 8 | AttributeAnnotationNode, 9 | IndexAnnotationNode, 10 | ElipsisAnnotationNode, 11 | ) 12 | from .visitor import TypeAnnotationVisitor 13 | 14 | __all__ = ["AliasReplacementVisitor"] 15 | 16 | 17 | class AliasReplacementVisitor(TypeAnnotationVisitor): 18 | """Replace Nodes with Aliases. Assumes recursion has been resolved in replacement_map""" 19 | 20 | def __init__(self, replacement_map: Dict[TypeAnnotationNode, TypeAnnotationNode]): 21 | self.__replacement_map = replacement_map 22 | 23 | def __replace_full(self, node: TypeAnnotationNode) -> Tuple[TypeAnnotationNode, bool]: 24 | replaced = False 25 | seen_names = {node} 26 | while node in self.__replacement_map: 27 | node = self.__replacement_map[node] 28 | replaced = True 29 | if node in seen_names: 30 | print(f"WARNING: Circle between {seen_names}. Picking the {node} for now.") 31 | break 32 | else: 33 | seen_names.add(node) 34 | return node, replaced 35 | 36 | def visit_subscript_annotation(self, node: SubscriptAnnotationNode): 37 | replacement, replaced = self.__replace_full(node) 38 | if replaced: 39 | return replacement 40 | return SubscriptAnnotationNode( 41 | value=node.value.accept_visitor(self), 42 | slice=node.slice.accept_visitor(self) if node.slice is not None else None, 43 | ) 44 | 45 | def visit_tuple_annotation(self, node: TupleAnnotationNode): 46 | replacement, replaced = self.__replace_full(node) 47 | if replaced: 48 | return replacement 49 | return TupleAnnotationNode((e.accept_visitor(self) for e in node.elements)) 50 | 51 | def visit_name_annotation(self, node): 52 | return self.__replace_full(node)[0] 53 | 54 | def visit_list_annotation(self, node: ListAnnotationNode): 55 | replacement, replaced = self.__replace_full(node) 56 | if replaced: 57 | return replacement 58 | return ListAnnotationNode((e.accept_visitor(self) for e in node.elements)) 59 | 60 | def visit_attribute_annotation(self, node: AttributeAnnotationNode): 61 | replacement, replaced = self.__replace_full(node) 62 | if replaced: 63 | return replacement 64 | return AttributeAnnotationNode(node.value.accept_visitor(self), node.attribute) 65 | 66 | def visit_index_annotation(self, node: IndexAnnotationNode): 67 | replacement, replaced = self.__replace_full(node) 68 | if replaced: 69 | return replacement 70 | return IndexAnnotationNode(node.value.accept_visitor(self)) 71 | 72 | def visit_elipsis_annotation(self, node: ElipsisAnnotationNode): 73 | return node 74 | 75 | def visit_name_constant_annotation(self, node): 76 | return self.__replace_full(node)[0] 77 | 78 | def visit_unknown_annotation(self, node): 79 | return node 80 | -------------------------------------------------------------------------------- /src/graph_generator/typeparsing/erasure.py: -------------------------------------------------------------------------------- 1 | from itertools import product 2 | 3 | from .nodes import ( 4 | SubscriptAnnotationNode, 5 | TupleAnnotationNode, 6 | ListAnnotationNode, 7 | AttributeAnnotationNode, 8 | IndexAnnotationNode, 9 | ElipsisAnnotationNode, 10 | ) 11 | from .visitor import TypeAnnotationVisitor 12 | 13 | __all__ = ["EraseOnceTypeRemoval"] 14 | 15 | 16 | class EraseOnceTypeRemoval(TypeAnnotationVisitor): 17 | """Replace Nodes with Aliases. Assumes recursion has been resolved in replacement_map""" 18 | 19 | def __init__(self): 20 | pass 21 | 22 | def visit_subscript_annotation(self, node: SubscriptAnnotationNode): 23 | if node.slice is None: 24 | erasure_happened_at_a_slice = False 25 | else: 26 | next_slices, erasure_happened_at_a_slice = node.slice.accept_visitor(self) 27 | 28 | if not erasure_happened_at_a_slice: 29 | return [node, node.value], True # Erase type parameters 30 | 31 | return ( 32 | [SubscriptAnnotationNode(value=node.value, slice=s) for s in next_slices], 33 | True, 34 | ) 35 | 36 | def visit_tuple_annotation(self, node: TupleAnnotationNode): 37 | elements = [e.accept_visitor(self) for e in node.elements] 38 | 39 | erasure_happened_before = any(e[1] for e in elements) 40 | return ( 41 | [TupleAnnotationNode(t) for t in product(*(e[0] for e in elements))], 42 | erasure_happened_before, 43 | ) 44 | 45 | def visit_name_annotation(self, node): 46 | return [node], False 47 | 48 | def visit_list_annotation(self, node: ListAnnotationNode): 49 | elements = [e.accept_visitor(self) for e in node.elements] 50 | 51 | erasure_happened_before = any(e[1] for e in elements) 52 | return ( 53 | [ListAnnotationNode(t) for t in product(*(e[0] for e in elements))], 54 | erasure_happened_before, 55 | ) 56 | 57 | def visit_attribute_annotation(self, node: AttributeAnnotationNode): 58 | return [node], False 59 | 60 | def visit_index_annotation(self, node: IndexAnnotationNode): 61 | next_values, erasure_happened = node.value.accept_visitor(self) 62 | return [IndexAnnotationNode(v) for v in next_values], erasure_happened 63 | 64 | def visit_elipsis_annotation(self, node: ElipsisAnnotationNode): 65 | return [node], False 66 | 67 | def visit_name_constant_annotation(self, node): 68 | return [node], False 69 | 70 | def visit_unknown_annotation(self, node): 71 | return [node], False 72 | -------------------------------------------------------------------------------- /src/graph_generator/typeparsing/inheritancerewrite.py: -------------------------------------------------------------------------------- 1 | from itertools import product 2 | from typing import Callable, Iterator, Set 3 | import random 4 | 5 | from .nodes import ( 6 | TypeAnnotationNode, 7 | SubscriptAnnotationNode, 8 | TupleAnnotationNode, 9 | ListAnnotationNode, 10 | AttributeAnnotationNode, 11 | IndexAnnotationNode, 12 | ElipsisAnnotationNode, 13 | ) 14 | from .visitor import TypeAnnotationVisitor 15 | 16 | __all__ = ["DirectInheritanceRewriting"] 17 | 18 | 19 | class DirectInheritanceRewriting(TypeAnnotationVisitor): 20 | """Replace Nodes their direct is-a relationships""" 21 | 22 | def __init__( 23 | self, 24 | is_a_info: Callable[[TypeAnnotationNode], Iterator[TypeAnnotationNode]], 25 | non_generic_types: Set[TypeAnnotationNode], 26 | limit_combinations_to: int = 10000, 27 | ): 28 | self.__is_a = is_a_info 29 | self.__non_generic_types = non_generic_types 30 | self.__limit_combinations_to = limit_combinations_to 31 | 32 | def visit_subscript_annotation(self, node: SubscriptAnnotationNode): 33 | value_node_options = node.value.accept_visitor(self) 34 | if node.slice is None: 35 | slice_node_options = [None] 36 | else: 37 | slice_node_options = node.slice.accept_visitor(self) 38 | 39 | all_children = [] 40 | for v in value_node_options: 41 | if v in self.__non_generic_types: 42 | all_children.append(v) 43 | continue 44 | for s in slice_node_options: 45 | all_children.append(SubscriptAnnotationNode(v, s)) 46 | 47 | return all_children 48 | 49 | def visit_tuple_annotation(self, node: TupleAnnotationNode): 50 | all_elements_options = [e.accept_visitor(self) for e in node.elements] 51 | r = [TupleAnnotationNode(t) for t in product(*all_elements_options)] 52 | 53 | if len(r) > self.__limit_combinations_to: 54 | random.shuffle(r) 55 | return r[: self.__limit_combinations_to] 56 | return r 57 | 58 | def visit_name_annotation(self, node): 59 | return [node] + list(self.__is_a(node)) 60 | 61 | def visit_list_annotation(self, node: ListAnnotationNode): 62 | all_elements_options = [e.accept_visitor(self) for e in node.elements] 63 | r = [ListAnnotationNode(t) for t in product(*all_elements_options)] 64 | if len(r) > self.__limit_combinations_to: 65 | random.shuffle(r) 66 | return r[: self.__limit_combinations_to] 67 | return r 68 | 69 | def visit_attribute_annotation(self, node: AttributeAnnotationNode): 70 | v = [node] + list(self.__is_a(node)) 71 | return v 72 | 73 | def visit_index_annotation(self, node: IndexAnnotationNode): 74 | next_values = node.value.accept_visitor(self) 75 | return [IndexAnnotationNode(v) for v in next_values] 76 | 77 | def visit_elipsis_annotation(self, node: ElipsisAnnotationNode): 78 | return [node] 79 | 80 | def visit_name_constant_annotation(self, node): 81 | return [node] 82 | 83 | def visit_unknown_annotation(self, node): 84 | return [node] 85 | -------------------------------------------------------------------------------- /src/graph_generator/typeparsing/nodes.py: -------------------------------------------------------------------------------- 1 | import traceback 2 | from abc import ABC, abstractmethod 3 | from typing import Any, Optional, Iterator 4 | 5 | import typed_ast 6 | from typed_ast.ast3 import parse 7 | 8 | from .visitor import TypeAnnotationVisitor 9 | 10 | __all__ = [ 11 | "FaultyAnnotation", 12 | "TypeAnnotationNode", 13 | "SubscriptAnnotationNode", 14 | "TupleAnnotationNode", 15 | "NameAnnotationNode", 16 | "ListAnnotationNode", 17 | "AttributeAnnotationNode", 18 | "IndexAnnotationNode", 19 | "ElipsisAnnotationNode", 20 | "NameConstantAnnotationNode", 21 | "UnknownAnnotationNode", 22 | "parse_type_annotation_node", 23 | "parse_type_comment", 24 | ] 25 | 26 | 27 | class FaultyAnnotation(Exception): 28 | pass 29 | 30 | 31 | class TypeAnnotationNode(ABC): 32 | @abstractmethod 33 | def size(self) -> int: 34 | pass 35 | 36 | @abstractmethod 37 | def accept_visitor(self, visitor: TypeAnnotationVisitor, *args) -> Any: 38 | pass 39 | 40 | @abstractmethod 41 | def __repr__(self): 42 | pass 43 | 44 | @staticmethod 45 | @abstractmethod 46 | def parse(node) -> "SubscriptAnnotationNode": 47 | pass 48 | 49 | 50 | class SubscriptAnnotationNode(TypeAnnotationNode): 51 | def __init__(self, value: TypeAnnotationNode, slice: Optional[TypeAnnotationNode]): 52 | self.value = value 53 | self.slice = slice 54 | 55 | def size(self) -> int: 56 | size = 1 + self.value.size() 57 | if self.slice is not None: 58 | size += self.slice.size() 59 | return size 60 | 61 | def accept_visitor(self, visitor: TypeAnnotationVisitor, *args) -> Any: 62 | return visitor.visit_subscript_annotation(self, *args) 63 | 64 | def __repr__(self): 65 | return repr(self.value) + "[" + repr(self.slice) + "]" 66 | 67 | def __hash__(self): 68 | return hash(self.value) ^ (hash(self.slice) + 13) 69 | 70 | def __eq__(self, other): 71 | if not isinstance(other, SubscriptAnnotationNode): 72 | return False 73 | else: 74 | return self.value == other.value and self.slice == other.slice 75 | 76 | @staticmethod 77 | def parse(node) -> "SubscriptAnnotationNode": 78 | assert hasattr(node, "value") 79 | assert hasattr(node, "slice") 80 | 81 | v = _parse_recursive(node.value) 82 | s = _parse_recursive(node.slice) 83 | assert v is not None 84 | return SubscriptAnnotationNode(v, s) 85 | 86 | 87 | class TupleAnnotationNode(TypeAnnotationNode): 88 | def __init__(self, elements: Iterator[TypeAnnotationNode]): 89 | self.elements = tuple(elements) 90 | 91 | def size(self) -> int: 92 | return sum(e.size() for e in self.elements) + 1 93 | 94 | def accept_visitor(self, visitor: TypeAnnotationVisitor, *args) -> Any: 95 | return visitor.visit_tuple_annotation(self, *args) 96 | 97 | def __repr__(self): 98 | return ", ".join(repr(e) for e in self.elements) 99 | 100 | def __hash__(self): 101 | if len(self.elements) > 0: 102 | return hash(self.elements) 103 | else: 104 | return 1 105 | 106 | def __eq__(self, other): 107 | if not isinstance(other, TupleAnnotationNode): 108 | return False 109 | else: 110 | return self.elements == other.elements 111 | 112 | @staticmethod 113 | def parse(node) -> "TupleAnnotationNode": 114 | assert hasattr(node, "elts") 115 | return TupleAnnotationNode((_parse_recursive(el) for el in node.elts)) 116 | 117 | 118 | class NameAnnotationNode(TypeAnnotationNode): 119 | def __init__(self, identifier: str): 120 | self.identifier = identifier 121 | 122 | def size(self) -> int: 123 | return 1 124 | 125 | def accept_visitor(self, visitor: TypeAnnotationVisitor, *args) -> Any: 126 | return visitor.visit_name_annotation(self, *args) 127 | 128 | def __repr__(self): 129 | return self.identifier 130 | 131 | def __hash__(self): 132 | return hash(self.identifier) 133 | 134 | def __eq__(self, other): 135 | if not isinstance(other, NameAnnotationNode): 136 | return False 137 | return self.identifier == other.identifier 138 | 139 | @staticmethod 140 | def parse(node) -> "NameAnnotationNode": 141 | assert hasattr(node, "id") 142 | return NameAnnotationNode(node.id) 143 | 144 | 145 | class ListAnnotationNode(TypeAnnotationNode): 146 | def __init__(self, elements: Iterator[TypeAnnotationNode]): 147 | self.elements = tuple(elements) 148 | 149 | def size(self) -> int: 150 | return sum(e.size() for e in self.elements) + 1 151 | 152 | def accept_visitor(self, visitor: TypeAnnotationVisitor, *args) -> Any: 153 | return visitor.visit_list_annotation(self, *args) 154 | 155 | def __repr__(self): 156 | return "[" + ", ".join(repr(e) for e in self.elements) + "]" 157 | 158 | def __hash__(self): 159 | if len(self.elements) > 0: 160 | return hash(self.elements) 161 | else: 162 | return 2 163 | 164 | def __eq__(self, other): 165 | if not isinstance(other, ListAnnotationNode): 166 | return False 167 | return self.elements == other.elements 168 | 169 | @staticmethod 170 | def parse(node) -> "ListAnnotationNode": 171 | assert hasattr(node, "elts") 172 | return ListAnnotationNode((_parse_recursive(el) for el in node.elts)) 173 | 174 | 175 | class AttributeAnnotationNode(TypeAnnotationNode): 176 | def __init__(self, value: TypeAnnotationNode, attribute: str): 177 | self.value = value 178 | assert isinstance(attribute, str), type(attribute) 179 | self.attribute = attribute 180 | 181 | def size(self) -> int: 182 | return 1 + self.value.size() 183 | 184 | def accept_visitor(self, visitor: TypeAnnotationVisitor, *args) -> Any: 185 | return visitor.visit_attribute_annotation(self, *args) 186 | 187 | def __repr__(self): 188 | return repr(self.value) + "." + self.attribute 189 | 190 | def __hash__(self): 191 | return hash(self.attribute) ^ (hash(self.value) + 13) 192 | 193 | def __eq__(self, other): 194 | if not isinstance(other, AttributeAnnotationNode): 195 | return False 196 | else: 197 | return self.attribute == other.attribute and self.value == other.value 198 | 199 | @staticmethod 200 | def parse(node) -> "AttributeAnnotationNode": 201 | assert hasattr(node, "value") 202 | assert hasattr(node, "attr") 203 | return AttributeAnnotationNode(_parse_recursive(node.value), node.attr) 204 | 205 | 206 | class IndexAnnotationNode(TypeAnnotationNode): 207 | def __init__(self, value: TypeAnnotationNode): 208 | self.value = value 209 | 210 | def size(self) -> int: 211 | return 1 + self.value.size() 212 | 213 | def accept_visitor(self, visitor: TypeAnnotationVisitor, *args) -> Any: 214 | return visitor.visit_index_annotation(self, *args) 215 | 216 | def __repr__(self): 217 | return repr(self.value) 218 | 219 | def __hash__(self): 220 | return hash(self.value) 221 | 222 | def __eq__(self, other): 223 | if not isinstance(other, IndexAnnotationNode): 224 | return False 225 | return self.value == other.value 226 | 227 | @staticmethod 228 | def parse(node) -> "IndexAnnotationNode": 229 | assert hasattr(node, "value") 230 | return IndexAnnotationNode(_parse_recursive(node.value)) 231 | 232 | 233 | class ElipsisAnnotationNode(TypeAnnotationNode): 234 | def __init__(self): 235 | pass 236 | 237 | def size(self) -> int: 238 | return 1 239 | 240 | def accept_visitor(self, visitor: TypeAnnotationVisitor, *args) -> Any: 241 | return visitor.visit_elipsis_annotation(self, *args) 242 | 243 | def __repr__(self): 244 | return "..." 245 | 246 | def __hash__(self): 247 | return 3 248 | 249 | def __eq__(self, other): 250 | return isinstance(other, ElipsisAnnotationNode) 251 | 252 | @staticmethod 253 | def parse(node) -> "ElipsisAnnotationNode": 254 | return ElipsisAnnotationNode() 255 | 256 | 257 | class NameConstantAnnotationNode(TypeAnnotationNode): 258 | def __init__(self, value: Any): 259 | self.value = value 260 | 261 | def size(self) -> int: 262 | return 1 263 | 264 | def accept_visitor(self, visitor: TypeAnnotationVisitor, *args) -> Any: 265 | return visitor.visit_name_constant_annotation(self, *args) 266 | 267 | def __repr__(self): 268 | return repr(self.value) 269 | 270 | def __hash__(self): 271 | return hash(self.value) 272 | 273 | def __eq__(self, other): 274 | if not isinstance(other, NameConstantAnnotationNode): 275 | return False 276 | return self.value == other.value 277 | 278 | @staticmethod 279 | def parse(node) -> "NameConstantAnnotationNode": 280 | if hasattr(node, "value"): 281 | return NameConstantAnnotationNode(node.value) 282 | return NameConstantAnnotationNode(None) 283 | 284 | 285 | class UnknownAnnotationNode(TypeAnnotationNode): 286 | def __init__(self): 287 | pass 288 | 289 | def size(self) -> int: 290 | return 1 291 | 292 | def accept_visitor(self, visitor: TypeAnnotationVisitor, *args) -> Any: 293 | return visitor.visit_unknown_annotation(self, *args) 294 | 295 | def __repr__(self): 296 | return "%UNKNOWN%" 297 | 298 | def __hash__(self): 299 | return 4 300 | 301 | def __eq__(self, other): 302 | return isinstance(other, UnknownAnnotationNode) 303 | 304 | @staticmethod 305 | def parse(node) -> "UnknownAnnotationNode": 306 | raise NotImplementedError() 307 | 308 | 309 | def _parse_string_annotation(node): 310 | assert hasattr(node, "s") 311 | try: 312 | node = parse(node.s, "", mode="eval") 313 | return _parse_recursive(node.body) 314 | except SyntaxError: 315 | return None 316 | 317 | 318 | def _parse_recursive(node) -> TypeAnnotationNode: 319 | if isinstance(node, typed_ast._ast3.Subscript): # pytype: disable=module-attr 320 | return SubscriptAnnotationNode.parse(node) 321 | elif isinstance(node, typed_ast._ast3.Tuple): # pytype: disable=module-attr 322 | return TupleAnnotationNode.parse(node) 323 | elif isinstance(node, typed_ast._ast3.Name): # pytype: disable=module-attr 324 | return NameAnnotationNode.parse(node) 325 | elif isinstance(node, typed_ast._ast3.List): # pytype: disable=module-attr 326 | return ListAnnotationNode.parse(node) 327 | elif isinstance(node, typed_ast._ast3.Attribute): # pytype: disable=module-attr 328 | return AttributeAnnotationNode.parse(node) 329 | elif isinstance(node, typed_ast._ast3.Str): # pytype: disable=module-attr 330 | return _parse_string_annotation(node) 331 | elif isinstance(node, typed_ast._ast3.Index): # pytype: disable=module-attr 332 | return IndexAnnotationNode.parse(node) 333 | elif isinstance(node, typed_ast._ast3.Ellipsis): # pytype: disable=module-attr 334 | return ElipsisAnnotationNode.parse(node) 335 | elif isinstance(node, typed_ast._ast3.NameConstant): # pytype: disable=module-attr 336 | return NameConstantAnnotationNode.parse(node) 337 | elif isinstance(node, typed_ast._ast3.Num): # pytype: disable=module-attr 338 | return NameConstantAnnotationNode.parse(node) 339 | else: 340 | raise Exception("Unparsable type node.") 341 | 342 | 343 | def parse_type_annotation_node(node) -> Optional[TypeAnnotationNode]: 344 | """ 345 | Processes the node containing the type annotation and return the object corresponding to the node type. 346 | """ 347 | try: 348 | if isinstance(node, str): 349 | r = parse_type_comment(node) 350 | else: 351 | r = _parse_recursive(node) 352 | return r 353 | except Exception as e: 354 | pass 355 | return None 356 | 357 | 358 | def parse_type_comment(annotation: str) -> Optional[TypeAnnotationNode]: 359 | try: 360 | node = parse(annotation, "", mode="eval") 361 | except SyntaxError: 362 | return None 363 | try: 364 | return _parse_recursive(node.body) 365 | except Exception as e: 366 | return None 367 | -------------------------------------------------------------------------------- /src/graph_generator/typeparsing/pruneannotations.py: -------------------------------------------------------------------------------- 1 | from .nodes import ( 2 | TypeAnnotationNode, 3 | SubscriptAnnotationNode, 4 | TupleAnnotationNode, 5 | ElipsisAnnotationNode, 6 | ListAnnotationNode, 7 | AttributeAnnotationNode, 8 | IndexAnnotationNode, 9 | ) 10 | from .visitor import TypeAnnotationVisitor 11 | 12 | __all__ = ["PruneAnnotationVisitor"] 13 | 14 | 15 | class PruneAnnotationVisitor(TypeAnnotationVisitor): 16 | """Prune Long Annotations""" 17 | 18 | def __init__(self, replacement_node: TypeAnnotationNode, max_list_size: int): 19 | self.__max_list_size = max_list_size 20 | self.__replacement_node = replacement_node 21 | 22 | def visit_subscript_annotation( 23 | self, node: SubscriptAnnotationNode, current_remaining_depth: int 24 | ): 25 | if current_remaining_depth == 0: 26 | return self.__replacement_node 27 | 28 | if node.slice is None: 29 | pruned_slice = None 30 | elif current_remaining_depth <= 2: # Remove the subscript completely. 31 | return node.value.accept_visitor(self, current_remaining_depth - 1) 32 | else: 33 | pruned_slice = node.slice.accept_visitor(self, current_remaining_depth - 1) 34 | 35 | return SubscriptAnnotationNode( 36 | value=node.value.accept_visitor(self, current_remaining_depth - 1), slice=pruned_slice, 37 | ) 38 | 39 | def visit_tuple_annotation(self, node: TupleAnnotationNode, current_remaining_depth: int): 40 | if len(node.elements) > self.__max_list_size: 41 | elements = node.elements[: self.__max_list_size - 1] + (ElipsisAnnotationNode(),) 42 | else: 43 | elements = node.elements 44 | 45 | if len(node.elements) == 0: 46 | pruned = node.elements 47 | elif current_remaining_depth <= 1 and len(node.elements) == 1: 48 | pruned = [self.__replacement_node] 49 | elif current_remaining_depth <= 1: 50 | pruned = [ElipsisAnnotationNode()] 51 | else: 52 | pruned = (e.accept_visitor(self, current_remaining_depth - 1) for e in elements) 53 | 54 | return TupleAnnotationNode(pruned) 55 | 56 | def visit_name_annotation(self, node, current_remaining_depth: int): 57 | return node 58 | 59 | def visit_list_annotation(self, node: ListAnnotationNode, current_remaining_depth: int): 60 | if len(node.elements) > self.__max_list_size: 61 | pruned = node.elements[: self.__max_list_size - 1] + (ElipsisAnnotationNode(),) 62 | 63 | if len(node.elements) == 0: 64 | pruned = node.elements 65 | elif current_remaining_depth <= 1 and len(node.elements) == 1: 66 | pruned = [self.__replacement_node] 67 | elif current_remaining_depth <= 1: 68 | pruned = [ElipsisAnnotationNode()] 69 | else: 70 | pruned = (e.accept_visitor(self, current_remaining_depth - 1) for e in node.elements) 71 | 72 | return ListAnnotationNode(pruned) 73 | 74 | def visit_attribute_annotation( 75 | self, node: AttributeAnnotationNode, current_remaining_depth: int 76 | ): 77 | return node 78 | 79 | def visit_index_annotation(self, node: IndexAnnotationNode, current_remaining_depth: int): 80 | if current_remaining_depth == 0: 81 | return self.__replacement_node 82 | 83 | return IndexAnnotationNode(node.value.accept_visitor(self, current_remaining_depth)) 84 | 85 | def visit_elipsis_annotation(self, node, current_remaining_depth: int): 86 | return node 87 | 88 | def visit_name_constant_annotation(self, node, current_remaining_depth: int): 89 | return node 90 | 91 | def visit_unknown_annotation(self, node, current_remaining_depth: int): 92 | return node 93 | -------------------------------------------------------------------------------- /src/graph_generator/typeparsing/rewriterules/__init__.py: -------------------------------------------------------------------------------- 1 | from .rewriterule import RewriteRule 2 | from .removerecursivegenerics import RemoveRecursiveGenerics 3 | from .removestandalones import RemoveStandAlones 4 | from .removeunionwithanys import RemoveUnionWithAnys 5 | from .removegenericwithany import RemoveGenericWithAnys 6 | -------------------------------------------------------------------------------- /src/graph_generator/typeparsing/rewriterules/removegenericwithany.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from graph_generator.typeparsing.nodes import ( 4 | parse_type_annotation_node, 5 | TypeAnnotationNode, 6 | SubscriptAnnotationNode, 7 | IndexAnnotationNode, 8 | ElipsisAnnotationNode, 9 | TupleAnnotationNode, 10 | ) 11 | from .rewriterule import RewriteRule 12 | 13 | __all__ = ["RemoveGenericWithAnys"] 14 | 15 | 16 | class RemoveGenericWithAnys(RewriteRule): 17 | 18 | ANY_NODE = parse_type_annotation_node("typing.Any") 19 | 20 | def matches(self, node: TypeAnnotationNode, parent: Optional[TypeAnnotationNode]) -> bool: 21 | if not isinstance(node, SubscriptAnnotationNode): 22 | return False 23 | 24 | slice = node.slice 25 | if isinstance(slice, IndexAnnotationNode): 26 | slice = slice.value 27 | if isinstance(slice, ElipsisAnnotationNode): 28 | return True 29 | if isinstance(slice, TupleAnnotationNode): 30 | return all( 31 | s == self.ANY_NODE or isinstance(s, ElipsisAnnotationNode) for s in slice.elements 32 | ) 33 | 34 | return False 35 | 36 | def apply(self, matching_node: TypeAnnotationNode) -> TypeAnnotationNode: 37 | return matching_node.value 38 | -------------------------------------------------------------------------------- /src/graph_generator/typeparsing/rewriterules/removerecursivegenerics.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from graph_generator.typeparsing.nodes import ( 4 | parse_type_annotation_node, 5 | TypeAnnotationNode, 6 | SubscriptAnnotationNode, 7 | IndexAnnotationNode, 8 | TupleAnnotationNode, 9 | ) 10 | from .rewriterule import RewriteRule 11 | 12 | __all__ = ["RemoveRecursiveGenerics"] 13 | 14 | 15 | class RemoveRecursiveGenerics(RewriteRule): 16 | 17 | GENERIC_NODE = parse_type_annotation_node("typing.Generic") 18 | 19 | def matches(self, node: TypeAnnotationNode, parent: Optional[TypeAnnotationNode]) -> bool: 20 | if not isinstance(node, SubscriptAnnotationNode): 21 | return False 22 | if node.value != self.GENERIC_NODE: 23 | return False 24 | 25 | slice = node.slice 26 | if not isinstance(slice, IndexAnnotationNode): 27 | return False 28 | slice = slice.value 29 | 30 | if isinstance(slice, TupleAnnotationNode): 31 | return any( 32 | s == self.GENERIC_NODE 33 | or (isinstance(s, SubscriptAnnotationNode) and s.value == self.GENERIC_NODE) 34 | for s in slice.elements 35 | ) 36 | 37 | return False 38 | 39 | def apply(self, matching_node: TypeAnnotationNode) -> TypeAnnotationNode: 40 | slice = matching_node.slice 41 | if not isinstance(slice, IndexAnnotationNode): 42 | return matching_node 43 | slice = slice.value 44 | 45 | next_slice = set() 46 | for s in slice.elements: 47 | if s == self.GENERIC_NODE: 48 | pass # has no arguments 49 | elif isinstance(s, SubscriptAnnotationNode) and s.value == self.GENERIC_NODE: 50 | if isinstance(s.slice.value, TupleAnnotationNode): 51 | next_slice |= set(s.slice.value.elements) 52 | else: 53 | next_slice.add(s.slice.value) 54 | else: 55 | next_slice.add(s) 56 | return SubscriptAnnotationNode( 57 | matching_node.value, IndexAnnotationNode(TupleAnnotationNode(next_slice)) 58 | ) 59 | -------------------------------------------------------------------------------- /src/graph_generator/typeparsing/rewriterules/removestandalones.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from graph_generator.typeparsing.nodes import ( 4 | parse_type_annotation_node, 5 | TypeAnnotationNode, 6 | SubscriptAnnotationNode, 7 | ) 8 | from .rewriterule import RewriteRule 9 | 10 | __all__ = ["RemoveStandAlones"] 11 | 12 | 13 | class RemoveStandAlones(RewriteRule): 14 | 15 | UNION_NODE = parse_type_annotation_node("typing.Union") 16 | OPTIONAL_NODE = parse_type_annotation_node("typing.Optional") 17 | GENERIC_NODE = parse_type_annotation_node("typing.Generic") 18 | ANY_NODE = parse_type_annotation_node("typing.Any") 19 | 20 | def matches(self, node: TypeAnnotationNode, parent: Optional[TypeAnnotationNode]) -> bool: 21 | if ( 22 | not node == self.UNION_NODE 23 | and not node == self.OPTIONAL_NODE 24 | and not node == self.GENERIC_NODE 25 | ): 26 | return False 27 | return not isinstance(parent, SubscriptAnnotationNode) 28 | 29 | def apply(self, matching_node: TypeAnnotationNode) -> TypeAnnotationNode: 30 | return self.ANY_NODE 31 | -------------------------------------------------------------------------------- /src/graph_generator/typeparsing/rewriterules/removeunionwithanys.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from graph_generator.typeparsing.nodes import ( 4 | parse_type_annotation_node, 5 | TypeAnnotationNode, 6 | SubscriptAnnotationNode, 7 | IndexAnnotationNode, 8 | ElipsisAnnotationNode, 9 | TupleAnnotationNode, 10 | ) 11 | from .rewriterule import RewriteRule 12 | 13 | __all__ = ["RemoveUnionWithAnys"] 14 | 15 | 16 | class RemoveUnionWithAnys(RewriteRule): 17 | 18 | UNION_NODE = parse_type_annotation_node("typing.Union") 19 | ANY_NODE = parse_type_annotation_node("typing.Any") 20 | 21 | def matches(self, node: TypeAnnotationNode, parent: Optional[TypeAnnotationNode]) -> bool: 22 | if not isinstance(node, SubscriptAnnotationNode): 23 | return False 24 | if node.value != self.UNION_NODE: 25 | return False 26 | 27 | slice = node.slice 28 | if isinstance(slice, IndexAnnotationNode): 29 | slice = slice.value 30 | if isinstance(slice, ElipsisAnnotationNode): 31 | return True 32 | if isinstance(slice, TupleAnnotationNode): 33 | return any(s == self.ANY_NODE for s in slice.elements) 34 | 35 | return False 36 | 37 | def apply(self, matching_node: TypeAnnotationNode) -> TypeAnnotationNode: 38 | return self.ANY_NODE 39 | -------------------------------------------------------------------------------- /src/graph_generator/typeparsing/rewriterules/rewriterule.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | from graph_generator.typeparsing.nodes import TypeAnnotationNode 4 | 5 | 6 | class RewriteRule(ABC): 7 | @abstractmethod 8 | def matches(self, node: TypeAnnotationNode, parent: TypeAnnotationNode) -> bool: 9 | pass 10 | 11 | @abstractmethod 12 | def apply(self, matching_node: TypeAnnotationNode) -> TypeAnnotationNode: 13 | pass 14 | -------------------------------------------------------------------------------- /src/graph_generator/typeparsing/rewriterulevisitor.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from .nodes import ( 4 | TypeAnnotationNode, 5 | SubscriptAnnotationNode, 6 | TupleAnnotationNode, 7 | ListAnnotationNode, 8 | AttributeAnnotationNode, 9 | IndexAnnotationNode, 10 | ElipsisAnnotationNode, 11 | ) 12 | from .rewriterules import RewriteRule 13 | from .visitor import TypeAnnotationVisitor 14 | 15 | 16 | class RewriteRuleVisitor(TypeAnnotationVisitor): 17 | """Replace Nodes based on a list of rules.""" 18 | 19 | def __init__(self, rules: List[RewriteRule]): 20 | self.__rules = rules 21 | 22 | def __apply_on_match( 23 | self, original_node: TypeAnnotationNode, parent: TypeAnnotationNode 24 | ) -> TypeAnnotationNode: 25 | for rule in self.__rules: 26 | if rule.matches(original_node, parent): 27 | return rule.apply(original_node) 28 | return original_node 29 | 30 | def visit_subscript_annotation( 31 | self, node: SubscriptAnnotationNode, parent: TypeAnnotationNode 32 | ) -> SubscriptAnnotationNode: 33 | node = SubscriptAnnotationNode( 34 | value=node.value.accept_visitor(self, node), 35 | slice=node.slice.accept_visitor(self, node) if node.slice is not None else None, 36 | ) 37 | return self.__apply_on_match(node, parent) 38 | 39 | def visit_tuple_annotation( 40 | self, node: TupleAnnotationNode, parent: TypeAnnotationNode 41 | ) -> TupleAnnotationNode: 42 | node = TupleAnnotationNode((e.accept_visitor(self, node) for e in node.elements)) 43 | return self.__apply_on_match(node, parent) 44 | 45 | def visit_name_annotation(self, node, parent: TypeAnnotationNode): 46 | return self.__apply_on_match(node, parent) 47 | 48 | def visit_list_annotation(self, node: ListAnnotationNode, parent: TypeAnnotationNode): 49 | node = ListAnnotationNode((e.accept_visitor(self, node) for e in node.elements)) 50 | return self.__apply_on_match(node, parent) 51 | 52 | def visit_attribute_annotation(self, node: AttributeAnnotationNode, parent: TypeAnnotationNode): 53 | node = AttributeAnnotationNode(node.value.accept_visitor(self, node), node.attribute) 54 | return self.__apply_on_match(node, parent) 55 | 56 | def visit_index_annotation(self, node: IndexAnnotationNode, parent: TypeAnnotationNode): 57 | node = IndexAnnotationNode(node.value.accept_visitor(self, node)) 58 | return self.__apply_on_match(node, parent) 59 | 60 | def visit_elipsis_annotation(self, node: ElipsisAnnotationNode, parent: TypeAnnotationNode): 61 | return self.__apply_on_match(node, parent) 62 | 63 | def visit_name_constant_annotation(self, node, parent: TypeAnnotationNode): 64 | return self.__apply_on_match(node, parent) 65 | 66 | def visit_unknown_annotation(self, node, parent: TypeAnnotationNode): 67 | return self.__apply_on_match(node, parent) 68 | -------------------------------------------------------------------------------- /src/graph_generator/typeparsing/visitor.py: -------------------------------------------------------------------------------- 1 | from abc import ABC 2 | from typing import Any 3 | 4 | __all__ = ["TypeAnnotationVisitor"] 5 | 6 | 7 | class TypeAnnotationVisitor(ABC): 8 | def visit_subscript_annotation(self, node, *args) -> Any: 9 | pass 10 | 11 | def visit_tuple_annotation(self, node, *args) -> Any: 12 | pass 13 | 14 | def visit_name_annotation(self, node, *args) -> Any: 15 | pass 16 | 17 | def visit_list_annotation(self, node, *args) -> Any: 18 | pass 19 | 20 | def visit_attribute_annotation(self, node, *args) -> Any: 21 | pass 22 | 23 | def visit_index_annotation(self, node, *args) -> Any: 24 | pass 25 | 26 | def visit_elipsis_annotation(self, node, *args) -> Any: 27 | pass 28 | 29 | def visit_name_constant_annotation(self, node, *args) -> Any: 30 | pass 31 | 32 | def visit_unknown_annotation(self, node, *args) -> Any: 33 | pass 34 | -------------------------------------------------------------------------------- /src/metadata/typingRules.json: -------------------------------------------------------------------------------- 1 | { 2 | "aliasing_rules": [ 3 | [ 4 | "nothing", 5 | "typing.Any" 6 | ], 7 | [ 8 | "typing.Hashable", 9 | "collections.abc.Hashable" 10 | ], 11 | [ 12 | "typing.Sized", 13 | "collections.abc.Sized" 14 | ], 15 | [ 16 | "Optional[typing.Any]", 17 | "typing.Any" 18 | ], 19 | [ 20 | "Generic[typing.Any]", 21 | "typing.Any" 22 | ], 23 | [ 24 | "str", 25 | "typing.Text" 26 | ], 27 | [ 28 | "builtins.globals", 29 | "globals" 30 | ], 31 | [ 32 | "builtins.NotADirectoryError", 33 | "NotADirectoryError" 34 | ], 35 | [ 36 | "builtins.BrokenPipeError", 37 | "BrokenPipeError" 38 | ], 39 | [ 40 | "builtins.SystemError", 41 | "SystemError" 42 | ], 43 | [ 44 | "builtins.ConnectionError", 45 | "ConnectionError" 46 | ], 47 | [ 48 | "builtins.oct", 49 | "oct" 50 | ], 51 | [ 52 | "builtins.any", 53 | "any" 54 | ], 55 | [ 56 | "builtins.ConnectionRefusedError", 57 | "ConnectionRefusedError" 58 | ], 59 | [ 60 | "builtins.list", 61 | "list" 62 | ], 63 | [ 64 | "builtins.enumerate", 65 | "enumerate" 66 | ], 67 | [ 68 | "builtins.OSError", 69 | "OSError" 70 | ], 71 | [ 72 | "builtins.compile", 73 | "compile" 74 | ], 75 | [ 76 | "builtins.ascii", 77 | "ascii" 78 | ], 79 | [ 80 | "builtins.bytearray", 81 | "bytearray" 82 | ], 83 | [ 84 | "builtins.len", 85 | "len" 86 | ], 87 | [ 88 | "builtins.IndexError", 89 | "IndexError" 90 | ], 91 | [ 92 | "builtins.IsADirectoryError", 93 | "IsADirectoryError" 94 | ], 95 | [ 96 | "builtins.min", 97 | "min" 98 | ], 99 | [ 100 | "builtins.repr", 101 | "repr" 102 | ], 103 | [ 104 | "builtins.bytes", 105 | "bytes" 106 | ], 107 | [ 108 | "builtins.slice", 109 | "slice" 110 | ], 111 | [ 112 | "builtins.__debug__", 113 | "__debug__" 114 | ], 115 | [ 116 | "builtins.setattr", 117 | "setattr" 118 | ], 119 | [ 120 | "builtins.range", 121 | "range" 122 | ], 123 | [ 124 | "builtins.Warning", 125 | "Warning" 126 | ], 127 | [ 128 | "builtins.LookupError", 129 | "LookupError" 130 | ], 131 | [ 132 | "builtins.ChildProcessError", 133 | "ChildProcessError" 134 | ], 135 | [ 136 | "builtins.callable", 137 | "callable" 138 | ], 139 | [ 140 | "builtins.UserWarning", 141 | "UserWarning" 142 | ], 143 | [ 144 | "builtins.open", 145 | "open" 146 | ], 147 | [ 148 | "builtins.RuntimeWarning", 149 | "RuntimeWarning" 150 | ], 151 | [ 152 | "builtins.chr", 153 | "chr" 154 | ], 155 | [ 156 | "builtins.copyright", 157 | "copyright" 158 | ], 159 | [ 160 | "builtins.dict", 161 | "dict" 162 | ], 163 | [ 164 | "builtins.int", 165 | "int" 166 | ], 167 | [ 168 | "builtins.quit", 169 | "quit" 170 | ], 171 | [ 172 | "builtins.BytesWarning", 173 | "BytesWarning" 174 | ], 175 | [ 176 | "builtins.exit", 177 | "exit" 178 | ], 179 | [ 180 | "builtins.MemoryError", 181 | "MemoryError" 182 | ], 183 | [ 184 | "builtins.staticmethod", 185 | "staticmethod" 186 | ], 187 | [ 188 | "builtins.isinstance", 189 | "isinstance" 190 | ], 191 | [ 192 | "builtins.tuple", 193 | "tuple" 194 | ], 195 | [ 196 | "typing.NamedTuple", 197 | "tuple" 198 | ], 199 | [ 200 | "typing.NamedTuple", 201 | "collections.namedtuple" 202 | ], 203 | [ 204 | "typing.NamedDict", 205 | "dict" 206 | ], 207 | [ 208 | "builtins.getattr", 209 | "getattr" 210 | ], 211 | [ 212 | "builtins.UnicodeEncodeError", 213 | "UnicodeEncodeError" 214 | ], 215 | [ 216 | "builtins.ValueError", 217 | "ValueError" 218 | ], 219 | [ 220 | "builtins.ord", 221 | "ord" 222 | ], 223 | [ 224 | "builtins.set", 225 | "set" 226 | ], 227 | [ 228 | "builtins.ConnectionAbortedError", 229 | "ConnectionAbortedError" 230 | ], 231 | [ 232 | "builtins.Ellipsis", 233 | "Ellipsis" 234 | ], 235 | [ 236 | "builtins.dir", 237 | "dir" 238 | ], 239 | [ 240 | "builtins.NotImplemented", 241 | "NotImplemented" 242 | ], 243 | [ 244 | "builtins.FloatingPointError", 245 | "FloatingPointError" 246 | ], 247 | [ 248 | "builtins.StopIteration", 249 | "StopIteration" 250 | ], 251 | [ 252 | "builtins.PermissionError", 253 | "PermissionError" 254 | ], 255 | [ 256 | "builtins.__package__", 257 | "__package__" 258 | ], 259 | [ 260 | "builtins.map", 261 | "map" 262 | ], 263 | [ 264 | "builtins.vars", 265 | "vars" 266 | ], 267 | [ 268 | "builtins.EOFError", 269 | "EOFError" 270 | ], 271 | [ 272 | "builtins.IOError", 273 | "IOError" 274 | ], 275 | [ 276 | "builtins.iter", 277 | "iter" 278 | ], 279 | [ 280 | "builtins.UnicodeError", 281 | "UnicodeError" 282 | ], 283 | [ 284 | "builtins.__build_class__", 285 | "__build_class__" 286 | ], 287 | [ 288 | "builtins.issubclass", 289 | "issubclass" 290 | ], 291 | [ 292 | "builtins.float", 293 | "float" 294 | ], 295 | [ 296 | "builtins.sum", 297 | "sum" 298 | ], 299 | [ 300 | "builtins.ImportWarning", 301 | "ImportWarning" 302 | ], 303 | [ 304 | "builtins.locals", 305 | "locals" 306 | ], 307 | [ 308 | "builtins.UnicodeDecodeError", 309 | "UnicodeDecodeError" 310 | ], 311 | [ 312 | "builtins.next", 313 | "next" 314 | ], 315 | [ 316 | "builtins.TypeError", 317 | "TypeError" 318 | ], 319 | [ 320 | "builtins.all", 321 | "all" 322 | ], 323 | [ 324 | "builtins.help", 325 | "help" 326 | ], 327 | [ 328 | "builtins.filter", 329 | "filter" 330 | ], 331 | [ 332 | "builtins.property", 333 | "property" 334 | ], 335 | [ 336 | "builtins.DeprecationWarning", 337 | "DeprecationWarning" 338 | ], 339 | [ 340 | "builtins.max", 341 | "max" 342 | ], 343 | [ 344 | "builtins.BlockingIOError", 345 | "BlockingIOError" 346 | ], 347 | [ 348 | "builtins.SyntaxWarning", 349 | "SyntaxWarning" 350 | ], 351 | [ 352 | "builtins.pow", 353 | "pow" 354 | ], 355 | [ 356 | "builtins.StopAsyncIteration", 357 | "StopAsyncIteration" 358 | ], 359 | [ 360 | "builtins.UnicodeTranslateError", 361 | "UnicodeTranslateError" 362 | ], 363 | [ 364 | "builtins.NotImplementedError", 365 | "NotImplementedError" 366 | ], 367 | [ 368 | "builtins.FutureWarning", 369 | "FutureWarning" 370 | ], 371 | [ 372 | "builtins.credits", 373 | "credits" 374 | ], 375 | [ 376 | "builtins.license", 377 | "license" 378 | ], 379 | [ 380 | "builtins.classmethod", 381 | "classmethod" 382 | ], 383 | [ 384 | "builtins.RecursionError", 385 | "RecursionError" 386 | ], 387 | [ 388 | "builtins.SyntaxError", 389 | "SyntaxError" 390 | ], 391 | [ 392 | "builtins.abs", 393 | "abs" 394 | ], 395 | [ 396 | "builtins.zip", 397 | "zip" 398 | ], 399 | [ 400 | "builtins.GeneratorExit", 401 | "GeneratorExit" 402 | ], 403 | [ 404 | "builtins.__doc__", 405 | "__doc__" 406 | ], 407 | [ 408 | "builtins.Exception", 409 | "Exception" 410 | ], 411 | [ 412 | "builtins.bool", 413 | "bool" 414 | ], 415 | [ 416 | "builtins.__loader__", 417 | "__loader__" 418 | ], 419 | [ 420 | "builtins.id", 421 | "id" 422 | ], 423 | [ 424 | "builtins.hex", 425 | "hex" 426 | ], 427 | [ 428 | "builtins.__spec__", 429 | "__spec__" 430 | ], 431 | [ 432 | "builtins.AssertionError", 433 | "AssertionError" 434 | ], 435 | [ 436 | "builtins.BufferError", 437 | "BufferError" 438 | ], 439 | [ 440 | "builtins.UnicodeWarning", 441 | "UnicodeWarning" 442 | ], 443 | [ 444 | "builtins.exec", 445 | "exec" 446 | ], 447 | [ 448 | "builtins.UnboundLocalError", 449 | "UnboundLocalError" 450 | ], 451 | [ 452 | "builtins.bin", 453 | "bin" 454 | ], 455 | [ 456 | "builtins.str", 457 | "str" 458 | ], 459 | [ 460 | "builtins.RuntimeError", 461 | "RuntimeError" 462 | ], 463 | [ 464 | "builtins.SystemExit", 465 | "SystemExit" 466 | ], 467 | [ 468 | "builtins.__import__", 469 | "__import__" 470 | ], 471 | [ 472 | "builtins.ImportError", 473 | "ImportError" 474 | ], 475 | [ 476 | "builtins.ResourceWarning", 477 | "ResourceWarning" 478 | ], 479 | [ 480 | "builtins.super", 481 | "super" 482 | ], 483 | [ 484 | "builtins.AttributeError", 485 | "AttributeError" 486 | ], 487 | [ 488 | "builtins.hash", 489 | "hash" 490 | ], 491 | [ 492 | "builtins.PendingDeprecationWarning", 493 | "PendingDeprecationWarning" 494 | ], 495 | [ 496 | "builtins.BaseException", 497 | "BaseException" 498 | ], 499 | [ 500 | "builtins.ModuleNotFoundError", 501 | "ModuleNotFoundError" 502 | ], 503 | [ 504 | "builtins.object", 505 | "object" 506 | ], 507 | [ 508 | "builtins.ArithmeticError", 509 | "ArithmeticError" 510 | ], 511 | [ 512 | "builtins.__name__", 513 | "__name__" 514 | ], 515 | [ 516 | "builtins.sorted", 517 | "sorted" 518 | ], 519 | [ 520 | "builtins.EnvironmentError", 521 | "EnvironmentError" 522 | ], 523 | [ 524 | "builtins.InterruptedError", 525 | "InterruptedError" 526 | ], 527 | [ 528 | "builtins.FileNotFoundError", 529 | "FileNotFoundError" 530 | ], 531 | [ 532 | "builtins.ProcessLookupError", 533 | "ProcessLookupError" 534 | ], 535 | [ 536 | "builtins.type", 537 | "type" 538 | ], 539 | [ 540 | "builtins.ReferenceError", 541 | "ReferenceError" 542 | ], 543 | [ 544 | "builtins.TabError", 545 | "TabError" 546 | ], 547 | [ 548 | "builtins.round", 549 | "round" 550 | ], 551 | [ 552 | "builtins.memoryview", 553 | "memoryview" 554 | ], 555 | [ 556 | "builtins.frozenset", 557 | "frozenset" 558 | ], 559 | [ 560 | "builtins.eval", 561 | "eval" 562 | ], 563 | [ 564 | "builtins.format", 565 | "format" 566 | ], 567 | [ 568 | "builtins.FileExistsError", 569 | "FileExistsError" 570 | ], 571 | [ 572 | "builtins.KeyboardInterrupt", 573 | "KeyboardInterrupt" 574 | ], 575 | [ 576 | "builtins.divmod", 577 | "divmod" 578 | ], 579 | [ 580 | "builtins.NameError", 581 | "NameError" 582 | ], 583 | [ 584 | "builtins.hasattr", 585 | "hasattr" 586 | ], 587 | [ 588 | "builtins.print", 589 | "print" 590 | ], 591 | [ 592 | "builtins.ZeroDivisionError", 593 | "ZeroDivisionError" 594 | ], 595 | [ 596 | "builtins.TimeoutError", 597 | "TimeoutError" 598 | ], 599 | [ 600 | "builtins.IndentationError", 601 | "IndentationError" 602 | ], 603 | [ 604 | "builtins.OverflowError", 605 | "OverflowError" 606 | ], 607 | [ 608 | "builtins.ConnectionResetError", 609 | "ConnectionResetError" 610 | ], 611 | [ 612 | "builtins.delattr", 613 | "delattr" 614 | ], 615 | [ 616 | "builtins.reversed", 617 | "reversed" 618 | ], 619 | [ 620 | "builtins.KeyError", 621 | "KeyError" 622 | ], 623 | [ 624 | "builtins.complex", 625 | "complex" 626 | ], 627 | [ 628 | "builtins.input", 629 | "input" 630 | ], 631 | [ 632 | "__builtin__.globals", 633 | "globals" 634 | ], 635 | [ 636 | "__builtin__.NotADirectoryError", 637 | "NotADirectoryError" 638 | ], 639 | [ 640 | "__builtin__.BrokenPipeError", 641 | "BrokenPipeError" 642 | ], 643 | [ 644 | "__builtin__.SystemError", 645 | "SystemError" 646 | ], 647 | [ 648 | "__builtin__.ConnectionError", 649 | "ConnectionError" 650 | ], 651 | [ 652 | "__builtin__.oct", 653 | "oct" 654 | ], 655 | [ 656 | "__builtin__.any", 657 | "any" 658 | ], 659 | [ 660 | "__builtin__.ConnectionRefusedError", 661 | "ConnectionRefusedError" 662 | ], 663 | [ 664 | "__builtin__.list", 665 | "list" 666 | ], 667 | [ 668 | "__builtin__.enumerate", 669 | "enumerate" 670 | ], 671 | [ 672 | "__builtin__.OSError", 673 | "OSError" 674 | ], 675 | [ 676 | "__builtin__.compile", 677 | "compile" 678 | ], 679 | [ 680 | "__builtin__.ascii", 681 | "ascii" 682 | ], 683 | [ 684 | "__builtin__.bytearray", 685 | "bytearray" 686 | ], 687 | [ 688 | "__builtin__.len", 689 | "len" 690 | ], 691 | [ 692 | "__builtin__.IndexError", 693 | "IndexError" 694 | ], 695 | [ 696 | "__builtin__.IsADirectoryError", 697 | "IsADirectoryError" 698 | ], 699 | [ 700 | "__builtin__.min", 701 | "min" 702 | ], 703 | [ 704 | "__builtin__.repr", 705 | "repr" 706 | ], 707 | [ 708 | "__builtin__.bytes", 709 | "bytes" 710 | ], 711 | [ 712 | "__builtin__.slice", 713 | "slice" 714 | ], 715 | [ 716 | "__builtin__.__debug__", 717 | "__debug__" 718 | ], 719 | [ 720 | "__builtin__.setattr", 721 | "setattr" 722 | ], 723 | [ 724 | "__builtin__.range", 725 | "range" 726 | ], 727 | [ 728 | "__builtin__.Warning", 729 | "Warning" 730 | ], 731 | [ 732 | "__builtin__.LookupError", 733 | "LookupError" 734 | ], 735 | [ 736 | "__builtin__.ChildProcessError", 737 | "ChildProcessError" 738 | ], 739 | [ 740 | "__builtin__.callable", 741 | "callable" 742 | ], 743 | [ 744 | "__builtin__.UserWarning", 745 | "UserWarning" 746 | ], 747 | [ 748 | "__builtin__.open", 749 | "open" 750 | ], 751 | [ 752 | "__builtin__.RuntimeWarning", 753 | "RuntimeWarning" 754 | ], 755 | [ 756 | "__builtin__.chr", 757 | "chr" 758 | ], 759 | [ 760 | "__builtin__.copyright", 761 | "copyright" 762 | ], 763 | [ 764 | "__builtin__.dict", 765 | "dict" 766 | ], 767 | [ 768 | "__builtin__.int", 769 | "int" 770 | ], 771 | [ 772 | "__builtin__.quit", 773 | "quit" 774 | ], 775 | [ 776 | "__builtin__.BytesWarning", 777 | "BytesWarning" 778 | ], 779 | [ 780 | "__builtin__.exit", 781 | "exit" 782 | ], 783 | [ 784 | "__builtin__.MemoryError", 785 | "MemoryError" 786 | ], 787 | [ 788 | "__builtin__.staticmethod", 789 | "staticmethod" 790 | ], 791 | [ 792 | "__builtin__.isinstance", 793 | "isinstance" 794 | ], 795 | [ 796 | "__builtin__.tuple", 797 | "tuple" 798 | ], 799 | [ 800 | "__builtin__.getattr", 801 | "getattr" 802 | ], 803 | [ 804 | "__builtin__.UnicodeEncodeError", 805 | "UnicodeEncodeError" 806 | ], 807 | [ 808 | "__builtin__.ValueError", 809 | "ValueError" 810 | ], 811 | [ 812 | "__builtin__.ord", 813 | "ord" 814 | ], 815 | [ 816 | "__builtin__.set", 817 | "set" 818 | ], 819 | [ 820 | "__builtin__.ConnectionAbortedError", 821 | "ConnectionAbortedError" 822 | ], 823 | [ 824 | "__builtin__.Ellipsis", 825 | "Ellipsis" 826 | ], 827 | [ 828 | "__builtin__.dir", 829 | "dir" 830 | ], 831 | [ 832 | "__builtin__.NotImplemented", 833 | "NotImplemented" 834 | ], 835 | [ 836 | "__builtin__.FloatingPointError", 837 | "FloatingPointError" 838 | ], 839 | [ 840 | "__builtin__.StopIteration", 841 | "StopIteration" 842 | ], 843 | [ 844 | "__builtin__.PermissionError", 845 | "PermissionError" 846 | ], 847 | [ 848 | "__builtin__.__package__", 849 | "__package__" 850 | ], 851 | [ 852 | "__builtin__.map", 853 | "map" 854 | ], 855 | [ 856 | "__builtin__.vars", 857 | "vars" 858 | ], 859 | [ 860 | "__builtin__.EOFError", 861 | "EOFError" 862 | ], 863 | [ 864 | "__builtin__.IOError", 865 | "IOError" 866 | ], 867 | [ 868 | "__builtin__.iter", 869 | "iter" 870 | ], 871 | [ 872 | "__builtin__.UnicodeError", 873 | "UnicodeError" 874 | ], 875 | [ 876 | "__builtin__.__build_class__", 877 | "__build_class__" 878 | ], 879 | [ 880 | "__builtin__.issubclass", 881 | "issubclass" 882 | ], 883 | [ 884 | "__builtin__.float", 885 | "float" 886 | ], 887 | [ 888 | "__builtin__.sum", 889 | "sum" 890 | ], 891 | [ 892 | "__builtin__.ImportWarning", 893 | "ImportWarning" 894 | ], 895 | [ 896 | "__builtin__.locals", 897 | "locals" 898 | ], 899 | [ 900 | "__builtin__.UnicodeDecodeError", 901 | "UnicodeDecodeError" 902 | ], 903 | [ 904 | "__builtin__.next", 905 | "next" 906 | ], 907 | [ 908 | "__builtin__.TypeError", 909 | "TypeError" 910 | ], 911 | [ 912 | "__builtin__.all", 913 | "all" 914 | ], 915 | [ 916 | "__builtin__.help", 917 | "help" 918 | ], 919 | [ 920 | "__builtin__.filter", 921 | "filter" 922 | ], 923 | [ 924 | "__builtin__.property", 925 | "property" 926 | ], 927 | [ 928 | "__builtin__.DeprecationWarning", 929 | "DeprecationWarning" 930 | ], 931 | [ 932 | "__builtin__.max", 933 | "max" 934 | ], 935 | [ 936 | "__builtin__.BlockingIOError", 937 | "BlockingIOError" 938 | ], 939 | [ 940 | "__builtin__.SyntaxWarning", 941 | "SyntaxWarning" 942 | ], 943 | [ 944 | "__builtin__.pow", 945 | "pow" 946 | ], 947 | [ 948 | "__builtin__.StopAsyncIteration", 949 | "StopAsyncIteration" 950 | ], 951 | [ 952 | "__builtin__.UnicodeTranslateError", 953 | "UnicodeTranslateError" 954 | ], 955 | [ 956 | "__builtin__.NotImplementedError", 957 | "NotImplementedError" 958 | ], 959 | [ 960 | "__builtin__.FutureWarning", 961 | "FutureWarning" 962 | ], 963 | [ 964 | "__builtin__.credits", 965 | "credits" 966 | ], 967 | [ 968 | "__builtin__.license", 969 | "license" 970 | ], 971 | [ 972 | "__builtin__.classmethod", 973 | "classmethod" 974 | ], 975 | [ 976 | "__builtin__.RecursionError", 977 | "RecursionError" 978 | ], 979 | [ 980 | "__builtin__.SyntaxError", 981 | "SyntaxError" 982 | ], 983 | [ 984 | "__builtin__.abs", 985 | "abs" 986 | ], 987 | [ 988 | "__builtin__.zip", 989 | "zip" 990 | ], 991 | [ 992 | "__builtin__.GeneratorExit", 993 | "GeneratorExit" 994 | ], 995 | [ 996 | "__builtin__.__doc__", 997 | "__doc__" 998 | ], 999 | [ 1000 | "__builtin__.Exception", 1001 | "Exception" 1002 | ], 1003 | [ 1004 | "__builtin__.bool", 1005 | "bool" 1006 | ], 1007 | [ 1008 | "__builtin__.__loader__", 1009 | "__loader__" 1010 | ], 1011 | [ 1012 | "__builtin__.id", 1013 | "id" 1014 | ], 1015 | [ 1016 | "__builtin__.hex", 1017 | "hex" 1018 | ], 1019 | [ 1020 | "__builtin__.__spec__", 1021 | "__spec__" 1022 | ], 1023 | [ 1024 | "__builtin__.AssertionError", 1025 | "AssertionError" 1026 | ], 1027 | [ 1028 | "__builtin__.BufferError", 1029 | "BufferError" 1030 | ], 1031 | [ 1032 | "__builtin__.UnicodeWarning", 1033 | "UnicodeWarning" 1034 | ], 1035 | [ 1036 | "__builtin__.exec", 1037 | "exec" 1038 | ], 1039 | [ 1040 | "__builtin__.UnboundLocalError", 1041 | "UnboundLocalError" 1042 | ], 1043 | [ 1044 | "__builtin__.bin", 1045 | "bin" 1046 | ], 1047 | [ 1048 | "__builtin__.str", 1049 | "str" 1050 | ], 1051 | [ 1052 | "__builtin__.RuntimeError", 1053 | "RuntimeError" 1054 | ], 1055 | [ 1056 | "__builtin__.SystemExit", 1057 | "SystemExit" 1058 | ], 1059 | [ 1060 | "__builtin__.__import__", 1061 | "__import__" 1062 | ], 1063 | [ 1064 | "__builtin__.ImportError", 1065 | "ImportError" 1066 | ], 1067 | [ 1068 | "__builtin__.ResourceWarning", 1069 | "ResourceWarning" 1070 | ], 1071 | [ 1072 | "__builtin__.super", 1073 | "super" 1074 | ], 1075 | [ 1076 | "__builtin__.AttributeError", 1077 | "AttributeError" 1078 | ], 1079 | [ 1080 | "__builtin__.hash", 1081 | "hash" 1082 | ], 1083 | [ 1084 | "__builtin__.PendingDeprecationWarning", 1085 | "PendingDeprecationWarning" 1086 | ], 1087 | [ 1088 | "__builtin__.BaseException", 1089 | "BaseException" 1090 | ], 1091 | [ 1092 | "__builtin__.ModuleNotFoundError", 1093 | "ModuleNotFoundError" 1094 | ], 1095 | [ 1096 | "__builtin__.object", 1097 | "object" 1098 | ], 1099 | [ 1100 | "__builtin__.ArithmeticError", 1101 | "ArithmeticError" 1102 | ], 1103 | [ 1104 | "__builtin__.__name__", 1105 | "__name__" 1106 | ], 1107 | [ 1108 | "__builtin__.sorted", 1109 | "sorted" 1110 | ], 1111 | [ 1112 | "__builtin__.EnvironmentError", 1113 | "EnvironmentError" 1114 | ], 1115 | [ 1116 | "__builtin__.InterruptedError", 1117 | "InterruptedError" 1118 | ], 1119 | [ 1120 | "__builtin__.FileNotFoundError", 1121 | "FileNotFoundError" 1122 | ], 1123 | [ 1124 | "__builtin__.ProcessLookupError", 1125 | "ProcessLookupError" 1126 | ], 1127 | [ 1128 | "__builtin__.type", 1129 | "type" 1130 | ], 1131 | [ 1132 | "__builtin__.ReferenceError", 1133 | "ReferenceError" 1134 | ], 1135 | [ 1136 | "__builtin__.TabError", 1137 | "TabError" 1138 | ], 1139 | [ 1140 | "__builtin__.round", 1141 | "round" 1142 | ], 1143 | [ 1144 | "__builtin__.memoryview", 1145 | "memoryview" 1146 | ], 1147 | [ 1148 | "__builtin__.frozenset", 1149 | "frozenset" 1150 | ], 1151 | [ 1152 | "__builtin__.eval", 1153 | "eval" 1154 | ], 1155 | [ 1156 | "__builtin__.format", 1157 | "format" 1158 | ], 1159 | [ 1160 | "__builtin__.FileExistsError", 1161 | "FileExistsError" 1162 | ], 1163 | [ 1164 | "__builtin__.KeyboardInterrupt", 1165 | "KeyboardInterrupt" 1166 | ], 1167 | [ 1168 | "__builtin__.divmod", 1169 | "divmod" 1170 | ], 1171 | [ 1172 | "__builtin__.NameError", 1173 | "NameError" 1174 | ], 1175 | [ 1176 | "__builtin__.hasattr", 1177 | "hasattr" 1178 | ], 1179 | [ 1180 | "__builtin__.print", 1181 | "print" 1182 | ], 1183 | [ 1184 | "__builtin__.ZeroDivisionError", 1185 | "ZeroDivisionError" 1186 | ], 1187 | [ 1188 | "__builtin__.TimeoutError", 1189 | "TimeoutError" 1190 | ], 1191 | [ 1192 | "__builtin__.IndentationError", 1193 | "IndentationError" 1194 | ], 1195 | [ 1196 | "__builtin__.OverflowError", 1197 | "OverflowError" 1198 | ], 1199 | [ 1200 | "__builtin__.ConnectionResetError", 1201 | "ConnectionResetError" 1202 | ], 1203 | [ 1204 | "__builtin__.delattr", 1205 | "delattr" 1206 | ], 1207 | [ 1208 | "__builtin__.reversed", 1209 | "reversed" 1210 | ], 1211 | [ 1212 | "__builtin__.KeyError", 1213 | "KeyError" 1214 | ], 1215 | [ 1216 | "__builtin__.complex", 1217 | "complex" 1218 | ], 1219 | [ 1220 | "__builtin__.input", 1221 | "input" 1222 | ] 1223 | ], 1224 | "is_a_relations": [ 1225 | [ 1226 | "typing.Type", 1227 | "typing.Generic" 1228 | ], 1229 | [ 1230 | "typing.Iterable", 1231 | "typing.Generic" 1232 | ], 1233 | [ 1234 | "typing.Iterable", 1235 | "collections.abc.Iterable" 1236 | ], 1237 | [ 1238 | "typing.Tuple", 1239 | "typing.Generic" 1240 | ], 1241 | [ 1242 | "typing.Iterator", 1243 | "typing.Iterable" 1244 | ], 1245 | [ 1246 | "typing.Iterator", 1247 | "collections.abc.Iterator" 1248 | ], 1249 | [ 1250 | "typing.NamedTuple", 1251 | "collections.namedtuple" 1252 | ], 1253 | [ 1254 | "typing.Reversible", 1255 | "typing.Iterable" 1256 | ], 1257 | [ 1258 | "typing.Reversible", 1259 | "collections.abc.Reversible" 1260 | ], 1261 | [ 1262 | "typing.Container", 1263 | "typing.Generic" 1264 | ], 1265 | [ 1266 | "typing.Container", 1267 | "collections.abc.Container" 1268 | ], 1269 | [ 1270 | "typing.Callable", 1271 | "typing.Generic" 1272 | ], 1273 | [ 1274 | "typing.Callable", 1275 | "collections.abc.Callable" 1276 | ], 1277 | [ 1278 | "typing.Collection", 1279 | "typing.Sized" 1280 | ], 1281 | [ 1282 | "typing.Collection", 1283 | "typing.Iterable" 1284 | ], 1285 | [ 1286 | "typing.Collection", 1287 | "typing.Container" 1288 | ], 1289 | [ 1290 | "typing.Collection", 1291 | "collections.abc.Collection" 1292 | ], 1293 | [ 1294 | "typing.AbstractSet", 1295 | "collection.abc.AbstractSet" 1296 | ], 1297 | [ 1298 | "typing.AbstractSet", 1299 | "typing.Collection" 1300 | ], 1301 | [ 1302 | "typing.AbstractSet", 1303 | "typing.Sized" 1304 | ], 1305 | [ 1306 | "typing.MutableSet", 1307 | "typing.AbstractSet" 1308 | ], 1309 | [ 1310 | "typing.Mapping", 1311 | "typing.Sized" 1312 | ], 1313 | [ 1314 | "typing.Mapping", 1315 | "typing.Collection" 1316 | ], 1317 | [ 1318 | "typing.Mapping", 1319 | "typing.Generic" 1320 | ], 1321 | [ 1322 | "typing.Mapping", 1323 | "collections.abc.Mapping" 1324 | ], 1325 | [ 1326 | "typing.MutableMapping", 1327 | "typing.Mapping" 1328 | ], 1329 | [ 1330 | "typing.MutableMapping", 1331 | "collections.abc.MutableMapping" 1332 | ], 1333 | [ 1334 | "typing.Sequence", 1335 | "typing.Reversible" 1336 | ], 1337 | [ 1338 | "typing.Sequence", 1339 | "collections.abc.Sequence" 1340 | ], 1341 | [ 1342 | "typing.Sequence", 1343 | "typing.Collection" 1344 | ], 1345 | [ 1346 | "typing.MutableSequence", 1347 | "typing.Sequence" 1348 | ], 1349 | [ 1350 | "typing.MutableSequence", 1351 | "collections.abc.MutableSequence" 1352 | ], 1353 | [ 1354 | "typing.ByteString", 1355 | "typing.Sequence[int]" 1356 | ], 1357 | [ 1358 | "typing.ByteString", 1359 | "collections.abc.ByteString" 1360 | ], 1361 | [ 1362 | "typing.Deque", 1363 | "deque" 1364 | ], 1365 | [ 1366 | "typing.Deque", 1367 | "typing.MutableSequence" 1368 | ], 1369 | [ 1370 | "typing.List", 1371 | "list" 1372 | ], 1373 | [ 1374 | "typing.List", 1375 | "typing.MutableSequence" 1376 | ], 1377 | [ 1378 | "typing.Set", 1379 | "set" 1380 | ], 1381 | [ 1382 | "typing.Set", 1383 | "typing.MutableSet" 1384 | ], 1385 | [ 1386 | "typing.Set", 1387 | "collections.abc.Set" 1388 | ], 1389 | [ 1390 | "typing.MutableSet", 1391 | "typing.AbstractSet" 1392 | ], 1393 | [ 1394 | "typing.FrozenSet", 1395 | "frozenset" 1396 | ], 1397 | [ 1398 | "typing.FrozenSet", 1399 | "typing.AbstractSet" 1400 | ], 1401 | [ 1402 | "typing.MappingView", 1403 | "typing.Sized" 1404 | ], 1405 | [ 1406 | "typing.MappingView", 1407 | "typing.Iterable" 1408 | ], 1409 | [ 1410 | "typing.KeysView", 1411 | "typing.MappingView" 1412 | ], 1413 | [ 1414 | "typing.KeysView", 1415 | "typing.AbstractSet" 1416 | ], 1417 | [ 1418 | "typing.ItemsView", 1419 | "typing.MappingView" 1420 | ], 1421 | [ 1422 | "typing.ItemsView", 1423 | "typing.Generic" 1424 | ], 1425 | [ 1426 | "typing.ValuesView", 1427 | "typing.MappingView" 1428 | ], 1429 | [ 1430 | "typing.Awaitable", 1431 | "typing.Generic" 1432 | ], 1433 | [ 1434 | "typing.Awaitable", 1435 | "collections.abc.Awaitable" 1436 | ], 1437 | [ 1438 | "typing.Coroutine", 1439 | "typing.Awaitable" 1440 | ], 1441 | [ 1442 | "typing.Coroutine", 1443 | "typing.Generic" 1444 | ], 1445 | [ 1446 | "typing.AsyncIterable", 1447 | "typing.Generic" 1448 | ], 1449 | [ 1450 | "typing.AsyncIterable", 1451 | "collections.abc.AsyncIterable" 1452 | ], 1453 | [ 1454 | "typing.AsyncIterator", 1455 | "typing.AsyncIterable" 1456 | ], 1457 | [ 1458 | "typing.AsyncIterator", 1459 | "collections.abc.AsyncIterable" 1460 | ], 1461 | [ 1462 | "typing.ContextManager", 1463 | "typing.Generic" 1464 | ], 1465 | [ 1466 | "typing.ContextManager", 1467 | "contextlib.AbstractContextManager" 1468 | ], 1469 | [ 1470 | "typing.AsyncContextManager", 1471 | "typing.Generic" 1472 | ], 1473 | [ 1474 | "typing.AsyncContextManager", 1475 | "contextlib.AbstractAsyncContextManager" 1476 | ], 1477 | [ 1478 | "typing.Dict", 1479 | "dict" 1480 | ], 1481 | [ 1482 | "typing.Dict", 1483 | "typing.MutableMapping" 1484 | ], 1485 | [ 1486 | "typing.DefaultDict", 1487 | "collections.defaultdict" 1488 | ], 1489 | [ 1490 | "typing.DefaultDict", 1491 | "typing.MutableMapping" 1492 | ], 1493 | [ 1494 | "typing.OrderedDict", 1495 | "collections.OrderedDict" 1496 | ], 1497 | [ 1498 | "typing.OrderedDict", 1499 | "typing.MutableMapping" 1500 | ], 1501 | [ 1502 | "typing.Counter", 1503 | "collections.Counter" 1504 | ], 1505 | [ 1506 | "typing.Counter", 1507 | "typing.Dict" 1508 | ], 1509 | [ 1510 | "typing.ChainMap", 1511 | "collections.ChainMap" 1512 | ], 1513 | [ 1514 | "typing.ChainMap", 1515 | "typing.MutableMapping" 1516 | ], 1517 | [ 1518 | "typing.Generator", 1519 | "typing.Iterator" 1520 | ], 1521 | [ 1522 | "typing.Generator", 1523 | "collections.abc.Generator" 1524 | ], 1525 | [ 1526 | "typing.Generator", 1527 | "typing.Generic" 1528 | ], 1529 | [ 1530 | "typing.AsyncGenerator", 1531 | "typing.AsyncIterator" 1532 | ], 1533 | [ 1534 | "typing.AsyncGenerator", 1535 | "typing.Generic" 1536 | ] 1537 | ] 1538 | } 1539 | --------------------------------------------------------------------------------