├── .github └── workflows │ └── CI.yml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── docs ├── Makefile ├── make.bat └── src │ ├── 404.rst │ ├── conf.py │ ├── gallery.rst │ ├── guide.rst │ ├── index.rst │ ├── reference.rst │ └── release-notes.rst ├── pyfactor ├── VERSION ├── __init__.py ├── _cli.py ├── _graph.py ├── _gv.py ├── _io.py └── _visit │ ├── __init__.py │ └── base.py ├── readme-pypi.rst ├── readme.rst ├── readthedocs.yml ├── setup.py ├── tests ├── __init__.py ├── cli.py └── parsing │ ├── __init__.py │ ├── _util.py │ ├── assign.py │ ├── docs.py │ ├── flow.py │ ├── import.py │ └── scoped.py └── tox.ini /.github/workflows/CI.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | push: 4 | branches: 5 | - master 6 | tags-ignore: 7 | - '*' 8 | pull_request: 9 | 10 | jobs: 11 | matrix: 12 | strategy: 13 | matrix: 14 | python-version: [3.6, 3.7, 3.9] 15 | name: Pytest on ${{matrix.python-version}} 16 | runs-on: ubuntu-latest 17 | 18 | steps: 19 | - uses: actions/checkout@v2 20 | - name: Set up Python ${{matrix.python-version}} 21 | uses: actions/setup-python@v1 22 | with: 23 | python-version: ${{matrix.python-version}} 24 | - name: Install package 25 | run: | 26 | python -m pip install --upgrade pip 27 | pip install .[tests] 28 | - name: Run test suite 29 | run: pytest 30 | 31 | full-build: 32 | name: Full 3.8 build 33 | runs-on: ubuntu-20.04 34 | steps: 35 | - uses: actions/checkout@v2 36 | - name: Set up Python 37 | uses: actions/setup-python@v1 38 | with: 39 | python-version: 3.8 40 | - name: Install package 41 | run: | 42 | sudo apt install graphviz 43 | sudo dot -c 44 | python -m pip install --upgrade pip 45 | pip install .[dev] 46 | - name: Run tox environments 47 | run: tox 48 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Package specific 2 | docs/src/gallery/ 3 | 4 | # Python 5 | *.py[cod] 6 | **__pycache__/ 7 | build/ 8 | dist/ 9 | *.egg-info/ 10 | 11 | # Editors 12 | .idea/ 13 | .vim/ 14 | 15 | # Tools 16 | .coverage 17 | coverage.xml 18 | docs/build 19 | .tox/ 20 | 21 | # Environments 22 | .env 23 | .venv 24 | env/ 25 | venv/ 26 | env.bak/ 27 | venv.bak/ 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Felix Hildén 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include readme-pypi.rst 2 | include pyfactor/VERSION 3 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = src 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=src 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/src/404.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Pyfactor 3 | ======== 4 | Oops! The page you are looking for was not found. 5 | Maybe you'll find what you're looking for by searching the documentation 6 | or returning to the `home page `_. 7 | 8 | .. _rtd: https://pyfactor.rtfd.org 9 | -------------------------------------------------------------------------------- /docs/src/conf.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import textwrap 4 | import subprocess 5 | 6 | import requests 7 | import pyfactor 8 | from pathlib import Path 9 | 10 | _root = Path(os.path.realpath(__file__)).parent.parent.parent 11 | sys.path.insert(0, _root) 12 | 13 | project = 'pyfactor' 14 | author = 'Felix Hildén' 15 | copyright = '2020, Felix Hildén' 16 | release = Path(_root, 'pyfactor', 'VERSION').read_text().strip() 17 | 18 | extensions = [ 19 | 'sphinx.ext.autodoc', 20 | 'sphinx.ext.autosummary', 21 | 'sphinx.ext.extlinks', 22 | 'sphinx.ext.napoleon', 23 | 'sphinx_autodoc_typehints', 24 | 'sphinx_rtd_theme', 25 | 'sphinxarg.ext', 26 | ] 27 | 28 | master_doc = 'index' 29 | exclude_patterns = ['build'] 30 | autosummary_generate = True 31 | html_theme = 'sphinx_rtd_theme' 32 | 33 | extlinks = { 34 | 'issue': ('https://github.com/felix-hilden/pyfactor/issues/%s', '#'), 35 | } 36 | 37 | # ----- Generate gallery entries ----- 38 | gallery_path = _root / 'docs' / 'src' / 'gallery' 39 | public_examples = [ 40 | 'felix-hilden/pyfactor/522f3ee5/pyfactor/_parse.py', 41 | 'pydot/pydot/5c9b2ce7/pydot.py', 42 | 'PyCQA/flake8/e0116d8e/src/flake8/style_guide.py', 43 | 'psf/black/c702588d/src/black/__init__.py', 44 | 'agronholm/sphinx-autodoc-typehints/49face65/sphinx_autodoc_typehints.py', 45 | 'pytest-dev/pytest/0061ec55/src/_pytest/python.py', 46 | ] 47 | builtin_examples = ['concurrent', 'json', 'importlib'] 48 | 49 | 50 | def public_doc(name: str, url: str) -> str: 51 | """Generate public module gallery docs from template.""" 52 | summary = f'This example was generated from `{name} source <{url}>`_' 53 | summary = '\n'.join(textwrap.wrap(summary, width=79)) 54 | return f""".. _gallery-{name}: 55 | 56 | {name} 57 | {'=' * len(name)} 58 | {summary} 59 | with :code:`pyfactor source.py --skip-external`. 60 | Click the image to enlarge. 61 | 62 | .. image:: {name}.svg 63 | :target: ../_images/{name}.svg 64 | :alt: {name} visualisation 65 | """ 66 | 67 | 68 | def builtin_doc(name: str) -> str: 69 | """Generate builtin module gallery docs from template.""" 70 | summary = f'This example was generated from the builtin ``{name}`` module' 71 | summary = '\n'.join(textwrap.wrap(summary, width=79)) 72 | return f""".. _gallery-{name}: 73 | 74 | {name} 75 | {'=' * len(name)} 76 | {summary} 77 | with :code:`pyfactor {name} --skip-external`. 78 | Click the image to enlarge. 79 | 80 | .. image:: {name}.svg 81 | :target: ../_images/{name}.svg 82 | :alt: {name} visualisation 83 | """ 84 | 85 | 86 | # Hack for RTD: install Graphviz 87 | if os.environ.get('PYFACTOR_RTD_BUILD', False): 88 | install_proc = subprocess.run(['apt', 'install', 'graphviz']) 89 | setup_proc = subprocess.run(['dot', '-c']) 90 | 91 | # Generate legend 92 | legend_path = gallery_path / 'legend' 93 | pyfactor.legend(str(legend_path), {'chain': 2}, {'format': 'svg'}) 94 | 95 | # Generate examples 96 | gallery_path.mkdir(exist_ok=True) 97 | parse_kwargs = {'skip_external': True} 98 | preprocess_kwargs = {'stagger': 10, 'fanout': True, 'chain': 5} 99 | render_kwargs = {'format': 'svg'} 100 | 101 | for example in public_examples: 102 | print('Generating gallery example:', example) 103 | raw_url = 'https://raw.githubusercontent.com/' + example 104 | url_parts = example.split('/') 105 | url_parts.insert(2, 'blob') 106 | ui_url = 'https://github.com/' + '/'.join(url_parts) 107 | repository_name = url_parts[1] 108 | 109 | source_text = requests.get(raw_url).text 110 | doc_text = public_doc(repository_name, ui_url) 111 | 112 | source_path = gallery_path / (repository_name + '.py') 113 | doc_path = source_path.with_suffix('.rst') 114 | image_path = source_path.with_suffix('') 115 | source_path = source_path.with_name(repository_name + '_example.py') 116 | 117 | source_path.write_text(source_text, encoding='utf-8') 118 | doc_path.write_text(doc_text, encoding='utf-8') 119 | 120 | pyfactor.pyfactor( 121 | [str(source_path)], None, str(image_path), 122 | parse_kwargs=parse_kwargs, 123 | preprocess_kwargs=preprocess_kwargs, 124 | render_kwargs=render_kwargs, 125 | ) 126 | 127 | for example in builtin_examples: 128 | print('Generating gallery example:', example) 129 | doc_text = builtin_doc(example) 130 | image_path = gallery_path / example 131 | doc_path = image_path.with_suffix('.rst') 132 | doc_path.write_text(doc_text, encoding='utf-8') 133 | 134 | pyfactor.pyfactor( 135 | [example], None, str(image_path), 136 | parse_kwargs=parse_kwargs, 137 | preprocess_kwargs=preprocess_kwargs, 138 | render_kwargs=render_kwargs, 139 | ) 140 | -------------------------------------------------------------------------------- /docs/src/gallery.rst: -------------------------------------------------------------------------------- 1 | .. _gallery: 2 | 3 | Gallery 4 | ======= 5 | This gallery contains example visualisations 6 | of builtin modules and public libraries. 7 | Note that because the public library examples refer to specific Git commits, 8 | they may be outdated. 9 | 10 | .. toctree:: 11 | :caption: Contents 12 | :glob: 13 | 14 | gallery/* 15 | 16 | Legend 17 | ------ 18 | Legend information is available in the image below (click to enlarge). 19 | 20 | .. image:: gallery/legend.svg 21 | :target: _images/legend.svg 22 | :alt: legend visualisation 23 | 24 | Nodes represent different types of source objects. 25 | Edges represent dependencies. 26 | The node from which the arrow starts 27 | depends on the node that the arrow head points to. 28 | 29 | In addition to type and connectivity information the nodes contain 30 | a line number indicating the location of the definition. 31 | Multiple line numbers are given if the name has multiple definitions. 32 | A single node can also be colored with two colors, 33 | indicating for example a central leaf node. 34 | 35 | Nodes are divided into subgraphs separated with bounding rectangles 36 | according to their source module. 37 | 38 | .. note:: 39 | 40 | Docstrings are provided as tooltips: hover over nodes of the SVG image 41 | to view the tooltip. 42 | 43 | Node shapes 44 | *********** 45 | - Unknown: node type unknown for some reason 46 | - Multiple: there are multiple definitions with different types for a name 47 | 48 | Node colours 49 | ************ 50 | - Centrality: the number of connections that a given node has, 51 | deeper red indicates an increased centrality 52 | - Waypoint: a node whose children can only be reached from its parents 53 | via that node 54 | - Collapsed: waypoint with its child nodes collapsed (see CLI options) 55 | - Leaf: has no child nodes 56 | - Root: has no parent nodes 57 | - Isolated: has no dependencies 58 | 59 | Edge styles 60 | *********** 61 | - Bridge: a dependency that when removed, would break the graph into pieces 62 | - Import: import referencing a node in a different module 63 | -------------------------------------------------------------------------------- /docs/src/guide.rst: -------------------------------------------------------------------------------- 1 | .. _guide: 2 | 3 | Guide 4 | ===== 5 | Here are some tips and tricks to using *Pyfactor*. 6 | 7 | Many configuration parameters are dedicated to managing 8 | the amount of information in the graph. 9 | While sometimes having extra information is useful, 10 | particularly with lengthy files, nested modules and many imports 11 | the graph structure can become messy. 12 | 13 | Controlling imports 14 | ------------------- 15 | Skipping external imports with ``--skip-external`` is likely the first useful 16 | reduction of detail that can greatly simplify the visualisation. 17 | Often tracking imports to external modules is not essential. 18 | 19 | With lots of references to only a few import targets, 20 | duplicating imports with ``--imports duplicate`` might consolidate imports 21 | before referencing the original sources, which reduces inter-module edges. 22 | Conversely if there are less references per import, resolving the nodes 23 | with ``--imports resolve`` can reduce the number of redundant nodes. 24 | 25 | Affecting specific nodes 26 | ------------------------ 27 | Sometimes very busy nodes can be a distraction to the overall graph. 28 | They can be manually excluded from the visualisation with ``--exclude``. 29 | If instead a part of the graph is particularly interesting, 30 | a node can be set as the graph root with ``--root``. 31 | -------------------------------------------------------------------------------- /docs/src/index.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Pyfactor 3 | ======== 4 | Welcome to the documentation of *Pyfactor* - a refactoring tool that visualises 5 | Python source files, modules and importable packages as a graph of dependencies 6 | between Python constructs like variables, functions and classes. 7 | 8 | .. code:: sh 9 | 10 | $ pyfactor --help 11 | $ pyfactor script.py 12 | $ pyfactor script.py --skip-external --view 13 | 14 | See our `PyPI`_ page for installation instructions and package information. 15 | If you've found a bug or would like to propose a feature, 16 | please submit an issue on `GitHub`_. 17 | 18 | For a glimpse into what is possible, here's a graph of our parsing module: 19 | 20 | .. image:: gallery/pyfactor.svg 21 | :target: _images/pyfactor.svg 22 | :alt: pyfactor visualisation 23 | 24 | More examples can be found in our :ref:`gallery`. 25 | 26 | *Pyfactor* is fundamentally a command line tool. 27 | However, the functionality is also exposed for use in code. 28 | See :ref:`reference` for CLI help and :ref:`guide` for configuration tips. 29 | 30 | Motivation 31 | ========== 32 | *Pyfactor* exists to make refactoring long scripts easier 33 | and understanding large code bases quicker. 34 | Seeing a graph makes it possible to easily discover structure in the code 35 | that is harder to grasp when simply reading the file, 36 | especially for those that are not intimately familiar with the code. 37 | For example, such a graph could reveal collections of definitions 38 | or connection hubs that could be easily extracted to sub-modules, 39 | or give insight into the code's complexity. 40 | 41 | Still, simply moving definitions around into several files 42 | is not the be-all end-all of refactoring and code style. 43 | It is up to the user to make decisions, 44 | but *Pyfactor* is here to help! 45 | 46 | .. toctree:: 47 | :hidden: 48 | :caption: Pyfactor 49 | 50 | release-notes 51 | reference 52 | guide 53 | gallery 54 | 55 | .. _pypi: https://pypi.org/project/pyfactor 56 | .. _github: https://github.com/felix-hilden/pyfactor 57 | -------------------------------------------------------------------------------- /docs/src/reference.rst: -------------------------------------------------------------------------------- 1 | .. _reference: 2 | 3 | Reference 4 | ========= 5 | This document contains the command line help and public API of *Pyfactor*. 6 | 7 | Command line interface 8 | ---------------------- 9 | .. argparse:: 10 | :ref: pyfactor._cli.parser 11 | :prog: pyfactor 12 | 13 | High-level Python API 14 | --------------------- 15 | .. autofunction:: pyfactor.pyfactor 16 | .. autofunction:: pyfactor.legend 17 | 18 | Low-level Python API 19 | -------------------- 20 | .. autofunction:: pyfactor.parse 21 | .. autofunction:: pyfactor.preprocess 22 | .. autofunction:: pyfactor.render 23 | .. autofunction:: pyfactor.create_legend 24 | -------------------------------------------------------------------------------- /docs/src/release-notes.rst: -------------------------------------------------------------------------------- 1 | .. _release-notes: 2 | 3 | Release notes 4 | ============= 5 | 0.4.1 (2021-04-06) 6 | ------------------ 7 | - Fix collapsing waypoints attribute error on graph conversion 8 | 9 | 0.4.0 (2021-04-06) 10 | ------------------ 11 | - Add multi-file, recursive and importable module analysis (:issue:`5`) 12 | - Split CLI file name specification to separate arguments (:issue:`5`) 13 | - Add option to specify graph root (:issue:`14`) 14 | - Expand assignment parsing (:issue:`18`) 15 | - Fix CLI and source gathering logic 16 | 17 | 0.3.0 (2021-03-05) 18 | ------------------ 19 | - Parse docstrings and provide them as tooltips (:issue:`8`) 20 | - Change default render format to SVG (for doc tooltips) (:issue:`8`) 21 | - Improve visual representation and legend, analyse waypoints 22 | (:issue:`4`, :issue:`12`, :issue:`13`) 23 | 24 | 0.2.0 (2021-03-01) 25 | ------------------ 26 | - Add handlers for most Python constructs 27 | - Handle existing constructs more correctly 28 | - Improve visual representation and legend 29 | - Improve command line interface 30 | 31 | 0.1.0 (2021-01-25) 32 | ------------------ 33 | Initial release with some missing functionality. 34 | -------------------------------------------------------------------------------- /pyfactor/VERSION: -------------------------------------------------------------------------------- 1 | 0.4.1 2 | -------------------------------------------------------------------------------- /pyfactor/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Script dependency visualisation. 3 | 4 | See online documentation on `RTD `_. 5 | """ 6 | import os as _os 7 | from sys import stderr as _stderr 8 | from typing import List as _List, Dict as _Dict 9 | from pathlib import Path as _Path 10 | 11 | _version_file = _Path(_os.path.realpath(__file__)).parent / 'VERSION' 12 | __version__ = _version_file.read_text().strip() 13 | 14 | from . import _cli, _visit, _graph, _io 15 | from ._graph import create_legend 16 | from ._gv import preprocess, render 17 | 18 | 19 | def parse( 20 | source_paths: _List[str], 21 | graph_path: str, 22 | skip_external: bool = False, 23 | imports: str = 'interface', 24 | exclude: _List[str] = None, 25 | root: str = None, 26 | collapse_waypoints: bool = False, 27 | collapse_exclude: _List[str] = None, 28 | graph_attrs: _Dict[str, str] = None, 29 | node_attrs: _Dict[str, str] = None, 30 | edge_attrs: _Dict[str, str] = None, 31 | ) -> None: 32 | """ 33 | Parse source and create graph file. 34 | 35 | Parameters 36 | ---------- 37 | source_paths 38 | paths to Python source files to read 39 | graph_path 40 | path to graph file to write 41 | skip_external 42 | do not visualise imports to external modules (reducing clutter) 43 | imports 44 | import duplication/resolving mode 45 | exclude 46 | exclude nodes in the graph 47 | root 48 | only show root and its children in the graph 49 | collapse_waypoints 50 | collapse waypoint nodes 51 | collapse_exclude 52 | exclude nodes from being collapsed 53 | graph_attrs 54 | Graphviz graph attributes (overrided by Pyfactor) 55 | node_attrs 56 | Graphviz node attributes (overrided by Pyfactor) 57 | edge_attrs 58 | Graphviz edge attributes (overrided by Pyfactor) 59 | """ 60 | sources = _io.resolve_sources(source_paths) 61 | for s in sources: 62 | s.content = _io.read_source(s.file) 63 | parsed = [_visit.parse_lines(s) for s in sources] 64 | graph = _graph.create_graph( 65 | list(zip(sources, parsed)), 66 | skip_external=skip_external, 67 | imports=imports, 68 | exclude=exclude, 69 | root=root, 70 | collapse_waypoints=collapse_waypoints, 71 | collapse_exclude=collapse_exclude, 72 | graph_attrs=graph_attrs, 73 | node_attrs=node_attrs, 74 | edge_attrs=edge_attrs, 75 | ) 76 | _io.write_graph(graph, graph_path) 77 | 78 | 79 | def legend(path: str, preprocess_kwargs: dict, render_kwargs: dict) -> None: 80 | """ 81 | Create and render a legend. 82 | 83 | Parameters 84 | ---------- 85 | path 86 | legend image file 87 | preprocess_kwargs 88 | keyword arguments for :func:`preprocess` 89 | render_kwargs 90 | keyword arguments for :func:`render` 91 | """ 92 | source = create_legend() 93 | source = preprocess(source, **preprocess_kwargs) 94 | render(source, path, **render_kwargs) 95 | 96 | 97 | def pyfactor( 98 | source_paths: _List[str] = None, 99 | graph_path: str = None, 100 | render_path: str = None, 101 | parse_kwargs: dict = None, 102 | preprocess_kwargs: dict = None, 103 | render_kwargs: dict = None, 104 | ) -> None: 105 | """ 106 | Pyfactor Python endpoint. 107 | 108 | See the command line help for more information. 109 | 110 | Parameters 111 | ---------- 112 | source_paths 113 | Python source files 114 | graph_path 115 | graph definition file 116 | render_path 117 | image file 118 | parse_kwargs 119 | keyword arguments for :func:`parse` 120 | preprocess_kwargs 121 | keyword arguments for :func:`preprocess` 122 | render_kwargs 123 | keyword arguments for :func:`render` 124 | """ 125 | source_paths = source_paths or [] 126 | parse_kwargs = parse_kwargs or {} 127 | preprocess_kwargs = preprocess_kwargs or {} 128 | render_kwargs = render_kwargs or {} 129 | 130 | graph_temp = graph_path or str(_cli.infer_graph_from_sources(source_paths)) 131 | 132 | if source_paths: 133 | parse(source_paths, graph_temp, **parse_kwargs) 134 | 135 | if render_path is not None: 136 | source = _io.read_graph(graph_temp) 137 | source = preprocess(source, **preprocess_kwargs) 138 | render(source, render_path, **render_kwargs) 139 | 140 | if graph_path is None: 141 | _Path(graph_temp).unlink() 142 | 143 | 144 | def _attrs_to_dict(attrs: _List[str] = None) -> _Dict[str, str]: 145 | split = [attr.split(':', 1) for attr in attrs or []] 146 | return {n: v for n, v in split} 147 | 148 | 149 | def main() -> None: 150 | """Pyfactor CLI endpoint.""" 151 | args = _cli.parser.parse_args() 152 | 153 | if args.version: 154 | print(f'Pyfactor v.{__version__}', file=_stderr) 155 | exit(0) 156 | 157 | parse_kwargs = { 158 | 'skip_external': args.skip_external, 159 | 'imports': args.imports, 160 | 'exclude': args.exclude, 161 | 'collapse_waypoints': args.collapse_waypoints, 162 | 'collapse_exclude': args.collapse_exclude, 163 | 'root': args.root, 164 | 'graph_attrs': _attrs_to_dict(args.graph_attr), 165 | 'node_attrs': _attrs_to_dict(args.node_attr), 166 | 'edge_attrs': _attrs_to_dict(args.edge_attr), 167 | } 168 | preprocess_kwargs = { 169 | 'stagger': args.stagger, 170 | 'fanout': not args.no_fanout, 171 | 'chain': args.chain, 172 | } 173 | render_kwargs = { 174 | 'view': args.view, 175 | 'format': args.format, 176 | 'renderer': args.renderer, 177 | 'formatter': args.formatter, 178 | } 179 | 180 | if args.legend: 181 | legend(args.legend, preprocess_kwargs, render_kwargs) 182 | if args.sources: 183 | try: 184 | source_paths, graph_path, render_path = _cli.parse_names( 185 | args.sources, args.graph, args.output 186 | ) 187 | except _cli.ArgumentError as e: 188 | print(str(e), file=_stderr) 189 | exit(1) 190 | 191 | pyfactor( 192 | source_paths, 193 | graph_path, 194 | render_path, 195 | parse_kwargs, 196 | preprocess_kwargs, 197 | render_kwargs, 198 | ) 199 | if not args.sources and not args.legend: 200 | _cli.parser.print_help(_stderr) 201 | exit(1) 202 | -------------------------------------------------------------------------------- /pyfactor/_cli.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | from pathlib import Path 3 | from typing import List, Optional 4 | 5 | parser = ArgumentParser( 6 | allow_abbrev=False, description='Script dependency visualiser.' 7 | ) 8 | 9 | group_mode = parser.add_argument_group('Source and output') 10 | group_mode.add_argument('sources', nargs='*', help=( 11 | 'source file names. If sources was disabled by providing no names, ' 12 | '--graph is used as direct input for rendering. Disabling two or more of ' 13 | 'SOURCES, --graph and --output will return with an error code 1.' 14 | )) 15 | group_mode.add_argument('--graph', '-g', nargs='?', default='-', const=None, help=( 16 | 'write or read intermediate graph file. Graph output is disabled by default. ' 17 | 'If a value is specified, it is used as the file name. ' 18 | 'If no value is provided, the name is inferred from combining SOURCES. ' 19 | 'See SOURCES for more information.' 20 | )) 21 | group_mode.add_argument('--output', '-o', help=( 22 | 'render file name. By default the name is inferred from --graph. ' 23 | 'If the name is a single hyphen, render output is disabled ' 24 | 'and a graph is written to --graph. See SOURCES for more information. ' 25 | 'NOTE: --format is appended to the name' 26 | )) 27 | group_mode.add_argument('--format', '-f', default='svg', help=( 28 | 'render file format, appended to all render file names (default: %(default)s) ' 29 | 'NOTE: displaying docstring tooltips is only available in svg and cmap formats' 30 | )) 31 | group_mode.add_argument( 32 | '--legend', nargs='?', default=None, const='pyfactor-legend', help=( 33 | 'render a legend, optionally specify a file name (default: %(const)s)' 34 | ) 35 | ) 36 | 37 | group_parse = parser.add_argument_group('Parsing options') 38 | group_parse.add_argument( 39 | '--imports', '-i', default='interface', help=( 40 | 'duplicate or resolve import nodes. ' 41 | 'Valid values are duplicate, interface and resolve (default: %(default)s). ' 42 | 'Duplicating produces a node for each import in the importing source. ' 43 | 'Resolving imports links edges directly to the original definitions instead. ' 44 | '"interface" leaves import nodes that reference definitions directly below ' 45 | 'the import in the module hierarchy and resolves others.' 46 | ) 47 | ) 48 | group_parse.add_argument( 49 | '--skip-external', '-se', action='store_true', help=( 50 | 'do not visualise imports to external modules' 51 | ) 52 | ) 53 | group_parse.add_argument( 54 | '--exclude', '-e', action='append', help='exclude nodes in the source' 55 | ) 56 | group_parse.add_argument( 57 | '--collapse-waypoints', '-cw', action='store_true', help=( 58 | 'remove children of waypoint nodes and mark them as collapsed' 59 | ) 60 | ) 61 | group_parse.add_argument( 62 | '--collapse-exclude', '-ce', action='append', help=( 63 | 'exclude waypoint nodes from being collapsed' 64 | 'when --collapse-waypoints is set' 65 | ) 66 | ) 67 | group_parse.add_argument( 68 | '--root', '-r', default=None, help=( 69 | 'only show root and its children in the graph ' 70 | 'NOTE: does not affect graph coloring' 71 | ) 72 | ) 73 | 74 | group_graph = parser.add_argument_group('Graph appearance') 75 | group_graph.add_argument( 76 | '--stagger', type=int, default=2, help='max Graphviz unflatten stagger' 77 | ) 78 | group_graph.add_argument( 79 | '--no-fanout', action='store_true', help='disable Graphviz unflatten fanout' 80 | ) 81 | group_graph.add_argument( 82 | '--chain', type=int, default=1, help='max Graphviz unflatten chain' 83 | ) 84 | group_graph.add_argument( 85 | '--graph-attr', '-ga', action='append', help=( 86 | 'Graphviz graph attributes as colon-separated name-value pairs ' 87 | '(e.g. -ga overlap:false) NOTE: overrided by Pyfactor' 88 | ) 89 | ) 90 | group_graph.add_argument( 91 | '--node-attr', '-na', action='append', help=( 92 | 'Graphviz node attributes as colon-separated name-value pairs ' 93 | '(e.g. -na style:filled,rounded) NOTE: overrided by Pyfactor' 94 | ) 95 | ) 96 | group_graph.add_argument( 97 | '--edge-attr', '-ea', action='append', help=( 98 | 'Graphviz edge attributes as colon-separated name-value pairs ' 99 | '(e.g. -ea arrowsize:2) NOTE: overrided by Pyfactor' 100 | ) 101 | ) 102 | group_graph.add_argument('--engine', help='Graphviz layout engine') 103 | 104 | group_misc = parser.add_argument_group('Miscellaneous options') 105 | group_misc.add_argument('--view', action='store_true', help=( 106 | 'open result in default application after rendering' 107 | )) 108 | group_misc.add_argument( 109 | '--renderer', help='Graphviz output renderer' 110 | ) 111 | group_misc.add_argument( 112 | '--formatter', help='Graphviz output formatter' 113 | ) 114 | group_misc.add_argument( 115 | '--version', '-v', action='store_true', help='display version number and exit' 116 | ) 117 | 118 | 119 | class ArgumentError(RuntimeError): 120 | """Invalid command line arguments given.""" 121 | 122 | 123 | def make_absolute(path: Path) -> Path: 124 | """Make absolute path out of a potentially relative one.""" 125 | return path if path.is_absolute() else Path.cwd() / path 126 | 127 | 128 | def infer_graph_from_sources(sources: List[str]) -> Path: 129 | """Infer graph name from sources.""" 130 | parts = [make_absolute(Path(s)).stem for s in sources] 131 | return Path('-'.join(parts)).with_suffix('.gv') 132 | 133 | 134 | def parse_names(sources: List[str], graph: Optional[str], output: Optional[str]): 135 | """Parse file names from arguments.""" 136 | if not sources and (not graph or graph == '-'): 137 | raise ArgumentError('Pyfactor: no input specified!') 138 | if graph == '-' and output == '-': 139 | raise ArgumentError('Pyfactor: all output disabled!') 140 | if not sources and output == '-': 141 | raise ArgumentError('Pyfactor: only graph name specified!') 142 | 143 | if not sources: 144 | o = output or str(Path(graph).with_suffix('')) 145 | return None, graph, o 146 | 147 | inferred = infer_graph_from_sources(sources) 148 | if graph == '-': 149 | o = output or str(inferred.with_suffix('')) 150 | return sources, None, o 151 | 152 | if output == '-': 153 | g = graph or str(inferred) 154 | return sources, g, None 155 | else: 156 | g = Path(graph) if graph else inferred 157 | o = output or str(g.with_suffix('')) 158 | return sources, str(g), o 159 | -------------------------------------------------------------------------------- /pyfactor/_graph.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | import networkx as nx 4 | import graphviz as gv 5 | 6 | from dataclasses import dataclass 7 | from enum import Enum 8 | from pathlib import Path 9 | from textwrap import dedent 10 | from warnings import warn 11 | from typing import List, Dict, Set, Optional, Tuple 12 | from ._visit import Line 13 | from ._io import Source 14 | from ._cli import ArgumentError 15 | 16 | 17 | class NodeType(Enum): 18 | """Shorthands for node types.""" 19 | 20 | var = 'V' 21 | func = 'F' 22 | class_ = 'C' 23 | import_ = 'I' 24 | unknown = '?' 25 | multiple = '+' 26 | 27 | 28 | def get_type(node: ast.AST) -> NodeType: 29 | """Determine general type of AST node.""" 30 | if isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign)): 31 | return NodeType.var 32 | elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): 33 | return NodeType.func 34 | elif isinstance(node, ast.ClassDef): 35 | return NodeType.class_ 36 | elif isinstance(node, (ast.Import, ast.ImportFrom)): 37 | return NodeType.import_ 38 | else: 39 | return NodeType.unknown 40 | 41 | 42 | @dataclass 43 | class GraphNode: 44 | name: str 45 | deps: Set[str] 46 | type: NodeType 47 | lineno_str: str 48 | docstring: Optional[str] 49 | import_sources: Set[str] 50 | 51 | 52 | def resolve_import(import_: str, location: str, file: Path) -> str: 53 | """Resolve potentially relative import inside a package.""" 54 | if not import_.startswith('.'): 55 | return import_ 56 | 57 | import_parts = import_.split('.') 58 | up_levels = len([p for p in import_parts if p == '']) 59 | import_down = import_parts[up_levels:] 60 | 61 | if file.stem == '__init__': 62 | up_levels -= 1 63 | location_parts = location.split('.') 64 | 65 | # up_levels can be 0 66 | location_parts = location_parts[:-up_levels or None] 67 | return '.'.join(location_parts + import_down) 68 | 69 | 70 | def merge_nodes(location: str, file: Path, lines: List[Line]) -> List[GraphNode]: 71 | """Merge name definitions and references on lines to unique graph nodes.""" 72 | nodes = {} 73 | for line in lines: 74 | for name in line.names: 75 | node = GraphNode( 76 | name.name, 77 | name.deps, 78 | get_type(line.ast_node), 79 | str(line.ast_node.lineno), 80 | line.docstring, 81 | set(), 82 | ) 83 | if name.source: 84 | node.import_sources.add( 85 | resolve_import(name.source, location, file) 86 | ) 87 | if node.name not in nodes: 88 | nodes[node.name] = [node, name.is_definition] 89 | else: 90 | merge = nodes[node.name][0] 91 | was_defined = nodes[node.name][1] 92 | 93 | merge.deps = merge.deps | node.deps 94 | merge.lineno_str += ',' + node.lineno_str 95 | if merge.type != node.type and name.is_definition and was_defined: 96 | merge.type = NodeType.multiple 97 | if name.is_definition: 98 | nodes[node.name][1] = True 99 | merge.import_sources = merge.import_sources | node.import_sources 100 | 101 | return [node for node, _ in nodes.values()] 102 | 103 | 104 | class MiscColor(Enum): 105 | """Colors for miscellaneous attributes.""" 106 | 107 | bridge = '#990000' 108 | 109 | 110 | class ConnectivityColor(Enum): 111 | """Colors for node connectivity degrees.""" 112 | 113 | default = '#FFFFFF' 114 | isolated = '#E5E5E5' 115 | root = '#BBEEFF' 116 | leaf = '#E4FFE4' 117 | waypoint = '#EFC4FF' 118 | 119 | 120 | type_shape = { 121 | NodeType.var: 'box', 122 | NodeType.func: 'ellipse', 123 | NodeType.class_: 'parallelogram', 124 | NodeType.import_: 'note', 125 | NodeType.unknown: 'ellipse', 126 | NodeType.multiple: 'ellipse', 127 | } 128 | centrality_color = { 129 | 0.997: '#FF3030', 130 | 0.95: '#FFA0A0', 131 | 0.68: '#FFE0E0', 132 | } 133 | 134 | 135 | def create_legend() -> gv.Source: 136 | """Create legend source.""" 137 | graph = gv.Digraph() 138 | 139 | with graph.subgraph(name='cluster1') as s: 140 | s.attr(label='Node shapes') 141 | s.node_attr.update(style='filled', fillcolor='#FFFFFF') 142 | types = [ 143 | ('variable', NodeType.var), 144 | ('function', NodeType.func), 145 | ('class', NodeType.class_), 146 | ('import', NodeType.import_), 147 | ('unknown', NodeType.unknown), 148 | ('multiple', NodeType.multiple), 149 | ] 150 | for name, t in types: 151 | s.node(f'{name} ({t.value})', shape=type_shape[t]) 152 | 153 | with graph.subgraph(name='cluster2') as s: 154 | s.attr(label='Node colours') 155 | s.node_attr.update(style='filled') 156 | for e in ConnectivityColor: 157 | s.node(e.name, fillcolor=e.value) 158 | for deg, col in centrality_color.items(): 159 | s.node(f'Centrality {deg}', fillcolor=col) 160 | s.node( 161 | 'collapsed', peripheries='2', fillcolor=ConnectivityColor.waypoint.value 162 | ) 163 | 164 | with graph.subgraph(name='cluster3') as s: 165 | s.node_attr.update(style='filled') 166 | s.attr(label='Edge styles') 167 | s.node('conn1', label=' ') 168 | s.node('conn2', label=' ') 169 | s.edge('conn1', 'conn2', label=' default') 170 | s.node('bridge1', label=' ') 171 | s.node('bridge2', label=' ') 172 | s.edge('bridge1', 'bridge2', label=' bridge', color=MiscColor.bridge.value) 173 | s.node('imp1', label=' ') 174 | s.node('imp2', label=' ') 175 | s.edge('imp1', 'imp2', label=' import', style='dashed') 176 | 177 | return gv.Source(graph.source) 178 | 179 | 180 | def append_color(node, color: str) -> None: 181 | """Append or replace default color.""" 182 | if node['fillcolor'] != ConnectivityColor.default.value: 183 | node['fillcolor'] = node['fillcolor'] + ';0.5:' + color 184 | node['gradientangle'] = '305' 185 | else: 186 | node['fillcolor'] = color 187 | 188 | 189 | class MissingNode(RuntimeWarning): 190 | """Node could not be found.""" 191 | 192 | 193 | class AmbiguousNode(RuntimeWarning): 194 | """Node could not be determined unambiguously.""" 195 | 196 | 197 | def guess_node(graph: nx.Graph, ref: str) -> Optional[str]: 198 | """Determine an unambiguous node that ref refers to, or return None.""" 199 | potential = {node for node in graph.nodes if node.endswith(ref)} 200 | if len(potential) == 1: 201 | return potential.pop() 202 | elif len(potential) == 0: 203 | msg = f'Node `{ref}` could not be found!' 204 | cls = MissingNode 205 | else: 206 | msg = f'Reference to `{ref}` is ambiguous!' 207 | cls = AmbiguousNode 208 | warn(msg, cls, stacklevel=4) 209 | 210 | 211 | cluster_invis_node = 'cluster-invis-node' 212 | 213 | 214 | def gen_cluster_nodes(graph: nx.DiGraph, levels: str) -> None: 215 | """Generate invis cluster nodes.""" 216 | parts = levels.split() 217 | for i in range(len(parts)): 218 | level = '.'.join(parts[:i + 1]) 219 | node = level + '.' + cluster_invis_node 220 | if not graph.has_node(node): 221 | graph.add_node(node, shape='point', style='invis') 222 | 223 | 224 | def create_graph( 225 | sources: List[Tuple[Source, List[Line]]], 226 | skip_external: bool = False, 227 | imports: str = 'interface', 228 | exclude: List[str] = None, 229 | root: str = None, 230 | collapse_waypoints: bool = False, 231 | collapse_exclude: List[str] = None, 232 | graph_attrs: Dict[str, str] = None, 233 | node_attrs: Dict[str, str] = None, 234 | edge_attrs: Dict[str, str] = None, 235 | ) -> gv.Digraph: 236 | """Create and populate a graph from references.""" 237 | exclude = set(exclude or []) 238 | collapse_exclude = set(collapse_exclude or []) 239 | graph_attrs = graph_attrs or {} 240 | node_attrs = node_attrs or {} 241 | edge_attrs = edge_attrs or {} 242 | graph_attrs.update({ 243 | 'compound': 'true', 244 | 'newrank': 'true', 245 | 'mclimit': '10.0', 246 | 'searchsize': '300', 247 | }) 248 | 249 | graph = nx.DiGraph() 250 | prefix_nodes = { 251 | s.name + '.': merge_nodes(s.name, s.file, ln) for s, ln in sources 252 | } 253 | for prefix, nodes in prefix_nodes.items(): 254 | for node in nodes: 255 | name = node.name.center(12, ' ') 256 | doc = node.docstring or f'{node.name} - no docstring' 257 | doc = dedent(doc).replace('\n', '\\n') 258 | attrs = { 259 | 'label': f'{name}\\n{node.type.value}:{node.lineno_str}', 260 | 'shape': type_shape[node.type], 261 | 'style': 'filled', 262 | 'tooltip': doc, 263 | } 264 | n_attrs = node_attrs.copy() 265 | n_attrs.update(attrs) 266 | graph.add_node(prefix + node.name, **n_attrs) 267 | graph.add_edges_from([ 268 | (prefix + node.name, prefix + d) for d in node.deps 269 | ], **edge_attrs) 270 | gen_cluster_nodes(graph, prefix[:-1]) 271 | 272 | import_sources = set() 273 | for _, nodes in prefix_nodes.items(): 274 | for node in nodes: 275 | import_sources.update(node.import_sources) 276 | for source in import_sources: 277 | if '.' in source: 278 | source = '.'.join(source.split('.')[:-1]) 279 | gen_cluster_nodes(graph, source) 280 | 281 | for prefix, nodes in prefix_nodes.items(): 282 | for node in nodes: 283 | if not node.import_sources: 284 | continue 285 | for s in node.import_sources: 286 | e_attrs = edge_attrs.copy() 287 | e_attrs['style'] = 'dashed' 288 | if graph.has_node(s + '.' + cluster_invis_node): 289 | e_attrs.update({'lhead': 'cluster_' + s}) 290 | graph.add_edge( 291 | prefix + node.name, s + '.' + cluster_invis_node, **e_attrs 292 | ) 293 | continue 294 | 295 | if not graph.has_node(s): 296 | graph.add_node( 297 | s, label=s.split('.')[-1], shape=type_shape[NodeType.import_] 298 | ) 299 | graph.add_edge(prefix + node.name, s, **e_attrs) 300 | 301 | for name in exclude: 302 | resolved = guess_node(graph, name) 303 | if resolved: 304 | graph.remove_node(resolved) 305 | 306 | if skip_external: 307 | internal = {p.split('.')[0] for p in prefix_nodes.keys()} 308 | removed = set() 309 | for node, data in graph.nodes.items(): 310 | if not data['shape'] == type_shape[NodeType.import_]: 311 | continue 312 | if all(v.split('.')[0] not in internal for _, v in graph.out_edges(node)): 313 | removed.add(node) 314 | graph.remove_nodes_from(removed) 315 | 316 | removed = set() 317 | for node in graph.nodes: 318 | if node.split('.')[0] not in internal: 319 | removed.add(node) 320 | graph.remove_nodes_from(removed) 321 | 322 | if imports == 'duplicate': 323 | pass 324 | elif imports in ('resolve', 'interface'): 325 | removed = set() 326 | for node, data in graph.nodes.items(): 327 | if not data['shape'] == type_shape[NodeType.import_]: 328 | continue 329 | 330 | out_edges = [(v, d) for _, v, d in graph.out_edges(node, data=True)] 331 | if len(out_edges) != 1: 332 | continue 333 | out_edge, data = out_edges[0] 334 | 335 | if imports == 'interface': 336 | location = '.'.join(node.split('.')[:-1]) 337 | target = out_edge.replace('.' + cluster_invis_node, '') 338 | if location in out_edge and node != target: 339 | continue 340 | 341 | in_edges = [(u, d) for u, _, d in graph.in_edges(node, data=True)] 342 | for in_edge, d in in_edges: 343 | attrs = d.copy() 344 | attrs.update(**data) 345 | graph.add_edge(in_edge, out_edge, **attrs) 346 | removed.add(node) 347 | graph.remove_nodes_from(removed) 348 | else: 349 | raise ArgumentError(f'Pyfactor: invalid imports mode `{imports}`!') 350 | 351 | conn = {} 352 | for node in graph.nodes: 353 | in_deg = len([0 for u, v in graph.in_edges(node) if u != node]) 354 | out_deg = len([0 for u, v in graph.out_edges(node) if v != node]) 355 | conn[node] = (in_deg, out_deg) 356 | 357 | centralities = sorted(i + o for i, o in conn.values()) 358 | 359 | for node in graph.nodes: 360 | in_deg, out_deg = conn[node] 361 | 362 | if in_deg == 0 and out_deg == 0: 363 | fill = ConnectivityColor.isolated 364 | elif in_deg == 0: 365 | fill = ConnectivityColor.root 366 | elif out_deg == 0: 367 | fill = ConnectivityColor.leaf 368 | else: 369 | fill = ConnectivityColor.default 370 | graph.nodes[node]['fillcolor'] = fill.value 371 | 372 | c = in_deg + out_deg 373 | central = sum(c > ct for ct in centralities) / len(centralities) 374 | for level, color in centrality_color.items(): 375 | if central > level: 376 | append_color(graph.nodes[node], color) 377 | break 378 | 379 | undirected = graph.to_undirected() 380 | bridge = MiscColor.bridge.value 381 | for from_, to in nx.bridges(undirected): 382 | if not graph.has_edge(from_, to): 383 | from_, to = to, from_ 384 | graph.edges[from_, to]['color'] = bridge 385 | 386 | i = -1 387 | graph_nodes = list(graph.nodes) 388 | removed_nodes = set() 389 | collapse_exclude = {guess_node(graph, n) for n in collapse_exclude} 390 | collapse_exclude = {n for n in collapse_exclude if n is not None} 391 | while i + 1 < len(graph_nodes): 392 | i += 1 393 | node = graph_nodes[i] 394 | 395 | if node in removed_nodes or conn[node][0] == 0 or conn[node][1] == 0: 396 | continue 397 | 398 | undirected = graph.to_undirected() 399 | undirected.remove_node(node) 400 | components = list(nx.connected_components(undirected)) 401 | 402 | in_nodes = {u for u, _ in graph.in_edges(node) if u != node} 403 | out_nodes = {v for _, v in graph.out_edges(node) if v != node} 404 | for comp in components: 405 | if len(comp & in_nodes) and len(comp & out_nodes): 406 | break 407 | else: 408 | append_color(graph.nodes[node], ConnectivityColor.waypoint.value) 409 | if collapse_waypoints and node not in collapse_exclude: 410 | graph.nodes[node]['peripheries'] = '2' 411 | for comp in components: 412 | if len(comp & out_nodes): 413 | removed_nodes = removed_nodes | comp 414 | graph.remove_nodes_from(comp) 415 | 416 | if root: 417 | root_ref = guess_node(graph, root) 418 | if root_ref: 419 | done = set() 420 | potential = {root_ref} 421 | while potential: 422 | n = potential.pop() 423 | done.add(n) 424 | succ = graph.successors(n) 425 | potential.update({s for s in succ if s not in done}) 426 | graph = graph.subgraph(done) 427 | 428 | # Construct module hierarchy 429 | hierarchy = Level({}, {}) 430 | for node, data in graph.nodes.items(): 431 | parts = node.split('.') 432 | tmp = hierarchy 433 | for part in parts[:-1]: 434 | if part not in tmp.sub: 435 | tmp.sub[part] = Level({}, {}) 436 | tmp = tmp.sub[part] 437 | tmp.names[parts[-1]] = data 438 | 439 | gv_graph = gv.Digraph() 440 | gv_graph.attr(**graph_attrs) 441 | make_subgraphs(gv_graph, hierarchy, []) 442 | for from_, to, data in graph.edges.data(): 443 | gv_graph.edge(from_, to, **data) 444 | return gv_graph 445 | 446 | 447 | @dataclass 448 | class Level: 449 | """Subgraph level.""" 450 | 451 | sub: Dict[str, 'Level'] 452 | names: Dict[str, Dict] 453 | 454 | 455 | def make_subgraphs( 456 | graph: gv.Digraph, hierarchy: Level, location: List 457 | ) -> None: 458 | """Recursively construct subgraph hierarchy.""" 459 | for name, data in hierarchy.names.items(): 460 | graph.node('.'.join(location + [name]), **data) 461 | 462 | for name, sub in hierarchy.sub.items(): 463 | new_loc = location + [name] 464 | loc_str = '.'.join(new_loc) 465 | name = 'cluster_' + loc_str 466 | attrs = { 467 | 'label': loc_str.center(12, ' '), 'fontsize': '22.0', 'penwidth': '2.5' 468 | } 469 | with graph.subgraph(name=name, graph_attr=attrs) as subgraph: 470 | make_subgraphs(subgraph, sub, new_loc) 471 | -------------------------------------------------------------------------------- /pyfactor/_gv.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | import graphviz as gv 3 | 4 | 5 | def preprocess( 6 | source: gv.Source, 7 | stagger: int = None, 8 | fanout: bool = False, 9 | chain: int = None, 10 | ) -> gv.Source: 11 | """ 12 | Preprocess source for rendering. 13 | 14 | Parameters 15 | ---------- 16 | source 17 | Graphviz source to preprocess 18 | stagger 19 | maximum Graphviz unflatten stagger 20 | fanout 21 | enable Graphviz unflatten fanout 22 | chain 23 | maximum Graphviz unflatten chain 24 | """ 25 | return source.unflatten(stagger=stagger, fanout=fanout, chain=chain) 26 | 27 | 28 | def render( 29 | source: gv.Source, 30 | out_path: str, 31 | format: str = None, 32 | engine: str = None, 33 | renderer: str = None, 34 | formatter: str = None, 35 | view: bool = False, 36 | ) -> None: 37 | """ 38 | Render source with Graphviz. 39 | 40 | Parameters 41 | ---------- 42 | source 43 | Graphviz source to render 44 | out_path 45 | path to visualisation file to write 46 | format 47 | Graphviz render file format 48 | engine 49 | Graphviz layout engine 50 | renderer 51 | Graphviz output renderer 52 | formatter 53 | Graphviz output formatter 54 | view 55 | after rendering, display with the default application 56 | """ 57 | image_bytes = gv.pipe( 58 | engine or source.engine, 59 | format, 60 | str(source).encode(), 61 | renderer=renderer, 62 | formatter=formatter, 63 | ) 64 | out_path = Path(out_path).with_suffix('.' + format) 65 | out_path.parent.mkdir(parents=True, exist_ok=True) 66 | out_path.write_bytes(image_bytes) 67 | if view: 68 | gv.view(str(out_path)) 69 | -------------------------------------------------------------------------------- /pyfactor/_io.py: -------------------------------------------------------------------------------- 1 | import graphviz as gv 2 | 3 | from dataclasses import dataclass 4 | from typing import List 5 | from pathlib import Path 6 | from importlib.util import find_spec 7 | 8 | from ._cli import ArgumentError, make_absolute 9 | 10 | 11 | @dataclass 12 | class Source: 13 | file: Path 14 | name: str 15 | content: str = None 16 | 17 | 18 | def find_package_top(file: Path): 19 | """Find package top directory.""" 20 | while True: 21 | if len(file.parts) == 1: 22 | raise ValueError('Package top was not found!') 23 | if not file.with_name('__init__.py').exists(): 24 | break 25 | file = file.parent 26 | return file 27 | 28 | 29 | def resolve_sources(paths: List[str]) -> List[Source]: 30 | """Resolve sources from paths and importable modules.""" 31 | singles = [] 32 | packages = [] 33 | importable = [] 34 | for path in paths: 35 | p = make_absolute(Path(path).resolve()) 36 | if p.is_dir(): 37 | packages.append(p) 38 | elif p.exists(): 39 | singles.append(p) 40 | elif find_spec(str(path)): 41 | importable.append(path) 42 | else: 43 | msg = ( 44 | f'Pyfactor: could not find `{path}`! ' 45 | 'Expected a file, a directory or an importable package.' 46 | ) 47 | raise ArgumentError(msg) 48 | 49 | if len(singles) == 0 and len(packages) == 1: 50 | if not (packages[0] / '__init__.py').exists(): 51 | folder = packages[0] 52 | singles = list(folder.glob('*.py')) 53 | packages = [path.parent for path in folder.glob('*/__init__.py')] 54 | 55 | local_sources = singles + packages 56 | if not all(p.parent == local_sources[0].parent for p in local_sources[1:]): 57 | raise ArgumentError('Pyfactor: sources are in different directories!') 58 | 59 | for i in importable: 60 | spec = find_spec(i) 61 | if spec.submodule_search_locations is None: 62 | singles.append(Path(spec.origin)) 63 | else: 64 | packages.extend([Path(p) for p in spec.submodule_search_locations]) 65 | 66 | sources = [Source(s, s.stem) for s in singles] 67 | for package in packages: 68 | for path in package.glob('**/*.py'): 69 | if not path.with_name('__init__.py').exists(): 70 | continue 71 | rel = path.relative_to(find_package_top(package).parent) 72 | if path.stem == '__init__': 73 | name = '.'.join(rel.parent.parts) 74 | else: 75 | name = '.'.join(rel.with_suffix('').parts) 76 | sources.append(Source(path, name)) 77 | return sources 78 | 79 | 80 | def read_source(path: Path) -> str: 81 | """Read Python source code with 'utf-8' encoding.""" 82 | return path.read_text(encoding='utf-8') 83 | 84 | 85 | def write_graph(graph: gv.Digraph, path: str) -> None: 86 | """Write graph to Graphviz dot file.""" 87 | with open(path, 'w') as f: 88 | f.write(str(graph.source)) 89 | 90 | 91 | def read_graph(path: str) -> gv.Source: 92 | """ 93 | Read graph file. 94 | 95 | Parameters 96 | ---------- 97 | path 98 | path to graph file to read 99 | """ 100 | return gv.Source.from_file(path) 101 | -------------------------------------------------------------------------------- /pyfactor/_visit/__init__.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | from typing import List, Set, Iterable, Tuple, Optional 4 | from .base import Visitor, Name, Scope, Line 5 | from .._io import Source 6 | 7 | 8 | def multi_union(sets: Iterable[Set]) -> Set: 9 | """Union of multiple sets.""" 10 | return set().union(*sets) 11 | 12 | 13 | def collect_names(node: ast.AST) -> Set[str]: 14 | """Collect all names that a node has as children taking scope into account.""" 15 | visitor = cast(node) 16 | scope = visitor.create_scope() 17 | parse_scoped(visitor, scope) 18 | merged = Scope() 19 | visitor.merge_scopes(merged, scope) 20 | return merged.inner_potential | merged.inner_globaled 21 | 22 | 23 | def collect_args(args: ast.arguments): 24 | """Collect function arguments.""" 25 | all_args = args.args + args.kwonlyargs 26 | all_args += [args.vararg, args.kwarg] 27 | return [arg for arg in all_args if arg is not None] 28 | 29 | 30 | class DefaultVisitor(Visitor): 31 | def children(self) -> List[ast.AST]: 32 | return list(ast.iter_child_nodes(self.node)) 33 | 34 | 35 | class UpdateScopeForwardVisitor(Visitor): 36 | def update_scope(self, scope: Scope) -> None: 37 | scope.used.update(self.forward_deps()) 38 | 39 | 40 | class NameVisitor(Visitor): 41 | def update_scope(self, scope: Scope) -> None: 42 | scope.used.add(self.node.id) 43 | 44 | 45 | class ScopedVisitor(Visitor): 46 | @property 47 | def breaks_scope(self) -> bool: 48 | return True 49 | 50 | 51 | def assign_target_name(node) -> Name: 52 | """Generate name for an assign target component.""" 53 | if isinstance(node, ast.Starred): 54 | node = node.value 55 | 56 | if isinstance(node, ast.Name): 57 | return Name(node.id, deps=set(), is_definition=True) 58 | elif isinstance(node, ast.Subscript): 59 | name = assign_target_name(node.value) 60 | deps = collect_names(node.slice) 61 | name.deps = name.deps | deps 62 | name.is_definition = False 63 | return name 64 | elif isinstance(node, ast.Attribute): 65 | name = assign_target_name(node.value) 66 | name.is_definition = False 67 | return name 68 | else: 69 | deps = collect_names(node) 70 | return Name(None, deps=deps, is_definition=False) 71 | 72 | 73 | def flatten_assign_targets(targets: List[ast.AST]) -> List[ast.AST]: 74 | """Extract assign targets from nested structures.""" 75 | unresolved = targets 76 | resolved = [] 77 | 78 | i = 0 79 | while i < len(unresolved): 80 | target = unresolved[i] 81 | if isinstance(target, (ast.List, ast.Tuple)): 82 | unresolved.extend(target.elts) 83 | else: 84 | resolved.append(target) 85 | i += 1 86 | return resolved 87 | 88 | 89 | class AssignVisitor(ScopedVisitor): 90 | """Fake scoped to handle possible inner scoped nodes.""" 91 | 92 | def children(self) -> List[ast.AST]: 93 | return [self.node.value] 94 | 95 | @property 96 | def _assign_targets(self) -> List[ast.AST]: 97 | return self.node.targets 98 | 99 | def parse_names(self) -> List[Name]: 100 | targets = flatten_assign_targets(self._assign_targets) 101 | return [assign_target_name(t) for t in targets] 102 | 103 | 104 | class AugAssignVisitor(AssignVisitor): 105 | @property 106 | def _assign_targets(self) -> List[ast.AST]: 107 | return [self.node.target] 108 | 109 | 110 | class AnnAssignVisitor(AssignVisitor): 111 | def children(self) -> List[ast.AST]: 112 | return [self.node.annotation, self.node.value] 113 | 114 | @property 115 | def _assign_targets(self) -> List[ast.AST]: 116 | return [self.node.target] 117 | 118 | 119 | class ImportVisitor(Visitor): 120 | def parse_names(self) -> List[Name]: 121 | names = [] 122 | for n in self.node.names: 123 | source_parts = [] 124 | if isinstance(self.node, ast.ImportFrom): 125 | source_parts += [''] * self.node.level 126 | if self.node.module: 127 | source_parts += [self.node.module] 128 | if n.asname: 129 | name = n.asname 130 | source_parts += [n.name] 131 | else: 132 | name = n.name.split('.')[0] 133 | source_parts += [name] 134 | names.append( 135 | Name( 136 | name, 137 | deps=set(), 138 | is_definition=True, 139 | source='.'.join(source_parts), 140 | ) 141 | ) 142 | return names 143 | 144 | 145 | class TryVisitor(Visitor): 146 | def children(self) -> List[ast.AST]: 147 | n = self.node 148 | return n.body + n.handlers + n.orelse + n.finalbody 149 | 150 | 151 | class ExceptHandlerVisitor(UpdateScopeForwardVisitor): 152 | def forward_deps(self) -> Set[str]: 153 | if self.node.type is not None: 154 | return collect_names(self.node.type) 155 | else: 156 | return set() 157 | 158 | def children(self) -> List[ast.AST]: 159 | return self.node.body 160 | 161 | 162 | class IfVisitor(UpdateScopeForwardVisitor): 163 | def forward_deps(self) -> Set[str]: 164 | return collect_names(self.node.test) 165 | 166 | def children(self) -> List[ast.AST]: 167 | return self.node.body + self.node.orelse 168 | 169 | 170 | class WithVisitor(Visitor): 171 | def parse_names(self) -> List[Name]: 172 | names = [] 173 | for item in self.node.items: 174 | if item.optional_vars is None: 175 | continue 176 | targets = flatten_assign_targets([item.optional_vars]) 177 | i_names = [assign_target_name(t) for t in targets] 178 | i_deps = collect_names(item.context_expr) 179 | for name in i_names: 180 | name.deps = name.deps | i_deps 181 | names.extend(i_names) 182 | return names 183 | 184 | def children(self) -> List[ast.AST]: 185 | return self.node.body 186 | 187 | def update_scope(self, scope: Scope) -> None: 188 | for item in self.node.items: 189 | if item.optional_vars is not None: 190 | continue 191 | scope.used.update(collect_names(item.context_expr)) 192 | 193 | 194 | class WhileVisitor(UpdateScopeForwardVisitor): 195 | def forward_deps(self) -> Set[str]: 196 | return collect_names(self.node.test) 197 | 198 | def children(self) -> List[ast.AST]: 199 | return self.node.body + self.node.orelse 200 | 201 | 202 | class ForVisitor(Visitor): 203 | def parse_names(self) -> List[Name]: 204 | names = collect_names(self.node.target) 205 | deps = collect_names(self.node.iter) 206 | return [Name(name, deps, is_definition=True) for name in names] 207 | 208 | def forward_deps(self) -> Set[str]: 209 | return collect_names(self.node.iter) 210 | 211 | def children(self) -> List[ast.AST]: 212 | return self.node.body + self.node.orelse 213 | 214 | 215 | class FunctionVisitor(ScopedVisitor): 216 | def parse_names(self) -> List[Name]: 217 | return [Name(self.node.name, deps=set(), is_definition=True)] 218 | 219 | def children(self) -> List[ast.AST]: 220 | return self.node.body 221 | 222 | def create_scope(self) -> Scope: 223 | scope = Scope() 224 | all_args = collect_args(self.node.args) 225 | scope.assigned.update(a.arg for a in all_args) 226 | annotation_names = ( 227 | collect_names(a.annotation) 228 | for a in all_args if a.annotation is not None 229 | ) 230 | scope.globaled.update(multi_union(annotation_names)) 231 | fdef_sources = self.node.args.kw_defaults + self.node.args.defaults 232 | fdef_sources += self.node.decorator_list + [self.node.returns] 233 | fdef_names = (collect_names(d) for d in fdef_sources if d is not None) 234 | scope.globaled.update(multi_union(fdef_names)) 235 | return scope 236 | 237 | 238 | class LambdaVisitor(ScopedVisitor): 239 | def children(self) -> List[ast.AST]: 240 | return [self.node.body] 241 | 242 | def create_scope(self) -> Scope: 243 | scope = Scope() 244 | all_args = collect_args(self.node.args) 245 | scope.assigned.update(a.arg for a in all_args) 246 | return scope 247 | 248 | 249 | class ClassVisitor(ScopedVisitor): 250 | def parse_names(self) -> List[Name]: 251 | return [Name(self.node.name, deps=set(), is_definition=True)] 252 | 253 | def children(self) -> List[ast.AST]: 254 | return self.node.body 255 | 256 | def create_scope(self) -> Scope: 257 | scope = Scope() 258 | cdef_sources = self.node.bases + self.node.keywords 259 | cdef_sources += self.node.decorator_list 260 | cdef_names = (collect_names(b) for b in cdef_sources) 261 | scope.globaled.update(multi_union(cdef_names)) 262 | return scope 263 | 264 | @staticmethod 265 | def merge_scopes(outer: Scope, inner: Scope) -> None: 266 | from_this_scope = (inner.assigned | inner.nonlocaled) - inner.globaled 267 | outer.inner_potential.update( 268 | (inner.used - from_this_scope - inner.globaled) | inner.inner_potential 269 | ) 270 | outer.inner_globaled.update( 271 | inner.globaled | inner.inner_globaled 272 | ) 273 | 274 | 275 | class ComprehensionVisitor(ScopedVisitor): 276 | def children(self) -> List[ast.AST]: 277 | return [self.node.elt] 278 | 279 | def create_scope(self) -> Scope: 280 | scope = Scope() 281 | for gen in self.node.generators: 282 | iters = collect_names(gen.iter) 283 | targets = collect_names(gen.target) 284 | ifs = multi_union(collect_names(f) for f in gen.ifs) 285 | 286 | scope.globaled.update(iters - scope.assigned) 287 | scope.assigned.update(targets) 288 | scope.globaled.update(ifs - scope.assigned) 289 | return scope 290 | 291 | 292 | class DictCompVisitor(ComprehensionVisitor): 293 | def children(self) -> List[ast.AST]: 294 | return [self.node.key, self.node.value] 295 | 296 | 297 | class GlobalVisitor(Visitor): 298 | def update_scope(self, scope: Scope) -> None: 299 | scope.globaled.update(self.node.names) 300 | 301 | 302 | class NonlocalVisitor(Visitor): 303 | def update_scope(self, scope: Scope) -> None: 304 | scope.nonlocaled.update(self.node.names) 305 | 306 | 307 | def cast(node: ast.AST) -> Visitor: 308 | """Cast node to a visitor.""" 309 | if isinstance(node, ast.Name): 310 | return NameVisitor(node) 311 | 312 | # Assign visitors 313 | elif isinstance(node, (ast.Import, ast.ImportFrom)): 314 | return ImportVisitor(node) 315 | elif isinstance(node, ast.Assign): 316 | return AssignVisitor(node) 317 | elif isinstance(node, ast.AugAssign): 318 | return AugAssignVisitor(node) 319 | elif isinstance(node, ast.AnnAssign): 320 | return AnnAssignVisitor(node) 321 | 322 | # Flow control visitors 323 | elif isinstance(node, ast.Try): 324 | return TryVisitor(node) 325 | elif isinstance(node, ast.ExceptHandler): 326 | return ExceptHandlerVisitor(node) 327 | elif isinstance(node, ast.If): 328 | return IfVisitor(node) 329 | elif isinstance(node, (ast.With, ast.AsyncWith)): 330 | return WithVisitor(node) 331 | elif isinstance(node, (ast.For, ast.AsyncFor)): 332 | return ForVisitor(node) 333 | elif isinstance(node, ast.While): 334 | return WhileVisitor(node) 335 | 336 | # Scoped visitors 337 | elif isinstance(node, ast.ClassDef): 338 | return ClassVisitor(node) 339 | elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): 340 | return FunctionVisitor(node) 341 | elif isinstance(node, ast.Lambda): 342 | return LambdaVisitor(node) 343 | elif isinstance(node, ast.DictComp): 344 | return DictCompVisitor(node) 345 | elif isinstance(node, (ast.ListComp, ast.SetComp, ast.GeneratorExp)): 346 | return ComprehensionVisitor(node) 347 | 348 | # Miscellaneous visitors 349 | elif isinstance(node, ast.Global): 350 | return GlobalVisitor(node) 351 | elif isinstance(node, ast.Nonlocal): 352 | return NonlocalVisitor(node) 353 | 354 | # Default to DefaultVisitor 355 | elif isinstance(node, ast.AST): 356 | return DefaultVisitor(node) 357 | 358 | 359 | def maybe_get_docstring(node: ast.AST): 360 | """Get docstring from a constant expression, or return None.""" 361 | if ( 362 | isinstance(node, ast.Expr) 363 | and isinstance(node.value, ast.Constant) 364 | and isinstance(node.value.value, str) 365 | ): 366 | return node.value.value 367 | elif ( 368 | isinstance(node, ast.Expr) 369 | and isinstance(node.value, ast.Str) 370 | ): 371 | return node.value.s 372 | 373 | 374 | def parse_scoped(visitor: Visitor, scope: Scope) -> Tuple[List[Name], Optional[str]]: 375 | """ 376 | Parse nodes in a scope with shortcuts. 377 | 378 | Returns parsed names and possible docstring for parent visitor. 379 | """ 380 | if not visitor.breaks_scope: 381 | # Was called on an arbitrary node 382 | visitor.update_scope(scope) 383 | 384 | parent_doc = None 385 | children = visitor.children() 386 | if children: 387 | parent_doc = maybe_get_docstring(children[0]) 388 | 389 | for child in children: 390 | child = cast(child) 391 | if child is None: 392 | continue 393 | 394 | if child.breaks_scope: 395 | new_scope = child.create_scope() 396 | c_names, _ = parse_scoped(child, new_scope) 397 | child.merge_scopes(scope, new_scope) 398 | else: 399 | c_names, _ = parse_scoped(child, scope) 400 | 401 | child.update_scope(scope) 402 | 403 | assigns = {n.name for n in c_names if n.name is not None} 404 | uses = multi_union(n.deps for n in c_names) 405 | scope.assigned.update(assigns - scope.used) 406 | scope.used.update(uses) 407 | 408 | return visitor.parse_names(), parent_doc 409 | 410 | 411 | def parse_no_scope(visitor: Visitor) -> List[Line]: 412 | """Fully parse nodes as in outermost scope.""" 413 | names = [n for n in visitor.parse_names() if n.name is not None] 414 | self_line = Line(visitor.node, names, docstring=None) 415 | lines = [] 416 | previous = visitor 417 | for child in visitor.children(): 418 | if previous is visitor: 419 | self_line.docstring = maybe_get_docstring(child) 420 | previous = None 421 | elif previous is not None: 422 | previous.docstring = maybe_get_docstring(child) 423 | previous = None 424 | 425 | child = cast(child) 426 | if child is None: 427 | continue 428 | 429 | if child.breaks_scope: 430 | scope = child.create_scope() 431 | c_names, doc = parse_scoped(child, scope) 432 | c_names = [n for n in c_names if n.name is not None] 433 | merged = Scope() 434 | child.merge_scopes(merged, scope) 435 | deps = merged.inner_globaled | merged.inner_potential 436 | for name in c_names: 437 | name.deps = name.deps | deps 438 | line = Line(child.node, c_names, docstring=doc) 439 | lines.append(line) 440 | previous = line if doc is None else None 441 | else: 442 | lines.extend(parse_no_scope(child)) 443 | previous = None 444 | 445 | forward = visitor.forward_deps() 446 | for line in lines: 447 | for name in line.names: 448 | name.deps = name.deps | forward 449 | 450 | lines = [self_line] + lines 451 | return [line for line in lines if line.names or line.docstring] 452 | 453 | 454 | def parse_lines(source: Source) -> List[Line]: 455 | """Parse name definitions and references on lines from source.""" 456 | tree = ast.parse(source.content) 457 | lines = parse_no_scope(cast(tree)) 458 | defined_names = {n.name for line in lines for n in line.names} 459 | 460 | for line in lines: 461 | for name in line.names: 462 | name.deps = name.deps.intersection(defined_names) 463 | 464 | return lines 465 | -------------------------------------------------------------------------------- /pyfactor/_visit/base.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | from dataclasses import dataclass 4 | from typing import List, Set, Optional 5 | 6 | 7 | @dataclass 8 | class Name: 9 | """Name dependencies.""" 10 | 11 | name: Optional[str] 12 | deps: Set[str] 13 | is_definition: bool 14 | source: str = None 15 | 16 | 17 | @dataclass 18 | class Line: 19 | """Line of multiple names.""" 20 | 21 | ast_node: ast.AST 22 | names: List[Name] 23 | docstring: Optional[str] 24 | 25 | 26 | @dataclass 27 | class Scope: 28 | """Scope state.""" 29 | 30 | used: Set[str] = None 31 | assigned: Set[str] = None 32 | globaled: Set[str] = None 33 | nonlocaled: Set[str] = None 34 | inner_potential: Set[str] = None 35 | inner_globaled: Set[str] = None 36 | 37 | def __post_init__(self): 38 | """Ensure members are sets.""" 39 | self.used = self.used or set() 40 | self.assigned = self.assigned or set() 41 | self.globaled = self.globaled or set() 42 | self.nonlocaled = self.nonlocaled or set() 43 | self.inner_potential = self.inner_potential or set() 44 | self.inner_globaled = self.inner_globaled or set() 45 | 46 | 47 | @dataclass 48 | class Visitor: 49 | """AST node visitor base.""" 50 | 51 | node: ast.AST 52 | 53 | @staticmethod 54 | def forward_deps() -> Set[str]: 55 | """ 56 | Dependencies to propagate forward to child nodes. 57 | 58 | Forward dependencies do not apply to the parsed names of the node. 59 | This method is used only in the outermost scope, 60 | functionality should be mirrored in :meth:`update_scope`. 61 | """ 62 | return set() 63 | 64 | @staticmethod 65 | def parse_names() -> List[Name]: 66 | """ 67 | Parse names from this node. 68 | 69 | Names from child nodes should not be parsed, 70 | but instead returned in :meth:`children`. 71 | """ 72 | return [] 73 | 74 | def children(self) -> List[ast.AST]: 75 | """Child nodes to be inspected next.""" 76 | return [] 77 | 78 | @property 79 | def breaks_scope(self) -> bool: 80 | """ 81 | Node breaks scope. 82 | 83 | Inner definitions should not be added to the current scope. 84 | """ 85 | return False 86 | 87 | @staticmethod 88 | def create_scope() -> Scope: 89 | """Create and populate a new inner scope.""" 90 | return Scope() 91 | 92 | def update_scope(self, scope: Scope) -> None: 93 | """ 94 | Update scope of an inner scope. 95 | 96 | For statements that neither break scope nor define names. 97 | """ 98 | 99 | @staticmethod 100 | def merge_scopes(outer: Scope, inner: Scope) -> None: 101 | """Merge inner scope to outer when moving back up in the ast tree.""" 102 | # Nonlocaled can be considered "from this scope" because they require 103 | # an inner function scope, so we don't care about them in the outermost one 104 | from_this_scope = (inner.assigned | inner.nonlocaled) - inner.globaled 105 | outer.inner_potential.update( 106 | (inner.used | inner.inner_potential) - from_this_scope - inner.globaled 107 | ) 108 | outer.inner_globaled.update( 109 | inner.globaled | inner.inner_globaled 110 | ) 111 | -------------------------------------------------------------------------------- /readme-pypi.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Pyfactor 3 | ======== 4 | |python| 5 | 6 | Welcome to the PyPI page of *Pyfactor* 7 | - a refactoring tool that `visualises `_ 8 | Python source files, modules and importable packages as a graph of dependencies 9 | between Python constructs like variables, functions and classes. 10 | 11 | .. code:: sh 12 | 13 | $ pyfactor --help 14 | $ pyfactor script.py 15 | $ pyfactor script.py --skip-external --view 16 | 17 | Visit our online documentation on `Read The Docs`_ 18 | for reference documentation, examples and release notes. 19 | If you've found a bug or would like to propose a feature, 20 | please submit an issue on `GitHub`_. 21 | 22 | Installation 23 | ============ 24 | *Pyfactor* can be installed from the Package Index via ``pip``. 25 | 26 | .. code:: sh 27 | 28 | $ pip install pyfactor 29 | 30 | **Additionally**, *Pyfactor* depends on a free graph visualisation software 31 | `Graphviz `_, available for Linux, Windows and Mac. 32 | See also the documentation of the `Graphviz Python package 33 | `_ for more help. 34 | 35 | Release notes 36 | ============= 37 | Release notes can be found on our 38 | `Read The Docs site `_. 39 | 40 | .. |python| image:: https://img.shields.io/pypi/pyversions/pyfactor 41 | :alt: python version 42 | 43 | .. _read the docs: https://pyfactor.rtfd.org 44 | .. _rtd-gallery: https://pyfactor.rtfd.org/en/stable/gallery.html 45 | .. _github: https://github.com/felix-hilden/pyfactor 46 | -------------------------------------------------------------------------------- /readme.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Pyfactor 3 | ======== 4 | |build| |documentation| 5 | 6 | Welcome to the GitHub repository of *Pyfactor* 7 | - a refactoring tool that `visualises `_ 8 | Python source files, modules and importable packages as a graph of dependencies 9 | between Python constructs like variables, functions and classes. 10 | 11 | .. code:: sh 12 | 13 | $ pyfactor --help 14 | $ pyfactor script.py 15 | $ pyfactor script.py --skip-external --view 16 | 17 | See our `PyPI`_ page for installation instructions and package information. 18 | Visit our online documentation on `Read The Docs`_ 19 | for reference documentation, examples and release notes. 20 | 21 | Contributing 22 | ============ 23 | New contributors are always welcome! 24 | If you've found a bug or would like to propose a feature, 25 | please submit an `issue `_. 26 | If you'd like to get 27 | `more involved `_, 28 | you can start by cloning the most recent version from GitHub 29 | and installing it as an editable package with development dependencies. 30 | 31 | .. code:: sh 32 | 33 | $ git clone https://github.com/felix-hilden/pyfactor 34 | $ cd pyfactor 35 | $ pip install -e .[dev] 36 | 37 | For specialised uses, sets of extras can be installed separately. 38 | ``tests`` installs dependencies related to executing tests, 39 | ``docs`` is for building documentation locally, 40 | and ``checks`` contains ``tox`` and tools for static checking. 41 | The install can be verified by running all essential tasks with tox. 42 | 43 | .. code:: sh 44 | 45 | $ tox 46 | 47 | Now tests have been run and documentation has been built. 48 | A list of all individual tasks can be viewed with their descriptions. 49 | 50 | .. code:: sh 51 | 52 | $ tox -a -v 53 | 54 | Please have a look at the following sections for additional information 55 | regarding specific tasks and configuration. 56 | 57 | Documentation 58 | ------------- 59 | Documentation can be built locally with Sphinx. 60 | 61 | .. code:: sh 62 | 63 | $ cd docs 64 | $ make html 65 | 66 | The main page ``index.html`` can be found in ``build/html``. 67 | If tox is installed, this is equivalent to running ``tox -e docs``. 68 | 69 | Code style 70 | ---------- 71 | A set of code style rules is followed. 72 | To check for violations, run ``flake8``. 73 | 74 | .. code:: sh 75 | 76 | $ flake8 pyfactor 77 | 78 | Style checks for docstrings and documentation files are also available. 79 | To run all style checks use ``tox -e lint``. 80 | 81 | Running tests 82 | ------------- 83 | The repository contains a suite of test cases 84 | which can be studied and run to ensure the package works as intended. 85 | 86 | .. code:: sh 87 | 88 | $ pytest 89 | 90 | For tox, this is the default command when running e.g. ``tox -e py``. 91 | To measure test coverage and view uncovered lines or branches run ``coverage``. 92 | 93 | .. code:: sh 94 | 95 | $ coverage run 96 | $ coverage report 97 | 98 | This can be achieved with tox by running ``tox -e coverage``. 99 | 100 | .. |build| image:: https://github.com/felix-hilden/pyfactor/workflows/CI/badge.svg 101 | :target: https://github.com/felix-hilden/pyfactor/actions 102 | :alt: build status 103 | 104 | .. |documentation| image:: https://rtfd.org/projects/pyfactor/badge/?version=latest 105 | :target: https://pyfactor.rtfd.org/en/latest 106 | :alt: documentation status 107 | 108 | .. _pypi: https://pypi.org/project/pyfactor 109 | .. _read the docs: https://pyfactor.rtfd.org 110 | .. _rtd-gallery: https://pyfactor.rtfd.org/en/stable/gallery.html 111 | -------------------------------------------------------------------------------- /readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sphinx: 4 | builder: html 5 | configuration: docs/src/conf.py 6 | fail_on_warning: false 7 | 8 | formats: all 9 | 10 | python: 11 | version: 3.7 12 | install: 13 | - method: pip 14 | path: . 15 | extra_requirements: 16 | - docs 17 | system_packages: false -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | import os 3 | from pathlib import Path 4 | 5 | root = Path(os.path.realpath(__file__)).parent 6 | version_file = root / 'pyfactor' / 'VERSION' 7 | readme_file = root / 'readme-pypi.rst' 8 | 9 | pypi_url = 'https://pypi.org/project/pyfactor' 10 | github_url = 'https://github.com/felix-hilden/pyfactor' 11 | documentation_url = 'https://pyfactor.rtfd.org' 12 | 13 | extras_require = { 14 | 'docs': [ 15 | 'sphinx', 16 | 'sphinx-rtd-theme', 17 | 'sphinx-autodoc-typehints', 18 | 'sphinx-argparse', 19 | ], 20 | 'tests': [ 21 | 'coverage', 22 | 'pytest', 23 | ], 24 | 'checks': [ 25 | 'tox', 26 | 'doc8', 27 | 'flake8', 28 | 'flake8-bugbear', 29 | 'pydocstyle', 30 | 'pygments', 31 | ] 32 | } 33 | extras_require['dev'] = ( 34 | extras_require['docs'] + extras_require['tests'] + extras_require['checks'] 35 | ) 36 | 37 | setuptools.setup( 38 | name='pyfactor', 39 | version=version_file.read_text().strip(), 40 | description='A script dependency visualiser.', 41 | long_description=readme_file.read_text(), 42 | long_description_content_type='text/x-rst', 43 | 44 | url=documentation_url, 45 | download_url=pypi_url, 46 | project_urls={ 47 | 'Source': github_url, 48 | 'Issues': github_url + '/issues', 49 | 'Documentation': documentation_url, 50 | }, 51 | 52 | author='Felix Hildén', 53 | author_email='felix.hilden@gmail.com', 54 | maintainer='Felix Hildén', 55 | maintainer_email='felix.hilden@gmail.com', 56 | 57 | license='MIT', 58 | keywords='dependency visualiser', 59 | packages=setuptools.find_packages(exclude=('tests', 'tests.*',)), 60 | include_package_data=True, 61 | package_data={ 62 | 'pyfactor': ['VERSION'] 63 | }, 64 | entry_points={ 65 | 'console_scripts': ['pyfactor=pyfactor:main'] 66 | }, 67 | 68 | python_requires='>=3.6', 69 | install_requires=[ 70 | 'dataclasses;python_version<"3.7"', 71 | 'pydot', 72 | 'networkx', 73 | 'graphviz', 74 | ], 75 | extras_require=extras_require, 76 | 77 | classifiers=[ 78 | 'Development Status :: 3 - Alpha', 79 | 'Intended Audience :: Developers', 80 | 'License :: OSI Approved :: MIT License', 81 | 'Programming Language :: Python', 82 | 'Programming Language :: Python :: 3.6', 83 | 'Programming Language :: Python :: 3.7', 84 | 'Programming Language :: Python :: 3.8', 85 | 'Programming Language :: Python :: 3.9', 86 | 'Programming Language :: Python :: 3 :: Only', 87 | ], 88 | ) 89 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felix-hilden/pyfactor/c07c4f4369a9fa3678a0103670484a6fdf1b56b0/tests/__init__.py -------------------------------------------------------------------------------- /tests/cli.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pyfactor._cli import parse_names, ArgumentError 3 | 4 | 5 | class TestCLI: 6 | def test_all_specified(self): 7 | s, g, o = parse_names(['s'], 'g', 'o') 8 | assert (s, g, o) == (['s'], 'g', 'o') 9 | 10 | def test_source_disabled(self): 11 | s, g, o = parse_names([], 'g', 'o') 12 | assert s is None 13 | assert (g, o) == ('g', 'o') 14 | 15 | def test_graph_disabled(self): 16 | s, g, o = parse_names(['s'], '-', 'o') 17 | assert g is None 18 | assert (s, o) == (['s'], 'o') 19 | 20 | def test_output_disabled(self): 21 | s, g, o = parse_names(['s'], 'g', '-') 22 | assert o is None 23 | assert (s, g) == (['s'], 'g') 24 | 25 | def test_source_graph_disabled(self): 26 | with pytest.raises(ArgumentError): 27 | parse_names([], '-', 'o') 28 | 29 | def test_source_output_disabled(self): 30 | with pytest.raises(ArgumentError): 31 | parse_names([], 'g', '-') 32 | 33 | def test_graph_output_disabled(self): 34 | with pytest.raises(ArgumentError): 35 | parse_names(['s'], '-', '-') 36 | 37 | def test_all_disabled(self): 38 | with pytest.raises(ArgumentError): 39 | parse_names([], '-', '-') 40 | 41 | def test_infer_graph(self): 42 | s, g, o = parse_names(['s'], None, 'o') 43 | assert (s, g, o) == (['s'], 's.gv', 'o') 44 | 45 | def test_infer_output(self): 46 | s, g, o = parse_names(['s'], 'g', None) 47 | assert (s, g, o) == (['s'], 'g', 'g') 48 | 49 | def test_infer_graph_output(self): 50 | s, g, o = parse_names(['s'], None, None) 51 | assert (s, g, o) == (['s'], 's.gv', 's') 52 | 53 | def test_disabled_source_infer_graph(self): 54 | with pytest.raises(ArgumentError): 55 | parse_names([], None, 'o') 56 | 57 | def test_disabled_source_infer_output(self): 58 | s, g, o = parse_names([], 'g', None) 59 | assert s is None 60 | assert (g, o) == ('g', 'g') 61 | 62 | def test_disabled_graph_infer_output(self): 63 | s, g, o = parse_names(['s'], '-', None) 64 | assert g is None 65 | assert (s, o) == (['s'], 's') 66 | 67 | def test_infer_graph_disabled_output(self): 68 | s, g, o = parse_names(['s'], None, '-') 69 | assert o is None 70 | assert (s, g) == (['s'], 's.gv') 71 | 72 | def test_disabled_source_infer_graph_source(self): 73 | with pytest.raises(ArgumentError): 74 | parse_names([], None, None) 75 | 76 | def test_disabled_source_graph_infer_source(self): 77 | with pytest.raises(ArgumentError): 78 | parse_names([], '-', None) 79 | 80 | def test_disabled_source_output_infer_graph(self): 81 | with pytest.raises(ArgumentError): 82 | parse_names([], None, '-') 83 | -------------------------------------------------------------------------------- /tests/parsing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/felix-hilden/pyfactor/c07c4f4369a9fa3678a0103670484a6fdf1b56b0/tests/parsing/__init__.py -------------------------------------------------------------------------------- /tests/parsing/_util.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | from pathlib import Path 3 | from pyfactor._visit import parse_lines 4 | from pyfactor._io import Source 5 | 6 | 7 | def parse(source: str): 8 | lines = parse_lines(Source(Path('./nonfile'), '', source)) 9 | return [name for line in lines for name in line.names] 10 | 11 | 12 | def refs_equal(func): 13 | @wraps(func) 14 | def wrapper(self): 15 | source, expected = func(self) 16 | nodes = parse(source) 17 | assert len(nodes) == len(expected), 'Wrong number of nodes!' 18 | for n, e in zip(nodes, expected): 19 | assert n.name == e[0], f'Wrong name! Expected\n{e}\ngot\n{n}' 20 | assert n.deps == e[1], f'Wrong deps! Expected\n{e}\ngot\n{n}' 21 | return wrapper 22 | 23 | 24 | def refs_in(func): 25 | @wraps(func) 26 | def wrapper(self): 27 | source, expected = func(self) 28 | nodes = parse(source) 29 | assert len(nodes) == len(expected), 'Wrong number of nodes!' 30 | for n in nodes: 31 | if (n.name, n.deps) not in expected: 32 | raise AssertionError( 33 | f'Missing node! Parsed\n{n.name} with deps: {n.deps}' 34 | ) 35 | return wrapper 36 | 37 | 38 | def docs_equal(func): 39 | @wraps(func) 40 | def wrapper(self): 41 | source, expected = func(self) 42 | lines = parse_lines(Source(Path('./nonfile'), '', source)) 43 | docs = [line.docstring for line in lines] 44 | assert len(docs) == len(expected), ( 45 | f'Wrong number of docs! Expected\n{expected}\ngot\n{docs}' 46 | ) 47 | for d, e in zip(docs, expected): 48 | err = f'Wrong doc! Expected\n{e}\ngot\n{d}' 49 | assert d == e or d is None and e is None, err 50 | return wrapper 51 | 52 | 53 | def import_equal(func): 54 | @wraps(func) 55 | def wrapper(self): 56 | source, expected = func(self) 57 | nodes = parse(source) 58 | assert len(nodes) == len(expected), 'Wrong number of nodes!' 59 | for n, e in zip(nodes, expected): 60 | assert n.name == e[0], f'Wrong name! Expected\n{e}\ngot\n{n}' 61 | assert n.source == e[1], f'Wrong source! Expected\n{e}\ngot\n{n}' 62 | return wrapper 63 | -------------------------------------------------------------------------------- /tests/parsing/assign.py: -------------------------------------------------------------------------------- 1 | from ._util import refs_equal 2 | 3 | 4 | class TestAssign: 5 | @refs_equal 6 | def test_assign_uses_none(self): 7 | source = 'a = 1' 8 | refs = [('a', set())] 9 | return source, refs 10 | 11 | @refs_equal 12 | def test_assign_uses_var(self): 13 | source = 'a = 1\nb = a' 14 | refs = [('a', set()), ('b', {'a'})] 15 | return source, refs 16 | 17 | @refs_equal 18 | def test_assign_uses_undeclared_var(self): 19 | source = 'b = a' 20 | refs = [('b', set())] 21 | return source, refs 22 | 23 | @refs_equal 24 | def test_multiple_assign_uses_var(self): 25 | source = 'c = 1\na = b = c' 26 | refs = [('c', set()), ('a', {'c'}), ('b', {'c'})] 27 | return source, refs 28 | 29 | @refs_equal 30 | def test_unpack_assign_uses_none(self): 31 | source = 'a, b = (1, 2)' 32 | refs = [('a', set()), ('b', set())] 33 | return source, refs 34 | 35 | @refs_equal 36 | def test_nested_unpack_assign_uses_none(self): 37 | source = 'a, (b, c) = 1' 38 | refs = [('a', set()), ('b', set()), ('c', set())] 39 | return source, refs 40 | 41 | @refs_equal 42 | def test_unpack_repack_assign_uses_none(self): 43 | source = 'a, *b = (1, 2)' 44 | refs = [('a', set()), ('b', set())] 45 | return source, refs 46 | 47 | @refs_equal 48 | def test_unpack_assign_uses_var(self): 49 | source = 'c = 1\na, b = (c, 2)' 50 | refs = [('c', set()), ('a', {'c'}), ('b', {'c'})] 51 | return source, refs 52 | 53 | @refs_equal 54 | def test_aug_assign_to_var(self): 55 | source = 'a = 1\na += 1' 56 | refs = [('a', set()), ('a', set())] 57 | return source, refs 58 | 59 | @refs_equal 60 | def test_aug_assign_uses_var(self): 61 | source = 'a = 1\nb = 1\nb += a' 62 | refs = [('a', set()), ('b', set()), ('b', {'a'})] 63 | return source, refs 64 | 65 | @refs_equal 66 | def test_ann_assign_uses_var(self): 67 | source = 'a = 1\nb: int = a' 68 | refs = [('a', set()), ('b', {'a'})] 69 | return source, refs 70 | 71 | @refs_equal 72 | def test_ann_assign_hint_uses_var(self): 73 | source = 'a = int\nb: a = 1' 74 | refs = [('a', set()), ('b', {'a'})] 75 | return source, refs 76 | 77 | @refs_equal 78 | def test_assign_to_attribute(self): 79 | source = 'a = 1\nb = 2\na.b = 2' 80 | refs = [('a', set()), ('b', set()), ('a', set())] 81 | return source, refs 82 | 83 | @refs_equal 84 | def test_assign_var_to_attribute(self): 85 | source = 'a = 1\nb = 2\na.attr = b' 86 | refs = [('a', set()), ('b', set()), ('a', {'b'})] 87 | return source, refs 88 | 89 | @refs_equal 90 | def test_assign_var_to_subscript(self): 91 | source = 'a = [0, 1]\nb = 1\na[0] = b' 92 | refs = [('a', set()), ('b', set()), ('a', {'b'})] 93 | return source, refs 94 | 95 | @refs_equal 96 | def test_assign_to_subscript_with_var(self): 97 | source = 'a = [0, 1]\nb = 1\na[b] = 1' 98 | refs = [('a', set()), ('b', set()), ('a', {'b'})] 99 | return source, refs 100 | 101 | @refs_equal 102 | def test_assign_to_subscript_with_comp_shadows_var(self): 103 | source = 'a = [0, 1]\nb = 1\na[{b for b in range(2)}] = 1' 104 | refs = [('a', set()), ('b', set()), ('a', set())] 105 | return source, refs 106 | 107 | @refs_equal 108 | def test_assign_to_subscript_with_comp_uses_var(self): 109 | source = 'a = [0, 1]\nb = 1\na[{b for i in range(2)}] = 1' 110 | refs = [('a', set()), ('b', set()), ('a', {'b'})] 111 | return source, refs 112 | 113 | @refs_equal 114 | def test_assign_to_subscript_attribute(self): 115 | source = 'a = 0\nb = 1\na[b].c = 1' 116 | refs = [('a', set()), ('b', set()), ('a', {'b'})] 117 | return source, refs 118 | 119 | @refs_equal 120 | def test_assign_to_attribute_subscript(self): 121 | source = 'a = 0\nb = 1\na.c[b] = 1' 122 | refs = [('a', set()), ('b', set()), ('a', {'b'})] 123 | return source, refs 124 | 125 | @refs_equal 126 | def test_assign_to_call_subscript(self): 127 | source = 'a = 0\na()["b"] = 1' 128 | refs = [('a', set())] 129 | return source, refs 130 | 131 | @refs_equal 132 | def test_assign_to_call_attribute(self): 133 | source = 'a = 0\na().b = 1' 134 | refs = [('a', set())] 135 | return source, refs 136 | 137 | @refs_equal 138 | def test_assign_to_comp_subscript(self): 139 | source = '{i: i for i in range(3)}["a"] = 1' 140 | refs = [] 141 | return source, refs 142 | 143 | @refs_equal 144 | def test_assign_to_comp_subscript_uses_var(self): 145 | source = 'a = 1\n{i: i for i in range(3)}[a] = 1' 146 | refs = [('a', set())] 147 | return source, refs 148 | 149 | @refs_equal 150 | def test_assign_in_func_call_subscript_uses_var(self): 151 | source = 'a = 1\ndef foo():\n a().b = 1' 152 | refs = [('a', set()), ('foo', {'a'})] 153 | return source, refs 154 | -------------------------------------------------------------------------------- /tests/parsing/docs.py: -------------------------------------------------------------------------------- 1 | from ._util import docs_equal 2 | 3 | 4 | class TestAssign: 5 | @docs_equal 6 | def test_module_doc_attached(self): 7 | source = '"doc"' 8 | docs = ['doc'] 9 | return source, docs 10 | 11 | @docs_equal 12 | def test_module_non_str_const_not_attached(self): 13 | source = '1' 14 | docs = [] 15 | return source, docs 16 | 17 | @docs_equal 18 | def test_assignment_doc_attached(self): 19 | source = 'a = 1\n"doc"' 20 | docs = ['doc'] 21 | return source, docs 22 | 23 | @docs_equal 24 | def test_assignment_non_str_const_not_attached(self): 25 | source = 'a = 1\n2' 26 | docs = [None] 27 | return source, docs 28 | 29 | @docs_equal 30 | def test_func_doc_attached(self): 31 | source = 'def foo():\n """doc"""' 32 | docs = ['doc'] 33 | return source, docs 34 | 35 | @docs_equal 36 | def test_func_non_str_const_not_attached(self): 37 | source = 'def foo():\n 1' 38 | docs = [None] 39 | return source, docs 40 | 41 | @docs_equal 42 | def test_class_doc_attached(self): 43 | source = 'class A:\n """doc"""' 44 | docs = ['doc'] 45 | return source, docs 46 | 47 | @docs_equal 48 | def test_class_non_str_const_not_attached(self): 49 | source = 'class A:\n 1' 50 | docs = [None] 51 | return source, docs 52 | 53 | @docs_equal 54 | def test_const_in_comprehension_not_interpreted_as_doc(self): 55 | source = '{1 for _ in range(1)}' 56 | docs = [] 57 | return source, docs 58 | -------------------------------------------------------------------------------- /tests/parsing/flow.py: -------------------------------------------------------------------------------- 1 | from ._util import refs_equal, refs_in 2 | 3 | 4 | class TestFlow: 5 | @refs_equal 6 | def test_if_test_comprehension_shadows(self): 7 | source = 'a = 1\nif {a for a in range(2)}:\n b = 2' 8 | refs = [('a', set()), ('b', set())] 9 | return source, refs 10 | 11 | @refs_equal 12 | def test_if_test_comprehension_uses(self): 13 | source = 'a = 1\nif {a for i in range(2)}:\n b = 2' 14 | refs = [('a', set()), ('b', {'a'})] 15 | return source, refs 16 | 17 | @refs_equal 18 | def test_if_test_propagated_to_body(self): 19 | source = 'a = 1\nif a:\n b = 2' 20 | refs = [('a', set()), ('b', {'a'})] 21 | return source, refs 22 | 23 | @refs_equal 24 | def test_if_test_propagated_to_else(self): 25 | source = 'a = 1\nif a:\n pass\nelse:\n b = 2' 26 | refs = [('a', set()), ('b', {'a'})] 27 | return source, refs 28 | 29 | @refs_equal 30 | def test_elif_test_propagated_to_body(self): 31 | source = """ 32 | a = 1 33 | b = 2 34 | if a: 35 | pass 36 | elif b: 37 | c = 3 38 | """ 39 | refs = [('a', set()), ('b', set()), ('c', {'a', 'b'})] 40 | return source, refs 41 | 42 | @refs_equal 43 | def test_elif_test_propagated_to_else(self): 44 | source = """ 45 | a = 1 46 | b = 2 47 | if a: 48 | pass 49 | elif b: 50 | pass 51 | else: 52 | c = 3 53 | """ 54 | refs = [('a', set()), ('b', set()), ('c', {'a', 'b'})] 55 | return source, refs 56 | 57 | @refs_equal 58 | def test_with_const(self): 59 | source = 'with 1:\n pass' 60 | refs = [] 61 | return source, refs 62 | 63 | @refs_equal 64 | def test_with_assigns_name(self): 65 | source = 'with 1 as a:\n pass' 66 | refs = [('a', set())] 67 | return source, refs 68 | 69 | @refs_equal 70 | def test_with_assigns_names(self): 71 | source = 'with 1 as a, 2 as b:\n pass' 72 | refs = [('a', set()), ('b', set())] 73 | return source, refs 74 | 75 | @refs_equal 76 | def test_with_assigns_name_using_var(self): 77 | source = 'a = 1\nwith a as b:\n pass' 78 | refs = [('a', set()), ('b', {'a'})] 79 | return source, refs 80 | 81 | @refs_equal 82 | def test_with_assigns_nested_names(self): 83 | source = 'with 1 as (a, (b, c)):\n pass' 84 | refs = [('a', set()), ('b', set()), ('c', set())] 85 | return source, refs 86 | 87 | @refs_equal 88 | def test_with_name_not_propagated_forward(self): 89 | source = 'with 1 as a:\n b = 1' 90 | refs = [('a', set()), ('b', set())] 91 | return source, refs 92 | 93 | @refs_equal 94 | def test_try(self): 95 | source = """ 96 | try: 97 | a = 1 98 | except: 99 | b = 2 100 | else: 101 | c = 3 102 | finally: 103 | d = 4 104 | """ 105 | refs = [('a', set()), ('b', set()), ('c', set()), ('d', set())] 106 | return source, refs 107 | 108 | @refs_equal 109 | def test_try_handler_propagated_forward(self): 110 | source = 'a = 1\ntry:\n pass\nexcept a:\n b = 1' 111 | refs = [('a', set()), ('b', {'a'})] 112 | return source, refs 113 | 114 | @refs_equal 115 | def test_try_handler_as_tuple(self): 116 | source = 'a = 1\ntry:\n pass\nexcept (a, ValueError):\n b = 1' 117 | refs = [('a', set()), ('b', {'a'})] 118 | return source, refs 119 | 120 | @refs_equal 121 | def test_while(self): 122 | source = 'while True:\n a = 1' 123 | refs = [('a', set())] 124 | return source, refs 125 | 126 | @refs_equal 127 | def test_while_test_propagated_forward(self): 128 | source = 'a = 1\nwhile a:\n b = 2\nelse:\n c = 3' 129 | refs = [('a', set()), ('b', {'a'}), ('c', {'a'})] 130 | return source, refs 131 | 132 | @refs_equal 133 | def test_for_assigns(self): 134 | source = 'for a in range(3):\n pass' 135 | refs = [('a', set())] 136 | return source, refs 137 | 138 | @refs_equal 139 | def test_for_iter_uses_var(self): 140 | source = 'a = 1\nfor b in range(a):\n pass' 141 | refs = [('a', set()), ('b', {'a'})] 142 | return source, refs 143 | 144 | @refs_in 145 | def test_for_nested_assign(self): 146 | source = 'for a, (b, c) in range(3):\n pass' 147 | refs = [('a', set()), ('b', set()), ('c', set())] 148 | return source, refs 149 | 150 | @refs_equal 151 | def test_for_iter_propagated_forward(self): 152 | source = 'a = 1\nfor b in range(a):\n c = 3\nelse: d = 4' 153 | refs = [('a', set()), ('b', {'a'}), ('c', {'a'}), ('d', {'a'})] 154 | return source, refs 155 | -------------------------------------------------------------------------------- /tests/parsing/import.py: -------------------------------------------------------------------------------- 1 | from ._util import refs_equal, import_equal 2 | 3 | 4 | class TestImport: 5 | @import_equal 6 | def test_import_module(self): 7 | source = 'import a' 8 | refs = [('a', 'a')] 9 | return source, refs 10 | 11 | @import_equal 12 | def test_import_modules(self): 13 | source = 'import a, b' 14 | refs = [('a', 'a'), ('b', 'b')] 15 | return source, refs 16 | 17 | @import_equal 18 | def test_import_module_as(self): 19 | source = 'import a as b' 20 | refs = [('b', 'a')] 21 | return source, refs 22 | 23 | @import_equal 24 | def test_import_modules_as(self): 25 | source = 'import a as b, c as d' 26 | refs = [('b', 'a'), ('d', 'c')] 27 | return source, refs 28 | 29 | @import_equal 30 | def test_import_submodule(self): 31 | source = 'import a.b' 32 | refs = [('a', 'a')] 33 | return source, refs 34 | 35 | @import_equal 36 | def test_import_submodule_as(self): 37 | source = 'import a.b as c' 38 | refs = [('c', 'a.b')] 39 | return source, refs 40 | 41 | @import_equal 42 | def test_from_import_module(self): 43 | source = 'from a import b' 44 | refs = [('b', 'a.b')] 45 | return source, refs 46 | 47 | @import_equal 48 | def test_from_import_module_as(self): 49 | source = 'from a import b as c' 50 | refs = [('c', 'a.b')] 51 | return source, refs 52 | 53 | @import_equal 54 | def test_from_import_modules(self): 55 | source = 'from a import b, c' 56 | refs = [('b', 'a.b'), ('c', 'a.c')] 57 | return source, refs 58 | 59 | @import_equal 60 | def test_import_modules_mixed(self): 61 | source = 'import a as b, c' 62 | refs = [('b', 'a'), ('c', 'c')] 63 | return source, refs 64 | 65 | @refs_equal 66 | def test_use_import_in_var(self): 67 | source = 'import a\nb = a' 68 | refs = [('a', set()), ('b', {'a'})] 69 | return source, refs 70 | -------------------------------------------------------------------------------- /tests/parsing/scoped.py: -------------------------------------------------------------------------------- 1 | from ._util import refs_equal 2 | 3 | 4 | class TestScoped: 5 | @refs_equal 6 | def test_function_uses_none(self): 7 | source = 'def a():\n pass' 8 | refs = [('a', set())] 9 | return source, refs 10 | 11 | @refs_equal 12 | def test_function_uses_undeclared_var(self): 13 | source = 'def a():\n return b' 14 | refs = [('a', set())] 15 | return source, refs 16 | 17 | @refs_equal 18 | def test_function_uses_var(self): 19 | source = 'b = 1\ndef a():\n return b' 20 | refs = [('b', set()), ('a', {'b'})] 21 | return source, refs 22 | 23 | @refs_equal 24 | def test_async_function_uses_var(self): 25 | source = 'b = 1\nasync def a():\n return b' 26 | refs = [('b', set()), ('a', {'b'})] 27 | return source, refs 28 | 29 | @refs_equal 30 | def test_var_in_func_shadows(self): 31 | source = 'a = 1\ndef foo():\n a = 2' 32 | refs = [('a', set()), ('foo', set())] 33 | return source, refs 34 | 35 | @refs_equal 36 | def test_func_parameter_shadows(self): 37 | source = 'a = 1\ndef foo(a):\n return a' 38 | refs = [('a', set()), ('foo', set())] 39 | return source, refs 40 | 41 | @refs_equal 42 | def test_func_parameter_default_uses_var(self): 43 | source = 'a = 1\ndef foo(p=a):\n return p' 44 | refs = [('a', set()), ('foo', {'a'})] 45 | return source, refs 46 | 47 | @refs_equal 48 | def test_func_parameter_annotation_uses_var(self): 49 | source = 'a = 1\ndef foo(p: a):\n return p' 50 | refs = [('a', set()), ('foo', {'a'})] 51 | return source, refs 52 | 53 | @refs_equal 54 | def test_func_return_annotation_uses_var(self): 55 | source = 'a = 1\ndef foo() -> a:\n pass' 56 | refs = [('a', set()), ('foo', {'a'})] 57 | return source, refs 58 | 59 | @refs_equal 60 | def test_func_return_annotation_shadows_inner_var(self): 61 | source = 'a = 1\ndef foo(p: a):\n a = 2' 62 | refs = [('a', set()), ('foo', {'a'})] 63 | return source, refs 64 | 65 | @refs_equal 66 | def test_func_decorator_uses_var(self): 67 | source = 'a = 1\n@a\ndef foo():\n pass' 68 | refs = [('a', set()), ('foo', {'a'})] 69 | return source, refs 70 | 71 | @refs_equal 72 | def test_var_in_func_double_shadows(self): 73 | source = """ 74 | a = 1 75 | def foo(): 76 | a = 2 77 | def bar(): 78 | a = 3 79 | """ 80 | refs = [('a', set()), ('foo', set())] 81 | return source, refs 82 | 83 | @refs_equal 84 | def test_inner_func_uses_var(self): 85 | source = """ 86 | a = 1 87 | def foo(): 88 | def bar(): 89 | b = a 90 | """ 91 | refs = [('a', set()), ('foo', {'a'})] 92 | return source, refs 93 | 94 | @refs_equal 95 | def test_inner_func_uses_nonlocal(self): 96 | source = """ 97 | a = 1 98 | def foo(): 99 | a = 2 100 | def bar(): 101 | b = a 102 | """ 103 | refs = [('a', set()), ('foo', set())] 104 | return source, refs 105 | 106 | @refs_equal 107 | def test_inner_func_uses_global(self): 108 | source = """ 109 | a = 1 110 | def foo(): 111 | a = 2 112 | def bar(): 113 | global a 114 | b = a 115 | """ 116 | refs = [('a', set()), ('foo', {'a'})] 117 | return source, refs 118 | 119 | @refs_equal 120 | def test_inner_func_name_shadows(self): 121 | source = """ 122 | a = 1 123 | def foo(): 124 | def a(): 125 | pass 126 | b = a 127 | """ 128 | refs = [('a', set()), ('foo', set())] 129 | return source, refs 130 | 131 | @refs_equal 132 | def test_func_called(self): 133 | source = 'def foo():\n return\na = foo()' 134 | refs = [('foo', set()), ('a', {'foo'})] 135 | return source, refs 136 | 137 | @refs_equal 138 | def test_with_without_target_in_func(self): 139 | source = 'a = 1\ndef foo():\n with a:\n pass' 140 | refs = [('a', set()), ('foo', {'a'})] 141 | return source, refs 142 | 143 | @refs_equal 144 | def test_class_body_uses_undeclared_var(self): 145 | source = 'class A:\n b = 1' 146 | refs = [('A', set())] 147 | return source, refs 148 | 149 | @refs_equal 150 | def test_class_body_uses_var(self): 151 | source = 'b = 1\nclass A:\n b' 152 | refs = [('b', set()), ('A', {'b'})] 153 | return source, refs 154 | 155 | @refs_equal 156 | def test_var_in_class_shadows(self): 157 | source = 'a = 1\nclass A:\n a = 2' 158 | refs = [('a', set()), ('A', set())] 159 | return source, refs 160 | 161 | @refs_equal 162 | def test_var_in_class_uses_then_shadows(self): 163 | source = 'a = 1\nclass A:\n b = a\n a = 2' 164 | refs = [('a', set()), ('A', {'a'})] 165 | return source, refs 166 | 167 | @refs_equal 168 | def test_class_bases_uses_var(self): 169 | source = 'a = 1\nclass A(a):\n pass' 170 | refs = [('a', set()), ('A', {'a'})] 171 | return source, refs 172 | 173 | @refs_equal 174 | def test_class_meta_uses_var(self): 175 | source = 'a = 1\nclass A(metaclass=a):\n pass' 176 | refs = [('a', set()), ('A', {'a'})] 177 | return source, refs 178 | 179 | @refs_equal 180 | def test_class_decorator_uses_var(self): 181 | source = 'a = 1\n@a\nclass A:\n pass' 182 | refs = [('a', set()), ('A', {'a'})] 183 | return source, refs 184 | 185 | @refs_equal 186 | def test_instance_var_in_class_shadows(self): 187 | source = 'a = 1\nclass A:\n def __init__(self):\n self.a = 2' 188 | refs = [('a', set()), ('A', set())] 189 | return source, refs 190 | 191 | @refs_equal 192 | def test_instance_var_in_class_uses_then_shadows(self): 193 | source = 'a = 1\nclass A:\n def __init__(self):\n self.a = a' 194 | refs = [('a', set()), ('A', {'a'})] 195 | return source, refs 196 | 197 | @refs_equal 198 | def test_instance_var_in_class_as_param(self): 199 | source = 'a = 1\nclass A:\n def __init__(self, a):\n self.a = a' 200 | refs = [('a', set()), ('A', set())] 201 | return source, refs 202 | 203 | @refs_equal 204 | def test_class_body_scope_not_propagated_to_methods(self): 205 | source = """ 206 | a = 1 207 | class A: 208 | a = 2 209 | def __init__(self): 210 | self.a = a 211 | """ 212 | refs = [('a', set()), ('A', {'a'})] 213 | return source, refs 214 | 215 | @refs_equal 216 | def test_class_instantiated(self): 217 | source = 'class A:\n pass\na = A()' 218 | refs = [('A', set()), ('a', {'A'})] 219 | return source, refs 220 | 221 | @refs_equal 222 | def test_classmethod_used(self): 223 | source = 'class A:\n pass\na = A.meth()' 224 | refs = [('A', set()), ('a', {'A'})] 225 | return source, refs 226 | 227 | @refs_equal 228 | def test_lambda_uses_var(self): 229 | source = "a = 1\nb = lambda x: a" 230 | refs = [('a', set()), ('b', {'a'})] 231 | return source, refs 232 | 233 | @refs_equal 234 | def test_lambda_argument_shadows(self): 235 | source = "a = 1\nb = lambda a: a" 236 | refs = [('a', set()), ('b', set())] 237 | return source, refs 238 | 239 | @refs_equal 240 | def test_lambda_in_call_shadows(self): 241 | source = "a = 1\nb = foo(lambda a: a)" 242 | refs = [('a', set()), ('b', set())] 243 | return source, refs 244 | 245 | @refs_equal 246 | def test_lambda_in_scope_shadows(self): 247 | source = """ 248 | a = 1 249 | def foo(): 250 | return lambda a: a 251 | """ 252 | refs = [('a', set()), ('foo', set())] 253 | return source, refs 254 | 255 | @refs_equal 256 | def test_comprehension_body_uses_var(self): 257 | source = "a = 1\nb = [a for _ in range(3)]" 258 | refs = [('a', set()), ('b', {'a'})] 259 | return source, refs 260 | 261 | @refs_equal 262 | def test_comprehension_source_uses_var(self): 263 | source = "a = 1\nb = [i for i in range(a)]" 264 | refs = [('a', set()), ('b', {'a'})] 265 | return source, refs 266 | 267 | @refs_equal 268 | def test_comprehension_condition_uses_var(self): 269 | source = "a = 1\nb = [i for i in range(3) if i > a]" 270 | refs = [('a', set()), ('b', {'a'})] 271 | return source, refs 272 | 273 | @refs_equal 274 | def test_comprehension_shadows(self): 275 | source = "a = 1\nb = [a for a in range(3)]" 276 | refs = [('a', set()), ('b', set())] 277 | return source, refs 278 | 279 | @refs_equal 280 | def test_comprehensions_shadow_and_use(self): 281 | source = "a = 1\nb = [i for a in range(a) for i in range(a)]" 282 | refs = [('a', set()), ('b', {'a'})] 283 | return source, refs 284 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = flake8,doc8,pydocstyle,coverage,docs 3 | 4 | [flake8] 5 | select = C,E,F,W,B,B9 6 | ignore = E402,E501,W503 7 | 8 | [pydocstyle] 9 | ignore = D101,D102,D203,D212,D413 10 | 11 | [pytest] 12 | python_files = *.py 13 | testpaths = tests 14 | 15 | [coverage:run] 16 | source = pyfactor 17 | branch = True 18 | command_line = -m pytest 19 | 20 | [coverage:report] 21 | precision = 1 22 | show_missing = True 23 | skip_covered = True 24 | 25 | [doc8] 26 | ignore = D002,D004 27 | max-line-length = 80 28 | 29 | [testenv] 30 | description = Run test suite with pytest 31 | extras = test 32 | commands = pytest {posargs} 33 | whitelist_externals = pytest 34 | 35 | [testenv:test] 36 | ; Inherit everything from testenv 37 | 38 | [testenv:docs] 39 | description = Build Sphinx HTML documentation 40 | extras = docs 41 | changedir = docs 42 | whitelist_externals = make 43 | commands = make html 44 | 45 | [testenv:doc8] 46 | description = Check documentation .rst files 47 | extras = checks 48 | whitelist_externals = doc8 49 | commands = doc8 docs/src 50 | 51 | [testenv:flake8] 52 | description = Check code style 53 | extras = checks 54 | whitelist_externals = flake8 55 | commands = flake8 pyfactor tests setup.py 56 | 57 | [testenv:pydocstyle] 58 | description = Check documentation string style 59 | extras = checks 60 | whitelist_externals = pydocstyle 61 | commands = pydocstyle pyfactor 62 | 63 | [testenv:lint] 64 | ; Duplication needed https://github.com/tox-dev/tox/issues/647 65 | description = Run all static checks 66 | extras = checks 67 | whitelist_externals = 68 | doc8 69 | flake8 70 | pydocstyle 71 | commands = 72 | flake8 pyfactor tests setup.py 73 | doc8 docs/src 74 | pydocstyle pyfactor 75 | 76 | [testenv:coverage] 77 | description = Run test suite with code coverage 78 | whitelist_externals = coverage 79 | commands = coverage run 80 | coverage report 81 | --------------------------------------------------------------------------------