├── .github └── workflows │ └── workflow.yml ├── .gitignore ├── .readthedocs.yaml ├── LICENSE ├── MANIFEST.in ├── README.rst ├── docs ├── Makefile ├── conf.py ├── index.rst ├── installation.rst ├── make.bat ├── memestra-cache.rst ├── memestra.rst └── pyls-plugin.rst ├── memestra ├── __init__.py ├── caching.py ├── docparse.py ├── memestra.py ├── nbmemestra.py ├── preprocessor.py ├── utils.py └── version.py ├── requirements-dev.txt ├── requirements.txt ├── setup.py └── tests ├── misc ├── README.rst ├── decoratortest.py ├── ipy │ ├── __init__.py │ └── a │ │ ├── __init__.py │ │ └── b.py ├── memestra_nb_demo.ipynb ├── memestra_nb_demo.rst ├── module_defining_symbol.py ├── module_forwarding_all_symbols.py ├── module_forwarding_symbol.py ├── phantom.py ├── pkg │ ├── __init__.py │ ├── helper.py │ ├── sub │ │ ├── __init__.py │ │ └── other.py │ └── test.py ├── some_module.py └── some_rec_module.py ├── notebook.py ├── share └── memestra │ └── gast │ ├── __init__.yml │ └── astn.yml ├── test_basic.py ├── test_basic_decorator_deprecated.py ├── test_caching.py ├── test_docparse.py ├── test_imports.py ├── test_multiattr.py └── test_notebook.py /.github/workflows/workflow.yml: -------------------------------------------------------------------------------- 1 | name: testing 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | jobs: 11 | build: 12 | runs-on: ubuntu-18.04 13 | strategy: 14 | matrix: 15 | python-version: [3.6, 3.7, 3.8, 3.9] 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: Setup Python ${{ matrix.python-version }} 19 | uses: actions/setup-python@v2 20 | with: 21 | python-version: ${{ matrix.python-version }} 22 | - name: Install dependencies 23 | run: | 24 | python --version 25 | python -m pip install --upgrade pip 26 | wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; 27 | bash miniconda.sh -b -p $HOME/miniconda 28 | source "$HOME/miniconda/etc/profile.d/conda.sh" 29 | hash -r 30 | conda config --set always_yes yes --set changeps1 no 31 | conda update -q conda 32 | conda info -a 33 | # nbconvert needs pandoc 34 | conda create -q -n test-environment -c conda-forge python=${{ matrix.python-version }} pandoc 35 | conda activate test-environment 36 | pip install -v .[test] 37 | - name: Testing 38 | run: | 39 | source "$HOME/miniconda/etc/profile.d/conda.sh" 40 | conda activate test-environment 41 | pytest -v 42 | pytest -v # run test twice because the first run populates the cache, so they are not exactly equivalent 43 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | dist 3 | *.py[co] 4 | __pycache__ 5 | *.egg-info 6 | .ipynb_checkpoints 7 | .DS_Store 8 | .pytest_cache 9 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the version of Python and other tools you might need 9 | build: 10 | os: ubuntu-22.04 11 | tools: 12 | python: "3.11" 13 | 14 | # Build documentation in the docs/ directory with Sphinx 15 | sphinx: 16 | configuration: docs/conf.py 17 | 18 | # We recommend specifying your dependencies to enable reproducible builds: 19 | # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 20 | # python: 21 | # install: 22 | # - requirements: docs/requirements.txt 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020, QuantStack 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of the copyright holder nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements-dev.txt 2 | include requirements.txt 3 | include LICENSE 4 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Memestra! 2 | ========= 3 | 4 | Memestra, a static analysis tool for Python, which detects the use of deprecated APIs. 5 | 6 | 7 | Documentation 8 | ------------- 9 | 10 | Check out Memestra's full documentation: 11 | 12 | https://memestra.readthedocs.io/ 13 | 14 | Sample Usages 15 | ------------- 16 | 17 | Track usage of functions decorated by ``@deprecated.deprecated`` from the 18 | `deprecated `_ package: 19 | 20 | .. code-block:: console 21 | 22 | > pip install deprecated 23 | > cat test.py 24 | import deprecated 25 | 26 | @deprecated.deprecated 27 | def foo(): pass 28 | 29 | def bar(): 30 | foo() 31 | 32 | foo() 33 | 34 | > python memestra.py test.py 35 | foo used at test.py:7:5 36 | foo used at test.py:9:1 37 | 38 | Track usage of functions decorated by ``deprecated`` imported from 39 | ``decorator``: 40 | 41 | .. code-block:: console 42 | 43 | > cat test2.py 44 | from deprecated import deprecated 45 | 46 | @deprecated 47 | def foo(): pass 48 | 49 | def bar(): 50 | foo() 51 | 52 | foo() 53 | 54 | > python memestra.py test2.py 55 | foo used at test2.py:7:5 56 | foo used at test2.py:9:1 57 | 58 | Track usage of functions decorated by ``deprecated`` imported from 59 | ``decorator`` and aliased: 60 | 61 | .. code-block:: console 62 | 63 | > cat test3.py 64 | from deprecated import deprecated as dp 65 | 66 | @dp 67 | def foo(): pass 68 | 69 | def bar(): 70 | foo() 71 | 72 | foo() 73 | 74 | > python memestra.py test3.py 75 | foo used at test3.py:7:5 76 | foo used at test3.py:9:1 77 | 78 | 79 | License 80 | ------- 81 | 82 | We use a shared copyright model that enables all contributors to maintain the copyright on their contributions. 83 | 84 | This software is licensed under the BSD-3-Clause license. See the [LICENSE](LICENSE) file for details. 85 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | 13 | import os 14 | import sys 15 | sys.path.insert(0, os.path.abspath('.')) 16 | 17 | 18 | # -- Project information ----------------------------------------------------- 19 | 20 | project = 'Memestra' 21 | copyright = '2020, serge-sans-paille' 22 | author = 'serge-sans-paille' 23 | 24 | versionfile = os.path.join('..', 'memestra', 'version.py') 25 | exec(open(versionfile).read()) 26 | 27 | # The full version, including alpha/beta/rc tags 28 | release = __version__ 29 | 30 | 31 | # -- General configuration --------------------------------------------------- 32 | 33 | # Add any Sphinx extension module names here, as strings. They can be 34 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 35 | # ones. 36 | extensions = [ 37 | ] 38 | 39 | # Add any paths that contain templates here, relative to this directory. 40 | templates_path = ['_templates'] 41 | 42 | # List of patterns, relative to source directory, that match files and 43 | # directories to ignore when looking for source files. 44 | # This pattern also affects html_static_path and html_extra_path. 45 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 46 | 47 | 48 | # -- Options for HTML output ------------------------------------------------- 49 | 50 | # The theme to use for HTML and HTML Help pages. See the documentation for 51 | # a list of builtin themes. 52 | # 53 | html_theme = 'alabaster' 54 | 55 | # Add any paths that contain custom static files (such as style sheets) here, 56 | # relative to this directory. They are copied after the builtin static files, 57 | # so a file named "default.css" will overwrite the builtin "default.css". 58 | html_static_path = ['_static'] 59 | 60 | master_doc = 'index' -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. Copyright (c) 2020, Serge Guelton, Johan Mabille, and Mariana Meireles 2 | 3 | Distributed under the terms of the BSD 3-Clause License. 4 | 5 | The full license is in the file LICENSE, distributed with this software. 6 | 7 | 8 | Introduction 9 | ------------ 10 | 11 | Memestra checks code for places where deprecated functions are called. 12 | 13 | Memestra looks for decorators in your code that can be either specified by you or be the default ``@decorator.deprecated``. 14 | 15 | Licensing 16 | --------- 17 | 18 | We use a shared copyright model that enables all contributors to maintain the 19 | copyright on their contributions. 20 | 21 | This software is licensed under the BSD-3-Clause license. See the LICENSE file for details. 22 | 23 | 24 | .. toctree:: 25 | :caption: Installation 26 | :maxdepth: 2 27 | 28 | installation 29 | 30 | .. toctree:: 31 | :caption: Usage 32 | :maxdepth: 2 33 | 34 | memestra 35 | memestra-cache 36 | pyls-plugin 37 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | .. Copyright (c) 2020, Serge Guelton, Johan Mabille, and Mariana Meireles 2 | 3 | Distributed under the terms of the BSD 3-Clause License. 4 | 5 | The full license is in the file LICENSE, distributed with this software. 6 | 7 | Installation 8 | ============ 9 | 10 | From PyPI 11 | --------- 12 | 13 | .. code-block:: console 14 | 15 | pip install memestra 16 | 17 | From source 18 | ----------- 19 | 20 | Clone Memestra's repository. 21 | 22 | .. code-block:: console 23 | 24 | git clone git@github.com:QuantStack/memestra.git 25 | cd memestra 26 | python -m pip install -e . 27 | 28 | Using the mamba package 29 | ----------------------- 30 | 31 | A package for memestra is available on the mamba package manager: 32 | 33 | .. code-block:: console 34 | 35 | mamba install -c conda-forge memestra 36 | 37 | Using the conda package 38 | ----------------------- 39 | 40 | A package for memestra is available on the conda package manager: 41 | 42 | .. code-block:: console 43 | 44 | conda install -c conda-forge memestra 45 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/memestra-cache.rst: -------------------------------------------------------------------------------- 1 | Memestra-cache 2 | ============== 3 | 4 | *Memestra* uses two caches to speedup some computations. 5 | 6 | One is a declarative cache, installed in ``/share/memestra``. The second 7 | one is an automatic cache, installed in ``/.memestra``. 8 | 9 | Declarative Cache 10 | ----------------- 11 | 12 | The declarative cache is a file-based cache manually managed by users or 13 | third-party packages. Its structure is very simple: to describe a Python file 14 | whose path is ``/numpy/random/__init__.py``, one need to drop a file in 15 | ``/share/memestra/numpy/random/__init__.yml``. The yaml content looks 16 | like the following: 17 | 18 | .. code:: yaml 19 | 20 | deprecated: ['deprecated_function1', 'deprecated_function2:some reason'] 21 | generator: manual 22 | name: numpy.random 23 | version: 1 24 | 25 | - The ``deprecated`` field is the most important one. It contains a list of 26 | strings, each element being the name of a deprecated identifier. Text after 27 | the first (optional) ``:`` is usead as deprectation documentation. 28 | 29 | - The ``generator`` field must be set to ``manual``. 30 | 31 | - The ``name`` field is informative, it documents the cache entry and is usually 32 | set to the entry path. 33 | 34 | - The ``version`` field is used to track compatibility with further format 35 | changes. Current value is ``1``. 36 | 37 | When hitting an entry in the declarative cache, *memestra* does **not** 38 | process the content of the file, and uses the entry content instead. 39 | 40 | 41 | Automatic Cache 42 | --------------- 43 | 44 | To avoid redundant computations, *memestra* also maintains a cache of the visited file 45 | and the associated deprecation results. 46 | 47 | To handle the cache, *memestra* provides a tool named ``memestra-cache``. 48 | 49 | *Memestra*'s caching infrastructure is a file-based cache, located in 50 | home//.memestra (RW). The key is a 51 | hash of the file content and the value contains deprecation information, generator 52 | used, etc. 53 | 54 | There are two kind of keys: recursive and non-recursive. The recursive one also 55 | uses the hash of imported modules, so that if an imported module changes, the 56 | hash of the importing module also changes. 57 | 58 | To interact with *memestra* caches: 59 | 60 | **Positional arguments:** 61 | 62 | ``-set`` 63 | 64 | Set a cache entry in the automatic cache 65 | 66 | ``-list`` 67 | 68 | List cache entries for both caches 69 | 70 | ``-clear`` 71 | 72 | Remove all cache entries from the automatic cache 73 | 74 | ``-docparse`` 75 | 76 | Set cache entry from docstring in the automatic cache 77 | 78 | 79 | **Optional arguments:** 80 | 81 | ``-h, --help`` 82 | 83 | Show this help message and exit 84 | -------------------------------------------------------------------------------- /docs/memestra.rst: -------------------------------------------------------------------------------- 1 | .. Copyright (c) 2020, Serge Guelton, Johan Mabille, and Mariana Meireles 2 | 3 | Distributed under the terms of the BSD 3-Clause License. 4 | 5 | The full license is in the file LICENSE, distributed with this software. 6 | 7 | Memestra 8 | ======== 9 | 10 | Using memestra 11 | -------------- 12 | 13 | To use memestra in your code simply decorate your functions with the decorator you wish to use. The default decorator for memestra is ``@decorator.deprecated`` and you can find more about it here_. 14 | 15 | Memestra receives one mandatory argument in the command line, which is a positional argument, or the path to file that you want to scan. 16 | 17 | .. code-block:: console 18 | 19 | memestra path/to/file 20 | 21 | Besides that there are a few optional arguments that you can use. 22 | 23 | **Optional arguments:** 24 | 25 | ``--decorator`` 26 | 27 | Path to the decorator to check. Allows the use of a personalized decorator. 28 | If you're using the default one there is no need to write it again. If you're using something different it's necessary to pass your decorator to memestra when calling it. 29 | 30 | .. code-block:: console 31 | 32 | memestra path/to/file --decorator=userdecorator.deprecated 33 | 34 | ``--reason-keyword`` 35 | 36 | It's possible to show a message to the user specifying the reason why the code was deprecated. With this flag you can choose a different keyword, the default is ``reason``. 37 | If the user doesn't specify a reason keyword and doesn't pass any other keyword but still add a string in the wrapper call this string will be shown as the reason for the deprecation. 38 | 39 | ``--recursive`` 40 | 41 | Traverses the whole module hierarchy, including imported modules down to the Python standard library. Is deactivated by default. 42 | 43 | ``-h, --help`` 44 | 45 | Show a help message and exit. 46 | 47 | .. _here: https://github.com/vilic/deprecated-decorator 48 | -------------------------------------------------------------------------------- /docs/pyls-plugin.rst: -------------------------------------------------------------------------------- 1 | .. Copyright (c) 2020, Serge Guelton, Johan Mabille, and Mariana Meireles 2 | 3 | Distributed under the terms of the BSD 3-Clause License. 4 | 5 | The full license is in the file LICENSE, distributed with this software. 6 | 7 | Pyls-Memestra 8 | ============= 9 | 10 | Memestra offers integration with the Python Language Protocol, meaning that you can use it inside different programs like Jupyter Lab or other IDEs that support the protocol. 11 | 12 | Pyls-Memestra will run automatically once you install the extension in your program and it will offer support for all functions marked as ``@decorator.deprecated`` in the code. 13 | 14 | More about the Pyls-Memestra use and implementation in the official documentation: https://pyls-memestra.readthedocs.io/en/latest/ -------------------------------------------------------------------------------- /memestra/__init__.py: -------------------------------------------------------------------------------- 1 | from memestra.memestra import memestra 2 | -------------------------------------------------------------------------------- /memestra/caching.py: -------------------------------------------------------------------------------- 1 | import os 2 | import hashlib 3 | import sys 4 | import yaml 5 | 6 | # not using gast because we only rely on Import and ImportFrom, which are 7 | # portable. Not using gast prevents an extra costly conversion step. 8 | import ast 9 | 10 | from memestra.docparse import docparse 11 | from memestra.utils import resolve_module 12 | 13 | 14 | class DependenciesResolver(ast.NodeVisitor): 15 | ''' 16 | Traverse a module an collect statically imported modules 17 | ''' 18 | 19 | 20 | def __init__(self): 21 | self.result = set() 22 | 23 | def add_module(self, module_name): 24 | module_path = resolve_module(module_name) 25 | if module_path is not None: 26 | self.result.add(module_path) 27 | 28 | def visit_Import(self, node): 29 | for alias in node.names: 30 | self.add_module(alias.name) 31 | 32 | def visit_ImportFrom(self, node): 33 | self.add_module(node.module) 34 | 35 | # All members below are specialized in order to improve performance: 36 | # It's useless to traverse leaf statements and expression when looking for 37 | # an import. 38 | 39 | def visit_stmt(self, node): 40 | pass 41 | 42 | visit_Assign = visit_AugAssign = visit_AnnAssign = visit_Expr = visit_stmt 43 | visit_Return = visit_Print = visit_Raise = visit_Assert = visit_stmt 44 | visit_Pass = visit_Break = visit_Continue = visit_Delete = visit_stmt 45 | visit_Global = visit_Nonlocal = visit_Exec = visit_stmt 46 | 47 | def visit_body(self, node): 48 | for stmt in node.body: 49 | self.visit(stmt) 50 | 51 | visit_FunctionDef = visit_ClassDef = visit_AsyncFunctionDef = visit_body 52 | visit_With = visit_AsyncWith = visit_body 53 | 54 | def visit_orelse(self, node): 55 | for stmt in node.body: 56 | self.visit(stmt) 57 | for stmt in node.orelse: 58 | self.visit(stmt) 59 | 60 | visit_For = visit_While = visit_If = visit_AsyncFor = visit_orelse 61 | 62 | def visit_Try(self, node): 63 | for stmt in node.body: 64 | self.visit(stmt) 65 | for stmt in node.orelse: 66 | self.visit(stmt) 67 | for stmt in node.finalbody: 68 | self.visit(stmt) 69 | 70 | 71 | class Format(object): 72 | 73 | version = 1 74 | 75 | fields = (('version', lambda: Format.version), 76 | ('name', str), 77 | ('deprecated', list), 78 | ('generator', lambda: 'manual')) 79 | 80 | generators = {'memestra', 'manual'} 81 | 82 | @staticmethod 83 | def setdefaults(data, **defaults): 84 | for field, default in Format.fields: 85 | data.setdefault(field, defaults.get(field, default())) 86 | 87 | @staticmethod 88 | def check(data): 89 | Format.check_keys(data) 90 | for k, _ in Format.fields: 91 | getattr(Format, 'check_{}'.format(k))(data) 92 | 93 | @staticmethod 94 | def check_name(data): 95 | if not isinstance(data['name'], str): 96 | raise ValueError("name must be an str") 97 | 98 | @staticmethod 99 | def check_keys(data): 100 | data_keys = set(data.keys()) 101 | format_keys = {k for k, _ in Format.fields} 102 | if not data_keys == format_keys: 103 | difference = data_keys.difference(format_keys) 104 | raise ValueError("Invalid field{}: {}".format( 105 | 's' * (len(difference) > 1), 106 | ', '.join(sorted(difference)))) 107 | 108 | @staticmethod 109 | def check_version(data): 110 | if data['version'] != Format.version: 111 | raise ValueError( 112 | "Invalid version, should be {}".format(Format.version)) 113 | 114 | @staticmethod 115 | def check_generator(data): 116 | if data['generator'] not in Format.generators: 117 | raise ValueError( 118 | "Invalid generator, should be one of {}" 119 | .format(', '.join(sorted(Format.generators)))) 120 | 121 | @staticmethod 122 | def check_deprecated(data): 123 | deprecated = data['deprecated'] 124 | if not isinstance(deprecated, list): 125 | raise ValueError("deprecated must be a list") 126 | if not all(isinstance(of, str) for of in deprecated): 127 | raise ValueError("deprecated must be a list of string") 128 | 129 | 130 | class CacheKeyFactoryBase(object): 131 | def __init__(self, keycls): 132 | self.keycls = keycls 133 | self.created = dict() 134 | 135 | def __call__(self, module_path, name_hint=None ): 136 | if module_path in self.created: 137 | return self.created[module_path] 138 | else: 139 | self.created[module_path] = None # creation in process 140 | key = self.keycls(module_path, self) 141 | if name_hint is None: 142 | name_hint = os.path.splitext(os.path.basename(module_path))[0] 143 | key.name = name_hint 144 | self.created[module_path] = key 145 | return key 146 | 147 | def get(self, *args): 148 | return self.created.get(*args) 149 | 150 | 151 | class CacheKeyFactory(CacheKeyFactoryBase): 152 | ''' 153 | Factory for non-recursive keys. 154 | Only the content of the module is taken into account 155 | ''' 156 | 157 | class CacheKey(object): 158 | 159 | def __init__(self, module_path, _): 160 | 161 | with open(module_path, 'rb') as fd: 162 | module_content = fd.read() 163 | module_hash = hashlib.sha256(module_content).hexdigest() 164 | self.module_hash = module_hash 165 | 166 | @property 167 | def path(self): 168 | return self.name.replace('.', os.path.sep) 169 | 170 | def __init__(self): 171 | super(CacheKeyFactory, self).__init__(CacheKeyFactory.CacheKey) 172 | 173 | 174 | class RecursiveCacheKeyFactory(CacheKeyFactoryBase): 175 | ''' 176 | Factory for recursive keys. 177 | This take into account the module content, and the content of *all* imported 178 | module. That way, a change in the module hierarchy implies a change in the 179 | key. 180 | ''' 181 | 182 | class CacheKey(object): 183 | 184 | def __init__(self, module_path, factory): 185 | assert module_path not in factory.created or factory.created[module_path] is None 186 | 187 | with open(module_path, 'rb') as fd: 188 | module_content = fd.read() 189 | 190 | code = ast.parse(module_content) 191 | dependencies_resolver = DependenciesResolver() 192 | dependencies_resolver.visit(code) 193 | 194 | new_deps = [] 195 | for dep in dependencies_resolver.result: 196 | if factory.get(dep, 1) is not None: 197 | new_deps.append(dep) 198 | 199 | module_hash = hashlib.sha256(module_content).hexdigest() 200 | 201 | hashes = [module_hash] 202 | 203 | for new_dep in sorted(new_deps): 204 | try: 205 | new_dep_key = factory(new_dep) 206 | # FIXME: this only happens on windows, maybe we could do 207 | # better? 208 | except UnicodeDecodeError: 209 | continue 210 | hashes.append(new_dep_key.module_hash) 211 | 212 | self.module_hash = hashlib.sha256("".join(hashes).encode("ascii")).hexdigest() 213 | 214 | @property 215 | def path(self): 216 | return self.name.replace('.', os.path.sep) 217 | 218 | def __init__(self): 219 | super(RecursiveCacheKeyFactory, self).__init__(RecursiveCacheKeyFactory.CacheKey) 220 | 221 | 222 | class SharedCache(object): 223 | 224 | def __init__(self): 225 | shared_dir = os.path.join(sys.prefix, 'share', 'memestra') 226 | self.cache_entries = {} 227 | 228 | for root, dirs, files in os.walk(shared_dir): 229 | for fname in files: 230 | if not fname.endswith('.yml'): 231 | continue 232 | # Not loading all entries on startup, 233 | # doing it lazily upon __getitem__ 234 | 235 | if fname == '__init__.yml': 236 | key = root[1 + len(shared_dir):] 237 | else: 238 | key = os.path.join(root[1 + len(shared_dir):], fname[:-4]) 239 | self.cache_entries[key] = os.path.join(root, fname) 240 | 241 | def __contains__(self, key): 242 | return key in self.cache_entries 243 | 244 | def __getitem__(self, key): 245 | cache_path = self.cache_entries[key] 246 | with open(cache_path, 'r') as yaml_fd: 247 | return yaml.load(yaml_fd, Loader=yaml.SafeLoader) 248 | 249 | def keys(self): 250 | return self.cache_entries.keys() 251 | 252 | 253 | class Cache(object): 254 | 255 | def __init__(self, cache_dir=None): 256 | self.shared_cache = SharedCache() 257 | 258 | if cache_dir is not None: 259 | self.cachedir = cache_dir 260 | else: 261 | xdg_config_home = os.environ.get('XDG_CONFIG_HOME', None) 262 | if xdg_config_home is None: 263 | user_config_dir = '~' 264 | memestra_dir = '.memestra' 265 | else: 266 | user_config_dir = xdg_config_home 267 | memestra_dir = 'memestra' 268 | self.cachedir = os.path.expanduser(os.path.join(user_config_dir, 269 | memestra_dir)) 270 | os.makedirs(self.cachedir, exist_ok=True) 271 | 272 | def _get_path(self, key): 273 | return os.path.join(self.cachedir, key.module_hash) 274 | 275 | def __contains__(self, key): 276 | if key.path in self.shared_cache: 277 | return True 278 | cache_path = self._get_path(key) 279 | return os.path.isfile(cache_path) 280 | 281 | def __getitem__(self, key): 282 | if key.path in self.shared_cache: 283 | return self.shared_cache[key.path] 284 | cache_path = self._get_path(key) 285 | with open(cache_path, 'r') as yaml_fd: 286 | return yaml.load(yaml_fd, Loader=yaml.SafeLoader) 287 | 288 | def __setitem__(self, key, data): 289 | data = data.copy() 290 | Format.setdefaults(data, name=key.name) 291 | Format.check(data) 292 | cache_path = self._get_path(key) 293 | with open(cache_path, 'w') as yaml_fd: 294 | yaml.dump(data, yaml_fd) 295 | 296 | def keys(self): 297 | return os.listdir(self.cachedir) 298 | 299 | def items(self): 300 | for key in self.keys(): 301 | cache_path = os.path.join(self.cachedir, key) 302 | with open(cache_path, 'r') as yaml_fd: 303 | yield key, yaml.load(yaml_fd, Loader=yaml.SafeLoader) 304 | 305 | 306 | def shared_items(self): 307 | for key in self.shared_cache.keys(): 308 | yield key, self.shared_cache[key] 309 | 310 | def clear(self): 311 | count = 0 312 | for key in self.keys(): 313 | cache_path = os.path.join(self.cachedir, key) 314 | os.remove(cache_path) 315 | count += 1 316 | return count 317 | 318 | 319 | def run_set(args): 320 | data = {'generator': 'manual', 321 | 'deprecated': args.deprecated} 322 | cache = Cache(cache_dir=args.cache_dir) 323 | if args.recursive: 324 | key_factory = RecursiveCacheKeyFactory() 325 | else: 326 | key_factory = CacheKeyFactory() 327 | key = key_factory(args.input) 328 | cache[key] = data 329 | 330 | 331 | def run_list(args): 332 | cache = Cache(cache_dir=args.cache_dir) 333 | print('declarative cache') 334 | print('-----------------') 335 | for k, v in cache.shared_items(): 336 | print('{}: {} ({})'.format(k, v['name'], len(v['deprecated']))) 337 | print() 338 | print('automatic cache') 339 | print('---------------') 340 | for k, v in cache.items(): 341 | print('{}: {} ({})'.format(k[:16], v['name'], len(v['deprecated']))) 342 | 343 | 344 | def run_clear(args): 345 | cache = Cache(cache_dir=args.cache_dir) 346 | nb_cleared = cache.clear() 347 | print('Cache cleared, {} element{} removed.'.format(nb_cleared, 's' * 348 | (nb_cleared > 1))) 349 | 350 | 351 | def run_docparse(args): 352 | deprecated = docparse(args.input, args.pattern) 353 | 354 | cache = Cache(cache_dir=args.cache_dir) 355 | if args.recursive: 356 | key_factory = RecursiveCacheKeyFactory() 357 | else: 358 | key_factory = CacheKeyFactory() 359 | key = key_factory(args.input) 360 | data = {'deprecated': deprecated, 361 | 'generator': 'manual'} 362 | cache[key] = data 363 | if args.verbose: 364 | print("Found {} deprecated identifier{}".format( 365 | len(deprecated), 366 | 's' * bool(deprecated))) 367 | for name in deprecated: 368 | print(name) 369 | 370 | 371 | def run(): 372 | import argparse 373 | parser = argparse.ArgumentParser( 374 | description='Interact with memestra cache') 375 | subparsers = parser.add_subparsers() 376 | 377 | parser.add_argument('--cache-dir', dest='cache_dir', 378 | default=None, 379 | action='store', 380 | help='The directory where the cache is located') 381 | 382 | parser_set = subparsers.add_parser('set', help='Set a cache entry') 383 | parser_set.add_argument('--deprecated', dest='deprecated', 384 | action='append', 385 | help='function to flag as deprecated') 386 | parser_set.add_argument('--recursive', action='store_true', 387 | help='set a dependency-aware cache key') 388 | parser_set.add_argument('input', type=str, 389 | help='module.py to edit') 390 | parser_set.set_defaults(runner=run_set) 391 | 392 | parser_list = subparsers.add_parser('list', help='List cache entries') 393 | parser_list.set_defaults(runner=run_list) 394 | 395 | parser_clear = subparsers.add_parser('clear', 396 | help='Remove all cache entries') 397 | parser_clear.set_defaults(runner=run_clear) 398 | 399 | parser_docparse = subparsers.add_parser( 400 | 'docparse', 401 | help='Set cache entry from docstring') 402 | parser_docparse.add_argument('-v', '--verbose', dest='verbose', 403 | action='store_true') 404 | parser_docparse.add_argument( 405 | '--pattern', dest='pattern', type=str, default=r'.*deprecated.*', 406 | help='pattern found in deprecated function docstring') 407 | parser_docparse.add_argument('--recursive', action='store_true', 408 | help='set a dependency-aware cache key') 409 | parser_docparse.add_argument('input', type=str, 410 | help='module.py to scan') 411 | parser_docparse.set_defaults(runner=run_docparse) 412 | 413 | args = parser.parse_args() 414 | if hasattr(args, 'runner'): 415 | args.runner(args) 416 | else: 417 | parser.print_help() 418 | -------------------------------------------------------------------------------- /memestra/docparse.py: -------------------------------------------------------------------------------- 1 | import gast as ast 2 | import re 3 | 4 | 5 | def docparse(module_path, pattern): 6 | 7 | with open(module_path) as fd: 8 | content = fd.read() 9 | tree = ast.parse(content) 10 | def_types = (ast.AsyncFunctionDef, 11 | ast.FunctionDef, 12 | ast.ClassDef) 13 | flags = re.DOTALL | re.MULTILINE 14 | 15 | deprecated = [] 16 | for stmt in tree.body: 17 | if not isinstance(stmt, def_types): 18 | continue 19 | fst_stmt = stmt.body[0] 20 | if not isinstance(fst_stmt, ast.Expr): 21 | continue 22 | if not isinstance(fst_stmt.value, ast.Constant): 23 | continue 24 | cst = fst_stmt.value 25 | if not isinstance(cst.value, str): 26 | continue 27 | if re.match(pattern, cst.value, flags): 28 | deprecated.append(stmt.name) 29 | return deprecated 30 | -------------------------------------------------------------------------------- /memestra/memestra.py: -------------------------------------------------------------------------------- 1 | import beniget 2 | import gast as ast 3 | import os 4 | import sys 5 | import warnings 6 | 7 | from importlib.util import resolve_name 8 | from collections import defaultdict 9 | from memestra.caching import Cache, CacheKeyFactory, RecursiveCacheKeyFactory 10 | from memestra.caching import Format 11 | from memestra.utils import resolve_module 12 | import frilouz 13 | 14 | _defs = ast.AsyncFunctionDef, ast.ClassDef, ast.FunctionDef 15 | 16 | class DeprecatedStar(object): 17 | 'Representation of a deprecated node imported through *' 18 | 19 | def __init__(self, name, dnode): 20 | assert dnode.name == '*' 21 | self.name = name 22 | self.dnode = dnode 23 | 24 | def symbol_name(sym): 25 | if getattr(sym, 'asname', None): 26 | return sym.asname 27 | if hasattr(sym, 'name'): 28 | return sym.name 29 | if hasattr(sym, 'id'): 30 | return sym.id 31 | raise NotImplementedError(sym) 32 | 33 | def make_deprecated(node, reason=None): 34 | return (node, reason) 35 | 36 | def store_deprecated(name, reason): 37 | return name if reason is None else '{}:{}'.format(name, reason) 38 | 39 | def load_deprecated(entry): 40 | if ':' in entry: 41 | return entry.split(':', maxsplit=1) 42 | else: 43 | return entry, None 44 | 45 | class SilentDefUseChains(beniget.DefUseChains): 46 | 47 | def unbound_identifier(self, name, node): 48 | pass 49 | 50 | 51 | class SilentDefUseChains(beniget.DefUseChains): 52 | 53 | def unbound_identifier(self, name, node): 54 | pass 55 | 56 | 57 | class ImportResolver(ast.NodeVisitor): 58 | 59 | def __init__(self, decorator, reason_keyword, search_paths=None, 60 | recursive=False, parent=None, pkg_name=None, 61 | cache_dir=None): 62 | ''' 63 | Create an ImportResolver that finds deprecated identifiers. 64 | 65 | A deprecated identifier is an identifier which is decorated 66 | by `decorator', or which uses a deprecated identifier. 67 | 68 | if `recursive' is greater than 0, it considers identifiers 69 | from imported module, with that depth in the import tree. 70 | 71 | `parent' is used internally to handle imports. 72 | ''' 73 | self.deprecated = None 74 | self.decorator = tuple(decorator) 75 | self.search_paths = search_paths 76 | self.recursive = recursive 77 | self.reason_keyword = reason_keyword 78 | self.pkg_name = pkg_name 79 | if parent: 80 | self.cache = parent.cache 81 | self.visited = parent.visited 82 | self.key_factory = parent.key_factory 83 | else: 84 | self.cache = Cache(cache_dir=cache_dir) 85 | self.visited = set() 86 | if recursive: 87 | self.key_factory = RecursiveCacheKeyFactory() 88 | else: 89 | self.key_factory = CacheKeyFactory() 90 | 91 | def load_deprecated_from_module(self, module_name, level=None): 92 | # level may be none when it's taken from the ImportFrom node 93 | if level is None: 94 | level = 0 95 | 96 | # update module/pkg based on level 97 | rmodule_name = '.' * level + module_name 98 | 99 | # perform the module lookup 100 | try: 101 | module_name = resolve_name(rmodule_name, self.pkg_name) 102 | except (ImportError, ValueError): 103 | return None 104 | module_path = resolve_module(module_name, self.search_paths) 105 | 106 | # hopefully a module was found 107 | if module_path is None: 108 | return None 109 | 110 | module_key = self.key_factory(module_path, name_hint=module_name) 111 | 112 | # either find it in the cache 113 | if module_key in self.cache: 114 | data = self.cache[module_key] 115 | if data['version'] == Format.version: 116 | return dict(load_deprecated(entry) 117 | for entry in data['deprecated']) 118 | elif data['generator'] == 'manual': 119 | warnings.warn( 120 | ("skipping module {} because it has an obsolete, " 121 | "manually generated, cache file: {}") 122 | .format(module_name, 123 | module_key.module_hash)) 124 | return {} 125 | 126 | # or fill a new entry 127 | 128 | # To avoid loop, mark the key as in process 129 | self.cache[module_key] = {'generator': 'manual', 130 | 'deprecated': []} 131 | 132 | with open(module_path) as fd: 133 | try: 134 | module, syntax_errors = frilouz.parse(ast.parse, fd.read()) 135 | except UnicodeDecodeError: 136 | return {} 137 | duc = SilentDefUseChains() 138 | duc.visit(module) 139 | anc = beniget.Ancestors() 140 | anc.visit(module) 141 | 142 | # Collect deprecated functions 143 | if self.recursive and module_path not in self.visited: 144 | self.visited.add(module_path) 145 | current_pkg = ".".join(module_name.split('.')[:-1]) 146 | resolver = ImportResolver(self.decorator, 147 | self.reason_keyword, 148 | self.search_paths, 149 | self.recursive, 150 | parent=self, 151 | pkg_name=current_pkg) 152 | resolver.visit(module) 153 | deprecated_imports = [make_deprecated(d, reason) 154 | for _, _, d, reason in 155 | resolver.get_deprecated_users(duc, anc)] 156 | else: 157 | deprecated_imports = [] 158 | deprecated = self.collect_deprecated(module, duc, anc, 159 | pkg_name=module_name) 160 | deprecated.update(deprecated_imports) 161 | dl = {symbol_name(d[0]): d[1] for d in deprecated if d is not None} 162 | data = {'generator': 'memestra', 163 | 'deprecated': [store_deprecated(d, dl[d]) for d in 164 | sorted(dl)]} 165 | self.cache[module_key] = data 166 | return dl 167 | 168 | def get_deprecated_users(self, defuse, ancestors): 169 | deprecated_uses = [] 170 | visited = set() 171 | worklist = list(self.deprecated) 172 | while worklist: 173 | deprecated_node, reason = worklist.pop() 174 | if deprecated_node in visited: 175 | continue 176 | visited.add(deprecated_node) 177 | 178 | # special node: an imported name 179 | if isinstance(deprecated_node, ast.alias): 180 | deprecated_uses.append((deprecated_node, 181 | ancestors.parent(deprecated_node), 182 | deprecated_node, reason)) 183 | 184 | else: 185 | # There's a special handler in ImportFrom for these 186 | if isinstance(deprecated_node, DeprecatedStar): 187 | continue 188 | 189 | for user in defuse.chains[deprecated_node].users(): 190 | user_ancestors = [n 191 | for n in ancestors.parents(user.node) 192 | if isinstance(n, _defs)] 193 | if any(f in self.deprecated for f in user_ancestors): 194 | continue 195 | 196 | user_ancestor = user_ancestors[-1] if user_ancestors else user.node 197 | deprecated_uses.append((deprecated_node, user.node, 198 | user_ancestor, 199 | reason)) 200 | if self.recursive and isinstance(user_ancestor, _defs): 201 | worklist.append((user_ancestor, reason)) 202 | return deprecated_uses 203 | 204 | def visit_Import(self, node): 205 | for alias in node.names: 206 | deprecated = self.load_deprecated_from_module(alias.name) 207 | if deprecated is None: 208 | continue 209 | 210 | for user in self.def_use_chains.chains[alias].users(): 211 | parent = self.ancestors.parents(user.node)[-1] 212 | if isinstance(parent, ast.Attribute): 213 | if parent.attr in deprecated: 214 | reason = deprecated[parent.attr] 215 | self.deprecated.add(make_deprecated(parent, reason)) 216 | 217 | def visit_ImportFrom(self, node): 218 | if node.module is None: 219 | return None 220 | deprecated = self.load_deprecated_from_module(node.module, 221 | level=node.level) 222 | if deprecated is None: 223 | return 224 | 225 | aliases = [alias.name for alias in node.names] 226 | 227 | try: 228 | # if we're importing *, pick all deprecated items 229 | star_alias_index = aliases.index('*') 230 | star_alias = node.names[star_alias_index] 231 | 232 | def alias_extractor(deprec): 233 | return star_alias 234 | 235 | def alias_selector(deprec, node): 236 | return getattr(node, 'id', None) == deprec 237 | 238 | except ValueError: 239 | # otherwise only pick the imported one 240 | 241 | def alias_extractor(deprec): 242 | try: 243 | index = aliases.index(deprec) 244 | alias = node.names[index] 245 | return alias 246 | except ValueError: 247 | return None 248 | 249 | def alias_selector(deprec, node): 250 | return True 251 | 252 | for deprec, reason in deprecated.items(): 253 | deprec_alias = alias_extractor(deprec) 254 | if deprec_alias is None: 255 | continue 256 | 257 | for user in self.def_use_chains.chains[deprec_alias].users(): 258 | # if deprec_alias is '*' we need to be selective 259 | if not alias_selector(deprec, user.node): 260 | continue 261 | self.deprecated.add(make_deprecated(user.node, reason)) 262 | 263 | def visit_Module(self, node): 264 | duc = SilentDefUseChains() 265 | duc.visit(node) 266 | self.def_use_chains = duc 267 | 268 | ancestors = beniget.Ancestors() 269 | ancestors.visit(node) 270 | self.ancestors = ancestors 271 | 272 | self.deprecated = self.collect_deprecated(node, duc, ancestors) 273 | self.generic_visit(node) 274 | 275 | def collect_deprecated(self, node, duc, ancestors, pkg_name=None): 276 | deprecated = set() 277 | 278 | for dlocal in duc.locals[node]: 279 | dnode = dlocal.node 280 | if not isinstance(dnode, ast.alias): 281 | continue 282 | 283 | original_path = tuple(dnode.name.split('.')) 284 | 285 | # for imports with a "from" clause, such as 286 | # 287 | # from foo import bar 288 | # 289 | # the AST alias will be just `bar`, but we want any functions 290 | # defined as such: 291 | # 292 | # @bar 293 | # def foo(): pass 294 | # 295 | # to be picked up when `foo.bar` is used as the target decorator. we 296 | # check if the parent of the alias is an ImportFrom node and fix the 297 | # original path to be fully qualified here. In the example above, it 298 | # becomes `foo.bar` instead of just `bar`. 299 | alias_parent = ancestors.parents(dnode)[-1] 300 | if isinstance(alias_parent, ast.ImportFrom): 301 | module = "." 302 | # A module can be None if a relative import from "." occurs 303 | if alias_parent.module is not None: 304 | module = resolve_name(alias_parent.module, self.pkg_name) 305 | 306 | original_path = tuple(module.split('.')) + original_path 307 | 308 | nbterms = len(original_path) 309 | 310 | if original_path == self.decorator[:nbterms]: 311 | for user in dlocal.users(): 312 | parents = list(ancestors.parents(user.node)) 313 | attrs = list(reversed(self.decorator[nbterms:])) 314 | while attrs and parents: 315 | attr = attrs[-1] 316 | parent = parents.pop() 317 | if not isinstance(parent, (ast.Attribute)): 318 | break 319 | if parent.attr != attr: 320 | break 321 | attrs.pop() 322 | 323 | # path parsing fails if some attr left 324 | if attrs: 325 | continue 326 | 327 | # Only handle decorators attached to a def 328 | self.extract_decorator_from_parents( 329 | parents, 330 | deprecated) 331 | 332 | 333 | if not self.recursive: 334 | return deprecated 335 | 336 | # In recursive mode, we consider deprecated any imported 337 | # deprecated symbol. 338 | for dlocal in duc.locals[node]: 339 | dnode = dlocal.node 340 | if not isinstance(dnode, ast.alias): 341 | continue 342 | alias_parent = ancestors.parents(dnode)[-1] 343 | if isinstance(alias_parent, ast.ImportFrom): 344 | if not alias_parent.module: 345 | continue 346 | resolver = ImportResolver(self.decorator, 347 | self.reason_keyword, 348 | self.search_paths, 349 | self.recursive, 350 | self, 351 | pkg_name) 352 | 353 | imported_deprecated = resolver.load_deprecated_from_module( 354 | alias_parent.module, 355 | level=alias_parent.level) 356 | if not imported_deprecated: 357 | continue 358 | if dnode.name in imported_deprecated: 359 | dinfo = make_deprecated(dnode, 360 | imported_deprecated[dnode.name]) 361 | deprecated.add(dinfo) 362 | elif dnode.name == '*': 363 | for name, reason in imported_deprecated.items(): 364 | dinfo = make_deprecated(DeprecatedStar(name, dnode), 365 | reason) 366 | deprecated.add(dinfo) 367 | 368 | return deprecated 369 | 370 | def extract_decorator_from_parents(self, parents, deprecated): 371 | parent = parents[-1] 372 | if isinstance(parent, _defs): 373 | deprecated.add(make_deprecated(parent)) 374 | return 375 | if len(parents) == 1: 376 | return 377 | parent_p = parents[-2] 378 | if isinstance(parent, ast.Call) and isinstance(parent_p, _defs): 379 | reason = None 380 | # Output only the specified reason with the --reason-keyword flag 381 | if not parent.keywords and parent.args: 382 | reason = parent.args[0].value 383 | for keyword in parent.keywords: 384 | if self.reason_keyword == keyword.arg: 385 | reason = keyword.value.value 386 | deprecated.add(make_deprecated(parent_p, reason)) 387 | return 388 | 389 | def prettyname(node): 390 | if isinstance(node, _defs): 391 | return node.name 392 | if isinstance(node, ast.alias): 393 | return node.asname or node.name 394 | if isinstance(node, ast.Name): 395 | return node.id 396 | if isinstance(node, ast.alias): 397 | return node.asname or node.name 398 | if isinstance(node, ast.Attribute): 399 | return prettyname(node.value) + '.' + node.attr 400 | return repr(node) 401 | 402 | 403 | def memestra(file_descriptor, decorator, reason_keyword, 404 | search_paths=None, recursive=False, cache_dir=None): 405 | ''' 406 | Parse `file_descriptor` and returns a list of 407 | (function, filename, line, colno) tuples. Each elements 408 | represents a code location where a deprecated function is used. 409 | A deprecated function is a function flagged by `decorator`, where 410 | `decorator` is a tuple representing an import path, 411 | e.g. (module, attribute) 412 | 413 | If `recursive` is set to `True`, deprecated use are 414 | checked recursively throughout the *whole* module import tree. Otherwise, 415 | only one level of import is checked. 416 | ''' 417 | 418 | assert not isinstance(decorator, str) and \ 419 | len(decorator) > 1, "decorator is at least (module, attribute)" 420 | 421 | module, syntax_errors = frilouz.parse(ast.parse, file_descriptor.read()) 422 | # Collect deprecated functions 423 | resolver = ImportResolver(decorator, reason_keyword, search_paths, 424 | recursive, cache_dir=cache_dir) 425 | resolver.visit(module) 426 | 427 | ancestors = resolver.ancestors 428 | duc = resolver.def_use_chains 429 | 430 | # Find their users 431 | formated_deprecated = [] 432 | for deprecated_info in resolver.get_deprecated_users(duc, ancestors): 433 | deprecated_node, user_node, _, reason = deprecated_info 434 | formated_deprecated.append((prettyname(deprecated_node), 435 | getattr(file_descriptor, 'name', '<>'), 436 | user_node.lineno, 437 | user_node.col_offset, 438 | reason)) 439 | formated_deprecated.sort() 440 | return formated_deprecated 441 | 442 | 443 | def run(): 444 | 445 | import argparse 446 | from pkg_resources import iter_entry_points 447 | 448 | parser = argparse.ArgumentParser(description='Check decorator usage.') 449 | parser.add_argument('--decorator', dest='decorator', 450 | default='deprecated.deprecated', 451 | help='Path to the decorator to check') 452 | parser.add_argument('input', type=argparse.FileType('r'), 453 | help='file to scan') 454 | parser.add_argument('--reason-keyword', dest='reason_keyword', 455 | default='reason', 456 | action='store', 457 | help='Specify keyword for deprecation reason') 458 | parser.add_argument('--cache-dir', dest='cache_dir', 459 | default=None, 460 | action='store', 461 | help='The directory where the cache is located') 462 | parser.add_argument('--recursive', dest='recursive', 463 | action='store_true', 464 | help='Traverse the whole module hierarchy') 465 | 466 | dispatcher = defaultdict(lambda: memestra) 467 | for entry_point in iter_entry_points(group='memestra.plugins', name=None): 468 | entry_point.load()(dispatcher) 469 | 470 | args = parser.parse_args() 471 | 472 | _, extension = os.path.splitext(args.input.name) 473 | 474 | # Add the directory of the python file to the list of import paths 475 | # to search. 476 | search_paths = [os.path.dirname(os.path.abspath(args.input.name))] 477 | 478 | deprecate_uses = dispatcher[extension](args.input, 479 | args.decorator.split('.'), 480 | args.reason_keyword, 481 | search_paths, 482 | args.recursive, 483 | args.cache_dir) 484 | 485 | for fname, fd, lineno, colno, reason in deprecate_uses: 486 | formatted_reason = "" 487 | if reason: 488 | formatted_reason = " - {}".format(reason) 489 | print("{} used at {}:{}:{}{}".format(fname, fd, lineno, colno + 1, formatted_reason)) 490 | 491 | 492 | if __name__ == '__main__': 493 | run() 494 | -------------------------------------------------------------------------------- /memestra/nbmemestra.py: -------------------------------------------------------------------------------- 1 | from memestra import memestra 2 | from io import StringIO 3 | from collections import namedtuple 4 | import nbformat 5 | import os 6 | 7 | 8 | def nbmemestra_from_nbnode(nb, decorator, search_paths=None, reason_keyword='reason'): 9 | # Get code cells 10 | cells = nb.cells 11 | code_cells = [c for c in cells if c['cell_type'] == 'code'] 12 | Cell = namedtuple('Cell', 'id begin end') 13 | # Aggregate code cells and generate cells list 14 | code_list = [] 15 | cell_list = [] 16 | current_line = 1 17 | for (num, c) in enumerate(code_cells): 18 | nb_lines = c['source'].count('\n') + 1 19 | code_list.append(c['source']) 20 | end_line = current_line + nb_lines 21 | cell_list.append(Cell(num, current_line, end_line)) 22 | current_line = end_line 23 | code = '\n'.join(code_list) 24 | 25 | # Collect calls to deprecated functions 26 | deprecated_list = memestra(StringIO(code), decorator, reason_keyword, 27 | search_paths) 28 | 29 | # Map them to cells 30 | result = [] 31 | for d in deprecated_list: 32 | cell = next(x for x in cell_list if x.begin <= d[2] and d[2] < x.end) 33 | result.append((d[0], 34 | 'Cell[' + str(cell.id) + ']', d[2] - cell.begin + 1, 35 | d[3])) 36 | 37 | return result 38 | 39 | 40 | def nbmemestra(nbfile, decorator, reason_keyword='reason'): 41 | nb = nbformat.read(nbfile, 4) 42 | return nbmemestra_from_nbnode(nb, decorator, 43 | [os.path.dirname(nbfile)], reason_keyword) 44 | 45 | 46 | def register(dispatcher): 47 | dispatcher['.ipynb'] = nbmemestra 48 | -------------------------------------------------------------------------------- /memestra/preprocessor.py: -------------------------------------------------------------------------------- 1 | import re 2 | from nbconvert.preprocessors import Preprocessor 3 | from memestra import nbmemestra 4 | from traitlets import Tuple 5 | 6 | 7 | class MemestraDeprecationChecker(Preprocessor): 8 | """A memestra-based preprocessor that checks code for places where deprecated functions are called""" 9 | 10 | decorator = Tuple(help="path to import the deprecation decorator, e.g. ('module', 'attribute')").tag(config=True) 11 | 12 | def preprocess(self, nb, resources): 13 | self.log.info("Using decorator '%s.%s' to check deprecated functions", self.decorator[0], self.decorator[1]) 14 | deprecations = {} 15 | for d in nbmemestra.nbmemestra_from_nbnode(nb, self.decorator): 16 | code_cell_i = int(re.search(r"\[(\d+)\]", d[1]).group(1)) 17 | deprecation = deprecations.get(code_cell_i, []) 18 | deprecation.append(d) 19 | deprecations[code_cell_i] = deprecation 20 | # filter out non-code cell as it is done in nbmemestra 21 | code_cell_i = -1 22 | for cell in nb.cells: 23 | if cell['cell_type'] != 'code': 24 | continue 25 | code_cell_i += 1 26 | if code_cell_i not in deprecations: 27 | continue 28 | for d in deprecations[code_cell_i]: 29 | line, col = d[2:4] 30 | outputs = cell.get('outputs', []) 31 | message = ('On line {}:\n' 32 | '{}\n' 33 | '{}^\n' 34 | 'Warning: call to deprecated function {}'.format( 35 | line, 36 | cell['source'].split('\n')[line - 1], 37 | ' ' * col, 38 | d[0] 39 | )) 40 | 41 | outputs.append({'output_type': 'stream', 'name': 'stderr', 'text': message}) 42 | cell['outputs'] = outputs 43 | return nb, resources 44 | -------------------------------------------------------------------------------- /memestra/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from importlib import util 4 | from importlib.abc import SourceLoader 5 | 6 | def _resolve_module(module_name, additional_search_paths=None): 7 | if additional_search_paths is None: 8 | additional_search_paths = [] 9 | 10 | if module_name is None or module_name == "__main__": 11 | return None, None 12 | 13 | parent, _, module_name = module_name.rpartition(".") 14 | search_path = sys.path + additional_search_paths 15 | 16 | if parent != "": 17 | parent_spec, search_path = _resolve_module(parent) 18 | if parent_spec is None: 19 | return None, None 20 | 21 | try: 22 | spec = None 23 | for finder in sys.meta_path: 24 | try: 25 | spec = finder.find_spec(module_name, search_path) 26 | except AttributeError: 27 | pass 28 | 29 | if spec is not None: 30 | break 31 | 32 | origin = spec.origin if spec else None 33 | if spec is None or spec.origin is None: 34 | return None, None 35 | if isinstance(spec.loader, SourceLoader): 36 | return spec.origin, spec.submodule_search_locations 37 | else: 38 | return None, None 39 | 40 | except ModuleNotFoundError: 41 | return None, [] 42 | 43 | def resolve_module(module_name, additional_search_paths=None): 44 | return _resolve_module(module_name, additional_search_paths)[0] 45 | -------------------------------------------------------------------------------- /memestra/version.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.2.1' 2 | __url__ = 'https://github.com/QuantStack/memestra' 3 | __descr__ = 'Memestra checks code for places where deprecated functions are called' 4 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | gast 2 | beniget 3 | nbformat 4 | nbconvert 5 | pyyaml 6 | frilouz 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from setuptools import setup 3 | 4 | kw = {"test_suite": "tests"} 5 | 6 | dev_reqs = open("requirements-dev.txt").read().splitlines() 7 | extras_require = {"test": dev_reqs, "dev": dev_reqs} 8 | 9 | versionfile = os.path.join('memestra', 'version.py') 10 | exec(open(versionfile).read()) 11 | 12 | setup( 13 | name="memestra", 14 | version=__version__, 15 | packages=["memestra"], 16 | description=__descr__, 17 | long_description=""" 18 | A linter that tracks reference to deprecated functions. 19 | 20 | Memestra walks through your code base and tracks reference to function marks 21 | with a given decorator.""", 22 | author="serge-sans-paille", 23 | author_email="serge.guelton@telecom-bretagne.eu", 24 | url=__url__, 25 | license="BSD 3-Clause", 26 | install_requires=open("requirements.txt").read().splitlines(), 27 | extras_require=extras_require, 28 | entry_points={'console_scripts': 29 | ['memestra = memestra.memestra:run', 30 | 'memestra-cache = memestra.caching:run'], 31 | 'memestra.plugins': 32 | [".ipynb = memestra.nbmemestra:register", ], 33 | }, 34 | classifiers=[ 35 | "Development Status :: 3 - Alpha", 36 | "Environment :: Console", 37 | "Intended Audience :: Developers", 38 | "License :: OSI Approved :: BSD License", 39 | "Natural Language :: English", 40 | "Programming Language :: Python :: 2", 41 | "Programming Language :: Python :: 2.7", 42 | "Programming Language :: Python :: 3", 43 | "Programming Language :: Python :: 3.4", 44 | "Programming Language :: Python :: 3.5", 45 | "Programming Language :: Python :: 3.6", 46 | ], 47 | python_requires=">=3.4", 48 | **kw 49 | ) 50 | -------------------------------------------------------------------------------- /tests/misc/README.rst: -------------------------------------------------------------------------------- 1 | This directory contains a bunch of dummy packages / files 2 | 3 | These files are only used to test the import mechanism within memestra. 4 | -------------------------------------------------------------------------------- /tests/misc/decoratortest.py: -------------------------------------------------------------------------------- 1 | from decorator import decorate 2 | 3 | def _deprecated(func, *args, **kw): 4 | print('warning:', func.__name__, 'has been deprecated') 5 | return func(*args, **kw) 6 | 7 | def deprecated(func): 8 | return decorate(func, _deprecated) 9 | -------------------------------------------------------------------------------- /tests/misc/ipy/__init__.py: -------------------------------------------------------------------------------- 1 | from .a import * 2 | 3 | def useless(): 4 | pass 5 | -------------------------------------------------------------------------------- /tests/misc/ipy/a/__init__.py: -------------------------------------------------------------------------------- 1 | from .b import foo 2 | -------------------------------------------------------------------------------- /tests/misc/ipy/a/b.py: -------------------------------------------------------------------------------- 1 | from decoratortest import deprecated 2 | 3 | @deprecated("why") 4 | def foo():pass 5 | -------------------------------------------------------------------------------- /tests/misc/memestra_nb_demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "warning: foo has been deprecated\n", 13 | "warning: foo has been deprecated\n" 14 | ] 15 | } 16 | ], 17 | "source": [ 18 | "import some_module\n", 19 | "some_module.foo()\n", 20 | "some_module.foo()" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": 2, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "some_module.bar()" 30 | ] 31 | }, 32 | { 33 | "cell_type": "markdown", 34 | "metadata": {}, 35 | "source": [ 36 | "some text" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": 3, 42 | "metadata": {}, 43 | "outputs": [ 44 | { 45 | "name": "stdout", 46 | "output_type": "stream", 47 | "text": [ 48 | "warning: foo has been deprecated\n" 49 | ] 50 | } 51 | ], 52 | "source": [ 53 | "if True:\n", 54 | " some_module.foo()\n", 55 | "\n", 56 | "\n" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": null, 62 | "metadata": {}, 63 | "outputs": [], 64 | "source": [] 65 | } 66 | ], 67 | "metadata": { 68 | "kernelspec": { 69 | "display_name": "Python 3", 70 | "language": "python", 71 | "name": "python3" 72 | }, 73 | "language_info": { 74 | "codemirror_mode": { 75 | "name": "ipython", 76 | "version": 3 77 | }, 78 | "file_extension": ".py", 79 | "mimetype": "text/x-python", 80 | "name": "python", 81 | "nbconvert_exporter": "python", 82 | "pygments_lexer": "ipython3", 83 | "version": "3.8.2" 84 | } 85 | }, 86 | "nbformat": 4, 87 | "nbformat_minor": 4 88 | } 89 | -------------------------------------------------------------------------------- /tests/misc/memestra_nb_demo.rst: -------------------------------------------------------------------------------- 1 | .. code:: ipython3 2 | 3 | import some_module 4 | some_module.foo() 5 | some_module.foo() 6 | 7 | 8 | .. parsed-literal:: 9 | 10 | warning: foo has been deprecated 11 | warning: foo has been deprecated 12 | 13 | 14 | .. parsed-literal:: 15 | 16 | On line 2: 17 | some_module.foo() 18 | ^ 19 | Warning: call to deprecated function some_module.foo 20 | 21 | .. parsed-literal:: 22 | 23 | On line 3: 24 | some_module.foo() 25 | ^ 26 | Warning: call to deprecated function some_module.foo 27 | 28 | .. code:: ipython3 29 | 30 | some_module.bar() 31 | 32 | some text 33 | 34 | .. code:: ipython3 35 | 36 | if True: 37 | some_module.foo() 38 | 39 | 40 | 41 | 42 | 43 | .. parsed-literal:: 44 | 45 | warning: foo has been deprecated 46 | 47 | 48 | .. parsed-literal:: 49 | 50 | On line 2: 51 | some_module.foo() 52 | ^ 53 | Warning: call to deprecated function some_module.foo 54 | 55 | -------------------------------------------------------------------------------- /tests/misc/module_defining_symbol.py: -------------------------------------------------------------------------------- 1 | import decoratortest 2 | 3 | @decoratortest.deprecated 4 | class Test: 5 | pass 6 | -------------------------------------------------------------------------------- /tests/misc/module_forwarding_all_symbols.py: -------------------------------------------------------------------------------- 1 | from module_defining_symbol import * 2 | -------------------------------------------------------------------------------- /tests/misc/module_forwarding_symbol.py: -------------------------------------------------------------------------------- 1 | from module_defining_symbol import Test 2 | from module_defining_symbol import Test as Testosterone 3 | -------------------------------------------------------------------------------- /tests/misc/phantom.py: -------------------------------------------------------------------------------- 1 | from decoratortest import deprecated 2 | from endless_pit_of_useless_stuff import void 3 | 4 | @deprecated 5 | def empty(): pass 6 | -------------------------------------------------------------------------------- /tests/misc/pkg/__init__.py: -------------------------------------------------------------------------------- 1 | from pkg.test import test 2 | -------------------------------------------------------------------------------- /tests/misc/pkg/helper.py: -------------------------------------------------------------------------------- 1 | from decoratortest import deprecated 2 | 3 | @deprecated 4 | def helper(): 5 | pass 6 | -------------------------------------------------------------------------------- /tests/misc/pkg/sub/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/QuantStack/memestra/646b47b25fa620274d02c3d7922b8b6efa99a178/tests/misc/pkg/sub/__init__.py -------------------------------------------------------------------------------- /tests/misc/pkg/sub/other.py: -------------------------------------------------------------------------------- 1 | from ..helper import helper 2 | def other(): return helper() 3 | -------------------------------------------------------------------------------- /tests/misc/pkg/test.py: -------------------------------------------------------------------------------- 1 | from .helper import helper 2 | def test(): 3 | return helper() 4 | -------------------------------------------------------------------------------- /tests/misc/some_module.py: -------------------------------------------------------------------------------- 1 | from decoratortest import deprecated 2 | 3 | @deprecated 4 | def foo(): pass 5 | 6 | def bar(): pass 7 | 8 | @deprecated("because it's too old") 9 | def foobar(): pass 10 | -------------------------------------------------------------------------------- /tests/misc/some_rec_module.py: -------------------------------------------------------------------------------- 1 | import some_module 2 | 3 | def foo(): 4 | return some_module.foo() 5 | 6 | def bar(): 7 | return some_module.bar() 8 | 9 | def foobar(): 10 | return foo(), bar() 11 | -------------------------------------------------------------------------------- /tests/notebook.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from memestra import nbmemestra 3 | 4 | class NotebookTest(TestCase): 5 | 6 | def test_nb_demo(self): 7 | output = nbmemestra.nbmemestra("tests/misc/memestra_nb_demo.ipynb", ('decoratortest', 'deprecated'), "tests/misc/memestra_nb_demo.ipynb") 8 | expected_output = [('some_module.foo', 'Cell[0]', 2, 0), 9 | ('some_module.foo', 'Cell[0]', 3, 0), 10 | ('some_module.foo', 'Cell[2]', 1, 0)] 11 | self.assertEqual(output, expected_output) 12 | -------------------------------------------------------------------------------- /tests/share/memestra/gast/__init__.yml: -------------------------------------------------------------------------------- 1 | deprecated: ['parse'] 2 | generator: manual 3 | name: gast.__init__ 4 | version: 1 5 | -------------------------------------------------------------------------------- /tests/share/memestra/gast/astn.yml: -------------------------------------------------------------------------------- 1 | deprecated: ['AstToGAst:because'] 2 | generator: manual 3 | name: gast.astn 4 | version: 1 5 | -------------------------------------------------------------------------------- /tests/test_basic.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from textwrap import dedent 3 | from io import StringIO 4 | import memestra 5 | 6 | class TestBasic(TestCase): 7 | 8 | def checkDeprecatedUses(self, code, expected_output, decorator=('decoratortest', 'deprecated')): 9 | sio = StringIO(dedent(code)) 10 | output = memestra.memestra(sio, decorator, None) 11 | self.assertEqual(output, expected_output) 12 | 13 | 14 | def test_import(self): 15 | code = ''' 16 | import decoratortest 17 | 18 | @decoratortest.deprecated 19 | def foo(): pass 20 | 21 | def bar(): 22 | foo() 23 | 24 | foo()''' 25 | 26 | self.checkDeprecatedUses( 27 | code, 28 | [('foo', '<>', 8, 4, None), ('foo', '<>', 10, 0, None)]) 29 | 30 | def test_import_alias(self): 31 | code = ''' 32 | import decoratortest as dec 33 | 34 | @dec.deprecated 35 | def foo(): pass 36 | 37 | def bar(): 38 | foo() 39 | 40 | foo()''' 41 | 42 | self.checkDeprecatedUses( 43 | code, 44 | [('foo', '<>', 8, 4, None), ('foo', '<>', 10, 0, None)]) 45 | 46 | def test_import_from(self): 47 | code = ''' 48 | from decoratortest import deprecated 49 | 50 | @deprecated 51 | def foo(): pass 52 | 53 | def bar(): 54 | foo() 55 | 56 | foo()''' 57 | 58 | self.checkDeprecatedUses( 59 | code, 60 | [('foo', '<>', 8, 4, None), ('foo', '<>', 10, 0, None)]) 61 | 62 | def test_import_from_alias(self): 63 | code = ''' 64 | from decoratortest import deprecated as dp 65 | 66 | @dp 67 | def foo(): pass 68 | 69 | def bar(): 70 | foo() 71 | 72 | foo()''' 73 | 74 | self.checkDeprecatedUses( 75 | code, 76 | [('foo', '<>', 8, 4, None), ('foo', '<>', 10, 0, None)]) 77 | 78 | def test_call_from_deprecated(self): 79 | code = ''' 80 | from decoratortest import deprecated as dp 81 | 82 | @dp 83 | def foo(): pass 84 | 85 | @dp 86 | def bar(): 87 | foo() 88 | 89 | foo()''' 90 | 91 | self.checkDeprecatedUses( 92 | code, 93 | [('foo', '<>', 9, 4, None), ('foo', '<>', 11, 0, None)]) 94 | 95 | def test_import_from_same_module_and_decorator(self): 96 | code = ''' 97 | from deprecated import deprecated 98 | 99 | @deprecated 100 | def foo(): pass 101 | 102 | def bar(): 103 | foo() 104 | 105 | foo()''' 106 | 107 | self.checkDeprecatedUses( 108 | code, 109 | [('foo', '<>', 8, 4, None), ('foo', '<>', 10, 0, None)], 110 | ('deprecated', 'deprecated')) 111 | 112 | 113 | class TestClassBasic(TestCase): 114 | 115 | def checkDeprecatedUses(self, code, expected_output): 116 | sio = StringIO(dedent(code)) 117 | output = memestra.memestra(sio, ('decoratortest', 'deprecated'), None) 118 | self.assertEqual(output, expected_output) 119 | 120 | 121 | def test_import(self): 122 | code = ''' 123 | import decoratortest 124 | 125 | @decoratortest.deprecated 126 | class foo: pass 127 | 128 | def bar(): 129 | foo() 130 | 131 | foo()''' 132 | 133 | self.checkDeprecatedUses( 134 | code, 135 | [('foo', '<>', 8, 4, None), ('foo', '<>', 10, 0, None)]) 136 | 137 | def test_import_alias(self): 138 | code = ''' 139 | import decoratortest as dec 140 | 141 | @dec.deprecated 142 | class foo: pass 143 | 144 | def bar(): 145 | foo() 146 | 147 | foo()''' 148 | 149 | self.checkDeprecatedUses( 150 | code, 151 | [('foo', '<>', 8, 4, None), ('foo', '<>', 10, 0, None)]) 152 | 153 | def test_import_from(self): 154 | code = ''' 155 | from decoratortest import deprecated 156 | 157 | @deprecated 158 | class foo: pass 159 | 160 | def bar(): 161 | foo() 162 | 163 | foo()''' 164 | 165 | self.checkDeprecatedUses( 166 | code, 167 | [('foo', '<>', 8, 4, None), ('foo', '<>', 10, 0, None)]) 168 | 169 | def test_import_from_alias(self): 170 | code = ''' 171 | from decoratortest import deprecated as dp 172 | 173 | @dp 174 | class foo(object): pass 175 | 176 | def bar(): 177 | foo() 178 | 179 | foo()''' 180 | 181 | self.checkDeprecatedUses( 182 | code, 183 | [('foo', '<>', 8, 4, None), ('foo', '<>', 10, 0, None)]) 184 | 185 | def test_instance_from_deprecated(self): 186 | code = ''' 187 | from decoratortest import deprecated as dp 188 | 189 | @dp 190 | class foo(object): pass 191 | 192 | @dp 193 | def bar(): 194 | foo() 195 | 196 | foo()''' 197 | 198 | self.checkDeprecatedUses( 199 | code, 200 | [('foo', '<>', 9, 4, None), ('foo', '<>', 11, 0, None)]) 201 | 202 | def test_use_in_inheritance(self): 203 | code = ''' 204 | from decoratortest import deprecated as dp 205 | 206 | @dp 207 | class foo(object): pass 208 | 209 | class bar(foo): pass 210 | ''' 211 | 212 | self.checkDeprecatedUses( 213 | code, 214 | [('foo', '<>', 7, 10, None)]) 215 | 216 | def test_instance_from_deprecated_class(self): 217 | code = ''' 218 | from decoratortest import deprecated as dp 219 | 220 | @dp 221 | class foo(object): pass 222 | 223 | @dp 224 | class bar(object): 225 | foo() 226 | 227 | foo()''' 228 | 229 | self.checkDeprecatedUses( 230 | code, 231 | [('foo', '<>', 9, 4, None), ('foo', '<>', 11, 0, None)]) 232 | 233 | def test_decorator_with_param(self): 234 | code = ''' 235 | from decoratortest import deprecated as dp 236 | 237 | @dp() 238 | class foo(object): pass 239 | 240 | @dp("ignored") 241 | def bar(x): 242 | foo() 243 | 244 | bar(foo)''' 245 | 246 | self.checkDeprecatedUses( 247 | code, 248 | [('bar', '<>', 11, 0, 'ignored'), 249 | ('foo', '<>', 9, 4, None), 250 | ('foo', '<>', 11, 4, None)]) 251 | -------------------------------------------------------------------------------- /tests/test_basic_decorator_deprecated.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | from textwrap import dedent 3 | import tempfile 4 | import contextlib 5 | from io import StringIO 6 | 7 | import memestra 8 | 9 | import os 10 | import sys 11 | 12 | TESTS_PATHS = [os.path.abspath(os.path.join(os.path.dirname(__file__), 'misc'))] 13 | 14 | class TestBasic(TestCase): 15 | 16 | def checkDeprecatedUses(self, code, expected_output): 17 | sio = StringIO(dedent(code)) 18 | output = memestra.memestra(sio, ('deprecated', 'deprecated'), 'reason', 19 | search_paths=TESTS_PATHS) 20 | self.assertEqual(output, expected_output) 21 | 22 | 23 | def test_default_kwarg(self): 24 | code = ''' 25 | import deprecated 26 | 27 | @deprecated.deprecated(reason='use another function') 28 | def foo(): pass 29 | 30 | foo()''' 31 | 32 | self.checkDeprecatedUses( 33 | code, 34 | [('foo', '<>', 7, 0, 'use another function')]) 35 | 36 | def test_no_keyword(self): 37 | code = ''' 38 | import deprecated 39 | 40 | @deprecated.deprecated('use another function') 41 | def foo(): pass 42 | 43 | foo()''' 44 | 45 | self.checkDeprecatedUses(code, 46 | [('foo', '<>', 7, 0, 'use another function')]) 47 | 48 | def test_multiple_args(self): 49 | code = ''' 50 | import deprecated 51 | 52 | @deprecated.deprecated(unrelated='unrelated content', 53 | type='magical', reason='another reason') 54 | def foo(): pass 55 | 56 | foo()''' 57 | 58 | self.checkDeprecatedUses( 59 | code, 60 | [('foo', '<>', 8, 0, 'another reason')]) 61 | 62 | 63 | class TestCLI(TestCase): 64 | 65 | def test_default_kwarg(self): 66 | fid, tmppy = tempfile.mkstemp(suffix='.py') 67 | code = ''' 68 | import deprecated 69 | 70 | @deprecated.deprecated(reason='use another function') 71 | def foo(): pass 72 | 73 | foo()''' 74 | 75 | ref = 'foo used at {}:7:1 - use another function\n'.format(tmppy) 76 | os.write(fid, dedent(code).encode()) 77 | os.close(fid) 78 | test_args = ['memestra', tmppy] 79 | with mock.patch.object(sys, 'argv', test_args): 80 | from memestra.memestra import run 81 | with StringIO() as buf: 82 | with contextlib.redirect_stdout(buf): 83 | run() 84 | self.assertEqual(buf.getvalue(), ref) 85 | -------------------------------------------------------------------------------- /tests/test_caching.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | from textwrap import dedent 3 | import os 4 | import sys 5 | import tempfile 6 | import shutil 7 | import contextlib 8 | from io import StringIO 9 | 10 | import memestra 11 | 12 | class TestCaching(TestCase): 13 | 14 | def test_xdg_config(self): 15 | tmpdir = tempfile.mkdtemp() 16 | try: 17 | os.environ['XDG_CONFIG_HOME'] = tmpdir 18 | cache = memestra.caching.Cache() 19 | self.assertTrue(os.path.isdir(os.path.join(tmpdir, 'memestra'))) 20 | finally: 21 | shutil.rmtree(tmpdir) 22 | 23 | def test_cache_dir(self): 24 | tmpdir = tempfile.mkdtemp() 25 | cachedir = os.path.join(tmpdir, "memestra-test") 26 | try: 27 | cache = memestra.caching.Cache(cache_dir=cachedir) 28 | self.assertTrue(os.path.isdir(cachedir)) 29 | finally: 30 | shutil.rmtree(tmpdir) 31 | 32 | def test_contains(self): 33 | tmpdir = tempfile.mkdtemp() 34 | try: 35 | os.environ['XDG_CONFIG_HOME'] = tmpdir 36 | cache = memestra.caching.Cache() 37 | key = memestra.caching.CacheKeyFactory()(__file__) 38 | self.assertNotIn(key, cache) 39 | cache[key] = {} 40 | self.assertIn(key, cache) 41 | finally: 42 | shutil.rmtree(tmpdir) 43 | 44 | def test_defaults(self): 45 | tmpdir = tempfile.mkdtemp() 46 | try: 47 | os.environ['XDG_CONFIG_HOME'] = tmpdir 48 | cache = memestra.caching.Cache() 49 | key = memestra.caching.CacheKeyFactory()(__file__) 50 | self.assertNotIn(key, cache) 51 | cache[key] = {} 52 | data = cache[key] 53 | self.assertEqual(data['version'], memestra.caching.Format.version) 54 | self.assertEqual(data['name'], 'test_caching') 55 | self.assertEqual(data['deprecated'], []) 56 | self.assertEqual(data['generator'], 'manual') 57 | finally: 58 | shutil.rmtree(tmpdir) 59 | 60 | def test_invalid_version(self): 61 | tmpdir = tempfile.mkdtemp() 62 | try: 63 | os.environ['XDG_CONFIG_HOME'] = tmpdir 64 | cache = memestra.caching.Cache() 65 | key = memestra.caching.CacheKeyFactory()(__file__) 66 | with self.assertRaises(ValueError): 67 | cache[key] = {'version': -1, 68 | 'deprecated': [], 69 | 'generator': 'manual'} 70 | finally: 71 | shutil.rmtree(tmpdir) 72 | 73 | class TestCLI(TestCase): 74 | 75 | def test_docparse(self): 76 | fid, tmppy = tempfile.mkstemp(suffix='.py') 77 | code = ''' 78 | def foo(): "deprecated since 1999" 79 | 80 | foo()''' 81 | 82 | ref = 'Found 1 deprecated identifiers\nfoo\n' 83 | os.write(fid, dedent(code).encode()) 84 | os.close(fid) 85 | test_args = ['memestra-cache', 'docparse', 86 | '--pattern', 'deprecated since', 87 | '--verbose', 88 | tmppy] 89 | with mock.patch.object(sys, 'argv', test_args): 90 | from memestra.caching import run 91 | with StringIO() as buf: 92 | with contextlib.redirect_stdout(buf): 93 | run() 94 | self.assertEqual(buf.getvalue(), ref) 95 | 96 | def test_set_cache(self): 97 | with tempfile.NamedTemporaryFile(suffix='.py', delete=False) as tmppy: 98 | code = ''' 99 | def foo() 100 | pass 101 | 102 | def bar() 103 | pass 104 | 105 | foo() 106 | bar()''' 107 | 108 | tmppy.write(dedent(code).encode()) 109 | 110 | ref = '' 111 | set_args = ['memestra-cache', 'set', 112 | '--deprecated', 'foo', 113 | '--deprecated', 'bar', 114 | tmppy.name] 115 | with mock.patch.object(sys, 'argv', set_args): 116 | from memestra.caching import run 117 | run() 118 | 119 | expected = { 120 | 'deprecated': ['foo', 'bar'], 121 | 'generator': 'manual', 122 | 'name': os.path.splitext(os.path.basename(tmppy.name))[0], 123 | 'version': 1, 124 | } 125 | cache = memestra.caching.Cache() 126 | key = memestra.caching.CacheKeyFactory()(tmppy.name) 127 | os.remove(tmppy.name) 128 | self.assertEqual(cache[key], expected) 129 | 130 | def test_cache_dir(self): 131 | with tempfile.NamedTemporaryFile(suffix='.py', delete=False) as tmppy: 132 | code = ''' 133 | def foo() 134 | pass 135 | 136 | foo()''' 137 | 138 | tmppy.write(dedent(code).encode()) 139 | 140 | try: 141 | tmpdir = tempfile.mkdtemp() 142 | ref = '' 143 | set_args = ['memestra-cache', 144 | '--cache-dir=' + tmpdir, 145 | 'set', 146 | '--deprecated=foo', 147 | tmppy.name] 148 | with mock.patch.object(sys, 'argv', set_args): 149 | from memestra.caching import run 150 | run() 151 | 152 | key = memestra.caching.CacheKeyFactory()(tmppy.name) 153 | cachefile = os.path.join(tmpdir, key.module_hash) 154 | self.assertTrue(os.path.isfile(cachefile)) 155 | finally: 156 | os.remove(tmppy.name) 157 | shutil.rmtree(tmpdir) 158 | 159 | -------------------------------------------------------------------------------- /tests/test_docparse.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from memestra.docparse import docparse 3 | 4 | def sample_function(): 5 | """ 6 | I'm a deprecated function 7 | """ 8 | 9 | class sample_class: 10 | 'deprecated too' 11 | 12 | class TestDocparse(TestCase): 13 | 14 | def test_docparse(self): 15 | deprecated = docparse(__file__, r'.*deprecated.*') 16 | self.assertEqual(deprecated, ['sample_function', 'sample_class']) 17 | -------------------------------------------------------------------------------- /tests/test_imports.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | from textwrap import dedent 3 | from io import StringIO 4 | import memestra 5 | 6 | import os 7 | import sys 8 | 9 | TESTS_PATHS = [os.path.abspath(os.path.join(os.path.dirname(__file__), 'misc'))] 10 | 11 | class TestImports(TestCase): 12 | 13 | def checkDeprecatedUses(self, code, expected_output): 14 | sio = StringIO(dedent(code)) 15 | output = memestra.memestra(sio, ('decoratortest', 'deprecated'), None, 16 | search_paths=TESTS_PATHS) 17 | self.assertEqual(output, expected_output) 18 | 19 | def test_import_from(self): 20 | code = ''' 21 | from some_module import foo, bar, foobar 22 | foobar() 23 | 24 | def foobar(): 25 | foo() 26 | bar() 27 | 28 | foo()''' 29 | 30 | self.checkDeprecatedUses( 31 | code, 32 | [('foo', '<>', 6, 4, None), 33 | ('foo', '<>', 9, 0, None), 34 | ('foobar', '<>', 3, 0, "because it's too old")]) 35 | 36 | def test_import_from_as(self): 37 | code = ''' 38 | from some_module import foo as Foo, bar as Bar 39 | 40 | def foobar(): 41 | Foo() 42 | Bar() 43 | 44 | Foo()''' 45 | 46 | self.checkDeprecatedUses( 47 | code, 48 | [('Foo', '<>', 5, 4, None), ('Foo', '<>', 8, 0, None)]) 49 | 50 | def test_import(self): 51 | code = ''' 52 | import some_module 53 | 54 | def bar(): 55 | some_module.foo() 56 | some_module.bar() 57 | 58 | some_module.foo()''' 59 | 60 | self.checkDeprecatedUses( 61 | code, 62 | [('some_module.foo', '<>', 5, 4, None), ('some_module.foo', '<>', 8, 0, None)]) 63 | 64 | def test_import_as(self): 65 | code = ''' 66 | import some_module as Module 67 | 68 | def bar(): 69 | Module.foo() 70 | Module.bar() 71 | 72 | Module.foo()''' 73 | 74 | self.checkDeprecatedUses( 75 | code, 76 | [('Module.foo', '<>', 5, 4, None), ('Module.foo', '<>', 8, 0, None)]) 77 | 78 | class TestRecImports(TestCase): 79 | 80 | def checkDeprecatedUses(self, code, expected_output): 81 | sio = StringIO(dedent(code)) 82 | output = memestra.memestra(sio, ('decoratortest', 'deprecated'), None, 83 | search_paths=TESTS_PATHS, recursive=True) 84 | self.assertEqual(output, expected_output) 85 | 86 | def test_import(self): 87 | code = ''' 88 | import some_rec_module 89 | 90 | def foobar(): 91 | some_rec_module.foo() 92 | some_rec_module.bar() 93 | 94 | some_rec_module.foo()''' 95 | 96 | self.checkDeprecatedUses( 97 | code, 98 | [('some_rec_module.foo', '<>', 5, 4, None), 99 | ('some_rec_module.foo', '<>', 8, 0, None)]) 100 | 101 | def test_import_from0(self): 102 | code = ''' 103 | from some_rec_module import foo 104 | 105 | def foobar(): 106 | foo() 107 | 108 | foo()''' 109 | 110 | self.checkDeprecatedUses( 111 | code, 112 | [('foo', '<>', 2, 0, None), 113 | ('foo', '<>', 5, 4, None), 114 | ('foo', '<>', 7, 0, None)]) 115 | 116 | def test_import_from1(self): 117 | code = ''' 118 | from some_rec_module import bar 119 | 120 | def foobar(): 121 | bar() 122 | 123 | bar()''' 124 | 125 | self.checkDeprecatedUses( 126 | code, 127 | []) 128 | 129 | def test_import_from2(self): 130 | code = ''' 131 | from some_rec_module import foobar 132 | 133 | foobar()''' 134 | 135 | self.checkDeprecatedUses( 136 | code, 137 | [('foobar', '<>', 2, 0, None), ('foobar', '<>', 4, 0, None)]) 138 | 139 | def test_forwarding_symbol0(self): 140 | code = ''' 141 | from module_forwarding_symbol import Test 142 | t = Test()''' 143 | 144 | self.checkDeprecatedUses( 145 | code, 146 | [('Test', '<>', 2, 0, None), ('Test', '<>', 3, 4, None)]) 147 | 148 | def test_forwarding_symbol1(self): 149 | code = ''' 150 | import module_forwarding_symbol 151 | t = module_forwarding_symbol.Test()''' 152 | 153 | self.checkDeprecatedUses( 154 | code, 155 | [('module_forwarding_symbol.Test', '<>', 3, 4, None)]) 156 | 157 | def test_forwarding_symbol2(self): 158 | code = ''' 159 | from module_forwarding_symbol import Testosterone 160 | t = Testosterone()''' 161 | 162 | self.checkDeprecatedUses( 163 | code, 164 | [('Testosterone', '<>', 2, 0, None), 165 | ('Testosterone', '<>', 3, 4, None)]) 166 | 167 | def test_forwarding_all_symbols1(self): 168 | code = ''' 169 | from module_forwarding_all_symbols import Test 170 | t = Test()''' 171 | 172 | self.checkDeprecatedUses( 173 | code, 174 | [('Test', '<>', 2, 0, None), 175 | ('Test', '<>', 3, 4, None)]) 176 | 177 | def test_forwarding_all_symbols2(self): 178 | code = ''' 179 | from module_forwarding_all_symbols import * 180 | t = Test()''' 181 | 182 | self.checkDeprecatedUses( 183 | code, 184 | [('Test', '<>', 3, 4, None)]) 185 | 186 | def test_importing_non_existing_file(self): 187 | code = ''' 188 | from phantom import void, empty 189 | void() 190 | empty()''' 191 | 192 | self.checkDeprecatedUses( 193 | code, 194 | [('empty', '<>', 2, 0, None), 195 | ('empty', '<>', 4, 0, None)]) 196 | 197 | 198 | class TestImportPkg(TestCase): 199 | 200 | def checkDeprecatedUses(self, code, expected_output): 201 | sio = StringIO(dedent(code)) 202 | output = memestra.memestra(sio, ('decoratortest', 'deprecated'), None, 203 | search_paths=TESTS_PATHS, recursive=True) 204 | self.assertEqual(output, expected_output) 205 | 206 | def test_import_pkg(self): 207 | code = ''' 208 | import pkg 209 | 210 | def foobar(): 211 | pkg.test()''' 212 | 213 | self.checkDeprecatedUses( 214 | code, 215 | [('pkg.test', '<>', 5, 4, None)]) 216 | 217 | def test_import_pkg_level(self): 218 | code = ''' 219 | from pkg.sub.other import other 220 | 221 | def foobar(): 222 | other()''' 223 | 224 | self.checkDeprecatedUses( 225 | code, 226 | [('other', '<>', 2, 0, None), ('other', '<>', 5, 4, None)]) 227 | 228 | def test_import_pkg_level_star(self): 229 | code = ''' 230 | from ipy import foo 231 | 232 | b = foo()''' 233 | 234 | self.checkDeprecatedUses( 235 | code, 236 | [('foo', '<>', 2, 0, 'why'), ('foo', '<>', 4, 4, 'why')]) 237 | 238 | def test_import_pkg_level_star2(self): 239 | code = ''' 240 | from ipy import * 241 | a = useless() 242 | b = foo()''' 243 | 244 | self.checkDeprecatedUses( 245 | code, 246 | [('foo', '<>', 4, 4, 'why')]) 247 | 248 | def test_shared_cache(self): 249 | # We have a fake description for gast in tests/share/memestra 250 | # Setup the shared cache to use it. 251 | with mock.patch('sys.prefix', os.path.dirname(__file__)): 252 | self.checkDeprecatedUses( 253 | 'from gast import parse', 254 | [('parse', '<>', 1, 0, None)]) 255 | 256 | def test_shared_cache_sub(self): 257 | # We have a fake description for gast in tests/share/memestra 258 | # Setup the shared cache to use it. 259 | with mock.patch('sys.prefix', os.path.dirname(__file__)): 260 | self.checkDeprecatedUses( 261 | 'from gast.astn import AstToGAst', 262 | [('AstToGAst', '<>', 1, 0, 'because')]) 263 | -------------------------------------------------------------------------------- /tests/test_multiattr.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from textwrap import dedent 3 | from io import StringIO 4 | import memestra 5 | 6 | import os 7 | import sys 8 | 9 | TESTS_PATHS = [os.path.abspath(os.path.join(os.path.dirname(__file__), 'misc'))] 10 | 11 | class TestMultiAttr(TestCase): 12 | 13 | def checkDeprecatedUses(self, code, expected_output): 14 | sio = StringIO(dedent(code)) 15 | output = memestra.memestra(sio, ('decorator', 'sub', 'deprecated'), None, 16 | search_paths=TESTS_PATHS) 17 | self.assertEqual(output, expected_output) 18 | 19 | def test_import(self): 20 | code = ''' 21 | import decorator 22 | 23 | @decorator.sub.deprecated 24 | def foo(): pass 25 | 26 | def bar(): 27 | foo() 28 | 29 | foo()''' 30 | 31 | self.checkDeprecatedUses( 32 | code, 33 | [('foo', '<>', 8, 4, None), ('foo', '<>', 10, 0, None)]) 34 | 35 | def test_import_alias(self): 36 | code = ''' 37 | import decorator as dp 38 | 39 | @dp.sub.deprecated 40 | def foo(): pass 41 | 42 | def bar(): 43 | foo() 44 | 45 | foo()''' 46 | 47 | self.checkDeprecatedUses( 48 | code, 49 | [('foo', '<>', 8, 4, None), ('foo', '<>', 10, 0, None)]) 50 | 51 | def test_import_sub_alias(self): 52 | code = ''' 53 | import decorator.sub as dp 54 | 55 | @dp.deprecated 56 | def foo(): pass 57 | 58 | def bar(): 59 | foo() 60 | 61 | foo()''' 62 | 63 | self.checkDeprecatedUses( 64 | code, 65 | [('foo', '<>', 8, 4, None), ('foo', '<>', 10, 0, None)]) 66 | 67 | def test_import_from(self): 68 | code = ''' 69 | from decorator.sub import deprecated 70 | 71 | @deprecated 72 | def foo(): pass 73 | 74 | def bar(): 75 | foo() 76 | 77 | foo()''' 78 | 79 | self.checkDeprecatedUses( 80 | code, 81 | [('foo', '<>', 8, 4, None), ('foo', '<>', 10, 0, None)]) 82 | 83 | def test_import_from_alias(self): 84 | code = ''' 85 | from decorator.sub import deprecated as dp 86 | 87 | @dp 88 | def foo(): pass 89 | 90 | def bar(): 91 | foo() 92 | 93 | foo()''' 94 | 95 | self.checkDeprecatedUses( 96 | code, 97 | [('foo', '<>', 8, 4, None), ('foo', '<>', 10, 0, None)]) 98 | -------------------------------------------------------------------------------- /tests/test_notebook.py: -------------------------------------------------------------------------------- 1 | import os 2 | from unittest import TestCase 3 | from memestra import nbmemestra, preprocessor 4 | import nbformat 5 | from traitlets.config import Config 6 | from nbconvert import RSTExporter 7 | 8 | 9 | this_dir = os.path.dirname(os.path.abspath(__file__)) 10 | TESTS_NB_FILE = os.path.abspath(os.path.join(os.path.dirname(__file__), 'misc', 'memestra_nb_demo.ipynb')) 11 | import sys 12 | sys.path.insert(0, os.path.join(this_dir, 'misc')) 13 | 14 | class NotebookTest(TestCase): 15 | 16 | def test_nb_demo(self): 17 | output = nbmemestra.nbmemestra(TESTS_NB_FILE, 18 | ('decoratortest', 'deprecated'), 19 | TESTS_NB_FILE) 20 | expected_output = [('some_module.foo', 'Cell[0]', 2, 0), 21 | ('some_module.foo', 'Cell[0]', 3, 0), 22 | ('some_module.foo', 'Cell[2]', 2, 4)] 23 | self.assertEqual(output, expected_output) 24 | 25 | def test_nbconvert_demo(self): 26 | self.maxDiff = None 27 | with open(TESTS_NB_FILE) as f: 28 | notebook = nbformat.read(f, as_version=4) 29 | 30 | c = Config() 31 | c.MemestraDeprecationChecker.decorator = ('decoratortest', 'deprecated') 32 | c.RSTExporter.preprocessors = [preprocessor.MemestraDeprecationChecker] 33 | 34 | deprecation_checker = RSTExporter(config=c) 35 | 36 | # the RST exporter behaves differently on windows and on linux 37 | # there can be some lines with only whitespaces 38 | # so we ignore differences that only consist of empty lines 39 | rst = deprecation_checker.from_notebook_node(notebook)[0] 40 | lines = rst.split('\n') 41 | lines = [l.rstrip() for l in lines] 42 | rst = '\n'.join(lines) 43 | 44 | with open(os.path.join(this_dir, 'misc', 'memestra_nb_demo.rst')) as f: 45 | rst_true = f.read() 46 | 47 | self.assertEqual(rst, rst_true) 48 | --------------------------------------------------------------------------------