├── doc ├── requirements.txt ├── api.rst ├── index.rst ├── Makefile └── conf.py ├── .gitignore ├── astsearch ├── README.rst ├── pyproject.toml ├── .github └── workflows │ └── test.yml ├── LICENSE ├── test_astsearch.py └── astsearch.py /doc/requirements.txt: -------------------------------------------------------------------------------- 1 | astcheck >= 0.2.1 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /build/ 2 | /dist/ 3 | MANIFEST 4 | *.pyc 5 | /doc/_build 6 | -------------------------------------------------------------------------------- /astsearch: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """Command line entry point for astsearch""" 3 | from astsearch import main 4 | main() 5 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ASTsearch is an intelligent search tool for Python code. 2 | 3 | To get it:: 4 | 5 | pip install astsearch 6 | 7 | To use it:: 8 | 9 | # astsearch pattern [path] 10 | astsearch "?/?" # Division operations in all files in the current directory 11 | 12 | For more details, see `the documentation `_. 13 | -------------------------------------------------------------------------------- /doc/api.rst: -------------------------------------------------------------------------------- 1 | Python API 2 | ========== 3 | 4 | .. module:: astsearch 5 | 6 | .. autoclass:: ASTPatternFinder 7 | 8 | .. automethod:: scan_ast 9 | .. automethod:: scan_file 10 | .. automethod:: scan_directory 11 | 12 | .. autofunction:: prepare_pattern 13 | 14 | .. seealso:: 15 | 16 | `astcheck `_ 17 | The AST pattern matching used by ASTsearch 18 | 19 | `Green Tree Snakes `_ 20 | An overview of the available AST nodes and how to work with them 21 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["flit_core >=2,<4"] 3 | build-backend = "flit_core.buildapi" 4 | 5 | [project] 6 | name = "astsearch" 7 | authors = [ 8 | {name = "Thomas Kluyver", email = "thomas@kluyver.me.uk"}, 9 | ] 10 | readme = "README.rst" 11 | requires-python = ">=3.9" 12 | license = {file = "LICENSE"} 13 | dependencies = ["astcheck (>=0.4)"] 14 | classifiers = [ 15 | "Intended Audience :: Developers", 16 | "License :: OSI Approved :: MIT License", 17 | "Programming Language :: Python", 18 | "Programming Language :: Python :: 3", 19 | "Topic :: Software Development :: Libraries :: Python Modules", 20 | "Topic :: Software Development :: Testing" 21 | ] 22 | dynamic = ["version", "description"] 23 | 24 | [project.scripts] 25 | astsearch = "astsearch:main" 26 | 27 | [project.urls] 28 | Home = "https://github.com/takluyver/astsearch" 29 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | matrix: 10 | python-version: [ "3.9", "3.10", "3.11", "3.12" ] 11 | steps: 12 | - uses: actions/checkout@v4 13 | 14 | - name: Setup Python ${{ matrix.python-version }} 15 | uses: actions/setup-python@v5 16 | with: 17 | python-version: ${{ matrix.python-version }} 18 | 19 | - uses: actions/cache@v4 20 | with: 21 | path: ~/.cache/pip 22 | key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('pyproject.toml') }} 23 | 24 | - name: Install dependencies 25 | run: | 26 | python -m pip install --upgrade pip 27 | pip install . pytest pytest-cov 28 | 29 | - name: Run tests 30 | run: pytest --cov=astsearch --cov-report=xml 31 | 32 | - name: Upload coverage to codecov 33 | uses: codecov/codecov-action@v3 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Thomas Kluyver 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | ASTsearch |version| 2 | =================== 3 | 4 | ASTsearch is an intelligent search tool for Python code. 5 | 6 | To get it:: 7 | 8 | pip install astsearch 9 | 10 | To use it:: 11 | 12 | # astsearch pattern [path] 13 | astsearch "?/?" # Division operations in all files in the current directory 14 | 15 | .. program:: astsearch 16 | 17 | .. option:: pattern 18 | 19 | A search pattern, using ``?`` as a wildcard to match anything. The pattern 20 | must be a valid Python statement once all ``?`` wilcards have been replaced 21 | with a name. 22 | 23 | .. option:: path 24 | 25 | A Python file or a directory in which to search. Directories will be searched 26 | recursively for ``.py`` and ``.pyw`` files. 27 | 28 | .. option:: -m MAX_LINES, --max-lines MAX_LINES 29 | 30 | By default, on Python >=3.8, multiline matches are fully printed, up to a 31 | maximum of 10 lines. This maximum number of printed lines can be set by 32 | this option. Setting it to 0 disables multiline printing. 33 | 34 | .. option:: -l, --files-with-matches 35 | 36 | Output only the paths of matching files, not the lines that matched. 37 | 38 | Contents: 39 | 40 | .. toctree:: 41 | :maxdepth: 2 42 | 43 | api 44 | 45 | .. seealso:: 46 | 47 | `astpath `_ 48 | Search through ASTs using XPath syntax 49 | 50 | Indices and tables 51 | ================== 52 | 53 | * :ref:`genindex` 54 | * :ref:`modindex` 55 | * :ref:`search` 56 | 57 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/ASTsearch.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/ASTsearch.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/ASTsearch" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/ASTsearch" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # ASTsearch documentation build configuration file, created by 4 | # sphinx-quickstart on Sun Apr 27 22:04:50 2014. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | 18 | # If extensions (or modules to document with autodoc) are in another directory, 19 | # add these directories to sys.path here. If the directory is relative to the 20 | # documentation root, use os.path.abspath to make it absolute, like shown here. 21 | sys.path.insert(0, os.path.abspath('..')) 22 | 23 | # -- General configuration ------------------------------------------------ 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | #needs_sphinx = '1.0' 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be 29 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 30 | # ones. 31 | extensions = [ 32 | 'sphinx.ext.autodoc', 33 | 'sphinx.ext.intersphinx', 34 | ] 35 | 36 | # Add any paths that contain templates here, relative to this directory. 37 | templates_path = ['_templates'] 38 | 39 | # The suffix of source filenames. 40 | source_suffix = '.rst' 41 | 42 | # The encoding of source files. 43 | #source_encoding = 'utf-8-sig' 44 | 45 | # The master toctree document. 46 | master_doc = 'index' 47 | 48 | # General information about the project. 49 | project = u'ASTsearch' 50 | copyright = u'2014, Thomas Kluyver' 51 | 52 | # The version info for the project you're documenting, acts as replacement for 53 | # |version| and |release|, also used in various other places throughout the 54 | # built documents. 55 | 56 | import astsearch 57 | # The short X.Y version. 58 | version = astsearch.__version__ 59 | # The full version, including alpha/beta/rc tags. 60 | release = version 61 | 62 | # The language for content autogenerated by Sphinx. Refer to documentation 63 | # for a list of supported languages. 64 | #language = None 65 | 66 | # There are two options for replacing |today|: either, you set today to some 67 | # non-false value, then it is used: 68 | #today = '' 69 | # Else, today_fmt is used as the format for a strftime call. 70 | #today_fmt = '%B %d, %Y' 71 | 72 | # List of patterns, relative to source directory, that match files and 73 | # directories to ignore when looking for source files. 74 | exclude_patterns = ['_build'] 75 | 76 | # The reST default role (used for this markup: `text`) to use for all 77 | # documents. 78 | #default_role = None 79 | 80 | # If true, '()' will be appended to :func: etc. cross-reference text. 81 | #add_function_parentheses = True 82 | 83 | # If true, the current module name will be prepended to all description 84 | # unit titles (such as .. function::). 85 | #add_module_names = True 86 | 87 | # If true, sectionauthor and moduleauthor directives will be shown in the 88 | # output. They are ignored by default. 89 | #show_authors = False 90 | 91 | # The name of the Pygments (syntax highlighting) style to use. 92 | pygments_style = 'sphinx' 93 | 94 | # A list of ignored prefixes for module index sorting. 95 | #modindex_common_prefix = [] 96 | 97 | # If true, keep warnings as "system message" paragraphs in the built documents. 98 | #keep_warnings = False 99 | 100 | 101 | # -- Options for HTML output ---------------------------------------------- 102 | 103 | # The theme to use for HTML and HTML Help pages. See the documentation for 104 | # a list of builtin themes. 105 | html_theme = 'default' 106 | 107 | # Theme options are theme-specific and customize the look and feel of a theme 108 | # further. For a list of options available for each theme, see the 109 | # documentation. 110 | #html_theme_options = {} 111 | 112 | # Add any paths that contain custom themes here, relative to this directory. 113 | #html_theme_path = [] 114 | 115 | # The name for this set of Sphinx documents. If None, it defaults to 116 | # " v documentation". 117 | #html_title = None 118 | 119 | # A shorter title for the navigation bar. Default is the same as html_title. 120 | #html_short_title = None 121 | 122 | # The name of an image file (relative to this directory) to place at the top 123 | # of the sidebar. 124 | #html_logo = None 125 | 126 | # The name of an image file (within the static path) to use as favicon of the 127 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 128 | # pixels large. 129 | #html_favicon = None 130 | 131 | # Add any paths that contain custom static files (such as style sheets) here, 132 | # relative to this directory. They are copied after the builtin static files, 133 | # so a file named "default.css" will overwrite the builtin "default.css". 134 | html_static_path = ['_static'] 135 | 136 | # Add any extra paths that contain custom files (such as robots.txt or 137 | # .htaccess) here, relative to this directory. These files are copied 138 | # directly to the root of the documentation. 139 | #html_extra_path = [] 140 | 141 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 142 | # using the given strftime format. 143 | #html_last_updated_fmt = '%b %d, %Y' 144 | 145 | # If true, SmartyPants will be used to convert quotes and dashes to 146 | # typographically correct entities. 147 | #html_use_smartypants = True 148 | 149 | # Custom sidebar templates, maps document names to template names. 150 | #html_sidebars = {} 151 | 152 | # Additional templates that should be rendered to pages, maps page names to 153 | # template names. 154 | #html_additional_pages = {} 155 | 156 | # If false, no module index is generated. 157 | #html_domain_indices = True 158 | 159 | # If false, no index is generated. 160 | #html_use_index = True 161 | 162 | # If true, the index is split into individual pages for each letter. 163 | #html_split_index = False 164 | 165 | # If true, links to the reST sources are added to the pages. 166 | #html_show_sourcelink = True 167 | 168 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 169 | #html_show_sphinx = True 170 | 171 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 172 | #html_show_copyright = True 173 | 174 | # If true, an OpenSearch description file will be output, and all pages will 175 | # contain a tag referring to it. The value of this option must be the 176 | # base URL from which the finished HTML is served. 177 | #html_use_opensearch = '' 178 | 179 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 180 | #html_file_suffix = None 181 | 182 | # Output file base name for HTML help builder. 183 | htmlhelp_basename = 'ASTsearchdoc' 184 | 185 | 186 | # -- Options for LaTeX output --------------------------------------------- 187 | 188 | latex_elements = { 189 | # The paper size ('letterpaper' or 'a4paper'). 190 | #'papersize': 'letterpaper', 191 | 192 | # The font size ('10pt', '11pt' or '12pt'). 193 | #'pointsize': '10pt', 194 | 195 | # Additional stuff for the LaTeX preamble. 196 | #'preamble': '', 197 | } 198 | 199 | # Grouping the document tree into LaTeX files. List of tuples 200 | # (source start file, target name, title, 201 | # author, documentclass [howto, manual, or own class]). 202 | latex_documents = [ 203 | ('index', 'ASTsearch.tex', u'ASTsearch Documentation', 204 | u'Thomas Kluyver', 'manual'), 205 | ] 206 | 207 | # The name of an image file (relative to this directory) to place at the top of 208 | # the title page. 209 | #latex_logo = None 210 | 211 | # For "manual" documents, if this is true, then toplevel headings are parts, 212 | # not chapters. 213 | #latex_use_parts = False 214 | 215 | # If true, show page references after internal links. 216 | #latex_show_pagerefs = False 217 | 218 | # If true, show URL addresses after external links. 219 | #latex_show_urls = False 220 | 221 | # Documents to append as an appendix to all manuals. 222 | #latex_appendices = [] 223 | 224 | # If false, no module index is generated. 225 | #latex_domain_indices = True 226 | 227 | 228 | # -- Options for manual page output --------------------------------------- 229 | 230 | # One entry per manual page. List of tuples 231 | # (source start file, name, description, authors, manual section). 232 | man_pages = [ 233 | ('index', 'astsearch', u'ASTsearch Documentation', 234 | [u'Thomas Kluyver'], 1) 235 | ] 236 | 237 | # If true, show URL addresses after external links. 238 | #man_show_urls = False 239 | 240 | 241 | # -- Options for Texinfo output ------------------------------------------- 242 | 243 | # Grouping the document tree into Texinfo files. List of tuples 244 | # (source start file, target name, title, author, 245 | # dir menu entry, description, category) 246 | texinfo_documents = [ 247 | ('index', 'ASTsearch', u'ASTsearch Documentation', 248 | u'Thomas Kluyver', 'ASTsearch', 'One line description of project.', 249 | 'Miscellaneous'), 250 | ] 251 | 252 | # Documents to append as an appendix to all manuals. 253 | #texinfo_appendices = [] 254 | 255 | # If false, no module index is generated. 256 | #texinfo_domain_indices = True 257 | 258 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 259 | #texinfo_show_urls = 'footnote' 260 | 261 | # If true, do not generate a @detailmenu in the "Top" node's menu. 262 | #texinfo_no_detailmenu = False 263 | 264 | 265 | # Example configuration for intersphinx: refer to the Python standard library. 266 | intersphinx_mapping = {'python': ('http://docs.python.org/3', None)} 267 | -------------------------------------------------------------------------------- /test_astsearch.py: -------------------------------------------------------------------------------- 1 | import ast 2 | from io import StringIO 3 | import types 4 | import unittest 5 | 6 | import pytest 7 | 8 | from astcheck import assert_ast_like, listmiddle, name_or_attr 9 | from astsearch import ( 10 | prepare_pattern, ASTPatternFinder, must_exist_checker, must_not_exist_checker, 11 | ArgsDefChecker, 12 | ) 13 | 14 | def assert_iterator_finished(it): 15 | with pytest.raises(StopIteration): 16 | next(it) 17 | 18 | def get_matches(pattern, sample_code): 19 | sample_ast = ast.parse(sample_code) 20 | print(ast.dump(sample_ast)) 21 | return list(ASTPatternFinder(pattern).scan_ast(sample_ast)) 22 | 23 | 24 | # Tests of just preparing the pattern -------------------------------- 25 | 26 | def test_prepare_plain(): 27 | pat = prepare_pattern('1/2') 28 | assert_ast_like(pat, ast.BinOp( 29 | left=ast.Constant(1), op=ast.Div(), right=ast.Constant(2) 30 | )) 31 | 32 | def test_simple_wildcard(): 33 | pat = prepare_pattern('?/?') 34 | assert_ast_like(pat, ast.BinOp(op=ast.Div())) 35 | assert pat.left is must_exist_checker 36 | assert pat.right is must_exist_checker 37 | 38 | def test_wildcard_body(): 39 | pat = prepare_pattern('if True: ??\nelse: ??') 40 | assert isinstance(pat, ast.If) 41 | assert pat.body is must_exist_checker 42 | assert pat.orelse is must_exist_checker 43 | 44 | pat2 = prepare_pattern('if True: ??') 45 | assert isinstance(pat2, ast.If) 46 | assert pat.body is must_exist_checker 47 | assert not hasattr(pat2, 'orelse') 48 | 49 | def test_wildcard_body_part(): 50 | pat = prepare_pattern("def foo():\n ??\n return a") 51 | assert isinstance(pat, ast.FunctionDef) 52 | assert isinstance(pat.body, listmiddle) 53 | assert_ast_like(pat.body.back[0], ast.Return(ast.Name(id='a'))) 54 | 55 | def test_name_or_attr(): 56 | pat = prepare_pattern('a = 1') 57 | assert_ast_like(pat, ast.Assign(value=ast.Constant(1))) 58 | assert isinstance(pat.targets[0], name_or_attr) 59 | 60 | def test_wildcard_call_args(): 61 | pat = prepare_pattern("f(??)") 62 | assert isinstance(pat, ast.Call) 63 | assert isinstance(pat.args, listmiddle) 64 | assert pat.args.front == [] 65 | assert pat.args.back == [] 66 | assert not hasattr(pat, 'keywords') 67 | 68 | def test_wildcard_some_call_args(): 69 | pat = prepare_pattern("f(??, 1)") 70 | assert isinstance(pat.args, listmiddle) 71 | assert pat.args.front == [] 72 | assert_ast_like(pat.args.back[0], ast.Constant(1)) 73 | assert pat.keywords == must_not_exist_checker 74 | 75 | def test_wildcard_call_keywords(): 76 | pat = prepare_pattern("f(a=1, ??=??)") 77 | assert pat.args == must_not_exist_checker 78 | assert isinstance(pat.keywords, types.FunctionType) 79 | 80 | def test_wildcard_call_mixed_args(): 81 | pat = prepare_pattern("f(1, ??, a=2, **{'b':3})") 82 | assert isinstance(pat.args, listmiddle) 83 | assert_ast_like(pat.args.front[0], ast.Constant(1)) 84 | assert isinstance(pat.keywords, types.FunctionType) 85 | kwargs_dict = ast.Dict(keys=[ast.Constant('b')], values=[ast.Constant(3)]) 86 | pat.keywords([ast.keyword(arg=None, value=kwargs_dict), 87 | ast.keyword(arg='a', value=ast.Constant(2))], []) 88 | 89 | def test_wildcard_funcdef(): 90 | pat = prepare_pattern("def f(??): ??") 91 | assert_ast_like(pat, ast.FunctionDef(name='f')) 92 | assert isinstance(pat.args.args, listmiddle) 93 | assert pat.args.args.front == [] 94 | assert pat.args.args.back == [] 95 | assert not hasattr(pat.args.args, 'vararg') 96 | assert not hasattr(pat.args.args, 'kwonlyargs') 97 | assert not hasattr(pat.args.args, 'kwarg') 98 | 99 | def test_wildcard_funcdef_earlyargs(): 100 | pat = prepare_pattern("def f(??, a): ??") 101 | assert isinstance(pat.args.args, listmiddle) 102 | assert_ast_like(pat.args.args.back[0], ast.arg(arg='a')) 103 | assert pat.args.vararg is must_not_exist_checker 104 | assert pat.args.kwonly_args_dflts == [] 105 | 106 | def test_wildcard_funcdef_kwonlyargs(): 107 | pat = prepare_pattern("def f(*, a, ??): ??") 108 | assert isinstance(pat.args, ArgsDefChecker) 109 | assert [a.arg for a,d in pat.args.kwonly_args_dflts] == ['a'] 110 | assert pat.args.koa_subset 111 | assert pat.args.kwarg is None 112 | assert pat.args.args is must_not_exist_checker 113 | assert pat.args.vararg is must_not_exist_checker 114 | 115 | def test_attr_no_ctx(): 116 | pat = prepare_pattern('?.baz') 117 | assert_ast_like(pat, ast.Attribute(attr='baz')) 118 | assert not hasattr(pat, 'ctx') 119 | matches = get_matches(pat, 'foo.baz = 1') 120 | assert len(matches) == 1 121 | 122 | def test_subscript_no_ctx(): 123 | pat = prepare_pattern('?[2]') 124 | assert_ast_like(pat, ast.Subscript(slice=ast.Index(value=ast.Constant(2)))) 125 | assert not hasattr(pat, 'ctx') 126 | matches = get_matches(pat, 'd[2] = 1') 127 | assert len(matches) == 1 128 | 129 | def test_import(): 130 | pat = prepare_pattern("import ?") 131 | assert isinstance(pat, ast.Import) 132 | assert len(pat.names) == 1 133 | assert pat.names[0].name is must_exist_checker 134 | 135 | def test_import_multi(): 136 | pat = prepare_pattern("import ??") 137 | assert isinstance(pat, ast.Import) 138 | assert not hasattr(pat, 'names') 139 | 140 | pat = prepare_pattern("from x import ??") 141 | assert isinstance(pat, ast.ImportFrom) 142 | assert pat.module == 'x' 143 | assert not hasattr(pat, 'names') 144 | 145 | def test_import_from(): 146 | pat = prepare_pattern("from ? import ?") 147 | print(ast.dump(pat, indent=2)) 148 | assert isinstance(pat, ast.ImportFrom) 149 | assert pat.module is must_exist_checker 150 | assert len(pat.names) == 1 151 | assert pat.names[0].name is must_exist_checker 152 | assert len(get_matches(pat, "from foo import bar as foobar")) == 1 153 | 154 | def test_string_u_prefix(): 155 | pat = prepare_pattern('"foo"') 156 | assert len(get_matches(pat, "u'foo'")) == 1 157 | 158 | def test_bare_except(): 159 | pat = prepare_pattern("try: ??\nexcept: ??") 160 | print(ast.dump(pat, indent=2)) 161 | assert len(get_matches(pat, "try: pass\nexcept: pass")) == 1 162 | # 'except:' should only match a bare assert with no exception type 163 | assert len(get_matches(pat, "try: pass\nexcept Exception: pass")) == 0 164 | 165 | 166 | # Tests of general matching ----------------------------------------------- 167 | 168 | division_sample = """#!/usr/bin/python3 169 | 'not / division' 170 | 1/2 171 | a.b/c 172 | # 5/6 173 | 78//8 # FloorDiv is not the same 174 | 175 | def divide(x, y): 176 | return x/y 177 | """ 178 | 179 | if_sample = """ 180 | if a: 181 | pass 182 | else: 183 | pass 184 | 185 | if b: 186 | pass 187 | """ 188 | 189 | def test_match_plain(): 190 | pat = ast.BinOp(left=ast.Constant(1), right=ast.Constant(2), op=ast.Div()) 191 | it = ASTPatternFinder(pat).scan_file(StringIO(division_sample)) 192 | assert next(it).left.value == 1 193 | assert_iterator_finished(it) 194 | 195 | def test_all_divisions(): 196 | pat = ast.BinOp(op=ast.Div()) 197 | it = ASTPatternFinder(pat).scan_file(StringIO(division_sample)) 198 | assert_ast_like(next(it), ast.BinOp(left=ast.Constant(1))) 199 | assert_ast_like(next(it), ast.BinOp(right=ast.Name(id='c'))) 200 | assert_ast_like(next(it), ast.BinOp(left=ast.Name(id='x'))) 201 | assert_iterator_finished(it) 202 | 203 | def test_block_must_exist(): 204 | pat = ast.If(orelse=must_exist_checker) 205 | it = ASTPatternFinder(pat).scan_file(StringIO(if_sample)) 206 | assert_ast_like(next(it), ast.If(test=ast.Name(id='a'))) 207 | assert_iterator_finished(it) 208 | 209 | 210 | # Test matching of function calls ----------------------------------- 211 | 212 | func_call_sample = """ 213 | f1() 214 | f2(1) 215 | f3(1, 2) 216 | f4(1, 2, *c) 217 | f5(d=3) 218 | f6(d=3, e=4) 219 | f7(1, d=3, e=4) 220 | f8(1, d=4, **k) 221 | """ 222 | 223 | class FuncCallTests(unittest.TestCase): 224 | ast = ast.parse(func_call_sample) 225 | 226 | def get_matching_names(self, pat): 227 | apf = ASTPatternFinder(prepare_pattern(pat)) 228 | matches = apf.scan_ast(self.ast) 229 | return [f.func.id for f in matches] 230 | 231 | def test_wildcard_all(self): 232 | assert self.get_matching_names("?(??)") == [f"f{i}" for i in range(1, 9)] 233 | 234 | def test_pos_final_wildcard(self): 235 | assert self.get_matching_names("?(1, ??)") == ["f2", "f3", "f4", "f7", "f8"] 236 | 237 | def test_pos_leading_wildcard(self): 238 | assert self.get_matching_names("?(??, 2)") == ["f3"] 239 | 240 | def test_keywords_wildcard(self): 241 | assert self.get_matching_names("?(e=4, ??=??)") == ["f6"] 242 | 243 | def test_keywords_wildcard2(self): 244 | assert self.get_matching_names("?(d=?, ??=??)") == ["f5", "f6"] 245 | 246 | def test_mixed_wildcard(self): 247 | assert self.get_matching_names("?(??, d=?)") == ["f5", "f6", "f7", "f8"] 248 | 249 | def test_single_and_multi_wildcard(self): 250 | assert self.get_matching_names("?(?, ??)") == ["f2", "f3", "f4", "f7", "f8"] 251 | 252 | # Test matching of function definitions --------------------------------- 253 | 254 | func_def_samples = """ 255 | def f(): pass 256 | 257 | def g(a): pass 258 | 259 | def h(a, b): pass 260 | 261 | def i(a, b, *, c): pass 262 | 263 | def j(*a, c, d): pass 264 | 265 | def k(a, b, **k): pass 266 | 267 | def l(*, d, c, **k): pass 268 | 269 | def m(a, b=2, c=4): pass 270 | 271 | def n(a, b, c=4): pass 272 | """ 273 | 274 | class FuncDefTests(unittest.TestCase): 275 | ast = ast.parse(func_def_samples) 276 | 277 | def get_matching_names(self, pat): 278 | apf = ASTPatternFinder(prepare_pattern(pat)) 279 | matches = apf.scan_ast(self.ast) 280 | return {f.name for f in matches} 281 | 282 | def test_wildcard_all(self): 283 | matches = self.get_matching_names("def ?(??): ??") 284 | assert matches == {'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n'} 285 | 286 | def test_trailing_wildcard(self): 287 | matches = self.get_matching_names("def ?(a, ??): ??") 288 | assert matches == {'g', 'h', 'i', 'k', 'm', 'n'} 289 | 290 | def test_wildcard_kwonlyargs(self): 291 | matches = self.get_matching_names("def ?(*, c, ??): ??") 292 | assert matches == {'l'} 293 | 294 | def test_wildcard_w_defaults(self): 295 | matches = self.get_matching_names("def ?(a, b=2, ??=??, c=4): ??") 296 | assert matches == {'m'} 297 | 298 | def test_wildcard_w_defaults2(self): 299 | matches = self.get_matching_names("def ?(a, b=2, ??=??): ??") 300 | assert matches == {'m'} 301 | 302 | def test_no_wildcard(self): 303 | matches = self.get_matching_names("def ?(a, b, c=4): ??") 304 | assert matches == {'m', 'n'} 305 | 306 | def test_mix_wildcards(self): 307 | matches = self.get_matching_names("def ?(?, ??): ??") 308 | assert matches == {'g', 'h', 'i', 'k', 'm', 'n'} 309 | -------------------------------------------------------------------------------- /astsearch.py: -------------------------------------------------------------------------------- 1 | """Intelligently search Python source code""" 2 | import astcheck, ast 3 | from astcheck import assert_ast_like 4 | import os.path 5 | import sys 6 | import tokenize 7 | import warnings 8 | 9 | __version__ = '0.2.0' 10 | 11 | class ASTPatternFinder(object): 12 | """Scans Python code for AST nodes matching pattern. 13 | 14 | :param ast.AST pattern: The node pattern to search for 15 | """ 16 | def __init__(self, pattern): 17 | self.pattern = pattern 18 | 19 | def scan_ast(self, tree): 20 | """Walk an AST and yield nodes matching pattern. 21 | 22 | :param ast.AST tree: The AST in which to search 23 | """ 24 | nodetype = type(self.pattern) 25 | for node in ast.walk(tree): 26 | if isinstance(node, nodetype) and astcheck.is_ast_like(node, self.pattern): 27 | yield node 28 | 29 | def scan_file(self, file): 30 | """Parse a file and yield AST nodes matching pattern. 31 | 32 | :param file: Path to a Python file, or a readable file object 33 | """ 34 | if isinstance(file, str): 35 | with open(file, 'rb') as f: 36 | tree = ast.parse(f.read()) 37 | else: 38 | tree = ast.parse(file.read()) 39 | yield from self.scan_ast(tree) 40 | 41 | def filter_subdirs(self, dirnames): 42 | dirnames[:] = [d for d in dirnames if d != 'build'] 43 | 44 | def scan_directory(self, directory): 45 | """Walk files in a directory, yielding (filename, node) pairs matching 46 | pattern. 47 | 48 | :param str directory: Path to a directory to search 49 | 50 | Only files with a ``.py`` or ``.pyw`` extension will be scanned. 51 | """ 52 | for dirpath, dirnames, filenames in os.walk(directory): 53 | self.filter_subdirs(dirnames) 54 | 55 | for filename in filenames: 56 | if filename.endswith(('.py', '.pyw')): 57 | filepath = os.path.join(dirpath, filename) 58 | try: 59 | for match in self.scan_file(filepath): 60 | yield filepath, match 61 | except SyntaxError as e: 62 | warnings.warn("Failed to parse {}:\n{}".format(filepath, e)) 63 | 64 | def must_exist_checker(node, path): 65 | """Checker function to ensure a field is not empty""" 66 | if (node is None) or (node == []): 67 | raise astcheck.ASTMismatch(path, node, "non empty") 68 | 69 | def must_not_exist_checker(node, path): 70 | """Checker function to ensure a field is empty""" 71 | if (node is not None) and (node != []): 72 | raise astcheck.ASTMismatch(path, node, "empty") 73 | 74 | class ArgsDefChecker: 75 | """Checks the arguments of a function definition against pattern arguments. 76 | """ 77 | def __init__(self, args, defaults, vararg, kwonly_args_dflts, koa_subset, kwarg): 78 | self.args = args 79 | self.defaults = defaults 80 | self.vararg = vararg 81 | self.kwonly_args_dflts = kwonly_args_dflts 82 | self.koa_subset = koa_subset 83 | self.kwarg = kwarg 84 | 85 | def __repr__(self): 86 | return ("astsearch.ArgsDefChecker(args={s.args}, defaults={s.defaults}, " 87 | "vararg={s.vararg}, kwonly_args_dflts={s.kwonly_args_dflts}, " 88 | "koa_subset={s.koa_subset}, kwarg={s.kwarg}").format(s=self) 89 | 90 | def __call__(self, sample_node, path): 91 | # Check positional-or-keyword args 92 | if self.args: 93 | if isinstance(self.args, list): 94 | astcheck._check_node_list(path+['args'], sample_node.args, self.args) 95 | else: 96 | assert_ast_like(sample_node.args, self.args) 97 | 98 | # Check defaults for positional-or-keyword args 99 | if self.defaults: 100 | sample_args_w_defaults = sample_node.args[-len(sample_node.defaults):] 101 | sample_arg_defaults = {a.arg: d for a,d in 102 | zip(sample_args_w_defaults, sample_node.defaults)} 103 | for argname, dflt in self.defaults: 104 | try: 105 | sample_dflt = sample_arg_defaults[argname] 106 | except KeyError: 107 | raise astcheck.ASTMismatch(path + ['defaults', argname], 108 | "(missing default)", dflt) 109 | else: 110 | assert_ast_like(sample_dflt, dflt, path + ['defaults', argname]) 111 | 112 | # *args 113 | if self.vararg: 114 | assert_ast_like(sample_node.vararg, self.vararg) 115 | 116 | 117 | # keyword-only arguments 118 | sample_kwonlyargs = {k.arg: (k, d) for k, d in 119 | zip(sample_node.kwonlyargs, sample_node.kw_defaults)} 120 | 121 | for template_arg, template_dflt in self.kwonly_args_dflts: 122 | argname = template_arg.arg 123 | try: 124 | sample_arg, sample_dflt = sample_kwonlyargs[argname] 125 | except KeyError: 126 | raise astcheck.ASTMismatch(path+['kwonlyargs'], '(missing)', 'keyword arg %s' % argname) 127 | else: 128 | assert_ast_like(sample_arg, template_arg, path+['kwonlyargs', argname]) 129 | if template_dflt is not None: 130 | assert_ast_like(sample_dflt, template_dflt, path+['kw_defaults', argname]) 131 | 132 | # If keyword-only-args weren't wildcarded, then there shouldn't 133 | # be any more args in the sample than the template 134 | if not self.koa_subset: 135 | template_kwarg_names = {k.arg for k,d in self.kwonly_args_dflts} 136 | excess_names = set(sample_kwonlyargs) - template_kwarg_names 137 | if excess_names: 138 | raise astcheck.ASTMismatch(path+ ['kwonlyargs'], excess_names, "(not present in template)") 139 | 140 | # **kwargs 141 | if self.kwarg: 142 | assert_ast_like(sample_node.kwarg, self.kwarg) 143 | 144 | WILDCARD_NAME = "__astsearch_wildcard" 145 | MULTIWILDCARD_NAME = "__astsearch_multiwildcard" 146 | 147 | class TemplatePruner(ast.NodeTransformer): 148 | def visit_Name(self, node): 149 | if node.id == WILDCARD_NAME: 150 | return must_exist_checker # Allow any node type for a wildcard 151 | elif node.id == MULTIWILDCARD_NAME: 152 | # This shouldn't happen, but users will probably confuse their 153 | # wildcards at times. If it's in a block, it should have been 154 | # transformed before it's visited. 155 | return must_exist_checker 156 | 157 | # Generalise names to allow attributes as well, because these are often 158 | # interchangeable. 159 | return astcheck.name_or_attr(node.id) 160 | 161 | def prune_wildcard(self, node, attrname, must_exist=False): 162 | """Prunes a plain string attribute if it matches WILDCARD_NAME""" 163 | if getattr(node, attrname, None) in (WILDCARD_NAME, MULTIWILDCARD_NAME): 164 | setattr(node, attrname, must_exist_checker) 165 | 166 | def prune_wildcard_body(self, node, attrname, must_exist=False): 167 | """Prunes a code block (e.g. function body) if it is a wildcard""" 168 | body = getattr(node, attrname, []) 169 | def _is_multiwildcard(n): 170 | return astcheck.is_ast_like(n, 171 | ast.Expr(value=ast.Name(id=MULTIWILDCARD_NAME))) 172 | 173 | if len(body) == 1 and _is_multiwildcard(body[0]): 174 | setattr(node, attrname, must_exist_checker) 175 | return 176 | 177 | # Find a ?? node within the block, and replace it with listmiddle 178 | for i, n in enumerate(body): 179 | if _is_multiwildcard(n): 180 | newbody = body[:i] + astcheck.listmiddle() + body[i+1:] 181 | setattr(node, attrname, newbody) 182 | 183 | def visit_Attribute(self, node): 184 | self.prune_wildcard(node, 'attr') 185 | return self.generic_visit(node) 186 | 187 | def visit_Constant(self, node): 188 | # From Python 3.8, Constant nodes have a .kind attribute, which 189 | # distuingishes u"" from "": https://bugs.python.org/issue36280 190 | # astsearch isn't interested in that distinction. 191 | node.kind = None 192 | return self.generic_visit(node) 193 | 194 | def visit_FunctionDef(self, node): 195 | self.prune_wildcard(node, 'name') 196 | self.prune_wildcard_body(node, 'body') 197 | return self.generic_visit(node) 198 | 199 | visit_ClassDef = visit_FunctionDef 200 | 201 | def visit_arguments(self, node): 202 | positional_final_wildcard = False 203 | for i, a in enumerate(node.args): 204 | if a.arg == MULTIWILDCARD_NAME: 205 | from_end = len(node.args) - (i+1) 206 | if from_end == 0: 207 | # Last positional argument - wildcard may extend to other groups 208 | positional_final_wildcard = True 209 | 210 | args = self._visit_list(node.args[:i]) + astcheck.listmiddle() \ 211 | + self._visit_list(node.args[i+1:]) 212 | break 213 | else: 214 | if node.args: 215 | args = self._visit_list(node.args) 216 | else: 217 | args = must_not_exist_checker 218 | 219 | defaults = [(a.arg, self.visit(d)) 220 | for a,d in zip(node.args[-len(node.defaults):], node.defaults) 221 | if a.arg not in {WILDCARD_NAME, MULTIWILDCARD_NAME}] 222 | 223 | if node.vararg is None: 224 | if positional_final_wildcard: 225 | vararg = None 226 | else: 227 | vararg = must_not_exist_checker 228 | else: 229 | vararg = self.visit(node.vararg) 230 | 231 | kwonly_args_dflts = [(self.visit(a), (d if d is None else self.visit(d))) 232 | for a, d in zip(node.kwonlyargs, node.kw_defaults) 233 | if a.arg != MULTIWILDCARD_NAME] 234 | 235 | koa_subset = (positional_final_wildcard and vararg is None and (not node.kwonlyargs)) \ 236 | or any(a.arg == MULTIWILDCARD_NAME for a in node.kwonlyargs) 237 | 238 | if node.kwarg is None: 239 | if koa_subset: 240 | kwarg = None 241 | else: 242 | kwarg = must_not_exist_checker 243 | else: 244 | kwarg = self.visit(node.kwarg) 245 | 246 | return ArgsDefChecker(args=args, defaults=defaults, vararg=vararg, 247 | kwonly_args_dflts=kwonly_args_dflts, 248 | koa_subset=koa_subset, kwarg=kwarg) 249 | 250 | def visit_arg(self, node): 251 | self.prune_wildcard(node, 'arg') 252 | return self.generic_visit(node) 253 | 254 | def visit_If(self, node): 255 | self.prune_wildcard_body(node, 'body') 256 | self.prune_wildcard_body(node, 'orelse') 257 | return self.generic_visit(node) 258 | 259 | # All of these have body & orelse node lists 260 | visit_For = visit_While = visit_If 261 | 262 | def visit_Try(self, node): 263 | self.prune_wildcard_body(node, 'body') 264 | self.prune_wildcard_body(node, 'orelse') 265 | self.prune_wildcard_body(node, 'finalbody') 266 | return self.generic_visit(node) 267 | 268 | def visit_ExceptHandler(self, node): 269 | if node.type is None: 270 | node.type = must_not_exist_checker 271 | else: 272 | self.prune_wildcard(node, 'type') 273 | self.prune_wildcard(node, 'name') 274 | self.prune_wildcard_body(node, 'body') 275 | return self.generic_visit(node) 276 | 277 | def visit_With(self, node): 278 | self.prune_wildcard_body(node, 'body') 279 | return self.generic_visit(node) 280 | 281 | def visit_Call(self, node): 282 | kwargs_are_subset = False 283 | for i, n in enumerate(node.args): 284 | if astcheck.is_ast_like(n, ast.Name(id=MULTIWILDCARD_NAME)): 285 | if i + 1 == len(node.args): 286 | # Last positional argument - wildcard may extend to kwargs 287 | kwargs_are_subset = True 288 | 289 | node.args = self._visit_list( 290 | node.args[:i]) + astcheck.listmiddle() \ 291 | + self._visit_list(node.args[i + 1:]) 292 | 293 | # Don't try to handle multiple multiwildcards 294 | break 295 | 296 | if kwargs_are_subset or any( 297 | k.arg == MULTIWILDCARD_NAME for k in node.keywords): 298 | template_keywords = [self.visit(k) for k in node.keywords 299 | if k.arg != MULTIWILDCARD_NAME] 300 | 301 | def kwargs_checker(sample_keywords, path): 302 | sample_kwargs = {k.arg: k.value for k in sample_keywords} 303 | 304 | for k in template_keywords: 305 | if k.arg == MULTIWILDCARD_NAME: 306 | continue 307 | if k.arg in sample_kwargs: 308 | astcheck.assert_ast_like(sample_kwargs[k.arg], k.value, 309 | path + [k.arg]) 310 | else: 311 | raise astcheck.ASTMismatch(path, '(missing)', 312 | 'keyword arg %s' % k.arg) 313 | 314 | if template_keywords: 315 | node.keywords = kwargs_checker 316 | else: 317 | # Shortcut if there are no keywords to check 318 | del node.keywords 319 | 320 | # In block contexts, we want to avoid checking empty lists (for optional 321 | # nodes), but here, an empty list should mean that there are no 322 | # arguments in that group. So we need to override the behaviour in 323 | # generic_visit 324 | if node.args == []: 325 | node.args = must_not_exist_checker 326 | if getattr(node, 'keywords', None) == []: 327 | node.keywords = must_not_exist_checker 328 | return self.generic_visit(node) 329 | 330 | def prune_import_names(self, node): 331 | if len(node.names) == 1 and node.names[0].name == MULTIWILDCARD_NAME: 332 | del node.names 333 | else: 334 | for alias in node.names: 335 | self.visit_alias(alias) 336 | 337 | def visit_ImportFrom(self, node): 338 | self.prune_wildcard(node, 'module') 339 | self.prune_import_names(node) 340 | if node.level == 0: 341 | node.level = None 342 | return node 343 | 344 | def visit_Import(self, node): 345 | self.prune_import_names(node) 346 | return node 347 | 348 | def visit_alias(self, node): 349 | self.prune_wildcard(node, 'name') 350 | self.prune_wildcard(node, 'asname') 351 | 352 | def generic_visit(self, node): 353 | # Copied from ast.NodeTransformer; changes marked PATCH 354 | for field, old_value in ast.iter_fields(node): 355 | old_value = getattr(node, field, None) 356 | if isinstance(old_value, list): 357 | new_values = [] 358 | for value in old_value: 359 | if isinstance(value, ast.AST): 360 | value = self.visit(value) 361 | if value is None: 362 | continue 363 | # PATCH: We want to put checker functions in the AST 364 | #elif not isinstance(value, ast.AST): 365 | elif isinstance(value, list): 366 | # ------- 367 | new_values.extend(value) 368 | continue 369 | new_values.append(value) 370 | # PATCH: Delete field if list is empty 371 | if not new_values: 372 | delattr(node, field) 373 | # ------ 374 | old_value[:] = new_values 375 | elif isinstance(old_value, ast.AST): 376 | new_node = self.visit(old_value) 377 | setattr(node, field, new_node) 378 | return node 379 | 380 | def _visit_list(self, l): 381 | return [self.visit(n) for n in l] 382 | 383 | def prepare_pattern(s): 384 | """Turn a string pattern into an AST pattern 385 | 386 | This parses the string to an AST, and generalises it a bit for sensible 387 | matching. ``?`` is treated as a wildcard that matches anything. Names in 388 | the pattern will match names or attribute access (i.e. ``foo`` will match 389 | ``bar.foo`` in files). 390 | """ 391 | s = s.replace('??', MULTIWILDCARD_NAME).replace('?', WILDCARD_NAME) 392 | pattern = ast.parse(s).body[0] 393 | if isinstance(pattern, ast.Expr): 394 | pattern = pattern.value 395 | if isinstance(pattern, (ast.Attribute, ast.Subscript)): 396 | # If the root of the pattern is like a.b or a[b], we want to match it 397 | # regardless of context: `a.b=2` and `del a.b` should match as well as 398 | # `c = a.b` 399 | del pattern.ctx 400 | return TemplatePruner().visit(pattern) 401 | 402 | def main(argv=None): 403 | """Run astsearch from the command line. 404 | 405 | :param list argv: Command line arguments; defaults to :data:`sys.argv` 406 | """ 407 | import argparse 408 | ap = argparse.ArgumentParser() 409 | ap.add_argument('pattern', 410 | help="AST pattern to search for; see docs for examples") 411 | ap.add_argument('path', nargs='?', default='.', 412 | help="file or directory to search in") 413 | if sys.version_info >= (3, 8): 414 | ap.add_argument( 415 | '-m', '--max-lines', type=int, default=10, 416 | help="maximum number of lines printed per match (0 disables " 417 | "multiline printing)") 418 | ap.add_argument('-l', '--files-with-matches', action='store_true', 419 | help="output only the paths of matching files, not the " 420 | "lines that matched") 421 | ap.add_argument('--debug', action='store_true', help=argparse.SUPPRESS) 422 | 423 | args = ap.parse_args(argv) 424 | ast_pattern = prepare_pattern(args.pattern) 425 | if args.debug: 426 | print(ast.dump(ast_pattern)) 427 | 428 | patternfinder = ASTPatternFinder(ast_pattern) 429 | 430 | if getattr(args, 'max_lines'): 431 | def _printline(node, filelines): 432 | for lineno in range(node.lineno, node.end_lineno + 1)[:args.max_lines]: 433 | print("{:>4}│{}".format(lineno, filelines[lineno - 1].rstrip())) 434 | elided = max(node.end_lineno + 1 - node.lineno - args.max_lines, 0) 435 | if elided: 436 | print(" └<{} more line{}>".format(elided, ['', 's'][elided > 1])) 437 | print() 438 | else: 439 | def _printline(node, filelines): 440 | print("{:>4}|{}".format(node.lineno, filelines[node.lineno-1].rstrip())) 441 | 442 | current_filelines = [] 443 | if os.path.isdir(args.path): 444 | # Search directory 445 | current_filepath = None 446 | if args.files_with_matches: 447 | for filepath, node in patternfinder.scan_directory(args.path): 448 | if filepath != current_filepath: 449 | print(filepath) 450 | current_filepath = filepath 451 | else: 452 | for filepath, node in patternfinder.scan_directory(args.path): 453 | if filepath != current_filepath: 454 | with tokenize.open(filepath) as f: 455 | current_filelines = f.readlines() 456 | if current_filepath is not None: 457 | print() # Blank line between files 458 | current_filepath = filepath 459 | print(filepath) 460 | _printline(node, current_filelines) 461 | 462 | elif os.path.exists(args.path): 463 | # Search file 464 | if args.files_with_matches: 465 | try: 466 | node = next(patternfinder.scan_file(args.path)) 467 | except StopIteration: 468 | pass 469 | else: 470 | print(args.path) 471 | else: 472 | for node in patternfinder.scan_file(args.path): 473 | if not current_filelines: 474 | with tokenize.open(args.path) as f: 475 | current_filelines = f.readlines() 476 | _printline(node, current_filelines) 477 | 478 | else: 479 | sys.exit("No such file or directory: {}".format(args.path)) 480 | 481 | if __name__ == '__main__': 482 | main() 483 | --------------------------------------------------------------------------------