├── .coveragerc ├── .github └── workflows │ └── test.yml ├── .gitignore ├── CHANGES ├── LICENSE ├── MANIFEST.in ├── README.rst ├── dev-requirements.txt ├── docs ├── Makefile ├── conf.py ├── contents.rst ├── dictionary.txt ├── faq.rst ├── index.rst ├── make.bat └── runner.rst ├── dodo.py ├── pytest_incremental.py ├── setup.py ├── tests ├── sample-inc │ ├── dodo.py │ ├── mod1.py │ ├── mod2.py │ └── tt │ │ ├── conftest.py │ │ ├── tt_mod1.py │ │ └── tt_mod2.py ├── test_functional.py ├── test_graph.py └── test_tasks.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = pytest_incremental, tests -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | python-version: [3.6, 3.7, 3.8, 3.9] 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Set up Python ${{ matrix.python-version }} 20 | uses: actions/setup-python@v2 21 | with: 22 | python-version: ${{ matrix.python-version }} 23 | - name: Install dependencies 24 | run: | 25 | python -m pip install --upgrade pip 26 | python -m pip install . 27 | python -m pip install -r dev-requirements.txt 28 | 29 | - name: Lint with pyflakes 30 | run: | 31 | doit pyflakes 32 | - name: Test with pytest 33 | run: | 34 | pytest --json-report --json-report-indent 4 35 | - name: upload to testview 36 | run: 'curl -X POST https://testview.schettino72.net/api/projects/pytest-incremental/${{ github.sha }}/${{ github.job }}-${{ matrix.python-version}}?branch=${{ github.ref_name }} --header "Content-Type: application/json" -d "@.report.json"' 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.pyo 3 | *.egg-info 4 | __pycache__ 5 | docs/_build 6 | 7 | .coverage 8 | *.doit.db 9 | dist 10 | .pytest_cache 11 | .tox 12 | .pytest-incremental* 13 | MANIFEST 14 | revision.txt 15 | -------------------------------------------------------------------------------- /CHANGES: -------------------------------------------------------------------------------- 1 | 2 | ======= 3 | Changes 4 | ======= 5 | 6 | 0.6.0 (*2021-04-25*) 7 | ==================== 8 | 9 | - drop python 3.5 10 | - add python 3.8, 3.9 11 | - support pytest 6.x 12 | 13 | 14 | 0.5.0 (*2018-12-09*) 15 | ==================== 16 | 17 | - drop python 2 support 18 | - require doit==0.31, pytest==4.0 19 | 20 | 21 | 0.4.2 (*2015-05-08*) 22 | ==================== 23 | 24 | - fix tox. include dependencies 25 | 26 | 27 | 0.4.1 (*2015-05-08*) 28 | ==================== 29 | 30 | - include tests and docs on distribution package 31 | 32 | 33 | 0.4.0 (*2015-05-07*) 34 | ==================== 35 | 36 | - move development to github 37 | - major re-rewrite, renamed all command line options 38 | - support for python 3 39 | - re-ordering of test execution 40 | 41 | 42 | 0.3.0 (*2013-01-09*) 43 | ===================== 44 | 45 | - fixes to work with latest doit (0.20) 46 | 47 | 0.2.0 (*2011-05-07*) 48 | ===================== 49 | 50 | - added option --graph-dependencies 51 | - support for xdist --dist=load 52 | - added option --list-dependencies 53 | - added option --list-outdated 54 | - fix bug. correctly detect when a test fails 55 | - rename option --watch-pkg to --watch-path 56 | 57 | 0.1.0 (*2011-04-25*) 58 | ==================== 59 | 60 | - initial release 61 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | 3 | The MIT License 4 | 5 | Copyright (c) 2011-2018 Eduardo Naufel Schettino 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy 8 | of this software and associated documentation files (the "Software"), to deal 9 | in the Software without restriction, including without limitation the rights 10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom the Software is 12 | furnished to do so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in 15 | all copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include .coveragerc 2 | include .github/workflows/test.yml 3 | include .gitignore 4 | include CHANGES 5 | include LICENSE 6 | include MANIFEST.in 7 | include README.rst 8 | include dev-requirements.txt 9 | include docs/Makefile 10 | include docs/conf.py 11 | include docs/contents.rst 12 | include docs/dictionary.txt 13 | include docs/faq.rst 14 | include docs/index.rst 15 | include docs/make.bat 16 | include docs/runner.rst 17 | include dodo.py 18 | include pytest_incremental.py 19 | include setup.py 20 | include tests/sample-inc/dodo.py 21 | include tests/sample-inc/mod1.py 22 | include tests/sample-inc/mod2.py 23 | include tests/sample-inc/tt/conftest.py 24 | include tests/sample-inc/tt/tt_mod1.py 25 | include tests/sample-inc/tt/tt_mod2.py 26 | include tests/test_functional.py 27 | include tests/test_graph.py 28 | include tests/test_tasks.py 29 | include tox.ini 30 | include revision.txt 31 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://img.shields.io/pypi/v/pytest-incremental.svg 2 | :target: https://pypi.python.org/pypi/pytest-incremental 3 | 4 | .. image:: https://img.shields.io/pypi/pyversions/pytest-incremental.svg 5 | :target: https://pypi.python.org/pypi/pytest-incremental 6 | 7 | .. image:: https://github.com/pytest-dev/pytest-incremental/workflows/test/badge.svg 8 | :target: https://github.com/pytest-dev/pytest-incremental/actions?query=workflow%3Atest 9 | 10 | 11 | pytest-incremental 12 | ==================== 13 | 14 | an incremental test runner (pytest plug-in) 15 | 16 | *pytest-incremental* analyses your project structure and file 17 | modifications between test-runs 18 | to modify the order tests are executed and de-select tests. 19 | This allows a much faster feedback for interactive test execution. 20 | 21 | 22 | Project Details 23 | =============== 24 | 25 | - Website & docs - https://pytest-incremental.readthedocs.io 26 | - Project code + issue track on github - https://github.com/pytest-dev/pytest-incremental 27 | - PyPI - https://pypi.python.org/pypi/pytest-incremental 28 | - Discussion group - http://groups.google.co.in/group/python-doit 29 | 30 | 31 | license 32 | ======= 33 | 34 | The MIT License 35 | Copyright (c) 2015-2021 Eduardo Naufel Schettino 36 | 37 | see LICENSE file 38 | -------------------------------------------------------------------------------- /dev-requirements.txt: -------------------------------------------------------------------------------- 1 | pytest-xdist 2 | pyflakes 3 | coverage 4 | doit-py 5 | git+https://github.com/schettino72/pytest-json-report@schettino72-fork#egg=pytest-json-report 6 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " applehelp to make an Apple Help Book" 34 | @echo " devhelp to make HTML files and a Devhelp project" 35 | @echo " epub to make an epub" 36 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 37 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 38 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 39 | @echo " text to make text files" 40 | @echo " man to make manual pages" 41 | @echo " texinfo to make Texinfo files" 42 | @echo " info to make Texinfo files and run them through makeinfo" 43 | @echo " gettext to make PO message catalogs" 44 | @echo " changes to make an overview of all changed/added/deprecated items" 45 | @echo " xml to make Docutils-native XML files" 46 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 47 | @echo " linkcheck to check all external links for integrity" 48 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 49 | @echo " coverage to run coverage check of the documentation (if enabled)" 50 | 51 | clean: 52 | rm -rf $(BUILDDIR)/* 53 | 54 | html: 55 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 56 | @echo 57 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 58 | 59 | dirhtml: 60 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 61 | @echo 62 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 63 | 64 | singlehtml: 65 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 66 | @echo 67 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 68 | 69 | pickle: 70 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 71 | @echo 72 | @echo "Build finished; now you can process the pickle files." 73 | 74 | json: 75 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 76 | @echo 77 | @echo "Build finished; now you can process the JSON files." 78 | 79 | htmlhelp: 80 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 81 | @echo 82 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 83 | ".hhp project file in $(BUILDDIR)/htmlhelp." 84 | 85 | qthelp: 86 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 87 | @echo 88 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 89 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 90 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pytest-incremental.qhcp" 91 | @echo "To view the help file:" 92 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pytest-incremental.qhc" 93 | 94 | applehelp: 95 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 96 | @echo 97 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 98 | @echo "N.B. You won't be able to view it unless you put it in" \ 99 | "~/Library/Documentation/Help or install it in your application" \ 100 | "bundle." 101 | 102 | devhelp: 103 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 104 | @echo 105 | @echo "Build finished." 106 | @echo "To view the help file:" 107 | @echo "# mkdir -p $$HOME/.local/share/devhelp/pytest-incremental" 108 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pytest-incremental" 109 | @echo "# devhelp" 110 | 111 | epub: 112 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 113 | @echo 114 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 115 | 116 | latex: 117 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 118 | @echo 119 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 120 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 121 | "(use \`make latexpdf' here to do that automatically)." 122 | 123 | latexpdf: 124 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 125 | @echo "Running LaTeX files through pdflatex..." 126 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 127 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 128 | 129 | latexpdfja: 130 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 131 | @echo "Running LaTeX files through platex and dvipdfmx..." 132 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 133 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 134 | 135 | text: 136 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 137 | @echo 138 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 139 | 140 | man: 141 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 142 | @echo 143 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 144 | 145 | texinfo: 146 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 147 | @echo 148 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 149 | @echo "Run \`make' in that directory to run these through makeinfo" \ 150 | "(use \`make info' here to do that automatically)." 151 | 152 | info: 153 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 154 | @echo "Running Texinfo files through makeinfo..." 155 | make -C $(BUILDDIR)/texinfo info 156 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 157 | 158 | gettext: 159 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 160 | @echo 161 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 162 | 163 | changes: 164 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 165 | @echo 166 | @echo "The overview file is in $(BUILDDIR)/changes." 167 | 168 | linkcheck: 169 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 170 | @echo 171 | @echo "Link check complete; look for any errors in the above output " \ 172 | "or in $(BUILDDIR)/linkcheck/output.txt." 173 | 174 | doctest: 175 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 176 | @echo "Testing of doctests in the sources finished, look at the " \ 177 | "results in $(BUILDDIR)/doctest/output.txt." 178 | 179 | coverage: 180 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 181 | @echo "Testing of coverage in the sources finished, look at the " \ 182 | "results in $(BUILDDIR)/coverage/python.txt." 183 | 184 | xml: 185 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 186 | @echo 187 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 188 | 189 | pseudoxml: 190 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 191 | @echo 192 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 193 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # pytest-incremental documentation build configuration file, created by 4 | # sphinx-quickstart on Wed Apr 22 18:47:03 2015. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | import shlex 18 | 19 | # If extensions (or modules to document with autodoc) are in another directory, 20 | # add these directories to sys.path here. If the directory is relative to the 21 | # documentation root, use os.path.abspath to make it absolute, like shown here. 22 | #sys.path.insert(0, os.path.abspath('.')) 23 | 24 | # -- General configuration ------------------------------------------------ 25 | 26 | # If your documentation needs a minimal Sphinx version, state it here. 27 | #needs_sphinx = '1.0' 28 | 29 | # Add any Sphinx extension module names here, as strings. They can be 30 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 31 | # ones. 32 | extensions = ['sphinx.ext.graphviz'] 33 | 34 | # Add any paths that contain templates here, relative to this directory. 35 | templates_path = ['_templates'] 36 | 37 | # The suffix(es) of source filenames. 38 | # You can specify multiple suffix as a list of string: 39 | # source_suffix = ['.rst', '.md'] 40 | source_suffix = '.rst' 41 | 42 | # The encoding of source files. 43 | #source_encoding = 'utf-8-sig' 44 | 45 | # The master toctree document. 46 | master_doc = 'contents' 47 | 48 | # General information about the project. 49 | project = u'pytest-incremental' 50 | copyright = u'2018, schettino72' 51 | author = u'schettino72' 52 | 53 | # The version info for the project you're documenting, acts as replacement for 54 | # |version| and |release|, also used in various other places throughout the 55 | # built documents. 56 | # 57 | # The short X.Y version. 58 | version = '0.5' 59 | # The full version, including alpha/beta/rc tags. 60 | release = '0.5' 61 | 62 | # The language for content autogenerated by Sphinx. Refer to documentation 63 | # for a list of supported languages. 64 | # 65 | # This is also used if you do content translation via gettext catalogs. 66 | # Usually you set "language" from the command line for these cases. 67 | language = None 68 | 69 | # There are two options for replacing |today|: either, you set today to some 70 | # non-false value, then it is used: 71 | #today = '' 72 | # Else, today_fmt is used as the format for a strftime call. 73 | #today_fmt = '%B %d, %Y' 74 | 75 | # List of patterns, relative to source directory, that match files and 76 | # directories to ignore when looking for source files. 77 | exclude_patterns = ['_build'] 78 | 79 | # The reST default role (used for this markup: `text`) to use for all 80 | # documents. 81 | #default_role = None 82 | 83 | # If true, '()' will be appended to :func: etc. cross-reference text. 84 | #add_function_parentheses = True 85 | 86 | # If true, the current module name will be prepended to all description 87 | # unit titles (such as .. function::). 88 | #add_module_names = True 89 | 90 | # If true, sectionauthor and moduleauthor directives will be shown in the 91 | # output. They are ignored by default. 92 | #show_authors = False 93 | 94 | # The name of the Pygments (syntax highlighting) style to use. 95 | pygments_style = 'sphinx' 96 | 97 | # A list of ignored prefixes for module index sorting. 98 | #modindex_common_prefix = [] 99 | 100 | # If true, keep warnings as "system message" paragraphs in the built documents. 101 | #keep_warnings = False 102 | 103 | # If true, `todo` and `todoList` produce output, else they produce nothing. 104 | todo_include_todos = False 105 | 106 | 107 | # -- Options for HTML output ---------------------------------------------- 108 | 109 | # The theme to use for HTML and HTML Help pages. See the documentation for 110 | # a list of builtin themes. 111 | # on_rtd is whether we are on readthedocs.org 112 | 113 | # import os 114 | # on_rtd = os.environ.get('READTHEDOCS', None) == 'True' 115 | 116 | # if not on_rtd: # only import and set the theme if we're building docs locally 117 | # html_theme = 'sphinxdoc' 118 | # # else RTD use its default theme 119 | html_theme = 'sphinxdoc' 120 | 121 | # Theme options are theme-specific and customize the look and feel of a theme 122 | # further. For a list of options available for each theme, see the 123 | # documentation. 124 | #html_theme_options = {} 125 | 126 | # Add any paths that contain custom themes here, relative to this directory. 127 | #html_theme_path = [] 128 | 129 | # The name for this set of Sphinx documents. If None, it defaults to 130 | # " v documentation". 131 | #html_title = None 132 | 133 | # A shorter title for the navigation bar. Default is the same as html_title. 134 | #html_short_title = None 135 | 136 | # The name of an image file (relative to this directory) to place at the top 137 | # of the sidebar. 138 | #html_logo = None 139 | 140 | # The name of an image file (within the static path) to use as favicon of the 141 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 142 | # pixels large. 143 | #html_favicon = None 144 | 145 | # Add any paths that contain custom static files (such as style sheets) here, 146 | # relative to this directory. They are copied after the builtin static files, 147 | # so a file named "default.css" will overwrite the builtin "default.css". 148 | html_static_path = ['_static'] 149 | 150 | # Add any extra paths that contain custom files (such as robots.txt or 151 | # .htaccess) here, relative to this directory. These files are copied 152 | # directly to the root of the documentation. 153 | #html_extra_path = [] 154 | 155 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 156 | # using the given strftime format. 157 | #html_last_updated_fmt = '%b %d, %Y' 158 | 159 | # If true, SmartyPants will be used to convert quotes and dashes to 160 | # typographically correct entities. 161 | #html_use_smartypants = True 162 | 163 | # Custom sidebar templates, maps document names to template names. 164 | #html_sidebars = {} 165 | 166 | # Additional templates that should be rendered to pages, maps page names to 167 | # template names. 168 | #html_additional_pages = {} 169 | 170 | # If false, no module index is generated. 171 | #html_domain_indices = True 172 | 173 | # If false, no index is generated. 174 | #html_use_index = True 175 | 176 | # If true, the index is split into individual pages for each letter. 177 | #html_split_index = False 178 | 179 | # If true, links to the reST sources are added to the pages. 180 | #html_show_sourcelink = True 181 | 182 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 183 | #html_show_sphinx = True 184 | 185 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 186 | #html_show_copyright = True 187 | 188 | # If true, an OpenSearch description file will be output, and all pages will 189 | # contain a tag referring to it. The value of this option must be the 190 | # base URL from which the finished HTML is served. 191 | #html_use_opensearch = '' 192 | 193 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 194 | #html_file_suffix = None 195 | 196 | # Language to be used for generating the HTML full-text search index. 197 | # Sphinx supports the following languages: 198 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' 199 | # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' 200 | #html_search_language = 'en' 201 | 202 | # A dictionary with options for the search language support, empty by default. 203 | # Now only 'ja' uses this config value 204 | #html_search_options = {'type': 'default'} 205 | 206 | # The name of a javascript file (relative to the configuration directory) that 207 | # implements a search results scorer. If empty, the default will be used. 208 | #html_search_scorer = 'scorer.js' 209 | 210 | # Output file base name for HTML help builder. 211 | htmlhelp_basename = 'pytest-incrementaldoc' 212 | 213 | # -- Options for LaTeX output --------------------------------------------- 214 | 215 | latex_elements = { 216 | # The paper size ('letterpaper' or 'a4paper'). 217 | #'papersize': 'letterpaper', 218 | 219 | # The font size ('10pt', '11pt' or '12pt'). 220 | #'pointsize': '10pt', 221 | 222 | # Additional stuff for the LaTeX preamble. 223 | #'preamble': '', 224 | 225 | # Latex figure (float) alignment 226 | #'figure_align': 'htbp', 227 | } 228 | 229 | # Grouping the document tree into LaTeX files. List of tuples 230 | # (source start file, target name, title, 231 | # author, documentclass [howto, manual, or own class]). 232 | latex_documents = [ 233 | (master_doc, 'pytest-incremental.tex', u'pytest-incremental Documentation', 234 | u'schettino72', 'manual'), 235 | ] 236 | 237 | # The name of an image file (relative to this directory) to place at the top of 238 | # the title page. 239 | #latex_logo = None 240 | 241 | # For "manual" documents, if this is true, then toplevel headings are parts, 242 | # not chapters. 243 | #latex_use_parts = False 244 | 245 | # If true, show page references after internal links. 246 | #latex_show_pagerefs = False 247 | 248 | # If true, show URL addresses after external links. 249 | #latex_show_urls = False 250 | 251 | # Documents to append as an appendix to all manuals. 252 | #latex_appendices = [] 253 | 254 | # If false, no module index is generated. 255 | #latex_domain_indices = True 256 | 257 | 258 | # -- Options for manual page output --------------------------------------- 259 | 260 | # One entry per manual page. List of tuples 261 | # (source start file, name, description, authors, manual section). 262 | man_pages = [ 263 | (master_doc, 'pytest-incremental', u'pytest-incremental Documentation', 264 | [author], 1) 265 | ] 266 | 267 | # If true, show URL addresses after external links. 268 | #man_show_urls = False 269 | 270 | 271 | # -- Options for Texinfo output ------------------------------------------- 272 | 273 | # Grouping the document tree into Texinfo files. List of tuples 274 | # (source start file, target name, title, author, 275 | # dir menu entry, description, category) 276 | texinfo_documents = [ 277 | (master_doc, 'pytest-incremental', u'pytest-incremental Documentation', 278 | author, 'pytest-incremental', 'One line description of project.', 279 | 'Miscellaneous'), 280 | ] 281 | 282 | # Documents to append as an appendix to all manuals. 283 | #texinfo_appendices = [] 284 | 285 | # If false, no module index is generated. 286 | #texinfo_domain_indices = True 287 | 288 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 289 | #texinfo_show_urls = 'footnote' 290 | 291 | # If true, do not generate a @detailmenu in the "Top" node's menu. 292 | #texinfo_no_detailmenu = False 293 | -------------------------------------------------------------------------------- /docs/contents.rst: -------------------------------------------------------------------------------- 1 | 2 | pytest-incremental documentation 3 | ================================ 4 | 5 | 6 | documentation 7 | ------------- 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | 12 | index 13 | runner 14 | faq 15 | -------------------------------------------------------------------------------- /docs/dictionary.txt: -------------------------------------------------------------------------------- 1 | 's 2 | 3Atest 3 | API 4 | AST 5 | BT 6 | CWD 7 | Github 8 | Naufel 9 | PyPI 10 | README 11 | Schettino 12 | TextTestRunner 13 | addopts 14 | app 15 | de 16 | deps 17 | exitfirst 18 | faq 19 | fillcolor 20 | fontcolor 21 | foo 22 | github 23 | graphviz 24 | ini 25 | instafail 26 | maxdepth 27 | maxfail 28 | ok 29 | pdb 30 | plugin 31 | py 32 | py3rd 33 | pytest 34 | quickstart 35 | rankdir 36 | stdlib 37 | testmon 38 | toctree 39 | traceback 40 | unittest 41 | unittest's 42 | util 43 | -------------------------------------------------------------------------------- /docs/faq.rst: -------------------------------------------------------------------------------- 1 | 2 | What is an "incremental test runner" ? 3 | ======================================= 4 | 5 | The idea is to execute your tests faster by executing not all of them 6 | but only the "required" ones. 7 | 8 | When talking about build-tools it is common to refer to the terms: 9 | 10 | * initial (full) build - all files are compiled 11 | * incremental build (or partial rebuild) - just changed files are compiled 12 | * no-op build - no files are compiled (none changed since last execution) 13 | 14 | So an "incremental test runner" will only re-execute tests that were affected 15 | by changes in the source code since last successful execution. 16 | 17 | 18 | How *pytest-incremental* compares to `testmon `_ ? 19 | ====================================================================================== 20 | 21 | Both projects have a similar goal but have completely different implementations. 22 | *testmon* uses *coverage.py* to trace the execution of tests while *pytest-incremental* analyses the imported dependencies. 23 | 24 | - *testmon* can track sub-module dependencies, so it can de-select more 25 | tests that *pytest-incremental* 26 | - *testmon* does not re-order tests according to the source structure 27 | - because *testmon* traces the code execution the test execution is slower 28 | than normally 29 | 30 | 31 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. pytest-incremental documentation master file, created by 2 | sphinx-quickstart on Wed Apr 22 18:47:03 2015. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | ============================================================ 7 | pytest-incremental: py.test plugin - incremental test runner 8 | ============================================================ 9 | 10 | 11 | Github: 12 | https://github.com/pytest-dev/pytest-incremental 13 | 14 | PyPI: 15 | https://pypi.python.org/pypi/pytest-incremental 16 | 17 | 18 | 19 | `pytest-incremental` is a `py.test `_ plug-in. 20 | It analyses your project structure and file modifications between test-runs 21 | to modify the order tests are executed and de-select tests. 22 | This allows a much faster feedback for interactive test execution. 23 | 24 | Check the :ref:`documentation ` for more details. 25 | 26 | Note that *py.test* has support to run standard unittest's and nose's tests. 27 | So even if you don't use *py.test* as a test framework you might be able to 28 | use it as a test runner. 29 | 30 | 31 | 32 | Install 33 | ========= 34 | 35 | pytest-incremental is tested on python 3.6-3.9, pytest 6.x. 36 | 37 | ``pip install pytest-incremental`` 38 | 39 | 40 | 41 | Usage 42 | ====== 43 | 44 | Just pass the parameter ``--inc`` when calling from the command line:: 45 | 46 | $ py.test --inc 47 | 48 | 49 | You can also enable it by default adding the following 50 | line to your ``pytest.ini``:: 51 | 52 | [pytest] 53 | addopts = --inc 54 | 55 | 56 | watched packages 57 | ------------------ 58 | 59 | By default all modules from within your *CWD* will be counted as dependencies 60 | if imported. In order to limit or extend the watched folders you must use 61 | the parameter ``--inc-path`` 62 | 63 | 64 | This can be used in case you want to watch for changes in modules that are 65 | in another project. 66 | For example if you are testing ``my_lib`` and want to check for changes 67 | in dependencies from your `py3rd` package:: 68 | 69 | $ py.test --inc --inc-path my_lib --inc-path ../py3rd-trunk/py3rd 70 | 71 | 72 | dependencies 73 | -------------- 74 | 75 | You can check what are the actual dependencies detected by running the command:: 76 | 77 | $ py.test --inc-deps 78 | 79 | for better visualization you can create a graph file in "dot" format 80 | (see `graphviz `_ ):: 81 | 82 | $ py.test --inc-graph 83 | 84 | To generate an image:: 85 | 86 | $ py.test --inc-graph-image 87 | 88 | 89 | You can also check what are the outdated tests without executing them:: 90 | 91 | $ py.test --inc-outdated 92 | 93 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | echo. coverage to run coverage check of the documentation if enabled 41 | goto end 42 | ) 43 | 44 | if "%1" == "clean" ( 45 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 46 | del /q /s %BUILDDIR%\* 47 | goto end 48 | ) 49 | 50 | 51 | REM Check if sphinx-build is available and fallback to Python version if any 52 | %SPHINXBUILD% 2> nul 53 | if errorlevel 9009 goto sphinx_python 54 | goto sphinx_ok 55 | 56 | :sphinx_python 57 | 58 | set SPHINXBUILD=python -m sphinx.__init__ 59 | %SPHINXBUILD% 2> nul 60 | if errorlevel 9009 ( 61 | echo. 62 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 63 | echo.installed, then set the SPHINXBUILD environment variable to point 64 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 65 | echo.may add the Sphinx directory to PATH. 66 | echo. 67 | echo.If you don't have Sphinx installed, grab it from 68 | echo.http://sphinx-doc.org/ 69 | exit /b 1 70 | ) 71 | 72 | :sphinx_ok 73 | 74 | 75 | if "%1" == "html" ( 76 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 77 | if errorlevel 1 exit /b 1 78 | echo. 79 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 80 | goto end 81 | ) 82 | 83 | if "%1" == "dirhtml" ( 84 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 85 | if errorlevel 1 exit /b 1 86 | echo. 87 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 88 | goto end 89 | ) 90 | 91 | if "%1" == "singlehtml" ( 92 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 93 | if errorlevel 1 exit /b 1 94 | echo. 95 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 96 | goto end 97 | ) 98 | 99 | if "%1" == "pickle" ( 100 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 101 | if errorlevel 1 exit /b 1 102 | echo. 103 | echo.Build finished; now you can process the pickle files. 104 | goto end 105 | ) 106 | 107 | if "%1" == "json" ( 108 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 109 | if errorlevel 1 exit /b 1 110 | echo. 111 | echo.Build finished; now you can process the JSON files. 112 | goto end 113 | ) 114 | 115 | if "%1" == "htmlhelp" ( 116 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 117 | if errorlevel 1 exit /b 1 118 | echo. 119 | echo.Build finished; now you can run HTML Help Workshop with the ^ 120 | .hhp project file in %BUILDDIR%/htmlhelp. 121 | goto end 122 | ) 123 | 124 | if "%1" == "qthelp" ( 125 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 129 | .qhcp project file in %BUILDDIR%/qthelp, like this: 130 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\pytest-incremental.qhcp 131 | echo.To view the help file: 132 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\pytest-incremental.ghc 133 | goto end 134 | ) 135 | 136 | if "%1" == "devhelp" ( 137 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 138 | if errorlevel 1 exit /b 1 139 | echo. 140 | echo.Build finished. 141 | goto end 142 | ) 143 | 144 | if "%1" == "epub" ( 145 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 146 | if errorlevel 1 exit /b 1 147 | echo. 148 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 149 | goto end 150 | ) 151 | 152 | if "%1" == "latex" ( 153 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 154 | if errorlevel 1 exit /b 1 155 | echo. 156 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 157 | goto end 158 | ) 159 | 160 | if "%1" == "latexpdf" ( 161 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 162 | cd %BUILDDIR%/latex 163 | make all-pdf 164 | cd %~dp0 165 | echo. 166 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 167 | goto end 168 | ) 169 | 170 | if "%1" == "latexpdfja" ( 171 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 172 | cd %BUILDDIR%/latex 173 | make all-pdf-ja 174 | cd %~dp0 175 | echo. 176 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 177 | goto end 178 | ) 179 | 180 | if "%1" == "text" ( 181 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 182 | if errorlevel 1 exit /b 1 183 | echo. 184 | echo.Build finished. The text files are in %BUILDDIR%/text. 185 | goto end 186 | ) 187 | 188 | if "%1" == "man" ( 189 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 190 | if errorlevel 1 exit /b 1 191 | echo. 192 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 193 | goto end 194 | ) 195 | 196 | if "%1" == "texinfo" ( 197 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 198 | if errorlevel 1 exit /b 1 199 | echo. 200 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 201 | goto end 202 | ) 203 | 204 | if "%1" == "gettext" ( 205 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 206 | if errorlevel 1 exit /b 1 207 | echo. 208 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 209 | goto end 210 | ) 211 | 212 | if "%1" == "changes" ( 213 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 214 | if errorlevel 1 exit /b 1 215 | echo. 216 | echo.The overview file is in %BUILDDIR%/changes. 217 | goto end 218 | ) 219 | 220 | if "%1" == "linkcheck" ( 221 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 222 | if errorlevel 1 exit /b 1 223 | echo. 224 | echo.Link check complete; look for any errors in the above output ^ 225 | or in %BUILDDIR%/linkcheck/output.txt. 226 | goto end 227 | ) 228 | 229 | if "%1" == "doctest" ( 230 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 231 | if errorlevel 1 exit /b 1 232 | echo. 233 | echo.Testing of doctests in the sources finished, look at the ^ 234 | results in %BUILDDIR%/doctest/output.txt. 235 | goto end 236 | ) 237 | 238 | if "%1" == "coverage" ( 239 | %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage 240 | if errorlevel 1 exit /b 1 241 | echo. 242 | echo.Testing of coverage in the sources finished, look at the ^ 243 | results in %BUILDDIR%/coverage/python.txt. 244 | goto end 245 | ) 246 | 247 | if "%1" == "xml" ( 248 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 249 | if errorlevel 1 exit /b 1 250 | echo. 251 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 252 | goto end 253 | ) 254 | 255 | if "%1" == "pseudoxml" ( 256 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 257 | if errorlevel 1 exit /b 1 258 | echo. 259 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 260 | goto end 261 | ) 262 | 263 | :end 264 | -------------------------------------------------------------------------------- /docs/runner.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _motivation: 3 | 4 | Test Runners 5 | ============ 6 | 7 | Let's start by looking at what is a test-runner. 8 | We need to differentiate a **test-runner** from a **test-framework**. 9 | 10 | From python `documentation `_: 11 | A test runner is a component which orchestrates the execution of tests and 12 | provides the outcome to the user. The runner may use a graphical interface, a 13 | textual interface, or return a special value to indicate the results of 14 | executing the tests. 15 | 16 | A test-framework defines an API that is used to write the tests. 17 | For example the `unittest` module from python's stdlib. 18 | 19 | The `unittest` module also defines an API for creating test runners, and 20 | provides a basic test runner `implementation `_. 21 | Apart from this, there are other runners that has support for running 22 | `unittest`\'s tests like `nose` and `py.test`. 23 | 24 | On the other hand `py.test` defines its own test-framework but only 25 | its own runner is capable of running its tests. 26 | 27 | 28 | Test Runner Features 29 | ==================== 30 | 31 | When using a test runner interactively in the command line I expect two 32 | main things: 33 | 34 | - fast feedback: I want to know as fast as possible if my latest changes are 35 | OK (tests are executed successfully or not). 36 | - in case of test failures, how easy it is to find the problem. 37 | 38 | Note that I said **fast feedback** not faster **tests**. 39 | Of course the actual test 40 | (and not the test runner) plays the main role in the time to execute a test. 41 | But you should not always need to wait until **all** tests are executed before 42 | getting some feedback. 43 | 44 | For example *py.test* offers the option `-x/--exitfirst` and `--maxfail`, 45 | to display information on failures without waiting for all tests to finish. 46 | Also check the `instafail plugin `_. 47 | 48 | Another way to achieve faster feedback is by executing just a sub-set of 49 | your tests. 50 | Using *py.test*, apart from selecting tests from just a package or module, 51 | also has a very powerful system to select tests based 52 | on *keywords* using the option `-k`. 53 | While this feature is extremely useful it 54 | 55 | *py.test* has great tools to help you debug failed tests like colorful output, 56 | *pdb* integration, assertion re-write and control of traceback verbosity. 57 | 58 | 59 | Importance of Test Ordering 60 | =========================== 61 | 62 | By default *py.test* will execute tests grouped by test module. 63 | Modules are ordered alphabetically, tests from each module are executed 64 | in the order they are defined. 65 | Although unit-tests *should* work when executed in any order, 66 | it is important to execute them in a defined order so failures can be easily 67 | reproduced. 68 | 69 | But using a simple alphabetical order that does not take into account 70 | the structure of the code has several disadvantages. 71 | 72 | #. To achieve faster feedback it is important that the most **relevant** tests 73 | should be executed first. Using alphabetical order you might spend a long 74 | time executing tests that were not affected by recent changes, or execute 75 | tests that have little chance to fail. 76 | 77 | #. It is common that a single change break several tests. 78 | In order to easily identify the cause of the problem it is important to 79 | look at the test that is directly testing the point where the code changed. 80 | It might not be easy to pin-point the problem when looking at failures 81 | in tests that were broken but not directly test the problem. 82 | Executing the most **relevant** tests first you could make sure to get 83 | *direct* failures first. 84 | 85 | 86 | How to order tests 87 | ================== 88 | 89 | There are two main factors to determine a most relevant order to execute tests. 90 | 91 | - The source code inter-dependency structure 92 | (this is done by analyzing the *imports*) 93 | 94 | - Modified modules since last successful execution 95 | 96 | 97 | Lets look at simple example project that contains four modules, 98 | and each module a corresponding test module. 99 | Look at the **imports** graph below where an edge ``bar -> util`` 100 | means that ``bar.py`` imports ``util.py``. 101 | 102 | .. graphviz:: 103 | 104 | digraph imports { 105 | rankdir = BT 106 | util [shape=box, color=blue] 107 | bar [shape=box, color=blue] 108 | foo [shape=box, color=blue] 109 | app [shape=box, color=blue] 110 | 111 | "bar" -> "util" 112 | "foo" -> "util" 113 | "app" -> "bar" 114 | "app" -> "foo" 115 | 116 | "test_util" -> "util" 117 | "test_bar" -> "bar" 118 | "test_foo" -> "foo" 119 | "test_app" -> "app" 120 | } 121 | 122 | 123 | 124 | Initial (full) run 125 | ------------------ 126 | 127 | On the first run all tests must be executed. 128 | Since ``bar`` and ``foo`` depends on ``util``, we want to execute 129 | ``test_util`` first to make sure any problems on ``util`` are caught 130 | first by its direct tests on ``test_util``. 131 | 132 | The same applies for ``app`` in relation to ``foo`` and ``bar``. 133 | ``foo`` and ``bar`` both have the same *level* in the structure, 134 | so they are just ordered in alphabetically. 135 | 136 | So we execute tests in the following order:: 137 | 138 | test_util, test_bar, test_foo, test_app 139 | 140 | 141 | incremental run - test modified 142 | ------------------------------- 143 | 144 | Now let's say that we modify the file ``test_foo``. 145 | We know that all tests were OK before this modification, 146 | so the most relevant tests to be execute are on ``test_foo`` itself. 147 | 148 | Not only ``test_foo`` should be executed first, all other tests 149 | do not need to be executed at all because a change in ``test_foo`` 150 | does not affect any other tests since no other module 151 | depends (imports) ``test_foo``. 152 | 153 | .. graphviz:: 154 | 155 | digraph imports { 156 | rankdir = BT 157 | util [shape=box, color=blue] 158 | bar [shape=box, color=blue] 159 | foo [shape=box, color=blue] 160 | app [shape=box, color=blue] 161 | 162 | "bar" -> "util" 163 | "foo" -> "util" 164 | "app" -> "bar" 165 | "app" -> "foo" 166 | 167 | "test_util" -> "util" 168 | "test_bar" -> "bar" 169 | "test_foo" -> "foo" 170 | "test_app" -> "app" 171 | 172 | test_foo [color=red, fontcolor=red, style=filled, fillcolor=yellow] 173 | } 174 | 175 | 176 | The same behavior can be observed for a change in any other test module 177 | in this example. 178 | Since there are not dependencies between test modules, a change in a test 179 | module will require the execution only of the modified module. 180 | 181 | 182 | incremental run - source modified 183 | --------------------------------- 184 | 185 | Let's check now what happens when ``foo`` is modified. 186 | Looking at the graph it is easy to see which tests are going to 187 | be affected. 188 | 189 | .. graphviz:: 190 | 191 | digraph imports { 192 | rankdir = BT 193 | util [shape=box, color=blue] 194 | bar [shape=box, color=blue] 195 | foo [shape=box, color=blue] 196 | app [shape=box, color=blue] 197 | 198 | "bar" -> "util" 199 | "foo" -> "util" 200 | "app" -> "bar" 201 | "app" -> "foo" [color=red] 202 | 203 | "test_util" -> "util" 204 | "test_bar" -> "bar" 205 | "test_foo" -> "foo" [color=red] 206 | "test_app" -> "app" [color=red] 207 | 208 | foo [fontcolor=red, color=red] 209 | app [color=red] 210 | test_foo [color=red, style=filled, fillcolor=yellow] 211 | test_app [color=red, style=filled, fillcolor=yellow] 212 | } 213 | 214 | The order of test execution is ``test_foo`` then ``test_app``. 215 | Other tests are not executed at all. 216 | 217 | Analyzing the graph is easy to see that a change in ``app`` would cause only 218 | ``test_app`` to be execute. And a change in ``util`` would cause all tests 219 | to be executed. 220 | 221 | 222 | 223 | 224 | pytest-incremental 225 | ================== 226 | 227 | Hopefully by now it is clear that by taking in account the structure of the 228 | code to order the tests, the test-runner can: 229 | 230 | - reduce total execution time for incremental changes 231 | - get faster feedback by executing first the tests which have direct code 232 | under test changes 233 | - easier to debug test failures because of more relevant test ordering 234 | 235 | ``pytest-incremental`` is a *py.test* plugin that analyses the source 236 | code and changes between runs to re-order and de-select tests cases. 237 | 238 | 239 | caveats 240 | ======= 241 | 242 | ``pytest-incremental`` looks for imports recursively to find dependencies (using 243 | AST). But given the very dynamic nature of python there are still some cases 244 | that a module can be affected by a module that are not detected. 245 | 246 | * modules imported not using the *import* statement 247 | * modules not explicitly imported but used at run-time 248 | * monkey-patching. (i.e. A imports X. B monkey-patches X. In this case A might 249 | depend on B) 250 | * others ? 251 | 252 | 253 | cyclic dependencies 254 | ------------------- 255 | 256 | If your project has dependency cycles will negatively affect the efficacy 257 | of *pytest-incremental*. 258 | Dependency cycles are bad not only for *pytest-incremental*, it makes the code 259 | hard to understand and modify. *pytest-incremental* does not try to be smart 260 | handling it, so you better **fix** your code and remove the cycles! 261 | 262 | .. graphviz:: 263 | 264 | digraph imports { 265 | rankdir = BT 266 | util [shape=box, color=blue] 267 | bar [shape=box, color=blue] 268 | foo [shape=box, color=blue] 269 | app [shape=box, color=blue] 270 | 271 | "bar" -> "util" 272 | "foo" -> "util" 273 | "app" -> "bar" 274 | "app" -> "foo" 275 | "util" -> "app" 276 | "bar" -> "app" 277 | } 278 | 279 | When you have cycles any change end up affecting all modules! 280 | -------------------------------------------------------------------------------- /dodo.py: -------------------------------------------------------------------------------- 1 | 2 | import glob 3 | 4 | from doitpy.pyflakes import Pyflakes 5 | from doitpy import docs 6 | from doitpy.package import Package 7 | 8 | 9 | DOIT_CONFIG = {'default_tasks': ['pyflakes',]} 10 | 11 | 12 | def task_pyflakes(): 13 | flakes = Pyflakes() 14 | yield flakes.tasks('*.py') 15 | yield flakes.tasks('tests/*.py') 16 | 17 | 18 | 19 | CODE_FILES = glob.glob("pytest_incremental.py") 20 | TEST_FILES = glob.glob("tests/test_*.py") 21 | 22 | def task_coverage(): 23 | """show coverage for all modules including tests""" 24 | all_files = " ".join(CODE_FILES + TEST_FILES) 25 | return { 26 | 'actions': [ 27 | 'coverage run `which py.test` {}'.format(all_files), 28 | "coverage report --show-missing {}".format(all_files), 29 | ], 30 | 'verbosity': 2, 31 | } 32 | 33 | 34 | def task_docs(): 35 | doc_files = glob.glob('docs/*.rst') + ['README.rst', ] 36 | yield docs.spell(doc_files, 'docs/dictionary.txt') 37 | 38 | 39 | def task_package(): 40 | """create/upload package to pypi""" 41 | pkg = Package() 42 | yield pkg.revision_git() 43 | yield pkg.manifest_git() 44 | yield pkg.sdist() 45 | yield pkg.sdist_upload() 46 | -------------------------------------------------------------------------------- /pytest_incremental.py: -------------------------------------------------------------------------------- 1 | """ 2 | pytest-incremental : an incremental test runner (pytest plugin) 3 | https://pypi.python.org/pypi/pytest-incremental 4 | 5 | The MIT License - see LICENSE file 6 | Copyright (c) 2011-2018 Eduardo Naufel Schettino 7 | """ 8 | 9 | __version__ = (0, 5, 0) 10 | 11 | import os 12 | import json 13 | import functools 14 | from collections import defaultdict 15 | from io import StringIO 16 | 17 | from import_deps import ModuleSet, PyModule 18 | from doit.task import Task, DelayedLoader 19 | from doit.cmd_base import ModuleTaskLoader 20 | from doit.cmd_run import Run 21 | from doit.reporter import ZeroReporter 22 | from doit import doit_cmd 23 | from doit.tools import config_changed 24 | 25 | 26 | ######### Graph implementation 27 | 28 | class GNode(object): 29 | '''represents a node in a direct graph 30 | 31 | Designed to return a list of all nodes in a sub-graph. 32 | The sub-graph from each node is built on demand and cached after built. 33 | ''' 34 | def __init__(self, name): 35 | self.name = name 36 | self.deps = set() # of direct GNode deps 37 | self.implicit_deps = [] 38 | # all_deps are lazily calculated and cached 39 | self._all_deps = None # set of all (recursive) deps names 40 | 41 | def __repr__(self): 42 | return "".format(self.name) 43 | 44 | def add_dep(self, dep): 45 | """add a dependency of self""" 46 | self.deps.add(dep) 47 | 48 | def all_deps(self): 49 | """return set of GNode with all deps from this node (including self)""" 50 | if self._all_deps is not None: 51 | return self._all_deps 52 | todo = set() 53 | done = set() 54 | todo.add(self) 55 | while todo: 56 | node = todo.pop() 57 | if node._all_deps: 58 | done.update(node._all_deps) 59 | else: 60 | todo.update(n for n in node.deps if n not in done) 61 | done.add(node) 62 | done.update(self.implicit_deps) 63 | self._all_deps = done 64 | return done 65 | 66 | 67 | class DepGraph(object): 68 | '''A direct graph used to track python module dependencies''' 69 | NODE_CLASS = GNode 70 | def __init__(self, dep_dict): 71 | """ 72 | :param dep_dict: (dict) key: (str) node name 73 | value: (list - str) direct deps 74 | """ 75 | self.nodes = {} 76 | for name, deps in dep_dict.items(): 77 | node = self._node(name) 78 | for dep in deps: 79 | node.add_dep(self._node(dep)) 80 | 81 | def _node(self, name): 82 | """get or create node""" 83 | node = self.nodes.get(name, None) 84 | if not node: 85 | node = self.nodes[name] = self.NODE_CLASS(name) 86 | return node 87 | 88 | 89 | def write_dot(self, stream): 90 | """write dot file 91 | :param stream: Any object with a `write()` method 92 | """ 93 | stream.write("digraph imports {\nrankdir = BT\n") 94 | for node in sorted(self.nodes.values(), key=lambda x:x.name): 95 | # FIXME add option to include test files or not 96 | #if node.name.startswith('test'): 97 | # continue 98 | node_path = os.path.relpath(node.name) 99 | if node.deps: 100 | for dep in sorted(node.deps, key=lambda x: x.name): 101 | dep_path = os.path.relpath(dep.name) 102 | stream.write('"{}" -> "{}"\n'.format(node_path, dep_path)) 103 | else: 104 | stream.write('"{}"\n'.format(node_path)) 105 | stream.write("}\n") 106 | 107 | 108 | def topsort(self): 109 | '''return list of node names in topological order 110 | 111 | If A has deps [B, C]. We say that A is a target, B and C are sources 112 | ''' 113 | num_src = {} 114 | targets = defaultdict(list) 115 | for target in self.nodes.values(): 116 | num_src[target.name] = len(target.deps) 117 | for source in target.deps: 118 | targets[source.name].append(target.name) 119 | 120 | result = [] 121 | next_level = [n for n in num_src if num_src[n]==0] 122 | 123 | # each iteration is get all nodes from a level 124 | while len(result) != len(self.nodes): 125 | # if there is a cycle all nodes have one or more srcs 126 | if not next_level: 127 | lowest = None 128 | for node_name, srcs in num_src.items(): 129 | if lowest is None or srcs < lowest: 130 | lowest = srcs 131 | next_level = [node_name] 132 | elif srcs == lowest: 133 | next_level.append(node_name) 134 | 135 | # remove elements from num_src so they are not taken account 136 | # when removing a cycle. 137 | for n in next_level: 138 | del num_src[n] 139 | 140 | # sort nodes of this level 141 | result.extend(sorted(next_level)) 142 | 143 | # process nodes preparing for next level 144 | level = next_level 145 | next_level = [] 146 | for n in level: 147 | for target in targets[n]: 148 | try: 149 | num_src[target] -= 1 150 | # cycle might already have remove target 151 | except KeyError: 152 | continue 153 | if num_src[target] == 0: 154 | next_level.append(target) 155 | return result 156 | 157 | 158 | 159 | ######### start doit section 160 | 161 | def gen_after(name, after_task): 162 | '''decorator for function creating a DelayedTask''' 163 | def decorated(fn_creator): 164 | """yield DelayedTasks executed after `after_task` is executed""" 165 | def task_creator(self): 166 | '''create a Task setting its loader''' 167 | creator = functools.partial(fn_creator, self) 168 | loader = DelayedLoader(creator, executed=after_task) 169 | return Task(name, None, loader=loader) 170 | return task_creator 171 | return decorated 172 | 173 | 174 | class PyTasks(object): 175 | """generate doit tasks related to python modules import dependencies 176 | 177 | :ivar ModuleSet py_mods: 178 | :ivar py_files: (list - str) files being watched for changes 179 | :ivar json_file str: name of intermediate file with import info from all 180 | modules 181 | """ 182 | def __init__(self, py_files, json_file='deps.json'): 183 | self.json_file = json_file 184 | self.py_files = list(set(py_files)) 185 | self.py_mods = ModuleSet(self.py_files) 186 | self._graph = None # DepGraph cached on first use 187 | 188 | 189 | def create_graph(self): 190 | """create Graph from json file""" 191 | with open(self.json_file) as fp: 192 | deps = json.load(fp) 193 | return DepGraph(deps) 194 | 195 | 196 | @property 197 | def graph(self): 198 | """cache graph object""" 199 | if self._graph is None: 200 | self._graph = self.create_graph() 201 | return self._graph 202 | 203 | 204 | def action_get_dep(self, module_path): 205 | """action: return list of direct imports from a single py module 206 | 207 | :return dict: single value 'imports', value set of str file paths 208 | """ 209 | mod = self.py_mods.by_path[module_path] 210 | return {'imports': list(str(s) for s in self.py_mods.get_imports(mod))} 211 | 212 | 213 | def action_write_json_deps(self, imports): 214 | """write JSON file with direct imports of all modules""" 215 | result = {k: v['imports'] for k, v in imports.items()} 216 | with open(self.json_file, 'w') as fp: 217 | json.dump(result, fp) 218 | 219 | def gen_deps(self): 220 | """generate doit tasks to find imports 221 | 222 | generated tasks: 223 | * get_dep: => find imported moudules 224 | * dep-json => save import info in a JSON file 225 | """ 226 | watched_modules = str(list(sorted(self.py_files))) 227 | for mod in self.py_files: 228 | # direct dependencies 229 | yield { 230 | 'basename': 'get_dep', 231 | 'name': mod, 232 | 'actions':[(self.action_get_dep, [mod])], 233 | 'file_dep': [mod], 234 | 'uptodate': [config_changed(watched_modules)], 235 | } 236 | 237 | # Create an intermediate json file with import information. 238 | # It is required to create an intermediate file because DelayedTasks 239 | # can not have get_args to use values from other tasks. 240 | yield { 241 | 'basename': 'dep-json', 242 | 'actions': [self.action_write_json_deps], 243 | 'task_dep': ['get_dep'], 244 | 'getargs': {'imports': ('get_dep', None)}, 245 | 'targets': [self.json_file], 246 | 'doc': 'save dep info in {}'.format(self.json_file), 247 | } 248 | 249 | 250 | @staticmethod 251 | def action_print_dependencies(node): 252 | '''print a node's name and its dependencies to SDTOUT''' 253 | node_list = sorted(n.name for n in node.all_deps()) 254 | node_path = os.path.relpath(node.name) 255 | deps_path = (os.path.relpath(p) for p in node_list) 256 | print(' - {}: {}'.format(node_path, ', '.join(deps_path))) 257 | 258 | @gen_after(name='print-deps', after_task='dep-json') 259 | def gen_print_deps(self): 260 | '''create tasks for printing node info to STDOUT''' 261 | for node in self.graph.nodes.values(): 262 | yield { 263 | 'basename': 'print-deps', 264 | 'name': node.name, 265 | 'actions': [(self.action_print_dependencies, [node])], 266 | 'verbosity': 2, 267 | } 268 | 269 | 270 | 271 | @staticmethod 272 | def action_write_dot(file_name, graph): 273 | """write a dot-file(graphviz) with import relation of modules""" 274 | with open(file_name, "w") as fp: 275 | graph.write_dot(fp) 276 | 277 | 278 | @gen_after(name='dep-dot', after_task='dep-json') 279 | def gen_dep_graph_dot(self, dot_file='deps.dot'): 280 | """generate tasks for creating a `dot` graph of module imports""" 281 | yield { 282 | 'basename': 'dep-dot', 283 | 'actions': [(self.action_write_dot, ['deps.dot', self.graph])], 284 | 'file_dep': [self.json_file], 285 | 'targets': [dot_file], 286 | } 287 | 288 | @gen_after(name='dep-image', after_task='dep-json') 289 | def gen_dep_graph_image(self, dot_file='deps.dot', img_file='deps.svg'): 290 | # generate SVG with bottom-up tree 291 | dot_cmd = 'dot -Tsvg ' 292 | yield { 293 | 'basename': 'dep-image', 294 | 'actions': [dot_cmd + " -o %(targets)s %(dependencies)s"], 295 | 'file_dep': [dot_file], 296 | 'targets': [img_file], 297 | } 298 | 299 | 300 | 301 | 302 | class IncrementalTasks(PyTasks): 303 | """Manage creation of all tasks for pytest-incremental plugin""" 304 | 305 | def __init__(self, pyfiles, test_files=None, **kwargs): 306 | PyTasks.__init__(self, pyfiles, **kwargs) 307 | self.test_files = test_files 308 | 309 | def create_graph(self): 310 | """overwrite to add implicit dep to conftest file""" 311 | graph = super(IncrementalTasks, self).create_graph() 312 | conftest = [mod for mod in graph.nodes.keys() 313 | if mod.endswith('conftest.py')] 314 | for conf in conftest: 315 | conftest_node = graph.nodes[conf] 316 | base_dir = os.path.dirname(conf) 317 | for path, node in graph.nodes.items(): 318 | if path.startswith(base_dir) and path != conf: 319 | node.implicit_deps.append(conftest_node) 320 | return graph 321 | 322 | def check_success(self): 323 | """check if task should succeed based on GLOBAL parameter""" 324 | return doit_cmd.get_var('success', False) 325 | 326 | @gen_after(name='outdated', after_task='dep-json') 327 | def gen_outdated(self): 328 | """generate tasks used by py.test to keep-track of successful results""" 329 | nodes = self.graph.nodes 330 | for test in self.test_files: 331 | yield { 332 | 'basename': 'outdated', 333 | 'name': test, 334 | 'actions': [self.check_success], 335 | 'file_dep': [n.name for n in nodes[test].all_deps()], 336 | 'verbosity': 0, 337 | } 338 | 339 | def create_doit_tasks(self): 340 | '''create all tasks used by the incremental plugin 341 | This method is a hook used by doit 342 | ''' 343 | yield self.gen_deps() 344 | yield self.gen_print_deps() 345 | yield self.gen_dep_graph_dot() 346 | yield self.gen_dep_graph_image() 347 | yield self.gen_outdated() 348 | 349 | 350 | 351 | class OutdatedReporter(ZeroReporter): 352 | """A doit reporter specialized to return list of outdated tasks""" 353 | def __init__(self, outstream, options): 354 | self.outdated = [] 355 | self.outstream = outstream 356 | 357 | def execute_task(self, task): 358 | if task.name.startswith('outdated:'): 359 | self.outdated.append(task.name.split(':', 1)[1]) 360 | 361 | def add_failure(self, task, exception): 362 | if task.name.startswith('outdated'): 363 | return 364 | raise pytest.UsageError("%s:%s" % (task.name, exception)) 365 | 366 | def runtime_error(self, msg): 367 | raise Exception(msg) 368 | 369 | def complete_run(self): 370 | outdated_info = json.dumps(self.outdated) 371 | self.outstream.write(outdated_info) 372 | 373 | 374 | ##################### end doit section 375 | 376 | 377 | class IncrementalControl(object): 378 | '''control which modules need to execute tests 379 | 380 | :cvar str DB_FILE: file name used as doit db file 381 | :ivar py_files: (list - str) relative path of test and code under test 382 | ''' 383 | DB_FILE = '.pytest-incremental' 384 | 385 | def __init__(self, pkg_folders): 386 | assert isinstance(pkg_folders, list) 387 | self.test_files = None 388 | self.py_files = [] 389 | for pkg in pkg_folders: 390 | self.py_files.extend(self._get_pkg_modules(pkg)) 391 | 392 | def _get_pkg_modules(self, pkg_name, get_sub_folders=True): 393 | """get all package modules recursively 394 | :param pkg_name: (str) path to search for python modules 395 | :param get_sub_folders: (bool) search sub-folders even if they are 396 | not python packages 397 | """ 398 | pkg_glob = os.path.join(pkg_name, "*.py") 399 | this_modules = glob.glob(pkg_glob) 400 | for dirname, dirnames, filenames in os.walk(pkg_name): 401 | for subdirname in dirnames: 402 | sub_path = os.path.join(dirname, subdirname) 403 | if get_sub_folders or PyModule.is_pkg(sub_path): 404 | this_modules.extend(self._get_pkg_modules(sub_path)) 405 | return this_modules 406 | 407 | def _run_doit(self, sel_tasks, reporter=None, doit_vars=None): 408 | """load this file as dodo file to collect tasks""" 409 | inc = IncrementalTasks(self.py_files, test_files=list(self.test_files)) 410 | output = StringIO() 411 | config = { 412 | 'dep_file': self.DB_FILE, 413 | 'continue': True, 414 | 'outfile': output, 415 | } 416 | if reporter: 417 | config['reporter'] = reporter 418 | 419 | ctx = { 420 | 'tasks_generator': inc, 421 | 'DOIT_CONFIG': config, 422 | } 423 | doit_cmd.reset_vars() 424 | if doit_vars: 425 | for key, value in doit_vars.items(): 426 | doit_cmd.set_var(key, value) 427 | loader = ModuleTaskLoader(ctx) 428 | cmd = Run(task_loader=loader) 429 | cmd.parse_execute(sel_tasks) 430 | output.seek(0) 431 | return inc.graph, output.read() 432 | 433 | 434 | def get_outdated(self): 435 | """run doit to find out which test files are "outdated" 436 | A test file is outdated if there was a change in the content in any 437 | import (direct or indirect) since last succesful execution 438 | 439 | :return set(str): list of outdated files 440 | """ 441 | outdated_tasks = ['outdated'] 442 | graph, output_str = self._run_doit(outdated_tasks, 443 | reporter=OutdatedReporter) 444 | outdated_list = json.loads(output_str) 445 | # dict of outdated with position 446 | outdated = {} 447 | order = {p:i for i,p in enumerate(graph.topsort())} 448 | for test in outdated_list: 449 | outdated[test] = order[test] 450 | return outdated 451 | 452 | def save_success(self, success): 453 | """mark doit test tasks as sucessful""" 454 | tasks = ['dep-json'] 455 | for path in success: 456 | tasks.append("outdated:%s" % path) 457 | self._run_doit(tasks, doit_vars={'success':True}) 458 | 459 | 460 | def print_deps(self): 461 | """print list of all python modules being tracked and its dependencies""" 462 | self._run_doit(['print-deps']) 463 | 464 | def create_dot_graph(self, graph_type='dot'): 465 | """create a graph of imports in dot format 466 | """ 467 | tasks = ['dep-dot', 'dep-image'] if graph_type=='image' else ['dep-dot'] 468 | self._run_doit(tasks) 469 | 470 | 471 | 472 | ### py.test integration 473 | 474 | import glob 475 | import pytest 476 | 477 | def pytest_addoption(parser): 478 | '''py.test hook: register argparse-style options and config values''' 479 | group = parser.getgroup("incremental", "incremental testing") 480 | group.addoption( 481 | '--inc', action="store_true", 482 | dest="incremental", default=False, 483 | help="execute only outdated tests (based on modified files)") 484 | group.addoption( 485 | '--inc-path', action="append", 486 | dest="watch_path", default=[], 487 | help="file path of a package. watch for file changes in packages (multi-allowed)") 488 | group.addoption( 489 | '--inc-outdated', action="store_true", 490 | dest="list_outdated", default=False, 491 | help="print list of outdated test files") 492 | group.addoption( 493 | '--inc-deps', action="store_true", 494 | dest="list_dependencies", default=False, 495 | help="print list of python modules being tracked and its dependencies") 496 | group.addoption( 497 | '--inc-graph', action="store_const", const='dot', 498 | dest="graph_dependencies", default=None, 499 | help="create graph file of dependencies in dot format 'deps.dot'") 500 | group.addoption( 501 | '--inc-graph-image', action="store_const", const='image', 502 | dest="graph_dependencies", default=None, 503 | help="create graph file of dependencies in SVG format 'deps.svg'") 504 | 505 | 506 | def pytest_configure(config): 507 | '''Register incremental plugin only if any of its options is specified 508 | 509 | py.test hook: called after parsing cmd optins and loading plugins. 510 | ''' 511 | opt = config.option 512 | if any((opt.incremental, opt.list_outdated, opt.list_dependencies, 513 | opt.graph_dependencies)): 514 | config._incremental = IncrementalPlugin() 515 | config.pluginmanager.register(config._incremental) 516 | 517 | 518 | def pytest_unconfigure(config): 519 | '''py.test hook: called before test process is exited.''' 520 | incremental_plugin = getattr(config, '_incremental', None) 521 | if incremental_plugin: 522 | del config._incremental 523 | config.pluginmanager.unregister(incremental_plugin) 524 | 525 | 526 | 527 | class IncrementalPlugin(object): 528 | """pytest-incremental plugin class 529 | 530 | how it works 531 | ============= 532 | 533 | * pytest_sessionstart: check configuration, 534 | find python files (if pkg specified) 535 | * pytest_collection_modifyitems (get_outdated): run doit and remove 536 | up-to-date tests from test items 537 | * pytest_runtestloop: print info on up-to-date (not excuted) on terminal 538 | * pytest_runtest_logreport: collect result from individual tests 539 | * pytest_sessionfinish (save_success): save successful tasks in doit db 540 | """ 541 | 542 | def __init__(self): 543 | # command line options 544 | self.list_outdated = False 545 | self.list_dependencies = False 546 | self.graph_dependencies = None 547 | self.run = None 548 | 549 | # IncrementalControl, set on sessionstart 550 | self.control = None 551 | 552 | # test information gathering during collect phase 553 | self.uptodate_paths = set() # test paths that are up-to-date 554 | self.outofdate = defaultdict(list) # path: list of nodeid 555 | self.test_files = None # list of collected test files 556 | 557 | # sets of nodeid's set on logreport 558 | self.passed = set() 559 | self.failed = set() 560 | 561 | 562 | def pytest_sessionstart(self, session): 563 | """initialization and sanity checking""" 564 | if session.config.pluginmanager.hasplugin('dsession'): 565 | msg = 'Plugin incremental is not compatible with plugin xdist.' 566 | raise pytest.UsageError(msg) 567 | 568 | opts = session.config.option 569 | self.list_outdated = opts.list_outdated 570 | self.list_dependencies = opts.list_dependencies 571 | self.graph_dependencies = opts.graph_dependencies 572 | self.run = not any((self.list_outdated, 573 | self.list_dependencies, 574 | self.graph_dependencies)) 575 | 576 | # pkg_folders to watch can never be empty, if not specified use CWD 577 | pkg_folders = [os.path.abspath(p) for p in 578 | session.config.option.watch_path] 579 | if not pkg_folders: 580 | pkg_folders = [os.getcwd()] 581 | 582 | self.control = IncrementalControl(pkg_folders) 583 | 584 | 585 | def pytest_collection_modifyitems(self, session, config, items): 586 | """py.test hook: reset `items` removing tests from up-to-date modules 587 | 588 | side-effects: 589 | - set self.test_files with all test modules 590 | - set self.uptodate_paths with all test files that wont be executed 591 | - set the param `items` with items to be executed 592 | """ 593 | # save reference of all found test modules 594 | test_files = set((str(i.fspath) for i in items)) 595 | self.test_files = test_files 596 | self.control.test_files = test_files 597 | 598 | # list dependencies doesnt care about current state of outdated 599 | if self.list_dependencies or self.graph_dependencies: 600 | return 601 | 602 | # execute doit to figure out which test modules are outdated 603 | # dict test_moodule path: relative order position 604 | outdated = self.control.get_outdated() 605 | 606 | # split items into 2 groups to be executed or not 607 | item_by_mod = defaultdict(list) 608 | deselected = [] 609 | for colitem in items: 610 | path = str(colitem.fspath) 611 | if path in outdated: 612 | self.outofdate[path].append(colitem.nodeid) 613 | item_by_mod[path].append(colitem) 614 | else: 615 | self.uptodate_paths.add(path) 616 | deselected.append(colitem) 617 | 618 | selected = [] 619 | for path, _ in sorted(outdated.items(), key=lambda x: x[1]): 620 | selected.extend(item_by_mod[path]) 621 | items[:] = selected 622 | 623 | # include number of tests deselected in report footer 624 | if deselected: 625 | config.hook.pytest_deselected(items=deselected) 626 | 627 | 628 | 629 | 630 | # FIXME should use termial to print stuff 631 | def pytest_runtestloop(self): 632 | """print up-to-date tests info before running tests or... 633 | """ 634 | # print info commands 635 | if not self.run: 636 | if self.list_outdated: 637 | self.print_outdated() 638 | elif self.list_dependencies: 639 | self.control.print_deps() 640 | elif self.graph_dependencies: 641 | self.control.create_dot_graph(self.graph_dependencies) 642 | print('Graph dot file written in deps.dot') 643 | if self.graph_dependencies == 'image': 644 | print('Graph image file written in deps.svg') 645 | return 0 # dont execute tests 646 | 647 | self.print_uptodate_test_files() 648 | 649 | 650 | def print_uptodate_test_files(self): 651 | """print info on up-to-date tests""" 652 | if self.uptodate_paths: 653 | print() 654 | rel_paths = (os.path.relpath(p) for p in self.uptodate_paths) 655 | for test_file in sorted(rel_paths): 656 | print("{} [up-to-date]".format(test_file)) 657 | 658 | def print_outdated(self): 659 | """print list of outdated test files""" 660 | outdated = [] 661 | for test in self.test_files: 662 | if test not in self.uptodate_paths: 663 | outdated.append(test) 664 | 665 | print() 666 | if outdated: 667 | print("List of outdated test files:") 668 | rel_paths = (os.path.relpath(p) for p in outdated) 669 | for test in sorted(rel_paths): 670 | print(test) 671 | else: 672 | print("All test files are up to date") 673 | 674 | 675 | def pytest_runtest_logreport(self, report): 676 | """save success and failures result so we can decide which files 677 | should be marked as successful in doit 678 | 679 | py.test hook: called on setup/call/teardown 680 | """ 681 | if report.failed: 682 | self.failed.add(report.nodeid) 683 | else: 684 | self.passed.add(report.nodeid) 685 | 686 | def pytest_sessionfinish(self, session): 687 | """save success in doit""" 688 | if not self.run: 689 | return 690 | 691 | # if some tests were deselected by a keyword we cant assure all tests 692 | # passed 693 | if getattr(session.config.option, 'keyword', None): 694 | print("\nWARNING: incremental not saving results because -k was used") 695 | return 696 | 697 | successful = [] 698 | for path in self.test_files: 699 | for nodeid in self.outofdate[path]: 700 | if nodeid in self.failed: 701 | break 702 | # check all items were really executed 703 | # when user hits Ctrl-C sessionfinish still gets called 704 | if nodeid not in self.passed: 705 | break 706 | else: 707 | successful.append(path) 708 | 709 | self.control.save_success(os.path.abspath(f) for f in successful) 710 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | 3 | from distutils.core import setup 4 | 5 | 6 | with open(os.path.join(os.path.dirname(__file__), 'README.rst'), 'r') as fp: 7 | README_TEXT = fp.read() 8 | 9 | 10 | setup(name = 'pytest-incremental', 11 | description = 'an incremental test runner (pytest plugin)', 12 | version = '0.6.0', 13 | license = 'MIT', 14 | author = 'Eduardo Naufel Schettino', 15 | author_email = 'schettino72@gmail.com', 16 | url = 'https://pytest-incremental.readthedocs.io', 17 | classifiers = ['Development Status :: 5 - Production/Stable', 18 | 'Environment :: Console', 19 | 'Intended Audience :: Developers', 20 | 'License :: OSI Approved :: MIT License', 21 | 'Natural Language :: English', 22 | 'Operating System :: OS Independent', 23 | 'Operating System :: POSIX', 24 | 'Programming Language :: Python :: 3', 25 | 'Programming Language :: Python :: 3.6', 26 | 'Programming Language :: Python :: 3.7', 27 | 'Programming Language :: Python :: 3.8', 28 | 'Programming Language :: Python :: 3.9', 29 | 'Topic :: Software Development :: Testing', 30 | ], 31 | py_modules = ['pytest_incremental'], 32 | install_requires = [ 33 | 'import_deps >= 0.1.0', 34 | 'doit >= 0.31.1', 35 | 'pytest >= 6.0', 36 | ], 37 | entry_points = { 38 | 'pytest11': ['pytest_incremental = pytest_incremental'], 39 | }, 40 | long_description = README_TEXT, 41 | keywords = "pytest unittest test", 42 | project_urls = { 43 | 'Documentation': 'https://pytest-incremental.readthedocs.io', 44 | 'Source': 'https://github.com/pytest-dev/pytest-incremental', 45 | 'Tracker': 'https://github.com/pytest-dev/pytest-incremental/issues', 46 | }, 47 | ) 48 | 49 | -------------------------------------------------------------------------------- /tests/sample-inc/dodo.py: -------------------------------------------------------------------------------- 1 | from glob import glob 2 | 3 | from pytest_incremental import IncrementalTasks 4 | 5 | def task_x(): 6 | src_files = glob("*.py") 7 | test_files = glob("tt/*.py") 8 | inc = IncrementalTasks(src_files + test_files, test_files=test_files) 9 | yield inc.create_doit_tasks() 10 | 11 | 12 | -------------------------------------------------------------------------------- /tests/sample-inc/mod1.py: -------------------------------------------------------------------------------- 1 | def sum(a, b): 2 | return a + b 3 | -------------------------------------------------------------------------------- /tests/sample-inc/mod2.py: -------------------------------------------------------------------------------- 1 | import mod1 2 | 3 | def plus5(a): 4 | return mod1.sum(5 + a) 5 | -------------------------------------------------------------------------------- /tests/sample-inc/tt/conftest.py: -------------------------------------------------------------------------------- 1 | xxx_in_use = True 2 | -------------------------------------------------------------------------------- /tests/sample-inc/tt/tt_mod1.py: -------------------------------------------------------------------------------- 1 | import mod1 2 | 3 | def test_sum(): 4 | assert 7 == mod1.sum(3,4) 5 | -------------------------------------------------------------------------------- /tests/sample-inc/tt/tt_mod2.py: -------------------------------------------------------------------------------- 1 | import mod2 2 | 3 | def test_plus5(): 4 | assert 8 == mod2.plus5(3) 5 | -------------------------------------------------------------------------------- /tests/test_functional.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | pytest_plugins = 'pytester', 'pytest_incremental' 5 | 6 | 7 | def get_results(recorder): 8 | '''filter records to get only call results''' 9 | results = {} 10 | for result in recorder.getreports(): 11 | when = getattr(result, 'when', None) 12 | if when is None: 13 | continue 14 | test_name = result.nodeid.split('::')[-1] 15 | results[test_name, when] = result.outcome 16 | return results 17 | 18 | def count_calls(results): 19 | """count number of "call" nodeid's in results""" 20 | return len(list(r for r in results.keys() if r[1]=='call')) 21 | 22 | 23 | 24 | TEST_SAMPLE = """ 25 | def test_foo(): 26 | assert True 27 | def test_bar(): 28 | assert True 29 | """ 30 | 31 | 32 | def test_list_deps(testdir, capsys): 33 | test = testdir.makepyfile(TEST_SAMPLE) 34 | args = ['--inc-deps', test] 35 | testdir.inline_run(*args) 36 | out = capsys.readouterr()[0].splitlines() 37 | assert ' - test_list_deps.py: test_list_deps.py' in out 38 | 39 | 40 | def test_list_outdated(testdir, capsys): 41 | test = testdir.makepyfile(TEST_SAMPLE) 42 | args = ['--inc-outdated', test] 43 | testdir.inline_run(*args) 44 | out = list(reversed(capsys.readouterr()[0].splitlines())) 45 | while(out): 46 | line = out.pop() 47 | if line == 'List of outdated test files:': 48 | outdated_list = out.pop() 49 | assert 'test_list_outdated.py' in outdated_list 50 | break 51 | else: # pragma: no cover 52 | assert False, 'outdated list not found' 53 | 54 | 55 | def test_list_outdated_none(testdir, capsys): 56 | test = testdir.makepyfile(TEST_SAMPLE) 57 | testdir.inline_run('--inc', test) # run so tests are not outdated 58 | testdir.inline_run('--inc-outdated', test) 59 | out = capsys.readouterr()[0].splitlines() 60 | assert 'All test files are up to date' in out 61 | 62 | 63 | def test_graph(testdir, capsys): 64 | test = testdir.makepyfile(TEST_SAMPLE) 65 | args = ['-v', '--inc-graph', test] 66 | testdir.inline_run(*args) 67 | out = capsys.readouterr()[0].splitlines() 68 | assert 'Graph dot file written in deps.dot' in out 69 | 70 | 71 | def test_fail_always_reexecute_test(testdir): 72 | TEST_FAIL = """ 73 | def foo(): 74 | return 'foo' 75 | def test_foo(): 76 | assert 'bar' == foo() 77 | """ 78 | 79 | test = testdir.makepyfile(TEST_FAIL) 80 | args = ['--inc', test] 81 | 82 | # first time failed 83 | rec = testdir.inline_run(*args) 84 | results = get_results(rec) 85 | assert results['test_foo', 'call'] == 'failed' 86 | 87 | # second time re-executed 88 | rec2 = testdir.inline_run(*args) 89 | results2 = get_results(rec2) 90 | assert results2['test_foo', 'call'] == 'failed' 91 | 92 | 93 | 94 | def test_ok_reexecute_only_if_changed(testdir, capsys): 95 | TEST_OK = """ 96 | def foo(): 97 | return 'foo' 98 | def test_foo(): 99 | assert 'foo' == foo() 100 | """ 101 | 102 | TEST_OK_2 = """ 103 | def foo(): 104 | return 'foo' 105 | def test_foo(): 106 | assert 'foo' == foo() 107 | def test_bar(): 108 | assert True 109 | """ 110 | # first time 111 | test = testdir.makepyfile(TEST_OK) 112 | args = ['--inc', str(test)] 113 | 114 | # first time passed 115 | rec = testdir.inline_run(*args) 116 | results = get_results(rec) 117 | assert results['test_foo', 'call'] == 'passed' 118 | assert count_calls(results) == 1 119 | 120 | # second time not executed because up-to-date 121 | rec2 = testdir.inline_run(*args) 122 | results2 = get_results(rec2) 123 | assert count_calls(results2) == 0 124 | out = capsys.readouterr()[0].splitlines() 125 | assert '== 1 deselected' in out[-1] 126 | 127 | # TODO remove when py.test 2.8 is released 128 | if 'test_ok_reexecute_only_if_changed' in sys.modules: # pragma: no cover 129 | del sys.modules['test_ok_reexecute_only_if_changed'] 130 | 131 | # change module 132 | test.write(TEST_OK_2) 133 | # re-execute tests 134 | rec3 = testdir.inline_run(*args) 135 | results3 = get_results(rec3) 136 | print(rec3.getreports(), results3) 137 | assert results3['test_foo', 'call'] == 'passed' 138 | assert results3['test_bar', 'call'] == 'passed' 139 | assert count_calls(results3) == 2 140 | 141 | 142 | 143 | 144 | def test_skip_same_behaviour_as_passed(testdir): 145 | TEST_SKIP = """ 146 | import pytest 147 | 148 | @pytest.mark.skipif("True") 149 | def test_my_skip(): 150 | assert False # not executed 151 | 152 | @pytest.mark.xfail 153 | def test_my_fail(): 154 | assert False 155 | """ 156 | # first time 157 | test = testdir.makepyfile(TEST_SKIP) 158 | args = ['--inc', test] 159 | 160 | rec = testdir.inline_run(*args) 161 | results = get_results(rec) 162 | assert results['test_my_skip', 'setup'] == 'skipped' 163 | assert results['test_my_fail', 'call'] == 'skipped' 164 | 165 | # second time not executed because up-to-date 166 | rec2 = testdir.inline_run(*args) 167 | results2 = get_results(rec2) 168 | assert count_calls(results2) == 0 169 | 170 | 171 | def test_keyword_dont_save_success(testdir, capsys): 172 | test = testdir.makepyfile(TEST_SAMPLE) 173 | testdir.inline_run('--inc', '-k', 'foo', test) 174 | out = capsys.readouterr()[0].splitlines() 175 | assert 'WARNING: incremental not saving results because -k was used' in out 176 | 177 | rec = testdir.inline_run('--inc', test) 178 | results = get_results(rec) 179 | assert results['test_foo', 'call'] == 'passed' 180 | assert results['test_bar', 'call'] == 'passed' 181 | 182 | 183 | def test_xdist_not_supported(testdir, capsys): 184 | from _pytest.main import ExitCode 185 | test = testdir.makepyfile(TEST_SAMPLE) 186 | got = testdir.inline_run('--inc', '-n', '2', test) 187 | assert got.ret == ExitCode.USAGE_ERROR 188 | err = capsys.readouterr()[1].splitlines() 189 | msg = 'ERROR: Plugin incremental is not compatible with plugin xdist.' 190 | assert msg in err 191 | 192 | 193 | def test_inc_path(testdir, capsys): 194 | sub = testdir.mkdir('sub') 195 | base = os.path.relpath(str(sub)) # pass a relative path 196 | with open('sub/test_x.py', 'w') as fp: 197 | fp.write(TEST_SAMPLE) 198 | args = ['-v', '--inc', '--inc-path', base, base] 199 | testdir.inline_run(*args) 200 | out = capsys.readouterr()[0].splitlines() 201 | assert any( 202 | line.startswith('sub/test_x.py::test_foo PASSED') 203 | for line in out 204 | ) 205 | -------------------------------------------------------------------------------- /tests/test_graph.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from pytest_incremental import StringIO, GNode, DepGraph 4 | 5 | class Test_GNode(object): 6 | def test_repr(self): 7 | node = GNode('foo') 8 | assert "" == repr(node) 9 | 10 | def test_add_dep(self): 11 | node = GNode('foo') 12 | dep1 = GNode('bar1') 13 | dep2 = GNode('bar2') 14 | node.add_dep(dep1) 15 | node.add_dep(dep2) 16 | assert 2 == len(node.deps) 17 | assert dep1 in node.deps 18 | assert dep2 in node.deps 19 | 20 | def test_all_deps(self): 21 | node = GNode('foo') 22 | dep1 = GNode('bar1') 23 | dep2 = GNode('bar2') 24 | dep3 = GNode('bar3') 25 | node.add_dep(dep1) 26 | dep1.add_dep(dep2) 27 | dep2.add_dep(dep3) 28 | all_deps = node.all_deps() 29 | assert 4 == len(all_deps) 30 | assert node in all_deps 31 | assert dep1 in all_deps 32 | assert dep2 in all_deps 33 | assert dep3 in all_deps 34 | 35 | 36 | def test_all_deps_include_implicit_dep(self): 37 | node = GNode('foo') 38 | dep1 = GNode('bar') 39 | conf = GNode('conf') 40 | node.add_dep(dep1) 41 | node.implicit_deps.append(conf) 42 | all_deps = node.all_deps() 43 | assert 3 == len(all_deps) 44 | assert node in all_deps 45 | assert dep1 in all_deps 46 | assert conf in all_deps 47 | 48 | def test_recursive1(self): 49 | node = GNode('foo') 50 | dep1 = GNode('bar1') 51 | dep2 = GNode('bar2') 52 | dep3 = GNode('bar3') 53 | dep4 = GNode('bar4') 54 | node.add_dep(dep1) 55 | dep1.add_dep(dep2) 56 | dep2.add_dep(dep3) 57 | dep3.add_dep(dep1) 58 | dep3.add_dep(dep4) 59 | 60 | cycle_deps = set([dep1, dep2, dep3, dep4]) 61 | all_deps = cycle_deps.union(set([node])) 62 | assert all_deps == node.all_deps() 63 | assert cycle_deps == dep1.all_deps() 64 | assert cycle_deps == dep2.all_deps() 65 | assert cycle_deps == dep3.all_deps() 66 | assert set([dep4]) == dep4.all_deps() 67 | 68 | 69 | def test_recursive2(self): 70 | n1 = GNode('bar1') 71 | n2 = GNode('bar2') 72 | n3 = GNode('bar3') 73 | n4 = GNode('bar4') 74 | n5 = GNode('bar5') 75 | n1.add_dep(n2) 76 | n2.add_dep(n3) 77 | n3.add_dep(n4) 78 | n4.add_dep(n5) 79 | n5.add_dep(n1) 80 | n4.add_dep(n2) 81 | n2.add_dep(n4) 82 | 83 | cycle_deps = set([n1, n2, n3, n4, n5]) 84 | assert n1.all_deps() == cycle_deps 85 | assert n2.all_deps() == cycle_deps 86 | assert n3.all_deps() == cycle_deps 87 | assert n4.all_deps() == cycle_deps 88 | 89 | def test_recursive3(self): 90 | n1 = GNode('bar1') 91 | n2 = GNode('bar2') 92 | n3 = GNode('bar3') 93 | n1.add_dep(n2) 94 | n2.add_dep(n3) 95 | n3.add_dep(n2) 96 | 97 | assert n3.all_deps() == set([n2, n3]) 98 | assert n2.all_deps() == set([n2, n3]) 99 | assert n1.all_deps() == set([n1, n2, n3]) 100 | 101 | def test_all_deps_cached(self): 102 | node = GNode('foo') 103 | dep1 = GNode('bar') 104 | dep2 = GNode('baz') 105 | node.add_dep(dep1) 106 | all_deps = node.all_deps() 107 | assert 2 == len(all_deps) 108 | assert node in all_deps 109 | assert dep1 in all_deps 110 | # added dep not seen because of cache 111 | node.add_dep(dep2) 112 | assert 2 == len(node.all_deps()) 113 | 114 | 115 | 116 | class Test_DepGraph(object): 117 | graph = DepGraph({ 118 | 'a': ['b', 'c'], 119 | 'b': ['d'], 120 | 'd': ['c', 'e'], 121 | }) 122 | 123 | def test_nodes(self): 124 | b_deps = [n.name for n in self.graph.nodes['b'].all_deps()] 125 | assert set(b_deps) == set(['c', 'e', 'b', 'd']) 126 | 127 | def test_dot(self): 128 | output = StringIO() 129 | self.graph.write_dot(output) 130 | lines = output.getvalue().splitlines() 131 | assert '"a" -> "c"' in lines 132 | assert '"a" -> "b"' in lines 133 | assert '"b" -> "d"' in lines 134 | assert '"d" -> "c"' in lines 135 | assert '"d" -> "e"' in lines 136 | 137 | 138 | def test_topsort_simple(self): 139 | graph = DepGraph({ 140 | 'a': ['c'], 141 | 'b': [], 142 | 'c': ['b'], 143 | }) 144 | assert ['b', 'c', 'a'] == graph.topsort() 145 | 146 | 147 | def test_topsort_same_level(self): 148 | graph = DepGraph({ 149 | 'a': ['b', 'd', 'e', 'c'], 150 | 'e': [], 151 | 'c': [], 152 | 'd': [], 153 | 'b': [], 154 | }) 155 | # same level sort by name 156 | assert ['b', 'c', 'd', 'e', 'a'] == graph.topsort() 157 | 158 | def test_topsort_cycle(self): 159 | graph = DepGraph({ 160 | 'a': ['b'], 161 | 'c': ['a'], 162 | 'b': ['c'], 163 | }) 164 | assert ['a', 'b', 'c'] == graph.topsort() 165 | 166 | def test_topsort_double_cycle(self): 167 | graph = DepGraph({ 168 | 'a': ['b', 'c'], 169 | 'c': ['a', 'b'], 170 | 'b': ['c', 'a'], 171 | }) 172 | assert ['a', 'b', 'c'] == graph.topsort() 173 | 174 | def test_topsort_cycle_plus(self): 175 | graph = DepGraph({ 176 | 'a': ['b', 'c'], 177 | 'c': ['a', 'b'], 178 | 'b': ['c', 'a', 'd'], 179 | 'd': [] 180 | }) 181 | assert ['d', 'a', 'b', 'c'] == graph.topsort() 182 | -------------------------------------------------------------------------------- /tests/test_tasks.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | 4 | from io import StringIO 5 | import pytest 6 | from doit.cmd_run import Run 7 | from doit.cmd_base import DodoTaskLoader 8 | 9 | from pytest_incremental import IncrementalTasks 10 | from pytest_incremental import IncrementalControl, OutdatedReporter 11 | 12 | 13 | #### fixture for "doit.db". create/remove for every test 14 | # (copied from doit/tests/conftest.py) 15 | def remove_db(filename): 16 | """remove db file from anydbm""" 17 | # dbm on some systems add '.db' on others add ('.dir', '.pag') 18 | extensions = ['', #dbhash #gdbm 19 | '.bak', #dumbdb 20 | '.dat', #dumbdb 21 | '.dir', #dumbdb #dbm2 22 | '.db', #dbm1 23 | '.pag', #dbm2 24 | ] 25 | for ext in extensions: 26 | if os.path.exists(filename + ext): 27 | os.remove(filename + ext) 28 | 29 | @pytest.fixture 30 | def depfile_name(request): 31 | # copied from tempdir plugin 32 | name = request._pyfuncitem.name 33 | name = re.sub(r"[\W]", "_", name) 34 | my_tmpdir = request.config._tmpdirhandler.mktemp(name, numbered=True) 35 | depfile_name = (os.path.join(my_tmpdir.strpath, "testdb")) 36 | 37 | def remove_depfile(): 38 | remove_db(depfile_name) 39 | request.addfinalizer(remove_depfile) 40 | 41 | return depfile_name 42 | ########################################## 43 | 44 | SAMPLE_DIR = os.path.join(os.path.dirname(__file__), 'sample-inc') 45 | 46 | @pytest.fixture 47 | def cmd_run(request, depfile_name): 48 | output = StringIO() 49 | cmd = Run(task_loader=DodoTaskLoader()) 50 | params, _ = cmd.cmdparser.parse([]) 51 | params['outfile'] = output 52 | params['dodoFile'] = os.path.join(SAMPLE_DIR, 'dodo.py') 53 | params['cwdPath'] = SAMPLE_DIR 54 | params['dep_file'] = depfile_name 55 | cmd.params = params # hack to make params available from fixture 56 | return cmd 57 | 58 | @pytest.fixture 59 | def rm_generated_deps(request): 60 | """remove deps.* files generated from running tasks on sample-inc folder""" 61 | def remove(): 62 | for path in ('deps.json', 'deps.dot', 'deps.svg'): 63 | try: 64 | os.remove(os.path.join(SAMPLE_DIR, path)) 65 | except OSError: 66 | pass 67 | request.addfinalizer(remove) 68 | 69 | 70 | class TestTasks(object): 71 | def test_print_deps(self, cmd_run, rm_generated_deps, capsys): 72 | # all deps in all levels 73 | cmd_run.execute(cmd_run.params, ['print-deps']) 74 | got = cmd_run.outstream.getvalue().splitlines() 75 | # FIXME there is some bad interaction of doit and py.test 76 | # leaving the output from tasks directly into stdout but not on 77 | # specified outstream 78 | out = capsys.readouterr()[0].splitlines() 79 | assert '. dep-json' in got 80 | assert ' - mod2.py: mod1.py, mod2.py' in out 81 | mod2_deps = 'mod1.py, mod2.py, tt/conftest.py, tt/tt_mod2.py' 82 | assert ' - tt/tt_mod2.py: ' + mod2_deps in out 83 | 84 | def test_dot_graph(self, cmd_run, rm_generated_deps): 85 | cmd_run.execute(cmd_run.params, ['dep-dot']) 86 | got = cmd_run.outstream.getvalue().splitlines() 87 | assert '. dep-dot' in got 88 | dot = open(os.path.join(SAMPLE_DIR, 'deps.dot')).read() 89 | got = dot.splitlines() 90 | assert '''"mod1.py"''' in got 91 | assert '''"mod2.py" -> "mod1.py"''' in got 92 | assert '''"tt/tt_mod1.py" -> "mod1.py"''' in got 93 | assert '''"tt/tt_mod2.py" -> "mod2.py"''' in got 94 | 95 | def test_img_graph(self, cmd_run, rm_generated_deps): 96 | # dumb test just check task is created 97 | IncrementalTasks(['xxx.py'], ['yyy.py']) 98 | cmd_run.execute(cmd_run.params, ['dep-image']) 99 | got = cmd_run.outstream.getvalue().splitlines() 100 | assert '. dep-image' in got 101 | 102 | 103 | 104 | class FakeTask(object): 105 | def __init__(self, name): 106 | self.name = name 107 | class TestOutdatedRerporter(object): 108 | def test_output(self): 109 | output = StringIO() 110 | rep = OutdatedReporter(output, None) 111 | rep.execute_task(FakeTask('foo')) 112 | rep.execute_task(FakeTask('outdated:xxx')) 113 | rep.execute_task(FakeTask('NOToutdated:abc')) 114 | rep.execute_task(FakeTask('outdated:yyy')) 115 | rep.complete_run() 116 | assert output.getvalue() == '["xxx", "yyy"]' 117 | 118 | def test_failure(self): 119 | output = StringIO() 120 | rep = OutdatedReporter(output, None) 121 | # failures of outdated tasks are expected, so nothing happens 122 | rep.add_failure(FakeTask('outdated:xxx'), Exception()) 123 | # failures on any other task raise error 124 | pytest.raises(pytest.UsageError, rep.add_failure, 125 | FakeTask('x'), Exception()) 126 | 127 | def test_runtime_error(self): 128 | output = StringIO() 129 | rep = OutdatedReporter(output, None) 130 | pytest.raises(Exception, rep.runtime_error, 'error msg') 131 | 132 | 133 | class TestIncrementalControl(object): 134 | tt_conf = os.path.join(SAMPLE_DIR, 'tt/conftest.py') 135 | tt_mod1 = os.path.join(SAMPLE_DIR, 'tt/tt_mod1.py') 136 | tt_mod2 = os.path.join(SAMPLE_DIR, 'tt/tt_mod2.py') 137 | def test_py_files(self): 138 | control = IncrementalControl([SAMPLE_DIR]) 139 | assert len(control.py_files) == 6 140 | assert os.path.join(SAMPLE_DIR, 'mod1.py') in control.py_files 141 | assert os.path.join(SAMPLE_DIR, 'mod2.py') in control.py_files 142 | assert self.tt_conf in control.py_files 143 | assert self.tt_mod1 in control.py_files 144 | assert self.tt_mod2 in control.py_files 145 | 146 | def test_outdated(self, depfile_name, rm_generated_deps): 147 | control = IncrementalControl([SAMPLE_DIR]) 148 | control.DB_FILE = depfile_name 149 | control.test_files = [self.tt_mod1, self.tt_mod2] 150 | 151 | # at first all are outdated 152 | got = control.get_outdated() 153 | assert set(got.keys()) == set([self.tt_mod1, self.tt_mod2]) 154 | 155 | # save one success and check outdated 156 | control.save_success([self.tt_mod2]) 157 | assert set(control.get_outdated().keys()) == set([self.tt_mod1]) 158 | 159 | 160 | def test_list_deps(self, depfile_name, rm_generated_deps, capsys): 161 | control = IncrementalControl([SAMPLE_DIR]) 162 | control.DB_FILE = depfile_name 163 | control.test_files = [self.tt_mod1, self.tt_mod2] 164 | control.print_deps() 165 | out = capsys.readouterr()[0].splitlines() 166 | assert len(out) == 6 167 | assert ' - dodo.py: dodo.py' in out 168 | assert ' - mod1.py: mod1.py' in out 169 | assert ' - mod2.py: mod1.py, mod2.py' in out 170 | assert ' - tt/conftest.py: tt/conftest.py' in out 171 | assert ' - tt/tt_mod1.py: mod1.py, tt/conftest.py, tt/tt_mod1.py' in out 172 | mod2_deps = 'mod1.py, mod2.py, tt/conftest.py, tt/tt_mod2.py' 173 | assert ' - tt/tt_mod2.py: ' + mod2_deps in out 174 | 175 | def test_dot_graph(self, depfile_name, rm_generated_deps): 176 | control = IncrementalControl([SAMPLE_DIR]) 177 | control.DB_FILE = depfile_name 178 | control.test_files = [self.tt_mod1, self.tt_mod2] 179 | control.create_dot_graph() 180 | dot = open(os.path.join(SAMPLE_DIR, 'deps.dot')).read() 181 | out = dot.splitlines() 182 | assert len(out) == 9 183 | assert 'rankdir = BT' in out 184 | assert '"dodo.py"' in out 185 | assert '"mod1.py"' in out 186 | assert '"mod2.py" -> "mod1.py"' in out 187 | assert '"tt/conftest.py"' in out 188 | assert '"tt/tt_mod1.py" -> "mod1.py"' in out 189 | assert '"tt/tt_mod2.py" -> "mod2.py"' in out 190 | 191 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (http://tox.testrun.org/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | 6 | [tox] 7 | envlist = py35,py36,py37 8 | 9 | [testenv] 10 | commands = py.test 11 | deps = pytest-xdist 12 | --------------------------------------------------------------------------------