├── .gitignore ├── .readthedocs.yaml ├── AUTHORS ├── LICENSE ├── README ├── docs ├── Makefile ├── build │ └── .gitignore └── source │ ├── _templates │ └── layout.html │ ├── conf.py │ ├── crawler.rst │ ├── index.rst │ ├── keep_database_runner.rst │ ├── mocks.rst │ ├── running_tests.rst │ ├── settings.rst │ ├── test_runner.rst │ ├── testmaker.rst │ └── twill_runner.rst ├── setup.py ├── test_project ├── __init__.py ├── manage.py ├── polls │ ├── __init__.py │ ├── admin.py │ ├── fixtures │ │ └── polls_testmaker.json │ ├── models.py │ ├── templates │ │ ├── base.html │ │ └── polls │ │ │ ├── poll_detail.html │ │ │ ├── poll_list.html │ │ │ └── results.html │ ├── urls.py │ └── views.py ├── runtests.py ├── settings.py ├── test_app │ ├── __init__.py │ ├── models.py │ └── tests │ │ ├── __init__.py │ │ ├── assertions_tests.py │ │ ├── crawler_tests.py │ │ ├── templatetags_tests.py │ │ ├── testmaker_tests.py │ │ └── twill_tests.py └── urls.py └── test_utils ├── .gitignore ├── __init__.py ├── assertions.py ├── bin └── django_test_runner.py ├── crawler ├── __init__.py ├── base.py ├── plugins │ ├── __init__.py │ ├── base.py │ ├── graph.py │ ├── guppy_plugin.py │ ├── memory_plugin.py │ ├── pdb.py │ ├── query_count.py │ ├── sanitize.py │ ├── tidy.py │ ├── time_plugin.py │ └── urlconf.py └── signals.py ├── management ├── __init__.py └── commands │ ├── __init__.py │ ├── crawlurls.py │ ├── makefixture.py │ ├── quicktest.py │ ├── relational_dumpdata.py │ ├── testmaker.py │ └── testshell.py ├── mocks.py ├── models.py ├── templatetags └── __init__.py ├── test_runners ├── __init__.py ├── keep_database.py └── profile.py ├── testmaker ├── __init__.py ├── middleware │ ├── __init__.py │ └── testmaker.py ├── processors │ ├── __init__.py │ ├── base.py │ ├── django_processor.py │ └── twill_processor.py ├── replay.py └── serializers │ ├── __init__.py │ ├── base.py │ ├── json_serializer.py │ └── pickle_serializer.py ├── urls.py ├── utils ├── __init__.py └── twill_runner.py └── views.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file for Sphinx projects 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | # Required 5 | version: 2 6 | 7 | # Set the OS, Python version and other tools you might need 8 | build: 9 | os: ubuntu-22.04 10 | tools: 11 | python: "3.12" 12 | 13 | # Build documentation in the "docs/" directory with Sphinx 14 | sphinx: 15 | configuration: docs/source/conf.py 16 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Primary Author is Eric Holscher 2 | 3 | Some code is attributed in the files as well 4 | Greatly appreciated patches and other code from the following: 5 | 6 | Andrii Kurinnyi (zen4ever) 7 | Brian Luft (unbracketed) 8 | Chris Adams (acdha) 9 | mthornhill 10 | Michael Elsdörfer (miracle2k) 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2008 Eric Holscher 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following 10 | conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Welcome to django-test-utils. The Documentation for this project is located at https://django-test-utils.readthedocs.org/en/latest/ 2 | 3 | The tests are run on every commit on Devmason: http://devmason.com/pony_server/django-test-utils 4 | 5 | The mailing list is located on google groups: http://groups.google.com/group/django-testing 6 | 7 | Thanks for using test utils! 8 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | 9 | # Internal variables. 10 | PAPEROPT_a4 = -D latex_paper_size=a4 11 | PAPEROPT_letter = -D latex_paper_size=letter 12 | ALLSPHINXOPTS = -d build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 13 | 14 | .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest 15 | 16 | help: 17 | @echo "Please use \`make ' where is one of" 18 | @echo " html to make standalone HTML files" 19 | @echo " dirhtml to make HTML files named index.html in directories" 20 | @echo " pickle to make pickle files" 21 | @echo " json to make JSON files" 22 | @echo " htmlhelp to make HTML files and a HTML help project" 23 | @echo " qthelp to make HTML files and a qthelp project" 24 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 25 | @echo " changes to make an overview of all changed/added/deprecated items" 26 | @echo " linkcheck to check all external links for integrity" 27 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 28 | 29 | clean: 30 | -rm -rf build/* 31 | 32 | html: 33 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) build/html 34 | @echo 35 | @echo "Build finished. The HTML pages are in build/html." 36 | 37 | dirhtml: 38 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) build/dirhtml 39 | @echo 40 | @echo "Build finished. The HTML pages are in build/dirhtml." 41 | 42 | pickle: 43 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) build/pickle 44 | @echo 45 | @echo "Build finished; now you can process the pickle files." 46 | 47 | json: 48 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) build/json 49 | @echo 50 | @echo "Build finished; now you can process the JSON files." 51 | 52 | htmlhelp: 53 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) build/htmlhelp 54 | @echo 55 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 56 | ".hhp project file in build/htmlhelp." 57 | 58 | qthelp: 59 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) build/qthelp 60 | @echo 61 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 62 | ".qhcp project file in build/qthelp, like this:" 63 | @echo "# qcollectiongenerator build/qthelp/Django Test Utils.qhcp" 64 | @echo "To view the help file:" 65 | @echo "# assistant -collectionFile build/qthelp/Django Test Utils.qhc" 66 | 67 | latex: 68 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) build/latex 69 | @echo 70 | @echo "Build finished; the LaTeX files are in build/latex." 71 | @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ 72 | "run these through (pdf)latex." 73 | 74 | changes: 75 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) build/changes 76 | @echo 77 | @echo "The overview file is in build/changes." 78 | 79 | linkcheck: 80 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) build/linkcheck 81 | @echo 82 | @echo "Link check complete; look for any errors in the above output " \ 83 | "or in build/linkcheck/output.txt." 84 | 85 | doctest: 86 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) build/doctest 87 | @echo "Testing of doctests in the sources finished, look at the " \ 88 | "results in build/doctest/output.txt." 89 | -------------------------------------------------------------------------------- /docs/build/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | -------------------------------------------------------------------------------- /docs/source/_templates/layout.html: -------------------------------------------------------------------------------- 1 | {% extends "!layout.html" %} 2 | 3 | {% block footer %} 4 | {{ super() }} 5 | 9 | 14 | {% endblock %} 15 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Django Test Utils documentation build configuration file, created by 4 | # sphinx-quickstart on Fri Apr 3 16:36:58 2009. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | #sys.path.append(os.path.abspath('.')) 20 | 21 | # -- General configuration ----------------------------------------------------- 22 | 23 | # Add any Sphinx extension module names here, as strings. They can be extensions 24 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 25 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx'] 26 | 27 | # Add any paths that contain templates here, relative to this directory. 28 | templates_path = ['_templates'] 29 | 30 | # The suffix of source filenames. 31 | source_suffix = '.rst' 32 | 33 | # The encoding of source files. 34 | #source_encoding = 'utf-8' 35 | 36 | # The master toctree document. 37 | master_doc = 'index' 38 | 39 | # General information about the project. 40 | project = u'Django Test Utils' 41 | copyright = u'2009, Eric Holscher' 42 | 43 | # The version info for the project you're documenting, acts as replacement for 44 | # |version| and |release|, also used in various other places throughout the 45 | # built documents. 46 | # 47 | # The short X.Y version. 48 | version = '0.3' 49 | # The full version, including alpha/beta/rc tags. 50 | release = '0.3' 51 | 52 | # The language for content autogenerated by Sphinx. Refer to documentation 53 | # for a list of supported languages. 54 | #language = None 55 | 56 | # There are two options for replacing |today|: either, you set today to some 57 | # non-false value, then it is used: 58 | #today = '' 59 | # Else, today_fmt is used as the format for a strftime call. 60 | #today_fmt = '%B %d, %Y' 61 | 62 | # List of documents that shouldn't be included in the build. 63 | #unused_docs = [] 64 | 65 | # List of directories, relative to source directory, that shouldn't be searched 66 | # for source files. 67 | exclude_trees = [] 68 | 69 | # The reST default role (used for this markup: `text`) to use for all documents. 70 | #default_role = None 71 | 72 | # If true, '()' will be appended to :func: etc. cross-reference text. 73 | #add_function_parentheses = True 74 | 75 | # If true, the current module name will be prepended to all description 76 | # unit titles (such as .. function::). 77 | #add_module_names = True 78 | 79 | # If true, sectionauthor and moduleauthor directives will be shown in the 80 | # output. They are ignored by default. 81 | #show_authors = False 82 | 83 | # The name of the Pygments (syntax highlighting) style to use. 84 | pygments_style = 'sphinx' 85 | 86 | # A list of ignored prefixes for module index sorting. 87 | #modindex_common_prefix = [] 88 | 89 | 90 | # -- Options for HTML output --------------------------------------------------- 91 | 92 | # The theme to use for HTML and HTML Help pages. Major themes that come with 93 | # Sphinx are currently 'default' and 'sphinxdoc'. 94 | html_theme = 'default' 95 | 96 | # Theme options are theme-specific and customize the look and feel of a theme 97 | # further. For a list of options available for each theme, see the 98 | # documentation. 99 | #html_theme_options = {} 100 | 101 | # Add any paths that contain custom themes here, relative to this directory. 102 | #html_theme_path = [] 103 | 104 | # The name for this set of Sphinx documents. If None, it defaults to 105 | # " v documentation". 106 | #html_title = None 107 | 108 | # A shorter title for the navigation bar. Default is the same as html_title. 109 | #html_short_title = None 110 | 111 | # The name of an image file (relative to this directory) to place at the top 112 | # of the sidebar. 113 | #html_logo = None 114 | 115 | # The name of an image file (within the static path) to use as favicon of the 116 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 117 | # pixels large. 118 | #html_favicon = None 119 | 120 | # Add any paths that contain custom static files (such as style sheets) here, 121 | # relative to this directory. They are copied after the builtin static files, 122 | # so a file named "default.css" will overwrite the builtin "default.css". 123 | html_static_path = ['_static'] 124 | 125 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 126 | # using the given strftime format. 127 | #html_last_updated_fmt = '%b %d, %Y' 128 | 129 | # If true, SmartyPants will be used to convert quotes and dashes to 130 | # typographically correct entities. 131 | #html_use_smartypants = True 132 | 133 | # Custom sidebar templates, maps document names to template names. 134 | #html_sidebars = {} 135 | 136 | # Additional templates that should be rendered to pages, maps page names to 137 | # template names. 138 | #html_additional_pages = {} 139 | 140 | # If false, no module index is generated. 141 | #html_use_modindex = True 142 | 143 | # If false, no index is generated. 144 | #html_use_index = True 145 | 146 | # If true, the index is split into individual pages for each letter. 147 | #html_split_index = False 148 | 149 | # If true, links to the reST sources are added to the pages. 150 | #html_show_sourcelink = True 151 | 152 | # If true, an OpenSearch description file will be output, and all pages will 153 | # contain a tag referring to it. The value of this option must be the 154 | # base URL from which the finished HTML is served. 155 | #html_use_opensearch = '' 156 | 157 | # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). 158 | #html_file_suffix = '' 159 | 160 | # Output file base name for HTML help builder. 161 | htmlhelp_basename = 'DjangoTestUtilsdoc' 162 | 163 | 164 | # -- Options for LaTeX output -------------------------------------------------- 165 | 166 | # The paper size ('letter' or 'a4'). 167 | #latex_paper_size = 'letter' 168 | 169 | # The font size ('10pt', '11pt' or '12pt'). 170 | #latex_font_size = '10pt' 171 | 172 | # Grouping the document tree into LaTeX files. List of tuples 173 | # (source start file, target name, title, author, documentclass [howto/manual]). 174 | latex_documents = [ 175 | ('index', 'DjangoTestUtils.tex', ur'Django Test Utils Documentation', 176 | ur'Eric Holscher', 'manual'), 177 | ] 178 | 179 | # The name of an image file (relative to this directory) to place at the top of 180 | # the title page. 181 | #latex_logo = None 182 | 183 | # For "manual" documents, if this is true, then toplevel headings are parts, 184 | # not chapters. 185 | #latex_use_parts = False 186 | 187 | # Additional stuff for the LaTeX preamble. 188 | #latex_preamble = '' 189 | 190 | # Documents to append as an appendix to all manuals. 191 | #latex_appendices = [] 192 | 193 | # If false, no module index is generated. 194 | #latex_use_modindex = True 195 | 196 | 197 | # Example configuration for intersphinx: refer to the Python standard library. 198 | intersphinx_mapping = {'http://docs.python.org/': None} 199 | -------------------------------------------------------------------------------- /docs/source/crawler.rst: -------------------------------------------------------------------------------- 1 | .. _crawler: 2 | 3 | Django Crawler 4 | ----------------- 5 | 6 | 7 | Core features 8 | ~~~~~~~~~~~~~ 9 | 10 | The Crawler at the beginning loops through all of your URLConfs. It 11 | then loads up all of the regular expressions from these URLConfs to 12 | examine later. Once the crawler is done crawling your site, it will 13 | tell you what URLConf entries are not being hit. 14 | 15 | 16 | Usage 17 | ~~~~~ 18 | 19 | The crawler is implemented as a management command. 20 | 21 | Step 1: Add `test_utils` to your `INSTALLED_APPS` 22 | 23 | Step 2: The syntax for invoking the crawler looks like: 24 | 25 | .. sourcecode:: python 26 | 27 | ./manage.py crawlurls [options] [relative_start_url] 28 | 29 | Relative start URLs are assumed to be relative to the site root and should 30 | look like 'some/path', 'home', or even '/'. The relative start URL will be 31 | normalized with leading and trailing slashes if they are not provided. The 32 | default relative start URL is '/'. 33 | 34 | The crawler at the moment has 4 options implemented on it. It crawls your 35 | site using the `Django Test Client 36 | `__ (so no network traffic is required!) This 38 | allows the crawler to have intimate knowledge of your Django Code. 39 | This allows it to have features that other crawlers can't have. 40 | 41 | 42 | Options 43 | ~~~~~~~ 44 | 45 | -v --verbosity [0,1,2] 46 | `````````````````````` 47 | 48 | Same as most django apps. Set it to 2 to get a lot of output. 1 is the 49 | default, which will only output errors. 50 | 51 | 52 | -t --time 53 | ````````` 54 | 55 | The `-t` option, as the help says: Pass -t to time your requests. This 56 | outputs the time it takes to run each request on your site. This 57 | option also tells the crawler to output the top 10 URLs that took the 58 | most time at the end of it's run. Here is an example output from 59 | running it on my site with `-t -v 2`: 60 | 61 | .. sourcecode:: python 62 | 63 | Getting /blog/2007/oct/17/umw-blog-ring/ ({}) from (/blog/2007/oct/17/umw-blog-ring/) 64 | Time Elapsed: 0.256254911423 65 | Getting /blog/2007/dec/20/logo-lovers/ ({}) from (/blog/2007/dec/20/logo-lovers/) 66 | Time Elapsed: 0.06906914711 67 | Getting /blog/2007/dec/18/getting-real/ ({}) from (/blog/2007/dec/18/getting-real/) 68 | Time Elapsed: 0.211296081543 69 | Getting /blog/ ({u'page': u'5'}) from (/blog/?page=4) 70 | Time Elapsed: 0.165636062622 71 | NOT MATCHED: account/email/ 72 | NOT MATCHED: account/register/ 73 | NOT MATCHED: admin/doc/bookmarklets/ 74 | NOT MATCHED: admin/doc/tags/ 75 | NOT MATCHED: admin/(.*) 76 | NOT MATCHED: admin/doc/views/ 77 | NOT MATCHED: account/signin/complete/ 78 | NOT MATCHED: account/password/ 79 | NOT MATCHED: resume/ 80 | /blog/2008/feb/9/another-neat-ad/ took 0.743204 81 | /blog/2007/dec/20/browser-tabs/#comments took 0.637164 82 | /blog/2008/nov/1/blog-post-day-keeps-doctor-away/ took 0.522269 83 | 84 | 85 | -p --pdb 86 | ```````` 87 | 88 | This option allows you to drop into pdb on an error in your site. This 89 | lets you look around the response, context, and other things to see 90 | what happened to cause the error. I don't know how useful this will 91 | be, but it seems like a neat feature to be able to have. I stole this 92 | idea from nose tests. 93 | 94 | 95 | -s --safe 96 | ````````` 97 | 98 | This option alerts you when you have escaped HTML fragments in your 99 | templates. This is useful for tracking down places where you aren't 100 | applying safe correctly, and other HTML related failures. This isn't 101 | implemented well, and might be buggy because I didn't have any broken 102 | pages on my site to test on :) 103 | 104 | 105 | -r --response 106 | ````````````` 107 | 108 | This tells the crawler to store the response object for each site. 109 | This used to be the default behavior, but doing this bloats up memory. 110 | There isn't anything useful implemented on top of this feature, but 111 | with this feature you get a dictionary of request URLs with responses 112 | as their values. You can then go through and do whatever you want 113 | (including examine the Templates rendered and Contexts. 114 | 115 | 116 | Considerations 117 | ~~~~~~~~~~~~~~ 118 | 119 | At the moment, this crawler doesn't have a lot of end-user 120 | functionality. However, you can go in and edit the script at the end 121 | of the crawl to do almost anything. You are left with a dictionary of 122 | URLs crawled, and the time it took, and response (if you use the `-r` 123 | option). 124 | 125 | 126 | Future improvements 127 | ~~~~~~~~~~~~~~~~~~~ 128 | 129 | There are a lot of future improvements that I have planned. I want to 130 | enable the test client to login as a user, passed in from the command 131 | line. This should be pretty simple, I just haven't implemented it yet. 132 | 133 | Another thing that I want to do but isn't implemented is fixtures. I 134 | want to be able to output a copy of the data returned from the crawler 135 | run. This will allow for future runs of the crawler to diff against 136 | previous runs, creating a kind of regression test. 137 | 138 | A third thing I want to implement is an option to only evaluate each 139 | URLConf entry X times. Where you could say "only hit 140 | /blog/[year]/[month]/ 10 times". This goes on the assumption that you 141 | are looking for errors in your views or templates, and you only need 142 | to hit each URL a couple of times. This also shouldn't be hard, but 143 | isn't implemented yet. 144 | 145 | The big pony that I want to make is to use multiprocessing on the 146 | crawler. The crawler doesn't hit a network, so it is CPU-bound. 147 | However, running with CPUs with multiple cores, multiprocessing will 148 | speed this up. A problem with it is that some of the timing stuff and 149 | pdb things won't be as useful. 150 | 151 | I would love to hear some people's feedback and thoughts on this. I 152 | think that this could be made into a really awesome tool. At the 153 | moment it works well for smaller sites, but it would be nice to be 154 | able to test only certain URLs in an app. There are lots of neat 155 | things I have planned, but I like following the release early, release 156 | often mantra. 157 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. _index: 2 | 3 | Welcome to Django Test Utils's documentation! 4 | ============================================= 5 | 6 | Here you will find information on the utilities and other nice things located in this package. It is meant to make testing your Django applications easier. It also has things that will automate some parts of your testing. 7 | 8 | Source Code 9 | ----------- 10 | 11 | The code for Django-test-utils is available at `Github 12 | `_ and 13 | `Pypi `_. 14 | 15 | If you have any questions, please contact `Eric Holscher `_ at eric@ericholscher.com 16 | 17 | There is a `Bug Tracker `_ at Github as well, if you find any defects with the code. Pull requests and patches are better than filing a bug. 18 | 19 | The mailing list for the project is located on Google Groups: http://groups.google.com/group/django-testing 20 | 21 | Contents 22 | -------- 23 | 24 | .. toctree:: 25 | :maxdepth: 2 26 | 27 | testmaker 28 | crawler 29 | test_runner 30 | twill_runner 31 | keep_database_runner 32 | mocks 33 | settings 34 | running_tests 35 | 36 | Indices and tables 37 | ================== 38 | 39 | * :ref:`genindex` 40 | * :ref:`search` 41 | -------------------------------------------------------------------------------- /docs/source/keep_database_runner.rst: -------------------------------------------------------------------------------- 1 | 2 | Persistent Database Test Runner 3 | =============================== 4 | 5 | This code allows you to persist a database between test runs. It is really useful for running the same tests again and again, without incurring the cost of having to re-create the database each time. 6 | 7 | .. note:: 8 | 9 | Currently support for 1.2 is in a 1.2 compatible branch. This will be 10 | merged into trunk soon. 11 | 12 | 13 | Management Command 14 | ------------------ 15 | 16 | To call this function, simply use the ``quicktest`` management command, instead of the ``test`` command. If you need to alter the schema for your tests, simply run the normal ``test`` command, and the normal destroy/create cycle will take place. 17 | 18 | 19 | Test Runner 20 | ----------- 21 | 22 | The functionality is actually implemented in a Test Runner located at ``test_utils.test_runners.keep_database``. If you want to use this as your default test runner, you can set the ``TEST_RUNNER`` setting to that value. This is basically all that the management command does, but in a temporary way. 23 | -------------------------------------------------------------------------------- /docs/source/mocks.rst: -------------------------------------------------------------------------------- 1 | .. _mocks: 2 | 3 | 4 | Mock Objects 5 | ============ 6 | 7 | 8 | Mock Requests 9 | ------------- 10 | 11 | This mock allows you to make requests against a view that isn't included in any URLConf. 12 | 13 | RequestFactory 14 | ^^^^^^^^^^^^^^ 15 | 16 | Usage:: 17 | 18 | rf = RequestFactory() 19 | get_request = rf.get('/hello/') 20 | post_request = rf.post('/submit/', {'foo': 'bar'}) 21 | 22 | This class re-uses the django.test.client.Client interface, docs here: 23 | http://www.djangoproject.com/documentation/testing/#the-test-client 24 | 25 | Once you have a request object you can pass it to any view function, 26 | just as if that view had been hooked up using a URLconf. 27 | 28 | 29 | Original Source 30 | """"""""""""""" 31 | 32 | Taken from Djangosnippets.net_, originally by `Simon Willison `_. 33 | 34 | .. _Djangosnippets.net: http://www.djangosnippets.org/snippets/963/ 35 | -------------------------------------------------------------------------------- /docs/source/running_tests.rst: -------------------------------------------------------------------------------- 1 | .. _running_tests: 2 | 3 | 4 | ==================== 5 | How to run the tests 6 | ==================== 7 | 8 | Test utils does contain some tests. Not as many as I would like, but it has enough to check the basic functionality of the things it does. 9 | 10 | Running the tests is pretty simple. You just need to go into the test_project, and then run:: 11 | 12 | ./manage.py test --settings=settings 13 | 14 | In order to run just the tests for test utils do:: 15 | 16 | ./manage.py test test_app --settings=settings 17 | 18 | It is also possible to just run a single class of the tests if you so wish:: 19 | 20 | ./manage.py test test_app.TestMakerTests --settings=settings 21 | -------------------------------------------------------------------------------- /docs/source/settings.rst: -------------------------------------------------------------------------------- 1 | .. _settings: 2 | 3 | 4 | ================== 5 | Available Settings 6 | ================== 7 | 8 | This page contains the available settings for test utils. Broken down by the app that they affect. 9 | 10 | 11 | Testmaker 12 | ========= 13 | 14 | TESTMAKER_SERIALIZER 15 | -------------------- 16 | 17 | This is the serializer that Testmaker should use. By default it is `pickle`. 18 | 19 | TESTMAKER_PROCESSOR 20 | ------------------- 21 | 22 | This is the processor that Testmaker should use. By default it is `django`. 23 | 24 | TEST_PROCESSOR_MODULES 25 | ---------------------- 26 | 27 | This allows Testmaker to have access to your own custom processor modules. They are defined with the full path to the module import as the value. 28 | 29 | To add your own processors, use the TEST_PROCESSOR_MODULES setting:: 30 | 31 | TEST_PROCESSOR_MODULES = { 32 | 'awesome': 'my_sweet_app.processors.awesome', 33 | } 34 | 35 | TEST_SERIALIZATION_MODULES 36 | -------------------------- 37 | 38 | The same as the above `TEST_PROCESSOR_MODULES`, allowing you to augment Testmakers default serializers. 39 | 40 | To add your own serializers, use the TEST_SERIALIZATION_MODULES setting:: 41 | 42 | TEST_SERIALIZATION_MODULES = { 43 | 'awesome': 'my_sweet_app.serializers.awesome', 44 | } 45 | -------------------------------------------------------------------------------- /docs/source/test_runner.rst: -------------------------------------------------------------------------------- 1 | .. _test_runner: 2 | 3 | Django Test Runner 4 | ================== 5 | 6 | This is a tool for running Django app tests standalone 7 | 8 | Introduction 9 | ------------ 10 | 11 | This script is fairly basic. Here is a quick example of how to use it:: 12 | django_test_runner.py [path-to-app] 13 | 14 | You must have Django on the PYTHONPATH prior to running this script. This 15 | script basically will bootstrap a Django environment for you. 16 | 17 | By default this script with use SQLite and an in-memory database. If you 18 | are using Python 2.5 it will just work out of the box for you. 19 | -------------------------------------------------------------------------------- /docs/source/testmaker.rst: -------------------------------------------------------------------------------- 1 | .. _testmaker: 2 | 3 | ================ 4 | Django Testmaker 5 | ================ 6 | 7 | Source code 8 | =========== 9 | 10 | This projects is now a part of `Django test utils `__. The 0.2 release is available at `Pypi 12 | `__ 13 | 14 | 15 | What it does 16 | ============ 17 | 18 | Django testmaker is an application that writes tests for your Django 19 | views for you. You simply run a special development server, and it 20 | records tests for your application into your project for you. Tests 21 | will be in a Unit Test format, and it will create a separate test for 22 | each view that you hit. 23 | 24 | 25 | Usage 26 | ===== 27 | 28 | Step 1: Add `test_utils` to your INSTALLED_APPS settings. 29 | 30 | Step 2: 31 | 32 | .. sourcecode:: python 33 | 34 | ./manage.py testmaker -a APP_NAME 35 | 36 | 37 | This will start the development server with testmaker loaded in. APP_NAME must 38 | be in installed apps, and it will use Django's mechanism for finding it. It 39 | should look a little something like this: 40 | 41 | .. sourcecode:: python 42 | 43 | eric@Odin:~/EH$ ./manage.py testmaker mine 44 | Handling app 'mine' 45 | Logging tests to /home/eric/Python/EH/mine/tests/mine_testmaker.py 46 | Appending to current log file 47 | Inserting TestMaker logging server... 48 | Validating models... 49 | 0 errors found 50 | 51 | Django version 1.0.1 final, using settings 'EH.settings' 52 | Development server is running at http://127.0.0.1:8000/ 53 | Quit the server with CONTROL-C. 54 | 55 | 56 | Then, as you browse around your site it will create unit test files 57 | for you, outputting the context variables and status code for each 58 | view that you visit. The test file used is in 59 | `APP/tests/APP_testmaker.py`. Once you have your tests written, you 60 | simply have to add them into your `__init__.py`, and then run your 61 | tests. 62 | 63 | Step 3: 64 | 65 | .. sourcecode:: python 66 | 67 | ./manage.py test -a APP_NAME 68 | 69 | Testmaker Internal Basics 70 | ========================= 71 | 72 | Testmaker now has a pluggable backend architecture. It includes the concept of a `Processor` and a `Serializer`. A `Serializer` is responsible for serializing your request and responses, so that they can be run again later. A `Processor` is response for taking these request and response objects and turning them into the actual Unit Tests. 73 | 74 | API 75 | --- 76 | 77 | Both processors and serializers follow the standard API direction that django serializers started. For example, to grab instances of both of them, use the following code:: 78 | 79 | serializer = serializers.get_serializer('pickle')() 80 | processor = processors.get_processor('django')() 81 | 82 | Serializers 83 | ----------- 84 | 85 | Testmaker ships with 2 default serializers currently. They are the `json` and `pickle` backends. 86 | 87 | Processors 88 | ---------- 89 | 90 | Testmaker currently ships with just one Processor, the `django` processor, which produces Django Testcase-style Unit Tests. 91 | 92 | Extending Testmaker 93 | =================== 94 | 95 | Adding a new backend for a Processor or Serializer is easy. They both provide a similar interface, which can be located in the base.py file in their respective directories. 96 | 97 | The only two functions that are required for each backend are the ability to save a request and a response. They have obvious defitions, and you should look in their directory for examples. 98 | 99 | save_request(self, request) 100 | --------------------------- 101 | 102 | save_response(self, request, response) 103 | -------------------------------------- 104 | 105 | 106 | Options 107 | ======= 108 | 109 | 110 | -f --fixture 111 | ------------ 112 | 113 | If you pass the `-f` option to testmaker, it will create fixtures for 114 | you. They will be saved in `APP/fixtures/APP_fixtures.FORMAT`. The 115 | default format is XML because I was having problems with JSON. 116 | 117 | 118 | --format 119 | -------- 120 | 121 | Pass this in with a valid serialization format for Django. Options are 122 | currently json, yaml, or xml. 123 | 124 | 125 | --addrport 126 | ---------- 127 | 128 | This allows you to pass in the normal address and port options for 129 | runserver. 130 | 131 | 132 | Future improvements 133 | =================== 134 | 135 | Force app filtering 136 | ------------------- 137 | 138 | I plan on having an option that allows you to restrict the views to 139 | the app that you passed in on the command line. This would inspect the 140 | URLConf for the app, and only output tests matching those URLs. This 141 | would allow you to fine tune your tests so that it is guaranteed to 142 | only test views in the app. 143 | 144 | 145 | Better test naming scheme 146 | ------------------------- 147 | 148 | The current way of naming tests is a bit hackish, and could be 149 | improved. It works for now, and keeps names unique, so it's achieving 150 | that goal. Suggestions welcome for a better way to name things. 151 | -------------------------------------------------------------------------------- /docs/source/twill_runner.rst: -------------------------------------------------------------------------------- 1 | .. _twill_runner: 2 | 3 | Twill Runner 4 | ============ 5 | 6 | Integrates the twill web browsing scripting language with Django. 7 | 8 | Introducation 9 | -------------- 10 | 11 | Provides two main functions, ``setup()`` and ``teardown``, that hook 12 | (and unhook) a certain host name to the WSGI interface of your Django 13 | app, making it possible to test your site using twill without actually 14 | going through TCP/IP. 15 | 16 | It also changes the twill browsing behaviour, so that relative urls 17 | per default point to the intercept (e.g. your Django app), so long 18 | as you don't browse away from that host. Further, you are allowed to 19 | specify the target url as arguments to Django's ``reverse()``. 20 | 21 | Usage:: 22 | 23 | twill.setup() 24 | try: 25 | twill.go('/') # --> Django WSGI 26 | twill.code(200) 27 | 28 | twill.go('http://google.com') 29 | twill.go('/services') # --> http://google.com/services 30 | 31 | twill.go('/list', default=True) # --> back to Django WSGI 32 | 33 | twill.go('proj.app.views.func', 34 | args=[1,2,3]) 35 | finally: 36 | twill.teardown() 37 | 38 | For more information about twill, see: http://twill.idyll.org/ 39 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | setup( 4 | name = "django-test-utils", 5 | version = "0.3", 6 | packages = find_packages(), 7 | author = "Eric Holscher", 8 | author_email = "eric@ericholscher.com", 9 | description = "A package to help testing in Django", 10 | url = "http://github.com/ericholscher/django-test-utils/tree/master", 11 | download_url='http://www.github.com/ericholscher/django-test-utils/tarball/0.3.0', 12 | test_suite = "test_project.run_tests.run_tests", 13 | include_package_data = True, 14 | install_requires=[ 15 | 'BeautifulSoup', 16 | 'twill', 17 | ] 18 | ) 19 | -------------------------------------------------------------------------------- /test_project/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ericholscher/django-test-utils/0207f06e0ebaa5081dc8648815815929fbc9dea5/test_project/__init__.py -------------------------------------------------------------------------------- /test_project/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from django.core.management import execute_manager 3 | try: 4 | import settings # Assumed to be in the same directory. 5 | except ImportError: 6 | import sys 7 | sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n" % __file__) 8 | sys.exit(1) 9 | 10 | if __name__ == "__main__": 11 | execute_manager(settings) 12 | -------------------------------------------------------------------------------- /test_project/polls/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ericholscher/django-test-utils/0207f06e0ebaa5081dc8648815815929fbc9dea5/test_project/polls/__init__.py -------------------------------------------------------------------------------- /test_project/polls/admin.py: -------------------------------------------------------------------------------- 1 | from polls.models import Poll, Choice 2 | from django.contrib import admin 3 | 4 | class ChoiceInline(admin.StackedInline): 5 | model = Choice 6 | extra = 3 7 | 8 | class PollAdmin(admin.ModelAdmin): 9 | inlines = [ChoiceInline] 10 | 11 | admin.site.register(Poll, PollAdmin) 12 | -------------------------------------------------------------------------------- /test_project/polls/fixtures/polls_testmaker.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "pk": 1, 4 | "model": "polls.poll", 5 | "fields": { 6 | "pub_date": "2007-04-01 00:00:00", 7 | "question": "What's up?", 8 | "slug": "whats-up" 9 | } 10 | }, 11 | { 12 | "pk": 2, 13 | "model": "polls.poll", 14 | "fields": { 15 | "pub_date": "2009-04-25 21:46:44", 16 | "question": "Test poll", 17 | "slug": "test-poll" 18 | } 19 | }, 20 | { 21 | "pk": 1, 22 | "model": "polls.choice", 23 | "fields": { 24 | "votes": 3, 25 | "poll": 1, 26 | "choice": "Just hacking again" 27 | } 28 | }, 29 | { 30 | "pk": 2, 31 | "model": "polls.choice", 32 | "fields": { 33 | "votes": 2, 34 | "poll": 1, 35 | "choice": "Playing with fire" 36 | } 37 | }, 38 | { 39 | "pk": 3, 40 | "model": "polls.choice", 41 | "fields": { 42 | "votes": 0, 43 | "poll": 1, 44 | "choice": "Clapping my shoes together three times" 45 | } 46 | }, 47 | { 48 | "pk": 4, 49 | "model": "polls.choice", 50 | "fields": { 51 | "votes": 0, 52 | "poll": 2, 53 | "choice": "Choice 1 " 54 | } 55 | }, 56 | { 57 | "pk": 5, 58 | "model": "polls.choice", 59 | "fields": { 60 | "votes": 3, 61 | "poll": 2, 62 | "choice": "Choice too" 63 | } 64 | }, 65 | { 66 | "pk": 6, 67 | "model": "polls.choice", 68 | "fields": { 69 | "votes": 0, 70 | "poll": 2, 71 | "choice": "New Choice" 72 | } 73 | } 74 | ] 75 | -------------------------------------------------------------------------------- /test_project/polls/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | class Poll(models.Model): 4 | question = models.CharField(max_length=200) 5 | pub_date = models.DateTimeField('date published') 6 | slug = models.SlugField(null=True) 7 | 8 | def __unicode__(self): 9 | return self.question 10 | 11 | class Choice(models.Model): 12 | poll = models.ForeignKey(Poll) 13 | choice = models.CharField(max_length=200) 14 | votes = models.IntegerField() 15 | 16 | def __unicode__(self): 17 | return self.choice 18 | -------------------------------------------------------------------------------- /test_project/polls/templates/base.html: -------------------------------------------------------------------------------- 1 | {% block content %} 2 | {% endblock %} 3 | -------------------------------------------------------------------------------- /test_project/polls/templates/polls/poll_detail.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block content %} 3 | {% with object as poll %} 4 |

{{ poll.question }}

5 | 6 | {% if error_message %}

{{ error_message }}

{% endif %} 7 | 8 |
9 | {% for choice in poll.choice_set.all %} 10 | 11 |
12 | {% endfor %} 13 | 14 |
15 | {% endwith %} 16 | {% endblock %} 17 | -------------------------------------------------------------------------------- /test_project/polls/templates/polls/poll_list.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block content %} 3 | {% if object_list %} 4 | 9 | {% else %} 10 |

No polls are available.

11 | {% endif %} 12 | {% endblock %} 13 | -------------------------------------------------------------------------------- /test_project/polls/templates/polls/results.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block content %} 3 | {% with object as poll %} 4 |

{{ poll.question }}

5 | 6 |
    7 | {% for choice in poll.choice_set.all %} 8 |
  • {{ choice.choice }} -- {{ choice.votes }} vote{{ choice.votes|pluralize }}
    • 9 | {% endfor %} 10 |
11 | 12 | {% endwith %} 13 | {% endblock %} 14 | -------------------------------------------------------------------------------- /test_project/polls/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls.defaults import * 2 | from models import Poll, Choice 3 | 4 | info_dict = { 5 | 'queryset': Poll.objects.all(), 6 | } 7 | 8 | urlpatterns = patterns('', 9 | (r'^$', 'django.views.generic.list_detail.object_list', info_dict), 10 | (r'^(?P\d+)/$', 'django.views.generic.list_detail.object_detail', info_dict), 11 | url(r'^(?P\d+)/results/$', 'django.views.generic.list_detail.object_detail', dict(info_dict, template_name='polls/results.html'), 'poll_results'), 12 | (r'^(?P\d+)/vote/$', 'polls.views.vote'), 13 | ) 14 | -------------------------------------------------------------------------------- /test_project/polls/views.py: -------------------------------------------------------------------------------- 1 | # Create your views here. 2 | 3 | from django.shortcuts import get_object_or_404, render_to_response 4 | from django.http import HttpResponseRedirect 5 | from django.core.urlresolvers import reverse 6 | from polls.models import Poll, Choice 7 | 8 | def vote(request, poll_id): 9 | p = get_object_or_404(Poll, pk=poll_id) 10 | try: 11 | selected_choice = p.choice_set.get(pk=request.POST['choice']) 12 | except (KeyError, Choice.DoesNotExist): 13 | # Redisplay the poll voting form. 14 | return render_to_response('polls/detail.html', { 15 | 'poll': p, 16 | 'error_message': "You didn't select a choice.", 17 | }) 18 | else: 19 | selected_choice.votes += 1 20 | selected_choice.save() 21 | # Always return an HttpResponseRedirect after successfully dealing 22 | # with POST data. This prevents data from being posted twice if a 23 | # user hits the Back button. 24 | return HttpResponseRedirect("/polls/%s/results/" % p.id) 25 | 26 | def results(request, poll_id): 27 | p = get_object_or_404(Poll, pk=poll_id) 28 | return render_to_response('polls/results.html', {'poll': p}) 29 | -------------------------------------------------------------------------------- /test_project/runtests.py: -------------------------------------------------------------------------------- 1 | #This file mainly exists to allow python setup.py test to work. 2 | 3 | import os, sys 4 | os.environ['DJANGO_SETTINGS_MODULE'] = 'test_project.settings' 5 | 6 | test_dir = os.path.dirname(__file__) 7 | sys.path.insert(0, test_dir) 8 | 9 | from django.test.utils import get_runner 10 | from django.conf import settings 11 | 12 | def runtests(): 13 | test_runner = get_runner(settings) 14 | failures = test_runner([]) 15 | sys.exit(failures) 16 | 17 | if __name__ == '__main__': 18 | runtests() 19 | -------------------------------------------------------------------------------- /test_project/settings.py: -------------------------------------------------------------------------------- 1 | # Django settings for paradigm project. 2 | import os 3 | PROJECT_DIR = os.path.dirname(__file__) 4 | 5 | #DEBUG = True 6 | DEBUG = False 7 | TEMPLATE_DEBUG = DEBUG 8 | 9 | ADMINS = () 10 | 11 | MANAGERS = ADMINS 12 | 13 | DATABASE_ENGINE = 'sqlite3' # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. 14 | DATABASE_NAME = PROJECT_DIR + '/test_settings.db' # Or path to database file if using sqlite3. 15 | #mckenzie 16 | 17 | # Local time zone for this installation. Choices can be found here: 18 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name 19 | # although not all choices may be available on all operating systems. 20 | # If running in a Windows environment this must be set to the same as your 21 | # system time zone. 22 | TIME_ZONE = 'America/Chicago' 23 | 24 | # Language code for this installation. All choices can be found here: 25 | # http://www.i18nguy.com/unicode/language-identifiers.html 26 | LANGUAGE_CODE = 'en-us' 27 | 28 | SITE_ID = 1 29 | 30 | # If you set this to False, Django will make some optimizations so as not 31 | # to load the internationalization machinery. 32 | USE_I18N = True 33 | 34 | # Absolute path to the directory that holds media. 35 | # Example: "/home/media/media.lawrence.com/" 36 | MEDIA_ROOT = '' 37 | 38 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a 39 | # trailing slash if there is a path component (optional in other cases). 40 | # Examples: "http://media.lawrence.com", "http://example.com/media/" 41 | MEDIA_URL = '' 42 | 43 | # URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a 44 | # trailing slash. 45 | # Examples: "http://foo.com/media/", "/media/". 46 | ADMIN_MEDIA_PREFIX = '/media/' 47 | 48 | # Make this unique, and don't share it with anybody. 49 | SECRET_KEY = '' 50 | 51 | # List of callables that know how to import templates from various sources. 52 | TEMPLATE_LOADERS = ( 53 | 'django.template.loaders.filesystem.load_template_source', 54 | 'django.template.loaders.app_directories.load_template_source', 55 | ) 56 | 57 | MIDDLEWARE_CLASSES = ( 58 | 'django.middleware.common.CommonMiddleware', 59 | 'django.contrib.sessions.middleware.SessionMiddleware', 60 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 61 | ) 62 | 63 | ROOT_URLCONF = 'urls' 64 | 65 | TEMPLATE_DIRS = ( 66 | os.path.join(PROJECT_DIR, 'templates') 67 | ) 68 | 69 | INSTALLED_APPS = ( 70 | 'django.contrib.auth', 71 | 'django.contrib.contenttypes', 72 | 'django.contrib.comments', 73 | 'django.contrib.sessions', 74 | 'django.contrib.sites', 75 | 'django.contrib.admin', 76 | 'polls', 77 | 'test_app', 78 | ) 79 | -------------------------------------------------------------------------------- /test_project/test_app/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ericholscher/django-test-utils/0207f06e0ebaa5081dc8648815815929fbc9dea5/test_project/test_app/__init__.py -------------------------------------------------------------------------------- /test_project/test_app/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | # Create your models here. 4 | -------------------------------------------------------------------------------- /test_project/test_app/tests/__init__.py: -------------------------------------------------------------------------------- 1 | from assertions_tests import * 2 | from templatetags_tests import * 3 | from testmaker_tests import * 4 | from crawler_tests import * 5 | 6 | import twill_tests 7 | 8 | __test__ = { 9 | 'TWILL': twill_tests, 10 | } 11 | -------------------------------------------------------------------------------- /test_project/test_app/tests/assertions_tests.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from test_utils.assertions import DiffTestCaseMixin 3 | 4 | class TestAssertions(TestCase, DiffTestCaseMixin): 5 | """ 6 | Tests to test assertions in test utils. 7 | """ 8 | 9 | def test_assert_no_diff_dict(self): 10 | dict1 = {'I love': 'you'} 11 | dict2 = {'I love': 'moo'} 12 | try: 13 | self.failIfDiff(dict1, dict2) 14 | except AssertionError, e: 15 | self.failIfDiff(e.message, """\n--- First \n\n+++ Second \n\n@@ -1,1 +1,1 @@\n\n-'I love':'you'\n+'I love':'moo'\n""") 16 | 17 | def test_assert_no_diff_list(self): 18 | list1 = ['I love', 'you'] 19 | list2 = ['I love', 'to moo'] 20 | try: 21 | self.failIfDiff(list1, list2) 22 | except AssertionError, e: 23 | self.failIfDiff(e.message, """\n--- First \n\n+++ Second \n\n@@ -1,2 +1,2 @@\n\n 'I love'\n-'you'\n+'to moo'\n""") 24 | -------------------------------------------------------------------------------- /test_project/test_app/tests/crawler_tests.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file is to test testmaker. It will run over the polls app and with the crawler and with test maker outputting things. Hopefully this will provide a sane way to test testmaker. 3 | """ 4 | from django.test.testcases import TestCase 5 | from test_utils.crawler.base import Crawler 6 | import logging 7 | import os 8 | 9 | class CrawlerTests(TestCase): 10 | """ 11 | Tests to test the Crawler API 12 | """ 13 | urls = "test_project.polls.urls" 14 | fixtures = ['polls_testmaker.json'] 15 | 16 | def setUp(self): 17 | self.log = logging.getLogger('crawler') 18 | [self.log.removeHandler(h) for h in self.log.handlers] 19 | self.log.setLevel(logging.DEBUG) 20 | handler = logging.FileHandler('crawler_log', 'a') 21 | handler.setFormatter(logging.Formatter('%(message)s')) 22 | self.log.addHandler(handler) 23 | 24 | def tearDown(self): 25 | os.remove('crawler_log') 26 | 27 | def test_basic_crawling(self): 28 | c = Crawler('/') 29 | c.run() 30 | self.assertEqual(c.crawled, {'/': True, u'/1': True, u'/2': True}) 31 | 32 | def test_relative_crawling(self): 33 | c = Crawler('/1') 34 | c.run() 35 | self.assertEqual(c.crawled, {u'/1': True}) 36 | 37 | def test_url_plugin(self): 38 | conf_urls = {'this_wont_be_crawled': True} 39 | c = Crawler('/', conf_urls=conf_urls) 40 | c.run() 41 | logs = open('crawler_log') 42 | output = logs.read() 43 | self.assertTrue(output.find('These patterns were not matched during the crawl: this_wont_be_crawled') != -1) 44 | 45 | def test_time_plugin(self): 46 | #This isn't testing much, but I can't know how long the time will take 47 | c = Crawler('/') 48 | c.run() 49 | logs = open('crawler_log') 50 | output = logs.read() 51 | self.assertTrue(output.find('Time taken:') != -1) 52 | 53 | def test_memory_plugin(self): 54 | from test_utils.crawler.plugins.memory_plugin import Memory 55 | Memory.active = True 56 | c = Crawler('/') 57 | c.run() 58 | logs = open('crawler_log') 59 | output = logs.read() 60 | self.assertTrue(output.find('Memory consumed:') != -1) 61 | 62 | 63 | #Guppy makes the tests take a lot longer, uncomment this if you want to 64 | #test it. 65 | """ 66 | def test_guppy_plugin(self): 67 | #This isn't testing much, but I can't know how long the time will take 68 | from test_utils.crawler.plugins.guppy_plugin import ACTIVE, Heap 69 | if ACTIVE: 70 | Heap.active = True 71 | c = Crawler('/') 72 | c.run() 73 | logs = open('crawler_log') 74 | output = logs.read() 75 | import ipdb; ipdb.set_trace() 76 | self.assertTrue(output.find('heap') != -1) 77 | else: 78 | print "Skipping memory test, as guppy isn't installed" 79 | """ 80 | -------------------------------------------------------------------------------- /test_project/test_app/tests/templatetags_tests.py: -------------------------------------------------------------------------------- 1 | import os 2 | from django.test.testcases import TestCase 3 | from django.template import Context, Template 4 | from django.contrib.auth.models import User 5 | from test_utils.templatetags import TemplateParser 6 | from test_utils.testmaker import Testmaker 7 | 8 | from django.contrib.auth.models import User 9 | 10 | class Parsing(TestCase): 11 | """ 12 | Tests to test the parsing API 13 | """ 14 | 15 | def setUp(self): 16 | self.tm = Testmaker() 17 | self.tm.setup_logging('test_file', 'serialize_file') 18 | 19 | def tearDown(self): 20 | #Teardown logging somehow? 21 | os.remove('test_file') 22 | os.remove('serialize_file') 23 | 24 | def test_basic_parsing(self): 25 | user = User.objects.create_user('john', 'lennon@thebeatles.com', 'johnpassword') 26 | user.save() 27 | c = Context({'object': user}) 28 | t = TemplateParser('{% load comments %}{% get_comment_list for object as as_var %}{{ as_var }}', c) 29 | t.parse() 30 | self.assertEquals(t.template_calls[0], '{% get_comment_list for object as as_var %}') 31 | self.assertEquals(t.loaded_classes[0], '{% load comments %}') 32 | t.create_tests() 33 | logs = open('test_file') 34 | output = logs.read() 35 | self.assertTrue(output.find("{'object': get_model('auth', 'user')") != -1) 36 | -------------------------------------------------------------------------------- /test_project/test_app/tests/testmaker_tests.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file is to test testmaker. It will run over the polls app and with the crawler and with test maker outputting things. Hopefully this will provide a sane way to test testmaker. 3 | """ 4 | from django.test.testcases import TestCase 5 | from test_utils.testmaker import Testmaker 6 | from django.conf import settings 7 | import os 8 | 9 | class TestMakerTests(TestCase): 10 | """ 11 | Tests to test basic testmaker functionality. 12 | """ 13 | urls = "test_project.polls.urls" 14 | fixtures = ['polls_testmaker.json'] 15 | 16 | def setUp(self): 17 | self.tm = Testmaker() 18 | self.tm.setup_logging('test_file', 'serialize_file') 19 | Testmaker.enabled = True 20 | self.tm.insert_middleware() 21 | 22 | def tearDown(self): 23 | #Teardown logging somehow? 24 | os.remove('test_file') 25 | os.remove('serialize_file') 26 | 27 | def test_basic_testmaker(self): 28 | self.client.get('/') 29 | logs = open('test_file') 30 | output = logs.read() 31 | self.assertTrue(output.find('[, ]') != -1) 32 | 33 | def test_twill_processor(self): 34 | settings.TESTMAKER_PROCESSOR = 'twill' 35 | self.client.get('/') 36 | self.client.get('/1/') 37 | logs = open('test_file') 38 | output = logs.read() 39 | self.assertTrue(output.find('code 200') != -1) 40 | 41 | def test_not_inserting_multiple_times(self): 42 | """ 43 | Test that the middleware will only be inserted once. 44 | """ 45 | self.tm.insert_middleware() 46 | self.tm.insert_middleware() 47 | middleware = settings.MIDDLEWARE_CLASSES 48 | #A set of the middleware should be the same, meaning the item isn't in twice. 49 | self.assertEqual(sorted(list(middleware)), sorted(list(set(middleware)))) 50 | -------------------------------------------------------------------------------- /test_project/test_app/tests/twill_tests.py: -------------------------------------------------------------------------------- 1 | __doc__ = """ 2 | ### test setup() and teardown() logic 3 | 4 | >>> from test_utils.utils.twill_runner import * 5 | >>> from django.conf import settings 6 | >>> setup() 7 | <..._EasyTwillBrowser object at ...> 8 | >>> setup() # no duplicate registrations 9 | False 10 | >>> len(INSTALLED) 11 | 1 12 | >>> teardown() 13 | True 14 | >>> len(INSTALLED) 15 | 0 16 | 17 | >>> setup(host='myhost', port=40) 18 | <..._EasyTwillBrowser object at ...> 19 | >>> setup(host='myhost', port=10) 20 | <..._EasyTwillBrowser object at ...> 21 | >>> teardown(port=10) # exact match OR no arguments to pop last required 22 | False 23 | >>> teardown() # this will remove the last 24 | True 25 | >>> len(INSTALLED) # one handler is still registered 26 | 1 27 | >>> teardown(host='myhost', port=40) # remove it by exact match 28 | True 29 | >>> len(INSTALLED) 30 | 0 31 | 32 | >>> settings.DEBUG_PROPAGATE_EXCEPTIONS = False 33 | >>> setup(propagate=True) 34 | <..._EasyTwillBrowser object at ...> 35 | >>> settings.DEBUG_PROPAGATE_EXCEPTIONS 36 | True 37 | >>> teardown() 38 | True 39 | >>> settings.DEBUG_PROPAGATE_EXCEPTIONS 40 | False 41 | >>> len(INSTALLED) 42 | 0 43 | 44 | 45 | ### test relative url handling ### 46 | # Note that for simplicities sake we only 47 | # check whether our custom code appended a 48 | # host name; the twill browser base class 49 | # never gets to see the urls, and we don't 50 | # know what it makes of it. 51 | 52 | # put browser into testing mode 53 | >>> browser = get_browser() 54 | >>> browser._testing_ = True 55 | 56 | >>> setup(host='a', port=1) 57 | <..._EasyTwillBrowser object at ...> 58 | 59 | >>> browser.go('/') 60 | 'http://a:1/' 61 | >>> browser.go('/index') 62 | 'http://a:1/index' 63 | 64 | >>> browser.go('http://google.de') 65 | 'http://google.de' 66 | >>> browser.go('/services') 67 | '/services' 68 | >>> browser.go('') 69 | '' 70 | >>> browser.go('?foo=bar') 71 | '?foo=bar' 72 | 73 | >>> browser.go('/index', default=True) 74 | 'http://a:1/index' 75 | 76 | # TODO: since we don't work with real urls, we don't get anything back. Improve. 77 | >>> url() 78 | 79 | >>> teardown() 80 | True 81 | >>> len(INSTALLED) 82 | 0 83 | 84 | # leave testing mode again 85 | >>> browser._testing_ = False 86 | 87 | # TODO: test the login/logout methods 88 | """ 89 | -------------------------------------------------------------------------------- /test_project/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls.defaults import * 2 | 3 | from django.contrib import admin 4 | admin.autodiscover() 5 | 6 | urlpatterns = patterns('', 7 | # Example: 8 | (r'^polls/', include('polls.urls')), 9 | (r'^admin/doc/', include('django.contrib.admindocs.urls')), 10 | url(r'^admin/(.*)', admin.site.root, name='admin-root'), 11 | 12 | ) 13 | -------------------------------------------------------------------------------- /test_utils/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /test_utils/__init__.py: -------------------------------------------------------------------------------- 1 | VERSION = '0.3' 2 | -------------------------------------------------------------------------------- /test_utils/assertions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code originally from: http://www.aminus.org/blogs/index.php/2009/01/09/assertnodiff 3 | """ 4 | 5 | import difflib 6 | from pprint import pformat 7 | 8 | 9 | class DiffTestCaseMixin(object): 10 | 11 | def get_diff_msg(self, first, second, 12 | fromfile='First', tofile='Second'): 13 | """Return a unified diff between first and second.""" 14 | # Force inputs to iterables for diffing. 15 | # use pformat instead of str or repr to output dicts and such 16 | # in a stable order for comparison. 17 | if isinstance(first, (tuple, list)): 18 | first = [pformat(d) for d in first] 19 | elif isinstance(first, dict): 20 | first = ["%s:%s" % (pformat(key), pformat(val)) for key,val in first.iteritems()] 21 | else: 22 | first = [pformat(first)] 23 | 24 | if isinstance(second, (tuple, list)): 25 | second = [pformat(d) for d in second] 26 | elif isinstance(second, dict): 27 | second = ["%s:%s" % (pformat(key), pformat(val)) for key,val in second.iteritems()] 28 | else: 29 | second = [pformat(second)] 30 | 31 | diff = difflib.unified_diff( 32 | first, second, fromfile=fromfile, tofile=tofile) 33 | # Add line endings. 34 | return '\n' + ''.join([d + '\n' for d in diff]) 35 | 36 | def failIfDiff(self, first, second, fromfile='First', tofile='Second'): 37 | """If not first == second, fail with a unified diff.""" 38 | if not first == second: 39 | msg = self.get_diff_msg(first, second, fromfile, tofile) 40 | raise self.failureException, msg 41 | -------------------------------------------------------------------------------- /test_utils/bin/django_test_runner.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Many thanks to Brian Rosner for letting me include 4 | this code in Test Utils. 5 | """ 6 | 7 | import os 8 | import sys 9 | 10 | from optparse import OptionParser 11 | 12 | from django.conf import settings 13 | from django.core.management import call_command 14 | 15 | def main(): 16 | """ 17 | The entry point for the script. This script is fairly basic. Here is a 18 | quick example of how to use it:: 19 | 20 | django_test_runner.py [path-to-app] 21 | 22 | You must have Django on the PYTHONPATH prior to running this script. This 23 | script basically will bootstrap a Django environment for you. 24 | 25 | By default this script with use SQLite and an in-memory database. If you 26 | are using Python 2.5 it will just work out of the box for you. 27 | """ 28 | parser = OptionParser() 29 | parser.add_option("--DATABASE_ENGINE", dest="DATABASE_ENGINE", default="sqlite3") 30 | parser.add_option("--DATABASE_NAME", dest="DATABASE_NAME", default="") 31 | parser.add_option("--DATABASE_USER", dest="DATABASE_USER", default="") 32 | parser.add_option("--DATABASE_PASSWORD", dest="DATABASE_PASSWORD", default="") 33 | parser.add_option("--SITE_ID", dest="SITE_ID", type="int", default=1) 34 | 35 | options, args = parser.parse_args() 36 | 37 | # check for app in args 38 | try: 39 | app_path = args[0] 40 | except IndexError: 41 | print "You did not provide an app path." 42 | raise SystemExit 43 | else: 44 | if app_path.endswith("/"): 45 | app_path = app_path[:-1] 46 | parent_dir, app_name = os.path.split(app_path) 47 | sys.path.insert(0, parent_dir) 48 | 49 | settings.configure(**{ 50 | "DATABASE_ENGINE": options.DATABASE_ENGINE, 51 | "DATABASE_NAME": options.DATABASE_NAME, 52 | "DATABASE_USER": options.DATABASE_USER, 53 | "DATABASE_PASSWORD": options.DATABASE_PASSWORD, 54 | "SITE_ID": options.SITE_ID, 55 | "ROOT_URLCONF": "", 56 | "TEMPLATE_LOADERS": ( 57 | "django.template.loaders.filesystem.load_template_source", 58 | "django.template.loaders.app_directories.load_template_source", 59 | ), 60 | "TEMPLATE_DIRS": ( 61 | os.path.join(os.path.dirname(__file__), "templates"), 62 | ), 63 | "INSTALLED_APPS": ( 64 | # HACK: the admin app should *not* be required. Need to spend some 65 | # time looking into this. Django #8523 has a patch for this issue, 66 | # but was wrongly attached to that ticket. It should have its own 67 | # ticket. 68 | "django.contrib.admin", 69 | "django.contrib.auth", 70 | "django.contrib.contenttypes", 71 | "django.contrib.sessions", 72 | "django.contrib.sites", 73 | app_name, 74 | ), 75 | }) 76 | call_command("test") 77 | 78 | if __name__ == "__main__": 79 | main() 80 | -------------------------------------------------------------------------------- /test_utils/crawler/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ericholscher/django-test-utils/0207f06e0ebaa5081dc8648815815929fbc9dea5/test_utils/crawler/__init__.py -------------------------------------------------------------------------------- /test_utils/crawler/base.py: -------------------------------------------------------------------------------- 1 | from HTMLParser import HTMLParseError 2 | import logging 3 | import os 4 | import urlparse 5 | 6 | from django.conf import settings 7 | from django.db import transaction 8 | from django.views.debug import cleanse_setting 9 | from django.test.client import Client 10 | from django.test.utils import setup_test_environment, teardown_test_environment 11 | 12 | from test_utils.crawler import signals as test_signals 13 | from test_utils.crawler.plugins.base import Plugin 14 | 15 | LOG = logging.getLogger('crawler') 16 | 17 | try: 18 | import lxml.html 19 | def link_extractor(html): 20 | try: 21 | tree = lxml.html.document_fromstring(html) 22 | except lxml.etree.ParseError, e: 23 | raise HTMLParseError(str(e), e.position) 24 | 25 | for element, attribute, link, pos in tree.iterlinks(): 26 | yield link 27 | except ImportError: 28 | LOG.info("Processing documents with HTMLParser; install lxml for greater performance") 29 | 30 | from HTMLParser import HTMLParser 31 | 32 | def link_extractor(html): 33 | class LinkExtractor(HTMLParser): 34 | links = set() 35 | 36 | def handle_starttag(self, tag, attrs): 37 | self.links.update( 38 | v for k, v in attrs if k == "href" or k =="src" 39 | ) 40 | 41 | parser = LinkExtractor() 42 | parser.feed(html) 43 | parser.close() 44 | 45 | return parser.links 46 | 47 | 48 | class Crawler(object): 49 | """ 50 | This is a class that represents a URL crawler in python 51 | """ 52 | 53 | def __init__(self, base_url, conf_urls={}, verbosity=1, output_dir=None, ascend=True, **kwargs): 54 | self.base_url = base_url 55 | self.conf_urls = conf_urls 56 | self.verbosity = verbosity 57 | self.ascend = ascend 58 | 59 | auth = kwargs.get('auth') 60 | 61 | if output_dir: 62 | assert os.path.isdir(output_dir) 63 | self.output_dir = os.path.realpath(output_dir) 64 | LOG.info("Output will be saved to %s" % self.output_dir) 65 | else: 66 | self.output_dir = None 67 | 68 | #These two are what keep track of what to crawl and what has been. 69 | self.not_crawled = [(0, 'START',self.base_url)] 70 | self.crawled = {} 71 | 72 | self.c = Client(REMOTE_ADDR='127.0.0.1') 73 | 74 | if auth: 75 | printable_auth = ', '.join( 76 | '%s: %s' % (key, cleanse_setting(key.upper(), value)) 77 | for key, value in auth.items()) 78 | LOG.info('Log in with %s' % printable_auth) 79 | self.c.login(**auth) 80 | 81 | self.plugins = [] 82 | for plug in Plugin.__subclasses__(): 83 | active = getattr(plug, 'active', True) 84 | if active: 85 | #TODO: Check if plugin supports writing CSV (or to a file in general?) 86 | self.plugins.append(plug()) 87 | 88 | def _parse_urls(self, url, resp): 89 | parsed = urlparse.urlparse(url) 90 | 91 | if resp['Content-Type'] == "text/html; charset=utf-8": 92 | html = resp.content.decode("utf-8") 93 | else: 94 | html = resp.content 95 | 96 | returned_urls = [] 97 | 98 | for link in link_extractor(html): 99 | parsed_href = urlparse.urlparse(link) 100 | 101 | if not parsed_href.path: 102 | continue 103 | 104 | if parsed_href.scheme and not parsed_href.netloc.startswith("testserver"): 105 | LOG.debug("Skipping external link: %s", link) 106 | continue 107 | 108 | if parsed_href.path.startswith(settings.STATIC_URL) or \ 109 | parsed_href.path.startswith(settings.MEDIA_URL): 110 | LOG.debug("Skipping static/media link: %s", link) 111 | continue 112 | 113 | if parsed_href.path.startswith('/'): 114 | returned_urls.append(link) 115 | else: 116 | # We'll use urlparse's urljoin since that handles things like 117 | returned_urls.append(urlparse.urljoin(url, link)) 118 | 119 | return returned_urls 120 | 121 | def get_url(self, from_url, to_url): 122 | """ 123 | Takes a url, and returns it with a list of links 124 | This uses the Django test client. 125 | """ 126 | parsed = urlparse.urlparse(to_url) 127 | request_dict = dict(urlparse.parse_qsl(parsed.query)) 128 | url_path = parsed.path 129 | 130 | #url_path now contains the path, request_dict contains get params 131 | 132 | LOG.debug("%s: link to %s with parameters %s", from_url, to_url, request_dict) 133 | 134 | test_signals.pre_request.send(self, url=to_url, request_dict=request_dict) 135 | 136 | resp = self.c.get(url_path, request_dict, follow=False) 137 | 138 | test_signals.post_request.send(self, url=to_url, response=resp) 139 | 140 | if resp.status_code in (301, 302): 141 | location = resp["Location"] 142 | if location.startswith("http://testserver"): 143 | LOG.debug("%s: following redirect to %s", to_url, location) 144 | # Mmm, recursion TODO: add a max redirects limit? 145 | return self.get_url(from_url, location) 146 | else: 147 | LOG.info("%s: not following off-site redirect to %s", to_url, location) 148 | return (resp, ()) 149 | elif 400 <= resp.status_code < 600: 150 | # We'll avoid logging a warning for HTTP statuses which aren't in the 151 | # official error ranges: 152 | LOG.warning("%s links to %s, which returned HTTP status %d", from_url, url_path, resp.status_code) 153 | return (resp, ()) 154 | 155 | if resp['Content-Type'].startswith("text/html"): 156 | returned_urls = self._parse_urls(to_url, resp) 157 | test_signals.urls_parsed.send(self, fro=to_url, returned_urls=returned_urls) 158 | else: 159 | returned_urls = list() 160 | 161 | return (resp, returned_urls) 162 | 163 | def run(self, max_depth=3): 164 | for p in self.plugins: 165 | p.set_output_dir(self.output_dir) 166 | 167 | old_DEBUG = settings.DEBUG 168 | settings.DEBUG = False 169 | 170 | setup_test_environment() 171 | test_signals.start_run.send(self) 172 | 173 | # To avoid tainting our memory usage stats with startup overhead we'll 174 | # do one extra request for the first page now: 175 | self.c.get(*self.not_crawled[0][-1]) 176 | 177 | while self.not_crawled: 178 | #Take top off not_crawled and evaluate it 179 | current_depth, from_url, to_url = self.not_crawled.pop(0) 180 | if current_depth > max_depth: 181 | continue 182 | 183 | transaction.enter_transaction_management() 184 | try: 185 | resp, returned_urls = self.get_url(from_url, to_url) 186 | except HTMLParseError, e: 187 | LOG.error("%s: unable to parse invalid HTML: %s", to_url, e) 188 | except Exception, e: 189 | LOG.exception("%s had unhandled exception: %s", to_url, e) 190 | continue 191 | finally: 192 | transaction.rollback() 193 | 194 | self.crawled[to_url] = True 195 | #Find its links that haven't been crawled 196 | for base_url in returned_urls: 197 | if not self.ascend and not base_url.startswith(self.base_url): 198 | LOG.debug("Skipping %s - outside scope of %s", base_url, self.base_url) 199 | continue 200 | 201 | if base_url not in [to for dep,fro,to in self.not_crawled] and not self.crawled.has_key(base_url): 202 | self.not_crawled.append((current_depth+1, to_url, base_url)) 203 | 204 | test_signals.finish_run.send(self) 205 | 206 | teardown_test_environment() 207 | 208 | settings.DEBUG = old_DEBUG 209 | -------------------------------------------------------------------------------- /test_utils/crawler/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | from base import Plugin 2 | 3 | from time_plugin import Time 4 | from pdb import Pdb 5 | from urlconf import URLConf 6 | -------------------------------------------------------------------------------- /test_utils/crawler/plugins/base.py: -------------------------------------------------------------------------------- 1 | from test_utils.crawler import signals as test_signals 2 | 3 | class Plugin(object): 4 | """ 5 | This is a class to represent a plugin to the Crawler. 6 | Subclass it and define a start or stop function to be called on requests. 7 | Define a print_report function if your plugin outputs at the end of the run. 8 | """ 9 | global_data = {} 10 | 11 | def __init__(self): 12 | #This should be refactored to call each of the subclasses. 13 | #Having them use the signal function signature is hacky.. 14 | 15 | if hasattr(self, 'pre_request'): 16 | test_signals.pre_request.connect(self.pre_request) 17 | if hasattr(self, 'post_request'): 18 | test_signals.post_request.connect(self.post_request) 19 | if hasattr(self, 'start_run'): 20 | test_signals.start_run.connect(self.start_run) 21 | if hasattr(self, 'finish_run'): 22 | test_signals.finish_run.connect(self.finish_run) 23 | if hasattr(self, 'urls_parsed'): 24 | test_signals.urls_parsed.connect(self.urls_parsed) 25 | 26 | self.data = self.global_data[self.__class__.__name__] = {} 27 | 28 | # This will be updated when a run starts if the user wants output to 29 | # be saved: 30 | self.output_dir = None 31 | 32 | """ 33 | #These functions enable instance['test'] to save to instance.data 34 | def __setitem__(self, key, val): 35 | self.global_data[self.__class__.__name__][key] = val 36 | 37 | def __getitem__(self, key): 38 | return self.global_data[self.__class__.__name__][key] 39 | """ 40 | 41 | def set_output_dir(self, output_dir): 42 | """ 43 | Extension point for subclasses to open files, create directories, etc. 44 | """ 45 | 46 | self.output_dir = output_dir -------------------------------------------------------------------------------- /test_utils/crawler/plugins/graph.py: -------------------------------------------------------------------------------- 1 | from base import Plugin 2 | 3 | class Graph(Plugin): 4 | "Make pretty graphs of your requests" 5 | active = False 6 | 7 | def __init__(self): 8 | super(Graph, self).__init__() 9 | self.request_graph = self.data['request_graph'] = {} 10 | import pygraphviz 11 | self.graph = pygraphviz.AGraph(directed=True) 12 | 13 | def urls_parsed(self, sender, fro, returned_urls, **kwargs): 14 | from_node = self.graph.add_node(str(fro), shape='tripleoctagon') 15 | for url in returned_urls: 16 | if not self.graph.has_node(str(url)): 17 | node = self.graph.add_node(str(url)) 18 | self.graph.add_edge(str(fro), str(url)) 19 | 20 | def finish_run(self, sender, **kwargs): 21 | print "Making graph of your URLs, this may take a while" 22 | self.graph.layout(prog='fdp') 23 | self.graph.draw('my_urls.png') 24 | 25 | PLUGIN = Graph 26 | -------------------------------------------------------------------------------- /test_utils/crawler/plugins/guppy_plugin.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import logging 3 | import os 4 | 5 | from django.template.defaultfilters import filesizeformat 6 | 7 | from guppy import hpy 8 | 9 | from base import Plugin 10 | 11 | LOG = logging.getLogger("crawler") 12 | 13 | 14 | class Heap(Plugin): 15 | """ 16 | Calculate heap consumed before and after request 17 | """ 18 | 19 | def __init__(self): 20 | super(Heap, self).__init__() 21 | self.heap_urls = self.data['heap_urls'] = {} 22 | self.hp = hpy() 23 | self.csv_writer = None 24 | 25 | def set_output_dir(self, output_dir=None): 26 | super(Heap, self).set_output_dir(output_dir) 27 | 28 | if output_dir: 29 | self.csv_writer = csv.writer(open(os.path.join(output_dir, 'heap.csv'), 'w')) 30 | 31 | def pre_request(self, sender, **kwargs): 32 | url = kwargs['url'] 33 | self.hp.setrelheap() 34 | 35 | def post_request(self, sender, **kwargs): 36 | url = kwargs['url'] 37 | heap = self.hp.heap() 38 | self.heap_urls[url] = heap.size 39 | 40 | LOG.debug("%s: heap consumed: %s", url, filesizeformat(self.heap_urls[url])) 41 | 42 | if self.csv_writer: 43 | self.csv_writer.writerow([url, heap.size]) 44 | 45 | def finish_run(self, sender, **kwargs): 46 | "Print the most heap consumed by a view" 47 | 48 | alist = sorted(self.heap_urls.iteritems(), 49 | key=lambda (k,v): (v,k), 50 | reverse=True 51 | ) 52 | 53 | for url, mem in alist[:10]: 54 | LOG.info("%s: %s heap", url, filesizeformat(mem)) 55 | 56 | 57 | PLUGIN = Heap -------------------------------------------------------------------------------- /test_utils/crawler/plugins/memory_plugin.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import logging 3 | import os 4 | 5 | from base import Plugin 6 | 7 | LOG = logging.getLogger("crawler") 8 | 9 | # from python mailing list http://mail.python.org/pipermail/python-list/2004-June/266257.html 10 | _proc_status = '/proc/%d/status' % os.getpid() # Linux only? 11 | _scale = {'kB': 1024.0, 'mB': 1024.0*1024.0, 12 | 'KB': 1024.0, 'MB': 1024.0*1024.0} 13 | 14 | def _VmB(VmKey): 15 | global _scale 16 | try: # get the /proc//status pseudo file 17 | t = open(_proc_status) 18 | v = [v for v in t.readlines() if v.startswith(VmKey)] 19 | t.close() 20 | # convert Vm value to bytes 21 | if len(v) == 1: 22 | t = v[0].split() # e.g. 'VmRSS: 9999 kB' 23 | if len(t) == 3: ## and t[0] == VmKey: 24 | return float(t[1]) * _scale.get(t[2], 0.0) 25 | except: 26 | pass 27 | return 0.0 28 | 29 | def memory(since=0.0): 30 | '''Return process memory usage in bytes. 31 | ''' 32 | return _VmB('VmSize:') - since 33 | 34 | def stacksize(since=0.0): 35 | '''Return process stack size in bytes. 36 | ''' 37 | return _VmB('VmStk:') - since 38 | 39 | 40 | class Memory(Plugin): 41 | """ 42 | Calculate proc memory consumed before and after request 43 | """ 44 | active = False 45 | 46 | def __init__(self, write_csv=False): 47 | super(Memory, self).__init__() 48 | self.memory_urls = self.data['memory_urls'] = {} 49 | self.write_csv = write_csv 50 | if self.write_csv: 51 | self.csv_writer = csv.writer(open('memory.csv', 'w')) 52 | 53 | def pre_request(self, sender, **kwargs): 54 | url = kwargs['url'] 55 | self.memory_urls[url] = memory() 56 | 57 | def post_request(self, sender, **kwargs): 58 | cur = memory() 59 | url = kwargs['url'] 60 | old_memory = self.memory_urls[url] 61 | total_memory = cur - old_memory 62 | self.memory_urls[url] = total_memory 63 | LOG.info("Memory consumed: %s", self.memory_urls[url]) 64 | if self.write_csv: 65 | self.csv_writer.writerow([url,cur, old_memory, total_memory]) 66 | 67 | def finish_run(self, sender, **kwargs): 68 | "Print the most memory consumed by a view" 69 | alist = sorted(self.memory_urls.iteritems(), key=lambda (k,v): (v,k), reverse=True) 70 | for url, mem in alist[:10]: 71 | LOG.info("%s took %f of memory", url, mem) 72 | 73 | PLUGIN = Memory -------------------------------------------------------------------------------- /test_utils/crawler/plugins/pdb.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from base import Plugin 4 | 5 | LOG = logging.getLogger("crawler") 6 | 7 | class Pdb(Plugin): 8 | "Run pdb on fail" 9 | active = False 10 | 11 | def post_request(self, sender, **kwargs): 12 | url = kwargs['url'] 13 | resp = kwargs['response'] 14 | if hasattr(resp, 'status_code'): 15 | if not resp.status_code in (200, 302, 301): 16 | LOG.error("%s: Status Code: %s", url, resp.status_code) 17 | try: 18 | import ipdb; ipdb.set_trace() 19 | except ImportError: 20 | import pdb; pdb.set_trace() 21 | 22 | PLUGIN = Pdb -------------------------------------------------------------------------------- /test_utils/crawler/plugins/query_count.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import logging 3 | import os 4 | 5 | from django.conf import settings 6 | from django.db import connections 7 | 8 | from base import Plugin 9 | 10 | 11 | LOG = logging.getLogger('crawler') 12 | 13 | 14 | class QueryCount(Plugin): 15 | """ 16 | Report the number of queries used to serve a page 17 | """ 18 | 19 | def __init__(self): 20 | super(QueryCount, self).__init__() 21 | 22 | self.csv_writer = None 23 | 24 | self.query_counts = self.data['query_counts'] = {} 25 | 26 | # Horrible monkey-patch to log query counts when DEBUG = False: 27 | for conn in connections.all(): 28 | conn.dtu_query_count = 0 29 | self._monkey_cursor_execute(conn) 30 | 31 | def _monkey_cursor_execute(self, conn): 32 | old_cursor = conn.cursor 33 | def new_cursor(*args, **kwargs): 34 | c = old_cursor(*args, **kwargs) 35 | 36 | old_execute = c.execute 37 | def new_execute(*args, **kwargs): 38 | try: 39 | return old_execute(*args, **kwargs) 40 | finally: 41 | conn.dtu_query_count += 1 42 | c.execute = new_execute 43 | 44 | old_executemany = c.executemany 45 | def new_executemany(s, sql, param_list, *args, **kwargs): 46 | try: 47 | return old_executemany(s, sql, param_list, *args, **kwargs) 48 | finally: 49 | conn.dtu_query_count += len(param_list) 50 | c.executemany = new_executemany 51 | 52 | return c 53 | 54 | conn.cursor = new_cursor 55 | 56 | def set_output_dir(self, output_dir=None): 57 | super(QueryCount, self).set_output_dir(output_dir) 58 | 59 | if output_dir: 60 | self.csv_writer = csv.writer(open(os.path.join(output_dir, 'query_counts.csv'), 'w')) 61 | 62 | def pre_request(self, sender, **kwargs): 63 | url = kwargs['url'] 64 | self.query_counts[url] = dict((c.alias, c.dtu_query_count) for c in connections.all()) 65 | 66 | def post_request(self, sender, **kwargs): 67 | url = kwargs['url'] 68 | 69 | new_query_counts = [(c.alias, c.dtu_query_count) for c in connections.all()] 70 | 71 | deltas = {} 72 | for k, v in new_query_counts: 73 | # Skip inactive connections: 74 | delta = v - self.query_counts[url][k] 75 | if delta > 0: 76 | deltas[k] = delta 77 | 78 | for k, v in sorted(deltas.items(), reverse=True): 79 | if v > 50: 80 | log_f = LOG.critical 81 | elif v > 20: 82 | log_f = LOG.error 83 | elif v > 10: 84 | log_f = LOG.warning 85 | else: 86 | log_f = LOG.info 87 | log_f("%s: %s %d queries", url, k, v) 88 | 89 | if self.csv_writer: 90 | self.csv_writer.writerow((url, sum(deltas.values()))) 91 | 92 | 93 | PLUGIN = QueryCount -------------------------------------------------------------------------------- /test_utils/crawler/plugins/sanitize.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from BeautifulSoup import BeautifulSoup 4 | 5 | from base import Plugin 6 | 7 | LOG = logging.getLogger("crawler") 8 | 9 | class Sanitize(Plugin): 10 | "Make sure your response is good" 11 | 12 | def post_request(self, sender, response, **kwargs): 13 | if not response['Content-Type'].startswith("text/html"): 14 | return 15 | 16 | if response['Content-Type'] == "text/html; charset=utf-8": 17 | html = response.content.decode("utf-8") 18 | else: 19 | html = response.content 20 | 21 | try: 22 | soup = BeautifulSoup(html) 23 | if soup.find(text='<') or soup.find(text='>'): 24 | LOG.warning("%s has dirty html", kwargs['url']) 25 | except Exception, e: 26 | # TODO: Derive unique names so we can continue after errors without clobbering past error pages 27 | fo = open("temp.html", 'w') 28 | fo.write(kwargs['response'].content) 29 | fo.close() 30 | LOG.error('Saved bad html to file temp.html') 31 | raise e 32 | 33 | 34 | PLUGIN = Sanitize -------------------------------------------------------------------------------- /test_utils/crawler/plugins/tidy.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | HTML validation plugin which uses Jason Stitt's pytidylib wrapper for HTML Tidy 4 | 5 | Prerequisites: 6 | tidylib: http://tidy.sourceforge.net/ 7 | pytidylib: http://countergram.com/software/pytidylib/ 8 | """ 9 | 10 | import logging 11 | import re 12 | 13 | import tidylib 14 | 15 | from base import Plugin 16 | 17 | # Based on http://stackoverflow.com/questions/92438/stripping-non-printable-characters-from-a-string-in-python 18 | # 19 | # We omit chars 9-13 (tab, newline, vertical tab, form feed, return) and 32 20 | # (space) to avoid clogging our reports with warnings about common, 21 | # non-problematic codes but still allow stripping things which will cause most 22 | # XML parsers to choke 23 | 24 | CONTROL_CHAR_RE = re.compile('[%s]' % "".join( 25 | re.escape(unichr(c)) for c in range(0, 8) + range(14, 31) + range(127, 160) 26 | )) 27 | 28 | LOG = logging.getLogger("crawler") 29 | 30 | class Tidy(Plugin): 31 | "Make sure your response is good" 32 | 33 | def post_request(self, sender, response, url=None, **kwargs): 34 | if not response['Content-Type'].startswith("text/html"): 35 | return 36 | 37 | # Check for redirects to avoid validation errors for empty responses: 38 | if response.status_code in (301, 302): 39 | return 40 | elif 400 <= response.status_code < 600: 41 | # We'll still validate error pages (they have user-written HTML, too) 42 | LOG.warning( 43 | "%s: Validating HTTP %d error page", 44 | url, 45 | response.status_code 46 | ) 47 | elif response.status_code != 200: 48 | LOG.warning( 49 | "%s: Validating unusual HTTP %d response", 50 | url, 51 | response.status_code 52 | ) 53 | 54 | # TODO: Decide how to handle character encodings more 55 | # intelligently - sniff? Scream bloody murder if charset isn't in 56 | # the HTTP headers? 57 | if response['Content-Type'] == "text/html; charset=utf-8": 58 | html = response.content.decode("utf-8") 59 | else: 60 | html = response.content 61 | 62 | if not html: 63 | LOG.error("%s: not processing empty response", url) 64 | return 65 | 66 | # First, deal with embedded control codes: 67 | html, sub_count = CONTROL_CHAR_RE.subn(" ", html) 68 | if sub_count: 69 | LOG.warning("%s: Stripped %d control characters from body: %s", 70 | url, 71 | sub_count, 72 | set(hex(ord(i)) for i in CONTROL_CHAR_RE.findall(html)) 73 | ) 74 | 75 | tidied_html, messages = tidylib.tidy_document( 76 | html.strip(), 77 | { 78 | "char-encoding": "utf8", 79 | "clean": False, 80 | "drop-empty-paras": False, 81 | "drop-font-tags": False, 82 | "drop-proprietary-attributes": False, 83 | "fix-backslash": False, 84 | "indent": False, 85 | "output-xhtml": False, 86 | } 87 | ) 88 | 89 | messages = filter(None, (l.strip() for l in messages.split("\n"))) 90 | 91 | errors = [] 92 | warnings = [] 93 | 94 | for msg in messages: 95 | if "Error:" in msg: 96 | errors.append(msg) 97 | else: 98 | warnings.append(msg) 99 | 100 | if errors: 101 | LOG.error( 102 | "%s: HTML validation errors:\n\t%s", 103 | url, 104 | "\n\t".join(errors) 105 | ) 106 | 107 | if warnings: 108 | LOG.warning( 109 | "%s: HTML validation warnings:\n\t%s", 110 | url, 111 | "\n\t".join(warnings) 112 | ) 113 | 114 | 115 | PLUGIN = Tidy -------------------------------------------------------------------------------- /test_utils/crawler/plugins/time_plugin.py: -------------------------------------------------------------------------------- 1 | import time 2 | import logging 3 | import csv 4 | import os 5 | 6 | from base import Plugin 7 | 8 | LOG = logging.getLogger('crawler') 9 | 10 | class Time(Plugin): 11 | """ 12 | Follow the time it takes to run requests. 13 | """ 14 | csv_writer = None 15 | 16 | def __init__(self): 17 | super(Time, self).__init__() 18 | self.timed_urls = self.data['timed_urls'] = {} 19 | 20 | def set_output_dir(self, output_dir=None): 21 | super(Time, self).set_output_dir(output_dir) 22 | 23 | if output_dir: 24 | self.csv_writer = csv.writer(open(os.path.join(output_dir, 'url_times.csv'), 'w')) 25 | 26 | def pre_request(self, sender, **kwargs): 27 | url = kwargs['url'] 28 | self.timed_urls[url] = time.time() 29 | 30 | def post_request(self, sender, **kwargs): 31 | cur = time.time() 32 | url = kwargs['url'] 33 | old_time = self.timed_urls[url] 34 | total_time = cur - old_time 35 | self.timed_urls[url] = total_time 36 | LOG.debug("Time taken: %s", self.timed_urls[url]) 37 | 38 | if self.csv_writer: 39 | self.csv_writer.writerow((url, self.timed_urls[url])) 40 | 41 | def finish_run(self, sender, **kwargs): 42 | "Print the longest time it took for pages to load" 43 | alist = sorted(self.timed_urls.iteritems(), key=lambda (k,v): (v,k), reverse=True) 44 | for url, ttime in alist[:10]: 45 | LOG.info("%s took %f", url, ttime) 46 | 47 | PLUGIN = Time -------------------------------------------------------------------------------- /test_utils/crawler/plugins/urlconf.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import re 3 | 4 | from base import Plugin 5 | 6 | LOG = logging.getLogger("crawler") 7 | 8 | class URLConf(Plugin): 9 | """ 10 | Plugin to check validity of URLConf. 11 | Run after the spider is done to show what URLConf entries got hit. 12 | """ 13 | 14 | def finish_run(self, sender, **kwargs): 15 | normal_patterns = list() 16 | admin_patterns = list() 17 | 18 | for pattern in sender.conf_urls.keys(): 19 | pattern = pattern.replace('^', '').replace('$', '').replace('//', '/') 20 | curr = re.compile(pattern) 21 | 22 | if any(curr.search(url) for url in sender.crawled): 23 | continue 24 | 25 | if pattern.startswith("admin"): 26 | admin_patterns.append(pattern) 27 | else: 28 | normal_patterns.append(pattern) 29 | 30 | if admin_patterns: 31 | LOG.debug("These admin pages were not crawled: %s", "\n\t".join(sorted(admin_patterns))) 32 | 33 | if normal_patterns: 34 | LOG.info("These patterns were not matched during the crawl: %s", "\n\t".join(sorted(normal_patterns))) 35 | 36 | PLUGIN = URLConf -------------------------------------------------------------------------------- /test_utils/crawler/signals.py: -------------------------------------------------------------------------------- 1 | import django.dispatch 2 | 3 | pre_request = django.dispatch.Signal(providing_args=['url', 'request']) 4 | post_request = django.dispatch.Signal(providing_args=['url', 'response']) 5 | urls_parsed = django.dispatch.Signal(providing_args=['fro', 'returned_urls']) 6 | start_run = django.dispatch.Signal() 7 | finish_run = django.dispatch.Signal() 8 | -------------------------------------------------------------------------------- /test_utils/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ericholscher/django-test-utils/0207f06e0ebaa5081dc8648815815929fbc9dea5/test_utils/management/__init__.py -------------------------------------------------------------------------------- /test_utils/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ericholscher/django-test-utils/0207f06e0ebaa5081dc8648815815929fbc9dea5/test_utils/management/commands/__init__.py -------------------------------------------------------------------------------- /test_utils/management/commands/crawlurls.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | from optparse import make_option 3 | import logging 4 | import sys 5 | 6 | from django.conf import settings 7 | from django.core.management.base import BaseCommand, CommandError 8 | from django.contrib.admindocs.views import extract_views_from_urlpatterns 9 | 10 | from test_utils.crawler.base import Crawler 11 | 12 | class LogStatsHandler(logging.Handler): 13 | stats = defaultdict(int) 14 | 15 | def emit(self, record): 16 | self.stats[record.levelno] += 1 17 | 18 | class Command(BaseCommand): 19 | option_list = BaseCommand.option_list + ( 20 | make_option('-p', '--pdb', action='store_true', dest='pdb', default=False, 21 | help='Pass -p to drop into pdb on an error'), 22 | make_option('-d', '--depth', action='store', dest='depth', default=3, 23 | help='Specify the depth to crawl.'), 24 | make_option('-s', '--safe', action='store_true', dest='html', default=False, 25 | help='Pass -s to check for html fragments in your pages.'), 26 | make_option('-r', '--response', action='store_true', dest='response', default=False, 27 | help='Pass -r to store the response objects.'), 28 | make_option('-t', '--time', action='store_true', dest='time', default=False, 29 | help='Pass -t to time your requests.'), 30 | make_option('--enable-plugin', action='append', dest='plugins', default=[], 31 | help='Enable the specified plugin'), 32 | make_option("-o", '--output-dir', action='store', dest='output_dir', default=None, 33 | help='If specified, store plugin output in the provided directory'), 34 | make_option('--no-parent', action='store_true', dest="no_parent", default=False, 35 | help='Do not crawl URLs which do not start with your base URL'), 36 | make_option('-a', "--auth", action='store', dest='auth', default=None, 37 | help='Authenticate (login:user,password:secret) before crawl') 38 | ) 39 | 40 | help = "Displays all of the url matching routes for the project." 41 | args = "[relative start url]" 42 | 43 | def handle(self, *args, **options): 44 | verbosity = int(options.get('verbosity', 1)) 45 | depth = int(options.get('depth', 3)) 46 | 47 | auth = _parse_auth(options.get('auth')) 48 | 49 | if verbosity > 1: 50 | log_level = logging.DEBUG 51 | elif verbosity: 52 | log_level = logging.INFO 53 | else: 54 | log_level = logging.WARN 55 | 56 | crawl_logger = logging.getLogger('crawler') 57 | crawl_logger.setLevel(logging.DEBUG) 58 | crawl_logger.propagate = 0 59 | 60 | log_stats = LogStatsHandler() 61 | 62 | crawl_logger.addHandler(log_stats) 63 | 64 | console = logging.StreamHandler() 65 | console.setLevel(log_level) 66 | console.setFormatter(logging.Formatter("%(name)s [%(levelname)s] %(module)s: %(message)s")) 67 | 68 | crawl_logger.addHandler(console) 69 | 70 | if len(args) > 1: 71 | raise CommandError('Only one start url is currently supported.') 72 | else: 73 | start_url = args[0] if args else '/' 74 | 75 | if getattr(settings, 'ADMIN_FOR', None): 76 | settings_modules = [__import__(m, {}, {}, ['']) for m in settings.ADMIN_FOR] 77 | else: 78 | settings_modules = [settings] 79 | 80 | conf_urls = {} 81 | 82 | # Build the list URLs to test from urlpatterns: 83 | for settings_mod in settings_modules: 84 | try: 85 | urlconf = __import__(settings_mod.ROOT_URLCONF, {}, {}, ['']) 86 | except Exception, e: 87 | logging.exception("Error occurred while trying to load %s: %s", settings_mod.ROOT_URLCONF, str(e)) 88 | continue 89 | 90 | view_functions = extract_views_from_urlpatterns(urlconf.urlpatterns) 91 | for (func, regex, namespace, name) in view_functions: 92 | #Get function name and add it to the hash of URLConf urls 93 | func_name = hasattr(func, '__name__') and func.__name__ or repr(func) 94 | conf_urls[regex] = ['func.__module__', func_name] 95 | 96 | c = Crawler(start_url, 97 | conf_urls=conf_urls, 98 | verbosity=verbosity, 99 | output_dir=options.get("output_dir"), 100 | ascend=not options.get("no_parent"), 101 | auth=auth, 102 | ) 103 | 104 | # Load plugins: 105 | for p in options['plugins']: 106 | # This nested try is somewhat unsightly but allows easy Pythonic 107 | # usage ("--enable-plugin=tidy") instead of Java-esque 108 | # "--enable-plugin=test_utils.crawler.plugins.tidy" 109 | try: 110 | try: 111 | plugin_module = __import__(p) 112 | except ImportError: 113 | if not "." in p: 114 | plugin_module = __import__( 115 | "test_utils.crawler.plugins.%s" % p, 116 | fromlist=["test_utils.crawler.plugins"] 117 | ) 118 | else: 119 | raise 120 | 121 | c.plugins.append(plugin_module.PLUGIN()) 122 | except (ImportError, AttributeError), e: 123 | crawl_logger.critical("Unable to load plugin %s: %s", p, e) 124 | sys.exit(3) 125 | 126 | c.run(max_depth=depth) 127 | 128 | # We'll exit with a non-zero status if we had any errors 129 | max_log_level = max(log_stats.stats.keys()) 130 | if max_log_level >= logging.ERROR: 131 | sys.exit(2) 132 | elif max_log_level >= logging.WARNING: 133 | sys.exit(1) 134 | else: 135 | sys.exit(0) 136 | 137 | 138 | def _parse_auth(auth): 139 | """ 140 | Parse auth string and return dict. 141 | 142 | >>> _parse_auth('login:user,password:secret') 143 | {'login': 'user', 'password': 'secret'} 144 | 145 | >>> _parse_auth('name:user, token:top:secret') 146 | {'name': 'user', 'token': 'top:secret'} 147 | """ 148 | if not auth: 149 | return None 150 | items = auth.split(',') 151 | return dict(i.strip().split(':', 1) for i in items) 152 | -------------------------------------------------------------------------------- /test_utils/management/commands/makefixture.py: -------------------------------------------------------------------------------- 1 | """ 2 | "Make fixture" command. 3 | 4 | Highly useful for making test fixtures. Use it to pick only few items 5 | from your data to serialize, restricted by primary keys. By default 6 | command also serializes foreign keys and m2m relations. You can turn 7 | off related items serialization with --skip-related option. 8 | 9 | How to use: 10 | python manage.py makefixture 11 | 12 | will display what models are installed 13 | 14 | python manage.py makefixture User[:3] 15 | or 16 | python manage.py makefixture auth.User[:3] 17 | or 18 | python manage.py makefixture django.contrib.auth.User[:3] 19 | 20 | will serialize users with ids 1 and 2, with assigned groups, permissions 21 | and content types. 22 | 23 | python manage.py makefixture YourModel[3] YourModel[6:10] 24 | 25 | will serialize YourModel with key 3 and keys 6 to 9 inclusively. 26 | 27 | Of course, you can serialize whole tables, and also different tables at 28 | once, and use options of dumpdata: 29 | 30 | python manage.py makefixture --format=xml --indent=4 YourModel[3] AnotherModel auth.User[:5] auth.Group 31 | """ 32 | # From http://www.djangosnippets.org/snippets/918/ 33 | 34 | #save into anyapp/management/commands/makefixture.py 35 | #or back into django/core/management/commands/makefixture.py 36 | #v0.1 -- current version 37 | #known issues: 38 | #no support for generic relations 39 | #no support for one-to-one relations 40 | from optparse import make_option 41 | from django.core import serializers 42 | from django.core.management.base import BaseCommand 43 | from django.core.management.base import CommandError 44 | from django.core.management.base import LabelCommand 45 | from django.db.models.fields.related import ForeignKey 46 | from django.db.models.fields.related import ManyToManyField 47 | from django.db.models.loading import get_models 48 | 49 | DEBUG = False 50 | 51 | def model_name(m): 52 | module = m.__module__.split('.')[:-1] # remove .models 53 | return ".".join(module + [m._meta.object_name]) 54 | 55 | class Command(LabelCommand): 56 | help = 'Output the contents of the database as a fixture of the given format.' 57 | args = 'modelname[pk] or modelname[id1:id2] repeated one or more times' 58 | option_list = BaseCommand.option_list + ( 59 | make_option('--skip-related', default=True, action='store_false', dest='propagate', 60 | help='Specifies if we shall not add related objects.'), 61 | make_option('--reverse', default=[], action='append', dest='reverse', 62 | help="Reverse relations to follow (e.g. 'Job.task_set')."), 63 | make_option('--format', default='json', dest='format', 64 | help='Specifies the output serialization format for fixtures.'), 65 | make_option('--indent', default=None, dest='indent', type='int', 66 | help='Specifies the indent level to use when pretty-printing output'), 67 | ) 68 | def handle_reverse(self, **options): 69 | follow_reverse = options.get('reverse', []) 70 | to_reverse = {} 71 | for arg in follow_reverse: 72 | try: 73 | model_name, related_set_name = arg.rsplit(".", 1) 74 | except: 75 | raise CommandError("Bad fieldname on '--reverse %s'" % arg) 76 | model = self.get_model_from_name(model_name) 77 | try: 78 | getattr(model, related_set_name) 79 | except AttributeError: 80 | raise CommandError("Field '%s' does not exist on model '%s'" % ( 81 | related_set_name, model_name)) 82 | to_reverse.setdefault(model, []).append(related_set_name) 83 | return to_reverse 84 | 85 | def handle_models(self, models, **options): 86 | format = options.get('format','json') 87 | indent = options.get('indent',None) 88 | show_traceback = options.get('traceback', False) 89 | propagate = options.get('propagate', True) 90 | follow_reverse = self.handle_reverse(**options) 91 | 92 | # Check that the serialization format exists; this is a shortcut to 93 | # avoid collating all the objects and _then_ failing. 94 | if format not in serializers.get_public_serializer_formats(): 95 | raise CommandError("Unknown serialization format: %s" % format) 96 | 97 | try: 98 | serializers.get_serializer(format) 99 | except KeyError: 100 | raise CommandError("Unknown serialization format: %s" % format) 101 | 102 | objects = [] 103 | for model, slice in models: 104 | if isinstance(slice, basestring) and slice: 105 | objects.extend(model._default_manager.filter(pk__exact=slice)) 106 | elif not slice or type(slice) is list: 107 | items = model._default_manager.all() 108 | if slice and slice[0]: 109 | items = items.filter(pk__gte=slice[0]) 110 | if slice and slice[1]: 111 | items = items.filter(pk__lt=slice[1]) 112 | items = items.order_by(model._meta.pk.attname) 113 | objects.extend(items) 114 | else: 115 | raise CommandError("Wrong slice: %s" % slice) 116 | 117 | all = objects 118 | if propagate: 119 | collected = set([(x.__class__, x.pk) for x in all]) 120 | while objects: 121 | related = [] 122 | for x in objects: 123 | if DEBUG: 124 | print "Adding %s[%s]" % (model_name(x), x.pk) 125 | # follow forward relation fields 126 | for f in x.__class__._meta.fields + x.__class__._meta.many_to_many: 127 | if isinstance(f, ForeignKey): 128 | new = getattr(x, f.name) # instantiate object 129 | if new and not (new.__class__, new.pk) in collected: 130 | collected.add((new.__class__, new.pk)) 131 | related.append(new) 132 | if isinstance(f, ManyToManyField): 133 | for new in getattr(x, f.name).all(): 134 | if new and not (new.__class__, new.pk) in collected: 135 | collected.add((new.__class__, new.pk)) 136 | related.append(new) 137 | # follow reverse relations as requested 138 | for reverse_field in follow_reverse.get(x.__class__, []): 139 | mgr = getattr(x, reverse_field) 140 | for new in mgr.all(): 141 | if new and not (new.__class__, new.pk) in collected: 142 | collected.add((new.__class__, new.pk)) 143 | related.append(new) 144 | objects = related 145 | all.extend(objects) 146 | 147 | try: 148 | return serializers.serialize(format, all, indent=indent) 149 | except Exception, e: 150 | if show_traceback: 151 | raise 152 | raise CommandError("Unable to serialize database: %s" % e) 153 | 154 | def get_models(self): 155 | return [(m, model_name(m)) for m in get_models()] 156 | 157 | def get_model_from_name(self, search): 158 | """Given a name of a model, return the model object associated with it 159 | 160 | The name can be either fully specified or uniquely matching the 161 | end of the model name. e.g. 162 | django.contrib.auth.User 163 | or 164 | auth.User 165 | raises CommandError if model can't be found or uniquely determined 166 | """ 167 | models = [model for model, name in self.get_models() 168 | if name.endswith('.'+name) or name == search] 169 | if not models: 170 | raise CommandError("Unknown model: %s" % search) 171 | if len(models)>1: 172 | raise CommandError("Ambiguous model name: %s" % search) 173 | return models[0] 174 | 175 | def handle_label(self, labels, **options): 176 | parsed = [] 177 | for label in labels: 178 | search, pks = label, '' 179 | if '[' in label: 180 | search, pks = label.split('[', 1) 181 | slice = '' 182 | if ':' in pks: 183 | slice = pks.rstrip(']').split(':', 1) 184 | elif pks: 185 | slice = pks.rstrip(']') 186 | model = self.get_model_from_name(search) 187 | parsed.append((model, slice)) 188 | return self.handle_models(parsed, **options) 189 | 190 | def list_models(self): 191 | names = [name for _model, name in self.get_models()] 192 | raise CommandError('Neither model name nor slice given. Installed model names: \n%s' % ",\n".join(names)) 193 | 194 | def handle(self, *labels, **options): 195 | if not labels: 196 | self.list_models() 197 | 198 | output = [] 199 | label_output = self.handle_label(labels, **options) 200 | if label_output: 201 | output.append(label_output) 202 | return '\n'.join(output) 203 | -------------------------------------------------------------------------------- /test_utils/management/commands/quicktest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Original code and ideas from http://www.djangosnippets.org/snippets/1318/ 3 | Thanks crucialfelix 4 | """ 5 | from django.core.management.base import BaseCommand 6 | from optparse import make_option 7 | import sys 8 | 9 | class Command(BaseCommand): 10 | option_list = BaseCommand.option_list + ( 11 | make_option('--noinput', action='store_false', dest='interactive', default=True, 12 | help='Tells Django to NOT prompt the user for input of any kind.'), 13 | ) 14 | help = 'Runs the test suite, creating a test db IF NEEDED and NOT DESTROYING the test db afterwards. Otherwise operates exactly as does test.' 15 | args = '[appname ...]' 16 | 17 | requires_model_validation = False 18 | 19 | def handle(self, *test_labels, **options): 20 | from django.conf import settings 21 | from django.test.utils import get_runner 22 | 23 | verbosity = int(options.get('verbosity', 1)) 24 | interactive = options.get('interactive', True) 25 | 26 | settings.TEST_RUNNER = 'test_utils.test_runners.keep_database.run_tests' 27 | 28 | test_runner = get_runner(settings) 29 | 30 | failures = test_runner(test_labels, verbosity=verbosity, interactive=interactive) 31 | if failures: 32 | sys.exit(failures) 33 | -------------------------------------------------------------------------------- /test_utils/management/commands/relational_dumpdata.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand, CommandError 2 | from django.core import serializers 3 | 4 | from optparse import make_option 5 | from django.db.models.fields.related import ForeignKey, ManyToManyField 6 | from django.db.models import get_app, get_apps, get_models 7 | 8 | def _relational_dumpdata(app, collected): 9 | objects = [] 10 | for mod in get_models(app): 11 | objects.extend(mod._default_manager.all()) 12 | #Got models, now get their relationships. 13 | #Thanks to http://www.djangosnippets.org/snippets/918/ 14 | related = [] 15 | collected = collected.union(set([(x.__class__, x.pk) for x in objects])) 16 | for obj in objects: 17 | for f in obj._meta.fields : 18 | if isinstance(f, ForeignKey): 19 | new = getattr(obj, f.name) # instantiate object 20 | if new and not (new.__class__, new.pk) in collected: 21 | collected.add((new.__class__, new.pk)) 22 | related.append(new) 23 | for f in obj._meta.many_to_many: 24 | if isinstance(f, ManyToManyField): 25 | for new in getattr(obj, f.name).all(): 26 | if new and not (new.__class__, new.pk) in collected: 27 | collected.add((new.__class__, new.pk)) 28 | related.append(new) 29 | if related != []: 30 | objects.extend(related) 31 | return (objects, collected) 32 | 33 | 34 | class Command(BaseCommand): 35 | option_list = BaseCommand.option_list + ( 36 | make_option('--format', default='json', dest='format', 37 | help='Specifies the output serialization format for fixtures.'), 38 | make_option('--indent', default=None, dest='indent', type='int', 39 | help='Specifies the indent level to use when pretty-printing output'), 40 | make_option('-e', '--exclude', dest='exclude',action='append', default=[], 41 | help='App to exclude (use multiple --exclude to exclude multiple apps).'), 42 | ) 43 | help = 'Output the contents of the database as a fixture of the given format.' 44 | args = '[appname ...]' 45 | 46 | def handle(self, *app_labels, **options): 47 | 48 | format = options.get('format','json') 49 | indent = options.get('indent',None) 50 | exclude = options.get('exclude',[]) 51 | show_traceback = options.get('traceback', False) 52 | 53 | excluded_apps = [get_app(app_label) for app_label in exclude] 54 | 55 | if len(app_labels) == 0: 56 | app_list = [app for app in get_apps() if app not in excluded_apps] 57 | else: 58 | app_list = [get_app(app_label) for app_label in app_labels] 59 | 60 | # Check that the serialization format exists; this is a shortcut to 61 | # avoid collating all the objects and _then_ failing. 62 | try: 63 | serializers.get_serializer(format) 64 | except KeyError: 65 | raise CommandError("Unknown serialization format: %s" % format) 66 | 67 | objects = [] 68 | collected = set() 69 | for app in app_list: #Yey for ghetto recusion 70 | objects, collected = _relational_dumpdata(app, collected) 71 | #****End New stuff 72 | try: 73 | return serializers.serialize(format, objects, indent=indent) 74 | except Exception, e: 75 | if show_traceback: 76 | raise 77 | raise CommandError("Unable to serialize database: %s" % e) 78 | -------------------------------------------------------------------------------- /test_utils/management/commands/testmaker.py: -------------------------------------------------------------------------------- 1 | from optparse import make_option 2 | import logging, os 3 | from os import path 4 | 5 | from django.core.management.base import BaseCommand, CommandError 6 | from django.conf import settings 7 | from django.core.management import call_command 8 | from django.db import models 9 | 10 | from test_utils.testmaker import Testmaker 11 | 12 | 13 | class Command(BaseCommand): 14 | option_list = BaseCommand.option_list + ( 15 | make_option('-a', '--app', action='store', dest='application', 16 | default=None, help='The name of the application (in the current \ 17 | directory) to output data to. (defaults to currect directory)'), 18 | make_option('-l', '--logdir', action='store', dest='logdirectory', 19 | default=os.getcwd(), help='Directory to send tests and fixtures to. \ 20 | (defaults to currect directory)'), 21 | make_option('-x', '--loud', action='store', dest='verbosity', default='1', 22 | type='choice', choices=['0', '1', '2'], 23 | help='Verbosity level; 0=minimal output, 1=normal output, 2=all output'), 24 | make_option('-f', '--fixture', action='store_true', dest='fixture', default=False, 25 | help='Pass -f to not create a fixture for the data.'), 26 | make_option('--format', default='json', dest='format', 27 | help='Specifies the output serialization format for fixtures.'), 28 | ) 29 | 30 | help = 'Runs the test server with the testmaker output enabled' 31 | args = '[server:port]' 32 | 33 | def handle(self, addrport='', *args, **options): 34 | 35 | app = options.get("application") 36 | verbosity = int(options.get('verbosity', 1)) 37 | create_fixtures = options.get('fixture', False) 38 | logdir = options.get('logdirectory') 39 | fixture_format = options.get('format', 'xml') 40 | 41 | if app: 42 | app = models.get_app(app) 43 | 44 | if not app: 45 | #Don't serialize the whole DB :) 46 | create_fixtures = False 47 | 48 | testmaker = Testmaker(app, verbosity, create_fixtures, fixture_format, addrport) 49 | testmaker.prepare(insert_middleware=True) 50 | try: 51 | call_command('runserver', addrport=addrport, use_reloader=False) 52 | except SystemExit: 53 | if create_fixtures: 54 | testmaker.make_fixtures() 55 | else: 56 | raise 57 | -------------------------------------------------------------------------------- /test_utils/management/commands/testshell.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand 2 | 3 | from optparse import make_option 4 | 5 | class Command(BaseCommand): 6 | option_list = BaseCommand.option_list + ( 7 | make_option('--addrport', action='store', dest='addrport', 8 | type='string', default='', 9 | help='port number or ipaddr:port to run the server on'), 10 | ) 11 | help = 'Runs a development server with data from the given fixture(s).' 12 | args = '[fixture ...]' 13 | 14 | requires_model_validation = False 15 | 16 | def handle(self, *fixture_labels, **options): 17 | from django.core.management import call_command 18 | from django.db import connection 19 | from django.db.backends import creation 20 | from django.conf import settings 21 | 22 | verbosity = int(options.get('verbosity', 1)) 23 | addrport = options.get('addrport') 24 | 25 | # Create a test database. 26 | connection.creation.create_test_db(verbosity, autoclobber=True) 27 | 28 | if settings.TEST_DATABASE_NAME: 29 | settings.DATABASE_NAME = settings.TEST_DATABASE_NAME 30 | else: 31 | settings.DATABASE_NAME = creation.TEST_DATABASE_PREFIX + settings.DATABASE_NAME 32 | 33 | # Import the fixture data into the test database. 34 | call_command('loaddata', *fixture_labels, **{'verbosity': verbosity}) 35 | 36 | try: 37 | call_command('shell_plus') 38 | except: 39 | call_command('shell') 40 | -------------------------------------------------------------------------------- /test_utils/mocks.py: -------------------------------------------------------------------------------- 1 | from django.test import Client 2 | from django.core.handlers.wsgi import WSGIRequest 3 | 4 | class RequestFactory(Client): 5 | """ 6 | Class that lets you create mock Request objects for use in testing. 7 | 8 | Usage: 9 | 10 | rf = RequestFactory() 11 | get_request = rf.get('/hello/') 12 | post_request = rf.post('/submit/', {'foo': 'bar'}) 13 | 14 | This class re-uses the django.test.client.Client interface, docs here: 15 | http://www.djangoproject.com/documentation/testing/#the-test-client 16 | 17 | Once you have a request object you can pass it to any view function, 18 | just as if that view had been hooked up using a URLconf. 19 | 20 | """ 21 | def request(self, **request): 22 | """ 23 | Similar to parent class, but returns the request object as soon as it 24 | has created it. 25 | """ 26 | environ = { 27 | 'HTTP_COOKIE': self.cookies, 28 | 'PATH_INFO': '/', 29 | 'QUERY_STRING': '', 30 | 'REQUEST_METHOD': 'GET', 31 | 'SCRIPT_NAME': '', 32 | 'SERVER_NAME': 'testserver', 33 | 'SERVER_PORT': 80, 34 | 'SERVER_PROTOCOL': 'HTTP/1.1', 35 | } 36 | environ.update(self.defaults) 37 | environ.update(request) 38 | return WSGIRequest(environ) 39 | -------------------------------------------------------------------------------- /test_utils/models.py: -------------------------------------------------------------------------------- 1 | #This is here so that Django thinks we are a model so we can test it. 2 | -------------------------------------------------------------------------------- /test_utils/templatetags/__init__.py: -------------------------------------------------------------------------------- 1 | import re 2 | import os 3 | from django.conf import settings 4 | from django import template 5 | 6 | try: 7 | from django.template.loaders.filesystem import load_template_source 8 | except ImportError: 9 | from django.template.loaders.filesystem import Loader 10 | loader = Loader 11 | load_template_source = lambda name: loader.load_template_source(name) 12 | 13 | from test_utils.testmaker import Testmaker 14 | 15 | DEFAULT_TAGS = ['autoescape' , 'block' , 'comment' , 'cycle' , 'debug' , 16 | 'extends' , 'filter' , 'firstof' , 'if' , 'else', 'for', #No for so we can do loops 17 | 'ifchanged' , 'ifequal' , 'ifnotequal' , 'include' , 'load' , 'now' , 18 | 'regroup' , 'spaceless' , 'ssi' , 'templatetag' , 'url' , 'widthratio' , 19 | 'with' ] 20 | 21 | tag_re = re.compile('({% (.*?) %})') 22 | 23 | ### Template Tag Maker stuff 24 | 25 | class TemplateParser(object): 26 | 27 | def __init__(self, template, context=None): 28 | """ 29 | Set the initial value of the template to be parsed 30 | 31 | Allows for the template passed to be a string of a template name 32 | or a string that represents a template. 33 | """ 34 | self.template = template 35 | self.context = context 36 | #Contains the strings of all loaded classes 37 | self.loaded_classes = [] 38 | self.template_calls = [] 39 | self.tests = [] 40 | #Accept both template names and template strings 41 | try: 42 | self.template_string, self.filepath = load_template_source(template.name) 43 | except: 44 | self.template_string = template 45 | self.filepath = None 46 | 47 | 48 | def parse(self): 49 | """ 50 | Parse the template tag calls out of the template. 51 | This is ugly because of having more than 1 tag on a line. 52 | Thus we have to loop over the file, splitting on the regex, then 53 | looping over the split, matching for our regex again. 54 | Improvements welcome! 55 | 56 | End result:: 57 | self.loaded_classes contains the load commands of classes loaded 58 | self.template_calls contains the template calls 59 | """ 60 | for line in self.template_string.split('\n'): 61 | split_line = tag_re.split(line) 62 | if len(split_line) > 1: 63 | for matched in split_line: 64 | mat = tag_re.search(matched) 65 | if mat: 66 | full_command = mat.group(0) 67 | cmd = mat.group(2).split()[0].strip() #get_comment_form etc 68 | if cmd == 'load': 69 | self.loaded_classes.append(full_command) 70 | else: 71 | if cmd not in DEFAULT_TAGS and cmd not in 'end'.join(DEFAULT_TAGS): 72 | self.template_calls.append(full_command) 73 | 74 | 75 | def create_tests(self): 76 | """ 77 | This yields a rendered template string to assert Equals against with 78 | the outputted template. 79 | """ 80 | for tag_string in self.template_calls: 81 | out_context = {} 82 | context_name = "" 83 | #Try and find anything in the string that's in the context 84 | context_name = '' 85 | bits = tag_string.split() 86 | for bit_num, bit in enumerate(bits): 87 | try: 88 | out_context[bit] = template.Variable(bit).resolve(self.context) 89 | except: 90 | pass 91 | if bit == 'as': 92 | context_name = bits[bit_num+1] 93 | 94 | if context_name: 95 | con_string = "{{ %s }}" % context_name 96 | else: 97 | con_string = "" 98 | template_string = "%s%s%s" % (''.join(self.loaded_classes), tag_string, con_string) 99 | try: 100 | template_obj = template.Template(template_string) 101 | rendered_string = template_obj.render(template.Context(out_context)) 102 | except Exception, e: 103 | print "EXCEPTION: %s" % e.message 104 | rendered_string = '' 105 | #self.tests.append(rendered_string) 106 | self.output_ttag(template_string, rendered_string, out_context) 107 | 108 | def output_ttag(self, template_str, output_str, context): 109 | Testmaker.log.info(" tmpl = template.Template(u'%s')" % template_str) 110 | context_str = "{" 111 | for con in context: 112 | try: 113 | tmpl_obj = context[con] 114 | #TODO: This will blow up on anything but a model. 115 | #Would be cool to have a per-type serialization, prior art is 116 | #in django's serializers and piston. 117 | context_str += "'%s': get_model('%s', '%s').objects.get(pk=%s)," % (con, tmpl_obj._meta.app_label, tmpl_obj._meta.module_name, tmpl_obj.pk ) 118 | except: 119 | #sometimes there be integers here 120 | pass 121 | context_str += "}" 122 | 123 | #if output_str: 124 | # Testmaker.log.info(" tmpl = template.Template(u'%s')" % template_str) 125 | Testmaker.log.info(" context = template.Context(%s)" % context_str) 126 | Testmaker.log.info(" self.assertEqual(tmpl.render(context), u'%s')\n" % output_str) 127 | -------------------------------------------------------------------------------- /test_utils/test_runners/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ericholscher/django-test-utils/0207f06e0ebaa5081dc8648815815929fbc9dea5/test_utils/test_runners/__init__.py -------------------------------------------------------------------------------- /test_utils/test_runners/keep_database.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | from django.test.simple import build_test, reorder_suite, build_suite 3 | from django.test.utils import setup_test_environment, teardown_test_environment 4 | from django.test.testcases import connections_support_transactions 5 | from django.db.models import get_app, get_apps 6 | from django.conf import settings 7 | try: 8 | from django.utils import unittest # django's 1.3 copy of unittest2 9 | except ImportError: 10 | import unittest # system fallback 11 | import os 12 | 13 | def run_tests(test_labels, verbosity=1, interactive=True, extra_tests=[]): 14 | """ 15 | worsk exactly as per normal test 16 | but only creates the test_db if it doesn't yet exist 17 | and does not destroy it when done 18 | tables are flushed and fixtures loaded between tests as per usual 19 | but if your schema has not changed then this saves significant amounts of time 20 | and speeds up the test cycle 21 | 22 | Run the unit tests for all the test labels in the provided list. 23 | Labels must be of the form: 24 | - app.TestClass.test_method 25 | Run a single specific test method 26 | - app.TestClass 27 | Run all the test methods in a given class 28 | - app 29 | Search for doctests and unittests in the named application. 30 | 31 | When looking for tests, the test runner will look in the models and 32 | tests modules for the application. 33 | 34 | A list of 'extra' tests may also be provided; these tests 35 | will be added to the test suite. 36 | 37 | Returns the number of tests that failed. 38 | """ 39 | setup_test_environment() 40 | 41 | settings.DEBUG = False 42 | suite = unittest.TestSuite() 43 | 44 | if test_labels: 45 | for label in test_labels: 46 | if '.' in label: 47 | suite.addTest(build_test(label)) 48 | else: 49 | app = get_app(label) 50 | suite.addTest(build_suite(app)) 51 | else: 52 | for app in get_apps(): 53 | suite.addTest(build_suite(app)) 54 | 55 | for test in extra_tests: 56 | suite.addTest(test) 57 | 58 | suite = reorder_suite(suite, (TestCase,)) 59 | 60 | old_name = settings.DATABASES['default']['NAME'] 61 | 62 | ###Everything up to here is from django.test.simple 63 | 64 | from django.db.backends import creation 65 | from django.db import connection, DatabaseError 66 | 67 | if settings.DATABASES['default']['TEST_NAME']: 68 | settings.DATABASES['default']['NAME'] = settings.DATABASES['default']['TEST_NAME'] 69 | else: 70 | settings.DATABASES['default']['NAME'] = creation.TEST_DATABASE_PREFIX + settings.DATABASES['default']['NAME'] 71 | connection.settings_dict["DATABASE_NAME"] = settings.DATABASES['default']['NAME'] 72 | 73 | # does test db exist already ? 74 | try: 75 | if settings.DATABASES['default']['ENGINE'] == 'sqlite3': 76 | if not os.path.exists(settings.DATABASES['default']['NAME']): 77 | raise DatabaseError 78 | cursor = connection.cursor() 79 | except Exception: 80 | # db does not exist 81 | # juggling ! create_test_db switches the DATABASE_NAME to the TEST_DATABASE_NAME 82 | settings.DATABASES['default']['NAME'] = old_name 83 | connection.settings_dict["DATABASE_NAME"] = old_name 84 | connection.creation.create_test_db(verbosity, autoclobber=True) 85 | else: 86 | connection.close() 87 | 88 | settings.DATABASES['default']['SUPPORTS_TRANSACTIONS'] = connections_support_transactions() 89 | 90 | result = unittest.TextTestRunner(verbosity=verbosity).run(suite) 91 | 92 | #Since we don't call destory_test_db, we need to set the db name back. 93 | settings.DATABASES['default']['NAME'] = old_name 94 | connection.settings_dict["DATABASE_NAME"] = old_name 95 | teardown_test_environment() 96 | 97 | return len(result.failures) + len(result.errors) 98 | -------------------------------------------------------------------------------- /test_utils/test_runners/profile.py: -------------------------------------------------------------------------------- 1 | import cProfile 2 | import pstats 3 | from django.test.simple import run_tests as django_test_runner 4 | 5 | def run_tests(test_labels, verbosity=1, interactive=True, 6 | extra_tests=[], nodatabase=False): 7 | """ 8 | Test runner which displays basic profile data. 9 | Needs some improvement, mostly here for Continuous Integration purposes. 10 | """ 11 | print "Using profiling test runner" 12 | cProfile.runctx("django_test_runner(test_labels, verbosity, interactive, extra_tests)", globals(), locals(), filename="django_tests.profile") 13 | stats = pstats.Stats('django_tests.profile') 14 | stats.strip_dirs().sort_stats('time').print_stats(30) 15 | return 0 16 | -------------------------------------------------------------------------------- /test_utils/testmaker/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from os import path 4 | from django.core import serializers as django_serializers 5 | from test_utils.management.commands.relational_dumpdata import _relational_dumpdata 6 | from django.template import Context, Template 7 | from django.conf import settings 8 | 9 | TESTMAKER_TEMPLATE = """\ 10 | #coding: utf-8 11 | from django.test import TestCase 12 | from django.test import Client 13 | from django import template 14 | from django.db.models import get_model 15 | 16 | class Testmaker(TestCase): 17 | {% if create_fixtures %} 18 | fixtures = ["{{ fixture_file }}"] 19 | {% else %} 20 | #fixtures = ["{{ app_name }}_testmaker"] 21 | {% endif %} 22 | """ 23 | 24 | class Testmaker(object): 25 | enabled = False 26 | #Have global log and serializer objects so that we never log things twice. 27 | log = None 28 | serializer = None 29 | 30 | def __init__(self, app=None, verbosity=0, create_fixtures=False, fixture_format='xml', addrport='', **kwargs): 31 | self.app = app 32 | self.verbosity = verbosity 33 | self.create_fixtures = create_fixtures 34 | self.fixture_format = fixture_format 35 | self.addrport = addrport 36 | self.kwargs = kwargs 37 | #Assume we're writing new tests until proven otherwise 38 | self.new_tests = True 39 | 40 | def prepare(self, insert_middleware=False): 41 | self.set_paths() 42 | if not hasattr(self, 'has_run_logging'): 43 | self.setup_logging() 44 | self.prepare_test_file() 45 | if insert_middleware: 46 | self.insert_middleware() 47 | Testmaker.enabled = True 48 | 49 | 50 | def set_paths(self): 51 | if self.app: 52 | self.app_name = self.app.__name__.split('.')[-2] 53 | self.base_dir = path.dirname(self.app.__file__) 54 | else: 55 | self.app_name = 'tmp' 56 | #TODO: Need to make this platform independent. 57 | self.base_dir = '/tmp/testmaker/' 58 | if not path.exists(self.base_dir): 59 | os.mkdir(self.base_dir) 60 | 61 | 62 | #Figure out where to store data 63 | self.fixtures_dir = path.join(self.base_dir, 'fixtures') 64 | self.fixture_file = path.join(self.fixtures_dir, '%s_testmaker.%s' % (self.app_name, self.fixture_format)) 65 | if self.create_fixtures: 66 | if not path.exists(self.fixtures_dir): 67 | os.mkdir(self.fixtures_dir) 68 | 69 | #Setup test and serializer files 70 | self.tests_dir = path.join(self.base_dir, 'tests') 71 | self.test_file = path.join(self.tests_dir, '%s_testmaker.py' % (self.app_name)) 72 | #TODO: Make this have the correct file extension based on serializer used 73 | self.serialize_file = path.join(self.tests_dir, '%s_testdata.serialized' % (self.app_name)) 74 | 75 | if not path.exists(self.tests_dir): 76 | os.mkdir(self.tests_dir) 77 | if path.exists(self.test_file): 78 | #Already have tests there. 79 | self.new_tests = False 80 | 81 | if self.verbosity > 0: 82 | print "Handling app '%s'" % self.app_name 83 | print "Logging tests to %s" % self.test_file 84 | if self.create_fixtures: 85 | print "Logging fixtures to %s" % self.fixture_file 86 | 87 | def setup_logging(self, test_file=None, serialize_file=None): 88 | #supress other logging 89 | logging.basicConfig(level=logging.CRITICAL, 90 | filename=path.devnull) 91 | 92 | #Override default if its passed in 93 | if not test_file: 94 | test_file = self.test_file 95 | else: 96 | self.test_file = test_file 97 | log = logging.getLogger('testprocessor') 98 | [log.removeHandler(h) for h in log.handlers] 99 | log.setLevel(logging.INFO) 100 | handler = logging.FileHandler(test_file, 'a') 101 | handler.setFormatter(logging.Formatter('%(message)s')) 102 | log.addHandler(handler) 103 | Testmaker.log = log 104 | 105 | #Override default if its passed in 106 | if not serialize_file: 107 | serialize_file = self.serialize_file 108 | else: 109 | self.serialize_file = serialize_file 110 | log_s = logging.getLogger('testserializer') 111 | [log_s.removeHandler(h) for h in log_s.handlers] 112 | log_s.setLevel(logging.INFO) 113 | handler_s = logging.FileHandler(self.serialize_file, 'a') 114 | handler_s.setFormatter(logging.Formatter('%(message)s')) 115 | log_s.addHandler(handler_s) 116 | Testmaker.serializer = log_s 117 | 118 | self.has_run_logging = True 119 | 120 | def prepare_test_file(self): 121 | if self.new_tests: 122 | t = Template(TESTMAKER_TEMPLATE) 123 | c = Context({ 124 | 'create_fixtures': self.create_fixtures, 125 | 'app_name': self.app_name, 126 | 'fixture_file': self.fixture_file, 127 | }) 128 | self.log.info(t.render(c)) 129 | else: 130 | if self.verbosity > 0: 131 | print "Appending to current log file" 132 | 133 | def insert_middleware(self): 134 | if self.verbosity > 0: 135 | print "Inserting TestMaker logging server..." 136 | if 'test_utils.testmaker.middleware.testmaker.TestMakerMiddleware' not in settings.MIDDLEWARE_CLASSES: 137 | settings.MIDDLEWARE_CLASSES += ('test_utils.testmaker.middleware.testmaker.TestMakerMiddleware',) 138 | 139 | def make_fixtures(self): 140 | if self.verbosity > 0: 141 | print "Creating fixture at " + self.fixture_file 142 | objects, collected = _relational_dumpdata(self.app, set()) 143 | serial_file = open(self.fixture_file, 'a') 144 | try: 145 | django_serializers.serialize(self.fixture_format, objects, stream=serial_file, indent=4) 146 | except Exception, e: 147 | if self.verbosity > 0: 148 | print ("Unable to serialize database: %s" % e) 149 | 150 | @classmethod 151 | def logfile(klass): 152 | return klass.log.handlers[0].baseFilename 153 | -------------------------------------------------------------------------------- /test_utils/testmaker/middleware/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ericholscher/django-test-utils/0207f06e0ebaa5081dc8648815815929fbc9dea5/test_utils/testmaker/middleware/__init__.py -------------------------------------------------------------------------------- /test_utils/testmaker/middleware/testmaker.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.test import Client 3 | from django.test.utils import setup_test_environment 4 | from django.template import Template, Context 5 | 6 | from test_utils.testmaker import processors 7 | from test_utils.testmaker import serializers 8 | from test_utils.testmaker import Testmaker 9 | 10 | #Remove at your own peril. 11 | #Thar be sharks in these waters. 12 | debug = getattr(settings, 'DEBUG', False) 13 | """ 14 | if not debug: 15 | print "THIS CODE IS NOT MEANT FOR USE IN PRODUCTION" 16 | else: 17 | print "Loaded Testmaker Middleware" 18 | """ 19 | 20 | if not Testmaker.enabled: 21 | testmaker = Testmaker(verbosity=0) 22 | testmaker.prepare() 23 | 24 | 25 | SHOW_TESTMAKER_HEADER = getattr(settings, 'SHOW_TESTMAKER_HEADER', False) 26 | 27 | RESPONSE_TEMPLATE = Template(""" 28 |
29 | Testmaker: Logging to: {{ file }} 30 |
31 | 32 | 33 |
34 | Show Log 35 |
36 | """) 37 | 38 | 39 | class TestMakerMiddleware(object): 40 | def __init__(self): 41 | """ 42 | Assign a Serializer and Processer 43 | Serializers will be pluggable and allow for custom recording. 44 | Processers will process the serializations into test formats. 45 | """ 46 | serializer_pref = getattr(settings, 'TESTMAKER_SERIALIZER', 'pickle') 47 | processor_pref = getattr(settings, 'TESTMAKER_PROCESSOR', 'django') 48 | self.serializer = serializers.get_serializer(serializer_pref)() 49 | self.processor = processors.get_processor(processor_pref)() 50 | 51 | def process_request(self, request): 52 | """ 53 | Run the request through the testmaker middleware. 54 | This outputs the requests to the chosen Serializers. 55 | Possible running it through one or many Processors 56 | """ 57 | #This is request.REQUEST to catch POST and GET 58 | if 'test_client_true' not in request.REQUEST: 59 | request.logfile = Testmaker.logfile() 60 | self.serializer.save_request(request) 61 | self.processor.save_request(request) 62 | #We only want to re-run the request on idempotent requests 63 | if request.method.lower() == "get": 64 | setup_test_environment() 65 | c = Client(REMOTE_ADDR='127.0.0.1') 66 | getdict = request.GET.copy() 67 | getdict['test_client_true'] = 'yes' #avoid recursion 68 | response = c.get(request.path, getdict) 69 | self.serializer.save_response(request, response) 70 | self.processor.save_response(request, response) 71 | return None 72 | 73 | def process_response(self, request, response): 74 | if 'test_client_true' not in request.REQUEST \ 75 | and SHOW_TESTMAKER_HEADER: 76 | c = Context({'file': Testmaker.logfile()}) 77 | s = RESPONSE_TEMPLATE.render(c) 78 | response.content = str(s) + str(response.content) 79 | return response 80 | -------------------------------------------------------------------------------- /test_utils/testmaker/processors/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | Interfaces for processing Django tests. 4 | 5 | To add your own processors, use the TEST_PROCESSOR_MODULES setting:: 6 | 7 | TEST_PROCESSOR_MODULES = { 8 | 'django': 'test_utils.testmaker.processors.django', 9 | 'twill': 'test_utils.testmaker.processors.twill', 10 | } 11 | 12 | """ 13 | 14 | from django.conf import settings 15 | from django.utils import importlib 16 | 17 | # Built-in processors 18 | 19 | TEST_PROCESSORS = { 20 | 'django': 'test_utils.testmaker.processors.django_processor', 21 | 'twill': 'test_utils.testmaker.processors.twill_processor', 22 | } 23 | 24 | _test_processors = {} 25 | 26 | def register_processor(format, processor_module, processors=None): 27 | """"Register a new processor. 28 | 29 | ``processor_module`` should be the fully qualified module name 30 | for the processor. 31 | 32 | If ``processors`` is provided, the registration will be added 33 | to the provided dictionary. 34 | 35 | If ``processors`` is not provided, the registration will be made 36 | directly into the global register of processors. Adding processors 37 | directly is not a thread-safe operation. 38 | """ 39 | module = importlib.import_module(processor_module) 40 | if processors is None: 41 | _test_processors[format] = module 42 | else: 43 | processors[format] = module 44 | 45 | def unregister_processor(format): 46 | "Unregister a given processor. This is not a thread-safe operation." 47 | del _test_processors[format] 48 | 49 | def get_processor(format): 50 | if not _test_processors: 51 | _load_test_processors() 52 | return _test_processors[format].Processor 53 | 54 | def get_processor_formats(): 55 | if not _test_processors: 56 | _load_test_processors() 57 | return _test_processors.keys() 58 | 59 | def _load_test_processors(): 60 | """ 61 | Register built-in and settings-defined processors. This is done lazily so 62 | that user code has a chance to (e.g.) set up custom settings without 63 | needing to be careful of import order. 64 | """ 65 | global _test_processors 66 | processors = {} 67 | for format in TEST_PROCESSORS: 68 | register_processor(format, TEST_PROCESSORS[format], processors) 69 | if hasattr(settings, "TEST_PROCESSOR_MODULES"): 70 | for format in settings.TEST_PROCESSOR_MODULES: 71 | register_processor(format, settings.TEST_PROCESSOR_MODULES[format], processors) 72 | _test_processors = processors 73 | -------------------------------------------------------------------------------- /test_utils/testmaker/processors/base.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import re 3 | import time 4 | 5 | from django.template.defaultfilters import slugify as base_slugify 6 | from django.template import Template, Context 7 | from django.utils.encoding import force_unicode 8 | from django.utils.safestring import mark_safe 9 | 10 | from test_utils.templatetags import TemplateParser 11 | 12 | TEST_TEMPLATE = 'Override in Subclass' 13 | STATUS_TEMPLATE = 'Override in Subclass' 14 | CONTEXT_TEMPLATE = 'Override in Subclass' 15 | #DISCARD_CONTEXT_KEYS = ('LANGUAGES',) 16 | DISCARD_CONTEXT_KEYS = [] 17 | 18 | def safe_dict(dict): 19 | new_dic = {} 20 | for key,val in dict.iteritems(): 21 | new_dic[key] = mark_safe(val) 22 | return new_dic 23 | 24 | def slugify(toslug): 25 | """ 26 | Turn dashs into underscores to sanitize for filenames 27 | """ 28 | return re.sub("-", "_", base_slugify(toslug)) 29 | 30 | class Processer(object): 31 | """Processes the serialized data. Generally to create some sort of test cases""" 32 | 33 | def __init__(self, name): 34 | self.name = name 35 | self.log = logging.getLogger('testprocessor') 36 | #self.log = logging.getLogger('testprocessor-%s' % self.name) 37 | self.data = {} 38 | 39 | def shall_we_proceed(self, request): 40 | if 'media' in request.path or 'test_utils' in request.path: 41 | return False 42 | return True 43 | 44 | def process_request(self, request): 45 | raise NotImplementedError 46 | 47 | def save_request(self, request): 48 | """ Actually write the request out to a file """ 49 | if self.shall_we_proceed(request): 50 | self._log_request(request) 51 | 52 | def process_response(self, request, response): 53 | raise NotImplementedError 54 | 55 | def save_response(self, request, response): 56 | if self.shall_we_proceed(request): 57 | self._log_status(response) 58 | if response.context and response.status_code != 404: 59 | self._log_context(response.context) 60 | #This is where template tag outputting would go 61 | #Turned off until it gets betterer 62 | """ 63 | parser = TemplateParser(response.template[0], context) 64 | parser.parse() 65 | parser.create_tests() 66 | """ 67 | 68 | def _get_template(self, templatename): 69 | """Should be implemented in subclass""" 70 | raise NotImplementedError 71 | 72 | def _log_request(self, request): 73 | method = request.method.lower() 74 | request_str = "'%s', {" % request.path 75 | for dikt in request.REQUEST.dicts: 76 | for arg in dikt: 77 | request_str += "'%s': '%s', " % (arg, request.REQUEST[arg]) 78 | request_str += "}" 79 | 80 | template = Template(self._get_template('test')) 81 | context = { 82 | 'path': slugify(request.path), 83 | 'time': slugify(time.time()), 84 | 'method': method, 85 | 'request_str': request_str, 86 | } 87 | context = Context(safe_dict(context)) 88 | self.log.info(template.render(context)) 89 | 90 | def _log_status(self, response): 91 | template = Template(self._get_template('status')) 92 | context = { 93 | 'status_code': response.status_code, 94 | } 95 | if response.status_code in [301, 302]: 96 | context['location'] = response['Location'] 97 | context = Context(safe_dict(context)) 98 | self.log.info(template.render(context)) 99 | 100 | def _get_context_keys(self, context): 101 | """Get the keys from the contexts(list) """ 102 | keys = [] 103 | for d in context.dicts: 104 | if isinstance(d, Context): 105 | keys += self._get_context_keys(d) 106 | keys += d.keys() 107 | return keys 108 | 109 | def _log_context(self, context): 110 | template = Template(self._get_template('context')) 111 | keys = [] 112 | if isinstance(context, list): 113 | for c in context: 114 | keys += self._get_context_keys(c) 115 | else: 116 | keys += self._get_context_keys(context) 117 | keys = set(keys) 118 | 119 | # Skip some keys 120 | for discardkey in DISCARD_CONTEXT_KEYS: 121 | keys.discard(discardkey) 122 | 123 | for key in keys: 124 | val = force_unicode(context[key]) 125 | con = { 126 | 'key': key, 127 | 'value': val, 128 | } 129 | con = Context(safe_dict(con)) 130 | try: 131 | #Avoid memory addy's which will change. 132 | if not re.search("0x\w+", val): 133 | self.log.info(template.render(con)) 134 | except UnicodeDecodeError, e: 135 | pass 136 | -------------------------------------------------------------------------------- /test_utils/testmaker/processors/django_processor.py: -------------------------------------------------------------------------------- 1 | import base 2 | 3 | TEST_TEMPLATE = \ 4 | """ def test_{{path}}_{{time}}(self): 5 | r = self.client.{{method}}({{request_str}})""" 6 | 7 | STATUS_TEMPLATE = \ 8 | """ self.assertEqual(r.status_code, {{status_code}})""" 9 | 10 | CONTEXT_TEMPLATE = \ 11 | ''' self.assertEqual(unicode(r.context["{{key}}"]), u"""{{value}}""")''' 12 | 13 | class Processor(base.Processer): 14 | """Processes the serialized data. Generally to create some sort of test cases""" 15 | 16 | def __init__(self, name='django'): 17 | super(Processor, self).__init__(name) 18 | 19 | def _get_template(self, templatename): 20 | return { 21 | 'test': TEST_TEMPLATE, 22 | 'status': STATUS_TEMPLATE, 23 | 'context': CONTEXT_TEMPLATE, 24 | }[templatename] 25 | -------------------------------------------------------------------------------- /test_utils/testmaker/processors/twill_processor.py: -------------------------------------------------------------------------------- 1 | import base 2 | 3 | TEST_TEMPLATE = """go {{ path }}""" 4 | 5 | STATUS_TEMPLATE = """code {{ status_code }}""" 6 | 7 | #CONTEXT_TEMPLATE = '''find {{value}}''' 8 | CONTEXT_TEMPLATE = '' 9 | 10 | class Processor(base.Processer): 11 | """Processes the serialized data. Generally to create some sort of test cases""" 12 | 13 | def __init__(self, name='twill'): 14 | super(Processor, self).__init__(name) 15 | 16 | def _get_template(self, templatename): 17 | return { 18 | 'test': TEST_TEMPLATE, 19 | 'status': STATUS_TEMPLATE, 20 | 'context': CONTEXT_TEMPLATE, 21 | }[templatename] 22 | -------------------------------------------------------------------------------- /test_utils/testmaker/replay.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import re 3 | import cPickle as pickle 4 | from test_utils.testmaker import Testmaker 5 | from test_utils.testmaker.processors.django_processor import Processor 6 | 7 | class MockRequest(dict): 8 | 'Mocking a dict to allow attribute access' 9 | def __getattr__(self, name): 10 | return self[name] 11 | 12 | class Replay(object): 13 | 14 | def __init__(self, file_name, replay_file='replay_file'): 15 | self.file_name = file_name 16 | self.stream = open(self.file_name).readlines() 17 | self.tm = Testmaker() 18 | self.tm.setup_logging(replay_file, '/dev/null') 19 | self.processor = Processor('replay_processor') 20 | self.serial_obj = pickle 21 | 22 | def process(self): 23 | self.log = [] 24 | 25 | buffer = [] 26 | req_re = re.compile('---REQUEST_BREAK---') 27 | res_re = re.compile('---RESPONSE_BREAK---') 28 | 29 | for line in self.stream: 30 | if req_re.search(line): 31 | #process request 32 | to_pickle = ''.join(buffer) 33 | request = MockRequest(self.serial_obj.loads(to_pickle)) 34 | self.processor.save_request(request) 35 | print request['path'], request['time'] 36 | buffer = [] 37 | elif res_re.search(line): 38 | #process response 39 | to_pickle = ''.join(buffer) 40 | response = MockRequest(self.serial_obj.loads(to_pickle)) 41 | self.log.append(request, response) 42 | self.processer.save_response(request, response) 43 | print response['status_code'], response['time'] 44 | buffer = [] 45 | else: 46 | buffer.append(line) 47 | 48 | if __name__ == '__main__': 49 | if len(sys.argv) == 2: 50 | in_file = sys.argv[1] 51 | else: 52 | raise Exception('Need file name') 53 | 54 | replay = Replay(in_file) 55 | replay.process() -------------------------------------------------------------------------------- /test_utils/testmaker/serializers/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | Interfaces for serializing Django requests. 4 | 5 | To add your own serializers, use the TEST_SERIALIZATION_MODULES setting:: 6 | 7 | TEST_SERIALIZATION_MODULES = { 8 | 'pickle': 'test_utils.testmaker.serializers.pickle_serializer', 9 | 'json': 'test_utils.testmaker.json_serializer', 10 | } 11 | 12 | """ 13 | 14 | from django.conf import settings 15 | from django.utils import importlib 16 | 17 | # Built-in serialize 18 | TEST_SERIALIZERS = { 19 | 'pickle': 'test_utils.testmaker.serializers.pickle_serializer', 20 | 'json': 'test_utils.testmaker.serializers.json_serializer', 21 | } 22 | 23 | REQUEST_UNIQUE_STRING = '---REQUEST_BREAK---' 24 | RESPONSE_UNIQUE_STRING = '---RESPONSE_BREAK---' 25 | 26 | _test_serializers = {} 27 | 28 | def register_serializer(format, serializer_module, serializers=None): 29 | """"Register a new serializer. 30 | 31 | ``serializer_module`` should be the fully qualified module name 32 | for the serializer. 33 | 34 | If ``serializers`` is provided, the registration will be added 35 | to the provided dictionary. 36 | 37 | If ``serializers`` is not provided, the registration will be made 38 | directly into the global register of serializers. Adding serializers 39 | directly is not a thread-safe operation. 40 | """ 41 | module = importlib.import_module(serializer_module) 42 | if serializers is None: 43 | _test_serializers[format] = module 44 | else: 45 | serializers[format] = module 46 | 47 | def unregister_serializer(format): 48 | "Unregister a given serializer. This is not a thread-safe operation." 49 | del _test_serializers[format] 50 | 51 | def get_serializer(format): 52 | if not _test_serializers: 53 | _load_test_serializers() 54 | return _test_serializers[format].Serializer 55 | 56 | def get_serializer_formats(): 57 | if not _test_serializers: 58 | _load_test_serializers() 59 | return _test_serializers.keys() 60 | 61 | def get_deserializer(format): 62 | if not _test_serializers: 63 | _load_test_serializers() 64 | return _test_serializers[format].Deserializer 65 | 66 | def _load_test_serializers(): 67 | """ 68 | Register built-in and settings-defined serializers. This is done lazily so 69 | that user code has a chance to (e.g.) set up custom settings without 70 | needing to be careful of import order. 71 | """ 72 | global _test_serializers 73 | serializers = {} 74 | for format in TEST_SERIALIZERS: 75 | register_serializer(format, TEST_SERIALIZERS[format], serializers) 76 | if hasattr(settings, "TEST_SERIALIZATION_MODULES"): 77 | for format in settings.TEST_SERIALIZATION_MODULES: 78 | register_serializer(format, settings.TEST_SERIALIZATION_MODULES[format], serializers) 79 | _test_serializers = serializers 80 | -------------------------------------------------------------------------------- /test_utils/testmaker/serializers/base.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | 4 | class Serializer(object): 5 | """A pluggable Serializer class""" 6 | 7 | name = "base" 8 | 9 | def __init__(self, name): 10 | """Constructor""" 11 | self.name = name 12 | self.ser = logging.getLogger('testserializer') 13 | #self.ser = logging.getLogger('testserializer-%s' % self.name) 14 | self.data = {} 15 | 16 | def process_request(self, request): 17 | request_dict = { 18 | 'name': self.name, 19 | 'time': time.time(), 20 | 'path': request.path, 21 | 22 | 'GET': request.GET, 23 | 'POST': request.POST, 24 | 'REQUEST': request.REQUEST, 25 | 'method': request.method, 26 | } 27 | return request_dict 28 | 29 | def save_request(self, request): 30 | raise NotImplementedError 31 | 32 | def process_response(self, path, response): 33 | response_dict = { 34 | 'name': self.name, 35 | 'time': time.time(), 36 | 'path': path, 37 | 38 | 'context': response.context, 39 | 'content': response.content, 40 | 'status_code': response.status_code, 41 | 'cookies': response.cookies, 42 | 'headers': response._headers, 43 | } 44 | return response_dict 45 | 46 | def save_response(self, request, response): 47 | raise NotImplementedError 48 | -------------------------------------------------------------------------------- /test_utils/testmaker/serializers/json_serializer.py: -------------------------------------------------------------------------------- 1 | import base 2 | try: 3 | from django.utils import simplejson as json 4 | except ImportError: 5 | import json 6 | 7 | from test_utils.testmaker.serializers import REQUEST_UNIQUE_STRING, RESPONSE_UNIQUE_STRING 8 | 9 | class Serializer(base.Serializer): 10 | 11 | def __init__(self, name='pickle'): 12 | super(Serializer, self).__init__(name) 13 | 14 | def save_request(self, request): 15 | """Saves the Request to the serialization stream""" 16 | request_dict = self.process_request(request) 17 | try: 18 | self.ser.info(json.dumps(request_dict)) 19 | self.ser.info(REQUEST_UNIQUE_STRING) 20 | except TypeError, e: 21 | #Can't serialize wsgi.error objects 22 | pass 23 | 24 | def save_response(self, request, response): 25 | """Saves the Response-like objects information that might be tested""" 26 | response_dict = self.process_response(request.path, response) 27 | try: 28 | self.ser.info(json.dumps(response_dict)) 29 | self.ser.info(RESPONSE_UNIQUE_STRING) 30 | except TypeError, e: 31 | #Can't serialize wsgi.error objects 32 | pass 33 | -------------------------------------------------------------------------------- /test_utils/testmaker/serializers/pickle_serializer.py: -------------------------------------------------------------------------------- 1 | import base 2 | import cPickle as pickle 3 | from test_utils.testmaker.serializers import REQUEST_UNIQUE_STRING, RESPONSE_UNIQUE_STRING 4 | 5 | class Serializer(base.Serializer): 6 | 7 | def __init__(self, name='pickle'): 8 | super(Serializer, self).__init__(name) 9 | 10 | def save_request(self, request): 11 | """Saves the Request to the serialization stream""" 12 | request_dict = self.process_request(request) 13 | self.ser.info(pickle.dumps(request_dict)) 14 | self.ser.info(REQUEST_UNIQUE_STRING) 15 | 16 | def save_response(self, request, response): 17 | """Saves the Response-like objects information that might be tested""" 18 | response_dict = self.process_response(request.path, response) 19 | try: 20 | self.ser.info(pickle.dumps(response_dict)) 21 | self.ser.info(RESPONSE_UNIQUE_STRING) 22 | except (TypeError, pickle.PicklingError): 23 | #Can't pickle wsgi.error objects 24 | pass 25 | -------------------------------------------------------------------------------- /test_utils/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls.defaults import * 2 | 3 | import test_utils.views as test_views 4 | 5 | urlpatterns = patterns('', 6 | url(r'^set_logging/(?P.*?)/', 7 | test_views.set_logging, 8 | name='test_utils_set_logging'), 9 | url(r'^set_logging/', 10 | test_views.set_logging, 11 | name='test_utils_set_logging'), 12 | url(r'^show_log/', 13 | test_views.show_log, 14 | name='test_utils_show_log'), 15 | ) 16 | -------------------------------------------------------------------------------- /test_utils/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ericholscher/django-test-utils/0207f06e0ebaa5081dc8648815815929fbc9dea5/test_utils/utils/__init__.py -------------------------------------------------------------------------------- /test_utils/utils/twill_runner.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | This code is originally by miracle2k: 4 | http://bitbucket.org/miracle2k/djutils/src/97f92c32c621/djutils/test/twill.py 5 | 6 | 7 | Integrates the twill web browsing scripting language with Django. 8 | 9 | Provides two main functions, ``setup()`` and ``teardown``, that hook 10 | (and unhook) a certain host name to the WSGI interface of your Django 11 | app, making it possible to test your site using twill without actually 12 | going through TCP/IP. 13 | 14 | It also changes the twill browsing behaviour, so that relative urls 15 | per default point to the intercept (e.g. your Django app), so long 16 | as you don't browse away from that host. Further, you are allowed to 17 | specify the target url as arguments to Django's ``reverse()``. 18 | 19 | Usage: 20 | 21 | from test_utils.utils import twill_runner as twill 22 | twill.setup() 23 | try: 24 | twill.go('/') # --> Django WSGI 25 | twill.code(200) 26 | 27 | twill.go('http://google.com') 28 | twill.go('/services') # --> http://google.com/services 29 | 30 | twill.go('/list', default=True) # --> back to Django WSGI 31 | 32 | twill.go('proj.app.views.func', 33 | args=[1,2,3]) 34 | finally: 35 | twill.teardown() 36 | 37 | For more information about twill, see: 38 | http://twill.idyll.org/ 39 | """ 40 | 41 | # allows us to import global twill as opposed to this module 42 | from __future__ import absolute_import 43 | 44 | # TODO: import all names with a _-prefix to keep the namespace clean with the twill stuff? 45 | import urlparse 46 | import cookielib 47 | 48 | import twill 49 | import twill.commands 50 | import twill.browser 51 | 52 | from django.conf import settings 53 | from django.core.servers.basehttp import AdminMediaHandler 54 | from django.core.handlers.wsgi import WSGIHandler 55 | from django.core.urlresolvers import reverse, NoReverseMatch 56 | from django.http import HttpRequest 57 | from django.utils.datastructures import SortedDict 58 | from django.contrib import auth 59 | from django.core import signals 60 | from django.db import close_connection 61 | 62 | 63 | # make available through this module 64 | from twill.commands import * 65 | 66 | __all__ = ('INSTALLED', 'setup', 'teardown', 'reverse',) + tuple(twill.commands.__all__) 67 | 68 | 69 | DEFAULT_HOST = '127.0.0.1' 70 | DEFAULT_PORT = 9090 71 | INSTALLED = SortedDict() # keep track of the installed hooks 72 | 73 | 74 | class DjangoWsgiFix(object): 75 | """Django closes the database connection after every request; 76 | this breaks the use of transactions in your tests. This wraps 77 | around Django's WSGI interface and will disable the critical 78 | signal handler for every request served. 79 | 80 | Note that we really do need to do this individually a every 81 | request, not just once when our WSGI hook is installed, since 82 | Django's own test client does the same thing; it would reinstall 83 | the signal handler if used in combination with us. 84 | """ 85 | def __init__(self, app): 86 | self.app = app 87 | 88 | def __call__(self, environ, start_response): 89 | signals.request_finished.disconnect(close_connection) 90 | try: 91 | return self.app(environ, start_response) 92 | finally: 93 | signals.request_finished.connect(close_connection) 94 | 95 | 96 | def setup(host=None, port=None, allow_xhtml=True, propagate=True): 97 | """Install the WSGI hook for ``host`` and ``port``. 98 | 99 | The default values will be used if host or port are not specified. 100 | 101 | ``allow_xhtml`` enables a workaround for the "not viewer HTML" 102 | error when browsing sites that are determined to be XHTML, e.g. 103 | featuring xhtml-ish mimetypes. 104 | 105 | Unless ``propagate specifies otherwise``, the 106 | ``DEBUG_PROPAGATE_EXCEPTIONS`` will be enabled for better debugging: 107 | when using twill, we don't really want to see 500 error pages, 108 | but rather directly the exceptions that occured on the view side. 109 | 110 | Multiple calls to this function will only result in one handler 111 | for each host/port combination being installed. 112 | """ 113 | 114 | host = host or DEFAULT_HOST 115 | port = port or DEFAULT_PORT 116 | key = (host, port) 117 | 118 | if not key in INSTALLED: 119 | # installer wsgi handler 120 | app = DjangoWsgiFix(AdminMediaHandler(WSGIHandler())) 121 | twill.add_wsgi_intercept(host, port, lambda: app) 122 | 123 | # start browser fresh 124 | browser = get_browser() 125 | browser.diverged = False 126 | 127 | # enable xhtml mode if requested 128 | _enable_xhtml(browser, allow_xhtml) 129 | 130 | # init debug propagate setting, and remember old value 131 | if propagate: 132 | old_propgate_setting = settings.DEBUG_PROPAGATE_EXCEPTIONS 133 | settings.DEBUG_PROPAGATE_EXCEPTIONS = True 134 | else: 135 | old_propgate_setting = None 136 | 137 | INSTALLED[key] = (app, old_propgate_setting) 138 | return browser 139 | return False 140 | 141 | 142 | def teardown(host=None, port=None): 143 | """Remove an installed WSGI hook for ``host`` and ```port``. 144 | 145 | If no host or port is passed, the default values will be assumed. 146 | If no hook is installed for the defaults, and both the host and 147 | port are missing, the last hook installed will be removed. 148 | 149 | Returns True if a hook was removed, otherwise False. 150 | """ 151 | 152 | both_missing = not host and not port 153 | host = host or DEFAULT_HOST 154 | port = port or DEFAULT_PORT 155 | key = (host, port) 156 | 157 | key_to_delete = None 158 | if key in INSTALLED: 159 | key_to_delete = key 160 | if not key in INSTALLED and both_missing and len(INSTALLED) > 0: 161 | host, port = key_to_delete = INSTALLED.keys()[-1] 162 | 163 | if key_to_delete: 164 | _, old_propagate = INSTALLED[key_to_delete] 165 | del INSTALLED[key_to_delete] 166 | result = True 167 | if old_propagate is not None: 168 | settings.DEBUG_PROPAGATE_EXCEPTIONS = old_propagate 169 | else: 170 | result = False 171 | 172 | # note that our return value is just a guess according to our 173 | # own records, we pass the request on to twill in any case 174 | twill.remove_wsgi_intercept(host, port) 175 | return result 176 | 177 | 178 | def _enable_xhtml(browser, enable): 179 | """Twill (darcs from 19-09-2008) does not work with documents 180 | identifying themselves as XHTML. 181 | 182 | This is a workaround. 183 | """ 184 | factory = browser._browser._factory 185 | factory.basic_factory._response_type_finder._allow_xhtml = \ 186 | factory.soup_factory._response_type_finder._allow_xhtml = \ 187 | enable 188 | 189 | 190 | class _EasyTwillBrowser(twill.browser.TwillBrowser): 191 | """Custom version of twill's browser class that defaults relative 192 | URLs to the last installed hook, if available. 193 | 194 | It also supports reverse resolving, and some additional commands. 195 | """ 196 | 197 | def __init__(self, *args, **kwargs): 198 | self.diverged = False 199 | self._testing_ = False 200 | super(_EasyTwillBrowser, self).__init__(*args, **kwargs) 201 | 202 | def go(self, url, args=None, kwargs=None, default=None): 203 | assert not ((args or kwargs) and default==False) 204 | 205 | try: 206 | url = reverse(url, args=args, kwargs=kwargs) 207 | except NoReverseMatch: 208 | pass 209 | else: 210 | default = True # default is implied 211 | 212 | if INSTALLED: 213 | netloc = '%s:%s' % INSTALLED.keys()[-1] 214 | urlbits = urlparse.urlsplit(url) 215 | if not urlbits[0]: 216 | if default: 217 | # force "undiverge" 218 | self.diverged = False 219 | if not self.diverged: 220 | url = urlparse.urlunsplit(('http', netloc)+urlbits[2:]) 221 | else: 222 | self.diverged = True 223 | 224 | if self._testing_: # hack that makes it simple for us to test this 225 | return url 226 | return super(_EasyTwillBrowser, self).go(url) 227 | 228 | def login(self, **credentials): 229 | """Log the user with the given credentials into your Django 230 | site. 231 | 232 | To further simplify things, rather than giving the credentials, 233 | you may pass a ``user`` parameter with the ``User`` instance you 234 | want to login. Note that in this case the user will not be 235 | further validated, i.e. it is possible to login an inactive user 236 | this way. 237 | 238 | This works regardless of the url currently browsed, but does 239 | require the WSGI intercept to be setup. 240 | 241 | Returns ``True`` if login was possible; ``False`` if the 242 | provided credentials are incorrect, or the user is inactive, 243 | or if the sessions framework is not available. 244 | 245 | Based on ``django.test.client.Client.logout``. 246 | 247 | Note: A ``twill.reload()`` will not refresh the cookies sent 248 | with the request, so your login will not have any effect there. 249 | This is different for ``logout``, since it actually invalidates 250 | the session server-side, thus making the current key invalid. 251 | """ 252 | 253 | if not 'django.contrib.sessions' in settings.INSTALLED_APPS: 254 | return False 255 | 256 | host, port = INSTALLED.keys()[-1] 257 | 258 | # determine the user we want to login 259 | user = credentials.pop('user', None) 260 | if user: 261 | # Login expects the user object to reference it's backend. 262 | # Since we're not going through ``authenticate``, we'll 263 | # have to do this ourselves. 264 | backend = auth.get_backends()[0] 265 | user.backend = user.backend = "%s.%s" % ( 266 | backend.__module__, backend.__class__.__name__) 267 | else: 268 | user = auth.authenticate(**credentials) 269 | if not user or not user.is_active: 270 | return False 271 | 272 | # create a fake request to use with ``auth.login`` 273 | request = HttpRequest() 274 | request.session = __import__(settings.SESSION_ENGINE, {}, {}, ['']).SessionStore() 275 | auth.login(request, user) 276 | request.session.save() 277 | 278 | # set the cookie to represent the session 279 | self.cj.set_cookie(cookielib.Cookie( 280 | version=None, 281 | name=settings.SESSION_COOKIE_NAME, 282 | value=request.session.session_key, 283 | port=str(port), # must be a string 284 | port_specified = False, 285 | domain=host, #settings.SESSION_COOKIE_DOMAIN, 286 | domain_specified=True, 287 | domain_initial_dot=False, 288 | path='/', 289 | path_specified=True, 290 | secure=settings.SESSION_COOKIE_SECURE or None, 291 | expires=None, 292 | discard=None, 293 | comment=None, 294 | comment_url=None, 295 | rest=None 296 | )) 297 | 298 | return True 299 | 300 | def logout(self): 301 | """Log the current user out of your Django site. 302 | 303 | This works regardless of the url currently browsed, but does 304 | require the WSGI intercept to be setup. 305 | 306 | Based on ``django.test.client.Client.logout``. 307 | """ 308 | host, port = INSTALLED.keys()[-1] 309 | for cookie in self.cj: 310 | if cookie.name == settings.SESSION_COOKIE_NAME \ 311 | and cookie.domain==host \ 312 | and (not cookie.port or str(cookie.port)==str(port)): 313 | session = __import__(settings.SESSION_ENGINE, {}, {}, ['']).SessionStore() 314 | session.delete(session_key=cookie.value) 315 | self.cj.clear(cookie.domain, cookie.path, cookie.name) 316 | return True 317 | return False 318 | 319 | 320 | def go(*args, **kwargs): 321 | # replace the default ``go`` to make the additional 322 | # arguments that our custom browser provides available. 323 | browser = get_browser() 324 | browser.go(*args, **kwargs) 325 | return browser.get_url() 326 | 327 | def login(*args, **kwargs): 328 | return get_browser().login(*args, **kwargs) 329 | 330 | def logout(*args, **kwargs): 331 | return get_browser().logout(*args, **kwargs) 332 | 333 | def reset_browser(*args, **kwargs): 334 | # replace the default ``reset_browser`` to ensure 335 | # that our custom browser class is used 336 | result = twill.commands.reset_browser(*args, **kwargs) 337 | twill.commands.browser = _EasyTwillBrowser() 338 | return result 339 | 340 | # Monkey-patch our custom browser into twill; this will be global, but 341 | # will only have an actual effect when intercepts are installed through 342 | # our module (via ``setup``). 343 | # Unfortunately, twill pretty much forces us to use the same global 344 | # state it does itself, lest us reimplement everything from 345 | # ``twill.commands``. It's a bit of a shame, we could provide dedicated 346 | # browser instances for each call to ``setup()``. 347 | reset_browser() 348 | 349 | 350 | def url(should_be=None): 351 | """Like the default ``url()``, but can be called without arguments, 352 | in which case it returns the current url. 353 | """ 354 | 355 | if should_be is None: 356 | return get_browser().get_url() 357 | else: 358 | return twill.commands.url(should_be) 359 | -------------------------------------------------------------------------------- /test_utils/views.py: -------------------------------------------------------------------------------- 1 | from django.http import HttpResponse 2 | 3 | import logging 4 | 5 | from test_utils.testmaker.processors.base import slugify 6 | from test_utils.testmaker import Testmaker 7 | 8 | def set_logging(request, filename=None): 9 | if not filename: 10 | filename = request.REQUEST['filename'] 11 | filename = slugify(filename) 12 | log_file = '/tmp/testmaker/tests/%s_tests_custom.py' % filename 13 | serialize_file = '/tmp/testmaker/tests/%s_serial_custm.py' % filename 14 | tm = Testmaker() 15 | tm.setup_logging(test_file=log_file, serialize_file=serialize_file) 16 | #tm.app_name = 'tmp' 17 | #tm.prepare_test_file() 18 | return HttpResponse('Setup logging %s' % tm.test_file) 19 | 20 | def show_log(request): 21 | file = Testmaker.logfile() 22 | contents = open(file) 23 | return HttpResponse(contents.read(), content_type='text/plain') 24 | HttpResponse() 25 | --------------------------------------------------------------------------------