├── .editorconfig ├── .gitignore ├── .travis.yml ├── AUTHORS.rst ├── HISTORY.rst ├── LICENSE ├── MANIFEST.in ├── README.rst ├── logo.png ├── nose_randomly.py ├── requirements.in ├── requirements.txt ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── fixtures │ ├── abcd_cases.py │ ├── abcd_generator.py │ ├── abcd_package │ │ ├── __init__.py │ │ ├── test_a.py │ │ ├── test_b.py │ │ ├── test_c.py │ │ └── test_d.py │ ├── abcd_tests.py │ ├── random_number.py │ ├── random_number_class.py │ ├── random_number_numpy.py │ └── random_seed_not_100.py └── test_basic.py └── tox.ini /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.py] 14 | multi_line_output=5 15 | known_first_party=nose_randomly 16 | 17 | [*.bat] 18 | indent_style = tab 19 | end_of_line = crlf 20 | 21 | [LICENSE] 22 | insert_final_newline = false 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | .eggs 21 | 22 | # Installer logs 23 | pip-log.txt 24 | 25 | # Unit test / coverage reports 26 | .coverage 27 | .tox 28 | nosetests.xml 29 | htmlcov 30 | 31 | # Translations 32 | *.mo 33 | 34 | # Mr Developer 35 | .mr.developer.cfg 36 | .project 37 | .pydevproject 38 | 39 | # Complexity 40 | output/*.html 41 | output/*/index.html 42 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | dist: trusty 3 | 4 | notifications: 5 | email: false 6 | 7 | language: python 8 | python: '3.6' 9 | cache: pip 10 | 11 | install: pip install tox 12 | 13 | script: tox 14 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Credits 3 | ======= 4 | 5 | Development Lead 6 | ---------------- 7 | 8 | * Adam Johnson 9 | 10 | Contributors 11 | ------------ 12 | 13 | * Étienne BERSAC 14 | * Martin K. Scherer @marscher 15 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | .. :changelog: 2 | 3 | History 4 | ------- 5 | 6 | Pending release 7 | --------------- 8 | 9 | 1.2.6 (2019-02-07) 10 | ------------------ 11 | 12 | * Update PyPI development status as inactive. This package is no longer 13 | maintained, see README.rst. 14 | * Dropped Python 2.6 compatibility, as upstream dependency NumPy did. 15 | 16 | 1.2.5 (2016-10-28) 17 | ------------------ 18 | 19 | * Set a high plugin score to ensure that ``nose-randomly`` is loaded before 20 | other plugins. This fixes a bug where randomization would disapper when using 21 | the ``doctests`` plugin that is included with Nose. 22 | 23 | 1.2.4 (2016-10-27) 24 | ------------------ 25 | 26 | * Reset the random state for NumPy too. 27 | 28 | 1.2.3 (2016-08-19) 29 | ------------------ 30 | 31 | * Fixed output so the random seed is always output when the plugin is enabled, 32 | not just when resetting ``random.seed()`` at the start of tests. Thanks 33 | @amygdalama. 34 | 35 | 1.2.2 (2016-07-06) 36 | ------------------ 37 | 38 | * Fixed to work with ``python setup.py nosetests`` on Python 2 due to issue 39 | with ``unicode`` not working with ``distutils.fancy_getopt``. 40 | 41 | 1.2.1 (2016-06-01) 42 | ------------------ 43 | 44 | * Support test generators. 45 | 46 | 1.2.0 (2015-12-10) 47 | ------------------ 48 | 49 | * Reset the random state for Faker (pip package ``fake-factory``) too 50 | 51 | 1.1.0 (2015-08-27) 52 | ------------------ 53 | 54 | * Reset the random seed at the start of nose test contexts (TestCases 55 | etc.) too 56 | * Slight performance improvement by always using ``random.setstate()`` for 57 | reseeding 58 | 59 | 1.0.0 (2015-07-23) 60 | ------------------ 61 | 62 | * First release on PyPI. 63 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD License 2 | 3 | Copyright (c) 2017-2019 Adam Johnson 4 | 5 | All rights reserved. 6 | 7 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 12 | 13 | * Neither the name of nose-randomly nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS.rst 2 | include HISTORY.rst 3 | include LICENSE 4 | include README.rst 5 | 6 | recursive-include tests * 7 | recursive-exclude * __pycache__ 8 | recursive-exclude * *.py[co] 9 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | nose-randomly 3 | ============= 4 | 5 | .. image:: https://img.shields.io/travis/adamchainz/nose-randomly.svg 6 | :target: https://travis-ci.org/adamchainz/nose-randomly 7 | 8 | .. image:: https://img.shields.io/pypi/v/nose-randomly.svg 9 | :target: https://pypi.python.org/pypi/nose-randomly 10 | 11 | .. figure:: https://raw.githubusercontent.com/adamchainz/nose-randomly/master/logo.png 12 | :scale: 50% 13 | :alt: Randomness power. 14 | 15 | ---- 16 | 17 | **Unmaintained:** I'm no longer maintaining this package because I haven't used 18 | nose for some time, and nose itself has not seen a release since 2015, nor a 19 | commit since 2016 (nearly 3 years at time of writing). If you want to continue 20 | maintenance please contact me. 21 | 22 | ---- 23 | 24 | Nose plugin to randomly order tests and control ``random.seed``. (Also 25 | available `for pytest `_). 26 | 27 | Features 28 | -------- 29 | 30 | All of these features are on by default but can be disabled with flags. 31 | 32 | * Randomly shuffles the submodules, ``TestCase`` classes + test functions when 33 | loading a module of tests. 34 | * Randomly shuffles the test functions inside a ``TestCase`` when loading it. 35 | * Resets ``random.seed()`` at the start of every test case and test to a fixed 36 | number - this defaults to ``time.time()`` from the start of your test run, 37 | but you can pass in ``--randomly-seed`` to repeat a randomness-induced 38 | failure. 39 | * If 40 | `factory boy `_ 41 | is installed, its random state is reset at the start of every test. This 42 | allows for repeatable use of its random 'fuzzy' features. 43 | * If `faker `_ is installed, its 44 | random state is reset at the start of every test. This is also for repeatable 45 | fuzzy data in tests - factory boy uses faker for lots of data. 46 | 47 | About 48 | ----- 49 | 50 | Randomness in testing can be quite powerful to discover hidden flaws in the 51 | tests themselves, as well as giving a little more coverage to your system. 52 | 53 | By randomly ordering the tests, the risk of surprising inter-test dependencies 54 | is reduced - a technique used in many places, for example Google's C++ test 55 | runner `googletest 56 | `_. 57 | 58 | By resetting the random seed to a repeatable number for each test, tests can 59 | create data based on random numbers and yet remain repeatable, for example 60 | factory boy's fuzzy values. This is good for ensuring that tests specify the 61 | data they need and that the tested system is not affected by any data that is 62 | filled in randomly due to not being specified. 63 | 64 | Requirements 65 | ------------ 66 | 67 | Tested with: 68 | 69 | * Python 2.7, 3.6 70 | * The latest version of Nose 71 | 72 | Usage 73 | ----- 74 | 75 | Install from pip with: 76 | 77 | .. code-block:: bash 78 | 79 | pip install nose-randomly 80 | 81 | Nose will automatically find the plugin. 82 | 83 | To activate it on your test run, use the ``--with-randomly`` flag, for example: 84 | 85 | .. code-block:: bash 86 | 87 | nosetests -v --with-randomly 88 | 89 | The output will start with an extra line that tells you the random seed that is 90 | being used: 91 | 92 | .. code-block:: bash 93 | 94 | Using --randomly-seed=1234 95 | test_D (abcd_tests.Tests) ... ok 96 | ... 97 | 98 | If the tests then fail due to ordering or randomly created data, you can then 99 | restart them with that seed: 100 | 101 | .. code-block:: bash 102 | 103 | nosetests -v --with-randomly --randomly-seed=1234 104 | 105 | You can disable behaviours you don't like with the following flags: 106 | 107 | * ``--randomly-dont-shuffle-modules`` - turn off the shuffling of the contents 108 | of modules 109 | * ``--randomly-dont-shuffle-cases`` - turn off the shuffling of test functions 110 | inside ``TestCase`` classes 111 | * ``--randomly-dont-reset-seed`` - turn off the reset of ``random.seed()`` at 112 | the start of every test 113 | 114 | 115 | Background 116 | ---------- 117 | 118 | `nose` has an `unmerged pull request 119 | `_ from 2009 to add 120 | random ordering functionality. This is available in plugin format in the 121 | `nose-randomize `_ package. It 122 | works quite well but I found that since it replaces all of the test loading 123 | machinery inside `nose`, it can interact badly with other plugins. This plugin 124 | was developed as a thinner layer to achieve the same thing, plus the random 125 | seed resetting which was not available before. 126 | 127 | 128 | License 129 | ------- 130 | 131 | * BSD licensed, see LICENSE file 132 | * Logo by Christian Mohr from the Noun Project 133 | (`link `_). 134 | -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adamchainz/nose-randomly/8a3fbeaf7cc5452c44da8c7e7573fe89391c8260/logo.png -------------------------------------------------------------------------------- /nose_randomly.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import division, print_function, unicode_literals 3 | 4 | import random 5 | import sys 6 | import time 7 | 8 | from nose.plugins import Plugin 9 | from nose.suite import ContextList 10 | 11 | # factory-boy 12 | try: 13 | from factory.fuzzy import set_random_state as factory_set_random_state 14 | have_factory_boy = True 15 | except ImportError: 16 | have_factory_boy = False 17 | 18 | # fake-factory 19 | try: 20 | from faker.generator import random as faker_random 21 | have_faker = True 22 | except ImportError: 23 | have_faker = False 24 | 25 | try: 26 | from numpy import random as np_random 27 | have_numpy = True 28 | except ImportError: 29 | have_numpy = False 30 | 31 | # Compat 32 | if sys.version_info[0] == 2: # Python 2 33 | map_return_type = list 34 | else: 35 | map_return_type = map 36 | 37 | 38 | __version__ = '1.2.6' 39 | 40 | 41 | class RandomlyPlugin(Plugin): 42 | name = str('randomly') 43 | score = 10000 # Ensure randomly's logic is executed first 44 | 45 | def options(self, parser, env): 46 | """Register commandline options. 47 | """ 48 | super(RandomlyPlugin, self).options(parser, env) 49 | parser.add_option( 50 | str('--randomly-seed'), action='store', dest='seed', 51 | default=int(time.time()), type=int, 52 | help="""Set the seed that nose-randomly uses. Default behaviour: 53 | use time.time()""" 54 | ) 55 | parser.add_option( 56 | str('--randomly-dont-shuffle-modules'), action='store_false', 57 | dest='shuffle_modules', default=True, 58 | help="Stop nose-randomly from shuffling the tests inside modules" 59 | ) 60 | parser.add_option( 61 | str('--randomly-dont-shuffle-cases'), action='store_false', 62 | dest='shuffle_cases', default=True, 63 | help="""Stop nose-randomly from shuffling the tests inside TestCase 64 | classes""" 65 | ) 66 | parser.add_option( 67 | str('--randomly-dont-reset-seed'), action='store_false', 68 | dest='reset_seed', default=True, 69 | help="""Stop nose-randomly from resetting random.seed() at the 70 | start of every test context (TestCase) and test.""" 71 | ) 72 | 73 | def configure(self, options, conf): 74 | """ 75 | Configure plugin. 76 | """ 77 | super(RandomlyPlugin, self).configure(options, conf) 78 | 79 | if not self.enabled: 80 | return 81 | 82 | self.options = options 83 | 84 | def setOutputStream(self, stream): 85 | if not self.enabled: 86 | return 87 | 88 | self.output_stream = stream 89 | print( 90 | "Using --randomly-seed={seed}".format(seed=self.options.seed), 91 | file=self.output_stream 92 | ) 93 | 94 | def startContext(self, context): 95 | self.reset_random_seed() 96 | 97 | def startTest(self, test): 98 | self.reset_random_seed() 99 | 100 | def reset_random_seed(self): 101 | if not self.enabled: 102 | return 103 | 104 | if self.options.reset_seed: 105 | random.setstate(self.random_state) 106 | 107 | if have_factory_boy: 108 | factory_set_random_state(self.random_state) 109 | 110 | if have_faker: 111 | faker_random.setstate(self.random_state) 112 | 113 | if have_numpy: 114 | np_random.set_state(self.random_state_numpy) 115 | 116 | @property 117 | def random_state(self): 118 | if not hasattr(self, '_random_state'): 119 | random.seed(self.options.seed) 120 | self._random_state = random.getstate() 121 | return self._random_state 122 | 123 | @property 124 | def random_state_numpy(self): 125 | # numpy uses its own random state implementation. 126 | if not have_numpy: 127 | raise RuntimeError('numpy not installed') 128 | if not hasattr(self, '_random_state_numpy'): 129 | np_random.seed(self.options.seed) 130 | self._random_state_numpy = np_random.get_state() 131 | return self._random_state_numpy 132 | 133 | def prepareTestLoader(self, loader): 134 | """ 135 | Randomize the order of tests loaded from modules and from classes. 136 | 137 | This is a hack. We take the class of the existing, passed in loader 138 | (normally nose.loader.Loader) and subclass it to monkey-patch in 139 | shuffle calls for module and case loading when they requested, and then 140 | mutate the existing test loader's class to this new subclass. 141 | 142 | This is somewhat horrible, but nose's plugin infrastructure isn't so 143 | flexible - there is no way to wrap just the loader without risking 144 | interfering with other plugins (if you return anything, no other plugin 145 | may do anything to the loader). 146 | """ 147 | if not self.enabled: 148 | return 149 | 150 | options = self.options 151 | 152 | class ShuffledLoader(loader.__class__): 153 | def loadTestsFromModule(self, *args, **kwargs): 154 | """ 155 | Temporarily wrap self.suiteClass with a function that shuffles 156 | any ContextList instances that the super() call will pass it. 157 | """ 158 | if options.shuffle_modules: 159 | orig_suiteClass = self.suiteClass 160 | 161 | def hackSuiteClass(tests, **kwargs): 162 | if isinstance(tests, ContextList): 163 | random.seed(options.seed) 164 | random.shuffle(tests.tests) 165 | return orig_suiteClass(tests, **kwargs) 166 | 167 | self.suiteClass = hackSuiteClass 168 | suite = super(ShuffledLoader, self).loadTestsFromModule( 169 | *args, **kwargs) 170 | 171 | if options.shuffle_modules: 172 | self.suiteClass = orig_suiteClass 173 | 174 | return suite 175 | 176 | def loadTestsFromTestCase(self, testCaseClass): 177 | """ 178 | Temporarily wrap self.suiteClass with a function that shuffles 179 | any list of tests that the super() call will pass it. 180 | """ 181 | if options.shuffle_cases: 182 | orig_suiteClass = self.suiteClass 183 | 184 | def hackSuiteClass(tests, **kwargs): 185 | if isinstance(tests, map_return_type): 186 | tests = list(tests) 187 | random.seed(options.seed) 188 | random.shuffle(tests) 189 | return orig_suiteClass(tests, **kwargs) 190 | 191 | self.suiteClass = hackSuiteClass 192 | 193 | suite = super(ShuffledLoader, self).loadTestsFromTestCase( 194 | testCaseClass) 195 | 196 | if options.shuffle_cases: 197 | self.suiteClass = orig_suiteClass 198 | 199 | return suite 200 | 201 | # Directly mutate the class of loader... eww 202 | loader.__class__ = ShuffledLoader 203 | 204 | # Tell the plugin infrastructure we did nothing so 'loader', as mutated 205 | # above, continues to be used 206 | return None 207 | -------------------------------------------------------------------------------- /requirements.in: -------------------------------------------------------------------------------- 1 | docutils 2 | flake8 3 | futures<3.2.0 4 | isort 5 | multilint 6 | nose 7 | numpy 8 | Pygments 9 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --output-file requirements.txt requirements.in 6 | # 7 | configparser==3.7.1 # via flake8 8 | docutils==0.14 9 | enum34==1.1.6 # via flake8 10 | flake8==3.6.0 11 | futures==3.1.1 12 | isort==4.3.4 13 | mccabe==0.6.1 # via flake8 14 | multilint==2.4.0 15 | nose==1.3.7 16 | numpy==1.16.0 17 | pycodestyle==2.4.0 # via flake8 18 | pyflakes==2.0.0 # via flake8 19 | pygments==2.3.1 20 | six==1.12.0 # via multilint 21 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | 4 | [flake8] 5 | max-line-length = 120 6 | 7 | [isort] 8 | line_length = 120 9 | multi_line_output = 5 10 | not_skip = __init__.py 11 | 12 | [metadata] 13 | license_file = LICENSE 14 | 15 | [tool:multilint] 16 | paths = nose_randomly.py 17 | setup.py 18 | tests 19 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import codecs 3 | import re 4 | 5 | from setuptools import setup 6 | 7 | 8 | def get_version(filename): 9 | with codecs.open(filename, 'r', 'utf-8') as fp: 10 | contents = fp.read() 11 | return re.search(r"__version__ = ['\"]([^'\"]+)['\"]", contents).group(1) 12 | 13 | 14 | version = get_version('nose_randomly.py') 15 | 16 | with codecs.open('README.rst', 'r', 'utf-8') as readme_file: 17 | readme = readme_file.read() 18 | 19 | with codecs.open('HISTORY.rst', 'r', 'utf-8') as history_file: 20 | history = history_file.read().replace('.. :changelog:', '') 21 | 22 | setup( 23 | name='nose-randomly', 24 | version=version, 25 | description="Nose plugin to randomly order tests and control random.seed.", 26 | long_description=readme + '\n\n' + history, 27 | author="Adam Johnson", 28 | author_email='me@adamj.eu', 29 | url='https://github.com/adamchainz/nose-randomly', 30 | py_modules=['nose_randomly'], 31 | include_package_data=True, 32 | install_requires=[ 33 | 'nose', 34 | ], 35 | python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', 36 | license="BSD", 37 | zip_safe=False, 38 | keywords='nose, random, randomize, randomise, randomly', 39 | entry_points={ 40 | 'nose.plugins.0.10': ['randomly = nose_randomly:RandomlyPlugin'], 41 | }, 42 | classifiers=[ 43 | 'Development Status :: 7 - Inactive', 44 | 'Intended Audience :: Developers', 45 | 'License :: OSI Approved :: BSD License', 46 | 'Natural Language :: English', 47 | "Programming Language :: Python :: 2", 48 | 'Programming Language :: Python :: 2.7', 49 | 'Programming Language :: Python :: 3', 50 | 'Programming Language :: Python :: 3.3', 51 | 'Programming Language :: Python :: 3.4', 52 | 'Programming Language :: Python :: 3.5', 53 | 'Programming Language :: Python :: 3.6', 54 | ], 55 | ) 56 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /tests/fixtures/abcd_cases.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | class A(unittest.TestCase): 5 | def test_it(self): 6 | pass 7 | 8 | 9 | class B(unittest.TestCase): 10 | def test_it(self): 11 | pass 12 | 13 | 14 | class C(unittest.TestCase): 15 | def test_it(self): 16 | pass 17 | 18 | 19 | class D(unittest.TestCase): 20 | def test_it(self): 21 | pass 22 | -------------------------------------------------------------------------------- /tests/fixtures/abcd_generator.py: -------------------------------------------------------------------------------- 1 | def _test_func(arg): 2 | pass 3 | 4 | 5 | def test_generator(): 6 | yield _test_func, 'A' 7 | yield _test_func, 'B' 8 | yield _test_func, 'C' 9 | yield _test_func, 'D' 10 | -------------------------------------------------------------------------------- /tests/fixtures/abcd_package/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adamchainz/nose-randomly/8a3fbeaf7cc5452c44da8c7e7573fe89391c8260/tests/fixtures/abcd_package/__init__.py -------------------------------------------------------------------------------- /tests/fixtures/abcd_package/test_a.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | 4 | class A(TestCase): 5 | def test_it(self): 6 | pass 7 | -------------------------------------------------------------------------------- /tests/fixtures/abcd_package/test_b.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | 4 | class B(TestCase): 5 | def test_it(self): 6 | pass 7 | -------------------------------------------------------------------------------- /tests/fixtures/abcd_package/test_c.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | 4 | class C(TestCase): 5 | def test_it(self): 6 | pass 7 | -------------------------------------------------------------------------------- /tests/fixtures/abcd_package/test_d.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | 4 | class D(TestCase): 5 | def test_it(self): 6 | pass 7 | -------------------------------------------------------------------------------- /tests/fixtures/abcd_tests.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | class Tests(unittest.TestCase): 5 | 6 | def test_A(self): 7 | pass 8 | 9 | def test_B(self): 10 | pass 11 | 12 | def test_C(self): 13 | pass 14 | 15 | def test_D(self): 16 | pass 17 | -------------------------------------------------------------------------------- /tests/fixtures/random_number.py: -------------------------------------------------------------------------------- 1 | import random 2 | import unittest 3 | 4 | 5 | class Tests(unittest.TestCase): 6 | 7 | def test_random(self): 8 | self.assertEqual(random.random(), 0.13436424411240122) 9 | -------------------------------------------------------------------------------- /tests/fixtures/random_number_class.py: -------------------------------------------------------------------------------- 1 | import random 2 | import unittest 3 | 4 | 5 | class Tests(unittest.TestCase): 6 | 7 | @classmethod 8 | def setUpClass(cls): 9 | cls.random_number = random.random() 10 | 11 | def test_random(self): 12 | self.assertEqual(self.random_number, 0.13436424411240122) 13 | 14 | 15 | class TestsAgain(unittest.TestCase): 16 | 17 | @classmethod 18 | def setUpClass(cls): 19 | cls.random_number = random.random() 20 | 21 | def test_random_again(self): 22 | self.assertEqual(self.random_number, 0.13436424411240122) 23 | -------------------------------------------------------------------------------- /tests/fixtures/random_number_numpy.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import numpy as np 4 | 5 | 6 | class Tests(unittest.TestCase): 7 | 8 | def test_random(self): 9 | self.assertEqual(np.random.rand(), 0.417022004702574) 10 | -------------------------------------------------------------------------------- /tests/fixtures/random_seed_not_100.py: -------------------------------------------------------------------------------- 1 | import random 2 | import unittest 3 | 4 | 5 | class Tests(unittest.TestCase): 6 | def test_not_reseeded_to_100(self): 7 | numbers = [random.random() for i in range(100)] 8 | 9 | # Now reseed with 100 to find what the first 100 numbers *would* be 10 | random.seed(100) 11 | numbers_seed_100 = [random.random() for i in range(100)] 12 | 13 | self.assertNotEqual(numbers, numbers_seed_100) 14 | -------------------------------------------------------------------------------- /tests/test_basic.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | import os 5 | import sys 6 | from unittest import TestCase 7 | 8 | from nose.plugins import PluginTester 9 | 10 | from nose_randomly import RandomlyPlugin 11 | 12 | fixtures = os.path.join(os.path.dirname(__file__), 'fixtures') 13 | 14 | 15 | class RandomlyPluginTester(PluginTester): 16 | activate = '--with-randomly' 17 | 18 | def __init__(self, *args, **kwargs): 19 | super(RandomlyPluginTester, self).__init__(*args, **kwargs) 20 | self.maxDiff = 10000 21 | 22 | @property 23 | def plugins(self): 24 | return [RandomlyPlugin()] 25 | 26 | @property 27 | def suitepath(self): 28 | return os.path.join(fixtures, self.fixture_suite) 29 | 30 | def check_output_like(self, lines): 31 | self.output = list(self.output) 32 | output = [line.strip() for line in self.output[:len(lines)]] 33 | self.assertEqual(output, lines) 34 | 35 | 36 | class ShuffledSubmmodulesInPackageTest(RandomlyPluginTester, TestCase): 37 | """ 38 | Check that the submodules inside a package are shuffled. 39 | """ 40 | args = ['-v'] 41 | if sys.version_info >= (3, 0): # Python 3 random changes 42 | args.append('--randomly-seed=15') 43 | else: 44 | args.append('--randomly-seed=41') 45 | 46 | fixture_suite = 'abcd_package' 47 | 48 | def runTest(self): 49 | self.check_output_like([ 50 | 'Using ' + self.args[-1], 51 | 'test_it (abcd_package.test_d.D) ... ok', 52 | 'test_it (abcd_package.test_c.C) ... ok', 53 | 'test_it (abcd_package.test_a.A) ... ok', 54 | 'test_it (abcd_package.test_b.B) ... ok' 55 | ]) 56 | 57 | 58 | class ShuffledCasesInModuleTest(RandomlyPluginTester, TestCase): 59 | """ 60 | Check that the cases inside a module are shuffled. 61 | """ 62 | args = ['-v'] 63 | if sys.version_info >= (3, 0): # Python 3 random changes 64 | args.append('--randomly-seed=38') 65 | else: 66 | args.append('--randomly-seed=56') 67 | 68 | fixture_suite = 'abcd_cases.py' 69 | 70 | def runTest(self): 71 | self.check_output_like([ 72 | 'Using ' + self.args[-1], 73 | 'test_it (abcd_cases.C) ... ok', 74 | 'test_it (abcd_cases.A) ... ok', 75 | 'test_it (abcd_cases.B) ... ok', 76 | 'test_it (abcd_cases.D) ... ok' 77 | ]) 78 | 79 | 80 | class DontShuffledCasesInModuleTest(RandomlyPluginTester, TestCase): 81 | """ 82 | Check that the cases inside a module are not shuffled when the 'dont' flag 83 | is set. 84 | """ 85 | args = ['-v', '--randomly-seed=1', '--randomly-dont-shuffle-modules'] 86 | fixture_suite = 'abcd_cases.py' 87 | 88 | def runTest(self): 89 | self.check_output_like([ 90 | 'Using --randomly-seed=1', 91 | 'test_it (abcd_cases.A) ... ok', 92 | 'test_it (abcd_cases.B) ... ok', 93 | 'test_it (abcd_cases.C) ... ok', 94 | 'test_it (abcd_cases.D) ... ok' 95 | ]) 96 | 97 | 98 | class ShuffledTestsInTestCaseTest(RandomlyPluginTester, TestCase): 99 | """ 100 | Check that the tests inside a case are shuffled. 101 | """ 102 | args = ['-v'] 103 | if sys.version_info >= (3, 0): # Python 3 random changes 104 | args.append('--randomly-seed=126') 105 | else: 106 | args.append('--randomly-seed=1') 107 | 108 | fixture_suite = 'abcd_tests.py' 109 | 110 | def runTest(self): 111 | self.check_output_like([ 112 | 'Using ' + self.args[-1], 113 | 'test_D (abcd_tests.Tests) ... ok', 114 | 'test_B (abcd_tests.Tests) ... ok', 115 | 'test_C (abcd_tests.Tests) ... ok', 116 | 'test_A (abcd_tests.Tests) ... ok' 117 | ]) 118 | 119 | 120 | class DontShuffledTestsInTestCaseTest(RandomlyPluginTester, TestCase): 121 | """ 122 | Check that the tests inside a case are not shuffled when the 'dont' flag is 123 | set. 124 | """ 125 | args = ['-v', '--randomly-seed=1', '--randomly-dont-shuffle-cases'] 126 | fixture_suite = 'abcd_tests.py' 127 | 128 | def runTest(self): 129 | self.check_output_like([ 130 | 'Using --randomly-seed=1', 131 | 'test_A (abcd_tests.Tests) ... ok', 132 | 'test_B (abcd_tests.Tests) ... ok', 133 | 'test_C (abcd_tests.Tests) ... ok', 134 | 'test_D (abcd_tests.Tests) ... ok' 135 | ]) 136 | 137 | 138 | class NotAlwaysOnTests(RandomlyPluginTester, TestCase): 139 | """ 140 | Check that if we don't activate the plugin using --with-randomly, then it 141 | doesn't do any shuffling or output. 142 | """ 143 | activate = '-v' # Abuse of this to fill in args - it can't be None :( 144 | fixture_suite = 'abcd_tests.py' 145 | 146 | def runTest(self): 147 | self.check_output_like([ 148 | 'test_A (abcd_tests.Tests) ... ok', 149 | 'test_B (abcd_tests.Tests) ... ok', 150 | 'test_C (abcd_tests.Tests) ... ok', 151 | 'test_D (abcd_tests.Tests) ... ok' 152 | ]) 153 | 154 | 155 | class RandomSeedTest(RandomlyPluginTester, TestCase): 156 | """ 157 | Check that the random seed is being set. 158 | """ 159 | args = ['-v', '--randomly-seed=1'] 160 | fixture_suite = 'random_number.py' 161 | 162 | def runTest(self): 163 | # Just runs the test - the remaining logic is in the file itself, 164 | # checking that random.random() gives result it should when seed = 1 165 | self.check_output_like([ 166 | 'Using --randomly-seed=1', 167 | 'test_random (random_number.Tests) ... ok' 168 | ]) 169 | 170 | 171 | class RandomSeedTestNumPy(RandomlyPluginTester, TestCase): 172 | """ 173 | Check that the random seed is being set for numpy. 174 | """ 175 | args = ['-v', '--randomly-seed=1'] 176 | fixture_suite = 'random_number_numpy.py' 177 | 178 | def runTest(self): 179 | # Just runs the test - the remaining logic is in the file itself, 180 | # checking that np.random.rand() gives result it should when seed = 1 181 | self.check_output_like([ 182 | 'Using --randomly-seed=1', 183 | 'test_random (random_number_numpy.Tests) ... ok' 184 | ]) 185 | 186 | 187 | class RandomSeedClassTest(RandomlyPluginTester, TestCase): 188 | """ 189 | Check that the random seed is being set for any code that might run in 190 | setUpClass too. 191 | """ 192 | args = ['-v', '--randomly-seed=1'] 193 | fixture_suite = 'random_number_class.py' 194 | 195 | def runTest(self): 196 | # Just runs the test - the remaining logic is in the file itself, 197 | # checking that random.random() gives result it should when seed = 1 198 | self.check_output_like([ 199 | 'Using --randomly-seed=1', 200 | 'test_random_again (random_number_class.TestsAgain) ... ok', 201 | 'test_random (random_number_class.Tests) ... ok', 202 | ]) 203 | 204 | 205 | class DontRandomSeedTest(RandomlyPluginTester, TestCase): 206 | """ 207 | Check that the random seed is being set. 208 | """ 209 | args = ['-v', '--randomly-seed=1', '--randomly-dont-reset-seed'] 210 | fixture_suite = 'random_seed_not_100.py' 211 | 212 | def runTest(self): 213 | # Just runs the test - the remaining logic is in the file itself, 214 | # checking that random.random() does not look like the seed is 100 215 | self.check_output_like([ 216 | 'Using --randomly-seed=1', 217 | 'test_not_reseeded_to_100 (random_seed_not_100.Tests) ... ok' 218 | ]) 219 | 220 | 221 | class GeneratorTest(RandomlyPluginTester, TestCase): 222 | """ 223 | Check generator support. 224 | """ 225 | args = ['-v', '--randomly-seed=83'] 226 | fixture_suite = 'abcd_generator.py' 227 | 228 | def runTest(self): 229 | self.check_output_like([ 230 | "Using --randomly-seed=83", 231 | "abcd_generator.test_generator('A',) ... ok", 232 | "abcd_generator.test_generator('B',) ... ok", 233 | "abcd_generator.test_generator('C',) ... ok", 234 | "abcd_generator.test_generator('D',) ... ok", 235 | ]) 236 | 237 | 238 | class SetuptoolsIntegrationTest(TestCase): 239 | """ 240 | Ensure nose-randomly works when Nose is invoked via setuptools. 241 | See https://github.com/adamchainz/nose-randomly/issues/11 242 | """ 243 | def runTest(self): 244 | import nose.commands 245 | 246 | from setuptools import Command 247 | from distutils.dist import Distribution 248 | from nose.config import Config, user_config_files 249 | from nose.plugins import PluginManager 250 | 251 | class DummyNose(Command): 252 | description = "Dummy" 253 | manager = PluginManager() 254 | manager.plugins = [RandomlyPlugin()] 255 | __config = Config( 256 | files=user_config_files(), 257 | plugins=manager) 258 | __parser = __config.getParser() 259 | user_options = nose.commands.get_user_options(__parser) 260 | 261 | def initialize_options(self): 262 | pass 263 | 264 | def finalize_options(self): 265 | pass 266 | 267 | def run(self): 268 | pass 269 | 270 | dist = Distribution({'cmdclass': {'nosetests': DummyNose}}) 271 | dist.script_args = ['nosetests'] 272 | 273 | # This test should merely not throw an exception 274 | dist.parse_command_line() 275 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | py{27,36}, 4 | py{27,36}-codestyle 5 | 6 | [testenv] 7 | setenv = 8 | PYTHONDONTWRITEBYTECODE=1 9 | install_command = pip install --no-deps {opts} {packages} 10 | commands = nosetests {posargs} 11 | deps = -r{toxinidir}/requirements.txt 12 | 13 | [testenv:py27-codestyle] 14 | # setup.py check broken on travis python 2.7 15 | skip_install = true 16 | commands = multilint --skip setup.py 17 | 18 | [testenv:py36-codestyle] 19 | skip_install = true 20 | commands = multilint 21 | --------------------------------------------------------------------------------