├── .codecov.yml ├── .gitignore ├── .idea └── vcs.xml ├── .pylintrc ├── .travis.yml ├── HISTORY.rst ├── LICENSE ├── MANIFEST.in ├── README.rst ├── docs └── _static │ ├── retox_demo.gif │ ├── retox_demo_watcher.gif │ └── screenshot.jpeg ├── requirements.txt ├── requirements_dev.txt ├── retox ├── __init__.py ├── __main__.py ├── exclude.py ├── log.py ├── path.py ├── proclimit.py ├── reporter.py ├── service.py ├── ui.py └── watch.py ├── setup.py ├── test ├── __init__.py ├── test_exclude.py ├── test_log.py ├── test_nothing.py ├── test_watch │ ├── file1.py │ └── sub │ │ └── file2.py └── test_watcher.py └── tox.ini /.codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | #notify: 3 | #require_ci_to_pass: yes 4 | 5 | coverage: 6 | precision: 2 # decimal places to display: 0 <= value <= 4 7 | round: nearest 8 | range: 50...90 # custom range of coverage colors from red -> yellow -> green 9 | 10 | status: 11 | project: 12 | default: 13 | threshold: 2% 14 | patch: yes 15 | changes: no 16 | 17 | comment: 18 | layout: "header, diff, tree" 19 | behavior: default 20 | require_changes: false # if true: only post the comment if coverage changes 21 | branches: null 22 | flags: null 23 | paths: null -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .retox.json 2 | src/ 3 | .idea/ 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | env/ 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | local_settings.py 59 | 60 | # Flask stuff: 61 | instance/ 62 | .webassets-cache 63 | 64 | # Scrapy stuff: 65 | .scrapy 66 | 67 | # Sphinx documentation 68 | docs/_build/ 69 | 70 | # PyBuilder 71 | target/ 72 | 73 | # Jupyter Notebook 74 | .ipynb_checkpoints 75 | 76 | # pyenv 77 | .python-version 78 | 79 | # celery beat schedule file 80 | celerybeat-schedule 81 | 82 | # SageMath parsed files 83 | *.sage.py 84 | 85 | # dotenv 86 | .env 87 | 88 | # virtualenv 89 | .venv 90 | venv/ 91 | ENV/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | lib/ 107 | include/ 108 | bin/ 109 | pip-selfcheck.json 110 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | # Add to the black list. It should be a base name, not a 3 | # path. You may set this option multiple times. 4 | ignore=test 5 | 6 | 7 | # Pickle collected data for later comparisons. 8 | persistent=yes 9 | 10 | # List of plugins (as comma separated values of python modules names) to load, 11 | # usually to register additional checkers. 12 | load-plugins= 13 | 14 | 15 | [MESSAGES CONTROL] 16 | disable=redefined-builtin,too-many-arguments,too-few-public-methods,missing-docstring,invalid-name,abstract-method,unbalanced-tuple-unpacking 17 | 18 | 19 | [TYPECHECK] 20 | # List of members which are set dynamically and missed by pylint inference 21 | # system, and so shouldn't trigger E0201 when accessed. Python regular 22 | # expressions are accepted. 23 | generated-members=async_request,objects 24 | 25 | [VARIABLES] 26 | 27 | # Tells whether we should check for unused import in __init__ files. 28 | init-import=no 29 | 30 | # A regular expression matching names used for dummy variables (i.e. not used). 31 | dummy-variables-rgx=_|dummy 32 | 33 | # List of additional names supposed to be defined in builtins. Remember that 34 | # you should avoid to define new builtins when possible. 35 | additional-builtins= 36 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | - "3.4" 5 | - "3.5" 6 | - "3.6" 7 | 8 | install: pip install tox-travis 9 | script: tox -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | Release History 2 | =============== 3 | 4 | 1.4.0 (2017-12-23) 5 | ------------------ 6 | 7 | * Directory watching only includes .py files now, so log files are ignored 8 | * Fix bug where multiple watch folders would not be watched (@bstpierre) 9 | 10 | 1.3.1 (2017-12-19) 11 | ------------------ 12 | 13 | * Fix small issue in readme not rendering on Pypi 14 | 15 | 1.3.0 (2017-12-19) 16 | ------------------ 17 | 18 | * Added a dashboard at the top 19 | * Individual task/action feedback for each virtualenv 20 | * Capture crashes within the threadpool into log files 21 | * Added tests and test structure, more ongoing 22 | * Fixed unicode related issue when running in Python 3.x https://github.com/tonybaloney/retox/issues/1 23 | 24 | 1.2.1 (2017-12-17) 25 | ------------------ 26 | 27 | * Fixed issue where retox command was not starting, with error "TypeError: main() takes exactly 1 argument (0 given)" 28 | See https://github.com/tonybaloney/retox/issues/3 29 | 30 | 1.2.0 (2017-12-15) 31 | ------------------ 32 | 33 | * Support all tox command line parameters, like -e for environment selection 34 | * Fixed issue where exceptions raised by subprocesses could crash host screen 35 | * Removed dependency on click 36 | 37 | 1.1.1 (2017-12-14) 38 | ------------------ 39 | 40 | * Fix crash where venv.status would not resolve on a NoneType resource venv 41 | 42 | 1.1.0 (2017-12-14) 43 | ------------------ 44 | 45 | * Fix issue where after a failed test, the build would fail after install deps 46 | 47 | 1.0.0 (2017-12-14) 48 | ------------------ 49 | 50 | * Initial release -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Permission is hereby granted, free of charge, to any person obtaining a copy 3 | of this software and associated documentation files (the "Software"), to deal 4 | in the Software without restriction, including without limitation the rights 5 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 6 | copies of the Software, and to permit persons to whom the Software is 7 | furnished to do so, subject to the following conditions: 8 | 9 | The above copyright notice and this permission notice shall be included in all 10 | copies or substantial portions of the Software. 11 | 12 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 13 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 14 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 15 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 16 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 17 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 18 | SOFTWARE. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS.rst 2 | include CONTRIBUTING.rst 3 | include HISTORY.rst 4 | include LICENSE 5 | include README.rst 6 | 7 | recursive-include tests * 8 | recursive-exclude * __pycache__ 9 | recursive-exclude * *.py[co] 10 | 11 | recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif 12 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Retox 2 | ===== 3 | 4 | .. image:: https://img.shields.io/pypi/v/retox.svg 5 | :target: https://pypi.python.org/pypi/retox 6 | 7 | .. image:: https://img.shields.io/travis/tonybaloney/retox.svg 8 | :target: https://travis-ci.org/tonybaloney/retox 9 | 10 | .. image:: https://codecov.io/gh/tonybaloney/retox/branch/master/graph/badge.svg 11 | :target: https://codecov.io/gh/tonybaloney/retox 12 | 13 | .. image:: https://pyup.io/repos/github/tonybaloney/retox/shield.svg 14 | :target: https://pyup.io/repos/github/tonybaloney/retox/ 15 | :alt: Updates 16 | 17 | .. image:: https://pyup.io/repos/github/tonybaloney/retox/python-3-shield.svg 18 | :target: https://pyup.io/repos/github/tonybaloney/retox/ 19 | :alt: Python 3 20 | 21 | A command line service that runs your tox tests in parallel, using threading and multicore CPUs. 22 | 23 | See your tox environments in a dashboard and automatically watch source folders for file changes and re-run tests. 24 | 25 | See : https://github.com/tonybaloney/retox/raw/master/docs/_static/screenshot.jpeg for an example screenshot 26 | 27 | .. image:: https://github.com/tonybaloney/retox/raw/master/docs/_static/retox_demo.gif 28 | 29 | Requirements 30 | ------------ 31 | 32 | Linux users may need to install libncurses5-dev before using Tox. If you see an error "ImportError: No module named '_curses'" this is because of the Requirement. 33 | 34 | Usage 35 | ----- 36 | 37 | To install, run 38 | 39 | .. code-block:: bash 40 | 41 | pip install retox 42 | 43 | Then from any project that has a `tox.ini` file setup and using tox, you can simply run 44 | 45 | .. code-block:: bash 46 | 47 | retox 48 | 49 | This will start the service, from where you can press (b) to rebuild on demand. 50 | 51 | Watching folders 52 | ---------------- 53 | 54 | Retox can watch one or many directories for file changes and re-run the tox environments when changes are detected 55 | 56 | .. code-block:: bash 57 | 58 | retox -w my_project_folder -w my_test_folder 59 | 60 | Excluding paths 61 | --------------- 62 | 63 | Retox will ignore files matching a given regex: 64 | 65 | .. code-block:: bash 66 | 67 | retox -w my_project_folder --exclude='.*\.(swp|pyc)$' 68 | 69 | Tox support 70 | ----------- 71 | 72 | Any tox arguments can be given to the command, and using --help to get a full list of commands. Tox arguments will be passed to all virtualenvs 73 | 74 | .. code-block:: bash 75 | 76 | retox -e py27,py36 77 | 78 | multicore configuration 79 | ----------------------- 80 | 81 | The number of concurrent processes in the threadpool can be set using the -n parameter. 82 | By default this will be equal to the number of CPU's on the OS. If you want to expand or throttle this, use the 83 | flag to change the size of the threadpool. 84 | 85 | .. code-block:: bash 86 | 87 | retox -n 4 88 | 89 | Logging 90 | ------- 91 | 92 | 2 files will be created - .retox.log, which is a file for all runs of the logs for the virtual environments. This can be handy to tail to see live output 93 | .retox.json - a JSON file with the virtualenv tasks and specific command output. 94 | 95 | Credits 96 | ------- 97 | 98 | This was inspired by the detox project, which was created by the tox development team. I worked and then significantly changed the way it works 99 | to support re-running environments with ease. 100 | -------------------------------------------------------------------------------- /docs/_static/retox_demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tonybaloney/retox/4635e31001d2ac083423f46766249ac8daca7c9c/docs/_static/retox_demo.gif -------------------------------------------------------------------------------- /docs/_static/retox_demo_watcher.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tonybaloney/retox/4635e31001d2ac083423f46766249ac8daca7c9c/docs/_static/retox_demo_watcher.gif -------------------------------------------------------------------------------- /docs/_static/screenshot.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tonybaloney/retox/4635e31001d2ac083423f46766249ac8daca7c9c/docs/_static/screenshot.jpeg -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tox==3.0.0 2 | eventlet==0.23.0 3 | asciimatics==1.9.0 -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | tox-travis 2 | codecov==2.0.15 3 | pytest==3.6.1 4 | pytest-cov==2.5.1 5 | -------------------------------------------------------------------------------- /retox/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __version__ = '1.4.0' 4 | -------------------------------------------------------------------------------- /retox/__main__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from __future__ import absolute_import 4 | 5 | import time 6 | import re 7 | import sys 8 | 9 | from tox.session import prepare 10 | from asciimatics.screen import Screen 11 | from asciimatics.event import KeyboardEvent 12 | from retox.service import RetoxService 13 | from retox.ui import create_layout 14 | from retox.log import retox_log 15 | from retox.path import Path 16 | 17 | 18 | def main(args=sys.argv): 19 | retox_log.debug("Starting command") 20 | retox_log.info("System stdout encoding is %s" % sys.stdout.encoding) 21 | 22 | # Use the Tox argparse logic 23 | tox_args = prepare(args) 24 | tox_args.option.resultjson = '.retox.json' 25 | 26 | # Custom arguments for watching directories 27 | if tox_args.option.watch is None: 28 | tox_args.option.watch = [] 29 | elif not isinstance(tox_args.option.watch, list): 30 | tox_args.option.watch = [tox_args.option.watch] 31 | 32 | # Start a service and a green pool 33 | screen = Screen.open(unicode_aware=True) 34 | 35 | needs_update = True 36 | running = True 37 | 38 | env_frames, main_scene, log_scene, host_frame = create_layout(tox_args, screen) 39 | 40 | service = RetoxService(tox_args, screen, env_frames) 41 | service.start() 42 | 43 | host_frame.status = 'Starting' 44 | 45 | exclude = tox_args.option.exclude 46 | 47 | # Create a local dictionary of the files to see for differences 48 | _watches = [get_hashes(w, exclude) for w in tox_args.option.watch] 49 | 50 | try: 51 | screen.set_scenes([main_scene], start_scene=main_scene) 52 | 53 | while running: 54 | if needs_update: 55 | host_frame.status = 'Running' 56 | out = service.run(tox_args.envlist) 57 | host_frame.last_result = out 58 | needs_update = False 59 | else: 60 | time.sleep(.5) 61 | 62 | if tox_args.option.watch: 63 | # Refresh the watch folders and check for changes 64 | _new_watches = [get_hashes(w, exclude) for w in tox_args.option.watch] 65 | changes = zip(_watches, _new_watches) 66 | needs_update = any(x != y for x, y in changes) 67 | _watches = _new_watches 68 | 69 | host_frame.status = 'Waiting' 70 | 71 | event = screen.get_event() 72 | if isinstance(event, KeyboardEvent): 73 | if event.key_code == ord('q'): 74 | running = False 75 | elif event.key_code == ord('b'): 76 | needs_update = True 77 | elif event.key_code == ord('r'): 78 | needs_update = True 79 | # elif event.key_code == ord('l'): 80 | # show_logs(screen, log_scene) 81 | 82 | except Exception: 83 | import traceback 84 | retox_log.error("!!!!!! Process crash !!!!!!!") 85 | retox_log.error(traceback.format_exc()) 86 | finally: 87 | # TODO : Extra key for rebuilding tox virtualenvs 88 | retox_log.debug(u"Finished and exiting") 89 | screen.clear() 90 | screen.close(restore=True) 91 | 92 | 93 | def show_logs(screen, log_scene): 94 | screen.set_scenes([log_scene], start_scene=log_scene) 95 | running = True 96 | while running: 97 | event = screen.get_event() 98 | if isinstance(event, KeyboardEvent): 99 | if event.key_code == ord('q'): 100 | running = False 101 | 102 | 103 | def get_hashes(path, exclude=None): 104 | ''' 105 | Get a dictionary of file paths and timestamps. 106 | 107 | Paths matching `exclude` regex will be excluded. 108 | ''' 109 | out = {} 110 | for f in Path(path).rglob('*'): 111 | if f.is_dir(): 112 | # We want to watch files, not directories. 113 | continue 114 | if exclude and re.match(exclude, f.as_posix()): 115 | retox_log.debug("excluding '{}'".format(f.as_posix())) 116 | continue 117 | pytime = f.stat().st_mtime 118 | out[f.as_posix()] = pytime 119 | return out 120 | 121 | 122 | if __name__ == '__main__': 123 | main(sys.argv) 124 | -------------------------------------------------------------------------------- /retox/exclude.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from tox import hookimpl 4 | 5 | 6 | @hookimpl 7 | def tox_addoption(parser): 8 | parser.add_argument( 9 | '--exclude', metavar='REGEX', default=None, 10 | help="Exclude files matching REGEX from being watched") 11 | -------------------------------------------------------------------------------- /retox/log.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import logging 3 | from functools import wraps 4 | 5 | LEVEL = logging.INFO 6 | 7 | 8 | def catch_exceptions(f): 9 | @wraps(f) 10 | def wrapper(*args, **kwds): 11 | try: 12 | return f(*args, **kwds) 13 | except Exception: 14 | import traceback 15 | retox_log.error("!!!!!! Process crash !!!!!!!") 16 | retox_log.error(traceback.format_exc()) 17 | 18 | return wrapper 19 | 20 | 21 | class RetoxLogging(object): 22 | ''' 23 | Create a special logging class that redirects stdout 24 | so it doesnt interfere with the screen session 25 | Write logging output to a file retox.log 26 | ''' 27 | 28 | def __init__(self): 29 | self.logger = logging.getLogger('retox') 30 | self.logger.propagate = False 31 | handler = logging.FileHandler('retox.log') 32 | self.logger.handlers = [] 33 | self.logger.addHandler(handler) 34 | self.logger.level = LEVEL 35 | 36 | def debug(self, *args): 37 | self.logger.debug(*args) 38 | 39 | def info(self, *args): 40 | self.logger.info(*args) 41 | 42 | def warning(self, *args): 43 | self.logger.warning(*args) 44 | 45 | def error(self, *args): 46 | self.logger.error(*args) 47 | 48 | def critical(self, *args): 49 | self.logger.critical(*args) 50 | 51 | def getEffectiveLevel(self): 52 | return self.logger.getEffectiveLevel() 53 | 54 | def setLevel(self, level): 55 | self.logger.setLevel(level) 56 | 57 | def addHandler(self, handler): 58 | self.logger.addHandler(handler) 59 | 60 | 61 | logging.basicConfig(level=logging.ERROR) 62 | 63 | retox_log = RetoxLogging() 64 | -------------------------------------------------------------------------------- /retox/path.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Provide pathlib.Path here, using backported pathlib2 if needed. 4 | try: 5 | from pathlib import Path 6 | Path().expanduser() 7 | except (ImportError, AttributeError): 8 | from pathlib2 import Path 9 | -------------------------------------------------------------------------------- /retox/proclimit.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import argparse 4 | import multiprocessing 5 | 6 | from tox import hookimpl 7 | 8 | 9 | @hookimpl 10 | def tox_addoption(parser): 11 | def positive_integer(value): 12 | ivalue = int(value) 13 | if ivalue <= 0: 14 | raise argparse.ArgumentTypeError( 15 | "%s is an invalid positive int value" % value) 16 | return ivalue 17 | 18 | try: 19 | num_proc = multiprocessing.cpu_count() 20 | except Exception: 21 | num_proc = 2 22 | parser.add_argument( 23 | "-n", "--num", 24 | type=positive_integer, 25 | action="store", 26 | default=num_proc, 27 | dest="numproc", 28 | help="set the number of concurrent processes " 29 | "(default %s)." % num_proc) 30 | -------------------------------------------------------------------------------- /retox/reporter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import tox.session 4 | import eventlet 5 | 6 | from retox.log import retox_log, catch_exceptions 7 | 8 | SHIFT = 20 9 | 10 | 11 | class FakeTerminalWriter(object): 12 | ''' 13 | Redirect TerminalWriter messages to a log file 14 | ''' 15 | hasmarkup = True 16 | 17 | def sep(self, char, message, **kwargs): 18 | return '-' 19 | 20 | def line(self, msg, *args, **kargs): 21 | retox_log.info("tox: " + msg) 22 | 23 | 24 | class RetoxReporter(tox.session.Reporter): 25 | ''' 26 | A custom tox reporter designed for updating a live UI 27 | ''' 28 | 29 | screen = None 30 | env_frames = None 31 | 32 | def __init__(self, session): 33 | ''' 34 | Create a new reporter 35 | 36 | :param session: The Tox Session 37 | :type session: :class:`tox.session.Session` 38 | ''' 39 | super(RetoxReporter, self).__init__(session) 40 | self._actionmayfinish = set() 41 | 42 | # Override default reporter functionality 43 | self.tw = FakeTerminalWriter() 44 | 45 | @classmethod 46 | def set_env_frames(cls, env_frames): 47 | cls.env_frames = env_frames 48 | 49 | def _loopreport(self): 50 | ''' 51 | Loop over the report progress 52 | ''' 53 | while 1: 54 | eventlet.sleep(0.2) 55 | ac2popenlist = {} 56 | for action in self.session._actions: 57 | for popen in action._popenlist: 58 | if popen.poll() is None: 59 | lst = ac2popenlist.setdefault(action.activity, []) 60 | lst.append(popen) 61 | if not action._popenlist and action in self._actionmayfinish: 62 | super(RetoxReporter, self).logaction_finish(action) 63 | self._actionmayfinish.remove(action) 64 | 65 | self.screen.draw_next_frame(repeat=False) 66 | 67 | @catch_exceptions 68 | def logaction_start(self, action): 69 | if action.venv is not None: 70 | retox_log.debug("Started: %s %s" % (action.venv.name, action.activity)) 71 | self.env_frames[action.venv.name].start(action.activity, action) 72 | super(RetoxReporter, self).logaction_start(action) 73 | 74 | @catch_exceptions 75 | def logaction_finish(self, action): 76 | if action.venv is not None: 77 | retox_log.debug("Finished: %s %s" % (action.venv.name, action.activity)) 78 | self.env_frames[action.venv.name].stop(action.activity, action) 79 | super(RetoxReporter, self).logaction_finish(action) 80 | 81 | def error(self, msg): 82 | # TODO : Raise errors in a panel 83 | self.logline("ERROR: " + msg, red=True) 84 | 85 | @catch_exceptions 86 | def startsummary(self): 87 | retox_log.debug("Starting summary") 88 | for frame_name, frame in self.env_frames.items(): 89 | for venv in self.session.venvlist: 90 | if venv.name == frame_name: 91 | try: 92 | frame.finish(venv.status) 93 | except AttributeError: 94 | frame.finish(None) 95 | venv.finish() 96 | 97 | super(RetoxReporter, self).startsummary() 98 | 99 | def reset(self): 100 | self._actionmayfinish = set() 101 | 102 | for _, frame in self.env_frames.items(): 103 | frame.reset() 104 | -------------------------------------------------------------------------------- /retox/service.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from __future__ import absolute_import 4 | 5 | from eventlet.green.subprocess import Popen 6 | import eventlet 7 | import eventlet.debug 8 | from eventlet import GreenPool 9 | 10 | import tox.session 11 | 12 | from retox.reporter import RetoxReporter 13 | from retox.log import retox_log 14 | 15 | 16 | class RetoxService(object): 17 | def __init__(self, toxconfig, screen, env_frames): 18 | self._toxconfig = toxconfig 19 | self._logger = retox_log 20 | self._logger.debug('Instantiated service') 21 | self._resources = Resources(self) 22 | self._sdistpath = None 23 | 24 | RetoxReporter.screen = screen 25 | RetoxReporter.set_env_frames(env_frames) 26 | 27 | self.screen = screen 28 | 29 | # Disabled eventlet dumping exceptions in threads 30 | eventlet.debug.hub_exceptions(False) 31 | eventlet.debug.tpool_exceptions(False) 32 | 33 | def start(self): 34 | eventlet.spawn_n(self.toxsession.report._loopreport) 35 | 36 | @property 37 | def toxsession(self): 38 | try: 39 | return self._toxsession 40 | except AttributeError: 41 | self._logger.debug('Starting new session') 42 | self._toxsession = tox.session.Session( 43 | self._toxconfig, Report=RetoxReporter, popen=Popen) 44 | return self._toxsession 45 | 46 | def run(self, envlist): 47 | self._logger.info(' ') 48 | self._logger.info(' ') 49 | self._logger.info(' ') 50 | self._logger.info(' ---- Starting new test run ----') 51 | 52 | self._toxsession.report.reset() 53 | 54 | pool = GreenPool(size=self._toxconfig.option.numproc) 55 | 56 | for env in envlist: 57 | pool.spawn_n(self.runtests, env) 58 | 59 | pool.waitall() 60 | self.screen.refresh() 61 | 62 | if not self.toxsession.config.option.sdistonly: 63 | retcode = self._toxsession._summary() 64 | return retcode 65 | 66 | def provide_sdist(self): 67 | sdistpath = self.toxsession.get_installpkg_path() 68 | if not sdistpath: 69 | raise SystemExit(1) 70 | return sdistpath 71 | 72 | def provide_venv(self, venvname): 73 | venv = self.toxsession.getvenv(venvname) 74 | if self.toxsession.setupenv(venv): 75 | return venv 76 | 77 | def provide_installpkg(self, venvname, sdistpath): 78 | venv = self.toxsession.getvenv(venvname) 79 | return self.toxsession.installpkg(venv, sdistpath) 80 | 81 | def runtests(self, venvname): 82 | if self.toxsession.config.option.sdistonly: 83 | self._logger.debug('Getting sdist resources') 84 | self._sdistpath = self.getresources("sdist") 85 | 86 | return 87 | if self.toxsession.config.skipsdist: 88 | self._logger.debug('Skipping sdist') 89 | venv_resources = self.getresources("venv:%s" % venvname) 90 | if venv_resources and len(venv_resources) > 0: 91 | self.toxsession.runtestenv(venv_resources[0], redirect=True) 92 | 93 | else: 94 | venv_resources = self.getresources("venv:%s" % venvname, "sdist") 95 | self._sdistpath = venv_resources[1] 96 | self._logger.debug('Running tests') 97 | 98 | if len(venv_resources) > 1: 99 | venv = venv_resources[0] 100 | sdist = venv_resources[1] 101 | venv.status = 0 102 | if self.toxsession.installpkg(venv, sdist): 103 | self.toxsession.runtestenv(venv, redirect=True) 104 | else: 105 | self._logger.debug('Failed installing package') 106 | else: 107 | self._logger.debug('VirtualEnv doesnt exist') 108 | 109 | def getresources(self, *specs): 110 | return self._resources.getresources(*specs) 111 | 112 | 113 | class Resources(object): 114 | def __init__(self, providerbase): 115 | self._providerbase = providerbase 116 | self._spec2thread = {} 117 | self._pool = GreenPool() 118 | self._resources = {} 119 | 120 | def _dispatchprovider(self, spec): 121 | parts = spec.split(":") 122 | name = parts.pop(0) 123 | provider = getattr(self._providerbase, "provide_" + name) 124 | self._resources[spec] = res = provider(*parts) 125 | return res 126 | 127 | def getresources(self, *specs): 128 | for spec in specs: 129 | if spec not in self._resources: 130 | if spec not in self._spec2thread: 131 | t = self._pool.spawn(self._dispatchprovider, spec) 132 | self._spec2thread[spec] = t 133 | lst = [] 134 | for spec in specs: 135 | if spec not in self._resources: 136 | self._spec2thread[spec].wait() 137 | lst.append(self._resources[spec]) 138 | return lst 139 | -------------------------------------------------------------------------------- /retox/ui.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import asciimatics.widgets as widgets 4 | from asciimatics.screen import Screen 5 | from asciimatics.scene import Scene 6 | 7 | from retox.log import retox_log 8 | 9 | TASK_NAMES = { 10 | 'runtests': u"Run Tests", 11 | 'command': u"Custom Command", 12 | 'installdeps': u"Install Dependencies", 13 | 'installpkg': u"Install Package", 14 | 'inst': u"Install", 15 | 'inst-nodeps': u"Install no-deps", 16 | 'sdist-make': u"Make sdist", 17 | 'create': u"Create", 18 | 'recreate': u"Recreate", 19 | 'getenv': u"Get environment" 20 | } 21 | 22 | 23 | RESULT_MESSAGES = { 24 | 0: '[pass]', 25 | 'commands failed': '[fail]', 26 | 1: '[fail]' 27 | } 28 | 29 | 30 | def create_layout(config, screen): 31 | _env_screens = {} 32 | _log_screens = {} 33 | count = 0 34 | 35 | host_frame = RetoxFrame(screen, config) 36 | for env in config.envlist: 37 | _env_screens[env] = VirtualEnvironmentFrame( 38 | screen, 39 | env, 40 | len(config.envlist), 41 | count) 42 | _log_screens[env] = LogFrame( 43 | screen, 44 | env, 45 | len(config.envlist), 46 | count) 47 | count = count + 1 48 | _scene = Scene([host_frame] + [frame for _, frame in _env_screens.items()], -1, name="Retox") 49 | _log_scene = Scene([frame for _, frame in _log_screens.items()], duration=10, name="Logs") 50 | return _env_screens, _scene, _log_scene, host_frame 51 | 52 | 53 | class RetoxRefreshMixin(object): 54 | def refresh(self): 55 | ''' 56 | Refresh the list and the screen 57 | ''' 58 | self._screen.force_update() 59 | self._screen.refresh() 60 | self._update(1) 61 | 62 | 63 | class RetoxFrame(widgets.Frame, RetoxRefreshMixin): 64 | ''' 65 | A UI frame for hosting the details of the overall host 66 | ''' 67 | 68 | def __init__(self, screen, args): 69 | ''' 70 | Create a new frame 71 | 72 | :param screen: The screen instance 73 | :type screen: :class:`asciimatics.screen.Screen` 74 | 75 | :param args: The tox arguments 76 | :type args: ``object`` 77 | ''' 78 | super(RetoxFrame, self).__init__( 79 | screen, 80 | screen.height // 5, 81 | screen.width, 82 | x=0, 83 | y=0, 84 | has_border=True, 85 | hover_focus=True, 86 | title='Retox') 87 | self._screen = screen 88 | self._status = 'Starting' 89 | self._last_result = '' 90 | 91 | # Draw a box for the environment 92 | header_layout = widgets.Layout([10], fill_frame=False) 93 | self.add_layout(header_layout) 94 | 95 | self._status_label = widgets.Label('Status') 96 | header_layout.add_widget(self._status_label) 97 | 98 | self._last_result_label = widgets.Label('Last Result') 99 | header_layout.add_widget(self._last_result_label) 100 | 101 | if args.option.watch: 102 | self._watch_label = widgets.Label('Watching : %s ' % ', '.join(args.option.watch)) 103 | header_layout.add_widget(self._watch_label) 104 | 105 | if args.option.exclude: 106 | self._exclude_label = widgets.Label( 107 | 'Excluding : %s ' % args.option.exclude) 108 | header_layout.add_widget(self._exclude_label) 109 | 110 | self._commands_label = widgets.Label('Commands : (q) quit (b) build') 111 | header_layout.add_widget(self._commands_label) 112 | self.fix() 113 | self.refresh() 114 | 115 | @property 116 | def status(self): 117 | return self._status 118 | 119 | @status.setter 120 | def status(self, value): 121 | self._status = value 122 | self._status_label.text = 'Status : {0}'.format(value) 123 | self.refresh() 124 | 125 | @property 126 | def last_result(self): 127 | return self._last_result 128 | 129 | @last_result.setter 130 | def last_result(self, value): 131 | self._last_result = value 132 | self._last_result_label.text = u'{0} : {1}'.format( 133 | 'Result', 134 | RESULT_MESSAGES.get(value, str(value))) 135 | self.refresh() 136 | 137 | 138 | class LogFrame(widgets.Frame, RetoxRefreshMixin): 139 | ''' 140 | A UI frame for hosting the logs of a virtualenv 141 | ''' 142 | 143 | def __init__(self, screen, venv_name, venv_count, index): 144 | ''' 145 | Create a new frame 146 | 147 | :param screen: The screen instance 148 | :type screen: :class:`asciimatics.screen.Screen` 149 | 150 | :param venv_name: The name of this environment, e.g. py27 151 | :type venv_name: ``str`` 152 | 153 | :param venv_count: How many environments are there? 154 | :type venv_count: ``int`` 155 | 156 | :param index: which environment index is this 157 | :type index: ``int`` 158 | ''' 159 | super(LogFrame, self).__init__( 160 | screen, 161 | screen.height // 2, 162 | screen.width // venv_count, 163 | x=index * (screen.width // venv_count), 164 | has_border=True, 165 | hover_focus=True, 166 | title=venv_name) 167 | self.name = venv_name 168 | self._screen = screen 169 | 170 | # Draw a box for the environment 171 | layout = widgets.Layout([10], fill_frame=False) 172 | self.add_layout(layout) 173 | self._logs = widgets.ListBox( 174 | 10, 175 | [], 176 | name="Logs", 177 | label="Logs") 178 | layout.add_widget(self._logs) 179 | self.fix() 180 | 181 | 182 | class VirtualEnvironmentFrame(widgets.Frame, RetoxRefreshMixin): 183 | ''' 184 | A UI frame for hosting the details of a virtualenv 185 | ''' 186 | 187 | def __init__(self, screen, venv_name, venv_count, index): 188 | ''' 189 | Create a new frame 190 | 191 | :param screen: The screen instance 192 | :type screen: :class:`asciimatics.screen.Screen` 193 | 194 | :param venv_name: The name of this environment, e.g. py27 195 | :type venv_name: ``str`` 196 | 197 | :param venv_count: How many environments are there? 198 | :type venv_count: ``int`` 199 | 200 | :param index: which environment index is this 201 | :type index: ``int`` 202 | ''' 203 | super(VirtualEnvironmentFrame, self).__init__( 204 | screen, 205 | screen.height // 2, 206 | screen.width // venv_count, 207 | x=index * (screen.width // venv_count), 208 | has_border=True, 209 | hover_focus=True, 210 | title=venv_name) 211 | self.name = venv_name 212 | self._screen = screen 213 | 214 | # Draw a box for the environment 215 | task_layout = widgets.Layout([10], fill_frame=False) 216 | self.add_layout(task_layout) 217 | completed_layout = widgets.Layout([10], fill_frame=False) 218 | self.add_layout(completed_layout) 219 | self._task_view = widgets.ListBox( 220 | 10, 221 | [], 222 | name="Tasks", 223 | label="Running") 224 | self._completed_view = widgets.ListBox( 225 | 10, 226 | [], 227 | name="Completed", 228 | label="Completed") 229 | task_layout.add_widget(self._task_view) 230 | completed_layout.add_widget(self._completed_view) 231 | self.fix() 232 | 233 | def start(self, activity, action): 234 | ''' 235 | Mark an action as started 236 | 237 | :param activity: The virtualenv activity name 238 | :type activity: ``str`` 239 | 240 | :param action: The virtualenv action 241 | :type action: :class:`tox.session.Action` 242 | ''' 243 | try: 244 | self._start_action(activity, action) 245 | except ValueError: 246 | retox_log.debug("Could not find action %s in env %s" % (activity, self.name)) 247 | self.refresh() 248 | 249 | def stop(self, activity, action): 250 | ''' 251 | Mark a task as completed 252 | 253 | :param activity: The virtualenv activity name 254 | :type activity: ``str`` 255 | 256 | :param action: The virtualenv action 257 | :type action: :class:`tox.session.Action` 258 | ''' 259 | try: 260 | self._remove_running_action(activity, action) 261 | except ValueError: 262 | retox_log.debug("Could not find action %s in env %s" % (activity, self.name)) 263 | self._mark_action_completed(activity, action) 264 | self.refresh() 265 | 266 | def finish(self, status): 267 | ''' 268 | Move laggard tasks over 269 | 270 | :param activity: The virtualenv status 271 | :type activity: ``str`` 272 | ''' 273 | retox_log.info("Completing %s with status %s" % (self.name, status)) 274 | result = Screen.COLOUR_GREEN if not status else Screen.COLOUR_RED 275 | self.palette['title'] = (Screen.COLOUR_WHITE, Screen.A_BOLD, result) 276 | for item in list(self._task_view.options): 277 | self._task_view.options.remove(item) 278 | self._completed_view.options.append(item) 279 | self.refresh() 280 | 281 | def reset(self): 282 | ''' 283 | Reset the frame between jobs 284 | ''' 285 | self.palette['title'] = (Screen.COLOUR_WHITE, Screen.A_BOLD, Screen.COLOUR_BLUE) 286 | self._completed_view.options = [] 287 | self._task_view.options = [] 288 | self.refresh() 289 | 290 | def _start_action(self, activity, action): 291 | self._task_view.options.append(self._make_list_item_from_action(activity, action)) 292 | 293 | def _remove_running_action(self, activity, action): 294 | self._task_view.options.remove(self._make_list_item_from_action(activity, action)) 295 | 296 | def _mark_action_completed(self, activity, action): 297 | name, value = self._make_list_item_from_action(activity, action) 298 | name = RESULT_MESSAGES.get(action.venv.status, str(action.venv.status)) + ' ' + name 299 | self._completed_view.options.append((name, value)) 300 | 301 | def _make_list_item_from_action(self, activity, action): 302 | return TASK_NAMES.get(activity, activity), self.name 303 | -------------------------------------------------------------------------------- /retox/watch.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from tox import hookimpl 4 | 5 | 6 | @hookimpl 7 | def tox_addoption(parser): 8 | parser.add_argument( 9 | '-w', '--watch', 10 | action='append', 11 | help="Watch a folder for changes and rebuild when detected file changes/new files") 12 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | import retox 4 | 5 | with open('README.rst') as readme: 6 | long_description = readme.read() 7 | 8 | 9 | _version = retox.__version__ 10 | 11 | requirements = [ 12 | 'tox==2.9.1', 13 | 'eventlet==0.21.0', 14 | 'asciimatics==1.9.0', 15 | 'pathlib2==2.3.0', 16 | ] 17 | 18 | 19 | def main(): 20 | setup( 21 | name='retox', 22 | description='A parallel service for tox', 23 | long_description=long_description, 24 | version=_version, 25 | url='https://github.com/tonybaloney/retox', 26 | license='MIT', 27 | platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'], 28 | author='Anthony Shaw', 29 | classifiers=['Development Status :: 4 - Beta', 30 | 'Intended Audience :: Developers', 31 | 'License :: OSI Approved :: MIT License', 32 | 'Operating System :: POSIX', 33 | 'Operating System :: Microsoft :: Windows', 34 | 'Operating System :: MacOS :: MacOS X', 35 | 'Topic :: Software Development :: Testing', 36 | 'Topic :: Software Development :: Libraries', 37 | 'Topic :: Utilities', 38 | 'Programming Language :: Python', 39 | ], 40 | packages=['retox', ], 41 | install_requires=[requirements], 42 | entry_points={'console_scripts': 'retox=retox.__main__:main', 43 | 'tox': ['exclude = retox.exclude', 44 | 'proclimit = retox.proclimit', 45 | 'watch = retox.watch']}, 46 | ) 47 | 48 | if __name__ == '__main__': 49 | main() 50 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tonybaloney/retox/4635e31001d2ac083423f46766249ac8daca7c9c/test/__init__.py -------------------------------------------------------------------------------- /test/test_exclude.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from __future__ import absolute_import 4 | 5 | import argparse 6 | 7 | import retox.exclude 8 | 9 | 10 | def test_exclude_args(): 11 | ''' 12 | Test that `--exclude` flag is handled. 13 | ''' 14 | parser = argparse.ArgumentParser() 15 | retox.exclude.tox_addoption(parser) 16 | args = parser.parse_args(['--exclude=\\.pyc$']) 17 | assert args.exclude == '\\.pyc$' 18 | -------------------------------------------------------------------------------- /test/test_log.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from mock import patch 4 | 5 | from six.moves import cStringIO as StringIO 6 | import retox.log 7 | 8 | 9 | @patch('sys.stdout', new_callable=StringIO) 10 | def test_stdout_redirect(mocked_stdout): 11 | retox.log.retox_log.warning("Test Warning") 12 | assert 'test' not in mocked_stdout.getvalue() 13 | -------------------------------------------------------------------------------- /test/test_nothing.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | def test_nothing(): 5 | assert 1 == 1 6 | -------------------------------------------------------------------------------- /test/test_watch/file1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tonybaloney/retox/4635e31001d2ac083423f46766249ac8daca7c9c/test/test_watch/file1.py -------------------------------------------------------------------------------- /test/test_watch/sub/file2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tonybaloney/retox/4635e31001d2ac083423f46766249ac8daca7c9c/test/test_watch/sub/file2.py -------------------------------------------------------------------------------- /test/test_watcher.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from __future__ import absolute_import 4 | 5 | import argparse 6 | import os 7 | 8 | from retox.__main__ import get_hashes 9 | from retox.path import Path 10 | import retox.watch 11 | 12 | 13 | def test_get_simple_hashes(): 14 | ''' 15 | Test that the hash method returns back both files and files 16 | in subdirectories 17 | ''' 18 | hashes = get_hashes('test/') 19 | assert 'test/test_watch/sub/file2.py' in hashes.keys() 20 | assert 'test/test_watch/file1.py' in hashes.keys() 21 | 22 | 23 | def test_get_simple_hashes_subdir(): 24 | ''' 25 | Test that the hash method returns back only files 26 | in subdirectories 27 | ''' 28 | hashes = get_hashes('test/test_watch/sub') 29 | assert 'test/test_watch/sub/file2.py' in hashes.keys() 30 | assert 'test/test_watch/file1.py' not in hashes.keys() 31 | 32 | 33 | def test_get_simple_hashes_timestamps(): 34 | ''' 35 | Test that the hash method returns back a time 36 | ''' 37 | hashes = get_hashes('test/') 38 | os_time = os.path.getmtime('test/test_watch/sub/file2.py') 39 | assert hashes['test/test_watch/sub/file2.py'] == os_time 40 | 41 | os_time = os.path.getmtime('test/test_watch/file1.py') 42 | assert hashes['test/test_watch/file1.py'] == os_time 43 | 44 | 45 | def test_hashes_dont_include_dirs(): 46 | files = get_hashes('test/') 47 | paths = (Path(f) for f in files) 48 | assert not any(p.is_dir() for p in paths) 49 | 50 | 51 | def test_get_excluded_hashes(): 52 | ''' 53 | Test that the hash method excludes files properly. 54 | ''' 55 | files = get_hashes('test/', exclude='.*watch.*').keys() 56 | 57 | # This file should be excluded because it matches the regex. 58 | assert 'test/test_watcher.py' not in files 59 | 60 | # This directory is excluded, so nothing under it should be present. 61 | assert 'test/test_watch/file1.py' not in files 62 | assert 'test/test_watch/sub/file2.py' not in files 63 | 64 | # And we of course should still see (for example) test_log.py. 65 | assert 'test/test_log.py' in files 66 | 67 | 68 | def test_multiple_watch_args(): 69 | ''' 70 | Test that multiple `-w` flags are respected. 71 | ''' 72 | parser = argparse.ArgumentParser() 73 | retox.watch.tox_addoption(parser) 74 | args = parser.parse_args(['-w', 'dir1', '-w', 'dir2']) 75 | assert 'dir1' in args.watch 76 | assert 'dir2' in args.watch 77 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27, py35, py36, lint, pylint 3 | 4 | [travis] 5 | python = 6 | 2.7: py27 7 | 3.6: py36, lint, pylint 8 | 9 | [testenv] 10 | deps = pytest 11 | six 12 | mock 13 | codecov 14 | pathlib2 15 | pytest-cov 16 | setenv = 17 | PYTHONPATH = {toxinidir}:{toxinidir}/retox 18 | commands = python -m pytest test/ --cov=./retox 19 | codecov --token=5a05d21e-2153-47e3-991d-b53274a6c291 20 | 21 | [testenv:pylint] 22 | deps = pylint 23 | 24 | commands = pylint -E --rcfile=./.pylintrc retox/ 25 | 26 | [testenv:lint] 27 | deps = flake8 28 | commands = flake8 --ignore=E402 --max-line-length=100 retox/ 29 | --------------------------------------------------------------------------------