├── .codecov.yml
├── .gitignore
├── .idea
└── vcs.xml
├── .pylintrc
├── .travis.yml
├── HISTORY.rst
├── LICENSE
├── MANIFEST.in
├── README.rst
├── docs
└── _static
│ ├── retox_demo.gif
│ ├── retox_demo_watcher.gif
│ └── screenshot.jpeg
├── requirements.txt
├── requirements_dev.txt
├── retox
├── __init__.py
├── __main__.py
├── exclude.py
├── log.py
├── path.py
├── proclimit.py
├── reporter.py
├── service.py
├── ui.py
└── watch.py
├── setup.py
├── test
├── __init__.py
├── test_exclude.py
├── test_log.py
├── test_nothing.py
├── test_watch
│ ├── file1.py
│ └── sub
│ │ └── file2.py
└── test_watcher.py
└── tox.ini
/.codecov.yml:
--------------------------------------------------------------------------------
1 | codecov:
2 | #notify:
3 | #require_ci_to_pass: yes
4 |
5 | coverage:
6 | precision: 2 # decimal places to display: 0 <= value <= 4
7 | round: nearest
8 | range: 50...90 # custom range of coverage colors from red -> yellow -> green
9 |
10 | status:
11 | project:
12 | default:
13 | threshold: 2%
14 | patch: yes
15 | changes: no
16 |
17 | comment:
18 | layout: "header, diff, tree"
19 | behavior: default
20 | require_changes: false # if true: only post the comment if coverage changes
21 | branches: null
22 | flags: null
23 | paths: null
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .retox.json
2 | src/
3 | .idea/
4 | # Byte-compiled / optimized / DLL files
5 | __pycache__/
6 | *.py[cod]
7 | *$py.class
8 |
9 | # C extensions
10 | *.so
11 |
12 | # Distribution / packaging
13 | .Python
14 | env/
15 | build/
16 | develop-eggs/
17 | dist/
18 | downloads/
19 | eggs/
20 | .eggs/
21 | lib/
22 | lib64/
23 | parts/
24 | sdist/
25 | var/
26 | wheels/
27 | *.egg-info/
28 | .installed.cfg
29 | *.egg
30 |
31 | # PyInstaller
32 | # Usually these files are written by a python script from a template
33 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
34 | *.manifest
35 | *.spec
36 |
37 | # Installer logs
38 | pip-log.txt
39 | pip-delete-this-directory.txt
40 |
41 | # Unit test / coverage reports
42 | htmlcov/
43 | .tox/
44 | .coverage
45 | .coverage.*
46 | .cache
47 | nosetests.xml
48 | coverage.xml
49 | *.cover
50 | .hypothesis/
51 |
52 | # Translations
53 | *.mo
54 | *.pot
55 |
56 | # Django stuff:
57 | *.log
58 | local_settings.py
59 |
60 | # Flask stuff:
61 | instance/
62 | .webassets-cache
63 |
64 | # Scrapy stuff:
65 | .scrapy
66 |
67 | # Sphinx documentation
68 | docs/_build/
69 |
70 | # PyBuilder
71 | target/
72 |
73 | # Jupyter Notebook
74 | .ipynb_checkpoints
75 |
76 | # pyenv
77 | .python-version
78 |
79 | # celery beat schedule file
80 | celerybeat-schedule
81 |
82 | # SageMath parsed files
83 | *.sage.py
84 |
85 | # dotenv
86 | .env
87 |
88 | # virtualenv
89 | .venv
90 | venv/
91 | ENV/
92 |
93 | # Spyder project settings
94 | .spyderproject
95 | .spyproject
96 |
97 | # Rope project settings
98 | .ropeproject
99 |
100 | # mkdocs documentation
101 | /site
102 |
103 | # mypy
104 | .mypy_cache/
105 |
106 | lib/
107 | include/
108 | bin/
109 | pip-selfcheck.json
110 |
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.pylintrc:
--------------------------------------------------------------------------------
1 | [MASTER]
2 | # Add to the black list. It should be a base name, not a
3 | # path. You may set this option multiple times.
4 | ignore=test
5 |
6 |
7 | # Pickle collected data for later comparisons.
8 | persistent=yes
9 |
10 | # List of plugins (as comma separated values of python modules names) to load,
11 | # usually to register additional checkers.
12 | load-plugins=
13 |
14 |
15 | [MESSAGES CONTROL]
16 | disable=redefined-builtin,too-many-arguments,too-few-public-methods,missing-docstring,invalid-name,abstract-method,unbalanced-tuple-unpacking
17 |
18 |
19 | [TYPECHECK]
20 | # List of members which are set dynamically and missed by pylint inference
21 | # system, and so shouldn't trigger E0201 when accessed. Python regular
22 | # expressions are accepted.
23 | generated-members=async_request,objects
24 |
25 | [VARIABLES]
26 |
27 | # Tells whether we should check for unused import in __init__ files.
28 | init-import=no
29 |
30 | # A regular expression matching names used for dummy variables (i.e. not used).
31 | dummy-variables-rgx=_|dummy
32 |
33 | # List of additional names supposed to be defined in builtins. Remember that
34 | # you should avoid to define new builtins when possible.
35 | additional-builtins=
36 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: python
2 | python:
3 | - "2.7"
4 | - "3.4"
5 | - "3.5"
6 | - "3.6"
7 |
8 | install: pip install tox-travis
9 | script: tox
--------------------------------------------------------------------------------
/HISTORY.rst:
--------------------------------------------------------------------------------
1 | Release History
2 | ===============
3 |
4 | 1.4.0 (2017-12-23)
5 | ------------------
6 |
7 | * Directory watching only includes .py files now, so log files are ignored
8 | * Fix bug where multiple watch folders would not be watched (@bstpierre)
9 |
10 | 1.3.1 (2017-12-19)
11 | ------------------
12 |
13 | * Fix small issue in readme not rendering on Pypi
14 |
15 | 1.3.0 (2017-12-19)
16 | ------------------
17 |
18 | * Added a dashboard at the top
19 | * Individual task/action feedback for each virtualenv
20 | * Capture crashes within the threadpool into log files
21 | * Added tests and test structure, more ongoing
22 | * Fixed unicode related issue when running in Python 3.x https://github.com/tonybaloney/retox/issues/1
23 |
24 | 1.2.1 (2017-12-17)
25 | ------------------
26 |
27 | * Fixed issue where retox command was not starting, with error "TypeError: main() takes exactly 1 argument (0 given)"
28 | See https://github.com/tonybaloney/retox/issues/3
29 |
30 | 1.2.0 (2017-12-15)
31 | ------------------
32 |
33 | * Support all tox command line parameters, like -e for environment selection
34 | * Fixed issue where exceptions raised by subprocesses could crash host screen
35 | * Removed dependency on click
36 |
37 | 1.1.1 (2017-12-14)
38 | ------------------
39 |
40 | * Fix crash where venv.status would not resolve on a NoneType resource venv
41 |
42 | 1.1.0 (2017-12-14)
43 | ------------------
44 |
45 | * Fix issue where after a failed test, the build would fail after install deps
46 |
47 | 1.0.0 (2017-12-14)
48 | ------------------
49 |
50 | * Initial release
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 |
2 | Permission is hereby granted, free of charge, to any person obtaining a copy
3 | of this software and associated documentation files (the "Software"), to deal
4 | in the Software without restriction, including without limitation the rights
5 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
6 | copies of the Software, and to permit persons to whom the Software is
7 | furnished to do so, subject to the following conditions:
8 |
9 | The above copyright notice and this permission notice shall be included in all
10 | copies or substantial portions of the Software.
11 |
12 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
13 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
14 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
15 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
16 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
17 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
18 | SOFTWARE.
--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include AUTHORS.rst
2 | include CONTRIBUTING.rst
3 | include HISTORY.rst
4 | include LICENSE
5 | include README.rst
6 |
7 | recursive-include tests *
8 | recursive-exclude * __pycache__
9 | recursive-exclude * *.py[co]
10 |
11 | recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif
12 |
--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
1 | Retox
2 | =====
3 |
4 | .. image:: https://img.shields.io/pypi/v/retox.svg
5 | :target: https://pypi.python.org/pypi/retox
6 |
7 | .. image:: https://img.shields.io/travis/tonybaloney/retox.svg
8 | :target: https://travis-ci.org/tonybaloney/retox
9 |
10 | .. image:: https://codecov.io/gh/tonybaloney/retox/branch/master/graph/badge.svg
11 | :target: https://codecov.io/gh/tonybaloney/retox
12 |
13 | .. image:: https://pyup.io/repos/github/tonybaloney/retox/shield.svg
14 | :target: https://pyup.io/repos/github/tonybaloney/retox/
15 | :alt: Updates
16 |
17 | .. image:: https://pyup.io/repos/github/tonybaloney/retox/python-3-shield.svg
18 | :target: https://pyup.io/repos/github/tonybaloney/retox/
19 | :alt: Python 3
20 |
21 | A command line service that runs your tox tests in parallel, using threading and multicore CPUs.
22 |
23 | See your tox environments in a dashboard and automatically watch source folders for file changes and re-run tests.
24 |
25 | See : https://github.com/tonybaloney/retox/raw/master/docs/_static/screenshot.jpeg for an example screenshot
26 |
27 | .. image:: https://github.com/tonybaloney/retox/raw/master/docs/_static/retox_demo.gif
28 |
29 | Requirements
30 | ------------
31 |
32 | Linux users may need to install libncurses5-dev before using Tox. If you see an error "ImportError: No module named '_curses'" this is because of the Requirement.
33 |
34 | Usage
35 | -----
36 |
37 | To install, run
38 |
39 | .. code-block:: bash
40 |
41 | pip install retox
42 |
43 | Then from any project that has a `tox.ini` file setup and using tox, you can simply run
44 |
45 | .. code-block:: bash
46 |
47 | retox
48 |
49 | This will start the service, from where you can press (b) to rebuild on demand.
50 |
51 | Watching folders
52 | ----------------
53 |
54 | Retox can watch one or many directories for file changes and re-run the tox environments when changes are detected
55 |
56 | .. code-block:: bash
57 |
58 | retox -w my_project_folder -w my_test_folder
59 |
60 | Excluding paths
61 | ---------------
62 |
63 | Retox will ignore files matching a given regex:
64 |
65 | .. code-block:: bash
66 |
67 | retox -w my_project_folder --exclude='.*\.(swp|pyc)$'
68 |
69 | Tox support
70 | -----------
71 |
72 | Any tox arguments can be given to the command, and using --help to get a full list of commands. Tox arguments will be passed to all virtualenvs
73 |
74 | .. code-block:: bash
75 |
76 | retox -e py27,py36
77 |
78 | multicore configuration
79 | -----------------------
80 |
81 | The number of concurrent processes in the threadpool can be set using the -n parameter.
82 | By default this will be equal to the number of CPU's on the OS. If you want to expand or throttle this, use the
83 | flag to change the size of the threadpool.
84 |
85 | .. code-block:: bash
86 |
87 | retox -n 4
88 |
89 | Logging
90 | -------
91 |
92 | 2 files will be created - .retox.log, which is a file for all runs of the logs for the virtual environments. This can be handy to tail to see live output
93 | .retox.json - a JSON file with the virtualenv tasks and specific command output.
94 |
95 | Credits
96 | -------
97 |
98 | This was inspired by the detox project, which was created by the tox development team. I worked and then significantly changed the way it works
99 | to support re-running environments with ease.
100 |
--------------------------------------------------------------------------------
/docs/_static/retox_demo.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tonybaloney/retox/4635e31001d2ac083423f46766249ac8daca7c9c/docs/_static/retox_demo.gif
--------------------------------------------------------------------------------
/docs/_static/retox_demo_watcher.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tonybaloney/retox/4635e31001d2ac083423f46766249ac8daca7c9c/docs/_static/retox_demo_watcher.gif
--------------------------------------------------------------------------------
/docs/_static/screenshot.jpeg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tonybaloney/retox/4635e31001d2ac083423f46766249ac8daca7c9c/docs/_static/screenshot.jpeg
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | tox==3.0.0
2 | eventlet==0.23.0
3 | asciimatics==1.9.0
--------------------------------------------------------------------------------
/requirements_dev.txt:
--------------------------------------------------------------------------------
1 | tox-travis
2 | codecov==2.0.15
3 | pytest==3.6.1
4 | pytest-cov==2.5.1
5 |
--------------------------------------------------------------------------------
/retox/__init__.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | __version__ = '1.4.0'
4 |
--------------------------------------------------------------------------------
/retox/__main__.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | from __future__ import absolute_import
4 |
5 | import time
6 | import re
7 | import sys
8 |
9 | from tox.session import prepare
10 | from asciimatics.screen import Screen
11 | from asciimatics.event import KeyboardEvent
12 | from retox.service import RetoxService
13 | from retox.ui import create_layout
14 | from retox.log import retox_log
15 | from retox.path import Path
16 |
17 |
18 | def main(args=sys.argv):
19 | retox_log.debug("Starting command")
20 | retox_log.info("System stdout encoding is %s" % sys.stdout.encoding)
21 |
22 | # Use the Tox argparse logic
23 | tox_args = prepare(args)
24 | tox_args.option.resultjson = '.retox.json'
25 |
26 | # Custom arguments for watching directories
27 | if tox_args.option.watch is None:
28 | tox_args.option.watch = []
29 | elif not isinstance(tox_args.option.watch, list):
30 | tox_args.option.watch = [tox_args.option.watch]
31 |
32 | # Start a service and a green pool
33 | screen = Screen.open(unicode_aware=True)
34 |
35 | needs_update = True
36 | running = True
37 |
38 | env_frames, main_scene, log_scene, host_frame = create_layout(tox_args, screen)
39 |
40 | service = RetoxService(tox_args, screen, env_frames)
41 | service.start()
42 |
43 | host_frame.status = 'Starting'
44 |
45 | exclude = tox_args.option.exclude
46 |
47 | # Create a local dictionary of the files to see for differences
48 | _watches = [get_hashes(w, exclude) for w in tox_args.option.watch]
49 |
50 | try:
51 | screen.set_scenes([main_scene], start_scene=main_scene)
52 |
53 | while running:
54 | if needs_update:
55 | host_frame.status = 'Running'
56 | out = service.run(tox_args.envlist)
57 | host_frame.last_result = out
58 | needs_update = False
59 | else:
60 | time.sleep(.5)
61 |
62 | if tox_args.option.watch:
63 | # Refresh the watch folders and check for changes
64 | _new_watches = [get_hashes(w, exclude) for w in tox_args.option.watch]
65 | changes = zip(_watches, _new_watches)
66 | needs_update = any(x != y for x, y in changes)
67 | _watches = _new_watches
68 |
69 | host_frame.status = 'Waiting'
70 |
71 | event = screen.get_event()
72 | if isinstance(event, KeyboardEvent):
73 | if event.key_code == ord('q'):
74 | running = False
75 | elif event.key_code == ord('b'):
76 | needs_update = True
77 | elif event.key_code == ord('r'):
78 | needs_update = True
79 | # elif event.key_code == ord('l'):
80 | # show_logs(screen, log_scene)
81 |
82 | except Exception:
83 | import traceback
84 | retox_log.error("!!!!!! Process crash !!!!!!!")
85 | retox_log.error(traceback.format_exc())
86 | finally:
87 | # TODO : Extra key for rebuilding tox virtualenvs
88 | retox_log.debug(u"Finished and exiting")
89 | screen.clear()
90 | screen.close(restore=True)
91 |
92 |
93 | def show_logs(screen, log_scene):
94 | screen.set_scenes([log_scene], start_scene=log_scene)
95 | running = True
96 | while running:
97 | event = screen.get_event()
98 | if isinstance(event, KeyboardEvent):
99 | if event.key_code == ord('q'):
100 | running = False
101 |
102 |
103 | def get_hashes(path, exclude=None):
104 | '''
105 | Get a dictionary of file paths and timestamps.
106 |
107 | Paths matching `exclude` regex will be excluded.
108 | '''
109 | out = {}
110 | for f in Path(path).rglob('*'):
111 | if f.is_dir():
112 | # We want to watch files, not directories.
113 | continue
114 | if exclude and re.match(exclude, f.as_posix()):
115 | retox_log.debug("excluding '{}'".format(f.as_posix()))
116 | continue
117 | pytime = f.stat().st_mtime
118 | out[f.as_posix()] = pytime
119 | return out
120 |
121 |
122 | if __name__ == '__main__':
123 | main(sys.argv)
124 |
--------------------------------------------------------------------------------
/retox/exclude.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | from tox import hookimpl
4 |
5 |
6 | @hookimpl
7 | def tox_addoption(parser):
8 | parser.add_argument(
9 | '--exclude', metavar='REGEX', default=None,
10 | help="Exclude files matching REGEX from being watched")
11 |
--------------------------------------------------------------------------------
/retox/log.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | import logging
3 | from functools import wraps
4 |
5 | LEVEL = logging.INFO
6 |
7 |
8 | def catch_exceptions(f):
9 | @wraps(f)
10 | def wrapper(*args, **kwds):
11 | try:
12 | return f(*args, **kwds)
13 | except Exception:
14 | import traceback
15 | retox_log.error("!!!!!! Process crash !!!!!!!")
16 | retox_log.error(traceback.format_exc())
17 |
18 | return wrapper
19 |
20 |
21 | class RetoxLogging(object):
22 | '''
23 | Create a special logging class that redirects stdout
24 | so it doesnt interfere with the screen session
25 | Write logging output to a file retox.log
26 | '''
27 |
28 | def __init__(self):
29 | self.logger = logging.getLogger('retox')
30 | self.logger.propagate = False
31 | handler = logging.FileHandler('retox.log')
32 | self.logger.handlers = []
33 | self.logger.addHandler(handler)
34 | self.logger.level = LEVEL
35 |
36 | def debug(self, *args):
37 | self.logger.debug(*args)
38 |
39 | def info(self, *args):
40 | self.logger.info(*args)
41 |
42 | def warning(self, *args):
43 | self.logger.warning(*args)
44 |
45 | def error(self, *args):
46 | self.logger.error(*args)
47 |
48 | def critical(self, *args):
49 | self.logger.critical(*args)
50 |
51 | def getEffectiveLevel(self):
52 | return self.logger.getEffectiveLevel()
53 |
54 | def setLevel(self, level):
55 | self.logger.setLevel(level)
56 |
57 | def addHandler(self, handler):
58 | self.logger.addHandler(handler)
59 |
60 |
61 | logging.basicConfig(level=logging.ERROR)
62 |
63 | retox_log = RetoxLogging()
64 |
--------------------------------------------------------------------------------
/retox/path.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | # Provide pathlib.Path here, using backported pathlib2 if needed.
4 | try:
5 | from pathlib import Path
6 | Path().expanduser()
7 | except (ImportError, AttributeError):
8 | from pathlib2 import Path
9 |
--------------------------------------------------------------------------------
/retox/proclimit.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | import argparse
4 | import multiprocessing
5 |
6 | from tox import hookimpl
7 |
8 |
9 | @hookimpl
10 | def tox_addoption(parser):
11 | def positive_integer(value):
12 | ivalue = int(value)
13 | if ivalue <= 0:
14 | raise argparse.ArgumentTypeError(
15 | "%s is an invalid positive int value" % value)
16 | return ivalue
17 |
18 | try:
19 | num_proc = multiprocessing.cpu_count()
20 | except Exception:
21 | num_proc = 2
22 | parser.add_argument(
23 | "-n", "--num",
24 | type=positive_integer,
25 | action="store",
26 | default=num_proc,
27 | dest="numproc",
28 | help="set the number of concurrent processes "
29 | "(default %s)." % num_proc)
30 |
--------------------------------------------------------------------------------
/retox/reporter.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | import tox.session
4 | import eventlet
5 |
6 | from retox.log import retox_log, catch_exceptions
7 |
8 | SHIFT = 20
9 |
10 |
11 | class FakeTerminalWriter(object):
12 | '''
13 | Redirect TerminalWriter messages to a log file
14 | '''
15 | hasmarkup = True
16 |
17 | def sep(self, char, message, **kwargs):
18 | return '-'
19 |
20 | def line(self, msg, *args, **kargs):
21 | retox_log.info("tox: " + msg)
22 |
23 |
24 | class RetoxReporter(tox.session.Reporter):
25 | '''
26 | A custom tox reporter designed for updating a live UI
27 | '''
28 |
29 | screen = None
30 | env_frames = None
31 |
32 | def __init__(self, session):
33 | '''
34 | Create a new reporter
35 |
36 | :param session: The Tox Session
37 | :type session: :class:`tox.session.Session`
38 | '''
39 | super(RetoxReporter, self).__init__(session)
40 | self._actionmayfinish = set()
41 |
42 | # Override default reporter functionality
43 | self.tw = FakeTerminalWriter()
44 |
45 | @classmethod
46 | def set_env_frames(cls, env_frames):
47 | cls.env_frames = env_frames
48 |
49 | def _loopreport(self):
50 | '''
51 | Loop over the report progress
52 | '''
53 | while 1:
54 | eventlet.sleep(0.2)
55 | ac2popenlist = {}
56 | for action in self.session._actions:
57 | for popen in action._popenlist:
58 | if popen.poll() is None:
59 | lst = ac2popenlist.setdefault(action.activity, [])
60 | lst.append(popen)
61 | if not action._popenlist and action in self._actionmayfinish:
62 | super(RetoxReporter, self).logaction_finish(action)
63 | self._actionmayfinish.remove(action)
64 |
65 | self.screen.draw_next_frame(repeat=False)
66 |
67 | @catch_exceptions
68 | def logaction_start(self, action):
69 | if action.venv is not None:
70 | retox_log.debug("Started: %s %s" % (action.venv.name, action.activity))
71 | self.env_frames[action.venv.name].start(action.activity, action)
72 | super(RetoxReporter, self).logaction_start(action)
73 |
74 | @catch_exceptions
75 | def logaction_finish(self, action):
76 | if action.venv is not None:
77 | retox_log.debug("Finished: %s %s" % (action.venv.name, action.activity))
78 | self.env_frames[action.venv.name].stop(action.activity, action)
79 | super(RetoxReporter, self).logaction_finish(action)
80 |
81 | def error(self, msg):
82 | # TODO : Raise errors in a panel
83 | self.logline("ERROR: " + msg, red=True)
84 |
85 | @catch_exceptions
86 | def startsummary(self):
87 | retox_log.debug("Starting summary")
88 | for frame_name, frame in self.env_frames.items():
89 | for venv in self.session.venvlist:
90 | if venv.name == frame_name:
91 | try:
92 | frame.finish(venv.status)
93 | except AttributeError:
94 | frame.finish(None)
95 | venv.finish()
96 |
97 | super(RetoxReporter, self).startsummary()
98 |
99 | def reset(self):
100 | self._actionmayfinish = set()
101 |
102 | for _, frame in self.env_frames.items():
103 | frame.reset()
104 |
--------------------------------------------------------------------------------
/retox/service.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | from __future__ import absolute_import
4 |
5 | from eventlet.green.subprocess import Popen
6 | import eventlet
7 | import eventlet.debug
8 | from eventlet import GreenPool
9 |
10 | import tox.session
11 |
12 | from retox.reporter import RetoxReporter
13 | from retox.log import retox_log
14 |
15 |
16 | class RetoxService(object):
17 | def __init__(self, toxconfig, screen, env_frames):
18 | self._toxconfig = toxconfig
19 | self._logger = retox_log
20 | self._logger.debug('Instantiated service')
21 | self._resources = Resources(self)
22 | self._sdistpath = None
23 |
24 | RetoxReporter.screen = screen
25 | RetoxReporter.set_env_frames(env_frames)
26 |
27 | self.screen = screen
28 |
29 | # Disabled eventlet dumping exceptions in threads
30 | eventlet.debug.hub_exceptions(False)
31 | eventlet.debug.tpool_exceptions(False)
32 |
33 | def start(self):
34 | eventlet.spawn_n(self.toxsession.report._loopreport)
35 |
36 | @property
37 | def toxsession(self):
38 | try:
39 | return self._toxsession
40 | except AttributeError:
41 | self._logger.debug('Starting new session')
42 | self._toxsession = tox.session.Session(
43 | self._toxconfig, Report=RetoxReporter, popen=Popen)
44 | return self._toxsession
45 |
46 | def run(self, envlist):
47 | self._logger.info(' ')
48 | self._logger.info(' ')
49 | self._logger.info(' ')
50 | self._logger.info(' ---- Starting new test run ----')
51 |
52 | self._toxsession.report.reset()
53 |
54 | pool = GreenPool(size=self._toxconfig.option.numproc)
55 |
56 | for env in envlist:
57 | pool.spawn_n(self.runtests, env)
58 |
59 | pool.waitall()
60 | self.screen.refresh()
61 |
62 | if not self.toxsession.config.option.sdistonly:
63 | retcode = self._toxsession._summary()
64 | return retcode
65 |
66 | def provide_sdist(self):
67 | sdistpath = self.toxsession.get_installpkg_path()
68 | if not sdistpath:
69 | raise SystemExit(1)
70 | return sdistpath
71 |
72 | def provide_venv(self, venvname):
73 | venv = self.toxsession.getvenv(venvname)
74 | if self.toxsession.setupenv(venv):
75 | return venv
76 |
77 | def provide_installpkg(self, venvname, sdistpath):
78 | venv = self.toxsession.getvenv(venvname)
79 | return self.toxsession.installpkg(venv, sdistpath)
80 |
81 | def runtests(self, venvname):
82 | if self.toxsession.config.option.sdistonly:
83 | self._logger.debug('Getting sdist resources')
84 | self._sdistpath = self.getresources("sdist")
85 |
86 | return
87 | if self.toxsession.config.skipsdist:
88 | self._logger.debug('Skipping sdist')
89 | venv_resources = self.getresources("venv:%s" % venvname)
90 | if venv_resources and len(venv_resources) > 0:
91 | self.toxsession.runtestenv(venv_resources[0], redirect=True)
92 |
93 | else:
94 | venv_resources = self.getresources("venv:%s" % venvname, "sdist")
95 | self._sdistpath = venv_resources[1]
96 | self._logger.debug('Running tests')
97 |
98 | if len(venv_resources) > 1:
99 | venv = venv_resources[0]
100 | sdist = venv_resources[1]
101 | venv.status = 0
102 | if self.toxsession.installpkg(venv, sdist):
103 | self.toxsession.runtestenv(venv, redirect=True)
104 | else:
105 | self._logger.debug('Failed installing package')
106 | else:
107 | self._logger.debug('VirtualEnv doesnt exist')
108 |
109 | def getresources(self, *specs):
110 | return self._resources.getresources(*specs)
111 |
112 |
113 | class Resources(object):
114 | def __init__(self, providerbase):
115 | self._providerbase = providerbase
116 | self._spec2thread = {}
117 | self._pool = GreenPool()
118 | self._resources = {}
119 |
120 | def _dispatchprovider(self, spec):
121 | parts = spec.split(":")
122 | name = parts.pop(0)
123 | provider = getattr(self._providerbase, "provide_" + name)
124 | self._resources[spec] = res = provider(*parts)
125 | return res
126 |
127 | def getresources(self, *specs):
128 | for spec in specs:
129 | if spec not in self._resources:
130 | if spec not in self._spec2thread:
131 | t = self._pool.spawn(self._dispatchprovider, spec)
132 | self._spec2thread[spec] = t
133 | lst = []
134 | for spec in specs:
135 | if spec not in self._resources:
136 | self._spec2thread[spec].wait()
137 | lst.append(self._resources[spec])
138 | return lst
139 |
--------------------------------------------------------------------------------
/retox/ui.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | import asciimatics.widgets as widgets
4 | from asciimatics.screen import Screen
5 | from asciimatics.scene import Scene
6 |
7 | from retox.log import retox_log
8 |
9 | TASK_NAMES = {
10 | 'runtests': u"Run Tests",
11 | 'command': u"Custom Command",
12 | 'installdeps': u"Install Dependencies",
13 | 'installpkg': u"Install Package",
14 | 'inst': u"Install",
15 | 'inst-nodeps': u"Install no-deps",
16 | 'sdist-make': u"Make sdist",
17 | 'create': u"Create",
18 | 'recreate': u"Recreate",
19 | 'getenv': u"Get environment"
20 | }
21 |
22 |
23 | RESULT_MESSAGES = {
24 | 0: '[pass]',
25 | 'commands failed': '[fail]',
26 | 1: '[fail]'
27 | }
28 |
29 |
30 | def create_layout(config, screen):
31 | _env_screens = {}
32 | _log_screens = {}
33 | count = 0
34 |
35 | host_frame = RetoxFrame(screen, config)
36 | for env in config.envlist:
37 | _env_screens[env] = VirtualEnvironmentFrame(
38 | screen,
39 | env,
40 | len(config.envlist),
41 | count)
42 | _log_screens[env] = LogFrame(
43 | screen,
44 | env,
45 | len(config.envlist),
46 | count)
47 | count = count + 1
48 | _scene = Scene([host_frame] + [frame for _, frame in _env_screens.items()], -1, name="Retox")
49 | _log_scene = Scene([frame for _, frame in _log_screens.items()], duration=10, name="Logs")
50 | return _env_screens, _scene, _log_scene, host_frame
51 |
52 |
53 | class RetoxRefreshMixin(object):
54 | def refresh(self):
55 | '''
56 | Refresh the list and the screen
57 | '''
58 | self._screen.force_update()
59 | self._screen.refresh()
60 | self._update(1)
61 |
62 |
63 | class RetoxFrame(widgets.Frame, RetoxRefreshMixin):
64 | '''
65 | A UI frame for hosting the details of the overall host
66 | '''
67 |
68 | def __init__(self, screen, args):
69 | '''
70 | Create a new frame
71 |
72 | :param screen: The screen instance
73 | :type screen: :class:`asciimatics.screen.Screen`
74 |
75 | :param args: The tox arguments
76 | :type args: ``object``
77 | '''
78 | super(RetoxFrame, self).__init__(
79 | screen,
80 | screen.height // 5,
81 | screen.width,
82 | x=0,
83 | y=0,
84 | has_border=True,
85 | hover_focus=True,
86 | title='Retox')
87 | self._screen = screen
88 | self._status = 'Starting'
89 | self._last_result = ''
90 |
91 | # Draw a box for the environment
92 | header_layout = widgets.Layout([10], fill_frame=False)
93 | self.add_layout(header_layout)
94 |
95 | self._status_label = widgets.Label('Status')
96 | header_layout.add_widget(self._status_label)
97 |
98 | self._last_result_label = widgets.Label('Last Result')
99 | header_layout.add_widget(self._last_result_label)
100 |
101 | if args.option.watch:
102 | self._watch_label = widgets.Label('Watching : %s ' % ', '.join(args.option.watch))
103 | header_layout.add_widget(self._watch_label)
104 |
105 | if args.option.exclude:
106 | self._exclude_label = widgets.Label(
107 | 'Excluding : %s ' % args.option.exclude)
108 | header_layout.add_widget(self._exclude_label)
109 |
110 | self._commands_label = widgets.Label('Commands : (q) quit (b) build')
111 | header_layout.add_widget(self._commands_label)
112 | self.fix()
113 | self.refresh()
114 |
115 | @property
116 | def status(self):
117 | return self._status
118 |
119 | @status.setter
120 | def status(self, value):
121 | self._status = value
122 | self._status_label.text = 'Status : {0}'.format(value)
123 | self.refresh()
124 |
125 | @property
126 | def last_result(self):
127 | return self._last_result
128 |
129 | @last_result.setter
130 | def last_result(self, value):
131 | self._last_result = value
132 | self._last_result_label.text = u'{0} : {1}'.format(
133 | 'Result',
134 | RESULT_MESSAGES.get(value, str(value)))
135 | self.refresh()
136 |
137 |
138 | class LogFrame(widgets.Frame, RetoxRefreshMixin):
139 | '''
140 | A UI frame for hosting the logs of a virtualenv
141 | '''
142 |
143 | def __init__(self, screen, venv_name, venv_count, index):
144 | '''
145 | Create a new frame
146 |
147 | :param screen: The screen instance
148 | :type screen: :class:`asciimatics.screen.Screen`
149 |
150 | :param venv_name: The name of this environment, e.g. py27
151 | :type venv_name: ``str``
152 |
153 | :param venv_count: How many environments are there?
154 | :type venv_count: ``int``
155 |
156 | :param index: which environment index is this
157 | :type index: ``int``
158 | '''
159 | super(LogFrame, self).__init__(
160 | screen,
161 | screen.height // 2,
162 | screen.width // venv_count,
163 | x=index * (screen.width // venv_count),
164 | has_border=True,
165 | hover_focus=True,
166 | title=venv_name)
167 | self.name = venv_name
168 | self._screen = screen
169 |
170 | # Draw a box for the environment
171 | layout = widgets.Layout([10], fill_frame=False)
172 | self.add_layout(layout)
173 | self._logs = widgets.ListBox(
174 | 10,
175 | [],
176 | name="Logs",
177 | label="Logs")
178 | layout.add_widget(self._logs)
179 | self.fix()
180 |
181 |
182 | class VirtualEnvironmentFrame(widgets.Frame, RetoxRefreshMixin):
183 | '''
184 | A UI frame for hosting the details of a virtualenv
185 | '''
186 |
187 | def __init__(self, screen, venv_name, venv_count, index):
188 | '''
189 | Create a new frame
190 |
191 | :param screen: The screen instance
192 | :type screen: :class:`asciimatics.screen.Screen`
193 |
194 | :param venv_name: The name of this environment, e.g. py27
195 | :type venv_name: ``str``
196 |
197 | :param venv_count: How many environments are there?
198 | :type venv_count: ``int``
199 |
200 | :param index: which environment index is this
201 | :type index: ``int``
202 | '''
203 | super(VirtualEnvironmentFrame, self).__init__(
204 | screen,
205 | screen.height // 2,
206 | screen.width // venv_count,
207 | x=index * (screen.width // venv_count),
208 | has_border=True,
209 | hover_focus=True,
210 | title=venv_name)
211 | self.name = venv_name
212 | self._screen = screen
213 |
214 | # Draw a box for the environment
215 | task_layout = widgets.Layout([10], fill_frame=False)
216 | self.add_layout(task_layout)
217 | completed_layout = widgets.Layout([10], fill_frame=False)
218 | self.add_layout(completed_layout)
219 | self._task_view = widgets.ListBox(
220 | 10,
221 | [],
222 | name="Tasks",
223 | label="Running")
224 | self._completed_view = widgets.ListBox(
225 | 10,
226 | [],
227 | name="Completed",
228 | label="Completed")
229 | task_layout.add_widget(self._task_view)
230 | completed_layout.add_widget(self._completed_view)
231 | self.fix()
232 |
233 | def start(self, activity, action):
234 | '''
235 | Mark an action as started
236 |
237 | :param activity: The virtualenv activity name
238 | :type activity: ``str``
239 |
240 | :param action: The virtualenv action
241 | :type action: :class:`tox.session.Action`
242 | '''
243 | try:
244 | self._start_action(activity, action)
245 | except ValueError:
246 | retox_log.debug("Could not find action %s in env %s" % (activity, self.name))
247 | self.refresh()
248 |
249 | def stop(self, activity, action):
250 | '''
251 | Mark a task as completed
252 |
253 | :param activity: The virtualenv activity name
254 | :type activity: ``str``
255 |
256 | :param action: The virtualenv action
257 | :type action: :class:`tox.session.Action`
258 | '''
259 | try:
260 | self._remove_running_action(activity, action)
261 | except ValueError:
262 | retox_log.debug("Could not find action %s in env %s" % (activity, self.name))
263 | self._mark_action_completed(activity, action)
264 | self.refresh()
265 |
266 | def finish(self, status):
267 | '''
268 | Move laggard tasks over
269 |
270 | :param activity: The virtualenv status
271 | :type activity: ``str``
272 | '''
273 | retox_log.info("Completing %s with status %s" % (self.name, status))
274 | result = Screen.COLOUR_GREEN if not status else Screen.COLOUR_RED
275 | self.palette['title'] = (Screen.COLOUR_WHITE, Screen.A_BOLD, result)
276 | for item in list(self._task_view.options):
277 | self._task_view.options.remove(item)
278 | self._completed_view.options.append(item)
279 | self.refresh()
280 |
281 | def reset(self):
282 | '''
283 | Reset the frame between jobs
284 | '''
285 | self.palette['title'] = (Screen.COLOUR_WHITE, Screen.A_BOLD, Screen.COLOUR_BLUE)
286 | self._completed_view.options = []
287 | self._task_view.options = []
288 | self.refresh()
289 |
290 | def _start_action(self, activity, action):
291 | self._task_view.options.append(self._make_list_item_from_action(activity, action))
292 |
293 | def _remove_running_action(self, activity, action):
294 | self._task_view.options.remove(self._make_list_item_from_action(activity, action))
295 |
296 | def _mark_action_completed(self, activity, action):
297 | name, value = self._make_list_item_from_action(activity, action)
298 | name = RESULT_MESSAGES.get(action.venv.status, str(action.venv.status)) + ' ' + name
299 | self._completed_view.options.append((name, value))
300 |
301 | def _make_list_item_from_action(self, activity, action):
302 | return TASK_NAMES.get(activity, activity), self.name
303 |
--------------------------------------------------------------------------------
/retox/watch.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | from tox import hookimpl
4 |
5 |
6 | @hookimpl
7 | def tox_addoption(parser):
8 | parser.add_argument(
9 | '-w', '--watch',
10 | action='append',
11 | help="Watch a folder for changes and rebuild when detected file changes/new files")
12 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | from setuptools import setup
2 |
3 | import retox
4 |
5 | with open('README.rst') as readme:
6 | long_description = readme.read()
7 |
8 |
9 | _version = retox.__version__
10 |
11 | requirements = [
12 | 'tox==2.9.1',
13 | 'eventlet==0.21.0',
14 | 'asciimatics==1.9.0',
15 | 'pathlib2==2.3.0',
16 | ]
17 |
18 |
19 | def main():
20 | setup(
21 | name='retox',
22 | description='A parallel service for tox',
23 | long_description=long_description,
24 | version=_version,
25 | url='https://github.com/tonybaloney/retox',
26 | license='MIT',
27 | platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'],
28 | author='Anthony Shaw',
29 | classifiers=['Development Status :: 4 - Beta',
30 | 'Intended Audience :: Developers',
31 | 'License :: OSI Approved :: MIT License',
32 | 'Operating System :: POSIX',
33 | 'Operating System :: Microsoft :: Windows',
34 | 'Operating System :: MacOS :: MacOS X',
35 | 'Topic :: Software Development :: Testing',
36 | 'Topic :: Software Development :: Libraries',
37 | 'Topic :: Utilities',
38 | 'Programming Language :: Python',
39 | ],
40 | packages=['retox', ],
41 | install_requires=[requirements],
42 | entry_points={'console_scripts': 'retox=retox.__main__:main',
43 | 'tox': ['exclude = retox.exclude',
44 | 'proclimit = retox.proclimit',
45 | 'watch = retox.watch']},
46 | )
47 |
48 | if __name__ == '__main__':
49 | main()
50 |
--------------------------------------------------------------------------------
/test/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tonybaloney/retox/4635e31001d2ac083423f46766249ac8daca7c9c/test/__init__.py
--------------------------------------------------------------------------------
/test/test_exclude.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | from __future__ import absolute_import
4 |
5 | import argparse
6 |
7 | import retox.exclude
8 |
9 |
10 | def test_exclude_args():
11 | '''
12 | Test that `--exclude` flag is handled.
13 | '''
14 | parser = argparse.ArgumentParser()
15 | retox.exclude.tox_addoption(parser)
16 | args = parser.parse_args(['--exclude=\\.pyc$'])
17 | assert args.exclude == '\\.pyc$'
18 |
--------------------------------------------------------------------------------
/test/test_log.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | from mock import patch
4 |
5 | from six.moves import cStringIO as StringIO
6 | import retox.log
7 |
8 |
9 | @patch('sys.stdout', new_callable=StringIO)
10 | def test_stdout_redirect(mocked_stdout):
11 | retox.log.retox_log.warning("Test Warning")
12 | assert 'test' not in mocked_stdout.getvalue()
13 |
--------------------------------------------------------------------------------
/test/test_nothing.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 |
4 | def test_nothing():
5 | assert 1 == 1
6 |
--------------------------------------------------------------------------------
/test/test_watch/file1.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tonybaloney/retox/4635e31001d2ac083423f46766249ac8daca7c9c/test/test_watch/file1.py
--------------------------------------------------------------------------------
/test/test_watch/sub/file2.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tonybaloney/retox/4635e31001d2ac083423f46766249ac8daca7c9c/test/test_watch/sub/file2.py
--------------------------------------------------------------------------------
/test/test_watcher.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | from __future__ import absolute_import
4 |
5 | import argparse
6 | import os
7 |
8 | from retox.__main__ import get_hashes
9 | from retox.path import Path
10 | import retox.watch
11 |
12 |
13 | def test_get_simple_hashes():
14 | '''
15 | Test that the hash method returns back both files and files
16 | in subdirectories
17 | '''
18 | hashes = get_hashes('test/')
19 | assert 'test/test_watch/sub/file2.py' in hashes.keys()
20 | assert 'test/test_watch/file1.py' in hashes.keys()
21 |
22 |
23 | def test_get_simple_hashes_subdir():
24 | '''
25 | Test that the hash method returns back only files
26 | in subdirectories
27 | '''
28 | hashes = get_hashes('test/test_watch/sub')
29 | assert 'test/test_watch/sub/file2.py' in hashes.keys()
30 | assert 'test/test_watch/file1.py' not in hashes.keys()
31 |
32 |
33 | def test_get_simple_hashes_timestamps():
34 | '''
35 | Test that the hash method returns back a time
36 | '''
37 | hashes = get_hashes('test/')
38 | os_time = os.path.getmtime('test/test_watch/sub/file2.py')
39 | assert hashes['test/test_watch/sub/file2.py'] == os_time
40 |
41 | os_time = os.path.getmtime('test/test_watch/file1.py')
42 | assert hashes['test/test_watch/file1.py'] == os_time
43 |
44 |
45 | def test_hashes_dont_include_dirs():
46 | files = get_hashes('test/')
47 | paths = (Path(f) for f in files)
48 | assert not any(p.is_dir() for p in paths)
49 |
50 |
51 | def test_get_excluded_hashes():
52 | '''
53 | Test that the hash method excludes files properly.
54 | '''
55 | files = get_hashes('test/', exclude='.*watch.*').keys()
56 |
57 | # This file should be excluded because it matches the regex.
58 | assert 'test/test_watcher.py' not in files
59 |
60 | # This directory is excluded, so nothing under it should be present.
61 | assert 'test/test_watch/file1.py' not in files
62 | assert 'test/test_watch/sub/file2.py' not in files
63 |
64 | # And we of course should still see (for example) test_log.py.
65 | assert 'test/test_log.py' in files
66 |
67 |
68 | def test_multiple_watch_args():
69 | '''
70 | Test that multiple `-w` flags are respected.
71 | '''
72 | parser = argparse.ArgumentParser()
73 | retox.watch.tox_addoption(parser)
74 | args = parser.parse_args(['-w', 'dir1', '-w', 'dir2'])
75 | assert 'dir1' in args.watch
76 | assert 'dir2' in args.watch
77 |
--------------------------------------------------------------------------------
/tox.ini:
--------------------------------------------------------------------------------
1 | [tox]
2 | envlist = py27, py35, py36, lint, pylint
3 |
4 | [travis]
5 | python =
6 | 2.7: py27
7 | 3.6: py36, lint, pylint
8 |
9 | [testenv]
10 | deps = pytest
11 | six
12 | mock
13 | codecov
14 | pathlib2
15 | pytest-cov
16 | setenv =
17 | PYTHONPATH = {toxinidir}:{toxinidir}/retox
18 | commands = python -m pytest test/ --cov=./retox
19 | codecov --token=5a05d21e-2153-47e3-991d-b53274a6c291
20 |
21 | [testenv:pylint]
22 | deps = pylint
23 |
24 | commands = pylint -E --rcfile=./.pylintrc retox/
25 |
26 | [testenv:lint]
27 | deps = flake8
28 | commands = flake8 --ignore=E402 --max-line-length=100 retox/
29 |
--------------------------------------------------------------------------------