├── .gitignore ├── .travis.yml ├── COPYING ├── LICENSE.txt ├── MANIFEST.in ├── README ├── README.md ├── TODO.md ├── mutornadomon ├── __init__.py ├── collectors │ ├── __init__.py │ ├── ioloop_util.py │ └── web.py ├── config.py ├── external_interfaces │ ├── __init__.py │ ├── http_endpoints.py │ └── publish.py ├── monitor.py ├── net.py └── utils.py ├── requirements-py2.txt ├── requirements-test.txt ├── requirements.txt ├── sample_application.py ├── setup.cfg ├── setup.py └── tests ├── __init__.py ├── test_basic.py ├── test_config.py ├── test_external_interfaces.py └── test_net.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | 3 | env/ 4 | 5 | dist/ 6 | sdist/ 7 | build/ 8 | *.egg-info/ 9 | *.egg_info/ 10 | .idea 11 | .tox 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | - "3.4" 5 | - "3.5" 6 | - "3.6" 7 | - "3.7" 8 | install: 9 | - "pip install -r requirements.txt -r requirements-test.txt" 10 | - "if [[ ${TRAVIS_PYTHON_VERSION::1} != '3' ]]; then pip install -r requirements-py2.txt; fi" 11 | - "pip freeze" 12 | script: "nosetests --with-coverage --cover-package=mutornadomon" 13 | after_success: coveralls 14 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | LICENSE.txt -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Uber Technologies, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to 5 | deal in the Software without restriction, including without limitation the 6 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 7 | sell copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 19 | IN THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.txt 2 | include *.md 3 | include COPYING 4 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | README.md -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/uber/mutornadomon.png)](https://travis-ci.org/uber/mutornadomon) 2 | [![Coverage Status](https://coveralls.io/repos/uber/mutornadomon/badge.svg?branch=master&service=github)](https://coveralls.io/github/uber/mutornadomon?branch=master) 3 | 4 | # mutornadomon 5 | 6 | **µtornadomon** is a library designed to be used with Tornado web applications. It adds an endpoint 7 | (`/mutornadomon`) to HTTP servers which outputs application statistics for use with standard metric 8 | collectors. 9 | 10 | # Usage 11 | 12 | The monitor is initialized using `mutornadomon.config.initialize_mutornadomon`. 13 | 14 | ## Exposing an HTTP endpoint 15 | 16 | If you only pass a tornado web application, it will include request/response statistics, 17 | and expose an HTTP endpoint for polling by external processes: 18 | 19 | ``` 20 | from mutornadomon.config import initialize_mutornadomon 21 | import signal 22 | 23 | [...] 24 | 25 | application = tornado.web.Application(...) 26 | monitor = initialize_mutornadomon(application) 27 | 28 | def shut_down(*args): 29 | monitor.stop() 30 | some_other_application_stop_function() 31 | tornado.ioloop.IOLoop.current().stop() 32 | 33 | for sig in (signal.SIGQUIT, signal.SIGINT, signal.SIGTERM): 34 | signal.signal(sig, shut_down) 35 | ``` 36 | 37 | This will add a `/mutornadomon` endpoint to the web application. 38 | 39 | Here is an example request to that endpoint: 40 | 41 | ``` 42 | $ curl http://localhost:8080/mutornadomon 43 | {"process": {"uptime": 38.98995113372803, "num_fds": 8, "meminfo": {"rss_bytes": 14020608, "vsz_bytes": 2530562048}, "cpu": {"num_threads": 1, "system_time": 0.049356776, "user_time": 0.182635456}}, "max_gauges": {"ioloop_pending_callbacks": 0, "ioloop_handlers": 2, "ioloop_excess_callback_latency": 0.0006290912628173773}, "min_gauges": {"ioloop_pending_callbacks": 0, "ioloop_handlers": 2, "ioloop_excess_callback_latency": -0.004179096221923834}, "gauges": {"ioloop_pending_callbacks": 0, "ioloop_handlers": 2, "ioloop_excess_callback_latency": 0.0006290912628173773}, "counters": {"callbacks": 388, "requests": 6, "localhost_requests": 6, "private_requests": 6}} 44 | ``` 45 | 46 | If you want to add your own metrics, you can do so by calling the `.kv()` or 47 | `.count()` methods on the monitor object at any time. 48 | 49 | The HTTP endpoint is restricted to only respond to request from loopback. 50 | 51 | ## Providing a publishing callback 52 | 53 | Alternatively, instead of polling the HTTP interface, you can pass in a `publisher` callback: 54 | 55 | ``` 56 | import pprint 57 | 58 | def publisher(metrics): 59 | pprint.pprint(metrics) 60 | 61 | monitor = initialize_mutornadomon(application, publisher=publisher) 62 | ``` 63 | 64 | By default, this will call the publisher callback every 10 seconds. 65 | To override this pass the `publish_interval` parameter (in miliseconds). 66 | 67 | ## Monitoring non-web applications 68 | 69 | If you don't pass an application object, other stats can still be collected: 70 | 71 | ``` 72 | import pprint 73 | 74 | def publisher(metrics): 75 | pprint.pprint(metrics) 76 | 77 | monitor = initialize_mutornadomon(publisher=publisher) 78 | ``` 79 | 80 | This only works with the publisher callback interface. 81 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | - probably multithreading / multiprocessing support or something 2 | - hook into RequestHandler to figure out how many requests are currently in-flight 3 | -------------------------------------------------------------------------------- /mutornadomon/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .monitor import MuTornadoMon # noqa 4 | from .config import initialize_mutornadomon # noqa 5 | 6 | version_info = (0, 4, 0) 7 | __name__ = 'MuTornadoMon' 8 | __author__ = 'dev@uber.com' 9 | __version__ = '.'.join(map(str, version_info)) 10 | __license__ = 'MIT' 11 | 12 | __all__ = ['MuTornadoMon', initialize_mutornadomon, 'version_info'] 13 | -------------------------------------------------------------------------------- /mutornadomon/collectors/__init__.py: -------------------------------------------------------------------------------- 1 | from mutornadomon.collectors.web import WebCollector # noqa 2 | -------------------------------------------------------------------------------- /mutornadomon/collectors/ioloop_util.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | 4 | class UtilizationCollector(object): 5 | """Collects stats for overall callback durations""" 6 | 7 | def __init__(self, monitor): 8 | self.monitor = monitor 9 | self.monitor.profiler_init = False 10 | self.monitor.profiler_running = False 11 | self.monitor.profiler = None 12 | 13 | def start(self): 14 | self.original_run_callback = self.monitor.io_loop._run_callback 15 | self.original_add_handler = self.monitor.io_loop.add_handler 16 | 17 | def enable_profiler(): 18 | """Enables profiler if required""" 19 | if not self.monitor.profiler_running: 20 | self.monitor.profiler.enable() 21 | self.monitor.profiler_running = True 22 | 23 | def update_callback_stats(start_time): 24 | """Update callback stats""" 25 | duration = (time.time() - start_time) 26 | self.monitor.count('callback_duration', duration) 27 | self.monitor.count('callbacks_processed', 1) 28 | 29 | def run_timed_callback(callback): 30 | if self.monitor.profiler_init: 31 | enable_profiler() 32 | 33 | start_time = time.time() 34 | result = self.original_run_callback(callback) 35 | update_callback_stats(start_time) 36 | 37 | return result 38 | 39 | def add_timed_handler(fd, handler, events): 40 | def timed_handler(*args, **kwargs): 41 | if self.monitor.profiler_init: 42 | enable_profiler() 43 | 44 | start_time = time.time() 45 | result = handler(*args, **kwargs) 46 | update_callback_stats(start_time) 47 | 48 | return result 49 | 50 | self.original_add_handler(fd, timed_handler, events) 51 | 52 | self.monitor.io_loop.add_handler = add_timed_handler 53 | self.monitor.io_loop._run_callback = run_timed_callback 54 | 55 | def stop(self): 56 | self.monitor.io_loop._run_callback = self.original_run_callback 57 | self.monitor.io_loop.add_handler = self.original_add_handler 58 | -------------------------------------------------------------------------------- /mutornadomon/collectors/web.py: -------------------------------------------------------------------------------- 1 | from mutornadomon import net 2 | 3 | 4 | class NullTransform(object): 5 | 6 | def transform_first_chunk(self, status_code, headers, chunk, *args, **kwargs): 7 | return status_code, headers, chunk 8 | 9 | def transform_chunk(self, chunk, *args, **kwargs): 10 | return chunk 11 | 12 | 13 | class WebCollector(object): 14 | """Collects stats from a tornado.web.application.Application""" 15 | 16 | def __init__(self, monitor, tornado_app): 17 | self.monitor = monitor 18 | self.tornado_app = tornado_app 19 | 20 | def start(self): 21 | self.tornado_app.add_transform(self._request) 22 | 23 | def _request(self, request): 24 | self.monitor.count('requests', 1) 25 | if net.is_local_address(request.remote_ip): 26 | self.monitor.count('localhost_requests', 1) 27 | if net.is_private_address(request.remote_ip): 28 | self.monitor.count('private_requests', 1) 29 | return NullTransform() 30 | -------------------------------------------------------------------------------- /mutornadomon/config.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | 4 | from mutornadomon import MuTornadoMon 5 | from mutornadomon.external_interfaces.publish import PublishExternalInterface 6 | from mutornadomon.external_interfaces.http_endpoints import ( 7 | HTTPEndpointExternalInterface, HTTPEndpointMuTornadoMonProfiler) 8 | from mutornadomon.collectors.web import WebCollector 9 | from mutornadomon.collectors.ioloop_util import UtilizationCollector 10 | 11 | 12 | def initialize_mutornadomon( 13 | tornado_app=None, publisher=None, publish_interval=None, 14 | host_limit=None, request_filter=None, tracer_port=None, 15 | **monitor_config): 16 | """Register mutornadomon to get Tornado request metrics""" 17 | if not publisher and not tornado_app: 18 | raise ValueError( 19 | 'Must pass at least one of `publisher` and `tornado_app`') 20 | 21 | if publisher: 22 | external_interface = PublishExternalInterface( 23 | publisher, publish_interval=publish_interval) 24 | else: 25 | external_interface = HTTPEndpointExternalInterface( 26 | tornado_app, request_filter=request_filter, host_limit=host_limit) 27 | 28 | # Start collecting metrics and register endpoint with app 29 | monitor = MuTornadoMon(external_interface, **monitor_config) 30 | monitor.start() 31 | 32 | # If tracer_port is not provided then don't start the profiler 33 | if tracer_port is not None: 34 | profiler_ep = HTTPEndpointMuTornadoMonProfiler(request_filter) 35 | profiler_ep.start(monitor, tracer_port) 36 | 37 | if tornado_app: 38 | web_collector = WebCollector(monitor, tornado_app) 39 | web_collector.start() 40 | 41 | utilization_collector = UtilizationCollector(monitor) 42 | utilization_collector.start() 43 | 44 | return monitor 45 | -------------------------------------------------------------------------------- /mutornadomon/external_interfaces/__init__.py: -------------------------------------------------------------------------------- 1 | from mutornadomon.external_interfaces.http_endpoints import HTTPEndpointExternalInterface # noqa 2 | from mutornadomon.external_interfaces.publish import PublishExternalInterface # noqa 3 | -------------------------------------------------------------------------------- /mutornadomon/external_interfaces/http_endpoints.py: -------------------------------------------------------------------------------- 1 | import tornado 2 | import cProfile 3 | import pstats 4 | import time 5 | from tornado import gen 6 | import logging 7 | from mutornadomon import net 8 | from tornado.ioloop import IOLoop 9 | 10 | try: 11 | from StringIO import StringIO 12 | except ImportError: 13 | from io import StringIO 14 | 15 | logger = logging.getLogger('mutornadomon_profiler') 16 | 17 | 18 | def LOCALHOST(request): 19 | if not net.is_local_address(request.remote_ip): 20 | return False 21 | xff = request.headers.get('X-Forwarded-For', None) 22 | if not xff or net.is_local_address(xff): 23 | return True 24 | return False 25 | 26 | 27 | class HTTPEndpointMuTornadoMonProfiler(object): 28 | """Handles external HTTP requests for Profiler""" 29 | 30 | def __init__(self, request_filter): 31 | if request_filter is None: 32 | self.request_filter = LOCALHOST 33 | else: 34 | self.request_filter = request_filter 35 | 36 | def start(self, monitor, server_port): 37 | self.server_port = server_port 38 | self.profiler_app = tornado.web.Application([ 39 | (r'/profiler', TornadoStatsHandler, { 40 | 'monitor': monitor, 41 | 'request_filter': self.request_filter 42 | }), 43 | ]) 44 | 45 | # If listening is started directly then IOLoop started by service will 46 | # cause issue resulting in high CPU usage. So start listening after 47 | # IOLoop is started by the service 48 | io_loop = IOLoop.current(instance=False) 49 | if io_loop is None: 50 | logger.error('Cannot initialize Mutornadomon without IOLoop') 51 | else: 52 | io_loop.add_callback(self.start_listen) 53 | 54 | def start_listen(self): 55 | self.profiler_app.listen(self.server_port) 56 | logger.info('MuTornadoMon Profiler Listening on port %s', 57 | self.server_port) 58 | 59 | def stop(self): 60 | pass 61 | 62 | 63 | class TornadoStatsHandler(tornado.web.RequestHandler): 64 | """ 65 | Profile Tornado IOLoop. 66 | Profiler will be started when the url end point is hit & stopped after 67 | profiletime or default profile collection time expires. 68 | waittime starts the profiling periodically, profiling is done for the 69 | duration of profiletime after waiting for a period of waittime. 70 | Params for the url are 71 | :param sortby: specifies how the profiling data will be sorted 72 | (ex: tottime or cumtime) 73 | :param profiletime: specifies how long profiling will be done (msec) 74 | :param waittime: specifies how long to wait when profiling periodically 75 | ex: curl "localhost:5951/profiler?sortby=cumtime&&profiletime=4000" 76 | ex: curl "localhost:5951/profiler?profiletime=200&&waittime=10000" 77 | """ 78 | 79 | def initialize(self, monitor, request_filter): 80 | self.monitor = monitor 81 | self.request_filter = request_filter 82 | self.monitor.stop_profiler = False 83 | 84 | def prepare(self): 85 | if not self.request_filter(self.request): 86 | self.send_error(403) 87 | 88 | def print_profile_data(self, sortby, wait_time): 89 | ps = None 90 | # Stats fails if there is no profile data collected 91 | try: 92 | strm = StringIO() 93 | ps = pstats.Stats(self.monitor.profiler, stream=strm) 94 | except (TypeError, ValueError): 95 | self.write("No profiling data collected") 96 | return 97 | 98 | if ps is not None: 99 | ps.sort_stats(sortby) 100 | ps.print_stats() 101 | 102 | if wait_time == 0.0: 103 | self.write(strm.getvalue()) 104 | else: 105 | logger.info(time.time()) 106 | logger.info(strm.getvalue()) 107 | 108 | self.monitor.profiler.clear() 109 | 110 | def set_options(self): 111 | valid_sortby = ['calls', 'cumulative', 'cumtime', 'file', 'filename', 112 | 'module', 'ncalls', 'pcalls', 'line', 'name', 'nfl', 113 | 'stdname', 'time', 'tottime'] 114 | 115 | sortby = 'time' 116 | profile_time = 2.0 117 | wait_time = 0.0 118 | 119 | # Dictates how the profile data is sorted 120 | if 'sortby' in self.request.arguments: 121 | sortby = self.request.arguments['sortby'][0] 122 | 123 | if sortby not in valid_sortby: 124 | sortby = 'time' 125 | 126 | # profiletime(msec) indicates for how long each of the profiling is 127 | # done 128 | if 'profiletime' in self.request.arguments: 129 | profile_time = float(self.request.arguments['profiletime'][0])/1000 130 | 131 | # waittime(msec) indicates how long to wait between profiling 132 | if 'waittime' in self.request.arguments: 133 | wait_time = float(self.request.arguments['waittime'][0])/1000 134 | self.write("Profiling will be done for every " + 135 | str(wait_time * 1000) + " msec\n") 136 | 137 | return sortby, profile_time, wait_time 138 | 139 | def disable_profiler(self): 140 | self.monitor.profiler_init = False 141 | self.monitor.profiler_running = False 142 | self.monitor.profiler.disable() 143 | 144 | @gen.coroutine 145 | def get(self): 146 | # Dictates whether to stop any on going profiling 147 | if 'stopprofiler' in self.request.arguments: 148 | self.monitor.profiler_init = False 149 | self.monitor.stop_profiler = True 150 | self.write("Stopped Profiling") 151 | return 152 | 153 | sortby, profile_time, wait_time = self.set_options() 154 | 155 | # If profiling is not started, start it 156 | if self.monitor.profiler_init is False: 157 | self.write("Profiling done for " + str(profile_time * 1000) + 158 | " msec\n") 159 | if self.monitor.profiler is None: 160 | self.monitor.profiler = cProfile.Profile() 161 | else: 162 | self.monitor.profiler.clear() 163 | 164 | while True: 165 | # enable proflier for profile_time 166 | self.monitor.profiler_init = True 167 | yield gen.Task(self.monitor.io_loop.add_timeout, 168 | time.time() + profile_time) 169 | 170 | # disable profiling 171 | self.disable_profiler() 172 | 173 | self.print_profile_data(sortby, wait_time) 174 | 175 | # Stop profiling for the duration of the wait_time 176 | yield gen.Task(self.monitor.io_loop.add_timeout, 177 | time.time() + wait_time) 178 | 179 | # If wait_time is specified then continue profiling 180 | # All the profiling data will be logged using the logger 181 | if ((wait_time == 0) or (self.monitor.stop_profiler is True)): 182 | break 183 | 184 | 185 | class StatusHandler(tornado.web.RequestHandler): 186 | 187 | def initialize(self, monitor, request_filter): 188 | self.monitor = monitor 189 | self.request_filter = request_filter 190 | 191 | def prepare(self): 192 | if not self.request_filter(self.request): 193 | self.send_error(403) 194 | 195 | def get(self): 196 | self.write(self.monitor.metrics) 197 | 198 | 199 | class HTTPEndpointExternalInterface(object): 200 | """External interface that exposes HTTP endpoints for polling by an 201 | external process. 202 | """ 203 | 204 | def __init__(self, app, host_limit=None, request_filter=None): 205 | self.app = app 206 | if request_filter is None: 207 | self.request_filter = LOCALHOST 208 | else: 209 | self.request_filter = request_filter 210 | 211 | if host_limit is None: 212 | self._host_limit = r'.*' 213 | else: 214 | self._host_limit = host_limit 215 | 216 | def start(self, monitor): 217 | self.app.add_handlers(self._host_limit, [ 218 | (r'/mutornadomon', StatusHandler, { 219 | 'monitor': monitor, 220 | 'request_filter': self.request_filter 221 | }) 222 | ]) 223 | 224 | def stop(self): 225 | pass 226 | -------------------------------------------------------------------------------- /mutornadomon/external_interfaces/publish.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import tornado 4 | 5 | 6 | class PublishExternalInterface(object): 7 | """External interface that delegates metrics publishing to a publisher callback.""" 8 | PUBLISH_FREQUENCY = 10 * 1000 # ms 9 | 10 | logger = logging.getLogger('mutornadomon') 11 | 12 | def __init__(self, publisher, publish_interval=None): 13 | self.publisher = publisher 14 | self.publish_callback = None 15 | self.publish_interval = publish_interval or self.PUBLISH_FREQUENCY 16 | 17 | def start(self, monitor): 18 | if self.publish_callback is not None: 19 | raise ValueError('Publish callback already started') 20 | 21 | self.publish_callback = tornado.ioloop.PeriodicCallback( 22 | lambda: self._publish(monitor), 23 | self.publish_interval, 24 | ) 25 | self.publish_callback.start() 26 | 27 | def stop(self): 28 | if self.publish_callback is not None: 29 | self.publish_callback.stop() 30 | self.publish_callback = None 31 | 32 | def _publish(self, monitor): 33 | try: 34 | self.publisher(monitor.metrics) 35 | except: 36 | self.logger.exception('Metrics publisher raised an exception') 37 | -------------------------------------------------------------------------------- /mutornadomon/monitor.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import collections 4 | import time 5 | 6 | import tornado.ioloop 7 | import tornado.web 8 | import os 9 | import psutil 10 | import mock 11 | 12 | CALLBACK_FREQUENCY = 100 # ms 13 | 14 | 15 | class MuTornadoMon(object): 16 | 17 | def __init__( 18 | self, 19 | external_interface, 20 | collectors=None, 21 | io_loop=None, 22 | measure_interval=CALLBACK_FREQUENCY, 23 | ): 24 | """Constructor for MuTornadoMon monitor 25 | 26 | :param host_limit: Regular expression of vhost to match, if you're 27 | using Tornado's vhosting stuff. 28 | 29 | :param remote_client_filter: Function which, when called with the request 30 | will filter it. Defaults to a filter which only allows requests from 31 | 127.0.0.0/8. 32 | 33 | :param io_loop: IOLoop to run on if not using the standard singleton. NO LONGER USED 34 | 35 | :param external_interface: 36 | 37 | :param measure_interval: The interval at which the latency of the ioloop is measured. 38 | """ 39 | if collectors is None: 40 | self.collectors = [] 41 | else: 42 | self.collectors = collectors 43 | self.io_loop = tornado.ioloop.IOLoop.current() 44 | 45 | self.measure_interval = measure_interval 46 | 47 | self.measure_callback = tornado.ioloop.PeriodicCallback( 48 | self._cb, 49 | measure_interval 50 | ) 51 | 52 | self.external_interface = external_interface 53 | 54 | self._ioloop_exception_patch = None 55 | if hasattr(collections, 'Counter'): 56 | self._COUNTERS = collections.Counter() 57 | else: 58 | self._COUNTERS = collections.defaultdict(lambda: 0) 59 | self._GAUGES = {} 60 | self._reset_ephemeral() 61 | 62 | def __del__(self): 63 | self.stop() 64 | 65 | def _reset_ephemeral(self): 66 | """Reset ephemeral statistics. 67 | 68 | For some things, rather than recording all values or just the latest 69 | value, we want to record the highest or lowest value since the last 70 | time stats were sampled. This function resets those gauges. 71 | """ 72 | self._MIN_GAUGES = {} 73 | self._MAX_GAUGES = {} 74 | 75 | def count(self, stat, value=1): 76 | """Increment a counter by the given value""" 77 | self._COUNTERS[stat] += value 78 | 79 | def kv(self, stat, value): 80 | """Set a gauge to the given stat and value. 81 | 82 | The monitor also keeps track of the max and min value seen between subsequent calls 83 | to the .metrics property. 84 | """ 85 | self._GAUGES[stat] = value 86 | if stat not in self._MAX_GAUGES or value > self._MAX_GAUGES[stat]: 87 | self._MAX_GAUGES[stat] = value 88 | if stat not in self._MIN_GAUGES or value < self._MIN_GAUGES[stat]: 89 | self._MIN_GAUGES[stat] = value 90 | 91 | def start(self): 92 | for collector in self.collectors: 93 | collector.start(self) 94 | self.external_interface.start(self) 95 | self._last_cb_time = time.time() 96 | self.measure_callback.start() 97 | 98 | def stop(self): 99 | self.external_interface.stop() 100 | for collector in self.collectors: 101 | collector.stop() 102 | if self.measure_callback is not None: 103 | self.measure_callback.stop() 104 | self.measure_callback = None 105 | if self._ioloop_exception_patch is not None: 106 | self._ioloop_exception_patch.stop() 107 | self._ioloop_exception_patch = None 108 | 109 | def _cb(self): 110 | now = time.time() 111 | self.count('callbacks') 112 | latency = now - self._last_cb_time 113 | excess_latency = latency - (self.measure_interval / 1000.0) 114 | self._last_cb_time = now 115 | self.kv('ioloop_excess_callback_latency', excess_latency) 116 | if hasattr(self.io_loop, '_handlers'): 117 | self.kv('ioloop_handlers', len(self.io_loop._handlers)) 118 | if hasattr(self.io_loop, '_callbacks'): 119 | self.kv('ioloop_pending_callbacks', len(self.io_loop._callbacks)) 120 | 121 | @property 122 | def metrics(self): 123 | """Return the current metrics. Resets max gauges.""" 124 | me = psutil.Process(os.getpid()) 125 | 126 | # Starting with 2.0.0, get_* methods are deprecated. 127 | # At 3.1.1 they are dropped. 128 | if psutil.version_info < (2, 0, 0): 129 | meminfo = me.get_memory_info() 130 | cpuinfo = me.get_cpu_times() 131 | create_time = me.create_time 132 | num_threads = me.get_num_threads() 133 | num_fds = me.get_num_fds() 134 | else: 135 | meminfo = me.memory_info() 136 | cpuinfo = me.cpu_times() 137 | create_time = me.create_time() 138 | num_threads = me.num_threads() 139 | num_fds = me.num_fds() 140 | rv = { 141 | 'process': { 142 | 'uptime': time.time() - create_time, 143 | 'meminfo': { 144 | 'rss_bytes': meminfo.rss, 145 | 'vsz_bytes': meminfo.vms, 146 | }, 147 | 'cpu': { 148 | 'user_time': cpuinfo.user, 149 | 'system_time': cpuinfo.system, 150 | 'num_threads': num_threads, 151 | }, 152 | 'num_fds': num_fds 153 | }, 154 | 'counters': dict(self._COUNTERS), 155 | 'gauges': self._GAUGES, 156 | 'max_gauges': self._MAX_GAUGES, 157 | 'min_gauges': self._MIN_GAUGES, 158 | } 159 | self._reset_ephemeral() 160 | return rv 161 | -------------------------------------------------------------------------------- /mutornadomon/net.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import six 4 | 5 | try: 6 | import ipaddress 7 | Network = ipaddress.ip_network 8 | IP = ipaddress.ip_address 9 | except ImportError: 10 | import ipcalc 11 | Network = ipcalc.Network 12 | IP = ipcalc.IP 13 | 14 | 15 | PRIVATE_NETS = [ 16 | Network('10.0.0.0/8'), # RFC 1918 17 | Network('172.16.0.0/12'), # RFC 1918 18 | Network('192.168.0.0/16'), # RFC 1918 19 | Network('127.0.0.0/8'), # local ipv4 20 | Network('fc00::/7'), # ula ipv6 21 | Network('fe80::/10'), # link-local ipv6 22 | Network('2001:0002::/48'), # bench ipv6 23 | Network('2001:db8::/32'), # documentation-only 24 | ] 25 | 26 | 27 | LOCAL_NETS = [ 28 | Network('127.0.0.0/8'), # local ipv4 29 | Network('::1'), # local ipv6 30 | Network('fc00::/7'), # ula ipv6 31 | Network('fe80::/10'), # link-local ipv6 32 | ] 33 | 34 | 35 | def is_local_address(ip): 36 | ip = _convert_to_unicode(ip) 37 | ip = IP(ip) 38 | return any(ip in net for net in LOCAL_NETS) 39 | 40 | 41 | def is_private_address(ip): 42 | ip = _convert_to_unicode(ip) 43 | ip = IP(ip) 44 | return any(ip in net for net in PRIVATE_NETS) 45 | 46 | 47 | def _convert_to_unicode(ip): 48 | """Converts given ip to unicode if its str type for python2.""" 49 | if six.PY2 and type(ip) == str: 50 | return six.u(ip) 51 | return ip 52 | -------------------------------------------------------------------------------- /mutornadomon/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber/mutornadomon/c66f3d8bf3fef024cc09ef92adbe1d63b7b61dbe/mutornadomon/utils.py -------------------------------------------------------------------------------- /requirements-py2.txt: -------------------------------------------------------------------------------- 1 | ipcalc>=1.0 2 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | nose>=1.3 2 | coverage 3 | coveralls 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tornado>=3.0,<6 2 | psutil>=1.2.0 3 | six 4 | ipcalc 5 | -------------------------------------------------------------------------------- /sample_application.py: -------------------------------------------------------------------------------- 1 | import pprint 2 | import signal 3 | import sys 4 | 5 | import tornado.web 6 | import tornado.httpserver 7 | import tornado.ioloop 8 | 9 | from mutornadomon.config import initialize_mutornadomon 10 | 11 | 12 | def fail(): 13 | assert False, 'oops' 14 | 15 | 16 | class HeloHandler(tornado.web.RequestHandler): 17 | 18 | def get(self): 19 | self.write('HELO %s' % self.request.remote_ip) 20 | tornado.ioloop.IOLoop.current().add_callback(fail) 21 | 22 | 23 | def publisher(metrics): 24 | print('Publishing metrics') 25 | pprint.pprint(metrics) 26 | 27 | 28 | def main(publish=False, no_app=False): 29 | io_loop = tornado.ioloop.IOLoop.current() 30 | 31 | application = tornado.web.Application([ 32 | (r'/', HeloHandler) 33 | ]) 34 | server = tornado.httpserver.HTTPServer(application) 35 | server.listen(8080, '127.0.0.1') 36 | 37 | if no_app: 38 | tornado_app = None 39 | else: 40 | tornado_app = application 41 | 42 | if publish: 43 | monitor = initialize_mutornadomon(tornado_app=tornado_app, 44 | io_loop=io_loop, 45 | publisher=publisher, 46 | publish_interval=5 * 1000) 47 | else: 48 | monitor = initialize_mutornadomon(tornado_app=tornado_app) 49 | 50 | def stop(*args): 51 | print('Good bye') 52 | monitor.stop() 53 | io_loop.stop() 54 | 55 | for sig in signal.SIGINT, signal.SIGQUIT, signal.SIGTERM: 56 | signal.signal(sig, stop) 57 | 58 | tornado.ioloop.IOLoop.current().start() 59 | 60 | 61 | if __name__ == '__main__': 62 | main(publish='--publish' in sys.argv, 63 | no_app='--no-app' in sys.argv) 64 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length=120 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | from setuptools import setup, find_packages 5 | 6 | install_requires = [ 7 | 'tornado<6', 8 | 'psutil', 9 | 'mock', 10 | 'six', 11 | ] 12 | 13 | if sys.version_info < (3, 0): 14 | install_requires.append('ipcalc') 15 | 16 | 17 | def read_long_description(filename="README.md"): 18 | with open(filename) as f: 19 | return f.read().strip() 20 | 21 | 22 | setup( 23 | name="mutornadomon", 24 | version="0.5.2.dev0", 25 | author="Uber Technologies, Inc.", 26 | author_email="dev@uber.com", 27 | url="https://github.com/uber/mutornadomon", 28 | license="MIT", 29 | packages=find_packages(exclude=['tests']), 30 | keywords=["monitoring", "tornado"], 31 | description="Library of standard monitoring hooks for the Tornado framework", 32 | install_requires=install_requires, 33 | long_description=read_long_description(), 34 | long_description_content_type='text/markdown', 35 | test_suite="nose.collector", 36 | tests_require=[ 37 | 'nose', 38 | ], 39 | classifiers=[ 40 | "Development Status :: 3 - Alpha", 41 | "Environment :: Web Environment", 42 | "Programming Language :: Python", 43 | "Programming Language :: Python :: 2.7", 44 | "Programming Language :: Python :: 3.4", 45 | "Programming Language :: Python :: 3.5", 46 | "Programming Language :: Python :: 3.6", 47 | "Intended Audience :: System Administrators", 48 | "Operating System :: OS Independent", 49 | "License :: OSI Approved :: MIT License", 50 | "Topic :: System :: Monitoring", 51 | "Topic :: System :: Systems Administration", 52 | ] 53 | ) 54 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber/mutornadomon/c66f3d8bf3fef024cc09ef92adbe1d63b7b61dbe/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_basic.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import mock 4 | import psutil 5 | from tornado.ioloop import IOLoop 6 | import tornado.testing 7 | from mutornadomon.config import initialize_mutornadomon 8 | from tornado.httpclient import AsyncHTTPClient 9 | from six import b 10 | 11 | 12 | class HeloHandler(tornado.web.RequestHandler): 13 | def get(self): 14 | self.write('HELO %s' % self.request.remote_ip) 15 | 16 | 17 | class TestBasic(tornado.testing.AsyncHTTPTestCase): 18 | def get_app(self): 19 | self.app = tornado.web.Application([ 20 | (r'/', HeloHandler) 21 | ]) 22 | return self.app 23 | 24 | def setUp(self): 25 | super(TestBasic, self).setUp() 26 | self.monitor = initialize_mutornadomon(self.app) 27 | 28 | def tearDown(self): 29 | super(TestBasic, self).tearDown() 30 | self.monitor.stop() 31 | 32 | @mock.patch.object(psutil.Process, 'num_threads', autospec=True, return_value=5) 33 | def test_endpoint(self, mock_num_threads): 34 | resp = self.fetch('/') 35 | self.assertEqual(resp.body, b('HELO 127.0.0.1')) 36 | resp = self.fetch('/mutornadomon') 37 | self.assertEqual(resp.code, 200) 38 | resp = json.loads(resp.body.decode('utf-8')) 39 | expected = {'requests': 2, 'localhost_requests': 2, 'private_requests': 2}.items() 40 | self.assertTrue(all(pair in resp['counters'].items() for pair in expected)) 41 | self.assertEqual(resp['process']['cpu']['num_threads'], 5) 42 | assert resp['process']['cpu']['system_time'] < 1.0 43 | 44 | def test_endpoint_xff(self): 45 | resp = self.fetch('/mutornadomon', headers={'X-Forwarded-For': '127.0.0.2'}) 46 | self.assertEqual(resp.code, 200) 47 | 48 | def test_endpoint_not_public(self): 49 | resp = self.fetch('/mutornadomon', headers={'X-Forwarded-For': '8.8.8.8'}) 50 | self.assertEqual(resp.code, 403) 51 | 52 | 53 | class TestPublisher(tornado.testing.AsyncTestCase): 54 | 55 | @mock.patch.object(psutil.Process, 'num_threads', autospec=True, return_value=5) 56 | def test_publisher_called(self, mock_num_threads): 57 | publisher = mock.Mock(return_value=None) 58 | 59 | monitor = initialize_mutornadomon(io_loop=IOLoop.current(), publisher=publisher) 60 | monitor.count('my_counter', 2) 61 | monitor.external_interface._publish(monitor) 62 | 63 | self.assertTrue(publisher.called_once()) 64 | metrics = publisher.call_args_list[0][0][0] 65 | 66 | self.assertEqual( 67 | metrics['counters'], 68 | {'my_counter': 2} 69 | ) 70 | self.assertEqual(metrics['process']['cpu']['num_threads'], 5) 71 | assert metrics['process']['cpu']['system_time'] < 1.0 72 | 73 | 74 | class TestProfiler(tornado.testing.AsyncTestCase): 75 | 76 | def setUp(self): 77 | super(TestProfiler, self).setUp() 78 | self.publisher = mock.Mock(return_value=None) 79 | self.io_loop = IOLoop.current() 80 | self.monitor = initialize_mutornadomon( 81 | io_loop=self.io_loop, publisher=self.publisher, tracer_port=5989) 82 | 83 | def tearDown(self): 84 | super(TestProfiler, self).tearDown() 85 | self.monitor.stop() 86 | 87 | @tornado.testing.gen_test 88 | @mock.patch.object(psutil.Process, 'num_threads', 89 | autospec=True, return_value=5) 90 | def test_profiler_endpoint(self, mock_num_threads): 91 | client = AsyncHTTPClient(self.io_loop) 92 | resp = yield client.fetch("http://localhost:5989/profiler") 93 | 94 | profile_str = "Profiling done for" 95 | self.assertTrue(profile_str in str(resp.body)) 96 | 97 | @tornado.testing.gen_test 98 | def test_profiler_endpoint_params(self): 99 | client = AsyncHTTPClient(self.io_loop) 100 | resp = yield client.fetch( 101 | "http://localhost:5989/profiler?sortby=cumulative&&profiletime=200") 102 | 103 | profile_str = "Profiling done for" 104 | self.assertTrue(profile_str in str(resp.body), 105 | msg='{0}'.format(str(resp.body))) 106 | 107 | @tornado.testing.gen_test 108 | def test_profiler_endpoint_invalid_param(self): 109 | client = AsyncHTTPClient(self.io_loop) 110 | resp = yield client.fetch( 111 | "http://localhost:5989/profiler?sortby=badparam&&profiletime=200") 112 | 113 | profile_str = "Profiling done for" 114 | self.assertTrue(profile_str in str(resp.body), 115 | msg='{0}'.format(str(resp.body))) 116 | 117 | @tornado.testing.gen_test 118 | def test_profiler_endpoint_periodic(self): 119 | client = AsyncHTTPClient(self.io_loop) 120 | resp = yield client.fetch("http://localhost:5989/profiler?stopprofiler") 121 | 122 | profile_str = "Stopped Profiling" 123 | self.assertTrue(profile_str in str(resp.body), 124 | msg='{0}'.format(str(resp.body))) 125 | -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import mock 4 | from mock import sentinel 5 | import mutornadomon 6 | import tornado.ioloop 7 | import unittest 8 | 9 | from mutornadomon import MuTornadoMon 10 | from mutornadomon.config import initialize_mutornadomon 11 | from mutornadomon.external_interfaces import (HTTPEndpointExternalInterface, 12 | PublishExternalInterface) 13 | 14 | 15 | class TestInitializeMutornadomon(unittest.TestCase): 16 | 17 | @mock.patch('mutornadomon.config.MuTornadoMon') 18 | @mock.patch('mutornadomon.config.WebCollector') 19 | @mock.patch('mutornadomon.config.UtilizationCollector') 20 | def test_initialize_mutornadmon(self, utilization_collector_mock, web_collector_mock, mutornadomon_mock): 21 | """Test initialize_mutornadomon() sets up HTTP endpoints interface""" 22 | app = sentinel.application, 23 | result = initialize_mutornadomon(app, host_limit='test') 24 | monitor_inst = mutornadomon_mock.return_value 25 | 26 | # initialize_mutornadomon() should return the monitor instance 27 | self.assertEqual(result, monitor_inst) 28 | 29 | assert mutornadomon_mock.call_count == 1 30 | web_collector_mock.assert_called_once_with(monitor_inst, app) 31 | utilization_collector_mock.assert_called_once_with(monitor_inst) 32 | 33 | # MuTornadoMon was created with monitor config values 34 | arg_list = mutornadomon_mock.call_args_list 35 | 36 | self.assertEquals(len(arg_list), 1) 37 | args, kwargs = arg_list[0] 38 | self.assertEqual(len(args), 1) 39 | self.assertTrue(isinstance(args[0], HTTPEndpointExternalInterface)) 40 | 41 | self.assertEqual(kwargs, {}) 42 | 43 | @mock.patch('mutornadomon.config.MuTornadoMon') 44 | @mock.patch('mutornadomon.config.WebCollector') 45 | @mock.patch('mutornadomon.config.UtilizationCollector') 46 | def test_initialize_mutornadmon_passes_publisher( 47 | self, 48 | utilization_collector_mock, 49 | web_collector_mock, 50 | mutornadomon_mock 51 | ): 52 | """Test initialize_mutornadomon() sets up publishing interface""" 53 | 54 | def publisher(monitor): 55 | pass 56 | 57 | app = sentinel.application 58 | result = initialize_mutornadomon(app, 59 | publisher=publisher, 60 | host_limit='test') 61 | monitor_inst = mutornadomon_mock.return_value 62 | 63 | # initialize_mutornadomon() should return the monitor instance 64 | self.assertEqual(result, monitor_inst) 65 | 66 | web_collector_mock.assert_called_once_with(monitor_inst, app) 67 | utilization_collector_mock.assert_called_once_with(monitor_inst) 68 | 69 | assert mutornadomon_mock.call_count == 1 70 | arg_list = mutornadomon_mock.call_args_list 71 | 72 | args, kwargs = arg_list[0] 73 | self.assertEqual(len(args), 1) 74 | self.assertTrue(isinstance(args[0], PublishExternalInterface)) 75 | 76 | self.assertEqual(kwargs, {}) 77 | 78 | @mock.patch('mutornadomon.config.MuTornadoMon') 79 | def test_initialize_mutornadmon_works_with_publisher_and_no_app(self, mutornadomon_mock): 80 | """Test initialize_mutornadomon() works with publisher, but no web app passed""" 81 | 82 | def publisher(monitor): 83 | pass 84 | 85 | monitor_inst = mutornadomon_mock.return_value 86 | result = initialize_mutornadomon(publisher=publisher) 87 | 88 | # initialize_mutornadomon() should return the monitor instance 89 | self.assertEqual(result, monitor_inst) 90 | 91 | assert mutornadomon_mock.call_count == 1 92 | 93 | # MuTornadoMon was created with monitor config values 94 | arg_list = mutornadomon_mock.call_args_list 95 | 96 | self.assertEquals(len(arg_list), 1) 97 | args, kwargs = arg_list[0] 98 | self.assertEqual(len(args), 1) 99 | self.assertTrue(isinstance(args[0], PublishExternalInterface)) 100 | 101 | # No collectors are passed 102 | self.assertEqual(kwargs, {}) 103 | 104 | @mock.patch('tornado.ioloop.PeriodicCallback') 105 | def test_publisher_initializer(self, periodic_callback_mock): 106 | """Test instrument_ioloop() setups monitoring and creates a PeriodicCallback""" 107 | 108 | def publisher(): 109 | pass 110 | 111 | result = initialize_mutornadomon(io_loop=tornado.ioloop.IOLoop.current(), publisher=publisher) 112 | 113 | assert periodic_callback_mock.called 114 | 115 | self.assertTrue(isinstance(result, mutornadomon.MuTornadoMon)) 116 | 117 | @mock.patch('tornado.ioloop.PeriodicCallback') 118 | def test_initialize_mutornadomon_raises_when_no_publisher_and_no_app(self, periodic_callback_mock): 119 | """Test instrument_ioloop() setups monitoring and creates a PeriodicCallback""" 120 | self.assertRaises(ValueError, initialize_mutornadomon, 121 | io_loop=tornado.ioloop.IOLoop.current()) 122 | 123 | @mock.patch('psutil.Process') 124 | @mock.patch('os.getpid') 125 | @mock.patch('mutornadomon.external_interfaces.PublishExternalInterface') 126 | def test_MuTornadoMon(self, pub_ext_iface_mock, mock_os, mock_ps): 127 | external_interface = pub_ext_iface_mock.return_value 128 | monitor = MuTornadoMon(external_interface) 129 | 130 | monitor.start() 131 | 132 | stat = 'test' 133 | val = 2 134 | 135 | # __COUNTERS[stat] = 2 136 | monitor.count(stat, val) 137 | 138 | # __COUNTERS[stat] = 3 139 | monitor.count(stat) 140 | self.assertEqual(monitor.metrics['counters'][stat], 3) 141 | 142 | 143 | monitor.kv(stat, 1) 144 | self.assertEqual(monitor.metrics['gauges'][stat], 1) 145 | 146 | # To make sure no exceptions are thrown 147 | monitor._cb() 148 | 149 | monitor.__del__() 150 | 151 | @mock.patch('tornado.ioloop') 152 | def test_initialize_PublishExternalInterface(self, mock_ioloop): 153 | 154 | def publisher(monitor): 155 | pass 156 | 157 | pub_inst = PublishExternalInterface(publisher=publisher) 158 | monitor = mock.MagicMock() 159 | pub_inst.start(monitor) 160 | self.assertTrue(pub_inst.publish_callback != None) 161 | 162 | self.assertRaises(ValueError, pub_inst.start, monitor=monitor) 163 | 164 | pub_inst._publish(monitor) 165 | 166 | pub_inst.stop() 167 | self.assertTrue(pub_inst.publish_callback == None) 168 | -------------------------------------------------------------------------------- /tests/test_external_interfaces.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber/mutornadomon/c66f3d8bf3fef024cc09ef92adbe1d63b7b61dbe/tests/test_external_interfaces.py -------------------------------------------------------------------------------- /tests/test_net.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import mock 4 | import unittest 5 | 6 | from mutornadomon.net import is_local_address 7 | from mutornadomon.net import is_private_address 8 | 9 | 10 | class TestIsLocalAddress(unittest.TestCase): 11 | 12 | @mock.patch('six.PY2') 13 | def test_is_local_address_works_with_python2(self, is_python2_mock): 14 | """Test is_local_address method works for python2.""" 15 | is_python2_mock.return_value = True 16 | self.assertTrue(is_local_address(u'127.0.0.1')) 17 | self.assertTrue(is_local_address('127.0.0.1')) 18 | self.assertFalse(is_local_address('126.0.0.10')) 19 | 20 | @mock.patch('six.PY2') 21 | def test_is_local_address_works_with_python3(self, is_python2_mock): 22 | """Test is_local_address method works for python3.""" 23 | is_python2_mock.return_value = False 24 | self.assertTrue(is_local_address(u'127.0.0.1')) 25 | self.assertTrue(is_local_address('127.0.0.1')) 26 | self.assertFalse(is_local_address('126.0.0.10')) 27 | 28 | 29 | class TestIsPrivateAddress(unittest.TestCase): 30 | 31 | @mock.patch('six.PY2') 32 | def test_is_private_address_works_with_python2(self, is_python2_mock): 33 | """Test is_private_address method works for python2.""" 34 | is_python2_mock.return_value = True 35 | self.assertTrue(is_private_address(u'127.0.0.1')) 36 | self.assertTrue(is_private_address('127.0.0.1')) 37 | self.assertFalse(is_private_address('126.0.0.10')) 38 | 39 | @mock.patch('six.PY2') 40 | def test_is_private_address_works_with_python3(self, is_python2_mock): 41 | """Test is_private_address method works for python3.""" 42 | is_python2_mock.return_value = False 43 | self.assertTrue(is_private_address(u'127.0.0.1')) 44 | self.assertTrue(is_private_address('127.0.0.1')) 45 | self.assertFalse(is_private_address('126.0.0.10')) --------------------------------------------------------------------------------