├── .gitignore ├── README.markdown ├── demo ├── main.py └── screenshot.png └── tornado_tracing ├── __init__.py ├── config.py └── recording.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *~ -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | Tornado Tracing 2 | =============== 3 | This library instruments HTTP calls in a [Tornado](http://tornadoweb.org) 4 | application and provides visualizations to find where your app is spending 5 | its time and identify opportunities for parallelism. It uses the 6 | [Appstats](http://code.google.com/appengine/docs/python/tools/appstats.html) 7 | module from the Google App Engine SDK. 8 | 9 | Tornado Tracing is licensed under the Apache License, Version 2.0 10 | (http://www.apache.org/licenses/LICENSE-2.0.html). 11 | 12 | Installation 13 | ============ 14 | You will need: 15 | * [Tornado](http://tornadoweb.org), version 1.1 or higher 16 | * The [Google App Engine SDK](http://code.google.com/appengine/downloads.html) 17 | * A memcached server and the [python memcache client library](http://www.tummy.com/Community/software/python-memcached/) 18 | 19 | Usage 20 | ===== 21 | In most cases, you can simply use/subclass `RecordingRequestHandler` 22 | and `AsyncHTTPClient` from `tornado_tracing.recording` instead of the 23 | corresponding Tornado classes. These classes do nothing special unless 24 | the `--enable_appstats` flag is passed. 25 | 26 | At startup, you must call `tornado_tracing.config.setup_memcache()` to tell 27 | the system where your memcache server is running. You can also use 28 | `tornado_tracing.config.set_options()` to set certain other options, 29 | such as `KEY_PREFIX` if multiple applications are sharing a memcache server 30 | and you want to keep the data separate. 31 | 32 | Finally, add `tornado_tracing.config.get_urlspec()` to your `Application`'s 33 | list of handlers. This is where the trace output will be visible. 34 | Note that the UI does not have to be served from the same process where 35 | the data is generated, as long as they're all using the same memcache. 36 | 37 | Screenshot 38 | ========== 39 | ![screenshot](https://raw.githubusercontent.com/bdarnell/tornado_tracing/master/demo/screenshot.png) 40 | -------------------------------------------------------------------------------- /demo/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | '''Demo of appstats tracing. 3 | 4 | Starts a simple server with appstats enabled. Go to http://localhost:8888/ 5 | to generate some sample data, then go to http://localhost:8888/appstats/ 6 | to see the results. 7 | 8 | Requires tornado, tornado_tracing, and the google appengine sdk to be 9 | on $PYTHONPATH. It also doesn't like it when the app is started using 10 | a relative path, so run it with something like this: 11 | 12 | export PYTHONPATH=.:../tornado:/usr/local/google_appengine:/usr/local/google_appengine/lib/webob 13 | $PWD/demo/main.py 14 | ''' 15 | 16 | from tornado.httpserver import HTTPServer 17 | from tornado.ioloop import IOLoop 18 | from tornado.options import define, options, parse_command_line 19 | from tornado.web import Application, asynchronous 20 | from tornado_tracing import config 21 | from tornado_tracing import recording 22 | 23 | import time 24 | 25 | define('port', type=int, default=8888) 26 | define('memcache', default='localhost:11211') 27 | 28 | class DelayHandler(recording.RecordingRequestHandler): 29 | @asynchronous 30 | def get(self): 31 | IOLoop.instance().add_timeout( 32 | time.time() + int(self.get_argument('ms')) / 1000.0, 33 | self.handle_timeout) 34 | 35 | def handle_timeout(self): 36 | self.finish("ok") 37 | 38 | # A handler that performs several HTTP requests taking different amount of 39 | # time. It waits for the first request to finish, then issues three requests 40 | # in parallel. 41 | class RootHandler(recording.RecordingRequestHandler): 42 | @asynchronous 43 | def get(self): 44 | self.client = recording.AsyncHTTPClient() 45 | self.client.fetch('http://localhost:%d/delay?ms=100' % options.port, 46 | self.step2) 47 | 48 | def handle_step2_fetch(self, response): 49 | assert response.body == 'ok' 50 | self.fetches_remaining -= 1 51 | if self.fetches_remaining == 0: 52 | self.step3() 53 | 54 | def step2(self, response): 55 | assert response.body == 'ok' 56 | self.fetches_remaining = 3 57 | self.client.fetch('http://localhost:%d/delay?ms=50' % options.port, 58 | self.handle_step2_fetch) 59 | self.client.fetch('http://localhost:%d/delay?ms=20' % options.port, 60 | self.handle_step2_fetch) 61 | self.client.fetch('http://localhost:%d/delay?ms=30' % options.port, 62 | self.handle_step2_fetch) 63 | 64 | def step3(self): 65 | self.finish('All done. See results here.') 66 | 67 | def main(): 68 | parse_command_line() 69 | # doesn't make much sense to run this without appstats enabled 70 | options.enable_appstats = True 71 | config.setup_memcache([options.memcache]) 72 | 73 | app = Application([ 74 | ('/', RootHandler), 75 | ('/delay', DelayHandler), 76 | config.get_urlspec('/appstats/.*'), 77 | ], debug=True) 78 | server = HTTPServer(app) 79 | server.listen(options.port) 80 | IOLoop.instance().start() 81 | 82 | if __name__ == '__main__': 83 | main() 84 | -------------------------------------------------------------------------------- /demo/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bdarnell/tornado_tracing/587bcd67b5bda0ffabdc4587a9d5517bbe551453/demo/screenshot.png -------------------------------------------------------------------------------- /tornado_tracing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bdarnell/tornado_tracing/587bcd67b5bda0ffabdc4587a9d5517bbe551453/tornado_tracing/__init__.py -------------------------------------------------------------------------------- /tornado_tracing/config.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import memcache 3 | import tornado.web 4 | import tornado.wsgi 5 | import warnings 6 | 7 | with warnings.catch_warnings(): 8 | warnings.simplefilter('ignore', DeprecationWarning) 9 | from google.appengine.api import memcache as appengine_memcache 10 | from google.appengine.api import lib_config 11 | from google.appengine.ext import webapp 12 | 13 | def setup_memcache(*args, **kwargs): 14 | '''Configures the app engine memcache interface with a set of regular 15 | memcache servers. All arguments are passed to the memcache.Client 16 | constructor. 17 | 18 | Example: 19 | setup_memcache(["localhost:11211"]) 20 | ''' 21 | client = memcache.Client(*args, **kwargs) 22 | # The appengine memcache interface has some methods that aren't 23 | # currently available in the regular memcache module (at least 24 | # in version 1.4.4). Fortunately appstats doesn't use them, but 25 | # the setup_client function expects them to be there. 26 | client.add_multi = None 27 | client.replace_multi = None 28 | client.offset_multi = None 29 | # Appengine adds a 'namespace' parameter to many methods. Since 30 | # appstats.recording uses both namespace and key_prefix, just drop 31 | # the namespace. (This list of methods is not exhaustive, it's just 32 | # the ones appstats uses) 33 | for method in ('set_multi', 'set', 'add', 'delete', 'get', 'get_multi'): 34 | def wrapper(old_method, *args, **kwargs): 35 | # appstats.recording always passes namespace by keyword 36 | if 'namespace' in kwargs: 37 | del kwargs['namespace'] 38 | return old_method(*args, **kwargs) 39 | setattr(client, method, 40 | functools.partial(wrapper, getattr(client, method))) 41 | appengine_memcache.setup_client(client) 42 | 43 | def get_urlspec(prefix): 44 | '''Returns a tornado.web.URLSpec for the appstats UI. 45 | Should be mapped to a url prefix ending with 'stats/'. 46 | 47 | Example: 48 | app = tornado.web.Application([ 49 | ... 50 | tornado_tracing.config.get_urlspec(r'/_stats/.*'), 51 | ]) 52 | ''' 53 | # This import can't happen at the top level because things get horribly 54 | # confused if it happens before django settings are initialized. 55 | with warnings.catch_warnings(): 56 | warnings.simplefilter('ignore', DeprecationWarning) 57 | from google.appengine.ext.appstats import ui 58 | wsgi_app = tornado.wsgi.WSGIContainer(webapp.WSGIApplication(ui.URLMAP)) 59 | return tornado.web.url(prefix, 60 | tornado.web.FallbackHandler, 61 | dict(fallback=wsgi_app)) 62 | 63 | def set_options(**kwargs): 64 | '''Sets configuration options for appstats. See 65 | /usr/local/google_appengine/ext/appstats/recording.py for possible keys. 66 | 67 | Example: 68 | tornado_tracing.config.set_options(RECORD_FRACTION=0.1, 69 | KEY_PREFIX='__appstats_myapp__') 70 | ''' 71 | lib_config.register('appstats', kwargs) 72 | -------------------------------------------------------------------------------- /tornado_tracing/recording.py: -------------------------------------------------------------------------------- 1 | '''RPC Tracing support. 2 | 3 | Records timing information about rpcs and other operations for performance 4 | profiling. Currently just a wrapper around the Google App Engine appstats 5 | module. 6 | ''' 7 | 8 | import contextlib 9 | import functools 10 | import logging 11 | import tornado.httpclient 12 | import tornado.web 13 | import tornado.wsgi 14 | import warnings 15 | 16 | with warnings.catch_warnings(): 17 | warnings.simplefilter('ignore', DeprecationWarning) 18 | from google.appengine.ext.appstats import recording 19 | from tornado.httpclient import AsyncHTTPClient 20 | from tornado.options import define, options 21 | from tornado.stack_context import StackContext 22 | from tornado.web import RequestHandler 23 | 24 | define('enable_appstats', type=bool, default=False) 25 | 26 | # These methods from the appengine recording module are a part of our 27 | # public API. 28 | 29 | # start_recording(wsgi_environ) creates a recording context 30 | start_recording = recording.start_recording 31 | # end_recording(http_status) terminates a recording context 32 | end_recording = recording.end_recording 33 | 34 | # pre/post_call_hook(service, method, request, response) mark the 35 | # beginning/end of a time span to record in the trace. 36 | pre_call_hook = recording.pre_call_hook 37 | post_call_hook = recording.post_call_hook 38 | 39 | def save(): 40 | '''Returns an object that can be passed to restore() to resume 41 | a suspended record. 42 | ''' 43 | return recording.recorder 44 | 45 | def restore(recorder): 46 | '''Reactivates a previously-saved recording context.''' 47 | recording.recorder = recorder 48 | 49 | 50 | class RecordingRequestHandler(RequestHandler): 51 | '''RequestHandler subclass that establishes a recording context for each 52 | request. 53 | ''' 54 | def __init__(self, *args, **kwargs): 55 | super(RecordingRequestHandler, self).__init__(*args, **kwargs) 56 | self.__recorder = None 57 | 58 | def _execute(self, transforms, *args, **kwargs): 59 | if options.enable_appstats: 60 | start_recording(tornado.wsgi.WSGIContainer.environ(self.request)) 61 | recorder = save() 62 | @contextlib.contextmanager 63 | def transfer_recorder(): 64 | restore(recorder) 65 | yield 66 | with StackContext(transfer_recorder): 67 | super(RecordingRequestHandler, self)._execute(transforms, 68 | *args, **kwargs) 69 | else: 70 | super(RecordingRequestHandler, self)._execute(transforms, 71 | *args, **kwargs) 72 | 73 | def finish(self, chunk=None): 74 | super(RecordingRequestHandler, self).finish(chunk) 75 | if options.enable_appstats: 76 | end_recording(self._status_code) 77 | 78 | class RecordingFallbackHandler(tornado.web.FallbackHandler): 79 | '''FallbackHandler subclass that establishes a recording context for 80 | each request. 81 | ''' 82 | def prepare(self): 83 | if options.enable_appstats: 84 | recording.start_recording( 85 | tornado.wsgi.WSGIContainer.environ(self.request)) 86 | recorder = save() 87 | @contextlib.contextmanager 88 | def transfer_recorder(): 89 | restore(recorder) 90 | yield 91 | with StackContext(transfer_recorder): 92 | super(RecordingFallbackHandler, self).prepare() 93 | recording.end_recording(self._status_code) 94 | else: 95 | super(RecordingFallbackHandler, self).prepare() 96 | 97 | def _request_info(request): 98 | '''Returns a tuple (method, url) for use in recording traces. 99 | 100 | Accepts either a url or HTTPRequest object, like HTTPClient.fetch. 101 | ''' 102 | if isinstance(request, tornado.httpclient.HTTPRequest): 103 | return (request.method, request.url) 104 | else: 105 | return ('GET', request) 106 | 107 | class HTTPClient(tornado.httpclient.HTTPClient): 108 | def fetch(self, request, *args, **kwargs): 109 | method, url = _request_info(request) 110 | recording.pre_call_hook('HTTP', method, url, None) 111 | response = super(HTTPClient, self).fetch(request, *args, **kwargs) 112 | recording.post_call_hook('HTTP', method, url, None) 113 | return response 114 | 115 | class AsyncHTTPClient(AsyncHTTPClient): 116 | def fetch(self, request, callback, *args, **kwargs): 117 | method, url = _request_info(request) 118 | recording.pre_call_hook('HTTP', method, url, None) 119 | def wrapper(request, callback, response, *args): 120 | recording.post_call_hook('HTTP', method, url, None) 121 | callback(response) 122 | super(AsyncHTTPClient, self).fetch( 123 | request, 124 | functools.partial(wrapper, request, callback), 125 | *args, **kwargs) 126 | --------------------------------------------------------------------------------