├── turq ├── util │ ├── __init__.py │ ├── logging.py │ ├── text.py │ └── http.py ├── __metadata__.py ├── __init__.py ├── examples.py ├── editor │ ├── codemirror │ │ ├── LICENSE │ │ ├── addon │ │ │ └── runmode │ │ │ │ ├── colorize.js │ │ │ │ └── runmode.js │ │ ├── lib │ │ │ └── codemirror.css │ │ └── mode │ │ │ └── python │ │ │ └── python.js │ ├── editor.html.tpl │ ├── editor.css │ └── editor.js ├── mock.py ├── main.py ├── editor.py ├── examples.rst └── rules.py ├── setup.cfg ├── docs ├── history.rst ├── examples.rst ├── index.rst ├── guide.rst └── conf.py ├── screenshot.png ├── .gitignore ├── tools ├── requirements.in └── requirements.txt ├── MANIFEST.in ├── LICENSE ├── pylintrc ├── CHANGELOG.rst ├── setup.py ├── README.rst ├── HACKING.rst └── tests ├── conftest.py ├── test_control.py └── test_examples.py /turq/util/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | 3 | universal=1 4 | -------------------------------------------------------------------------------- /turq/__metadata__.py: -------------------------------------------------------------------------------- 1 | version = '0.3.2.dev1' 2 | -------------------------------------------------------------------------------- /docs/history.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CHANGELOG.rst 2 | -------------------------------------------------------------------------------- /turq/__init__.py: -------------------------------------------------------------------------------- 1 | from turq.__metadata__ import version as __version__ 2 | -------------------------------------------------------------------------------- /screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vfaronov/turq/HEAD/screenshot.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.egg-info 3 | .cache 4 | build 5 | dist 6 | docs/_build 7 | MANIFEST 8 | -------------------------------------------------------------------------------- /tools/requirements.in: -------------------------------------------------------------------------------- 1 | pip-tools 2 | pylint 3 | pytest 4 | requests 5 | Sphinx 6 | sphinx_rtd_theme 7 | check-manifest 8 | -------------------------------------------------------------------------------- /docs/examples.rst: -------------------------------------------------------------------------------- 1 | Examples 2 | ======== 3 | 4 | The rules language of Turq is documented only by example. 5 | 6 | .. include:: ../turq/examples.rst 7 | -------------------------------------------------------------------------------- /turq/util/logging.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=invalid-name 2 | 3 | import collections 4 | import logging 5 | import threading 6 | 7 | counts = collections.defaultdict(int) 8 | lock = threading.Lock() 9 | 10 | 11 | def getNextLogger(prefix): 12 | # This is an easy way to distinguish log messages from different objects, 13 | # and to selectively enable debug logging only for some of them. 14 | with lock: 15 | counts[prefix] += 1 16 | return logging.getLogger('%s.%d' % (prefix, counts[prefix])) 17 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include setup.cfg 2 | 3 | include README.rst 4 | include screenshot.png 5 | include LICENSE 6 | include turq/editor/codemirror/LICENSE 7 | include CHANGELOG.rst 8 | include HACKING.rst 9 | 10 | include pylintrc 11 | 12 | # Even though Travis and AppVeyor only works on a Git repo, 13 | # these files are important as references on the build procedures. 14 | include .travis.yml 15 | include appveyor.yml 16 | 17 | graft tests 18 | graft docs 19 | prune docs/_build 20 | graft tools 21 | 22 | global-exclude *.pyc 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright © 2017 Vasiliy Faronov. 2 | 3 | Permission to use, copy, modify, and/or distribute this software 4 | for any purpose with or without fee is hereby granted, provided that 5 | the above copyright notice and this permission notice appear in all copies. 6 | 7 | THE SOFTWARE IS PROVIDED “AS IS” AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES 9 | OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE 10 | FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN 12 | AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING 13 | OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Turq user manual 2 | ================ 3 | 4 | .. warning:: 5 | 6 | Turq is no longer maintained nor used by its author. 7 | Try `mitmproxy`__ with `scripting`__ instead. 8 | See also https://gist.github.com/vfaronov/3f7848932ed96a264c382902262ce7b3 9 | 10 | __ https://mitmproxy.org/ 11 | __ https://docs.mitmproxy.org/stable/addons-scripting/ 12 | 13 | `Turq`_ is a small HTTP server that can be scripted in a Python-based language. 14 | Use it to set up mock HTTP resources that respond with the status, headers, 15 | and body of your choosing. Turq is designed for quick interactive testing, 16 | but can be used in automated scenarios as well. 17 | 18 | .. _Turq: https://github.com/vfaronov/turq 19 | 20 | .. toctree:: 21 | :maxdepth: 2 22 | 23 | guide 24 | examples 25 | history 26 | -------------------------------------------------------------------------------- /pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # Do not remember and compare with previous results. That's just noise. 4 | persistent=no 5 | 6 | 7 | [REPORTS] 8 | 9 | reports=no 10 | 11 | 12 | [MESSAGES CONTROL] 13 | 14 | disable= 15 | # - Don't tell me that I've locally disabled a check. That's the point! 16 | locally-disabled,locally-enabled, 17 | # - No factoring checks. 18 | R, 19 | # - Missing docstrings are clearly visible, I will decide for myself. 20 | missing-docstring, 21 | 22 | 23 | [BASIC] 24 | 25 | # Single-letter variable names are fine in many cases. 26 | variable-rgx=[a-z_][a-z0-9_]* 27 | argument-rgx=[a-z_][a-z0-9_]* 28 | 29 | # What Pylint considers "constants" are sometimes valid global variables, 30 | # such as per-module `logger`. 31 | const-rgx=[A-Za-z_][A-Za-z0-9_]* 32 | 33 | 34 | [EXCEPTIONS] 35 | 36 | # Catching `Exception` is a perfectly good idea in many cases. 37 | overgeneral-exceptions= 38 | 39 | 40 | [TYPECHECK] 41 | 42 | # ``h11.NEED_DATA``, etc. 43 | generated-members=h11\.[A-Z][A-Z_]+ 44 | -------------------------------------------------------------------------------- /turq/examples.py: -------------------------------------------------------------------------------- 1 | import pkgutil 2 | import xml.etree.ElementTree 3 | 4 | import docutils.core 5 | 6 | 7 | def load_pairs(): 8 | # Load pairs of "example ID, rules code" for the test suite. 9 | rst_code = _load_rst() 10 | xml_code = docutils.core.publish_string(rst_code, writer_name='xml') 11 | tree = xml.etree.ElementTree.fromstring(xml_code) 12 | parsed = [] 13 | for section in tree.findall('./section'): 14 | slug = section.get('ids').replace('-', '_') 15 | for i, block in enumerate(section.findall('./literal_block'), start=1): 16 | parsed.append(('%s_%d' % (slug, i), block.text)) 17 | return parsed 18 | 19 | 20 | def load_html(initial_header_level): 21 | # Render an HTML fragment ready for inclusion into a page. 22 | rst_code = _load_rst() 23 | parts = docutils.core.publish_parts( 24 | rst_code, writer_name='html', 25 | settings_overrides={'initial_header_level': initial_header_level}) 26 | return parts['fragment'] 27 | 28 | 29 | def _load_rst(): 30 | return pkgutil.get_data('turq', 'examples.rst') 31 | -------------------------------------------------------------------------------- /turq/editor/codemirror/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (C) 2017 by Marijn Haverbeke and others 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /tools/requirements.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile 3 | # To update, run: 4 | # 5 | # pip-compile --output-file tools/requirements.txt tools/requirements.in 6 | # 7 | alabaster==0.7.10 # via sphinx 8 | appdirs==1.4.3 # via setuptools 9 | astroid==1.4.9 # via pylint 10 | babel==2.4.0 # via sphinx 11 | check-manifest==0.35 12 | click==6.7 # via pip-tools 13 | docutils==0.13.1 # via sphinx 14 | first==2.0.1 # via pip-tools 15 | imagesize==0.7.1 # via sphinx 16 | isort==4.2.5 # via pylint 17 | jinja2==2.9.6 # via sphinx 18 | lazy-object-proxy==1.2.2 # via astroid 19 | markupsafe==1.0 # via jinja2 20 | mccabe==0.6.1 # via pylint 21 | packaging==16.8 # via setuptools 22 | pip-tools==1.8.0 23 | py==1.4.33 # via pytest 24 | pygments==2.2.0 # via sphinx 25 | pylint==1.6.5 26 | pyparsing==2.2.0 # via packaging 27 | pytest==3.0.7 28 | pytz==2017.2 # via babel 29 | requests==2.13.0 30 | six==1.10.0 # via astroid, packaging, pip-tools, pylint, setuptools, sphinx 31 | snowballstemmer==1.2.1 # via sphinx 32 | sphinx-rtd-theme==0.2.4 33 | sphinx==1.5.5 34 | wrapt==1.10.10 # via astroid 35 | 36 | # The following packages are considered to be unsafe in a requirements file: 37 | # setuptools # via pytest 38 | -------------------------------------------------------------------------------- /turq/editor/codemirror/addon/runmode/colorize.js: -------------------------------------------------------------------------------- 1 | // CodeMirror, copyright (c) by Marijn Haverbeke and others 2 | // Distributed under an MIT license: http://codemirror.net/LICENSE 3 | 4 | (function(mod) { 5 | if (typeof exports == "object" && typeof module == "object") // CommonJS 6 | mod(require("../../lib/codemirror"), require("./runmode")); 7 | else if (typeof define == "function" && define.amd) // AMD 8 | define(["../../lib/codemirror", "./runmode"], mod); 9 | else // Plain browser env 10 | mod(CodeMirror); 11 | })(function(CodeMirror) { 12 | "use strict"; 13 | 14 | var isBlock = /^(p|li|div|h\\d|pre|blockquote|td)$/; 15 | 16 | function textContent(node, out) { 17 | if (node.nodeType == 3) return out.push(node.nodeValue); 18 | for (var ch = node.firstChild; ch; ch = ch.nextSibling) { 19 | textContent(ch, out); 20 | if (isBlock.test(node.nodeType)) out.push("\n"); 21 | } 22 | } 23 | 24 | CodeMirror.colorize = function(collection, defaultMode) { 25 | if (!collection) collection = document.body.getElementsByTagName("pre"); 26 | 27 | for (var i = 0; i < collection.length; ++i) { 28 | var node = collection[i]; 29 | var mode = node.getAttribute("data-lang") || defaultMode; 30 | if (!mode) continue; 31 | 32 | var text = []; 33 | textContent(node, text); 34 | node.innerHTML = ""; 35 | CodeMirror.runMode(text.join(""), mode, node); 36 | 37 | node.className += " cm-s-default"; 38 | } 39 | }; 40 | }); 41 | -------------------------------------------------------------------------------- /turq/editor/editor.html.tpl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Turq editor 7 | 8 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 |
22 |

Turq editor

23 |

24 | Mock server is listening on $mock_host port $mock_port — 25 | try $mock_url 26 |

27 | 28 |
29 | 30 |

31 | 32 | 33 |

34 |
35 |
36 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | History of changes 2 | ================== 3 | 4 | 0.3.1 - 2017-04-04 5 | ------------------ 6 | 7 | Packaging fixes. 8 | 9 | 10 | 0.3.0 - 2017-04-04 11 | ------------------ 12 | 13 | - Complete rewrite. Only the most notable changes are listed below. 14 | 15 | - Requires Python 3.4 or higher. 16 | 17 | - The rules language is completely different, simpler and more powerful. 18 | 19 | - Notable new features of the rules language: 20 | 21 | - forwarding to other servers ("reverse proxy"); 22 | - easy "RESTful" routing with path segments; 23 | - easy construction of arbitrary HTML pages (using `Dominate`_); 24 | - CORS support now handles preflight requests automatically; 25 | - control over finer aspects of the protocol: streaming, 1xx responses, 26 | ``Content-Length``, ``Transfer-Encoding``, keep-alive. 27 | 28 | - On the other hand, some features have been removed for now: 29 | 30 | - alternating responses (``first()``, ``next()``, ``then()``); 31 | - shortcuts for JavaScript and XML responses (``js()``, ``xml()``). 32 | 33 | - You can now choose which network interface Turq listens on, including IPv6. 34 | 35 | - The Turq editor (formerly known as "console") now has automatic indentation 36 | and syntax highlighting. 37 | 38 | - The Turq editor is now optional, listens on a separate port, 39 | and is protected with a password by default. 40 | 41 | - Turq can now print more information to the (system) console, including 42 | request and response headers. 43 | 44 | - Initial rules may now be read from a file at startup. This provides a simple 45 | way to use Turq programmatically. 46 | 47 | - Turq can now handle multiple concurrent requests. 48 | 49 | .. _Dominate: https://github.com/Knio/dominate 50 | 51 | 52 | 0.2.0 - 2012-12-09 53 | ------------------ 54 | 55 | - Stochastic responses (``maybe()``, ``otherwise()``). 56 | 57 | - Various features of the response can now be parametrized with lambdas. 58 | 59 | - ``body_file()`` now expands tilde to the user's home directory. 60 | 61 | 62 | 0.1.0 - 2012-11-17 63 | ------------------ 64 | 65 | Initial release. 66 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from setuptools import setup 4 | 5 | metadata = {} 6 | with open(os.path.join('turq', '__metadata__.py'), 'r') as f: 7 | exec(f.read(), metadata) # pylint: disable=exec-used 8 | 9 | with open('README.rst') as f: 10 | long_description = f.read() 11 | 12 | 13 | setup( 14 | name='turq', 15 | version=metadata['version'], 16 | description='Mock HTTP server', 17 | long_description=long_description, 18 | url='https://github.com/vfaronov/turq', 19 | author='Vasiliy Faronov', 20 | author_email='vfaronov@gmail.com', 21 | license='ISC', 22 | # Do not care about the ugliness of hardcoding all these names 23 | # because any discrepancies will be caught by check-manifest. 24 | packages=['turq', 'turq.util'], 25 | package_data={ 26 | 'turq': [ 27 | 'editor/*.tpl', 'editor/*.css', 'editor/*.js', 28 | 'editor/codemirror/lib/*.css', 'editor/codemirror/lib/*.js', 29 | 'editor/codemirror/mode/python/*.js', 30 | 'editor/codemirror/addon/runmode/*.js', 31 | 'examples.rst', 32 | ], 33 | }, 34 | entry_points={'console_scripts': ['turq=turq.main:main']}, 35 | install_requires=[ 36 | 'h11 >= 0.7.0', 37 | 'falcon >= 1.1.0', 38 | 'dominate >= 2.3.1', 39 | 'Werkzeug >= 0.12.1', 40 | 'docutils >= 0.13.1', 41 | 'colorlog >= 2.10.0', 42 | ], 43 | extras_require={ 44 | ':sys_platform == "win32"': [ 45 | 'colorama >= 0.3.7', # for colorlog 46 | ], 47 | }, 48 | classifiers=[ 49 | 'Development Status :: 7 - Inactive', 50 | 'Intended Audience :: Developers', 51 | 'License :: OSI Approved :: ISC License (ISCL)', 52 | 'Programming Language :: Python :: 3 :: Only', 53 | 'Programming Language :: Python :: 3.4', 54 | 'Programming Language :: Python :: 3.5', 55 | 'Programming Language :: Python :: 3.6', 56 | 'Topic :: Internet :: WWW/HTTP :: HTTP Servers', 57 | 'Topic :: Software Development :: Testing', 58 | 'Topic :: Utilities', 59 | ], 60 | keywords='HTTP Web server mock mocking test debug', 61 | ) 62 | -------------------------------------------------------------------------------- /turq/editor/editor.css: -------------------------------------------------------------------------------- 1 | body { 2 | color: #000000; 3 | background: #FDFDFD; 4 | font-family: sans-serif; 5 | margin: 0; 6 | } 7 | 8 | main { 9 | padding: 2vw; 10 | } 11 | 12 | aside { 13 | padding: 2vw; 14 | background: #EEEEEE; 15 | } 16 | 17 | /* Unless the display is super narrow, show the examples on the side. */ 18 | @media (min-width: 60em) { 19 | body { 20 | display: flex; 21 | overflow: hidden; 22 | } 23 | main { 24 | flex-grow: 3; 25 | width: 0; 26 | } 27 | aside { 28 | height: calc(100vh - 4vw); /* 4vw of padding */ 29 | overflow-y: scroll; 30 | flex-grow: 2; 31 | width: 0; 32 | } 33 | } 34 | 35 | a[href] { 36 | color: #0E807B; 37 | } 38 | 39 | h1, h2 { 40 | margin-top: 0; 41 | } 42 | 43 | input { 44 | font-size: 1em; /* somehow is smaller by default in Safari */ 45 | } 46 | 47 | .CodeMirror, textarea, pre, tt { 48 | /* http://meyerweb.com/eric/thoughts/2010/02/12/fixed-monospace-sizing/ 49 | * (current Safari doesn't seem to need that serif trick either) */ 50 | font-family: Consolas, monospace; 51 | font-size: 1em; 52 | } 53 | 54 | .CodeMirror, textarea { 55 | /* Take up all the space except for padding (4vw) and headers/buttons/etc. 56 | * (adding up to roughly 10em, precision doesn't matter here). */ 57 | height: calc(100vh - 4vw - 10em); 58 | border: 1px solid darkgrey; 59 | } 60 | 61 | .CodeMirror { 62 | width: calc(100% - 2px); /* 2px of border */ 63 | } 64 | 65 | textarea { 66 | padding: 0.15em; 67 | width: calc(100% - 0.3em - 2px); /* 0.3em of padding and 2px of border */ 68 | /* Do not wrap long lines. */ 69 | white-space: pre; 70 | overflow-x: scroll; 71 | } 72 | 73 | pre { 74 | background: #FAFAFA; 75 | border-left: solid 0.5em #0E807B; 76 | padding: 0.5em 1em; 77 | overflow: hidden; 78 | margin-right: 0; 79 | } 80 | 81 | p.submit { 82 | vertical-align: center; 83 | } 84 | 85 | .status { 86 | padding-left: 1em; 87 | } 88 | 89 | .status.error { 90 | color: #720000; 91 | font-weight: bold; 92 | } 93 | 94 | .try { 95 | white-space: nowrap; 96 | } 97 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Turq 2 | ==== 3 | 4 | .. warning:: 5 | 6 | Turq is no longer maintained nor used by its author. 7 | Try `mitmproxy`__ with `scripting`__ instead. 8 | See also https://gist.github.com/vfaronov/3f7848932ed96a264c382902262ce7b3 9 | 10 | __ https://mitmproxy.org/ 11 | __ https://docs.mitmproxy.org/stable/addons-scripting/ 12 | 13 | Turq is a small HTTP server that can be scripted in a Python-based language. 14 | Use it to set up **mock HTTP resources** that respond with the status, headers, 15 | and body of your choosing. Turq is designed for **quick interactive testing**, 16 | but can be used in automated scenarios as well. 17 | 18 | 19 | Lets you do things like 20 | ----------------------- 21 | 22 | "RESTful API" resource with cross-origin support:: 23 | 24 | if route('/v1/products/:product_id'): 25 | if GET or HEAD: 26 | json({'id': product_id, 'inStock': True}) 27 | elif PUT: 28 | json(request.json) # As if we saved it 29 | elif DELETE: 30 | status(204) 31 | cors() # Handles preflight requests automatically 32 | 33 | Redirect to an ``index.php``, which serves a gzipped, cacheable page 34 | after 3 seconds of "loading":: 35 | 36 | if path == '/': 37 | redirect('/index.php') 38 | 39 | if path == '/index.php': 40 | sleep(3) 41 | html() 42 | gzip() 43 | header('Cache-Control', 'max-age=3600') 44 | 45 | Stream `server-sent events`_:: 46 | 47 | header('Content-Type', 'text/event-stream') 48 | for i in range(9000): 49 | sleep(1) 50 | chunk('data: event number %d\r\n\r\n' % i) 51 | 52 | .. _server-sent events: https://en.wikipedia.org/wiki/Server-sent_events 53 | 54 | 55 | Built-in editor 56 | --------------- 57 | 58 | You don't even need to create any files, just use the built-in Web editor: 59 | 60 | .. image:: screenshot.png 61 | 62 | 63 | Get it now 64 | ---------- 65 | 66 | In any Python 3.4+ environment:: 67 | 68 | $ pip3 install turq 69 | $ turq 70 | 71 | `Read the docs `_ for more. 72 | 73 | 74 | Thanks 75 | ------ 76 | 77 | `BrowserStack`_ have kindly provided a free subscription for testing Turq. 78 | 79 | .. _BrowserStack: https://www.browserstack.com/ 80 | -------------------------------------------------------------------------------- /turq/util/text.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | 4 | LOREM_IPSUM_WORDS = ['a', 'ac', 'accumsan', 'adipiscing', 'aenean', 'aliquam', 5 | 'amet', 'ante', 'arcu', 'at', 'augue', 'blandit', 6 | 'condimentum', 'congue', 'consectetur', 'consequat', 7 | 'convallis', 'cras', 'cursus', 'dapibus', 'diam', 8 | 'dictum', 'dignissim', 'dolor', 'donec', 'dui', 'duis', 9 | 'egestas', 'eget', 'eleifend', 'elementum', 'elit', 10 | 'enim', 'est', 'et', 'etiam', 'eu', 'euismod', 'ex', 11 | 'facilisis', 'faucibus', 'felis', 'feugiat', 'finibus', 12 | 'fringilla', 'fusce', 'gravida', 'id', 'in', 'integer', 13 | 'ipsum', 'justo', 'lacinia', 'laoreet', 'lectus', 14 | 'libero', 'ligula', 'lorem', 'luctus', 'maecenas', 15 | 'magna', 'mattis', 'mauris', 'maximus', 'metus', 'mi', 16 | 'molestie', 'mollis', 'nec', 'neque', 'nibh', 'nisi', 17 | 'nisl', 'non', 'nulla', 'nunc', 'odio', 'orci', 18 | 'ornare', 'pellentesque', 'phasellus', 'porttitor', 19 | 'praesent', 'pretium', 'pulvinar', 'purus', 'quis', 20 | 'risus', 'rutrum', 'sagittis', 'sapien', 'scelerisque', 21 | 'sed', 'sem', 'sit', 'sollicitudin', 'suscipit', 22 | 'tellus', 'tempus', 'tincidunt', 'tristique', 'turpis', 23 | 'ullamcorper', 'urna', 'ut', 'vel', 'velit', 'vestibulum', 24 | 'vitae', 'vivamus', 'viverra', 'vulputate'] 25 | 26 | 27 | def force_bytes(x, encoding='iso-8859-1'): 28 | if isinstance(x, bytes): 29 | return x 30 | else: 31 | return x.encode(encoding) 32 | 33 | 34 | def lorem_ipsum(): 35 | return ' '.join( # sentences 36 | ' '.join( # words 37 | random.sample(LOREM_IPSUM_WORDS, random.randint(5, 10)) 38 | ).capitalize() + '.' 39 | for _ in range(random.randint(5, 10)) 40 | ) 41 | 42 | 43 | def ellipsize(s, max_length=60): 44 | """ 45 | >>> print(ellipsize(u'lorem ipsum dolor sit amet', 40)) 46 | lorem ipsum dolor sit amet 47 | >>> print(ellipsize(u'lorem ipsum dolor sit amet', 20)) 48 | lorem ipsum dolor... 49 | """ 50 | if len(s) > max_length: 51 | ellipsis = '...' 52 | return s[:(max_length - len(ellipsis))] + ellipsis 53 | else: 54 | return s 55 | -------------------------------------------------------------------------------- /HACKING.rst: -------------------------------------------------------------------------------- 1 | Developing Turq 2 | =============== 3 | 4 | Development environment 5 | ~~~~~~~~~~~~~~~~~~~~~~~ 6 | 7 | Set up:: 8 | 9 | $ virtualenv /path/to/env 10 | $ source /path/to/env/bin/activate 11 | $ pip install -e . 12 | $ pip install -r tools/requirements.txt 13 | $ pip install ... # any extra tools you like to have 14 | 15 | Run tests:: 16 | 17 | $ pytest 18 | 19 | The delivery pipeline (Travis) enforces some other checks; if you want to run 20 | them locally before pushing to GitHub, see ``.travis.yml``. 21 | 22 | Versions of development tools (pytest, Pylint...) are pinned down to make 23 | builds/QA reproducible. From time to time, they are `manually upgraded 24 | `_. 25 | 26 | 27 | Releasing a new version 28 | ~~~~~~~~~~~~~~~~~~~~~~~ 29 | 30 | #. Make sure that you're on master, it's clean and synced with GitHub, 31 | and that Travis and AppVeyor are green. 32 | 33 | #. If necessary, update the version number in ``turq/__metadata__.py`` 34 | (e.g. 0.12.0.dev4 → 0.12.0). 35 | 36 | #. If releasing a "stable" (not pre-release) version, update ``CHANGELOG.rst`` 37 | (usually just replace "Unreleased" with " - ", 38 | e.g. "0.12.0 - 2017-08-14"). 39 | 40 | #. Commit as necessary, for example:: 41 | 42 | $ git commit -am 'Version 0.12.0' 43 | 44 | #. Apply a Git tag for the version, for example:: 45 | 46 | $ git tag -a v0.12.0 -m v0.12.0 47 | 48 | #. Push master and tags:: 49 | 50 | $ git push --tags origin master 51 | 52 | #. Watch as Travis builds and uploads stuff to PyPI. 53 | 54 | #. If releasing a "stable" (not pre-release) version, check that the 55 | `stable docs `_ have been updated 56 | (you may need to force-refresh the page to see it). 57 | 58 | #. Bump the version number in ``turq/__metadata__.py`` 59 | (e.g. 0.12.0 → 0.13.0.dev1). 60 | 61 | #. Commit and push:: 62 | 63 | $ git commit -am 'Bump version to 0.13.0.dev1' 64 | $ git push 65 | 66 | 67 | Maintenance 68 | ~~~~~~~~~~~ 69 | 70 | - Watch for new versions of Python and dependencies (``install_requires``), 71 | and make sure Turq is compatible with them. 72 | 73 | - Update development dependencies: 74 | 75 | #. Review ``tools/requirements.in`` and update if necessary. 76 | 77 | #. Pin down new versions:: 78 | 79 | $ rm tools/requirements.txt 80 | $ pip-compile tools/requirements.in 81 | $ pip install -r tools/requirements.txt 82 | 83 | - Look at recent Travis and AppVeyor build logs and make sure everything there 84 | looks alright. 85 | 86 | - Check that the Python version trove classifiers in ``setup.py`` 87 | are up-to-date. 88 | 89 | -------------------------------------------------------------------------------- /turq/editor/editor.js: -------------------------------------------------------------------------------- 1 | /* jshint browser: true, -W097, -W040 */ 2 | /* globals CodeMirror */ 3 | 4 | 'use strict'; 5 | 6 | 7 | var codeMirror = null; 8 | 9 | 10 | function installCodeMirror() { 11 | var textArea = document.querySelector('textarea'); 12 | codeMirror = CodeMirror.fromTextArea(textArea, { 13 | lineNumbers: true, 14 | indentUnit: 4, 15 | extraKeys: { 16 | Tab: 'insertSoftTab' // insert spaces when Tab is pressed 17 | } 18 | }); 19 | CodeMirror.colorize(document.querySelectorAll('aside pre'), 'python'); 20 | } 21 | 22 | 23 | function interceptForm() { 24 | document.forms[0].addEventListener('submit', function (e) { 25 | e.preventDefault(); 26 | hideStatus(); 27 | // Send more or less the same data, but via XHR. 28 | var req = new XMLHttpRequest(); 29 | req.onreadystatechange = onReadyStateChange; 30 | codeMirror.save(); 31 | var data = new FormData(this); 32 | // https://blog.yorkxin.org/2014/02/06/ajax-with-formdata-is-broken-on-ie10-ie11 33 | data.append('workaround', 'IE'); 34 | req.open(this.method, this.action); 35 | req.timeout = 5000; 36 | req.send(data); 37 | codeMirror.focus(); 38 | }); 39 | } 40 | 41 | 42 | function onReadyStateChange() { 43 | if (this.readyState !== 4) return; // not DONE yet 44 | var text; 45 | var type = (this.getResponseHeader('Content-Type') || '').toLowerCase(); 46 | if (type.indexOf('text/plain') === 0) { 47 | text = this.responseText; 48 | } else if (this.status) { 49 | text = this.statusText; 50 | } else { 51 | text = 'Connection error'; 52 | } 53 | var isError = !this.status || (this.status >= 400); 54 | displayStatus(text, isError); 55 | } 56 | 57 | 58 | var timeoutId = null; 59 | 60 | 61 | function displayStatus(text, isError) { 62 | if (timeoutId !== null) { 63 | window.clearTimeout(timeoutId); 64 | } 65 | var status = document.querySelector('.status'); 66 | status.textContent = text; 67 | if (isError) { 68 | status.classList.add('error'); 69 | } else { 70 | status.classList.remove('error'); 71 | // An "okay" message is safe to hide after a short delay. 72 | timeoutId = window.setTimeout(hideStatus, 1000); 73 | } 74 | } 75 | 76 | 77 | function hideStatus() { 78 | document.querySelector('.status').textContent = ''; 79 | timeoutId = null; 80 | } 81 | 82 | 83 | document.addEventListener('DOMContentLoaded', function() { 84 | document.querySelector('textarea').focus(); 85 | installCodeMirror(); 86 | interceptForm(); 87 | }); 88 | -------------------------------------------------------------------------------- /turq/editor/codemirror/addon/runmode/runmode.js: -------------------------------------------------------------------------------- 1 | // CodeMirror, copyright (c) by Marijn Haverbeke and others 2 | // Distributed under an MIT license: http://codemirror.net/LICENSE 3 | 4 | (function(mod) { 5 | if (typeof exports == "object" && typeof module == "object") // CommonJS 6 | mod(require("../../lib/codemirror")); 7 | else if (typeof define == "function" && define.amd) // AMD 8 | define(["../../lib/codemirror"], mod); 9 | else // Plain browser env 10 | mod(CodeMirror); 11 | })(function(CodeMirror) { 12 | "use strict"; 13 | 14 | CodeMirror.runMode = function(string, modespec, callback, options) { 15 | var mode = CodeMirror.getMode(CodeMirror.defaults, modespec); 16 | var ie = /MSIE \d/.test(navigator.userAgent); 17 | var ie_lt9 = ie && (document.documentMode == null || document.documentMode < 9); 18 | 19 | if (callback.appendChild) { 20 | var tabSize = (options && options.tabSize) || CodeMirror.defaults.tabSize; 21 | var node = callback, col = 0; 22 | node.innerHTML = ""; 23 | callback = function(text, style) { 24 | if (text == "\n") { 25 | // Emitting LF or CRLF on IE8 or earlier results in an incorrect display. 26 | // Emitting a carriage return makes everything ok. 27 | node.appendChild(document.createTextNode(ie_lt9 ? '\r' : text)); 28 | col = 0; 29 | return; 30 | } 31 | var content = ""; 32 | // replace tabs 33 | for (var pos = 0;;) { 34 | var idx = text.indexOf("\t", pos); 35 | if (idx == -1) { 36 | content += text.slice(pos); 37 | col += text.length - pos; 38 | break; 39 | } else { 40 | col += idx - pos; 41 | content += text.slice(pos, idx); 42 | var size = tabSize - col % tabSize; 43 | col += size; 44 | for (var i = 0; i < size; ++i) content += " "; 45 | pos = idx + 1; 46 | } 47 | } 48 | 49 | if (style) { 50 | var sp = node.appendChild(document.createElement("span")); 51 | sp.className = "cm-" + style.replace(/ +/g, " cm-"); 52 | sp.appendChild(document.createTextNode(content)); 53 | } else { 54 | node.appendChild(document.createTextNode(content)); 55 | } 56 | }; 57 | } 58 | 59 | var lines = CodeMirror.splitLines(string), state = (options && options.state) || CodeMirror.startState(mode); 60 | for (var i = 0, e = lines.length; i < e; ++i) { 61 | if (i) callback("\n"); 62 | var stream = new CodeMirror.StringStream(lines[i]); 63 | if (!stream.string && mode.blankLine) mode.blankLine(state); 64 | while (!stream.eol()) { 65 | var style = mode.token(stream, state); 66 | callback(stream.current(), style, i, stream.start, state); 67 | stream.start = stream.pos; 68 | } 69 | } 70 | }; 71 | 72 | }); 73 | -------------------------------------------------------------------------------- /turq/util/http.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | import http.server 3 | from ipaddress import IPv6Address 4 | import re 5 | import socket 6 | 7 | import werkzeug.http 8 | 9 | 10 | # https://www.iana.org/assignments/http-methods/http-methods.xhtml 11 | KNOWN_METHODS = ['ACL', 'BASELINE-CONTROL', 'BIND', 'CHECKIN', 'CHECKOUT', 12 | 'CONNECT', 'COPY', 'DELETE', 'GET', 'HEAD', 'LABEL', 'LINK', 13 | 'LOCK', 'MERGE', 'MKACTIVITY', 'MKCALENDAR', 'MKCOL', 14 | 'MKREDIRECTREF', 'MKWORKSPACE', 'MOVE', 'OPTIONS', 15 | 'ORDERPATCH', 'PATCH', 'POST', 'PRI', 'PROPFIND', 'PROPPATCH', 16 | 'PUT', 'REBIND', 'REPORT', 'SEARCH', 'TRACE', 'UNBIND', 17 | 'UNCHECKOUT', 'UNLINK', 'UNLOCK', 'UPDATE', 18 | 'UPDATEREDIRECTREF', 'VERSION-CONTROL'] 19 | 20 | 21 | IPV4_REVERSE_DNS = re.compile(r'^' + r'([0-9]+)\.' * 4 + r'in-addr\.arpa\.?$', 22 | flags=re.IGNORECASE) 23 | IPV6_REVERSE_DNS = re.compile(r'^' + r'([0-9a-f])\.' * 32 + r'ip6\.arpa\.?$', 24 | flags=re.IGNORECASE) 25 | 26 | 27 | def default_reason(status_code): 28 | common_phrases = http.server.BaseHTTPRequestHandler.responses 29 | (reason, _) = common_phrases.get(status_code, ('Unknown', None)) 30 | return reason 31 | 32 | 33 | def error_explanation(status_code): 34 | common_phrases = http.server.BaseHTTPRequestHandler.responses 35 | (_, explanation) = common_phrases.get(status_code, 36 | (None, 'Something is wrong')) 37 | return explanation 38 | 39 | 40 | def date(): 41 | return werkzeug.http.http_date(datetime.utcnow()) 42 | 43 | 44 | def nice_header_name(name): 45 | # "cache-control" -> "Cache-Control" 46 | return '-'.join(word.capitalize() for word in name.split('-')) 47 | 48 | 49 | def guess_external_url(local_host, port): 50 | """Return a URL that is most likely to route to `local_host` from outside. 51 | 52 | The point is that we may be running on a remote host from the user's 53 | point of view, so they can't access `local_host` from a Web browser just 54 | by typing ``http://localhost:12345/``. 55 | """ 56 | if local_host in ['0.0.0.0', '::']: 57 | # The server is listening on all interfaces, but we have to pick one. 58 | # The system's FQDN should give us a hint. 59 | local_host = socket.getfqdn() 60 | 61 | # https://github.com/vfaronov/turq/issues/9 62 | match = IPV4_REVERSE_DNS.match(local_host) 63 | if match: 64 | local_host = '.'.join(reversed(match.groups())) 65 | else: 66 | match = IPV6_REVERSE_DNS.match(local_host) 67 | if match: 68 | address_as_int = int(''.join(reversed(match.groups())), 16) 69 | local_host = str(IPv6Address(address_as_int)) 70 | 71 | if ':' in local_host: 72 | # Looks like an IPv6 literal. Has to be wrapped in brackets in a URL. 73 | # Also, an IPv6 address can have a zone ID tacked on the end, 74 | # like "%3". RFC 6874 allows encoding them in URLs as well, 75 | # but in my experiments on Windows 8.1, I had more success 76 | # removing the zone ID altogether. After all this is just a guess. 77 | local_host = '[%s]' % local_host.rsplit('%', 1)[0] 78 | 79 | return 'http://%s:%d/' % (local_host, port) 80 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import subprocess 3 | import sys 4 | import time 5 | 6 | import h11 7 | import pytest 8 | import requests 9 | 10 | 11 | @pytest.fixture 12 | def turq_instance(): 13 | return TurqInstance() 14 | 15 | 16 | class TurqInstance: 17 | 18 | """Spins up and controls a live instance of Turq for testing.""" 19 | 20 | def __init__(self): 21 | self.host = 'localhost' 22 | # Test instance listens on port 13095 instead of the default 13085, 23 | # to make it easier to run tests while also testing Turq manually. 24 | # Of course, ideally it should be a random free port instead. 25 | self.mock_port = 13095 26 | self.editor_port = 13096 27 | self.password = '' 28 | self.extra_args = [] 29 | self.wait = True 30 | self._process = None 31 | self.console_output = None 32 | 33 | def __enter__(self): 34 | args = [sys.executable, '-m', 'turq.main', 35 | '--bind', self.host, '--mock-port', str(self.mock_port), 36 | '--editor-port', str(self.editor_port)] 37 | if self.password is not None: 38 | args += ['--editor-password', self.password] 39 | args += self.extra_args 40 | self._process = subprocess.Popen(args, stdin=subprocess.DEVNULL, 41 | stdout=subprocess.DEVNULL, 42 | stderr=subprocess.PIPE) 43 | if self.wait: 44 | self._wait_for_server() 45 | return self 46 | 47 | def __exit__(self, exc_type, exc_value, traceback): 48 | self._process.terminate() 49 | self._process.wait() 50 | self.console_output = self._process.stderr.read().decode() 51 | return False 52 | 53 | def _wait_for_server(self, timeout=3): 54 | # Wait until the mock server starts accepting connections, 55 | # but no more than `timeout` seconds. 56 | t0 = time.monotonic() 57 | while time.monotonic() - t0 < timeout: 58 | time.sleep(0.1) 59 | try: 60 | self.connect().close() 61 | self.connect_editor().close() 62 | return 63 | except OSError: 64 | pass 65 | raise RuntimeError('Turq failed to start') 66 | 67 | def connect(self): 68 | return socket.create_connection((self.host, self.mock_port), timeout=5) 69 | 70 | def connect_editor(self): 71 | return socket.create_connection((self.host, self.editor_port), 72 | timeout=5) 73 | 74 | def send(self, *events): 75 | hconn = h11.Connection(our_role=h11.CLIENT) 76 | with self.connect() as sock: 77 | for event in events: 78 | sock.sendall(hconn.send(event)) 79 | sock.shutdown(socket.SHUT_WR) 80 | while hconn.their_state is not h11.CLOSED: 81 | event = hconn.next_event() 82 | if event is h11.NEED_DATA: 83 | hconn.receive_data(sock.recv(4096)) 84 | elif not isinstance(event, h11.ConnectionClosed): 85 | yield event 86 | 87 | def request(self, method, url, **kwargs): 88 | full_url = 'http://%s:%d%s' % (self.host, self.mock_port, url) 89 | return requests.request(method, full_url, **kwargs) 90 | 91 | def request_editor(self, method, url, **kwargs): 92 | full_url = 'http://%s:%d%s' % (self.host, self.editor_port, url) 93 | return requests.request(method, full_url, **kwargs) 94 | -------------------------------------------------------------------------------- /turq/mock.py: -------------------------------------------------------------------------------- 1 | # This module, together with `turq.rules`, constitutes the Turq mock server. 2 | # It tries to be mostly HTTP-compliant by default, but it doesn't care at all 3 | # about performance. In particular, there are no explicit timeouts. 4 | 5 | import logging 6 | import socket 7 | import socketserver 8 | 9 | import h11 10 | 11 | from turq.rules import RULES_FILENAME, RulesContext 12 | import turq.util.http 13 | from turq.util.logging import getNextLogger 14 | 15 | 16 | class MockServer(socketserver.ThreadingMixIn, socketserver.TCPServer): 17 | 18 | allow_reuse_address = True # Prevent "Address already in use" on restart 19 | daemon_threads = True 20 | 21 | def __init__(self, host, port, ipv6, initial_rules, 22 | bind_and_activate=True): 23 | self.address_family = socket.AF_INET6 if ipv6 else socket.AF_INET 24 | super().__init__((host, port), MockHandler, bind_and_activate) 25 | self.install_rules(initial_rules) 26 | 27 | def install_rules(self, rules): 28 | self.compiled_rules = compile(rules, RULES_FILENAME, 'exec') 29 | self.rules = rules 30 | logging.getLogger('turq').info('new rules installed') 31 | 32 | 33 | class MockHandler(socketserver.StreamRequestHandler): 34 | 35 | def setup(self): 36 | super().setup() 37 | self._logger = getNextLogger('turq.connection') 38 | self._socket = self.request # To reduce confusion with HTTP requests 39 | self._hconn = h11.Connection(our_role=h11.SERVER) 40 | 41 | def handle(self): 42 | self._logger.info('new connection from %s', self.client_address[0]) 43 | try: 44 | while True: 45 | # pylint: disable=protected-access 46 | event = self.receive_event() 47 | if isinstance(event, h11.Request): # not `ConnectionClosed` 48 | # `RulesContext` takes care of handling one complete 49 | # request/response cycle. 50 | RulesContext(self.server.compiled_rules, self)._run(event) 51 | self._logger.debug('states: %r', self._hconn.states) 52 | if self._hconn.states == {h11.CLIENT: h11.DONE, 53 | h11.SERVER: h11.DONE}: 54 | # Connection persists, proceed to the next cycle. 55 | self._hconn.start_next_cycle() 56 | else: 57 | # Connection has to be closed (e.g. because HTTP/1.0 58 | # or because somebody sent "Connection: close"). 59 | break 60 | except Exception as e: 61 | self._logger.error('error: %s', e) 62 | self._logger.debug('states: %r', self._hconn.states) 63 | if self._hconn.our_state in [h11.SEND_RESPONSE, h11.IDLE]: 64 | self._send_fatal_error(e) 65 | 66 | @property 67 | def our_state(self): 68 | return self._hconn.our_state 69 | 70 | @property 71 | def their_state(self): 72 | return self._hconn.their_state 73 | 74 | def receive_event(self): 75 | while True: 76 | event = self._hconn.next_event() 77 | if event is h11.NEED_DATA: 78 | self._hconn.receive_data(self._socket.recv(4096)) 79 | else: 80 | return event 81 | 82 | def send_event(self, event): 83 | data = self._hconn.send(event) 84 | self._socket.sendall(data) 85 | 86 | def send_raw(self, data): 87 | self._socket.sendall(data) 88 | 89 | def _send_fatal_error(self, exc): 90 | status_code = getattr(exc, 'error_status_hint', 500) 91 | self._logger.debug('sending error response, status %d', status_code) 92 | try: 93 | self.send_event(h11.Response( 94 | status_code=status_code, 95 | reason=turq.util.http.default_reason(status_code).encode(), 96 | headers=[ 97 | (b'Date', turq.util.http.date().encode()), 98 | (b'Content-Type', b'text/plain'), 99 | (b'Connection', b'close'), 100 | ], 101 | )) 102 | self.send_event(h11.Data(data=('Error: %s\r\n' % exc).encode())) 103 | self.send_event(h11.EndOfMessage()) 104 | except Exception as e: 105 | self._logger.debug('cannot send error response: %s', e) 106 | 107 | # A crude way to avoid the TCP reset problem (RFC 7230 Section 6.6). 108 | try: 109 | self._socket.shutdown(socket.SHUT_WR) 110 | while self._socket.recv(1024): 111 | self._logger.debug('discarding data from client') 112 | except OSError: # The client may have already closed the connection 113 | pass 114 | -------------------------------------------------------------------------------- /docs/guide.rst: -------------------------------------------------------------------------------- 1 | User guide 2 | ========== 3 | 4 | .. highlight:: console 5 | 6 | Quick start 7 | ----------- 8 | 9 | To run Turq, you need `Python`_ 3.4 or higher. 10 | Once you have that, install the `turq`_ package with `pip`_:: 11 | 12 | $ pip3 install turq 13 | 14 | .. _turq: https://pypi.python.org/pypi/turq 15 | 16 | Start Turq:: 17 | 18 | $ turq 19 | 20 | You should see something like this:: 21 | 22 | 18:22:19 turq new rules installed 23 | 18:22:19 turq mock on port 13085 - try http://pergamon:13085/ 24 | 18:22:19 turq editor on port 13086 - try http://pergamon:13086/ 25 | 18:22:19 turq editor password: QGOf9Y9Eqjvz4XhY4JA3U7hG (any username) 26 | 27 | As you can see, Turq starts two HTTP servers. One is the *mock server* 28 | for the mocks you define. The other is the optional *rules editor* 29 | that makes writing mocks easier. 30 | 31 | First you probably want to open the editor. By default, Turq listens on all 32 | network interfaces, so you can open the editor at ``http://localhost:13086/`` 33 | in your Web browser. Turq also tries to guess and print a URL that doesn't 34 | include ``localhost``, which is useful when you run Turq on some remote 35 | machine via SSH. 36 | 37 | Turq will ask you for the password that it generated and printed for you. 38 | You can leave the username field blank, it is ignored. 39 | 40 | .. warning:: 41 | 42 | Anybody with access to the Turq editor can **execute arbitrary code** 43 | in the Turq process. The default password protection should keep you safe 44 | in most cases, but doesn't help against an active man-in-the-middle. 45 | If that's a problem, limit Turq to loopback with ``--bind localhost``, 46 | or `run without the editor `_. 47 | 48 | In the editor, you define your mock by writing rules in the big code area, 49 | using the examples on the right as your guide. The default rules are just 50 | ``error(404)``, which means that the mock server will respond with 404 51 | (Not Found) to every request. Let's check that with curl:: 52 | 53 | $ curl -i http://pergamon:13085/some/page.html 54 | HTTP/1.1 404 Not Found 55 | content-type: text/plain; charset=utf-8 56 | date: Tue, 04 Apr 2017 15:33:55 GMT 57 | transfer-encoding: chunked 58 | 59 | Error! Nothing matches the given URI 60 | 61 | Keep an eye on the system console where you launched ``turq`` --- 62 | all requests and responses are logged there:: 63 | 64 | 19:01:30 turq.connection.1 new connection from 127.0.0.1 65 | 19:01:30 turq.request.1 > GET /some/page.html HTTP/1.1 66 | 19:01:30 turq.request.1 < HTTP/1.1 404 Not Found 67 | 68 | When you are done, stop Turq by pressing Ctrl+C in the console. 69 | 70 | That's it, basically. Check ``turq --help`` for command-line options, 71 | or read on for more hints on how to use Turq. 72 | 73 | .. _Python: https://www.python.org/ 74 | .. _pip: https://pip.pypa.io/ 75 | 76 | 77 | Programmatic use 78 | ---------------- 79 | 80 | Turq was designed for interactive use; it trades precision for convenience 81 | and simplicity. However, you can use it non-interactively if you like:: 82 | 83 | $ turq --no-editor --rules /path/to/rules.py 84 | 85 | Give it a second to spin up, or just loop until you can ``connect()`` to it. 86 | Shut it down with SIGTERM like any other process:: 87 | 88 | $ pkill turq 89 | 90 | It goes without saying that Turq can’t be used anywhere near production. 91 | 92 | 93 | Using mitmproxy with Turq 94 | ------------------------- 95 | 96 | Put `mitmproxy`_ in front of Turq to: 97 | 98 | - enable TLS (``https``) access to the mock server; 99 | - inspect all requests and responses in detail; 100 | - validate them with `HTTPolice`_; and more. 101 | 102 | .. _mitmproxy: https://mitmproxy.org/ 103 | .. _HTTPolice: https://github.com/vfaronov/httpolice 104 | 105 | Assuming Turq runs on the default port, use a command like this:: 106 | 107 | $ mitmproxy -p 13185 --mode reverse:http://localhost:13085 108 | 109 | Then tell your client to connect to port 13185 (``http`` or ``https``) 110 | instead of 13085. 111 | 112 | 113 | Known issues 114 | ------------ 115 | 116 | .. highlight:: python 117 | 118 | Password protection in the rules editor does not work well in some browsers. 119 | For example, you may randomly get "Connection error" in Internet Explorer. 120 | To avoid this, you can disable password protection with ``-P ""``, but be sure 121 | to have some other protection instead. 122 | 123 | The mock server doesn't send any cache-related headers by default. As a result, 124 | some browsers may cache your mocks, leading to strange results. You can disable 125 | caching in your rules:: 126 | 127 | add_header('Cache-Control', 'no-store') 128 | 129 | Turq has limited options to control the addresses it listens on. 130 | You can forward its ports manually with `socat`_ or mitmproxy. 131 | 132 | .. _socat: http://www.dest-unreach.org/socat/ 133 | -------------------------------------------------------------------------------- /turq/main.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import base64 3 | import logging 4 | import os 5 | import sys 6 | import threading 7 | 8 | import colorlog 9 | 10 | import turq 11 | import turq.editor 12 | import turq.mock 13 | from turq.util.http import guess_external_url 14 | 15 | DEFAULT_ADDRESS = '' # All interfaces 16 | DEFAULT_MOCK_PORT = 13085 17 | DEFAULT_EDITOR_PORT = 13086 18 | DEFAULT_RULES = 'error(404)\n' 19 | 20 | logger = logging.getLogger('turq') 21 | 22 | 23 | def main(): 24 | args = parse_args(sys.argv) 25 | if not args.verbose: 26 | sys.excepthook = excepthook 27 | setup_logging(args) 28 | run(args) 29 | 30 | 31 | def parse_args(argv): 32 | parser = argparse.ArgumentParser() 33 | parser.add_argument('--version', action='version', 34 | version='Turq %s' % turq.__version__) 35 | parser.add_argument('-v', '--verbose', action='store_true', 36 | help='print more verbose diagnostics') 37 | parser.add_argument('--no-color', action='store_true', 38 | help='do not colorize console output') 39 | parser.add_argument('--no-editor', action='store_true', 40 | help='disable the built-in Web-based rules editor') 41 | parser.add_argument('-b', '--bind', metavar='ADDRESS', 42 | default=DEFAULT_ADDRESS, 43 | help='IP address or hostname to listen on') 44 | parser.add_argument('-p', '--mock-port', metavar='PORT', type=int, 45 | default=DEFAULT_MOCK_PORT, 46 | help='port for the mock server to listen on') 47 | parser.add_argument('--editor-port', metavar='PORT', type=int, 48 | default=DEFAULT_EDITOR_PORT, 49 | help='port for the rules editor to listen on') 50 | parser.add_argument('-6', '--ipv6', action='store_true', 51 | default=False, 52 | help=('listen on IPv6 instead of IPv4 ' 53 | '(or on both, depending on the system)')) 54 | parser.add_argument('-r', '--rules', metavar='PATH', 55 | type=argparse.FileType('r'), 56 | help='file with initial rules code') 57 | parser.add_argument('-P', '--editor-password', metavar='PASSWORD', 58 | default=random_password(), 59 | help='explicitly set editor password ' 60 | '(empty string to disable)') 61 | return parser.parse_args(argv[1:]) 62 | 63 | 64 | def excepthook(_type, exc, _traceback): 65 | sys.stderr.write('turq: error: %s\n' % exc) 66 | 67 | 68 | def setup_logging(args): 69 | if args.no_color: 70 | formatter = logging.Formatter( 71 | fmt='%(asctime)s %(name)s %(message)s', 72 | datefmt='%H:%M:%S') 73 | else: 74 | formatter = colorlog.ColoredFormatter( 75 | fmt=('%(asctime)s ' 76 | '%(name_log_color)s%(name)s%(reset)s ' 77 | '%(log_color)s%(message)s%(reset)s'), 78 | datefmt='%H:%M:%S', 79 | log_colors={'DEBUG': 'green', 'ERROR': 'red', 'CRITICAL': 'red'}, 80 | secondary_log_colors={ 81 | 'name': {'DEBUG': 'cyan', 'INFO': 'cyan', 82 | 'WARNING': 'cyan', 'ERROR': 'cyan', 83 | 'CRITICAL': 'cyan'}, 84 | }, 85 | ) 86 | handler = logging.StreamHandler() 87 | handler.setFormatter(formatter) 88 | logger.addHandler(handler) 89 | logger.setLevel(logging.DEBUG if args.verbose else logging.INFO) 90 | 91 | 92 | def run(args): 93 | rules = args.rules.read() if args.rules else DEFAULT_RULES 94 | mock_server = turq.mock.MockServer(args.bind, args.mock_port, args.ipv6, 95 | rules) 96 | 97 | if args.no_editor: 98 | editor_server = None 99 | else: 100 | editor_server = turq.editor.make_server( 101 | args.bind, args.editor_port, args.ipv6, 102 | args.editor_password, mock_server) 103 | threading.Thread(target=editor_server.serve_forever).start() 104 | 105 | # Show mock server info just before going into `serve_forever`, 106 | # to minimize the delay between printing it and actually listening 107 | show_server_info('mock', mock_server) 108 | if editor_server is not None: 109 | show_server_info('editor', editor_server) 110 | if args.editor_password: 111 | logger.info('editor password: %s (any username)', 112 | args.editor_password) 113 | 114 | try: 115 | mock_server.serve_forever() 116 | except KeyboardInterrupt: 117 | mock_server.server_close() 118 | sys.stderr.write('\n') 119 | 120 | if editor_server is not None: 121 | editor_server.shutdown() 122 | editor_server.server_close() 123 | 124 | 125 | def show_server_info(label, server): 126 | (host, port, *_) = server.server_address 127 | logger.info('%s on port %d - try %s', 128 | label, port, guess_external_url(host, port)) 129 | 130 | 131 | def random_password(): 132 | return base64.b64encode(os.urandom(18), altchars=b'Ab').decode() 133 | 134 | 135 | if __name__ == '__main__': 136 | main() 137 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Turq documentation build configuration file, created by 5 | # sphinx-quickstart on Tue Apr 4 17:53:11 2017. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # 20 | import os 21 | # import sys 22 | # sys.path.insert(0, os.path.abspath('.')) 23 | 24 | 25 | # -- General configuration ------------------------------------------------ 26 | 27 | # If your documentation needs a minimal Sphinx version, state it here. 28 | # 29 | # needs_sphinx = '1.0' 30 | 31 | # Add any Sphinx extension module names here, as strings. They can be 32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 33 | # ones. 34 | extensions = [] 35 | 36 | # Add any paths that contain templates here, relative to this directory. 37 | templates_path = ['_templates'] 38 | 39 | # The suffix(es) of source filenames. 40 | # You can specify multiple suffix as a list of string: 41 | # 42 | # source_suffix = ['.rst', '.md'] 43 | source_suffix = '.rst' 44 | 45 | # The master toctree document. 46 | master_doc = 'index' 47 | 48 | # General information about the project. 49 | project = 'Turq' 50 | copyright = '2017, Vasiliy Faronov' 51 | author = 'Vasiliy Faronov' 52 | 53 | # The version info for the project you're documenting, acts as replacement for 54 | # |version| and |release|, also used in various other places throughout the 55 | # built documents. 56 | metadata = {} 57 | with open(os.path.join('..', 'turq', '__metadata__.py'), 'r') as f: 58 | exec(f.read(), metadata) 59 | version = release = metadata['version'] 60 | 61 | # The language for content autogenerated by Sphinx. Refer to documentation 62 | # for a list of supported languages. 63 | # 64 | # This is also used if you do content translation via gettext catalogs. 65 | # Usually you set "language" from the command line for these cases. 66 | language = None 67 | 68 | # List of patterns, relative to source directory, that match files and 69 | # directories to ignore when looking for source files. 70 | # This patterns also effect to html_static_path and html_extra_path 71 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 72 | 73 | # The name of the Pygments (syntax highlighting) style to use. 74 | pygments_style = 'sphinx' 75 | 76 | # If true, `todo` and `todoList` produce output, else they produce nothing. 77 | todo_include_todos = False 78 | 79 | 80 | # -- Options for HTML output ---------------------------------------------- 81 | 82 | # The theme to use for HTML and HTML Help pages. See the documentation for 83 | # a list of builtin themes. 84 | # 85 | html_theme = 'sphinx_rtd_theme' 86 | 87 | # Theme options are theme-specific and customize the look and feel of a theme 88 | # further. For a list of options available for each theme, see the 89 | # documentation. 90 | # 91 | # html_theme_options = {} 92 | 93 | # Add any paths that contain custom static files (such as style sheets) here, 94 | # relative to this directory. They are copied after the builtin static files, 95 | # so a file named "default.css" will overwrite the builtin "default.css". 96 | html_static_path = [] 97 | 98 | 99 | # -- Options for HTMLHelp output ------------------------------------------ 100 | 101 | # Output file base name for HTML help builder. 102 | htmlhelp_basename = 'Turqdoc' 103 | 104 | 105 | # -- Options for LaTeX output --------------------------------------------- 106 | 107 | latex_elements = { 108 | # The paper size ('letterpaper' or 'a4paper'). 109 | # 110 | # 'papersize': 'letterpaper', 111 | 112 | # The font size ('10pt', '11pt' or '12pt'). 113 | # 114 | # 'pointsize': '10pt', 115 | 116 | # Additional stuff for the LaTeX preamble. 117 | # 118 | # 'preamble': '', 119 | 120 | # Latex figure (float) alignment 121 | # 122 | # 'figure_align': 'htbp', 123 | } 124 | 125 | # Grouping the document tree into LaTeX files. List of tuples 126 | # (source start file, target name, title, 127 | # author, documentclass [howto, manual, or own class]). 128 | latex_documents = [ 129 | (master_doc, 'Turq.tex', 'Turq Documentation', 130 | 'Vasiliy Faronov', 'manual'), 131 | ] 132 | 133 | 134 | # -- Options for manual page output --------------------------------------- 135 | 136 | # One entry per manual page. List of tuples 137 | # (source start file, name, description, authors, manual section). 138 | man_pages = [ 139 | (master_doc, 'turq', 'Turq Documentation', 140 | [author], 1) 141 | ] 142 | 143 | 144 | # -- Options for Texinfo output ------------------------------------------- 145 | 146 | # Grouping the document tree into Texinfo files. List of tuples 147 | # (source start file, target name, title, author, 148 | # dir menu entry, description, category) 149 | texinfo_documents = [ 150 | (master_doc, 'Turq', 'Turq Documentation', 151 | author, 'Turq', 'One line description of project.', 152 | 'Miscellaneous'), 153 | ] 154 | 155 | 156 | 157 | -------------------------------------------------------------------------------- /turq/editor.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=unused-argument 2 | 3 | import base64 4 | import hashlib 5 | import html 6 | import mimetypes 7 | import os 8 | import pkgutil 9 | import posixpath 10 | import socket 11 | import socketserver 12 | import string 13 | import threading 14 | import wsgiref.simple_server 15 | 16 | import falcon 17 | import werkzeug.formparser 18 | 19 | import turq.examples 20 | from turq.util.http import guess_external_url 21 | 22 | 23 | STATIC_PREFIX = '/static/' 24 | 25 | 26 | def make_server(host, port, ipv6, password, mock_server): 27 | editor = falcon.API(media_type='text/plain; charset=utf-8', 28 | middleware=[CommonHeaders()]) 29 | # Microsoft Edge doesn't send ``Authorization: Digest`` to ``/``. 30 | # Can be circumvented with ``/?``, but I think ``/editor`` is better. 31 | editor.add_route('/editor', EditorResource(mock_server, password)) 32 | editor.add_route('/', RedirectResource()) 33 | editor.add_sink(static_file, STATIC_PREFIX) 34 | editor.set_error_serializer(text_error_serializer) 35 | return wsgiref.simple_server.make_server( 36 | host, port, editor, 37 | IPv6EditorServer if ipv6 else EditorServer, 38 | EditorHandler) 39 | 40 | 41 | def text_error_serializer(req, resp, exc): 42 | resp.body = exc.title 43 | 44 | 45 | class EditorServer(socketserver.ThreadingMixIn, 46 | wsgiref.simple_server.WSGIServer): 47 | 48 | address_family = socket.AF_INET 49 | allow_reuse_address = True 50 | daemon_threads = True 51 | 52 | def handle_error(self, request, client_address): 53 | # Do not print tracebacks. 54 | pass 55 | 56 | 57 | class IPv6EditorServer(EditorServer): 58 | 59 | address_family = socket.AF_INET6 60 | 61 | 62 | class EditorHandler(wsgiref.simple_server.WSGIRequestHandler): 63 | 64 | def log_message(self, *args): # Do not log requests and responses. 65 | pass 66 | 67 | 68 | class EditorResource: 69 | 70 | realm = 'Turq editor' 71 | template = string.Template( 72 | pkgutil.get_data('turq', 'editor/editor.html.tpl').decode('utf-8')) 73 | 74 | def __init__(self, mock_server, password): 75 | self.mock_server = mock_server 76 | self.password = password 77 | self.nonce = self.new_nonce() 78 | self._lock = threading.Lock() 79 | 80 | def on_get(self, req, resp): 81 | self.check_auth(req) 82 | resp.content_type = 'text/html; charset=utf-8' 83 | (mock_host, mock_port, *_) = self.mock_server.server_address 84 | resp.body = self.template.substitute( 85 | mock_host=html.escape(mock_host), mock_port=mock_port, 86 | mock_url=html.escape(guess_external_url(mock_host, mock_port)), 87 | rules=html.escape(self.mock_server.rules), 88 | examples=turq.examples.load_html(initial_header_level=3)) 89 | 90 | def on_post(self, req, resp): 91 | self.check_auth(req) 92 | # Need `werkzeug.formparser` because JavaScript sends ``FormData``, 93 | # which is encoded as multipart. 94 | (_, form, _) = werkzeug.formparser.parse_form_data(req.env) 95 | if 'rules' not in form: 96 | raise falcon.HTTPBadRequest('Bad form') 97 | try: 98 | self.mock_server.install_rules(form['rules']) 99 | except SyntaxError as exc: 100 | resp.status = falcon.HTTP_422 # Unprocessable Entity 101 | resp.body = str(exc) 102 | else: 103 | resp.status = falcon.HTTP_303 # See Other 104 | resp.location = '/editor' 105 | resp.body = 'Rules installed successfully.' 106 | 107 | # We use HTTP digest authentication here, which provides a fairly high 108 | # level of protection. We use only one-time nonces, so replay attacks 109 | # should not be possible. An active man-in-the-middle could still intercept 110 | # a request and substitute their own rules; the ``auth-int`` option 111 | # is supposed to protect against that, but Chrome and Firefox (at least) 112 | # don't seem to support it. 113 | 114 | def check_auth(self, req): 115 | if not self.password: 116 | return 117 | auth = werkzeug.http.parse_authorization_header(req.auth) 118 | password_ok = False 119 | if self.check_password(req, auth): 120 | password_ok = True 121 | with self._lock: 122 | if auth.nonce == self.nonce: 123 | self.nonce = self.new_nonce() 124 | return 125 | raise falcon.HTTPUnauthorized(headers={ 126 | 'WWW-Authenticate': 127 | 'Digest realm="%s", qop="auth", charset=UTF-8, ' 128 | 'nonce="%s", stale=%s' % 129 | (self.realm, self.nonce, 'true' if password_ok else 'false')}) 130 | 131 | def check_password(self, req, auth): 132 | if not auth: 133 | return False 134 | a1 = '%s:%s:%s' % (auth.username, self.realm, self.password) 135 | a2 = '%s:%s' % (req.method, auth.uri) 136 | response = self.h('%s:%s:%s:%s:%s:%s' % (self.h(a1), 137 | auth.nonce, auth.nc, 138 | auth.cnonce, auth.qop, 139 | self.h(a2))) 140 | return auth.response == response 141 | 142 | @staticmethod 143 | def h(s): # pylint: disable=invalid-name 144 | return hashlib.md5(s.encode('utf-8')).hexdigest().lower() 145 | 146 | @staticmethod 147 | def new_nonce(): 148 | return base64.b64encode(os.urandom(18)).decode() 149 | 150 | 151 | class RedirectResource: 152 | 153 | def on_get(self, req, resp): 154 | raise falcon.HTTPFound('/editor') 155 | 156 | on_post = on_get 157 | 158 | 159 | def static_file(req, resp): 160 | path = '/' + req.path[len(STATIC_PREFIX):] 161 | path = posixpath.normpath(path.replace('\\', '/')) # Avoid path traversal 162 | try: 163 | resp.data = pkgutil.get_data('turq', 'editor%s' % path) 164 | except FileNotFoundError: 165 | raise falcon.HTTPNotFound() 166 | else: 167 | (resp.content_type, _) = mimetypes.guess_type(path) 168 | 169 | 170 | class CommonHeaders: 171 | 172 | def process_response(self, req, resp, resource, req_succeeded): 173 | # This server is very volatile: who knows what will be listening 174 | # on this host and port tomorrow? So, disable caching completely. 175 | # We don't want Chrome to "Show saved copy" when Turq is down, etc. 176 | resp.cache_control = ['no-store'] 177 | 178 | # For some reason, under some circumstances, Internet Explorer 11 179 | # falls back to IE 7 compatibility mode on the Turq editor. 180 | resp.append_header('X-UA-Compatible', 'IE=edge') 181 | -------------------------------------------------------------------------------- /turq/examples.rst: -------------------------------------------------------------------------------- 1 | Basics 2 | ------ 3 | 4 | :: 5 | 6 | # This is a comment. Normal Python syntax. 7 | if path == '/hello': 8 | header('Content-Type', 'text/plain') 9 | body('Hello world!\r\n') 10 | else: 11 | error(404) 12 | 13 | 14 | “RESTful” routing 15 | ----------------- 16 | 17 | :: 18 | 19 | if route('/v1/products/:product_id'): 20 | if GET or HEAD: 21 | json({'id': int(product_id), 22 | 'inStock': True}) 23 | elif PUT: 24 | # Pretend that we saved it 25 | json(request.json) 26 | elif DELETE: 27 | status(204) # No Content 28 | 29 | 30 | HTML pages 31 | ---------- 32 | 33 | To get a simple page:: 34 | 35 | html() 36 | 37 | If you want to change the contents of the page, the full `Dominate`_ library 38 | is at your service (``dominate.tags`` imported as ``H``):: 39 | 40 | with html(): 41 | H.h1('Welcome to our site') 42 | H.p('Have a look at our ', 43 | H.a('products', href='/products')) 44 | 45 | To change the ````:: 46 | 47 | with html() as document: 48 | with document.head: 49 | H.style('h1 {color: red}') 50 | H.h1('Welcome to our site') 51 | 52 | .. _Dominate: https://github.com/Knio/dominate 53 | 54 | 55 | Request details 56 | --------------- 57 | 58 | :: 59 | 60 | if request.json: # parsed JSON body 61 | name = request.json['name'] 62 | elif request.form: # URL-encoded or multipart 63 | name = request.form['name'] 64 | elif query: # query string parameters 65 | name = query['name'] 66 | else: 67 | raw_name = request.body # raw bytes 68 | name = raw_name.decode('utf-8') 69 | 70 | # Header names are case-insensitive 71 | if 'json' in request.headers['Accept']: 72 | json({'hello': name}) 73 | else: 74 | text('Hello %s!\r\n' % name) 75 | 76 | 77 | Response headers 78 | ---------------- 79 | 80 | ``header()`` *replaces* the given header, so this will send 81 | **only** ``max-age``:: 82 | 83 | header('Cache-Control', 'public') 84 | header('Cache-Control', 'max-age=3600') 85 | 86 | To *add* a header instead:: 87 | 88 | add_header('Set-Cookie', 'sessionid=123456') 89 | add_header('Set-Cookie', '__adtrack=abcdef') 90 | 91 | 92 | Custom status code and reason 93 | ----------------------------- 94 | 95 | :: 96 | 97 | status(567, 'Server Fell Over') 98 | text('Server crashed, sorry!\r\n') 99 | 100 | 101 | Redirection 102 | ----------- 103 | 104 | :: 105 | 106 | if path == '/': 107 | redirect('/index.html') 108 | elif path == '/index.html': 109 | html() 110 | 111 | ``redirect()`` sends `302 (Found)`_ by default, but you can override:: 112 | 113 | redirect('/index.html', 307) 114 | 115 | .. _302 (Found): https://tools.ietf.org/html/rfc7231#section-6.4.3 116 | 117 | 118 | Authentication 119 | -------------- 120 | 121 | To demand `basic`_ authentication:: 122 | 123 | basic_auth() 124 | with html(): 125 | H.h1('Super-secret page!') 126 | 127 | This sends `401 (Unauthorized)`_ unless the request had ``Authorization`` 128 | with the ``Basic`` scheme (credentials are ignored). 129 | 130 | Similarly for `digest`_:: 131 | 132 | digest_auth() 133 | 134 | And for `bearer`_:: 135 | 136 | bearer_auth() 137 | 138 | .. _basic: https://en.wikipedia.org/wiki/Basic_access_authentication 139 | .. _401 (Unauthorized): https://tools.ietf.org/html/rfc7235#section-3.1 140 | .. _digest: https://en.wikipedia.org/wiki/Digest_access_authentication 141 | .. _bearer: https://tools.ietf.org/html/rfc6750 142 | 143 | 144 | Body from file 145 | -------------- 146 | 147 | :: 148 | 149 | text(open('/etc/services')) 150 | 151 | 152 | Inspecting requests 153 | ------------------- 154 | 155 | To see what the client sends, including headers (but not the raw body), 156 | put ``debug()`` somewhere early in your rules:: 157 | 158 | debug() 159 | 160 | and watch the console output. Alternatively, for even more diagnostics, 161 | run Turq with the ``--verbose`` option. 162 | 163 | Or `use mitmproxy`_. 164 | 165 | .. _use mitmproxy: 166 | http://turq.readthedocs.io/en/stable/guide.html#using-mitmproxy-with-turq 167 | 168 | 169 | Forwarding requests 170 | ------------------- 171 | 172 | Turq can act as a gateway or “reverse proxy”:: 173 | 174 | forward('httpbin.org', 80, # host, port 175 | target) # path + query string 176 | # At this point, response from httpbin.org:80 177 | # has been copied to Turq, and can be tweaked: 178 | delete_header('Server') 179 | add_header('Cache-Control', 'max-age=86400') 180 | 181 | Turq uses TLS when connecting to port 443, but **ignores certificates**. 182 | You can override TLS like this:: 183 | 184 | forward('develop1.example', 8765, 185 | '/v1/articles', tls=True) 186 | 187 | 188 | Cross-origin resource sharing 189 | ----------------------------- 190 | 191 | ``cors()`` adds the right ``Access-Control-*`` headers, and handles 192 | preflight requests automatically:: 193 | 194 | cors() 195 | json({'some': 'data'}) 196 | 197 | For legacy systems, JSONP is also supported, reacting automatically 198 | to a ``callback`` query parameter:: 199 | 200 | json({'some': 'data'}, jsonp=True) 201 | 202 | 203 | Compression 204 | ----------- 205 | 206 | Call ``gzip()`` after setting the body:: 207 | 208 | with html(): 209 | # 100 paragraphs of text 210 | for i in range(100): 211 | H.p(lorem_ipsum()) 212 | gzip() 213 | 214 | 215 | Random responses 216 | ---------------- 217 | 218 | :: 219 | 220 | if maybe(0.1): # 10% probability 221 | error(503) 222 | else: 223 | html() 224 | 225 | 226 | Response framing 227 | ---------------- 228 | 229 | By default, if the client supports it, Turq uses ``Transfer-Encoding: chunked`` 230 | and keeps the connection alive. 231 | 232 | To use ``Content-Length`` instead of ``Transfer-Encoding``, 233 | call ``content_length()`` after you’ve set the body:: 234 | 235 | text('Hello world!\r\n') 236 | content_length() 237 | 238 | To close the connection after sending the response:: 239 | 240 | add_header('Connection', 'close') 241 | text('Hello world!\r\n') 242 | 243 | 244 | Streaming responses 245 | ------------------- 246 | 247 | :: 248 | 249 | header('Content-Type', 'text/event-stream') 250 | sleep(1) # 1 second delay 251 | chunk('data: my event 1\r\n\r\n') 252 | sleep(1) 253 | chunk('data: my event 2\r\n\r\n') 254 | sleep(1) 255 | chunk('data: my event 3\r\n\r\n') 256 | 257 | Once you call ``chunk()``, the response begins streaming. 258 | Any headers you set after that will be sent in the `trailer part`_:: 259 | 260 | header('Content-Type', 'text/plain') 261 | header('Trailer', 'Content-MD5') 262 | chunk('Hello, ') 263 | chunk('world!\n') 264 | header('Content-MD5', '746308829575e17c3331bbcb00c0898b') 265 | 266 | .. _trailer part: https://tools.ietf.org/html/rfc7230#section-4.1.2 267 | 268 | 269 | Handling ``Expect: 100-continue`` 270 | --------------------------------- 271 | 272 | :: 273 | 274 | with interim(): 275 | status(100) 276 | 277 | text('Resource updated OK') 278 | 279 | In the above example, `100 (Continue)`_ is sent immediately after the 280 | ``interim()`` block, but the final 200 (OK) response is sent only after 281 | reading the full request body. 282 | 283 | If instead you want to send a response *before* reading the request body:: 284 | 285 | error(403) # Forbidden 286 | flush() 287 | 288 | .. _100 (Continue): https://tools.ietf.org/html/rfc7231#section-6.2.1 289 | 290 | 291 | Custom methods 292 | -------------- 293 | 294 | :: 295 | 296 | if method != 'FROBNICATE': 297 | error(405) # Method Not Allowed 298 | header('Allow', 'FROBNICATE') 299 | 300 | 301 | Switching protocols 302 | ------------------- 303 | 304 | :: 305 | 306 | if request.headers['Upgrade'] == 'QXTP': 307 | with interim(): 308 | status(101) # Switching Protocols 309 | header('Upgrade', 'QXTP') 310 | header('Connection', 'upgrade') 311 | send_raw('This is no longer HTTP!\r\n') 312 | send_raw('This is QXTP now!\r\n') 313 | 314 | 315 | Anything else 316 | ------------- 317 | 318 | In the end, Turq rules are just Python code that is not sandboxed, 319 | so you can import and use anything you like. For example, 320 | to send random binary data:: 321 | 322 | import os 323 | header('Content-Type', 'application/octet-stream') 324 | body(os.urandom(128)) 325 | -------------------------------------------------------------------------------- /tests/test_control.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=invalid-name 2 | 3 | import re 4 | import socket 5 | import time 6 | 7 | import pytest 8 | import requests 9 | from requests.auth import HTTPDigestAuth 10 | 11 | 12 | @pytest.mark.parametrize('extra_args', [[], ['--no-color']]) 13 | def test_output(turq_instance, extra_args): 14 | turq_instance.extra_args = extra_args 15 | with turq_instance: 16 | turq_instance.request('GET', '/foo/bar') 17 | output = turq_instance.console_output 18 | # pylint: disable=superfluous-parens 19 | assert ('mock on port %d' % turq_instance.mock_port) in output 20 | assert ('editor on port %d' % turq_instance.editor_port) in output 21 | assert 'new connection from' in output 22 | assert '> GET /foo/bar HTTP/1.1' in output 23 | assert '+ User-Agent: python-requests' not in output 24 | assert '< HTTP/1.1 404 Not Found' in output 25 | assert '+ Content-Type: text/plain; charset=utf-8' not in output 26 | 27 | 28 | def test_verbose_output(turq_instance): 29 | turq_instance.extra_args = ['--verbose'] 30 | with turq_instance: 31 | turq_instance.request('GET', '/foo/bar') 32 | output = turq_instance.console_output 33 | assert '+ User-Agent: python-requests' in output 34 | assert '+ Content-Type: text/plain; charset=utf-8' in output 35 | assert 'states:' in output 36 | 37 | 38 | def test_editor(turq_instance): 39 | with turq_instance: 40 | resp = turq_instance.request_editor('GET', '/editor') 41 | assert resp.headers['Cache-Control'] == 'no-store' 42 | assert resp.headers['X-UA-Compatible'] == 'IE=edge' 43 | assert 'port %d' % turq_instance.mock_port in resp.text 44 | assert '>error(404)\n' in resp.text 45 | assert 'with html() as document:' in resp.text # from examples 46 | resp = turq_instance.request_editor('POST', '/editor', 47 | data={'rules': 'html()'}) 48 | assert '>html()' in resp.text 49 | resp = turq_instance.request('GET', '/') 50 | assert '

Hello world!

' in resp.text 51 | # Initial rules (``error(404)``) + the ones we posted 52 | assert turq_instance.console_output.count('new rules installed') == 2 53 | 54 | 55 | def test_no_editor(turq_instance): 56 | turq_instance.extra_args = ['--no-editor'] 57 | turq_instance.wait = False 58 | with turq_instance, pytest.raises(requests.exceptions.ConnectionError): 59 | time.sleep(1) 60 | turq_instance.request_editor('GET', '/editor') 61 | 62 | 63 | def test_editor_bad_syntax(turq_instance): 64 | with turq_instance: 65 | resp = turq_instance.request_editor('POST', '/editor', 66 | data={'rules': 'html('}) 67 | assert resp.status_code == 422 68 | assert resp.headers['Content-Type'] == 'text/plain; charset=utf-8' 69 | assert resp.text == 'unexpected EOF while parsing (, line 1)' 70 | 71 | 72 | def test_editor_bad_form(turq_instance): 73 | with turq_instance: 74 | resp = turq_instance.request_editor('POST', '/editor', 75 | data={'foo': 'bar'}) 76 | assert resp.status_code == 400 77 | assert resp.text == 'Bad form' 78 | 79 | 80 | def test_editor_static(turq_instance): 81 | with turq_instance: 82 | resp = turq_instance.request_editor('GET', '/static/editor.css') 83 | assert resp.headers['Content-Type'] == 'text/css' 84 | assert 'font-size' in resp.text 85 | resp = turq_instance.request_editor( 86 | 'GET', '/static/codemirror/lib/codemirror.js') 87 | assert resp.headers['Content-Type'] == 'application/javascript' 88 | 89 | 90 | def test_debug_output(turq_instance): 91 | with turq_instance: 92 | turq_instance.request_editor('POST', '/editor', 93 | data={'rules': 'debug(); html()'}) 94 | turq_instance.request('GET', '/') 95 | assert '+ User-Agent: python-requests' in turq_instance.console_output 96 | assert '+ Content-Type: text/html' in turq_instance.console_output 97 | assert 'states:' not in turq_instance.console_output 98 | 99 | 100 | def test_uncaught_exception(turq_instance): 101 | turq_instance.host = 'wololo.invalid' 102 | turq_instance.wait = False 103 | with turq_instance: 104 | time.sleep(1) 105 | assert 'turq: error: ' in turq_instance.console_output 106 | 107 | 108 | def test_uncaught_exception_traceback(turq_instance): 109 | turq_instance.host = 'wololo.invalid' 110 | turq_instance.extra_args = ['-v'] 111 | turq_instance.wait = False 112 | with turq_instance: 113 | time.sleep(1) 114 | assert 'Traceback (most recent call last):' in turq_instance.console_output 115 | assert 'turq: error: ' not in turq_instance.console_output 116 | 117 | 118 | def test_editor_password(turq_instance): 119 | turq_instance.password = 'wololo' 120 | with turq_instance: 121 | resp = turq_instance.request_editor( 122 | 'POST', '/editor', data={'rules': 'html()'}, 123 | auth=HTTPDigestAuth('', 'foobar')) 124 | assert resp.status_code == 401 125 | assert 'Hello world!' not in turq_instance.request('GET', '/').text 126 | resp = turq_instance.request_editor( 127 | 'POST', '/editor', data={'rules': 'html()'}, 128 | auth=HTTPDigestAuth('', 'wololo')) 129 | assert resp.status_code == 200 130 | assert 'Hello world!' in turq_instance.request('GET', '/').text 131 | 132 | 133 | def test_editor_password_auto_generated(turq_instance): 134 | turq_instance.password = None 135 | with turq_instance: 136 | time.sleep(1) 137 | assert re.search(r'editor password: [A-Za-z0-9]{24}\b', 138 | turq_instance.console_output) 139 | 140 | 141 | def test_new_rules_affect_existing_connection(turq_instance): 142 | with turq_instance, turq_instance.connect() as sock: 143 | sock.sendall(b'GET / HTTP/1.1\r\n' 144 | b'Host: example\r\n' 145 | b'\r\n') 146 | while b'Error! Nothing matches the given URI' not in sock.recv(4096): 147 | pass 148 | # Connection to mock server is kept open. Meanwhile, we post new rules. 149 | turq_instance.request_editor('POST', '/editor', 150 | data={'rules': 'text("Hi there!")'}) 151 | sock.sendall(b'GET / HTTP/1.1\r\n' 152 | b'Host: example\r\n' 153 | b'\r\n') 154 | while b'Hi there!' not in sock.recv(4096): 155 | pass 156 | 157 | 158 | def test_exception_in_rules(turq_instance): 159 | with turq_instance: 160 | turq_instance.request_editor('POST', '/editor', 161 | data={'rules': 162 | 'def helper():\n' 163 | ' html()\n' 164 | ' oops()\n' 165 | 'debug()\n' 166 | 'helper()\n' 167 | 'gzip()\n'}) 168 | resp = turq_instance.request('GET', '/') 169 | assert resp.status_code == 500 170 | output = turq_instance.console_output 171 | assert "error in rules, line 3: name 'oops' is not defined" in output 172 | assert 'Traceback (most recent call last):' in output # because `debug` 173 | 174 | 175 | def test_abruptly_closed_request(turq_instance): 176 | with turq_instance, turq_instance.connect() as sock: 177 | sock.sendall(b'POST / HTTP/1.1\r\n' 178 | b'Host: example\r\n' 179 | b'Content-Length: 9001\r\n' 180 | b'\r\n') 181 | sock.shutdown(socket.SHUT_WR) 182 | assert b'HTTP/1.1 400 Bad Request' in sock.recv(4096) 183 | assert 'error:' in turq_instance.console_output 184 | 185 | 186 | def test_no_premature_connection_close(turq_instance): 187 | with turq_instance, turq_instance.connect() as sock: 188 | sock.sendall(b'POST / HTTP/1.1\r\n' 189 | b'Content-Length: 14\r\n' 190 | b'\r\n') 191 | # Already at this point, the server will have sent a 400 (Bad Request), 192 | # because the ``Host`` header is missing. But we're a slow client; 193 | # we keep writing to the server. 194 | time.sleep(1) 195 | sock.sendall(b'Hello world!\r\n') 196 | assert b'HTTP/1.1 400 Bad Request' in sock.recv(4096) 197 | 198 | 199 | @pytest.mark.parametrize('bad_path', [ 200 | br'../../../../../../../../../../../../../etc/services', 201 | br'..\..\..\..\..\..\..\..\..\..\..\..\..\pagefile.sys']) 202 | def test_editor_no_path_traversal(turq_instance, bad_path): 203 | with turq_instance, turq_instance.connect_editor() as sock: 204 | sock.sendall(b'GET /static/' + bad_path + b' HTTP/1.1\r\n' 205 | b'Host: example\r\n' 206 | b'\r\n') 207 | assert b'HTTP/1.0 404 Not Found\r\n' in sock.recv(4096) 208 | -------------------------------------------------------------------------------- /turq/editor/codemirror/lib/codemirror.css: -------------------------------------------------------------------------------- 1 | /* BASICS */ 2 | 3 | .CodeMirror { 4 | /* Set height, width, borders, and global font properties here */ 5 | font-family: monospace; 6 | height: 300px; 7 | color: black; 8 | } 9 | 10 | /* PADDING */ 11 | 12 | .CodeMirror-lines { 13 | padding: 4px 0; /* Vertical padding around content */ 14 | } 15 | .CodeMirror pre { 16 | padding: 0 4px; /* Horizontal padding of content */ 17 | } 18 | 19 | .CodeMirror-scrollbar-filler, .CodeMirror-gutter-filler { 20 | background-color: white; /* The little square between H and V scrollbars */ 21 | } 22 | 23 | /* GUTTER */ 24 | 25 | .CodeMirror-gutters { 26 | border-right: 1px solid #ddd; 27 | background-color: #f7f7f7; 28 | white-space: nowrap; 29 | } 30 | .CodeMirror-linenumbers {} 31 | .CodeMirror-linenumber { 32 | padding: 0 3px 0 5px; 33 | min-width: 20px; 34 | text-align: right; 35 | color: #999; 36 | white-space: nowrap; 37 | } 38 | 39 | .CodeMirror-guttermarker { color: black; } 40 | .CodeMirror-guttermarker-subtle { color: #999; } 41 | 42 | /* CURSOR */ 43 | 44 | .CodeMirror-cursor { 45 | border-left: 1px solid black; 46 | border-right: none; 47 | width: 0; 48 | } 49 | /* Shown when moving in bi-directional text */ 50 | .CodeMirror div.CodeMirror-secondarycursor { 51 | border-left: 1px solid silver; 52 | } 53 | .cm-fat-cursor .CodeMirror-cursor { 54 | width: auto; 55 | border: 0 !important; 56 | background: #7e7; 57 | } 58 | .cm-fat-cursor div.CodeMirror-cursors { 59 | z-index: 1; 60 | } 61 | 62 | .cm-animate-fat-cursor { 63 | width: auto; 64 | border: 0; 65 | -webkit-animation: blink 1.06s steps(1) infinite; 66 | -moz-animation: blink 1.06s steps(1) infinite; 67 | animation: blink 1.06s steps(1) infinite; 68 | background-color: #7e7; 69 | } 70 | @-moz-keyframes blink { 71 | 0% {} 72 | 50% { background-color: transparent; } 73 | 100% {} 74 | } 75 | @-webkit-keyframes blink { 76 | 0% {} 77 | 50% { background-color: transparent; } 78 | 100% {} 79 | } 80 | @keyframes blink { 81 | 0% {} 82 | 50% { background-color: transparent; } 83 | 100% {} 84 | } 85 | 86 | /* Can style cursor different in overwrite (non-insert) mode */ 87 | .CodeMirror-overwrite .CodeMirror-cursor {} 88 | 89 | .cm-tab { display: inline-block; text-decoration: inherit; } 90 | 91 | .CodeMirror-rulers { 92 | position: absolute; 93 | left: 0; right: 0; top: -50px; bottom: -20px; 94 | overflow: hidden; 95 | } 96 | .CodeMirror-ruler { 97 | border-left: 1px solid #ccc; 98 | top: 0; bottom: 0; 99 | position: absolute; 100 | } 101 | 102 | /* DEFAULT THEME */ 103 | 104 | .cm-s-default .cm-header {color: blue;} 105 | .cm-s-default .cm-quote {color: #090;} 106 | .cm-negative {color: #d44;} 107 | .cm-positive {color: #292;} 108 | .cm-header, .cm-strong {font-weight: bold;} 109 | .cm-em {font-style: italic;} 110 | .cm-link {text-decoration: underline;} 111 | .cm-strikethrough {text-decoration: line-through;} 112 | 113 | .cm-s-default .cm-keyword {color: #708;} 114 | .cm-s-default .cm-atom {color: #219;} 115 | .cm-s-default .cm-number {color: #164;} 116 | .cm-s-default .cm-def {color: #00f;} 117 | .cm-s-default .cm-variable, 118 | .cm-s-default .cm-punctuation, 119 | .cm-s-default .cm-property, 120 | .cm-s-default .cm-operator {} 121 | .cm-s-default .cm-variable-2 {color: #05a;} 122 | .cm-s-default .cm-variable-3 {color: #085;} 123 | .cm-s-default .cm-comment {color: #a50;} 124 | .cm-s-default .cm-string {color: #a11;} 125 | .cm-s-default .cm-string-2 {color: #f50;} 126 | .cm-s-default .cm-meta {color: #555;} 127 | .cm-s-default .cm-qualifier {color: #555;} 128 | .cm-s-default .cm-builtin {color: #30a;} 129 | .cm-s-default .cm-bracket {color: #997;} 130 | .cm-s-default .cm-tag {color: #170;} 131 | .cm-s-default .cm-attribute {color: #00c;} 132 | .cm-s-default .cm-hr {color: #999;} 133 | .cm-s-default .cm-link {color: #00c;} 134 | 135 | .cm-s-default .cm-error {color: #f00;} 136 | .cm-invalidchar {color: #f00;} 137 | 138 | .CodeMirror-composing { border-bottom: 2px solid; } 139 | 140 | /* Default styles for common addons */ 141 | 142 | div.CodeMirror span.CodeMirror-matchingbracket {color: #0f0;} 143 | div.CodeMirror span.CodeMirror-nonmatchingbracket {color: #f22;} 144 | .CodeMirror-matchingtag { background: rgba(255, 150, 0, .3); } 145 | .CodeMirror-activeline-background {background: #e8f2ff;} 146 | 147 | /* STOP */ 148 | 149 | /* The rest of this file contains styles related to the mechanics of 150 | the editor. You probably shouldn't touch them. */ 151 | 152 | .CodeMirror { 153 | position: relative; 154 | overflow: hidden; 155 | background: white; 156 | } 157 | 158 | .CodeMirror-scroll { 159 | overflow: scroll !important; /* Things will break if this is overridden */ 160 | /* 30px is the magic margin used to hide the element's real scrollbars */ 161 | /* See overflow: hidden in .CodeMirror */ 162 | margin-bottom: -30px; margin-right: -30px; 163 | padding-bottom: 30px; 164 | height: 100%; 165 | outline: none; /* Prevent dragging from highlighting the element */ 166 | position: relative; 167 | } 168 | .CodeMirror-sizer { 169 | position: relative; 170 | border-right: 30px solid transparent; 171 | } 172 | 173 | /* The fake, visible scrollbars. Used to force redraw during scrolling 174 | before actual scrolling happens, thus preventing shaking and 175 | flickering artifacts. */ 176 | .CodeMirror-vscrollbar, .CodeMirror-hscrollbar, .CodeMirror-scrollbar-filler, .CodeMirror-gutter-filler { 177 | position: absolute; 178 | z-index: 6; 179 | display: none; 180 | } 181 | .CodeMirror-vscrollbar { 182 | right: 0; top: 0; 183 | overflow-x: hidden; 184 | overflow-y: scroll; 185 | } 186 | .CodeMirror-hscrollbar { 187 | bottom: 0; left: 0; 188 | overflow-y: hidden; 189 | overflow-x: scroll; 190 | } 191 | .CodeMirror-scrollbar-filler { 192 | right: 0; bottom: 0; 193 | } 194 | .CodeMirror-gutter-filler { 195 | left: 0; bottom: 0; 196 | } 197 | 198 | .CodeMirror-gutters { 199 | position: absolute; left: 0; top: 0; 200 | min-height: 100%; 201 | z-index: 3; 202 | } 203 | .CodeMirror-gutter { 204 | white-space: normal; 205 | height: 100%; 206 | display: inline-block; 207 | vertical-align: top; 208 | margin-bottom: -30px; 209 | } 210 | .CodeMirror-gutter-wrapper { 211 | position: absolute; 212 | z-index: 4; 213 | background: none !important; 214 | border: none !important; 215 | } 216 | .CodeMirror-gutter-background { 217 | position: absolute; 218 | top: 0; bottom: 0; 219 | z-index: 4; 220 | } 221 | .CodeMirror-gutter-elt { 222 | position: absolute; 223 | cursor: default; 224 | z-index: 4; 225 | } 226 | .CodeMirror-gutter-wrapper ::selection { background-color: transparent } 227 | .CodeMirror-gutter-wrapper ::-moz-selection { background-color: transparent } 228 | 229 | .CodeMirror-lines { 230 | cursor: text; 231 | min-height: 1px; /* prevents collapsing before first draw */ 232 | } 233 | .CodeMirror pre { 234 | /* Reset some styles that the rest of the page might have set */ 235 | -moz-border-radius: 0; -webkit-border-radius: 0; border-radius: 0; 236 | border-width: 0; 237 | background: transparent; 238 | font-family: inherit; 239 | font-size: inherit; 240 | margin: 0; 241 | white-space: pre; 242 | word-wrap: normal; 243 | line-height: inherit; 244 | color: inherit; 245 | z-index: 2; 246 | position: relative; 247 | overflow: visible; 248 | -webkit-tap-highlight-color: transparent; 249 | -webkit-font-variant-ligatures: contextual; 250 | font-variant-ligatures: contextual; 251 | } 252 | .CodeMirror-wrap pre { 253 | word-wrap: break-word; 254 | white-space: pre-wrap; 255 | word-break: normal; 256 | } 257 | 258 | .CodeMirror-linebackground { 259 | position: absolute; 260 | left: 0; right: 0; top: 0; bottom: 0; 261 | z-index: 0; 262 | } 263 | 264 | .CodeMirror-linewidget { 265 | position: relative; 266 | z-index: 2; 267 | overflow: auto; 268 | } 269 | 270 | .CodeMirror-widget {} 271 | 272 | .CodeMirror-rtl pre { direction: rtl; } 273 | 274 | .CodeMirror-code { 275 | outline: none; 276 | } 277 | 278 | /* Force content-box sizing for the elements where we expect it */ 279 | .CodeMirror-scroll, 280 | .CodeMirror-sizer, 281 | .CodeMirror-gutter, 282 | .CodeMirror-gutters, 283 | .CodeMirror-linenumber { 284 | -moz-box-sizing: content-box; 285 | box-sizing: content-box; 286 | } 287 | 288 | .CodeMirror-measure { 289 | position: absolute; 290 | width: 100%; 291 | height: 0; 292 | overflow: hidden; 293 | visibility: hidden; 294 | } 295 | 296 | .CodeMirror-cursor { 297 | position: absolute; 298 | pointer-events: none; 299 | } 300 | .CodeMirror-measure pre { position: static; } 301 | 302 | div.CodeMirror-cursors { 303 | visibility: hidden; 304 | position: relative; 305 | z-index: 3; 306 | } 307 | div.CodeMirror-dragcursors { 308 | visibility: visible; 309 | } 310 | 311 | .CodeMirror-focused div.CodeMirror-cursors { 312 | visibility: visible; 313 | } 314 | 315 | .CodeMirror-selected { background: #d9d9d9; } 316 | .CodeMirror-focused .CodeMirror-selected { background: #d7d4f0; } 317 | .CodeMirror-crosshair { cursor: crosshair; } 318 | .CodeMirror-line::selection, .CodeMirror-line > span::selection, .CodeMirror-line > span > span::selection { background: #d7d4f0; } 319 | .CodeMirror-line::-moz-selection, .CodeMirror-line > span::-moz-selection, .CodeMirror-line > span > span::-moz-selection { background: #d7d4f0; } 320 | 321 | .cm-searching { 322 | background: #ffa; 323 | background: rgba(255, 255, 0, .4); 324 | } 325 | 326 | /* Used to force a border model for a node */ 327 | .cm-force-border { padding-right: .1px; } 328 | 329 | @media print { 330 | /* Hide the cursor when printing */ 331 | .CodeMirror div.CodeMirror-cursors { 332 | visibility: hidden; 333 | } 334 | } 335 | 336 | /* See issue #2901 */ 337 | .cm-tab-wrap-hack:after { content: ''; } 338 | 339 | /* Help users use markselection to safely style text background */ 340 | span.CodeMirror-selectedtext { background: none; } 341 | -------------------------------------------------------------------------------- /turq/editor/codemirror/mode/python/python.js: -------------------------------------------------------------------------------- 1 | // CodeMirror, copyright (c) by Marijn Haverbeke and others 2 | // Distributed under an MIT license: http://codemirror.net/LICENSE 3 | 4 | (function(mod) { 5 | if (typeof exports == "object" && typeof module == "object") // CommonJS 6 | mod(require("../../lib/codemirror")); 7 | else if (typeof define == "function" && define.amd) // AMD 8 | define(["../../lib/codemirror"], mod); 9 | else // Plain browser env 10 | mod(CodeMirror); 11 | })(function(CodeMirror) { 12 | "use strict"; 13 | 14 | function wordRegexp(words) { 15 | return new RegExp("^((" + words.join(")|(") + "))\\b"); 16 | } 17 | 18 | var wordOperators = wordRegexp(["and", "or", "not", "is"]); 19 | var commonKeywords = ["as", "assert", "break", "class", "continue", 20 | "def", "del", "elif", "else", "except", "finally", 21 | "for", "from", "global", "if", "import", 22 | "lambda", "pass", "raise", "return", 23 | "try", "while", "with", "yield", "in"]; 24 | var commonBuiltins = ["abs", "all", "any", "bin", "bool", "bytearray", "callable", "chr", 25 | "classmethod", "compile", "complex", "delattr", "dict", "dir", "divmod", 26 | "enumerate", "eval", "filter", "float", "format", "frozenset", 27 | "getattr", "globals", "hasattr", "hash", "help", "hex", "id", 28 | "input", "int", "isinstance", "issubclass", "iter", "len", 29 | "list", "locals", "map", "max", "memoryview", "min", "next", 30 | "object", "oct", "open", "ord", "pow", "property", "range", 31 | "repr", "reversed", "round", "set", "setattr", "slice", 32 | "sorted", "staticmethod", "str", "sum", "super", "tuple", 33 | "type", "vars", "zip", "__import__", "NotImplemented", 34 | "Ellipsis", "__debug__"]; 35 | CodeMirror.registerHelper("hintWords", "python", commonKeywords.concat(commonBuiltins)); 36 | 37 | function top(state) { 38 | return state.scopes[state.scopes.length - 1]; 39 | } 40 | 41 | CodeMirror.defineMode("python", function(conf, parserConf) { 42 | var ERRORCLASS = "error"; 43 | 44 | var singleDelimiters = parserConf.singleDelimiters || /^[\(\)\[\]\{\}@,:`=;\.]/; 45 | var doubleOperators = parserConf.doubleOperators || /^([!<>]==|<>|<<|>>|\/\/|\*\*)/; 46 | var doubleDelimiters = parserConf.doubleDelimiters || /^(\+=|\-=|\*=|%=|\/=|&=|\|=|\^=)/; 47 | var tripleDelimiters = parserConf.tripleDelimiters || /^(\/\/=|>>=|<<=|\*\*=)/; 48 | 49 | var hangingIndent = parserConf.hangingIndent || conf.indentUnit; 50 | 51 | var myKeywords = commonKeywords, myBuiltins = commonBuiltins; 52 | if (parserConf.extra_keywords != undefined) 53 | myKeywords = myKeywords.concat(parserConf.extra_keywords); 54 | 55 | if (parserConf.extra_builtins != undefined) 56 | myBuiltins = myBuiltins.concat(parserConf.extra_builtins); 57 | 58 | var py3 = !(parserConf.version && Number(parserConf.version) < 3) 59 | if (py3) { 60 | // since http://legacy.python.org/dev/peps/pep-0465/ @ is also an operator 61 | var singleOperators = parserConf.singleOperators || /^[\+\-\*\/%&|\^~<>!@]/; 62 | var identifiers = parserConf.identifiers|| /^[_A-Za-z\u00A1-\uFFFF][_A-Za-z0-9\u00A1-\uFFFF]*/; 63 | myKeywords = myKeywords.concat(["nonlocal", "False", "True", "None", "async", "await"]); 64 | myBuiltins = myBuiltins.concat(["ascii", "bytes", "exec", "print"]); 65 | var stringPrefixes = new RegExp("^(([rbuf]|(br))?('{3}|\"{3}|['\"]))", "i"); 66 | } else { 67 | var singleOperators = parserConf.singleOperators || /^[\+\-\*\/%&|\^~<>!]/; 68 | var identifiers = parserConf.identifiers|| /^[_A-Za-z][_A-Za-z0-9]*/; 69 | myKeywords = myKeywords.concat(["exec", "print"]); 70 | myBuiltins = myBuiltins.concat(["apply", "basestring", "buffer", "cmp", "coerce", "execfile", 71 | "file", "intern", "long", "raw_input", "reduce", "reload", 72 | "unichr", "unicode", "xrange", "False", "True", "None"]); 73 | var stringPrefixes = new RegExp("^(([rubf]|(ur)|(br))?('{3}|\"{3}|['\"]))", "i"); 74 | } 75 | var keywords = wordRegexp(myKeywords); 76 | var builtins = wordRegexp(myBuiltins); 77 | 78 | // tokenizers 79 | function tokenBase(stream, state) { 80 | if (stream.sol()) state.indent = stream.indentation() 81 | // Handle scope changes 82 | if (stream.sol() && top(state).type == "py") { 83 | var scopeOffset = top(state).offset; 84 | if (stream.eatSpace()) { 85 | var lineOffset = stream.indentation(); 86 | if (lineOffset > scopeOffset) 87 | pushPyScope(state); 88 | else if (lineOffset < scopeOffset && dedent(stream, state) && stream.peek() != "#") 89 | state.errorToken = true; 90 | return null; 91 | } else { 92 | var style = tokenBaseInner(stream, state); 93 | if (scopeOffset > 0 && dedent(stream, state)) 94 | style += " " + ERRORCLASS; 95 | return style; 96 | } 97 | } 98 | return tokenBaseInner(stream, state); 99 | } 100 | 101 | function tokenBaseInner(stream, state) { 102 | if (stream.eatSpace()) return null; 103 | 104 | var ch = stream.peek(); 105 | 106 | // Handle Comments 107 | if (ch == "#") { 108 | stream.skipToEnd(); 109 | return "comment"; 110 | } 111 | 112 | // Handle Number Literals 113 | if (stream.match(/^[0-9\.]/, false)) { 114 | var floatLiteral = false; 115 | // Floats 116 | if (stream.match(/^[\d_]*\.\d+(e[\+\-]?\d+)?/i)) { floatLiteral = true; } 117 | if (stream.match(/^[\d_]+\.\d*/)) { floatLiteral = true; } 118 | if (stream.match(/^\.\d+/)) { floatLiteral = true; } 119 | if (floatLiteral) { 120 | // Float literals may be "imaginary" 121 | stream.eat(/J/i); 122 | return "number"; 123 | } 124 | // Integers 125 | var intLiteral = false; 126 | // Hex 127 | if (stream.match(/^0x[0-9a-f_]+/i)) intLiteral = true; 128 | // Binary 129 | if (stream.match(/^0b[01_]+/i)) intLiteral = true; 130 | // Octal 131 | if (stream.match(/^0o[0-7_]+/i)) intLiteral = true; 132 | // Decimal 133 | if (stream.match(/^[1-9][\d_]*(e[\+\-]?[\d_]+)?/)) { 134 | // Decimal literals may be "imaginary" 135 | stream.eat(/J/i); 136 | // TODO - Can you have imaginary longs? 137 | intLiteral = true; 138 | } 139 | // Zero by itself with no other piece of number. 140 | if (stream.match(/^0(?![\dx])/i)) intLiteral = true; 141 | if (intLiteral) { 142 | // Integer literals may be "long" 143 | stream.eat(/L/i); 144 | return "number"; 145 | } 146 | } 147 | 148 | // Handle Strings 149 | if (stream.match(stringPrefixes)) { 150 | state.tokenize = tokenStringFactory(stream.current()); 151 | return state.tokenize(stream, state); 152 | } 153 | 154 | // Handle operators and Delimiters 155 | if (stream.match(tripleDelimiters) || stream.match(doubleDelimiters)) 156 | return "punctuation"; 157 | 158 | if (stream.match(doubleOperators) || stream.match(singleOperators)) 159 | return "operator"; 160 | 161 | if (stream.match(singleDelimiters)) 162 | return "punctuation"; 163 | 164 | if (state.lastToken == "." && stream.match(identifiers)) 165 | return "property"; 166 | 167 | if (stream.match(keywords) || stream.match(wordOperators)) 168 | return "keyword"; 169 | 170 | if (stream.match(builtins)) 171 | return "builtin"; 172 | 173 | if (stream.match(/^(self|cls)\b/)) 174 | return "variable-2"; 175 | 176 | if (stream.match(identifiers)) { 177 | if (state.lastToken == "def" || state.lastToken == "class") 178 | return "def"; 179 | return "variable"; 180 | } 181 | 182 | // Handle non-detected items 183 | stream.next(); 184 | return ERRORCLASS; 185 | } 186 | 187 | function tokenStringFactory(delimiter) { 188 | while ("rubf".indexOf(delimiter.charAt(0).toLowerCase()) >= 0) 189 | delimiter = delimiter.substr(1); 190 | 191 | var singleline = delimiter.length == 1; 192 | var OUTCLASS = "string"; 193 | 194 | function tokenString(stream, state) { 195 | while (!stream.eol()) { 196 | stream.eatWhile(/[^'"\\]/); 197 | if (stream.eat("\\")) { 198 | stream.next(); 199 | if (singleline && stream.eol()) 200 | return OUTCLASS; 201 | } else if (stream.match(delimiter)) { 202 | state.tokenize = tokenBase; 203 | return OUTCLASS; 204 | } else { 205 | stream.eat(/['"]/); 206 | } 207 | } 208 | if (singleline) { 209 | if (parserConf.singleLineStringErrors) 210 | return ERRORCLASS; 211 | else 212 | state.tokenize = tokenBase; 213 | } 214 | return OUTCLASS; 215 | } 216 | tokenString.isString = true; 217 | return tokenString; 218 | } 219 | 220 | function pushPyScope(state) { 221 | while (top(state).type != "py") state.scopes.pop() 222 | state.scopes.push({offset: top(state).offset + conf.indentUnit, 223 | type: "py", 224 | align: null}) 225 | } 226 | 227 | function pushBracketScope(stream, state, type) { 228 | var align = stream.match(/^([\s\[\{\(]|#.*)*$/, false) ? null : stream.column() + 1 229 | state.scopes.push({offset: state.indent + hangingIndent, 230 | type: type, 231 | align: align}) 232 | } 233 | 234 | function dedent(stream, state) { 235 | var indented = stream.indentation(); 236 | while (state.scopes.length > 1 && top(state).offset > indented) { 237 | if (top(state).type != "py") return true; 238 | state.scopes.pop(); 239 | } 240 | return top(state).offset != indented; 241 | } 242 | 243 | function tokenLexer(stream, state) { 244 | if (stream.sol()) state.beginningOfLine = true; 245 | 246 | var style = state.tokenize(stream, state); 247 | var current = stream.current(); 248 | 249 | // Handle decorators 250 | if (state.beginningOfLine && current == "@") 251 | return stream.match(identifiers, false) ? "meta" : py3 ? "operator" : ERRORCLASS; 252 | 253 | if (/\S/.test(current)) state.beginningOfLine = false; 254 | 255 | if ((style == "variable" || style == "builtin") 256 | && state.lastToken == "meta") 257 | style = "meta"; 258 | 259 | // Handle scope changes. 260 | if (current == "pass" || current == "return") 261 | state.dedent += 1; 262 | 263 | if (current == "lambda") state.lambda = true; 264 | if (current == ":" && !state.lambda && top(state).type == "py") 265 | pushPyScope(state); 266 | 267 | var delimiter_index = current.length == 1 ? "[({".indexOf(current) : -1; 268 | if (delimiter_index != -1) 269 | pushBracketScope(stream, state, "])}".slice(delimiter_index, delimiter_index+1)); 270 | 271 | delimiter_index = "])}".indexOf(current); 272 | if (delimiter_index != -1) { 273 | if (top(state).type == current) state.indent = state.scopes.pop().offset - hangingIndent 274 | else return ERRORCLASS; 275 | } 276 | if (state.dedent > 0 && stream.eol() && top(state).type == "py") { 277 | if (state.scopes.length > 1) state.scopes.pop(); 278 | state.dedent -= 1; 279 | } 280 | 281 | return style; 282 | } 283 | 284 | var external = { 285 | startState: function(basecolumn) { 286 | return { 287 | tokenize: tokenBase, 288 | scopes: [{offset: basecolumn || 0, type: "py", align: null}], 289 | indent: basecolumn || 0, 290 | lastToken: null, 291 | lambda: false, 292 | dedent: 0 293 | }; 294 | }, 295 | 296 | token: function(stream, state) { 297 | var addErr = state.errorToken; 298 | if (addErr) state.errorToken = false; 299 | var style = tokenLexer(stream, state); 300 | 301 | if (style && style != "comment") 302 | state.lastToken = (style == "keyword" || style == "punctuation") ? stream.current() : style; 303 | if (style == "punctuation") style = null; 304 | 305 | if (stream.eol() && state.lambda) 306 | state.lambda = false; 307 | return addErr ? style + " " + ERRORCLASS : style; 308 | }, 309 | 310 | indent: function(state, textAfter) { 311 | if (state.tokenize != tokenBase) 312 | return state.tokenize.isString ? CodeMirror.Pass : 0; 313 | 314 | var scope = top(state), closing = scope.type == textAfter.charAt(0) 315 | if (scope.align != null) 316 | return scope.align - (closing ? 1 : 0) 317 | else 318 | return scope.offset - (closing ? hangingIndent : 0) 319 | }, 320 | 321 | electricInput: /^\s*[\}\]\)]$/, 322 | closeBrackets: {triples: "'\""}, 323 | lineComment: "#", 324 | fold: "indent" 325 | }; 326 | return external; 327 | }); 328 | 329 | CodeMirror.defineMIME("text/x-python", "python"); 330 | 331 | var words = function(str) { return str.split(" "); }; 332 | 333 | CodeMirror.defineMIME("text/x-cython", { 334 | name: "python", 335 | extra_keywords: words("by cdef cimport cpdef ctypedef enum except"+ 336 | "extern gil include nogil property public"+ 337 | "readonly struct union DEF IF ELIF ELSE") 338 | }); 339 | 340 | }); 341 | -------------------------------------------------------------------------------- /tests/test_examples.py: -------------------------------------------------------------------------------- 1 | # Test the examples from the Turq documentation (``turq/examples.rst``). 2 | # Every test function is matched (by function name) to exactly one 3 | # snippet of example code, which is loaded into Turq by the `example` fixture. 4 | 5 | # pylint: disable=redefined-outer-name,invalid-name 6 | 7 | import json 8 | import io 9 | import os 10 | import time 11 | 12 | import h11 13 | import pytest 14 | from requests.auth import HTTPDigestAuth 15 | 16 | import turq.examples 17 | 18 | examples = turq.examples.load_pairs() 19 | 20 | 21 | @pytest.fixture 22 | def example(request, turq_instance, tmpdir): 23 | test_name = request.node.name 24 | [example_code] = [code for (slug, code) in examples if slug in test_name] 25 | rules_path = tmpdir.join('rules.py') 26 | rules_path.write(example_code) 27 | turq_instance.extra_args = ['-r', str(rules_path)] 28 | with turq_instance: 29 | yield turq_instance 30 | 31 | 32 | def test_basics_1_ok(example): 33 | resp, data, _ = example.send(h11.Request(method='GET', target='/hello', 34 | headers=[('Host', 'example')]), 35 | h11.EndOfMessage()) 36 | assert resp.status_code == 200 37 | assert resp.reason == b'OK' 38 | assert (b'content-type', b'text/plain') in resp.headers 39 | assert data.data == b'Hello world!\r\n' 40 | 41 | 42 | def test_basics_1_not_found(example): 43 | resp, data, _ = example.send(h11.Request(method='GET', target='/', 44 | headers=[('Host', 'example')]), 45 | h11.EndOfMessage()) 46 | assert resp.status_code == 404 47 | assert resp.reason == b'Not Found' 48 | assert (b'content-type', b'text/plain; charset=utf-8') in resp.headers 49 | assert b'Error! ' in data.data 50 | 51 | 52 | def test_basics_1_head(example): 53 | resp, _ = example.send(h11.Request(method='HEAD', target='/hello', 54 | headers=[('Host', 'example')]), 55 | h11.EndOfMessage()) 56 | assert resp.status_code == 200 57 | assert resp.reason == b'OK' 58 | assert (b'content-type', b'text/plain') in resp.headers 59 | 60 | 61 | def test_basics_1_client_error(example): 62 | # Request a URL that is too long for h11's buffers. 63 | resp, data, _ = example.send(h11.Request(method='GET', target='/a' * 99999, 64 | headers=[('Host', 'example')]), 65 | h11.EndOfMessage()) 66 | assert 400 <= resp.status_code <= 499 67 | assert resp.reason 68 | assert (b'content-type', b'text/plain') in resp.headers 69 | assert b'Error: ' in data.data 70 | 71 | 72 | def test_basics_1_pipelining(example): 73 | with example.connect() as sock: 74 | # Send three requests in a row... 75 | sock.sendall(b'GET /hello HTTP/1.1\r\nHost: example\r\n\r\n' * 3) 76 | # ...and then read the three responses. 77 | n = 0 78 | while n < 3: 79 | n += sock.recv(4096).count(b'HTTP/1.1 200 OK\r\n') 80 | 81 | 82 | def test_response_headers_1(example): 83 | resp, _ = example.send(h11.Request(method='GET', target='/', 84 | headers=[('Host', 'example')]), 85 | h11.EndOfMessage()) 86 | assert (b'cache-control', b'max-age=3600') in resp.headers 87 | assert (b'cache-control', b'public') not in resp.headers 88 | 89 | 90 | def test_response_headers_2(example): 91 | resp, _ = example.send(h11.Request(method='GET', target='/', 92 | headers=[('Host', 'example')]), 93 | h11.EndOfMessage()) 94 | assert (b'set-cookie', b'sessionid=123456') in resp.headers 95 | assert (b'set-cookie', b'__adtrack=abcdef') in resp.headers 96 | 97 | 98 | def test_custom_status_code_and_reason_1(example): 99 | resp, _, _ = example.send(h11.Request(method='GET', target='/', 100 | headers=[('Host', 'example')]), 101 | h11.EndOfMessage()) 102 | assert resp.status_code == 567 103 | assert resp.reason == b'Server Fell Over' 104 | 105 | 106 | def test_response_framing_1_content_length(example): 107 | resp, data, _ = example.send(h11.Request(method='GET', target='/', 108 | headers=[('Host', 'example')]), 109 | h11.EndOfMessage()) 110 | assert (b'content-length', b'14') in resp.headers 111 | assert b'transfer-encoding' not in dict(resp.headers) 112 | assert data.data == b'Hello world!\r\n' 113 | 114 | 115 | def test_response_framing_1_keep_alive(example): 116 | with example.connect() as sock: 117 | for _ in range(3): # The same socket can be reused multiple times. 118 | sock.sendall(b'GET / HTTP/1.1\r\n' 119 | b'Host: example\r\n' 120 | b'\r\n') 121 | while b'HTTP/1.1 200 OK\r\n' not in sock.recv(4096): 122 | pass 123 | 124 | 125 | def test_response_framing_1_http10(example): 126 | with example.connect() as sock: 127 | sock.sendall(b'GET / HTTP/1.0\r\n' 128 | b'\r\n') 129 | time.sleep(1) 130 | assert b'HTTP/1.1 200 OK\r\n' in sock.recv(4096) 131 | assert sock.recv(4096) == b'' # Server closed the connection 132 | 133 | 134 | def test_response_framing_2_close(example): 135 | with example.connect() as sock: 136 | sock.sendall(b'GET / HTTP/1.1\r\n' 137 | b'Host: example\r\n' 138 | b'\r\n') 139 | time.sleep(1) 140 | assert b'HTTP/1.1 200 OK\r\n' in sock.recv(4096) 141 | assert sock.recv(4096) == b'' # Server closed the connection 142 | 143 | 144 | def test_streaming_responses_1(example): 145 | t0 = time.monotonic() 146 | resp, data1, data2, data3, _ = example.send( 147 | h11.Request(method='GET', target='/', headers=[('Host', 'example')]), 148 | h11.EndOfMessage()) 149 | t1 = time.monotonic() 150 | assert t1 - t0 >= 3 # At least 3 seconds elapsed, due to ``sleep()`` 151 | assert (b'content-type', b'text/event-stream') in resp.headers 152 | assert data1.data == b'data: my event 1\r\n\r\n' 153 | assert data2.data == b'data: my event 2\r\n\r\n' 154 | assert data3.data == b'data: my event 3\r\n\r\n' 155 | 156 | 157 | def test_streaming_responses_2(example): 158 | resp, data1, data2, end = example.send( 159 | h11.Request(method='GET', target='/', headers=[('Host', 'example')]), 160 | h11.EndOfMessage()) 161 | assert (b'content-type', b'text/plain') in resp.headers 162 | assert b'content-md5' not in dict(resp.headers) 163 | assert data1.data == b'Hello, ' 164 | assert data2.data == b'world!\n' 165 | assert b'content-md5' in dict(end.headers) # In the trailer part 166 | 167 | 168 | def test_handling_expect_100_continue_1(example): 169 | with example.connect() as sock: 170 | sock.sendall(b'POST / HTTP/1.1\r\n' 171 | b'Host: example\r\n' 172 | b'Content-Length: 14\r\n' 173 | b'\r\n') 174 | assert b'HTTP/1.1 100 Continue\r\n' in sock.recv(4096) 175 | sock.sendall(b'Hello world!\r\n') 176 | assert b'HTTP/1.1 200 OK\r\n' in sock.recv(4096) 177 | 178 | 179 | def test_handling_expect_100_continue_2(example): 180 | with example.connect() as sock: 181 | sock.sendall(b'POST / HTTP/1.1\r\n' 182 | b'Host: example\r\n' 183 | b'Content-Length: 14\r\n' 184 | b'\r\n') 185 | assert b'HTTP/1.1 403 Forbidden\r\n' in sock.recv(4096) 186 | 187 | 188 | def test_forwarding_requests_1(example): 189 | resp, data, _ = example.send( 190 | h11.Request(method='GET', target='/get', 191 | headers=[('Host', 'example'), 192 | ('User-Agent', 'test'), 193 | ('Upgrade', 'my-protocol'), 194 | ('Connection', 'upgrade')]), 195 | h11.EndOfMessage()) 196 | assert (b'content-type', b'application/json') in resp.headers 197 | assert (b'cache-control', b'max-age=86400') in resp.headers 198 | assert b'server' not in dict(resp.headers) 199 | # Headers were correctly forwarded by Turq. 200 | what_upstream_saw = json.loads(data.data.decode('utf-8')) 201 | assert 'Upgrade' not in what_upstream_saw['headers'] 202 | assert what_upstream_saw['headers']['User-Agent'] == 'test' 203 | # Unfortunately httpbin does not reflect ``Via`` in ``headers``. 204 | 205 | 206 | def test_forwarding_requests_2(example): 207 | resp, _, _ = example.send(h11.Request(method='GET', target='/', 208 | headers=[('Host', 'example')]), 209 | h11.EndOfMessage()) 210 | # ``develop1.example`` is unreachable 211 | assert 500 <= resp.status_code <= 599 212 | 213 | 214 | def test_request_details_1_json(example): 215 | resp = example.request('POST', '/', 216 | headers={'Accept': 'application/json'}, 217 | json={'name': 'Ленин'}) 218 | assert resp.json() == {'hello': 'Ленин'} 219 | 220 | 221 | def test_request_details_1_url_encoded_form(example): 222 | resp = example.request('POST', '/', 223 | data={'request_id': 'Adk347sQZ', 'name': 'Чубайс'}) 224 | assert resp.text == 'Hello Чубайс!\r\n' 225 | 226 | 227 | def test_request_details_1_multipart(example): 228 | resp = example.request('POST', '/', 229 | data={'name': 'Atatürk'}, 230 | files={'dummy': io.BytesIO(b'force multipart')}) 231 | assert resp.text == 'Hello Atatürk!\r\n' 232 | 233 | 234 | def test_request_details_1_query(example): 235 | resp = example.request('GET', '/', params={'name': 'Святослав'}) 236 | assert resp.text == 'Hello Святослав!\r\n' 237 | 238 | 239 | def test_request_details_1_raw(example): 240 | resp = example.request('POST', '/', data='Ramesses') 241 | assert resp.text == 'Hello Ramesses!\r\n' 242 | 243 | 244 | def test_restful_routing_1_hit(example): 245 | resp = example.request('GET', '/v1/products/12345') 246 | assert resp.json() == {'id': 12345, 'inStock': True} 247 | resp = example.request('HEAD', '/v1/products/12345') 248 | assert resp.text == '' 249 | resp = example.request('PUT', '/v1/products/12345', 250 | json={'id': 12345, 'name': 'Café arabica'}) 251 | assert resp.json() == {'id': 12345, 'name': 'Café arabica'} 252 | resp = example.request('DELETE', '/v1/products/12345') 253 | assert resp.status_code == 204 254 | 255 | 256 | def test_restful_routing_1_miss(example): 257 | resp = example.request('GET', '/') 258 | assert resp.text == '' 259 | resp = example.request('GET', '/v1/products/') 260 | assert resp.text == '' 261 | resp = example.request('GET', '/v1/products/12345/details') 262 | assert resp.text == '' 263 | 264 | 265 | def test_html_pages_1(example): 266 | resp = example.request('GET', '/') 267 | assert resp.headers['Content-Type'].startswith('text/html') 268 | assert '

Hello world!

' in resp.text 269 | 270 | 271 | def test_html_pages_2(example): 272 | resp = example.request('GET', '/') 273 | assert '

Welcome to our site

' in resp.text 274 | assert 'href="/products"' in resp.text 275 | 276 | 277 | def test_html_pages_3(example): 278 | resp = example.request('GET', '/') 279 | assert '' in resp.text 280 | 281 | 282 | def test_anything_else_1(example): 283 | resp = example.request('GET', '/') 284 | assert len(resp.content) == 128 285 | with pytest.raises(ValueError): # Random binary data, not UTF-8 286 | resp.content.decode('utf-8') 287 | 288 | 289 | def test_random_responses_1(example): 290 | # Yeah, this doesn't actually test the probability 291 | resp = example.request('GET', '/') 292 | assert resp.status_code == 503 or 'Hello world!' in resp.text 293 | 294 | 295 | def test_switching_protocols_1_upgrade(example): 296 | with example.connect() as sock: 297 | sock.sendall(b'GET / HTTP/1.1\r\n' 298 | b'Host: example\r\n' 299 | b'Upgrade: QXTP\r\n' 300 | b'Connection: Upgrade\r\n' 301 | b'\r\n') 302 | resp = [] 303 | while not resp or resp[-1]: # Read everything 304 | resp.append(sock.recv(4096)) 305 | assert b'HTTP/1.1 101 Switching Protocols\r\n' in b''.join(resp) 306 | assert b'This is QXTP now!\r\n' in b''.join(resp) 307 | 308 | 309 | def test_switching_protocols_1_no_upgrade(example): 310 | with example.connect() as sock: 311 | sock.sendall(b'GET / HTTP/1.1\r\n' 312 | b'Host: example\r\n' 313 | b'Upgrade: WZTP\r\n' 314 | b'Connection: Upgrade\r\n' 315 | b'\r\n') 316 | assert b'HTTP/1.1 200 OK\r\n' in sock.recv(4096) 317 | 318 | 319 | def test_cross_origin_resource_sharing_1_simple(example): 320 | resp = example.request('GET', '/', 321 | headers={'Origin': 'http://example.com'}) 322 | assert resp.status_code == 200 323 | assert resp.headers['Access-Control-Allow-Origin'] == 'http://example.com' 324 | assert resp.headers['Access-Control-Allow-Credentials'] == 'true' 325 | assert resp.json() == {'some': 'data'} 326 | 327 | 328 | def test_cross_origin_resource_sharing_1_preflight(example): 329 | resp = example.request( 330 | 'OPTIONS', '/', 331 | headers={'Origin': 'http://example.com', 332 | 'Access-Control-Request-Method': 'PUT', 333 | 'Access-Control-Request-Headers': 'content-type'}) 334 | assert resp.status_code == 200 335 | assert resp.headers['Access-Control-Allow-Origin'] == 'http://example.com' 336 | assert resp.headers['Access-Control-Allow-Credentials'] == 'true' 337 | assert resp.headers['Access-Control-Allow-Methods'] == 'PUT' 338 | assert resp.headers['Access-Control-Allow-Headers'] == 'content-type' 339 | assert resp.text == '' 340 | 341 | 342 | def test_cross_origin_resource_sharing_2_callback(example): 343 | resp = example.request('GET', '/', params={'callback': '__jsonp4321'}) 344 | assert resp.status_code == 200 345 | assert resp.headers['Content-Type'] == 'application/javascript' 346 | assert resp.text == '__jsonp4321({"some": "data"});' 347 | 348 | 349 | @pytest.mark.skipif(not os.path.exists('/etc/services'), 350 | reason='requires a specific file') 351 | def test_body_from_file_1(example): 352 | resp = example.request('GET', '/') 353 | assert '80/tcp' in resp.text 354 | 355 | 356 | def test_custom_methods_1_allowed(example): 357 | resp = example.request('FROBNICATE', '/some/resource') 358 | assert resp.status_code == 200 359 | 360 | 361 | def test_custom_methods_1_not_allowed(example): 362 | resp = example.request('GET', '/some/resource') 363 | assert resp.status_code == 405 364 | assert resp.headers['Allow'] == 'FROBNICATE' 365 | 366 | 367 | def test_authentication_1(example): 368 | resp = example.request('GET', '/') 369 | assert resp.status_code == 401 370 | assert 'Basic realm="Turq"' in resp.headers['WWW-Authenticate'] 371 | resp = example.request('GET', '/', auth=('user', 'passwd')) 372 | assert resp.status_code == 200 373 | assert 'Super-secret page' in resp.text 374 | 375 | 376 | def test_authentication_2(example): 377 | resp = example.request('GET', '/') 378 | assert resp.status_code == 401 379 | assert 'Digest realm="Turq"' in resp.headers['WWW-Authenticate'] 380 | resp = example.request('GET', '/', auth=HTTPDigestAuth('user', 'passwd')) 381 | assert resp.status_code == 200 382 | 383 | 384 | def test_authentication_3(example): 385 | resp = example.request('GET', '/') 386 | assert resp.status_code == 401 387 | assert 'Bearer scope="turq"' in resp.headers['WWW-Authenticate'] 388 | resp = example.request('GET', '/', 389 | headers={'Authorization': 'Bearer ZumuVcMi7N6u'}) 390 | assert resp.status_code == 200 391 | 392 | 393 | def test_compression_1(example): 394 | resp = example.request('GET', '/') 395 | assert resp.headers['Content-Encoding'] == 'gzip' 396 | # Requests automatically decodes gzip 397 | assert 'ipsum' in resp.text 398 | 399 | 400 | def test_redirection_1(example): 401 | resp = example.request('GET', '/') 402 | assert 'Hello world!' in resp.text 403 | assert len(resp.history) == 1 404 | assert resp.history[0].status_code == 302 405 | 406 | 407 | def test_redirection_2(example): 408 | resp = example.request('GET', '/', allow_redirects=False) 409 | assert resp.status_code == 307 410 | 411 | 412 | def test_inspecting_requests_1(example): 413 | resp = example.request('GET', '/') 414 | resp.raise_for_status() 415 | # The actual logging is checked in `test_control.test_debug_output`. 416 | -------------------------------------------------------------------------------- /turq/rules.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=protected-access 2 | 3 | import cgi 4 | import contextlib 5 | import gzip 6 | import io 7 | import json 8 | import logging 9 | import random 10 | import re 11 | import socket 12 | import ssl 13 | import time 14 | import traceback 15 | from urllib.parse import parse_qs, urlparse 16 | import wsgiref.headers 17 | 18 | import dominate 19 | import dominate.tags as H 20 | import h11 21 | 22 | from turq.util.http import (KNOWN_METHODS, date, default_reason, 23 | error_explanation, nice_header_name) 24 | from turq.util.logging import getNextLogger 25 | from turq.util.text import ellipsize, force_bytes, lorem_ipsum 26 | 27 | 28 | RULES_FILENAME = '' 29 | 30 | 31 | class RulesContext: 32 | 33 | # An instance of `RulesContext` is responsible for handling 34 | # one request according to the rules provided by the user. 35 | # It receives `h11` events from its `MockHandler`, 36 | # converts them into a convenient `Request` representation, 37 | # executes the rules code to fill out the `Response`, 38 | # converts the `Response` into `h11` events 39 | # and sends them back to the `MockHandler`. 40 | 41 | # pylint: disable=attribute-defined-outside-init 42 | 43 | def __init__(self, code, handler): 44 | self._code = code 45 | self._handler = handler 46 | self._logger = getNextLogger('turq.request') 47 | 48 | def _run(self, event): 49 | self.request = Request( 50 | self, event.method.decode(), event.target.decode(), 51 | event.http_version.decode(), _decode_headers(event.headers), 52 | ) 53 | self._logger.info('> %s', ellipsize(self.request.line, 100)) 54 | self._log_headers(self.request.raw_headers) 55 | self._response = Response() 56 | self._scope = self._build_scope() 57 | try: 58 | exec(self._code, self._scope) # pylint: disable=exec-used 59 | except SkipRemainingRules: 60 | pass 61 | except Exception as exc: 62 | self._log_rules_error(exc) 63 | if self._handler.our_state is h11.SEND_RESPONSE: 64 | # We can still replace the response with a 500. 65 | self._response = Response() 66 | self.error(500) 67 | 68 | # Depending on the rules, at this point the request body may or may not 69 | # have been received, and the response may or may not have been sent. 70 | # We need to make sure everything is flushed. 71 | self._ensure_request_received() 72 | self.flush() 73 | 74 | def _log_headers(self, headers): 75 | for (name, value) in headers: 76 | self._logger.debug('+ %s: %s', name, value) 77 | 78 | def _build_scope(self): 79 | # Assemble the global scope in which the rules will be executed. 80 | # This includes all "public" attributes of `RulesContext`... 81 | scope = {name: getattr(self, name) 82 | for name in dir(self) if not name.startswith('_')} 83 | # ...shortcuts for common request methods 84 | for method in KNOWN_METHODS: 85 | scope[method.replace('-', '_')] = (self.method == method) 86 | # ...Dominate's HTML tags library 87 | scope['H'] = H 88 | # ...utility functions 89 | for func in [lorem_ipsum, time.sleep]: 90 | scope[func.__name__] = func 91 | return scope 92 | 93 | def _log_rules_error(self, exc): 94 | # Extract the rules line number where the error happened. 95 | [lineno, *_] = [lineno 96 | for (filename, lineno, _, _) 97 | in reversed(traceback.extract_tb(exc.__traceback__)) 98 | if filename == RULES_FILENAME] 99 | self._logger.error('error in rules, line %d: %s', lineno, exc) 100 | self._logger.debug('details of this error:', exc_info=True) 101 | 102 | def _ensure_request_received(self): 103 | if self._handler.their_state is h11.SEND_BODY: 104 | self._receive_body() 105 | 106 | def flush(self, body_too=True): 107 | if self._handler.our_state is h11.SEND_RESPONSE: 108 | self._send_response() 109 | # Clear the list of response headers: from this point on, 110 | # any headers added will be sent in the trailer part. 111 | self._response.raw_headers[:] = [] 112 | if body_too and self._handler.our_state is h11.SEND_BODY: 113 | self._send_body() 114 | 115 | def _receive_body(self): 116 | chunks = [] 117 | while True: 118 | event = self._handler.receive_event() 119 | if isinstance(event, h11.Data): 120 | chunks.append(event.data) 121 | elif isinstance(event, h11.EndOfMessage): 122 | self.request._body = b''.join(chunks) 123 | self._logger.debug('received request body: %d bytes', 124 | len(self.request._body)) 125 | # Add any trailer part to the main headers list 126 | trailer = _decode_headers(event.headers) 127 | self._log_headers(trailer) 128 | self.request.raw_headers += trailer 129 | break 130 | 131 | def _send_response(self, interim=False): 132 | self._response.finalize() 133 | self._logger.info('< %s', self._response.status_line) 134 | self._log_headers(self._response.raw_headers) 135 | cls = h11.InformationalResponse if interim else h11.Response 136 | self._handler.send_event(cls( 137 | http_version=self._response.http_version, 138 | status_code=self._response.status_code, 139 | reason=force_bytes(self._response.reason), 140 | headers=_encode_headers(self._response.raw_headers), 141 | )) 142 | 143 | def _send_body(self): 144 | if self._response.body: 145 | self.chunk(self._response.body) 146 | self._log_headers(self._response.raw_headers) 147 | self._handler.send_event(h11.EndOfMessage( 148 | headers=_encode_headers(self._response.raw_headers), 149 | )) 150 | 151 | def debug(self): 152 | if self._logger.getEffectiveLevel() > logging.DEBUG: 153 | self._logger.setLevel(logging.DEBUG) 154 | # Request headers were logged earlier in `_run`, 155 | # but the user didn't have a chance to see them 156 | # because debug logging was not yet enabled. 157 | self._log_headers(self.request.raw_headers) 158 | 159 | method = property(lambda self: self.request.method) 160 | target = property(lambda self: self.request.target) 161 | path = property(lambda self: self.request.path) 162 | query = property(lambda self: self.request.query) 163 | 164 | def status(self, code, reason=None): 165 | self._response.status_code = code 166 | self._response.reason = reason 167 | 168 | def header(self, name, value): 169 | self._response.headers[name] = value 170 | 171 | def add_header(self, name, value): 172 | self._response.headers.add_header(name, value) 173 | 174 | def delete_header(self, name): 175 | del self._response.headers[name] 176 | 177 | def body(self, data): 178 | if hasattr(data, 'read'): # files 179 | data = data.read() 180 | self._response.body = force_bytes(data, 'utf-8') 181 | 182 | def chunk(self, data): 183 | self.flush(body_too=False) 184 | self._response.body = None # So that `_send_body` skips it 185 | # Responses to HEAD can't have a message body. We magically skip 186 | # sending data in that case, so the user doesn't have to remember. 187 | # (204 and 304 responses also can't have a body, but those have to be 188 | # explicitly selected by the user, so it's their problem.) 189 | if self.method == 'HEAD': 190 | self._logger.debug('not sending %d bytes of response body ' 191 | 'because request was HEAD', len(data)) 192 | else: 193 | self._logger.debug('sending %d bytes of response body', len(data)) 194 | self._handler.send_event(h11.Data(data=force_bytes(data))) 195 | 196 | def content_length(self): 197 | self._response.headers['Content-Length'] = \ 198 | str(len(self._response.body)) 199 | 200 | @contextlib.contextmanager 201 | def interim(self): 202 | main_response = self._response 203 | self._response = Response() 204 | self.status(100) 205 | yield 206 | self._send_response(interim=True) 207 | self._response = main_response 208 | 209 | def forward(self, hostname, port, target, tls=None): 210 | self._ensure_request_received() # Get the trailer part, if any 211 | self._logger.debug('forwarding to %s port %d', hostname, port) 212 | self._response = forward(self.request, hostname, port, target, tls) 213 | self._logger.debug('upstream response: %s', self._response.status_line) 214 | 215 | def text(self, content): 216 | self.header('Content-Type', 'text/plain; charset=utf-8') 217 | self.body(content) 218 | 219 | def error(self, code): 220 | self.status(code) 221 | self.text('Error! %s\r\n' % error_explanation(code)) 222 | 223 | def json(self, obj, jsonp=False): 224 | data = json.dumps(obj) 225 | if jsonp and self.request.query.get('callback'): 226 | self.header('Content-Type', 'application/javascript') 227 | # http://timelessrepo.com/json-isnt-a-javascript-subset 228 | data = (data. 229 | replace('\u2028', '\\u2028'). 230 | replace('\u2029', '\\u2029')) 231 | self.body('%s(%s);' % (self.request.query['callback'], data)) 232 | else: 233 | self.header('Content-Type', 'application/json') 234 | self.body(data) 235 | 236 | def route(self, spec): 237 | # Convert our simplistic route format to a regex to match the path. 238 | # First, we need to escape dots and such. 239 | spec = re.escape(spec) 240 | # This had the side effect of escaping colon as well, 241 | # so we need to compensate for it (note the backslashes): 242 | regex = '^%s$' % re.sub(r'\\:([A-Za-z_][A-Za-z0-9_]*)', 243 | lambda m: '(?P<%s>[^/]+)' % m.group(1), 244 | spec) 245 | match = re.match(regex, self.path) 246 | if match: 247 | self._scope.update(match.groupdict()) 248 | return True 249 | else: 250 | return False 251 | 252 | def html(self): 253 | # If the user just calls ``html()``, we fill out a basic page. 254 | with self._edit_html(): 255 | H.h1('Hello world!') 256 | H.p(lorem_ipsum()) 257 | H.p(lorem_ipsum()) 258 | # But then we also return the context manager that can be used 259 | # to rebuild the page as the user wishes. It does nothing unless 260 | # it is entered (``with``). 261 | return self._edit_html() 262 | 263 | @contextlib.contextmanager 264 | def _edit_html(self): 265 | document = dominate.document(title='Hello world') 266 | with document: 267 | yield document 268 | self.header('Content-Type', 'text/html; charset=utf-8') 269 | self.body(document.render()) 270 | 271 | @staticmethod 272 | def maybe(p): 273 | return random.random() < p 274 | 275 | def send_raw(self, data): 276 | self._logger.info('sending %d bytes of raw data', len(data)) 277 | self._handler.send_raw(force_bytes(data, 'utf-8')) 278 | 279 | def cors(self): 280 | headers = self.request.headers 281 | self.header('Access-Control-Allow-Origin', headers.get('Origin', '*')) 282 | self.header('Access-Control-Allow-Credentials', 'true') 283 | if self.method == 'OPTIONS' and 'Origin' in headers: 284 | self._logger.debug('responding to CORS preflight request') 285 | self.status(200) 286 | self.header('Access-Control-Allow-Methods', 287 | headers.get('Access-Control-Request-Method', '')) 288 | self.header('Access-Control-Allow-Headers', 289 | headers.get('Access-Control-Request-Headers', '')) 290 | self.body('') 291 | raise SkipRemainingRules() 292 | else: 293 | self.add_header('Vary', 'Origin') 294 | 295 | def basic_auth(self): 296 | self._require_auth('Basic', 'realm="Turq"') 297 | 298 | def digest_auth(self): 299 | self._require_auth('Digest', 'realm="Turq", qop="auth", nonce="12345"') 300 | 301 | def bearer_auth(self): 302 | self._require_auth('Bearer', 'scope="turq"') 303 | 304 | def _require_auth(self, scheme, challenge_params): 305 | authorization = self.request.headers.get('Authorization', '') 306 | if not authorization.lower().startswith(scheme.lower() + ' '): 307 | self._logger.debug('missing required Authorization: %s', scheme) 308 | self.error(401) 309 | self.header('WWW-Authenticate', 310 | '%s %s' % (scheme, challenge_params)) 311 | raise SkipRemainingRules() 312 | 313 | def gzip(self): 314 | buf = io.BytesIO() 315 | with gzip.GzipFile(mode='wb', compresslevel=4, fileobj=buf) as f: 316 | f.write(self._response.body) 317 | self.body(buf.getvalue()) 318 | self.add_header('Content-Encoding', 'gzip') 319 | 320 | def redirect(self, location, status=302): 321 | self.status(status) 322 | self.header('Location', location) 323 | self.text('Please see %s\r\n' % location) 324 | 325 | 326 | class SkipRemainingRules(Exception): 327 | 328 | pass 329 | 330 | 331 | class Request: 332 | 333 | def __init__(self, context, method, target, http_version, headers): 334 | self._context = context 335 | self.method = method 336 | self.target = target 337 | parsed_url = urlparse(target) 338 | self.path = parsed_url.path 339 | self.query = _single_values(parse_qs(parsed_url.query)) 340 | self.version = self.http_version = http_version 341 | self.raw_headers = headers 342 | self.headers = wsgiref.headers.Headers(self.raw_headers) 343 | self._body = None 344 | self._json = None 345 | self._form = None 346 | 347 | # Reconstructed request-line, for logging. 348 | self.line = '%s %s HTTP/%s' % (self.method, self.target, 349 | self.http_version) 350 | 351 | @property 352 | def body(self): 353 | # Request body is received lazily. This allows handling 354 | # finer aspects of the protocol, such as ``Expect: 100-continue``. 355 | if self._body is None: 356 | self._context._receive_body() 357 | return self._body 358 | 359 | # `json` and `form` are very heavy-handed with regard to encoding. 360 | # We don't care about applications that send JSON in UTF-16 or 361 | # Windows-1251 in URL encoding. Turq should be easy in the common case. 362 | 363 | @property 364 | def json(self): 365 | if self._json is None: 366 | try: 367 | self._json = json.loads(self.body.decode('utf-8')) 368 | except ValueError as exc: 369 | self._context._logger.debug('cannot read JSON: %s', exc) 370 | return self._json 371 | 372 | @property 373 | def form(self): 374 | if self._form is None: 375 | try: 376 | content_type = self.headers.get('Content-Type', '') 377 | type_, params = cgi.parse_header(content_type) 378 | if type_.lower() == 'multipart/form-data': 379 | self._form = _parse_multipart(self.body, params) 380 | else: # Assume URL-encoded 381 | self._form = _single_values(parse_qs(self.body.decode())) 382 | except ValueError as exc: 383 | self._context._logger.debug('cannot read form: %s', exc) 384 | return self._form 385 | 386 | 387 | class Response: 388 | 389 | def __init__(self): 390 | self.http_version = '1.1' 391 | self.status_code = 200 392 | self.reason = None 393 | self.raw_headers = [] 394 | self.headers = wsgiref.headers.Headers(self.raw_headers) 395 | self.body = b'' 396 | 397 | def finalize(self): 398 | # h11 only sends HTTP/1.1. 399 | self.http_version = '1.1' 400 | 401 | # h11 sends an empty reason phrase by default. While this is 402 | # correct with regard to the protocol, I think it will be 403 | # more convenient and less surprising to the user if we fill it. 404 | if self.reason is None: 405 | self.reason = default_reason(self.status_code) 406 | 407 | # RFC 7231 Section 7.1.1.2 requires a ``Date`` header 408 | # on all 2xx, 3xx, and 4xx responses. 409 | if 200 <= self.status_code <= 499 and 'Date' not in self.headers: 410 | self.headers['Date'] = date() 411 | 412 | @property 413 | def status_line(self): 414 | # Reconstructed status-line, for logging. 415 | return 'HTTP/%s %d %s' % (self.http_version, 416 | self.status_code, self.reason) 417 | 418 | 419 | def _decode_headers(headers): 420 | # Header values can contain arbitrary bytes. Decode them from ISO-8859-1, 421 | # which is the historical encoding of HTTP. Decoding bytes from ISO-8859-1 422 | # is a lossless operation, it cannot fail. Also, h11 gives us all header 423 | # names in lowercase, but we force them to Message-Case for a more readable 424 | # output in `_log_headers`. 425 | return [(nice_header_name(name.decode()), value.decode('iso-8859-1')) 426 | for (name, value) in headers] 427 | 428 | def _encode_headers(headers): 429 | return [(force_bytes(name), force_bytes(value)) 430 | for (name, value) in headers] 431 | 432 | 433 | def _parse_multipart(body, params): 434 | # Some ritual dance is required to get the `cgi` module work in 2017. 435 | body = io.BytesIO(body) 436 | params = {name: force_bytes(value) for (name, value) in params.items()} 437 | parsed = _single_values(cgi.parse_multipart(body, params)) 438 | return {name: value.decode('utf-8') for (name, value) in parsed.items()} 439 | 440 | 441 | def _single_values(parsed_dict): 442 | # For ease of use, leave only the first value for each name. 443 | return {name: value for name, (value, *_) in parsed_dict.items()} 444 | 445 | 446 | def forward(request, hostname, port, target, tls=None): 447 | hconn = h11.Connection(our_role=h11.CLIENT) 448 | if tls is None: 449 | tls = (port == 443) 450 | headers = _forward_headers(request.raw_headers, request.http_version, 451 | also_exclude=['Host']) 452 | # RFC 7230 recommends that ``Host`` be the first header. 453 | headers.insert(0, ('Host', _generate_host_header(hostname, port, tls))) 454 | headers.append(('Connection', 'close')) 455 | 456 | sock = socket.create_connection((hostname, port)) 457 | 458 | try: 459 | if tls: 460 | # We intentionally ignore server certificates. In this context, 461 | # they are more likely to be a nuisance than a boon. 462 | ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) 463 | sock = ssl_context.wrap_socket(sock, server_hostname=hostname) 464 | sock.sendall(hconn.send(h11.Request(method=request.method, 465 | target=target, 466 | headers=_encode_headers(headers)))) 467 | sock.sendall(hconn.send(h11.Data(data=request.body))) 468 | sock.sendall(hconn.send(h11.EndOfMessage())) 469 | 470 | response = Response() 471 | while True: 472 | # pylint: disable=no-member 473 | event = hconn.next_event() 474 | if event is h11.NEED_DATA: 475 | hconn.receive_data(sock.recv(4096)) 476 | elif isinstance(event, h11.Response): 477 | response.http_version = event.http_version.decode() 478 | response.status_code = event.status_code 479 | # Reason phrases can contain arbitrary bytes. 480 | # See above regarding ISO-8859-1. 481 | response.reason = event.reason.decode('iso-8859-1') 482 | response.raw_headers[:] = _forward_headers( 483 | _decode_headers(event.headers), response.http_version) 484 | elif isinstance(event, h11.Data): 485 | response.body += event.data 486 | elif isinstance(event, h11.EndOfMessage): 487 | return response 488 | 489 | except h11.RemoteProtocolError as exc: 490 | # https://github.com/njsmith/h11/issues/41 491 | raise RuntimeError(str(exc)) from exc 492 | 493 | finally: 494 | sock.close() 495 | 496 | 497 | def _forward_headers(headers, http_version, also_exclude=None): 498 | # RFC 7230 Section 5.7 499 | connection_options = [option.strip().lower() 500 | for (name, value) in headers 501 | if name.lower() == 'connection' 502 | for option in value.split(',')] 503 | also_exclude = [name.lower() for name in also_exclude or []] 504 | exclude = connection_options + ['connection'] + also_exclude 505 | filtered = [(name, value) 506 | for (name, value) in headers 507 | if name.lower() not in exclude] 508 | return filtered + [('Via', '%s turq' % http_version)] 509 | 510 | 511 | def _generate_host_header(hostname, port, tls): 512 | if ':' in hostname: # IPv6 literal 513 | hostname = '[%s]' % hostname 514 | if port == (443 if tls else 80): # Default port 515 | return hostname 516 | else: 517 | return '%s:%d' % (hostname, port) 518 | --------------------------------------------------------------------------------