├── .gitignore ├── LICENSE ├── MANIFEST.in ├── NOTICE ├── README.rst ├── THANKS ├── bin └── tproxy ├── examples ├── couchdbrouter.py ├── httpproxy.py ├── httprewrite.py ├── psock4.py ├── sendfile.py ├── ssl.py ├── transparent.py ├── transparentrw.py └── welcome.txt ├── requirements.txt ├── setup.py └── tproxy ├── __init__.py ├── _sendfile.py ├── app.py ├── arbiter.py ├── client.py ├── config.py ├── pidfile.py ├── proxy.py ├── rewrite.py ├── route.py ├── sendfile.py ├── server.py ├── tools.py ├── util.py ├── worker.py └── workertmp.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.swp 3 | *.pyc 4 | *#* 5 | build 6 | dist 7 | setuptools-* 8 | .svn/* 9 | .DS_Store 10 | *.so 11 | tproxy.egg-info 12 | nohup.out 13 | .coverage 14 | doc/.sass-cache 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2011 (c) Benoît Chesneau 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following 10 | conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include .gitignore 2 | include LICENSE 3 | include NOTICE 4 | include README.rst 5 | include THANKS 6 | include requirements.txt 7 | recursive-include examples * 8 | 9 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | tproxy 2 | 2011 (c) Benoît Chesneau 3 | 4 | tproxy is released under the MIT license. See the LICENSE 5 | file for the complete license. 6 | 7 | 8 | Some code from Gunicorn project: 9 | -------------------------------- 10 | 11 | 2009,2010 (c) Benoît Chesneau 12 | 2009,2010 (c) Paul J. Davis 13 | 14 | gunicorn/arbiter.py 15 | gunicorn/config.py 16 | gunicorn/pidfile.py 17 | gunicorn/workertmp.py 18 | 19 | backports from python 2.7/3.x 20 | ----------------------------- 21 | tools.import_module 22 | 23 | 24 | gunicorn/_sendfile.py 25 | --------------------- 26 | 27 | based on gunicorn/sendfile.py refactored 28 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | tproxy 2 | ------ 3 | 4 | tproxy is a simple TCP routing proxy (layer 7) built on 5 | Gevent_ that lets you configure the routine logic in Python. It's heavily 6 | inspired from `proxy machine `_ 7 | but have some unique features like the pre-fork worker model borrowed to 8 | Gunicorn_. 9 | 10 | 11 | Instalation 12 | ----------- 13 | 14 | tproxy requires **Python 2.x >= 2.5**. Python 3.x support is planned. 15 | 16 | :: 17 | 18 | $ pip install gevent 19 | $ pip install tproxy 20 | 21 | To install from source:: 22 | 23 | $ git clone git://github.com/benoitc/tproxy.git 24 | $ cd tproxy 25 | $ pip install -r requirements.txt 26 | $ python setup.py install 27 | 28 | 29 | Test your installation by running the command line:: 30 | 31 | $ tproxy examples/transparent.py 32 | 33 | And go on http://127.0.0.1:5000 , you should see the google homepage. 34 | 35 | 36 | Usage 37 | ----- 38 | 39 | :: 40 | 41 | $ tproxy -h 42 | 43 | Usage: tproxy [OPTIONS] script_path 44 | 45 | Options: 46 | --version show program's version number and exit 47 | -h, --help show this help message and exit 48 | --log-file=FILE The log file to write to. [-] 49 | --log-level=LEVEL The granularity of log outputs. [info] 50 | --log-config=FILE The log config file to use. [None] 51 | -n STRING, --name=STRING A base to use with setproctitle for process naming. 52 | [None] 53 | -D, --daemon Daemonize the tproxy process. [False] 54 | -p FILE, --pid=FILE A filename to use for the PID file. [None] 55 | -u USER, --user=USER Switch worker processes to run as this user. [501] 56 | -g GROUP, --group=GROUP 57 | Switch worker process to run as this group. [20] 58 | -m INT, --umask=INT A bit mask for the file mode on files written by 59 | tproxy. [0] 60 | -b ADDRESS, --bind=ADDRESS The socket to bind. [127.0.0.1:8000] 61 | --backlog=INT The maximum number of pending connections. [2048] 62 | --ssl-keyfile=STRING Ssl key file [None] 63 | --ssl-certfile=STRING Ssl ca certs file. contains concatenated 64 | "certification [None] 65 | --ssl-ca-certs=STRING Ssl ca certs file. contains concatenated 66 | "certification [None] 67 | --ssl-cert-reqs=INT Specifies whether a certificate is required from the 68 | other [0] 69 | -w INT, --workers=INT The number of worker process for handling requests. [1] 70 | --worker-connections=INT The maximum number of simultaneous clients per worker. 71 | [1000] 72 | -t INT, --timeout=INT Workers silent for more than this many seconds are 73 | killed and restarted. [30] 74 | 75 | Signals 76 | ------- 77 | :: 78 | 79 | QUIT - Graceful shutdown. Stop accepting connections immediatly 80 | and wait until all connections close 81 | 82 | TERM - Fast shutdown. Stop accepting and close all conections 83 | after 10s. 84 | INT - Same as TERM 85 | 86 | HUP - Graceful reloading. Reload all workers with the new code 87 | in your routing script. 88 | 89 | USR2 - Upgrade tproxy on the fly 90 | 91 | TTIN - Increase the number of worker from 1 92 | 93 | TTOU - Decrease the number of worker from 1 94 | 95 | 96 | Exemple of routing script 97 | ------------------------- 98 | 99 | :: 100 | 101 | import re 102 | re_host = re.compile("Host:\s*(.*)\r\n") 103 | 104 | class CouchDBRouter(object): 105 | # look at the routing table and return a couchdb node to use 106 | def lookup(self, name): 107 | """ do something """ 108 | 109 | router = CouchDBRouter() 110 | 111 | # Perform content-aware routing based on the stream data. Here, the 112 | # Host header information from the HTTP protocol is parsed to find the 113 | # username and a lookup routine is run on the name to find the correct 114 | # couchdb node. If no match can be made yet, do nothing with the 115 | # connection. (make your own couchone server...) 116 | 117 | def proxy(data): 118 | matches = re_host.findall(data) 119 | if matches: 120 | host = router.lookup(matches.pop()) 121 | return {"remote": host} 122 | return None 123 | 124 | Example SOCKS4 Proxy in 18 Lines 125 | -------------------------------- 126 | 127 | :: 128 | 129 | import socket 130 | import struct 131 | 132 | def proxy(data): 133 | if len(data) < 9: 134 | return 135 | 136 | command = ord(data[1]) 137 | ip, port = socket.inet_ntoa(data[4:8]), struct.unpack(">H", data[2:4])[0] 138 | idx = data.index("\0") 139 | userid = data[8:idx] 140 | 141 | if command == 1: #connect 142 | return dict(remote="%s:%s" % (ip, port), 143 | reply="\0\x5a\0\0\0\0\0\0", 144 | data=data[idx:]) 145 | else: 146 | return {"close": "\0\x5b\0\0\0\0\0\0"} 147 | 148 | Example of returning a file 149 | --------------------------- 150 | 151 | :: 152 | 153 | import os 154 | 155 | WELCOME_FILE = os.path.join(os.path.dirname(__file__), "welcome.txt") 156 | 157 | def proxy(data): 158 | fno = os.open(WELCOME_FILE, os.O_RDONLY) 159 | return { 160 | "file": fno, 161 | "reply": "HTTP/1.1 200 OK\r\n\r\n" 162 | } 163 | 164 | Valid return values 165 | ------------------- 166 | 167 | * { "remote:": string or tuple } - String is the host:port of the 168 | server that will be proxied. 169 | * { "remote": String, "data": String} - Same as above, but 170 | send the given data instead. 171 | * { "remote": String, "data": String, "reply": String} - Same as above, 172 | but reply with given data back to the client 173 | * None - Do nothing. 174 | * { "close": True } - Close the connection. 175 | * { "close": String } - Close the connection after sending 176 | the String. 177 | * { "file": String } - Return a file specify by the file path and close 178 | the connection. 179 | * { "file": String, "reply": String } - Return a file specify by the 180 | file path and close the connection. 181 | * { "file": Int, "reply": String} - Same as above but reply with given 182 | data back to the client 183 | * { "file": Int } - Return a file specify by 184 | its file descriptor 185 | * { "file": Int, "reply": String} - Same as above 186 | but reply with given data back to the client 187 | 188 | Notes: 189 | ++++++ 190 | 191 | If `sendfile `_ API available it 192 | will be used to send a file with "file" command. 193 | 194 | The **file** command can have 2 optionnnal parameters: 195 | 196 | - offset: argument specifies where to begin in the file. 197 | - nbytes: specifies how many bytes of the file should be sent 198 | 199 | 200 | To **handle ssl for remote connection** you can add these optionals 201 | arguments: 202 | 203 | - ssl: True or False, if you want to connect with ssl 204 | - ssl_args: dict, optionals ssl arguments. Read the `ssl documentation 205 | `_ for more informations about them. 206 | 207 | Handle errors 208 | ------------- 209 | 210 | You can easily handling error by adding a **proxy_error** function in 211 | your script:: 212 | 213 | def proxy_error(client, e): 214 | pass 215 | 216 | This function get the ClientConnection instance (current connection) as 217 | first arguments and the error exception in second argument. 218 | 219 | Rewrite requests & responses 220 | ---------------------------- 221 | 222 | Main goal of tproxy is to allows you to route transparently tcp to your 223 | applications. But some case you want to do more. For example you need in 224 | HTTP 1.1 to change the Host header to make sure remote HTTP server will 225 | know what to do if uses virtual hosting. 226 | 227 | To do that, add a **rewrite_request** function in your function to 228 | simply rewrite clienrt request and **rewrite_response** to rewrite the 229 | remote response. Both functions take a tproxy.rewrite.RewriteIO instance 230 | which is based on io.RawIOBase class. 231 | 232 | See the `httprewrite.py `_ example for an example of HTTP rewrite. 233 | 234 | 235 | Copyright 236 | --------- 237 | 2011 (c) Benoît Chesneau 238 | 239 | 240 | .. _Gevent: http://gevent.org 241 | .. _Gunicorn: http://gunicorn.org 242 | -------------------------------------------------------------------------------- /THANKS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/benoitc/tproxy/a4aeb19fc3012d286d3b4e2f2be693813ba8d0d1/THANKS -------------------------------------------------------------------------------- /bin/tproxy: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of tproxy released under the MIT license. 5 | # See the NOTICE for more information. 6 | 7 | from tproxy.app import run 8 | 9 | run() 10 | -------------------------------------------------------------------------------- /examples/couchdbrouter.py: -------------------------------------------------------------------------------- 1 | import re 2 | re_host = re.compile("Host:\s*(.*)\r\n") 3 | 4 | class CouchDBRouter(object): 5 | # look at the routing table and return a couchdb node to use 6 | def lookup(self, name): 7 | """ do something """ 8 | return ("127.0.0.1",5984) 9 | router = CouchDBRouter() 10 | 11 | # Perform content-aware routing based on the stream data. Here, the 12 | # Host header information from the HTTP protocol is parsed to find the 13 | # username and a lookup routine is run on the name to find the correct 14 | # couchdb node. If no match can be made yet, do nothing with the 15 | # connection. (make your own couchone server...) 16 | 17 | def proxy(data): 18 | matches = re_host.findall(data) 19 | if matches: 20 | host = router.lookup(matches.pop()) 21 | return {"remote": host} 22 | return None 23 | -------------------------------------------------------------------------------- /examples/httpproxy.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | # 3 | # This file is part of tproxy released under the MIT license. 4 | # See the NOTICE for more information. 5 | 6 | 7 | """ simple proxy that can be used behind a browser. 8 | This example require http_parser module: 9 | 10 | pip install http_parser 11 | 12 | """ 13 | 14 | import io 15 | import urlparse 16 | import socket 17 | 18 | from http_parser.http import HttpStream, NoMoreData, ParserError 19 | from http_parser.parser import HttpParser 20 | from tproxy.util import parse_address 21 | 22 | def get_host(addr, is_ssl=False): 23 | """ return a correct Host header """ 24 | host = addr[0] 25 | if addr[1] != (is_ssl and 443 or 80): 26 | host = "%s:%s" % (host, addr[1]) 27 | return host 28 | 29 | 30 | def write_chunk(to, data): 31 | """ send a chunk encoded """ 32 | chunk = "".join(("%X\r\n" % len(data), data, "\r\n")) 33 | to.writeall(chunk) 34 | 35 | def write(to, data): 36 | to.writeall(data) 37 | 38 | def send_body(to, body, chunked=False): 39 | if chunked: 40 | _write = write_chunk 41 | else: 42 | _write = write 43 | 44 | while True: 45 | data = body.read(io.DEFAULT_BUFFER_SIZE) 46 | if not data: 47 | break 48 | _write(to, data) 49 | 50 | if chunked: 51 | _write(to, "") 52 | 53 | def rewrite_request(req): 54 | try: 55 | while True: 56 | parser = HttpStream(req) 57 | headers = parser.headers() 58 | 59 | parsed_url = urlparse.urlparse(parser.url()) 60 | 61 | is_ssl = parsed_url.scheme == "https" 62 | 63 | host = get_host(parse_address(parsed_url.netloc, 80), 64 | is_ssl=is_ssl) 65 | headers['Host'] = host 66 | headers['Connection'] = 'close' 67 | 68 | if 'Proxy-Connection' in headers: 69 | del headers['Proxy-Connection'] 70 | 71 | 72 | location = urlparse.urlunparse(('', '', parsed_url.path, 73 | parsed_url.params, parsed_url.query, parsed_url.fragment)) 74 | 75 | httpver = "HTTP/%s" % ".".join(map(str, 76 | parser.version())) 77 | 78 | new_headers = ["%s %s %s\r\n" % (parser.method(), location, 79 | httpver)] 80 | 81 | new_headers.extend(["%s: %s\r\n" % (hname, hvalue) \ 82 | for hname, hvalue in headers.items()]) 83 | 84 | req.writeall(bytes("".join(new_headers) + "\r\n")) 85 | body = parser.body_file() 86 | send_body(req, body, parser.is_chunked()) 87 | 88 | except (socket.error, NoMoreData, ParserError): 89 | pass 90 | 91 | def proxy(data): 92 | recved = len(data) 93 | 94 | idx = data.find("\r\n") 95 | if idx <= 0: 96 | return 97 | 98 | line, rest = data[:idx], data[idx:] 99 | if line.startswith("CONNECT"): 100 | parts = line.split(None) 101 | netloc = parts[1] 102 | remote = parse_address(netloc, 80) 103 | 104 | reply_msg = "%s 200 OK\r\n\r\n" % parts[2] 105 | return {"remote": remote, 106 | "reply": reply_msg, 107 | "data": ""} 108 | 109 | 110 | parser = HttpParser() 111 | parsed = parser.execute(data, recved) 112 | if parsed != recved: 113 | return { 'close':'HTTP/1.0 502 Gateway Error\r\n\r\nError parsing request'} 114 | 115 | if not parser.get_url(): 116 | return 117 | 118 | parsed_url = urlparse.urlparse(parser.get_url()) 119 | 120 | is_ssl = parsed_url.scheme == "https" 121 | remote = parse_address(parsed_url.netloc, 80) 122 | 123 | return {"remote": remote, 124 | "ssl": is_ssl} 125 | -------------------------------------------------------------------------------- /examples/httprewrite.py: -------------------------------------------------------------------------------- 1 | import re 2 | from http_parser.http import HttpStream, NoMoreData 3 | from http_parser.reader import SocketReader 4 | import socket 5 | 6 | def rewrite_headers(parser, values=None): 7 | headers = parser.headers() 8 | if isinstance(values, dict): 9 | headers.update(values) 10 | 11 | httpver = "HTTP/%s" % ".".join(map(str, 12 | parser.version())) 13 | 14 | new_headers = ["%s %s %s\r\n" % (parser.method(), parser.url(), 15 | httpver)] 16 | 17 | new_headers.extend(["%s: %s\r\n" % (k, str(v)) for k, v in \ 18 | headers.items()]) 19 | 20 | return "".join(new_headers) + "\r\n" 21 | 22 | def rewrite_request(req): 23 | try: 24 | while True: 25 | parser = HttpStream(req) 26 | 27 | new_headers = rewrite_headers(parser, {'Host': 'gunicorn.org'}) 28 | if new_headers is None: 29 | break 30 | req.send(new_headers) 31 | body = parser.body_file() 32 | while True: 33 | data = body.read(8192) 34 | if not data: 35 | break 36 | req.writeall(data) 37 | except (socket.error, NoMoreData): 38 | pass 39 | 40 | def rewrite_response(resp): 41 | try: 42 | while True: 43 | parser = HttpStream(resp) 44 | # we aren't doing anything here 45 | for data in parser: 46 | resp.writeall(data) 47 | 48 | if not parser.should_keep_alive(): 49 | # close the connection. 50 | break 51 | except (socket.error, NoMoreData): 52 | pass 53 | 54 | def proxy(data): 55 | return {'remote': ('gunicorn.org', 80)} 56 | -------------------------------------------------------------------------------- /examples/psock4.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import struct 3 | 4 | def proxy(data): 5 | if len(data) < 9: 6 | return 7 | 8 | command = ord(data[1]) 9 | ip, port = socket.inet_ntoa(data[4:8]), struct.unpack(">H", data[2:4])[0] 10 | idx = data.index("\0") 11 | userid = data[8:idx] 12 | 13 | if command == 1: #connect 14 | return dict(remote="%s:%s" % (ip, port), 15 | reply="\0\x5a\0\0\0\0\0\0", 16 | data=data[idx:]) 17 | else: 18 | return {"close": "\0\x5b\0\0\0\0\0\0"} 19 | -------------------------------------------------------------------------------- /examples/sendfile.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | WELCOME_FILE = os.path.join(os.path.dirname(__file__), "welcome.txt") 4 | 5 | def proxy(data): 6 | fno = os.open(WELCOME_FILE, os.O_RDONLY) 7 | return { 8 | "file": fno, 9 | "reply": "HTTP/1.1 200 OK\r\n\r\n" 10 | } 11 | -------------------------------------------------------------------------------- /examples/ssl.py: -------------------------------------------------------------------------------- 1 | def proxy(data): 2 | return {'remote': ('encrypted.google.com', 443), "ssl": True} 3 | -------------------------------------------------------------------------------- /examples/transparent.py: -------------------------------------------------------------------------------- 1 | def proxy(data): 2 | return { "remote": "google.com:80" } 3 | -------------------------------------------------------------------------------- /examples/transparentrw.py: -------------------------------------------------------------------------------- 1 | import io 2 | import re 3 | 4 | def rewrite_request(req): 5 | while True: 6 | data = req.read(io.DEFAULT_BUFFER_SIZE) 7 | if not data: 8 | break 9 | req.writeall(data) 10 | 11 | def rewrite_response(resp): 12 | while True: 13 | data = resp.read(io.DEFAULT_BUFFER_SIZE) 14 | if not data: 15 | break 16 | resp.writeall(data) 17 | 18 | def proxy(data): 19 | return {'remote': ('google.com', 80)} 20 | -------------------------------------------------------------------------------- /examples/welcome.txt: -------------------------------------------------------------------------------- 1 | hello! 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | gevent>=0.13.4 2 | setproctitle>=1.1.2 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 - 3 | # 4 | # This file is part of tproxy released under the MIT license. 5 | # See the NOTICE for more information. 6 | 7 | from __future__ import with_statement 8 | 9 | from glob import glob 10 | from imp import load_source 11 | import os 12 | import sys 13 | 14 | 15 | CLASSIFIERS = [ 16 | 'Development Status :: 4 - Beta', 17 | 'Environment :: Other Environment', 18 | 'Intended Audience :: Developers', 19 | 'License :: OSI Approved :: MIT License', 20 | 'Operating System :: MacOS :: MacOS X', 21 | 'Operating System :: POSIX', 22 | 'Programming Language :: Python', 23 | 'Topic :: Internet', 24 | 'Topic :: Internet', 25 | 'Topic :: Internet :: Proxy Servers', 26 | 'Topic :: Utilities', 27 | 'Topic :: Software Development :: Libraries :: Python Modules', 28 | 'Topic :: System :: Networking' 29 | ] 30 | 31 | MODULES = ( 32 | 'tproxy', 33 | ) 34 | 35 | SCRIPTS = glob("bin/tproxy*") 36 | 37 | def main(): 38 | if "--setuptools" in sys.argv: 39 | sys.argv.remove("--setuptools") 40 | from setuptools import setup 41 | use_setuptools = True 42 | else: 43 | from distutils.core import setup 44 | use_setuptools = False 45 | 46 | tproxy = load_source("tproxy", os.path.join("tproxy", 47 | "__init__.py")) 48 | 49 | # read long description 50 | with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as f: 51 | long_description = f.read() 52 | 53 | 54 | PACKAGES = {} 55 | for name in MODULES: 56 | PACKAGES[name] = name.replace(".", "/") 57 | 58 | DATA_FILES = [ 59 | ('tproxy', ["LICENSE", "MANIFEST.in", "NOTICE", "README.rst", 60 | "THANKS",]) 61 | ] 62 | 63 | options = dict( 64 | name = 'tproxy', 65 | version = tproxy.__version__, 66 | description = 'WSGI HTTP Server for UNIX', 67 | long_description = long_description, 68 | author = 'Benoit Chesneau', 69 | author_email = 'benoitc@e-engura.com', 70 | license = 'MIT', 71 | url = 'http://github.com/benoitc/tproxy', 72 | classifiers = CLASSIFIERS, 73 | packages = PACKAGES.keys(), 74 | package_dir = PACKAGES, 75 | scripts = SCRIPTS, 76 | data_files = DATA_FILES, 77 | ) 78 | 79 | 80 | setup(**options) 81 | 82 | if __name__ == "__main__": 83 | main() 84 | 85 | -------------------------------------------------------------------------------- /tproxy/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | # 3 | # This file is part of tproxy released under the MIT license. 4 | # See the NOTICE for more information. 5 | 6 | version_info = (0, 5, 4) 7 | __version__ = ".".join(map(str, version_info)) 8 | -------------------------------------------------------------------------------- /tproxy/_sendfile.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | # 3 | # This file is part of tproxy released under the MIT license. 4 | # See the NOTICE for more information. 5 | 6 | import errno 7 | import os 8 | import sys 9 | 10 | try: 11 | import ctypes 12 | import ctypes.util 13 | except MemoryError: 14 | # selinux execmem denial 15 | # https://bugzilla.redhat.com/show_bug.cgi?id=488396 16 | raise ImportError 17 | 18 | SUPPORTED_PLATFORMS = ( 19 | 'darwin', 20 | 'freebsd', 21 | 'dragonfly' 22 | 'linux2') 23 | 24 | if sys.version_info < (2, 6) or \ 25 | sys.platform not in SUPPORTED_PLATFORMS: 26 | raise ImportError("sendfile isn't supported on this platform") 27 | 28 | _libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True) 29 | _sendfile = _libc.sendfile 30 | 31 | def sendfile(fdout, fdin, offset, nbytes): 32 | if sys.platform == 'darwin': 33 | _sendfile.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_uint64, 34 | ctypes.POINTER(ctypes.c_uint64), ctypes.c_voidp, 35 | ctypes.c_int] 36 | _nbytes = ctypes.c_uint64(nbytes) 37 | result = _sendfile(fdin, fdout, offset, _nbytes, None, 0) 38 | if result == -1: 39 | e = ctypes.get_errno() 40 | if e == errno.EAGAIN and _nbytes.value: 41 | return nbytes.value 42 | raise OSError(e, os.strerror(e)) 43 | return _nbytes.value 44 | elif sys.platform in ('freebsd', 'dragonfly',): 45 | _sendfile.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_uint64, 46 | ctypes.c_uint64, ctypes.c_voidp, 47 | ctypes.POINTER(ctypes.c_uint64), ctypes.c_int] 48 | _sbytes = ctypes.c_uint64() 49 | result = _sendfile(fdin, fdout, offset, nbytes, None, _sbytes, 0) 50 | if result == -1: 51 | e = ctypes.get_errno() 52 | if e == errno.EAGAIN and _sbytes.value: 53 | return _sbytes.value 54 | raise OSError(e, os.strerror(e)) 55 | return _sbytes.value 56 | 57 | else: 58 | _sendfile.argtypes = [ctypes.c_int, ctypes.c_int, 59 | ctypes.POINTER(ctypes.c_uint64), ctypes.c_size_t] 60 | 61 | _offset = ctypes.c_uint64(offset) 62 | sent = _sendfile(fdout, fdin, _offset, nbytes) 63 | if sent == -1: 64 | e = ctypess.get_errno() 65 | raise OSError(e, os.strerror(e)) 66 | return sent 67 | -------------------------------------------------------------------------------- /tproxy/app.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | # 3 | # This file is part of tproxy released under the MIT license. 4 | # See the NOTICE for more information. 5 | 6 | import imp 7 | import inspect 8 | import logging 9 | from logging.config import fileConfig 10 | import os 11 | import sys 12 | 13 | from gevent import core 14 | from gevent.hub import get_hub 15 | from gevent import monkey 16 | monkey.noisy = False 17 | monkey.patch_all() 18 | 19 | from . import util 20 | from .arbiter import Arbiter 21 | from .config import Config 22 | from .tools import import_module 23 | 24 | class Script(object): 25 | """ load a python file or module """ 26 | 27 | def __init__(self, script_uri, cfg=None): 28 | self.script_uri = script_uri 29 | self.cfg = cfg 30 | 31 | def load(self): 32 | if os.path.exists(self.script_uri): 33 | script = imp.load_source('_route', self.script_uri) 34 | else: 35 | if ":" in self.script_uri: 36 | parts = self.script_uri.rsplit(":", 1) 37 | name, objname = parts[0], parts[1] 38 | mod = import_module(name) 39 | 40 | script_class = getattr(mod, objname) 41 | if inspect.getargspec(script_class.__init__) > 1: 42 | script = script_class(self.cfg) 43 | else: 44 | script=script_class() 45 | else: 46 | script = import_module(self.script_uri) 47 | 48 | script.__dict__['__tproxy_cfg__'] = self.cfg 49 | return script 50 | 51 | class Application(object): 52 | 53 | LOG_LEVELS = { 54 | "critical": logging.CRITICAL, 55 | "error": logging.ERROR, 56 | "warning": logging.WARNING, 57 | "info": logging.INFO, 58 | "debug": logging.DEBUG 59 | } 60 | 61 | def __init__(self): 62 | self.logger = None 63 | self.cfg = Config("%prog [OPTIONS] script_path") 64 | self.script = None 65 | 66 | def load_config(self): 67 | # parse console args 68 | parser = self.cfg.parser() 69 | opts, args = parser.parse_args() 70 | 71 | if len(args) != 1: 72 | parser.error("No script or module specified.") 73 | 74 | script_uri = args[0] 75 | self.cfg.default_name = args[0] 76 | 77 | # Load conf 78 | try: 79 | for k, v in opts.__dict__.items(): 80 | if v is None: 81 | continue 82 | self.cfg.set(k.lower(), v) 83 | except Exception, e: 84 | sys.stderr.write("config error: %s\n" % str(e)) 85 | os._exit(1) 86 | 87 | # setup script 88 | self.script = Script(script_uri, cfg=self.cfg) 89 | sys.path.insert(0, os.getcwd()) 90 | 91 | 92 | def configure_logging(self): 93 | """\ 94 | Set the log level and choose the destination for log output. 95 | """ 96 | self.logger = logging.getLogger('tproxy') 97 | 98 | fmt = r"%(asctime)s [%(process)d] [%(levelname)s] %(message)s" 99 | datefmt = r"%Y-%m-%d %H:%M:%S" 100 | if not self.cfg.logconfig: 101 | handlers = [] 102 | if self.cfg.logfile != "-": 103 | handlers.append(logging.FileHandler(self.cfg.logfile)) 104 | else: 105 | handlers.append(logging.StreamHandler()) 106 | 107 | loglevel = self.LOG_LEVELS.get(self.cfg.loglevel.lower(), logging.INFO) 108 | self.logger.setLevel(loglevel) 109 | for h in handlers: 110 | h.setFormatter(logging.Formatter(fmt, datefmt)) 111 | self.logger.addHandler(h) 112 | else: 113 | if os.path.exists(self.cfg.logconfig): 114 | fileConfig(self.cfg.logconfig) 115 | else: 116 | raise RuntimeError("Error: logfile '%s' not found." % 117 | self.cfg.logconfig) 118 | 119 | def run(self): 120 | self.load_config() 121 | 122 | if self.cfg.daemon: 123 | util.daemonize() 124 | else: 125 | try: 126 | os.setpgrp() 127 | except OSError, e: 128 | if e[0] != errno.EPERM: 129 | raise 130 | 131 | self.configure_logging() 132 | try: 133 | Arbiter(self.cfg, self.script).run() 134 | except RuntimeError, e: 135 | sys.stderr.write("\nError: %s\n\n" % e) 136 | sys.stderr.flush() 137 | os._exit(1) 138 | 139 | def run(): 140 | Application().run() 141 | -------------------------------------------------------------------------------- /tproxy/arbiter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | # 3 | # This file is part of tproxy released under the MIT license. 4 | # See the NOTICE for more information. 5 | 6 | import errno 7 | import logging 8 | import os 9 | import signal 10 | import sys 11 | import time 12 | import traceback 13 | 14 | import gevent 15 | from gevent import select 16 | 17 | from . import __version__ 18 | from . import util 19 | from .pidfile import Pidfile 20 | from .proxy import tcp_listener 21 | from .worker import Worker 22 | 23 | 24 | 25 | class HaltServer(Exception): 26 | def __init__(self, reason, exit_status=1): 27 | self.reason = reason 28 | self.exit_status = exit_status 29 | 30 | def __str__(self): 31 | return "" % (self.reason, self.exit_status) 32 | 33 | class Arbiter(object): 34 | 35 | WORKER_BOOT_ERROR = 3 36 | 37 | START_CTX = {} 38 | 39 | LISTENER = None 40 | WORKERS = {} 41 | PIPE = [] 42 | 43 | # I love dynamic languages 44 | SIG_QUEUE = [] 45 | SIGNALS = map( 46 | lambda x: getattr(signal, "SIG%s" % x), 47 | "HUP QUIT INT TERM TTIN TTOU USR1 USR2 WINCH".split() 48 | ) 49 | SIG_NAMES = dict( 50 | (getattr(signal, name), name[3:].lower()) for name in dir(signal) 51 | if name[:3] == "SIG" and name[3] != "_" 52 | ) 53 | 54 | def __init__(self, cfg, script): 55 | self.cfg = cfg 56 | self.script = script 57 | self.num_workers = cfg.workers 58 | self.address = cfg.address 59 | self.timeout = cfg.timeout 60 | self.name = cfg.name 61 | 62 | self.pid = 0 63 | self.pidfile = None 64 | self.worker_age = 0 65 | self.reexec_pid = 0 66 | self.master_name = "master" 67 | self.log = logging.getLogger(__name__) 68 | 69 | # get current path, try to use PWD env first 70 | try: 71 | a = os.stat(os.environ['PWD']) 72 | b = os.stat(os.getcwd()) 73 | if a.ino == b.ino and a.dev == b.dev: 74 | cwd = os.environ['PWD'] 75 | else: 76 | cwd = os.getcwd() 77 | except: 78 | cwd = os.getcwd() 79 | 80 | args = sys.argv[:] 81 | args.insert(0, sys.executable) 82 | 83 | # init start context 84 | self.START_CTX = { 85 | "args": args, 86 | "cwd": cwd, 87 | 0: sys.executable 88 | } 89 | 90 | def start(self): 91 | self.pid = os.getpid() 92 | self.init_signals() 93 | if not self.LISTENER: 94 | self.LISTENER = tcp_listener(self.address, self.cfg.backlog) 95 | 96 | if self.cfg.pidfile is not None: 97 | self.pidfile = Pidfile(self.cfg.pidfile) 98 | self.pidfile.create(self.pid) 99 | 100 | util._setproctitle("master [%s]" % self.name) 101 | self.log.info("tproxy %s started" % __version__) 102 | self.log.info("Listening on %s:%s" % self.address) 103 | 104 | def init_signals(self): 105 | """\ 106 | Initialize master signal handling. Most of the signals 107 | are queued. Child signals only wake up the master. 108 | """ 109 | if self.PIPE: 110 | map(os.close, self.PIPE) 111 | self.PIPE = pair = os.pipe() 112 | map(util.set_non_blocking, pair) 113 | map(util.close_on_exec, pair) 114 | map(lambda s: signal.signal(s, self.signal), self.SIGNALS) 115 | signal.signal(signal.SIGCHLD, self.handle_chld) 116 | 117 | def signal(self, sig, frame): 118 | if len(self.SIG_QUEUE) < 5: 119 | self.SIG_QUEUE.append(sig) 120 | self.wakeup() 121 | else: 122 | self.log.warn("Dropping signal: %s" % sig) 123 | 124 | def run(self): 125 | self.start() 126 | self.manage_workers() 127 | while True: 128 | try: 129 | self.reap_workers() 130 | sig = self.SIG_QUEUE.pop(0) if len(self.SIG_QUEUE) else None 131 | if sig is None: 132 | self.sleep() 133 | self.murder_workers() 134 | self.manage_workers() 135 | continue 136 | 137 | if sig not in self.SIG_NAMES: 138 | self.log.info("Ignoring unknown signal: %s" % sig) 139 | continue 140 | 141 | signame = self.SIG_NAMES.get(sig) 142 | handler = getattr(self, "handle_%s" % signame, None) 143 | if not handler: 144 | self.log.error("Unhandled signal: %s" % signame) 145 | continue 146 | self.log.info("Handling signal: %s" % signame) 147 | handler() 148 | self.wakeup() 149 | except StopIteration: 150 | self.halt() 151 | except KeyboardInterrupt: 152 | self.halt() 153 | except HaltServer, inst: 154 | self.halt(reason=inst.reason, exit_status=inst.exit_status) 155 | except SystemExit: 156 | raise 157 | except Exception: 158 | self.log.info("Unhandled exception in main loop:\n%s" % 159 | traceback.format_exc()) 160 | self.stop(False) 161 | if self.pidfile is not None: 162 | self.pidfile.unlink() 163 | sys.exit(-1) 164 | 165 | def handle_chld(self, *args): 166 | "SIGCHLD handling" 167 | self.wakeup() 168 | self.reap_workers() 169 | 170 | def handle_hup(self): 171 | """\ 172 | HUP handling. 173 | - Reload configuration 174 | - Start the new worker processes with a new configuration 175 | - Gracefully shutdown the old worker processes 176 | """ 177 | self.log.info("Hang up: %s" % self.master_name) 178 | self.reload() 179 | 180 | def handle_quit(self): 181 | "SIGQUIT handling" 182 | raise StopIteration 183 | 184 | def handle_int(self): 185 | "SIGINT handling" 186 | self.stop(False) 187 | raise StopIteration 188 | 189 | def handle_term(self): 190 | "SIGTERM handling" 191 | self.stop(False) 192 | raise StopIteration 193 | 194 | def handle_ttin(self): 195 | """\ 196 | SIGTTIN handling. 197 | Increases the number of workers by one. 198 | """ 199 | self.num_workers += 1 200 | self.manage_workers() 201 | 202 | def handle_ttou(self): 203 | """\ 204 | SIGTTOU handling. 205 | Decreases the number of workers by one. 206 | """ 207 | if self.num_workers <= 1: 208 | return 209 | self.num_workers -= 1 210 | self.manage_workers() 211 | 212 | def handle_usr1(self): 213 | """\ 214 | SIGUSR1 handling. 215 | Kill all workers by sending them a SIGUSR1 216 | """ 217 | self.kill_workers(signal.SIGUSR1) 218 | 219 | def handle_usr2(self): 220 | """\ 221 | SIGUSR2 handling. 222 | Creates a new master/worker set as a slave of the current 223 | master without affecting old workers. Use this to do live 224 | deployment with the ability to backout a change. 225 | """ 226 | self.reexec() 227 | 228 | def handle_winch(self): 229 | "SIGWINCH handling" 230 | if os.getppid() == 1 or os.getpgrp() != os.getpid(): 231 | self.log.info("graceful stop of workers") 232 | self.num_workers = 0 233 | self.kill_workers(signal.SIGQUIT) 234 | else: 235 | self.log.info("SIGWINCH ignored. Not daemonized") 236 | 237 | def wakeup(self): 238 | """\ 239 | Wake up the arbiter by writing to the PIPE 240 | """ 241 | try: 242 | os.write(self.PIPE[1], '.') 243 | except IOError, e: 244 | if e.errno not in [errno.EAGAIN, errno.EINTR]: 245 | raise 246 | 247 | def halt(self, reason=None, exit_status=0): 248 | """ halt arbiter """ 249 | self.stop() 250 | self.log.info("Shutting down: %s" % self.master_name) 251 | if reason is not None: 252 | self.log.info("Reason: %s" % reason) 253 | if self.pidfile is not None: 254 | self.pidfile.unlink() 255 | sys.exit(exit_status) 256 | 257 | def sleep(self): 258 | """\ 259 | Sleep until PIPE is readable or we timeout. 260 | A readable PIPE means a signal occurred. 261 | """ 262 | try: 263 | ready = select.select([self.PIPE[0]], [], [], 1.0) 264 | if not ready[0]: 265 | return 266 | while os.read(self.PIPE[0], 1): 267 | pass 268 | except select.error, e: 269 | if e[0] not in [errno.EAGAIN, errno.EINTR]: 270 | raise 271 | except OSError, e: 272 | if e.errno not in [errno.EAGAIN, errno.EINTR]: 273 | raise 274 | except KeyboardInterrupt: 275 | sys.exit() 276 | 277 | 278 | def stop(self, graceful=True): 279 | """\ 280 | Stop workers 281 | 282 | :attr graceful: boolean, If True (the default) workers will be 283 | killed gracefully (ie. trying to wait for the current connection) 284 | """ 285 | self.LISTENER = None 286 | sig = signal.SIGQUIT 287 | if not graceful: 288 | sig = signal.SIGTERM 289 | limit = time.time() + self.timeout 290 | while self.WORKERS and time.time() < limit: 291 | self.kill_workers(sig) 292 | gevent.sleep(0.1) 293 | self.reap_workers() 294 | self.kill_workers(signal.SIGKILL) 295 | 296 | def reexec(self): 297 | """\ 298 | Relaunch the master and workers. 299 | """ 300 | if self.pidfile is not None: 301 | self.pidfile.rename("%s.oldbin" % self.pidfile.fname) 302 | 303 | self.reexec_pid = os.fork() 304 | if self.reexec_pid != 0: 305 | self.master_name = "Old Master" 306 | return 307 | 308 | os.environ['TPROXY_FD'] = str(self.LISTENER.fileno()) 309 | os.chdir(self.START_CTX['cwd']) 310 | self.cfg.pre_exec(self) 311 | os.execvpe(self.START_CTX[0], self.START_CTX['args'], os.environ) 312 | 313 | def reload(self): 314 | # spawn new workers with new app & conf 315 | for i in range(self.cfg.workers): 316 | self.spawn_worker() 317 | 318 | # unlink pidfile 319 | if self.pidfile is not None: 320 | self.pidfile.unlink() 321 | 322 | # create new pidfile 323 | if self.cfg.pidfile is not None: 324 | self.pidfile = Pidfile(self.cfg.pidfile) 325 | self.pidfile.create(self.pid) 326 | 327 | # manage workers 328 | self.manage_workers() 329 | 330 | def murder_workers(self): 331 | """\ 332 | Kill unused/idle workers 333 | """ 334 | for (pid, worker) in self.WORKERS.items(): 335 | try: 336 | diff = time.time() - os.fstat(worker.tmp.fileno()).st_ctime 337 | if diff <= self.timeout: 338 | continue 339 | except ValueError: 340 | continue 341 | 342 | self.log.critical("WORKER TIMEOUT (pid:%s)" % pid) 343 | self.kill_worker(pid, signal.SIGKILL) 344 | 345 | def reap_workers(self): 346 | """\ 347 | Reap workers to avoid zombie processes 348 | """ 349 | try: 350 | while True: 351 | wpid, status = os.waitpid(-1, os.WNOHANG) 352 | if not wpid: 353 | break 354 | if self.reexec_pid == wpid: 355 | self.reexec_pid = 0 356 | else: 357 | # A worker said it cannot boot. We'll shutdown 358 | # to avoid infinite start/stop cycles. 359 | exitcode = status >> 8 360 | if exitcode == self.WORKER_BOOT_ERROR: 361 | reason = "Worker failed to boot." 362 | raise HaltServer(reason, self.WORKER_BOOT_ERROR) 363 | worker = self.WORKERS.pop(wpid, None) 364 | if not worker: 365 | continue 366 | worker.tmp.close() 367 | except OSError, e: 368 | if e.errno == errno.ECHILD: 369 | pass 370 | 371 | def manage_workers(self): 372 | """\ 373 | Maintain the number of workers by spawning or killing 374 | as required. 375 | """ 376 | if len(self.WORKERS.keys()) < self.num_workers: 377 | self.spawn_workers() 378 | 379 | num_to_kill = len(self.WORKERS) - self.num_workers 380 | for i in range(num_to_kill, 0, -1): 381 | pid, age = 0, sys.maxint 382 | for (wpid, worker) in self.WORKERS.iteritems(): 383 | if worker.age < age: 384 | pid, age = wpid, worker.age 385 | self.kill_worker(pid, signal.SIGQUIT) 386 | 387 | def spawn_worker(self): 388 | self.worker_age += 1 389 | worker = Worker(self.worker_age, self.pid, self.LISTENER, self.cfg, 390 | self.script) 391 | pid = os.fork() 392 | if pid != 0: 393 | self.WORKERS[pid] = worker 394 | return 395 | 396 | # Process Child 397 | worker_pid = os.getpid() 398 | try: 399 | self.log.info("Booting worker with pid: %s" % worker_pid) 400 | worker.serve_forever() 401 | sys.exit(0) 402 | except SystemExit: 403 | raise 404 | except KeyboardInterrupt: 405 | sys.exit(0) 406 | except: 407 | self.log.exception("Exception in worker process:") 408 | if not worker.booted: 409 | sys.exit(self.WORKER_BOOT_ERROR) 410 | sys.exit(-1) 411 | finally: 412 | self.log.info("Worker exiting (pid: %s)" % worker_pid) 413 | try: 414 | worker.tmp.close() 415 | except: 416 | pass 417 | 418 | def spawn_workers(self): 419 | """\ 420 | Spawn new workers as needed. 421 | 422 | This is where a worker process leaves the main loop 423 | of the master process. 424 | """ 425 | 426 | for i in range(self.num_workers - len(self.WORKERS.keys())): 427 | self.spawn_worker() 428 | 429 | def kill_workers(self, sig): 430 | """\ 431 | Lill all workers with the signal `sig` 432 | :attr sig: `signal.SIG*` value 433 | """ 434 | for pid in self.WORKERS.keys(): 435 | self.kill_worker(pid, sig) 436 | 437 | def kill_worker(self, pid, sig): 438 | """\ 439 | Kill a worker 440 | 441 | :attr pid: int, worker pid 442 | :attr sig: `signal.SIG*` value 443 | """ 444 | try: 445 | os.kill(pid, sig) 446 | except OSError, e: 447 | if e.errno == errno.ESRCH: 448 | try: 449 | worker = self.WORKERS.pop(pid) 450 | worker.tmp.close() 451 | return 452 | except (KeyError, OSError): 453 | return 454 | raise 455 | -------------------------------------------------------------------------------- /tproxy/client.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | # 3 | # This file is part of tproxy released under the MIT license. 4 | # See the NOTICE for more information. 5 | 6 | import logging 7 | import os 8 | import ssl 9 | 10 | import gevent 11 | from gevent import coros 12 | from gevent import socket 13 | import greenlet 14 | 15 | from .server import ServerConnection, InactivityTimeout 16 | from .util import parse_address, is_ipv6 17 | from .sendfile import async_sendfile 18 | 19 | log = logging.getLogger(__name__) 20 | 21 | class ConnectionError(Exception): 22 | """ Exception raised when a connection is either rejected or a 23 | connection timeout occurs """ 24 | 25 | class ClientConnection(object): 26 | 27 | def __init__(self, sock, addr, worker): 28 | self.sock = sock 29 | self.addr = addr 30 | self.worker = worker 31 | 32 | self.route = self.worker.route 33 | self.buf = [] 34 | self.remote = None 35 | self.connected = False 36 | self._lock = coros.Semaphore() 37 | 38 | def handle(self): 39 | with self._lock: 40 | self.worker.nb_connections +=1 41 | self.worker.refresh_name() 42 | 43 | try: 44 | while not self.connected: 45 | data = self.sock.recv(1024) 46 | if not data: 47 | break 48 | self.buf.append(data) 49 | if self.remote is None: 50 | try: 51 | self.do_proxy() 52 | except StopIteration: 53 | break 54 | except ConnectionError, e: 55 | log.error("Error while connecting: [%s]" % str(e)) 56 | self.handle_error(e) 57 | except InactivityTimeout, e: 58 | log.warn("inactivity timeout") 59 | self.handle_error(e) 60 | except socket.error, e: 61 | log.error("socket.error: [%s]" % str(e)) 62 | self.handle_error(e) 63 | except greenlet.GreenletExit: 64 | pass 65 | except KeyboardInterrupt: 66 | pass 67 | except Exception, e: 68 | log.error("unknown error %s" % str(e)) 69 | finally: 70 | if self.remote is not None: 71 | log.debug("Close connection to %s:%s" % self.remote) 72 | 73 | with self._lock: 74 | self.worker.nb_connections -=1 75 | self.worker.refresh_name() 76 | _closesocket(self.sock) 77 | 78 | def handle_error(self, e): 79 | if hasattr(self.route, 'proxy_error'): 80 | self.route.proxy_error(self, e) 81 | 82 | def do_proxy(self): 83 | commands = self.route.proxy("".join(self.buf)) 84 | if commands is None: # do nothing 85 | return 86 | 87 | 88 | if not isinstance(commands, dict): 89 | raise StopIteration 90 | 91 | if 'remote' in commands: 92 | remote = parse_address(commands['remote']) 93 | if 'data' in commands: 94 | self.buf = [commands['data']] 95 | if 'reply' in commands: 96 | self.send_data(self.sock, commands['reply']) 97 | 98 | is_ssl = commands.get('ssl', False) 99 | ssl_args = commands.get('ssl_args', {}) 100 | extra = commands.get('extra') 101 | connect_timeout = commands.get('connect_timeout') 102 | inactivity_timeout = commands.get('inactivity_timeout') 103 | self.connect_to_resource(remote, is_ssl=is_ssl, connect_timeout=connect_timeout, 104 | inactivity_timeout=inactivity_timeout, extra=extra, 105 | **ssl_args) 106 | 107 | elif 'close' in commands: 108 | if isinstance(commands['close'], basestring): 109 | self.send_data(self.sock, commands['close']) 110 | raise StopIteration() 111 | 112 | elif 'file' in commands: 113 | # command to send a file 114 | if isinstance(commands['file'], basestring): 115 | fdin = os.open(commands['file'], os.O_RDONLY) 116 | else: 117 | fdin = commands['file'] 118 | 119 | offset = commands.get('offset', 0) 120 | nbytes = commands.get('nbytes', os.fstat(fdin).st_size) 121 | 122 | # send a reply if needed, useful in HTTP response. 123 | if 'reply' in commands: 124 | self.send_data(self.sock, commands['reply']) 125 | 126 | # use sendfile if possible to send the file content 127 | async_sendfile(self.sock.fileno(), fdin, offset, nbytes) 128 | raise StopIteration() 129 | else: 130 | raise StopIteration() 131 | 132 | def send_data(self, sock, data): 133 | if hasattr(data, 'read'): 134 | try: 135 | data.seek(0) 136 | except (ValueError, IOError): 137 | pass 138 | 139 | while True: 140 | chunk = data.readline() 141 | if not chunk: 142 | break 143 | sock.sendall(chunk) 144 | elif isinstance(data, basestring): 145 | sock.sendall(data) 146 | else: 147 | for chunk in data: 148 | sock.sendall(chunk) 149 | 150 | def connect_to_resource(self, addr, is_ssl=False, connect_timeout=None, 151 | inactivity_timeout=None, extra=None, **ssl_args): 152 | 153 | with gevent.Timeout(connect_timeout, ConnectionError): 154 | try: 155 | if is_ipv6(addr[0]): 156 | sock = socket.socket(socket.AF_INET6, 157 | socket.SOCK_STREAM) 158 | else: 159 | sock = socket.socket(socket.AF_INET, 160 | socket.SOCK_STREAM) 161 | 162 | if is_ssl: 163 | sock = ssl.wrap_socket(sock, **ssl_args) 164 | sock.connect(addr) 165 | except socket.error, e: 166 | raise ConnectionError( 167 | "socket error while connectinng: [%s]" % str(e)) 168 | 169 | self.remote = addr 170 | self.connected = True 171 | log.debug("Successful connection to %s:%s" % addr) 172 | 173 | if self.buf and self.route.empty_buf: 174 | self.send_data(sock, self.buf) 175 | self.buf = [] 176 | 177 | server = ServerConnection(sock, self, 178 | timeout=inactivity_timeout, extra=extra, buf=self.buf) 179 | server.handle() 180 | 181 | def _closesocket(sock): 182 | try: 183 | sock._sock.close() 184 | sock.close() 185 | except socket.error: 186 | pass 187 | -------------------------------------------------------------------------------- /tproxy/config.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | # 3 | # This file is part of tproxy released under the MIT license. 4 | # See the NOTICE for more information. 5 | 6 | class ConfigError(Exception): 7 | """ Exception raised on config error """ 8 | 9 | 10 | # -*- coding: utf-8 - 11 | # 12 | # This file is part of gunicorn released under the MIT license. 13 | # See the NOTICE for more information. 14 | 15 | import copy 16 | import grp 17 | import inspect 18 | import optparse 19 | import os 20 | import pwd 21 | import textwrap 22 | import types 23 | 24 | from . import __version__ 25 | from . import util 26 | 27 | KNOWN_SETTINGS = [] 28 | 29 | class ConfigError(Exception): 30 | """ Exception raised on config error """ 31 | 32 | def wrap_method(func): 33 | def _wrapped(instance, *args, **kwargs): 34 | return func(*args, **kwargs) 35 | return _wrapped 36 | 37 | def make_settings(ignore=None): 38 | settings = {} 39 | ignore = ignore or () 40 | for s in KNOWN_SETTINGS: 41 | setting = s() 42 | if setting.name in ignore: 43 | continue 44 | settings[setting.name] = setting.copy() 45 | return settings 46 | 47 | class Config(object): 48 | 49 | def __init__(self, usage=None): 50 | self.settings = make_settings() 51 | self.usage = usage 52 | self.default_name = None 53 | 54 | def __getattr__(self, name): 55 | if name not in self.settings: 56 | raise AttributeError("No configuration setting for: %s" % name) 57 | return self.settings[name].get() 58 | 59 | def __setattr__(self, name, value): 60 | if name != "settings" and name in self.settings: 61 | raise AttributeError("Invalid access!") 62 | super(Config, self).__setattr__(name, value) 63 | 64 | def set(self, name, value): 65 | if name not in self.settings: 66 | raise AttributeError("No configuration setting for: %s" % name) 67 | self.settings[name].set(value) 68 | 69 | def parser(self): 70 | kwargs = { 71 | "usage": self.usage, 72 | "version": __version__ 73 | } 74 | parser = optparse.OptionParser(**kwargs) 75 | 76 | keys = self.settings.keys() 77 | def sorter(k): 78 | return (self.settings[k].section, self.settings[k].order) 79 | keys.sort(key=sorter) 80 | for k in keys: 81 | self.settings[k].add_option(parser) 82 | return parser 83 | 84 | @property 85 | def workers(self): 86 | return self.settings['workers'].get() 87 | 88 | @property 89 | def address(self): 90 | bind = self.settings['bind'].get() 91 | return util.parse_address(str(bind)) 92 | 93 | @property 94 | def uid(self): 95 | return self.settings['user'].get() 96 | 97 | @property 98 | def gid(self): 99 | return self.settings['group'].get() 100 | 101 | @property 102 | def name(self): 103 | pn = self.settings['name'].get() 104 | if pn is not None: 105 | return pn 106 | else: 107 | return self.default_name or "" 108 | 109 | class SettingMeta(type): 110 | def __new__(cls, name, bases, attrs): 111 | super_new = super(SettingMeta, cls).__new__ 112 | parents = [b for b in bases if isinstance(b, SettingMeta)] 113 | if not parents: 114 | return super_new(cls, name, bases, attrs) 115 | 116 | attrs["order"] = len(KNOWN_SETTINGS) 117 | attrs["validator"] = wrap_method(attrs["validator"]) 118 | 119 | new_class = super_new(cls, name, bases, attrs) 120 | new_class.fmt_desc(attrs.get("desc", "")) 121 | KNOWN_SETTINGS.append(new_class) 122 | return new_class 123 | 124 | def fmt_desc(cls, desc): 125 | desc = textwrap.dedent(desc).strip() 126 | setattr(cls, "desc", desc) 127 | setattr(cls, "short", desc.splitlines()[0]) 128 | 129 | class Setting(object): 130 | __metaclass__ = SettingMeta 131 | 132 | name = None 133 | value = None 134 | section = None 135 | cli = None 136 | validator = None 137 | type = None 138 | meta = None 139 | action = None 140 | default = None 141 | short = None 142 | desc = None 143 | 144 | def __init__(self): 145 | if self.default is not None: 146 | self.set(self.default) 147 | 148 | def add_option(self, parser): 149 | if not self.cli: 150 | return 151 | args = tuple(self.cli) 152 | kwargs = { 153 | "dest": self.name, 154 | "metavar": self.meta or None, 155 | "action": self.action or "store", 156 | "type": self.type or "string", 157 | "default": None, 158 | "help": "%s [%s]" % (self.short, self.default) 159 | } 160 | if kwargs["action"] != "store": 161 | kwargs.pop("type") 162 | parser.add_option(*args, **kwargs) 163 | 164 | def copy(self): 165 | return copy.copy(self) 166 | 167 | def get(self): 168 | return self.value 169 | 170 | def set(self, val): 171 | assert callable(self.validator), "Invalid validator: %s" % self.name 172 | self.value = self.validator(val) 173 | 174 | def validate_bool(val): 175 | if isinstance(val, types.BooleanType): 176 | return val 177 | if not isinstance(val, basestring): 178 | raise TypeError("Invalid type for casting: %s" % val) 179 | if val.lower().strip() == "true": 180 | return True 181 | elif val.lower().strip() == "false": 182 | return False 183 | else: 184 | raise ValueError("Invalid boolean: %s" % val) 185 | 186 | def validate_pos_int(val): 187 | if not isinstance(val, (types.IntType, types.LongType)): 188 | val = int(val, 0) 189 | else: 190 | # Booleans are ints! 191 | val = int(val) 192 | if val < 0: 193 | raise ValueError("Value must be positive: %s" % val) 194 | return val 195 | 196 | def validate_string(val): 197 | if val is None: 198 | return None 199 | if not isinstance(val, basestring): 200 | raise TypeError("Not a string: %s" % val) 201 | return val.strip() 202 | 203 | def validate_callable(arity): 204 | def _validate_callable(val): 205 | if not callable(val): 206 | raise TypeError("Value is not callable: %s" % val) 207 | if arity != len(inspect.getargspec(val)[0]): 208 | raise TypeError("Value must have an arity of: %s" % arity) 209 | return val 210 | return _validate_callable 211 | 212 | 213 | def validate_user(val): 214 | if val is None: 215 | return os.geteuid() 216 | if isinstance(val, int): 217 | return val 218 | elif val.isdigit(): 219 | return int(val) 220 | else: 221 | try: 222 | return pwd.getpwnam(val).pw_uid 223 | except KeyError: 224 | raise ConfigError("No such user: '%s'" % val) 225 | 226 | def validate_group(val): 227 | if val is None: 228 | return os.getegid() 229 | 230 | if isinstance(val, int): 231 | return val 232 | elif val.isdigit(): 233 | return int(val) 234 | else: 235 | try: 236 | return grp.getgrnam(val).gr_gid 237 | except KeyError: 238 | raise ConfigError("No such group: '%s'" % val) 239 | 240 | 241 | class Bind(Setting): 242 | name = "bind" 243 | section = "Server Socket" 244 | cli = ["-b", "--bind"] 245 | meta = "ADDRESS" 246 | validator = validate_string 247 | default = "127.0.0.1:5000" 248 | desc = """\ 249 | The socket to bind. 250 | 251 | A string of the form: 'HOST', 'HOST:PORT', 'unix:PATH'. An IP is a valid 252 | HOST. 253 | """ 254 | 255 | class Backlog(Setting): 256 | name = "backlog" 257 | section = "Server Socket" 258 | cli = ["--backlog"] 259 | meta = "INT" 260 | validator = validate_pos_int 261 | type = "int" 262 | default = 2048 263 | desc = """\ 264 | The maximum number of pending connections. 265 | 266 | This refers to the number of clients that can be waiting to be served. 267 | Exceeding this number results in the client getting an error when 268 | attempting to connect. It should only affect servers under significant 269 | load. 270 | 271 | Must be a positive integer. Generally set in the 64-2048 range. 272 | """ 273 | 274 | class Workers(Setting): 275 | name = "workers" 276 | section = "Worker Processes" 277 | cli = ["-w", "--workers"] 278 | meta = "INT" 279 | validator = validate_pos_int 280 | type = "int" 281 | default = 1 282 | desc = """\ 283 | The number of worker process for handling requests. 284 | 285 | A positive integer generally in the 2-4 x $(NUM_CORES) range. You'll 286 | want to vary this a bit to find the best for your particular 287 | application's work load. 288 | """ 289 | class WorkerConnections(Setting): 290 | name = "worker_connections" 291 | section = "Worker Processes" 292 | cli = ["--worker-connections"] 293 | meta = "INT" 294 | validator = validate_pos_int 295 | type = "int" 296 | default = 1000 297 | desc = """\ 298 | The maximum number of simultaneous clients per worker. 299 | """ 300 | 301 | class Timeout(Setting): 302 | name = "timeout" 303 | section = "Worker Processes" 304 | cli = ["-t", "--timeout"] 305 | meta = "INT" 306 | validator = validate_pos_int 307 | type = "int" 308 | default = 30 309 | desc = """\ 310 | Workers silent for more than this many seconds are killed and restarted. 311 | 312 | Generally set to thirty seconds. Only set this noticeably higher if 313 | you're sure of the repercussions for sync workers. For the non sync 314 | workers it just means that the worker process is still communicating and 315 | is not tied to the length of time required to handle a single request. 316 | """ 317 | 318 | class Daemon(Setting): 319 | name = "daemon" 320 | section = "Server Mechanics" 321 | cli = ["-D", "--daemon"] 322 | validator = validate_bool 323 | action = "store_true" 324 | default = False 325 | desc = """\ 326 | Daemonize the tproxy process. 327 | 328 | Detaches the server from the controlling terminal and enters the 329 | background. 330 | """ 331 | 332 | class Pidfile(Setting): 333 | name = "pidfile" 334 | section = "Server Mechanics" 335 | cli = ["-p", "--pid"] 336 | meta = "FILE" 337 | validator = validate_string 338 | default = None 339 | desc = """\ 340 | A filename to use for the PID file. 341 | 342 | If not set, no PID file will be written. 343 | """ 344 | 345 | class User(Setting): 346 | name = "user" 347 | section = "Server Mechanics" 348 | cli = ["-u", "--user"] 349 | meta = "USER" 350 | validator = validate_user 351 | default = os.geteuid() 352 | desc = """\ 353 | Switch worker processes to run as this user. 354 | 355 | A valid user id (as an integer) or the name of a user that can be 356 | retrieved with a call to pwd.getpwnam(value) or None to not change 357 | the worker process user. 358 | """ 359 | 360 | class Group(Setting): 361 | name = "group" 362 | section = "Server Mechanics" 363 | cli = ["-g", "--group"] 364 | meta = "GROUP" 365 | validator = validate_group 366 | default = os.getegid() 367 | desc = """\ 368 | Switch worker process to run as this group. 369 | 370 | A valid group id (as an integer) or the name of a user that can be 371 | retrieved with a call to pwd.getgrnam(value) or None to not change 372 | the worker processes group. 373 | """ 374 | 375 | class Umask(Setting): 376 | name = "umask" 377 | section = "Server Mechanics" 378 | cli = ["-m", "--umask"] 379 | meta = "INT" 380 | validator = validate_pos_int 381 | type = "int" 382 | default = 0 383 | desc = """\ 384 | A bit mask for the file mode on files written by tproxy. 385 | 386 | Note that this affects unix socket permissions. 387 | 388 | A valid value for the os.umask(mode) call or a string compatible with 389 | int(value, 0) (0 means Python guesses the base, so values like "0", 390 | "0xFF", "0022" are valid for decimal, hex, and octal representations) 391 | """ 392 | 393 | class Logfile(Setting): 394 | name = "logfile" 395 | section = "Logging" 396 | cli = ["--log-file"] 397 | meta = "FILE" 398 | validator = validate_string 399 | default = "-" 400 | desc = """\ 401 | The log file to write to. 402 | 403 | "-" means log to stdout. 404 | """ 405 | 406 | class Loglevel(Setting): 407 | name = "loglevel" 408 | section = "Logging" 409 | cli = ["--log-level"] 410 | meta = "LEVEL" 411 | validator = validate_string 412 | default = "info" 413 | desc = """\ 414 | The granularity of log outputs. 415 | 416 | Valid level names are: 417 | 418 | * debug 419 | * info 420 | * warning 421 | * error 422 | * critical 423 | """ 424 | 425 | class LogConfig(Setting): 426 | name = "logconfig" 427 | section = "Logging" 428 | cli = ["--log-config"] 429 | meta = "FILE" 430 | validator = validate_string 431 | default = None 432 | desc = """\ 433 | The log config file to use. 434 | 435 | tproxy uses the standard Python logging module's Configuration 436 | file format. 437 | """ 438 | 439 | class Procname(Setting): 440 | name = "name" 441 | section = "Process Naming" 442 | cli = ["-n", "--name"] 443 | meta = "STRING" 444 | validator = validate_string 445 | default = None 446 | desc = """\ 447 | A base to use with setproctitle for process naming. 448 | 449 | This affects things like ``ps`` and ``top``. If you're going to be 450 | running more than one instance of tproxy you'll probably want to set a 451 | name to tell them apart. This requires that you install the setproctitle 452 | module. 453 | 454 | It defaults to 'tproxy'. 455 | """ 456 | 457 | class SslKeyFile(Setting): 458 | name = "ssl_keyfile" 459 | section = "Ssl" 460 | cli = ["--ssl-keyfile"] 461 | validator = validate_string 462 | meta = "STRING" 463 | default = None 464 | desc = """\ 465 | Ssl key file 466 | """ 467 | 468 | class SslCertFile(Setting): 469 | name = "ssl_certfile" 470 | section = "Ssl" 471 | cli = ["--ssl-certfile"] 472 | validator = validate_string 473 | meta = "STRING" 474 | default = None 475 | desc = """\ 476 | Ssl cert file 477 | """ 478 | 479 | class SslCertFile(Setting): 480 | name = "ssl_certfile" 481 | section = "Ssl" 482 | cli = ["--ssl-certfile"] 483 | validator = validate_string 484 | meta = "STRING" 485 | default = None 486 | desc = """\ 487 | Ssl ca certs file. contains concatenated "certification 488 | authority" certificates. 489 | """ 490 | 491 | class SslCACerts(Setting): 492 | name = "ssl_ca_certs" 493 | section = "Ssl" 494 | cli = ["--ssl-ca-certs"] 495 | validator = validate_string 496 | meta = "STRING" 497 | default = None 498 | desc = """\ 499 | Ssl ca certs file. contains concatenated "certification 500 | authority" certificates. 501 | """ 502 | 503 | class SSLCertReq(Setting): 504 | name = "ssl_cert_reqs" 505 | section = "Ssl" 506 | cli = ["--ssl-cert-reqs"] 507 | validator = validate_pos_int 508 | meta = "INT" 509 | default = 0 510 | desc = """\ 511 | Specifies whether a certificate is required from the other 512 | side of the connection, and whether it will be validated if 513 | provided. Values are: 0 (certificates ignored), 1 (not 514 | required, but validated if provided), 2 (required and 515 | validated). 516 | """ 517 | 518 | -------------------------------------------------------------------------------- /tproxy/pidfile.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | # 3 | # This file is part of gunicorn released under the MIT license. 4 | # See the NOTICE for more information. 5 | 6 | from __future__ import with_statement 7 | 8 | import errno 9 | import os 10 | import tempfile 11 | 12 | 13 | class Pidfile(object): 14 | """\ 15 | Manage a PID file. If a specific name is provided 16 | it and '"%s.oldpid" % name' will be used. Otherwise 17 | we create a temp file using os.mkstemp. 18 | """ 19 | 20 | def __init__(self, fname): 21 | self.fname = fname 22 | self.pid = None 23 | 24 | def create(self, pid): 25 | oldpid = self.validate() 26 | if oldpid: 27 | if oldpid == os.getpid(): 28 | return 29 | raise RuntimeError("Already running on PID %s " \ 30 | "(or pid file '%s' is stale)" % (os.getpid(), self.fname)) 31 | 32 | self.pid = pid 33 | 34 | # Write pidfile 35 | fdir = os.path.dirname(self.fname) 36 | if fdir and not os.path.isdir(fdir): 37 | raise RuntimeError("%s doesn't exist. Can't create pidfile." % fdir) 38 | fd, fname = tempfile.mkstemp(dir=fdir) 39 | os.write(fd, "%s\n" % self.pid) 40 | if self.fname: 41 | os.rename(fname, self.fname) 42 | else: 43 | self.fname = fname 44 | os.close(fd) 45 | 46 | # set permissions to -rw-r--r-- 47 | os.chmod(self.fname, 420) 48 | 49 | def rename(self, path): 50 | self.unlink() 51 | self.fname = path 52 | self.create(self.pid) 53 | 54 | def unlink(self): 55 | """ delete pidfile""" 56 | try: 57 | with open(self.fname, "r") as f: 58 | pid1 = int(f.read() or 0) 59 | 60 | if pid1 == self.pid: 61 | os.unlink(self.fname) 62 | except: 63 | pass 64 | 65 | def validate(self): 66 | """ Validate pidfile and make it stale if needed""" 67 | if not self.fname: 68 | return 69 | try: 70 | with open(self.fname, "r") as f: 71 | wpid = int(f.read() or 0) 72 | 73 | if wpid <= 0: 74 | return 75 | 76 | try: 77 | os.kill(wpid, 0) 78 | return wpid 79 | except OSError, e: 80 | if e[0] == errno.ESRCH: 81 | return 82 | raise 83 | except IOError, e: 84 | if e[0] == errno.ENOENT: 85 | return 86 | raise 87 | -------------------------------------------------------------------------------- /tproxy/proxy.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | # 3 | # This file is part of tproxy released under the MIT license. 4 | # See the NOTICE for more information. 5 | 6 | import errno 7 | import logging 8 | import os 9 | import signal 10 | import sys 11 | import time 12 | 13 | import gevent 14 | from gevent.server import StreamServer 15 | from gevent import socket 16 | 17 | # we patch all 18 | from gevent import monkey 19 | monkey.noisy = False 20 | monkey.patch_all() 21 | 22 | 23 | from .client import ClientConnection 24 | from .route import Route 25 | from . import util 26 | 27 | log = logging.getLogger(__name__) 28 | 29 | 30 | class ProxyServer(StreamServer): 31 | 32 | def __init__(self, listener, script, backlog=None, 33 | spawn='default', **sslargs): 34 | StreamServer.__init__(self, listener, backlog=backlog, 35 | spawn=spawn, **sslargs) 36 | 37 | self.script = script 38 | self.nb_connections = 0 39 | self.route = None 40 | self.rewrite_request = None 41 | self.rewrite_response = None 42 | 43 | def handle_quit(self, *args): 44 | """Graceful shutdown. Stop accepting connections immediately and 45 | wait as long as necessary for all connections to close. 46 | """ 47 | gevent.spawn(self.stop) 48 | 49 | def handle_exit(self, *args): 50 | """ Fast shutdown.Stop accepting connection immediatly and wait 51 | up to 10 seconds for connections to close before forcing the 52 | termination 53 | """ 54 | gevent.spawn(self.stop, 10.0) 55 | 56 | def handle_winch(self, *args): 57 | # Ignore SIGWINCH in worker. Fixes a crash on OpenBSD. 58 | return 59 | 60 | def pre_start(self): 61 | """ create socket if needed and bind SIGKILL, SIGINT & SIGTERM 62 | signals 63 | """ 64 | # setup the socket 65 | if not hasattr(self, 'socket'): 66 | self.socket = tcp_listener(self.address, self.backlog) 67 | self.address = self.socket.getsockname() 68 | self._stopped_event.clear() 69 | 70 | # make SSL work: 71 | if self.ssl_enabled: 72 | self._handle = self.wrap_socket_and_handle 73 | else: 74 | self._handle = self.handle 75 | 76 | # handle signals 77 | signal.signal(signal.SIGQUIT, self.handle_quit) 78 | signal.signal(signal.SIGTERM, self.handle_exit) 79 | signal.signal(signal.SIGINT, self.handle_exit) 80 | signal.signal(signal.SIGWINCH, self.handle_winch) 81 | 82 | def init_route(self): 83 | if self.route is not None: 84 | return 85 | 86 | self.route = Route(self.script) 87 | 88 | def start_accepting(self): 89 | self.init_route() 90 | super(ProxyServer, self).start_accepting() 91 | 92 | def handle(self, socket, address): 93 | """ handle the connection """ 94 | conn = ClientConnection(socket, address, self) 95 | conn.handle() 96 | 97 | def wrap_socket_and_handle(self, client_socket, address): 98 | # used in case of ssl sockets 99 | ssl_socket = self.wrap_socket(client_socket, **self.ssl_args) 100 | return self.handle(ssl_socket, address) 101 | 102 | def tcp_listener(address, backlog=None): 103 | backlog = backlog or 128 104 | 105 | if util.is_ipv6(address[0]): 106 | family = socket.AF_INET6 107 | else: 108 | family = socket.AF_INET 109 | 110 | bound = False 111 | if 'TPROXY_FD' in os.environ: 112 | fd = int(os.environ.pop('TPROXY_FD')) 113 | try: 114 | sock = socket.fromfd(fd, family, socket.SOCK_STREAM) 115 | except socket.error, e: 116 | if e[0] == errno.ENOTCONN: 117 | log.error("TPROXY_FD should refer to an open socket.") 118 | else: 119 | raise 120 | bound = True 121 | else: 122 | sock = socket.socket(family, socket.SOCK_STREAM) 123 | 124 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 125 | 126 | for i in range(5): 127 | try: 128 | if not bound: 129 | sock.bind(address) 130 | sock.setblocking(0) 131 | sock.listen(backlog) 132 | return sock 133 | except socket.error, e: 134 | if e[0] == errno.EADDRINUSE: 135 | log.error("Connection in use: %s" % str(address)) 136 | if e[0] == errno.EADDRNOTAVAIL: 137 | log.error("Invalid address: %s" % str(address)) 138 | sys.exit(1) 139 | if i < 5: 140 | log.error("Retrying in 1 second. %s" % str(e)) 141 | time.sleep(1) 142 | -------------------------------------------------------------------------------- /tproxy/rewrite.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | # 3 | # This file is part of tproxy released under the MIT license. 4 | # See the NOTICE for more information. 5 | 6 | import sys 7 | 8 | # backports socketio 9 | import io 10 | import inspect 11 | import socket 12 | 13 | try: 14 | import errno 15 | except ImportError: 16 | errno = None 17 | EBADF = getattr(errno, 'EBADF', 9) 18 | EINTR = getattr(errno, 'EINTR', 4) 19 | EAGAIN = getattr(errno, 'EAGAIN', 11) 20 | EWOULDBLOCK = getattr(errno, 'EWOULDBLOCK', 11) 21 | 22 | _blocking_errnos = ( EAGAIN, EWOULDBLOCK, EBADF) 23 | 24 | if sys.version_info[:2] < (2, 7): 25 | # in python 2.6 socket.recv_into doesn't support bytesarray 26 | def _readinto(sock, b): 27 | while True: 28 | try: 29 | data = sock.recv(len(b)) 30 | recved = len(data) 31 | b[0:recved] = data 32 | return recved 33 | except socket.error as e: 34 | n = e.args[0] 35 | 36 | if n == EINTR: 37 | continue 38 | if n in _blocking_errnos: 39 | return None 40 | raise 41 | _get_memory = buffer 42 | else: 43 | _readinto = None 44 | def _get_memory(string, offset): 45 | return memoryview(string)[offset:] 46 | 47 | class RewriteIO(io.RawIOBase): 48 | 49 | """Raw I/O implementation for stream sockets. 50 | 51 | It provides the raw I/O interface on top of a socket object. 52 | Backported from python 3. 53 | """ 54 | 55 | 56 | def __init__(self, src, dest, buf=None): 57 | 58 | io.RawIOBase.__init__(self) 59 | self._src = src 60 | self._dest = dest 61 | 62 | if not buf: 63 | buf = [] 64 | self._buf = buf 65 | 66 | def readinto(self, b): 67 | self._checkClosed() 68 | self._checkReadable() 69 | 70 | buf = bytes("".join(self._buf)) 71 | 72 | if buf and buf is not None: 73 | l = len(b) 74 | if len(self._buf) > l: 75 | del b[l:] 76 | 77 | b[0:l], buf = buf[:l], buf[l:] 78 | self._buf = [buf] 79 | return len(b) 80 | else: 81 | length = len(buf) 82 | del b[length:] 83 | b[0:length] = buf 84 | self._buf = [] 85 | return len(b) 86 | 87 | if _readinto is not None: 88 | return _readinto(self._src, b) 89 | 90 | while True: 91 | try: 92 | return self._src.recv_into(b) 93 | except socket.error as e: 94 | n = e.args[0] 95 | if n == EINTR: 96 | continue 97 | if n in _blocking_errnos: 98 | return None 99 | raise 100 | 101 | def write(self, b): 102 | self._checkClosed() 103 | self._checkWritable() 104 | 105 | try: 106 | return self._dest.send(bytes(b)) 107 | except socket.error as e: 108 | # XXX what about EINTR? 109 | if e.args[0] in _blocking_errnos: 110 | return None 111 | raise 112 | 113 | def writeall(self, b): 114 | sent = 0 115 | while sent < len(b): 116 | sent += self._dest.send(_get_memory(b, sent)) 117 | 118 | def readable(self): 119 | """True if the SocketIO is open for reading. 120 | """ 121 | return not self.closed 122 | 123 | def writable(self): 124 | """True if the SocketIO is open for writing. 125 | """ 126 | return not self.closed 127 | 128 | def recv(self, n=None): 129 | return self.read(n) 130 | 131 | def send(self, b): 132 | return self.write(b) 133 | 134 | def sendall(self, b): 135 | return self.writeall(b) 136 | 137 | 138 | class RewriteProxy(object): 139 | 140 | def __init__(self, src, dest, rewrite_fun, timeout=None, 141 | extra=None, buf=None): 142 | self.src = src 143 | self.dest = dest 144 | self.rewrite_fun = rewrite_fun 145 | self.timeout = timeout 146 | self.buf = buf 147 | self.extra = extra 148 | 149 | def run(self): 150 | pipe = RewriteIO(self.src, self.dest, self.buf) 151 | spec = inspect.getargspec(self.rewrite_fun) 152 | try: 153 | if len(spec.args) > 1: 154 | self.rewrite_fun(pipe, self.extra) 155 | else: 156 | self.rewrite_fun(pipe) 157 | finally: 158 | pipe.close() 159 | 160 | 161 | -------------------------------------------------------------------------------- /tproxy/route.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | # 3 | # This file is part of tproxy released under the MIT license. 4 | # See the NOTICE for more information. 5 | 6 | import io 7 | import logging 8 | 9 | from .rewrite import RewriteProxy 10 | 11 | class Route(object): 12 | """ toute object to handle real proxy """ 13 | 14 | def __init__(self, script): 15 | if hasattr(script, "load"): 16 | self.script = script.load() 17 | else: 18 | self.script = script 19 | 20 | self.empty_buf = True 21 | if hasattr(self.script, 'rewrite_request'): 22 | self.proxy_input = self.rewrite_request 23 | self.empty_buf = False 24 | else: 25 | self.proxy_input = self.proxy_io 26 | 27 | if hasattr(self.script, 'rewrite_response'): 28 | self.proxy_connected = self.rewrite_response 29 | else: 30 | self.proxy_connected = self.proxy_io 31 | 32 | self.log = logging.getLogger(__name__) 33 | 34 | def proxy(self, data): 35 | return self.script.proxy(data) 36 | 37 | def proxy_io(self, src, dest, buf=None, extra=None): 38 | while True: 39 | data = src.recv(io.DEFAULT_BUFFER_SIZE) 40 | if not data: 41 | break 42 | self.log.debug("got data from input") 43 | dest.sendall(data) 44 | 45 | def rewrite(self, src, dest, fun, buf=None, extra=None): 46 | rwproxy = RewriteProxy(src, dest, fun, extra=extra, buf=buf) 47 | rwproxy.run() 48 | 49 | def rewrite_request(self, src, dest, buf=None, extra=None): 50 | self.rewrite(src, dest, self.script.rewrite_request, buf=buf, 51 | extra=extra) 52 | 53 | def rewrite_response(self, src, dest, extra=None): 54 | self.rewrite(src, dest, self.script.rewrite_response, 55 | extra=extra) 56 | -------------------------------------------------------------------------------- /tproxy/sendfile.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | # 3 | # This file is part of tproxy released under the MIT license. 4 | # See the NOTICE for more information. 5 | 6 | import errno 7 | import io 8 | import os 9 | try: 10 | from os import sendfile 11 | except ImportError: 12 | try: 13 | from _sendfile import sendfile 14 | except ImportError: 15 | def sendfile(fdout, fdin, offset, nbytes): 16 | fsize = os.fstat(fdin).st_size 17 | 18 | # max to send 19 | length = min(fsize-offset, nbytes) 20 | 21 | with os.fdopen(fdin) as fin: 22 | fin.seek(offset) 23 | 24 | while length > 0: 25 | l = min(length, io.DEFAULT_BUFFER_SIZE) 26 | os.write(fdout, fin.read(l)) 27 | length = length - l 28 | 29 | return length 30 | 31 | from gevent.socket import wait_write 32 | 33 | 34 | def async_sendfile(fdout, fdin, offset, nbytes): 35 | total_sent = 0 36 | while total_sent < nbytes: 37 | try: 38 | sent = sendfile(fdout, fdin, offset + total_sent, 39 | nbytes - total_sent) 40 | total_sent += sent 41 | except OSError, e: 42 | if e.args[0] == errno.EAGAIN: 43 | wait_write(fdout) 44 | else: 45 | raise 46 | return total_sent 47 | -------------------------------------------------------------------------------- /tproxy/server.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | # 3 | # This file is part of tproxy released under the MIT license. 4 | # See the NOTICE for more information. 5 | 6 | import logging 7 | 8 | import greenlet 9 | import gevent 10 | from gevent.event import Event 11 | from gevent.pool import Group, Pool 12 | 13 | 14 | class InactivityTimeout(Exception): 15 | """ Exception raised when the configured timeout elapses without 16 | receiving any data from a connected server """ 17 | 18 | class Peers(Group): 19 | """ 20 | Peered greenlets. If one of greenlet is killed, all are killed. 21 | """ 22 | def discard(self, greenlet): 23 | super(Peers, self).discard(greenlet) 24 | if not hasattr(self, '_killing'): 25 | self._killing = True 26 | gevent.spawn(self.kill) 27 | 28 | class ServerConnection(object): 29 | 30 | def __init__(self, sock, client, timeout=None, extra=None, 31 | buf=None): 32 | self.sock = sock 33 | self.timeout = timeout 34 | self.client = client 35 | self.extra = extra 36 | self.buf = buf 37 | 38 | self.route = client.route 39 | 40 | self.log = logging.getLogger(__name__) 41 | self._stopped_event = Event() 42 | 43 | def handle(self): 44 | """ start to relay the response 45 | """ 46 | try: 47 | peers = Peers([ 48 | gevent.spawn(self.route.proxy_input, self.client.sock, 49 | self.sock, self.buf, self.extra), 50 | gevent.spawn(self.route.proxy_connected, self.sock, 51 | self.client.sock, self.extra)]) 52 | gevent.joinall(peers.greenlets) 53 | finally: 54 | self.sock.close() 55 | 56 | 57 | def proxy_input(self, src, dest, buf, extra): 58 | """ proxy innput to the connected host 59 | """ 60 | self.route.proxy_input(src, dest, buf=buf, extra=extra) 61 | 62 | def proxy_connected(self, src, dest, extra): 63 | """ proxy the response from the connected host to the client 64 | """ 65 | self.route.proxy_connected(src, dest, extra=extra) 66 | -------------------------------------------------------------------------------- /tproxy/tools.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | # 3 | # This file is part of tproxy released under the MIT license. 4 | # See the NOTICE for more information. 5 | 6 | 7 | try: 8 | from importlibe import import_module 9 | except ImportError: 10 | import sys 11 | 12 | def _resolve_name(name, package, level): 13 | """Return the absolute name of the module to be imported.""" 14 | if not hasattr(package, 'rindex'): 15 | raise ValueError("'package' not set to a string") 16 | dot = len(package) 17 | for x in xrange(level, 1, -1): 18 | try: 19 | dot = package.rindex('.', 0, dot) 20 | except ValueError: 21 | raise ValueError("attempted relative import beyond top-level " 22 | "package") 23 | return "%s.%s" % (package[:dot], name) 24 | 25 | 26 | def import_module(name, package=None): 27 | """Import a module. 28 | 29 | The 'package' argument is required when performing a relative import. It 30 | specifies the package to use as the anchor point from which to resolve the 31 | relative import to an absolute import. 32 | 33 | """ 34 | if name.startswith('.'): 35 | if not package: 36 | raise TypeError("relative imports require the 'package' argument") 37 | level = 0 38 | for character in name: 39 | if character != '.': 40 | break 41 | level += 1 42 | name = _resolve_name(name[level:], package, level) 43 | __import__(name) 44 | return sys.modules[name] 45 | -------------------------------------------------------------------------------- /tproxy/util.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | # 3 | # This file is part of tproxy released under the MIT license. 4 | # See the NOTICE for more information. 5 | 6 | 7 | try: 8 | import ctypes 9 | except MemoryError: 10 | # selinux execmem denial 11 | # https://bugzilla.redhat.com/show_bug.cgi?id=488396 12 | ctypes = None 13 | except ImportError: 14 | # Python on Solaris compiled with Sun Studio doesn't have ctypes 15 | ctypes = None 16 | 17 | import fcntl 18 | import os 19 | import random 20 | import resource 21 | import socket 22 | 23 | # add support for gevent 1.0 24 | from gevent import version_info 25 | if version_info[0] >0: 26 | from gevent.os import fork 27 | else: 28 | from gevent.hub import fork 29 | 30 | try: 31 | from setproctitle import setproctitle 32 | def _setproctitle(title): 33 | setproctitle("tproxy: %s" % title) 34 | except ImportError: 35 | def _setproctitle(title): 36 | return 37 | 38 | MAXFD = 1024 39 | if (hasattr(os, "devnull")): 40 | REDIRECT_TO = os.devnull 41 | else: 42 | REDIRECT_TO = "/dev/null" 43 | 44 | def is_ipv6(addr): 45 | try: 46 | socket.inet_pton(socket.AF_INET6, addr) 47 | except socket.error: # not a valid address 48 | return False 49 | return True 50 | 51 | 52 | def parse_address(netloc, default_port=5000): 53 | if isinstance(netloc, tuple): 54 | return netloc 55 | 56 | # get host 57 | if '[' in netloc and ']' in netloc: 58 | host = netloc.split(']')[0][1:].lower() 59 | elif ':' in netloc: 60 | host = netloc.split(':')[0].lower() 61 | elif netloc == "": 62 | host = "0.0.0.0" 63 | else: 64 | host = netloc.lower() 65 | 66 | #get port 67 | netloc = netloc.split(']')[-1] 68 | if ":" in netloc: 69 | port = netloc.split(':', 1)[1] 70 | if not port.isdigit(): 71 | raise RuntimeError("%r is not a valid port number." % port) 72 | port = int(port) 73 | else: 74 | port = default_port 75 | return (host, port) 76 | 77 | 78 | def set_owner_process(uid,gid): 79 | """ set user and group of workers processes """ 80 | if gid: 81 | try: 82 | os.setgid(gid) 83 | except OverflowError: 84 | if not ctypes: 85 | raise 86 | # versions of python < 2.6.2 don't manage unsigned int for 87 | # groups like on osx or fedora 88 | os.setgid(-ctypes.c_int(-gid).value) 89 | 90 | if uid: 91 | os.setuid(uid) 92 | 93 | def chown(path, uid, gid): 94 | try: 95 | os.chown(path, uid, gid) 96 | except OverflowError: 97 | if not ctypes: 98 | raise 99 | os.chown(path, uid, -ctypes.c_int(-gid).value) 100 | 101 | def get_maxfd(): 102 | maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1] 103 | if (maxfd == resource.RLIM_INFINITY): 104 | maxfd = MAXFD 105 | return maxfd 106 | 107 | def close_on_exec(fd): 108 | flags = fcntl.fcntl(fd, fcntl.F_GETFD) 109 | flags |= fcntl.FD_CLOEXEC 110 | fcntl.fcntl(fd, fcntl.F_SETFD, flags) 111 | 112 | def set_non_blocking(fd): 113 | flags = fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK 114 | fcntl.fcntl(fd, fcntl.F_SETFL, flags) 115 | 116 | def daemonize(close=False): 117 | """\ 118 | Standard daemonization of a process. 119 | http://www.svbug.com/documentation/comp.unix.programmer-FAQ/faq_2.html#SEC16 120 | """ 121 | if not 'TPROXY_FD' in os.environ: 122 | try: 123 | if fork(): 124 | os._exit(0) 125 | except OSError, e: 126 | sys.stderr.write("fork #1 failed: %s\n" % str(e)) 127 | sys.exit(1) 128 | 129 | os.setsid() 130 | 131 | try: 132 | if fork(): 133 | os._exit(0) 134 | except OSError, e: 135 | sys.stderr.write("fork #2 failed: %s\n" % str(e)) 136 | sys.exit(1) 137 | 138 | os.umask(0) 139 | 140 | if close: 141 | maxfd = get_maxfd() 142 | 143 | # Iterate through and close all file descriptors. 144 | for fd in range(0, maxfd): 145 | try: 146 | os.close(fd) 147 | except OSError: 148 | # ERROR, fd wasn't open to begin with (ignored) 149 | pass 150 | 151 | os.open(REDIRECT_TO, os.O_RDWR) 152 | os.dup2(0, 1) 153 | os.dup2(0, 2) 154 | 155 | def seed(): 156 | try: 157 | random.seed(os.urandom(64)) 158 | except NotImplementedError: 159 | random.seed(random.random()) 160 | -------------------------------------------------------------------------------- /tproxy/worker.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | # 3 | # This file is part of tproxy released under the MIT license. 4 | # See the NOTICE for more information. 5 | 6 | import os 7 | import logging 8 | import signal 9 | 10 | import gevent 11 | from gevent.pool import Pool 12 | from gevent.ssl import wrap_socket 13 | 14 | 15 | from . import util 16 | from .proxy import ProxyServer 17 | from .workertmp import WorkerTmp 18 | 19 | class Worker(ProxyServer): 20 | 21 | SIGNALS = map( 22 | lambda x: getattr(signal, "SIG%s" % x), 23 | "HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split() 24 | ) 25 | 26 | PIPE = [] 27 | 28 | def __init__(self, age, ppid, listener, cfg, script): 29 | ProxyServer.__init__(self, listener, script, 30 | spawn=Pool(cfg.worker_connections)) 31 | 32 | if cfg.ssl_keyfile and cfg.ssl_certfile: 33 | self.wrap_socket = wrap_socket 34 | self.ssl_args = dict( 35 | keyfile = cfg.ssl_keyfile, 36 | certfile = cfg.ssl_certfile, 37 | server_side = True, 38 | cert_reqs = cfg.ssl_cert_reqs, 39 | ca_certs = cfg.ssl_ca_certs, 40 | suppress_ragged_eofs=True, 41 | do_handshake_on_connect=True) 42 | self.ssl_enabled = True 43 | 44 | self.name = cfg.name 45 | self.age = age 46 | self.ppid = ppid 47 | self.cfg = cfg 48 | self.tmp = WorkerTmp(cfg) 49 | self.booted = False 50 | self.log = logging.getLogger(__name__) 51 | 52 | def __str__(self): 53 | return "" % self.pid 54 | 55 | @property 56 | def pid(self): 57 | return os.getpid() 58 | 59 | def init_process(self): 60 | #gevent doesn't reinitialize dns for us after forking 61 | #here's the workaround 62 | gevent.core.dns_shutdown(fail_requests=1) 63 | gevent.core.dns_init() 64 | 65 | util.set_owner_process(self.cfg.uid, self.cfg.gid) 66 | 67 | # Reseed the random number generator 68 | util.seed() 69 | 70 | # For waking ourselves up 71 | self.PIPE = os.pipe() 72 | map(util.set_non_blocking, self.PIPE) 73 | map(util.close_on_exec, self.PIPE) 74 | 75 | 76 | # Prevent fd inherientence 77 | util.close_on_exec(self.socket) 78 | util.close_on_exec(self.tmp.fileno()) 79 | 80 | map(lambda s: signal.signal(s, signal.SIG_DFL), self.SIGNALS) 81 | self.booted = True 82 | 83 | def start_heartbeat(self): 84 | def notify(): 85 | while self.started: 86 | gevent.sleep(self.cfg.timeout / 2.0) 87 | 88 | # If our parent changed then we shut down. 89 | if self.ppid != os.getppid(): 90 | self.log.info("Parent changed, shutting down: %s" % self) 91 | return 92 | 93 | self.tmp.notify() 94 | 95 | return gevent.spawn(notify) 96 | 97 | def serve_forever(self): 98 | self.init_process() 99 | self.start_heartbeat() 100 | super(Worker, self).serve_forever() 101 | 102 | def refresh_name(self): 103 | title = "worker" 104 | if self.name is not None: 105 | title += " [%s]" 106 | title = "%s - handling %s connections" % (title, self.nb_connections) 107 | util._setproctitle(title) 108 | 109 | def stop_accepting(self): 110 | title = "worker" 111 | if self.name is not None: 112 | title += " [%s]" 113 | title = "%s - stop accepting" % title 114 | util._setproctitle(title) 115 | super(Worker, self).stop_accepting() 116 | 117 | def start_accepting(self): 118 | self.refresh_name() 119 | super(Worker, self).start_accepting() 120 | 121 | def kill(self): 122 | """stop accepting.""" 123 | self.started = False 124 | try: 125 | self.stop_accepting() 126 | finally: 127 | self.__dict__.pop('socket', None) 128 | self.__dict__.pop('handle', None) 129 | -------------------------------------------------------------------------------- /tproxy/workertmp.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 - 2 | # 3 | # This file is part of tproxy released under the MIT license. 4 | # See the NOTICE for more information. 5 | 6 | import os 7 | import tempfile 8 | 9 | from . import util 10 | 11 | class WorkerTmp(object): 12 | 13 | def __init__(self, cfg): 14 | old_umask = os.umask(cfg.umask) 15 | fd, name = tempfile.mkstemp(prefix="wtproxy-") 16 | 17 | # allows the process to write to the file 18 | util.chown(name, cfg.uid, cfg.gid) 19 | os.umask(old_umask) 20 | 21 | # unlink the file so we don't leak tempory files 22 | try: 23 | os.unlink(name) 24 | self._tmp = os.fdopen(fd, 'w+b', 1) 25 | except: 26 | os.close(fd) 27 | raise 28 | 29 | self.spinner = 0 30 | 31 | def notify(self): 32 | try: 33 | self.spinner = (self.spinner+1) % 2 34 | os.fchmod(self._tmp.fileno(), self.spinner) 35 | except AttributeError: 36 | # python < 2.6 37 | self._tmp.truncate(0) 38 | os.write(self._tmp.fileno(), "X") 39 | 40 | def fileno(self): 41 | return self._tmp.fileno() 42 | 43 | def close(self): 44 | return self._tmp.close() 45 | --------------------------------------------------------------------------------