├── .gitignore ├── .pylintrc ├── .scrutinizer.yml ├── CHANGELOG ├── COPYRIGHT ├── QA ├── bin │ ├── exaproxy-coverage │ ├── python-profile │ └── python-stats ├── checks │ ├── checker │ ├── flakes │ └── lint ├── profile │ ├── analyse │ ├── pyprof2calltree.py │ └── tools │ │ ├── guppy-0.1.9.tar.gz │ │ └── pyprof2calltree-1.1.0.tar.gz ├── release │ ├── google │ └── names └── test │ ├── connect-http-127.0.0.1 │ ├── connect-http-google │ ├── connect-http-google-proxy-explanation │ ├── connect-http-test │ ├── get-127.0.0.1 │ ├── get-google │ ├── get-google-via │ ├── get-not-here │ ├── kill-stuck │ ├── loop-google │ ├── options-http-apache │ ├── options-http-apache-max-forwards │ ├── options-http-local │ ├── pipeline │ ├── pipeline-google │ ├── proxy │ └── chunk.py │ ├── reqmod.data │ ├── request.data │ └── run ├── README.FIRST ├── README.md ├── ROADMAP ├── etc ├── exaproxy │ ├── dns │ │ └── types │ ├── exaproxy.conf │ ├── html │ │ ├── classify.html │ │ ├── deny.html │ │ ├── dns.html │ │ ├── images │ │ │ ├── deny.gif │ │ │ ├── deny.jpg │ │ │ └── deny.png │ │ ├── internal_error.html │ │ ├── noconnect.html │ │ └── success.html │ └── redirector │ │ ├── icap-allow-204 │ │ ├── icap-allow-304 │ │ ├── icap-deny │ │ ├── icap-deny-proxy-explanation │ │ ├── icap-google-no-ssl │ │ ├── icap-google-to-others │ │ ├── icap-safe-youtube │ │ ├── icap-serve │ │ ├── icap-serve-file │ │ ├── url-allow │ │ ├── url-allow-net │ │ ├── url-deny │ │ ├── url-google-no-ssl │ │ ├── url-google-to-others │ │ ├── url-parrot │ │ └── url-serve-file └── systemd │ └── exaproxy.service ├── lib └── exaproxy │ ├── __init__.py │ ├── application.py │ ├── configuration.py │ ├── dns │ ├── __init__.py │ ├── codec.py │ ├── convert.py │ ├── definition.py │ ├── dnstype.py │ └── factory.py │ ├── html │ ├── __init__.py │ ├── graph.py │ ├── humans.py │ ├── images.py │ ├── img.py │ ├── index.py │ ├── licence.py │ ├── mail.py │ ├── menu.py │ └── page.py │ ├── http │ ├── __init__.py │ ├── factory.py │ ├── headers.py │ ├── message.py │ ├── request.py │ └── response.py │ ├── icap │ ├── __init__.py │ ├── header.py │ ├── parser.py │ ├── request.py │ └── response.py │ ├── leak │ ├── __init__.py │ ├── gcdump.py │ └── objgraph.py │ ├── monitor.py │ ├── network │ ├── __init__.py │ ├── async │ │ ├── __init__.py │ │ ├── epoll.py │ │ ├── interface.py │ │ ├── kqueue.py │ │ └── selectpoll.py │ ├── errno_list.py │ ├── functions.py │ ├── server.py │ └── splice.py │ ├── reactor │ ├── __init__.py │ ├── client │ │ ├── __init__.py │ │ ├── http.py │ │ ├── icap.py │ │ ├── manager.py │ │ ├── passthrough.py │ │ └── tls.py │ ├── content │ │ ├── __init__.py │ │ ├── manager.py │ │ └── worker.py │ ├── reactor.py │ ├── redirector │ │ ├── __init__.py │ │ ├── child.py │ │ ├── dispatch.py │ │ ├── icap.py │ │ ├── manager.py │ │ ├── messagebox.py │ │ ├── reactor.py │ │ ├── redirector.py │ │ ├── response.py │ │ ├── serialize │ │ │ ├── __init__.py │ │ │ ├── icap.py │ │ │ ├── passthrough.py │ │ │ └── tls.py │ │ ├── supervisor.py │ │ └── worker.py │ └── resolver │ │ ├── __init__.py │ │ ├── manager.py │ │ └── worker.py │ ├── supervisor.py │ ├── tls │ ├── __init__.py │ ├── decode.py │ ├── header.py │ ├── parser.py │ ├── request.py │ └── response.py │ └── util │ ├── __init__.py │ ├── alarm.py │ ├── cache.py │ ├── control.py │ ├── daemon.py │ ├── debug.py │ ├── interfaces.py │ ├── log │ ├── __init__.py │ ├── history.py │ ├── logger.py │ ├── message.py │ └── writer.py │ ├── messagebox.py │ ├── messagequeue.py │ ├── pid.py │ └── proxy.py ├── sbin ├── exaproxy └── icap-connector ├── scripts └── exaproxy ├── service ├── exaproxy │ ├── log │ │ └── run │ └── run └── surfprotect │ ├── log │ └── run │ └── run └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *pyc 3 | *.swp 4 | *.orig 5 | run 6 | -------------------------------------------------------------------------------- /.scrutinizer.yml: -------------------------------------------------------------------------------- 1 | filter: 2 | excluded_paths: 3 | - 'lib/exaproxy/dep/*' 4 | - 'bin/*' 5 | - 'sbin/*' 6 | - 'etc/*' 7 | - 'QA/*' 8 | - 'build/*' 9 | - 'dist/*' 10 | paths: { } 11 | tools: 12 | python_analyzer: 13 | enabled: true 14 | config: 15 | metrics: true 16 | duplication_detector: 17 | enabled: true 18 | min_mass: 25 19 | file_extensions: 20 | - .py 21 | filter: 22 | excluded_paths: 23 | - 'lib/exaproxy/dep/*' 24 | - 'bin/*' 25 | - 'sbin/*' 26 | - 'etc/*' 27 | - 'QA/*' 28 | - 'build/*' 29 | - 'dist/*' 30 | paths: { } 31 | pylint: 32 | enabled: true 33 | python_version: '2' 34 | plugins: { } 35 | config_file: '.pylintrc' 36 | filter: 37 | excluded_paths: 38 | - 'lib/exaproxy/dep/*' 39 | - 'bin/*' 40 | - 'sbin/*' 41 | - 'etc/*' 42 | - 'QA/*' 43 | - 'build/*' 44 | - 'dist/*' 45 | paths: { } 46 | -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | ExaProxy 2 | 3 | Copyright (c) 2011-2013, Exa Networks Limited 4 | Copyright (c) 2011-2013, Thomas Mangin 5 | Copyright (c) 2011-2013, David Farrar 6 | 7 | All rights reserved. 8 | 9 | Redistribution and use in source and binary forms, with or without 10 | modification, are permitted provided that the following conditions are met: 11 | 12 | 1. Redistributions of source code must retain the above copyright notice, this 13 | list of conditions and the following disclaimer. 14 | 2. Redistributions in binary form must reproduce the above copyright notice, 15 | this list of conditions and the following disclaimer in the documentation 16 | and/or other materials provided with the distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 25 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /QA/bin/exaproxy-coverage: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | dirname=`dirname $0` 4 | 5 | case $dirname in 6 | /*) 7 | cd $dirname/.. > /dev/null 8 | path=`pwd` 9 | cd - > /dev/null 10 | ;; 11 | *) 12 | cd `pwd`/$dirname/.. > /dev/null 13 | path=`pwd` 14 | cd - > /dev/null 15 | ;; 16 | esac 17 | 18 | export PYTHONPATH=$path/lib 19 | 20 | env PYTHONPATH=./lib \ 21 | `which python` \ 22 | -m exaproxy.util.debug \ 23 | `which coverage` \ 24 | run \ 25 | lib/exaproxy/application.py \ 26 | --release "coverage" 27 | $* 28 | 29 | -------------------------------------------------------------------------------- /QA/bin/python-profile: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | python -m cProfile -o stat.prof $* 3 | -------------------------------------------------------------------------------- /QA/bin/python-stats: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | python -m pstats stat.prof 3 | -------------------------------------------------------------------------------- /QA/checks/checker: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | env \ 3 | PYTHONPATH=$PYTHONPATH:lib \ 4 | python2.6 \ 5 | /opt/local/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/site-packages/pychecker/checker.py \ 6 | $1 7 | -------------------------------------------------------------------------------- /QA/checks/flakes: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | env \ 3 | PYTHONPATH=$PYTHONPATH:lib \ 4 | python2.6 \ 5 | /opt/local/bin/pyflakes-2.6 \ 6 | $1 7 | -------------------------------------------------------------------------------- /QA/checks/lint: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | env \ 3 | PYTHONPATH=$PYTHONPATH:lib \ 4 | pylint \ 5 | --variable-rgx="[a-z_][a-z0-9_]{0,30}$" \ 6 | --disable-msg=W0312 \ 7 | --disable-msg=C0324 \ 8 | --disable-msg=C0111 \ 9 | --disable-msg=C0321 \ 10 | --disable-msg=C0103 \ 11 | --max-line-length=200 \ 12 | $1 13 | 14 | # W0312 : Found indentation with tabs instead of spaces 15 | # C0324 : Comma not followed by a space 16 | # C0111 : Missing docstring 17 | # C0321 : More than one statement on a single line 18 | # C0103 : the regex for class and functions 19 | -------------------------------------------------------------------------------- /QA/profile/analyse: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import pstats 5 | 6 | prg = sys.argv[0] 7 | 8 | try: 9 | name = sys.argv[1] 10 | except IndexError: 11 | print "%s [ []] " % prg 12 | print "%s -^" % (' '*len(prg)) 13 | sys.exit(1) 14 | 15 | try: 16 | number = int(sys.argv[2]) 17 | except ValueError: 18 | print "%s [ []] " % prg 19 | print "%s -^" % (' '*len(prg)) 20 | sys.exit(1) 21 | except IndexError: 22 | number = 0 23 | 24 | try: 25 | field = sys.argv[3] 26 | except IndexError: 27 | field = 'time' 28 | 29 | options = { 30 | 'calls' : 'call count', 31 | 'cumulative' : 'cumulative time', 32 | 'file' : 'file name', 33 | 'module' : 'file name', 34 | 'pcalls' : 'primitive call count', 35 | 'line' : 'line number', 36 | 'name' : 'function name', 37 | 'nfl' : 'name/file/line', 38 | 'stdname' : 'standard name', 39 | 'time' : 'internal time', 40 | } 41 | 42 | if field not in options: 43 | print 'invalid sorting field, valid enties are :\n%s' % '\n'.join(" %-10s : %s" % (k,v) for (k,v) in options.iteritems()) 44 | sys.exit(1) 45 | 46 | try: 47 | stats = pstats.Stats(name) 48 | except IOError: 49 | print "can not open the file %s" % name 50 | sys.exit(1) 51 | 52 | if number: 53 | print stats.strip_dirs().sort_stats(field).print_stats(number) 54 | else: 55 | print stats.print_stats() 56 | -------------------------------------------------------------------------------- /QA/profile/tools/guppy-0.1.9.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Exa-Networks/exaproxy/8b7291b79c1cd6542213a5e7d8dda3cf5a676166/QA/profile/tools/guppy-0.1.9.tar.gz -------------------------------------------------------------------------------- /QA/profile/tools/pyprof2calltree-1.1.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Exa-Networks/exaproxy/8b7291b79c1cd6542213a5e7d8dda3cf5a676166/QA/profile/tools/pyprof2calltree-1.1.0.tar.gz -------------------------------------------------------------------------------- /QA/release/google: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | me=`id -u` 4 | 5 | if [ $me -eq 0 ]; then 6 | echo 'root is not allowed to use this script' 7 | exit 1 8 | fi 9 | 10 | dirname=`dirname $0` 11 | 12 | if [[ $dirname == /* ]]; then 13 | path=$dirname/../.. 14 | else 15 | cd `pwd`/$dirname/../.. > /dev/null 16 | path=`pwd` 17 | cd - > /dev/null 18 | fi 19 | 20 | # This is not safe against race condition, but I trust my own machine to not be hacked .. 21 | 22 | build=/tmp 23 | version=`tail -1 $path/.hgtags | cut -f2 -d\ ` 24 | name=exaproxy-$version 25 | destination=$build/$name 26 | 27 | echo destination is $destination 28 | rm -rf $destination 29 | mkdir $destination 30 | 31 | # Adding the files and folders 32 | 33 | echo adding the files 34 | 35 | cd $path 36 | for f in `ls`; 37 | do 38 | echo copying $f 39 | cp -r $f $destination/ 40 | done 41 | cp .hgtags $destination/ 42 | 43 | # Removing all the non-commited files (normally test configurations) 44 | 45 | echo removing unreleased file 46 | 47 | internal=`hg status | grep "^?" | cut -c3- | sed -e "s/ /|/g"` 48 | 49 | cd $destination 50 | for f in $internal; 51 | do 52 | file=`echo $f | sed -e "s/\|/ /g"` 53 | echo removing $file 54 | rm -f "$file" 55 | done 56 | cd - 57 | 58 | echo removing pyc files 59 | 60 | cd $destination 61 | for f in `find . -name "*.pyc"`; 62 | do 63 | echo removing $f 64 | rm -f $f 65 | done 66 | cd - 67 | 68 | echo removing dot files 69 | 70 | cd $destination 71 | for f in `find . -name ".?*"`; 72 | do 73 | if [[ $f != ./.hgtags ]]; then 74 | echo removing $f 75 | rm -f $f 76 | fi 77 | done 78 | cd - 79 | 80 | echo removing orig files 81 | 82 | cd $destination 83 | for f in `find . -name "*.orig"`; 84 | do 85 | echo removing $f 86 | rm -f "$f" 87 | done 88 | cd - 89 | 90 | cd $destination 91 | echo removing html folder 92 | rm -rf html 93 | echo removing QA folder 94 | rm -rf QA 95 | cd - 96 | 97 | # Making tarball 98 | 99 | echo making tarbal 100 | 101 | # Telling MACs to not include the ._ version of the files 102 | export COPY_EXTENDED_ATTRIBUTES_DISABLE=true 103 | export COPYFILE_DISABLE=true 104 | 105 | cd $path 106 | rm -f ${name}.tgz 107 | 108 | cd $build 109 | rm -f ${name}.tgz 110 | tar zcvf $path/${name}.tgz $name 111 | 112 | echo 'done' 113 | -------------------------------------------------------------------------------- /QA/release/names: -------------------------------------------------------------------------------- 1 | Release Name - perhaps :) 2 | first Aluminium Hydroxide 3 | next Better Bretta 4 | next Crazy Chlorine 5 | next Demineralized ... 6 | next E 7 | next Flocculated ... 8 | 9 | -------------------------------------------------------------------------------- /QA/test/connect-http-127.0.0.1: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ( \ 4 | echo "CONNECT 127.0.0.1:80 HTTP/1.1" ; 5 | echo "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:8.0.1) Gecko/20100101 Firefox/8.0.1"; 6 | echo "Host: 127.0.0.1"; 7 | echo 8 | echo "GET / HTTP/1.1"; 9 | echo "Host: 127.0.0.1" ; 10 | echo "X-Forwarded-For: 127.0.0.1"; 11 | echo "Connection: close"; 12 | echo; 13 | sleep 100) | telnet 127.0.0.1 3128 14 | -------------------------------------------------------------------------------- /QA/test/connect-http-google: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ( \ 4 | echo "CONNECT www.google.com:80 HTTP/1.1" ; 5 | echo "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:8.0.1) Gecko/20100101 Firefox/8.0.1"; 6 | echo 7 | echo "GET / HTTP/1.1"; 8 | echo "Host: www.google.com" ; 9 | echo "X-Forwarded-For: 127.0.0.1"; 10 | echo "Connection: close"; 11 | echo; 12 | sleep 100) | telnet 127.0.0.1 3128 13 | -------------------------------------------------------------------------------- /QA/test/connect-http-google-proxy-explanation: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # env exaproxy.http.connections=100 exaproxy.tls.enable=false exaproxy.security.connect=80 exaproxy.redirector.enable=true exaproxy.redirector.program=etc/exaproxy/redirector/icap-deny-proxy-explanation exaproxy.redirector.protocol='icap://' ./sbin/exaproxy 4 | 5 | ( \ 6 | echo "CONNECT www.google.com:80 HTTP/1.1" ; 7 | echo "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:8.0.1) Gecko/20100101 Firefox/8.0.1"; 8 | echo 9 | echo "GET / HTTP/1.1"; 10 | echo "Host: www.google.com" ; 11 | echo "Accept: application/proxy-explanation+json"; 12 | echo "Accept-Language: en-gb"; 13 | echo "Connection: close"; 14 | echo; 15 | sleep 100) | telnet 127.0.0.1 3128 16 | -------------------------------------------------------------------------------- /QA/test/connect-http-test: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import httplib 4 | 5 | while True: 6 | connection = httplib.HTTPConnection("www.google.com") 7 | connection.set_tunnel('127.0.0.1',3128, headers=None) 8 | connection.request("GET","/index.html") 9 | response = connection.getresponse() 10 | print response.status, response.reason 11 | page = response.read() 12 | print page 13 | connection.close() 14 | 15 | """ 16 | #!/bin/sh 17 | ( 18 | echo "GET / HTTP/1.1"; 19 | echo "Host: www.google.com" ; 20 | echo "X-Forwarded-For: 127.0.0.1"; 21 | echo "Connection: close"; 22 | echo; 23 | sleep 100) | telnet 127.0.0.1 3128 24 | """ 25 | -------------------------------------------------------------------------------- /QA/test/get-127.0.0.1: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ( 3 | echo "GET / HTTP/1.1"; 4 | echo "Host: 127.0.0.1" ; 5 | echo "X-Forwarded-For: 127.0.0.1"; 6 | echo "Connection: close"; 7 | echo; 8 | sleep 100) | telnet 127.0.0.1 3128 9 | -------------------------------------------------------------------------------- /QA/test/get-google: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import socket 4 | import sys 5 | 6 | PROXY_HOST = '127.0.0.1' 7 | PROXY_PORT = 3128 8 | 9 | HOST = 'www.google.com' 10 | GET = '/' 11 | 12 | try: 13 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 14 | except socket.error, msg: 15 | sys.stderr.write("[ERROR] %s\n" % msg[1]) 16 | sys.exit(1) 17 | 18 | try: 19 | sock.connect((PROXY_HOST, PROXY_PORT)) 20 | except socket.error, msg: 21 | sys.stderr.write("[ERROR] %s\n" % msg[1]) 22 | sys.exit(2) 23 | 24 | sock.send("GET %s HTTP/1.0\r\nHost: %s\r\n\r\n" % (GET, HOST)) 25 | 26 | data = sock.recv(1024) 27 | string = "" 28 | while len(data): 29 | string = string + data 30 | data = sock.recv(1024) 31 | sock.close() 32 | 33 | print string 34 | 35 | sys.exit(0) 36 | 37 | -------------------------------------------------------------------------------- /QA/test/get-google-via: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import socket 4 | import sys 5 | 6 | PROXY_HOST = '127.0.0.1' 7 | PROXY_PORT = 3128 8 | 9 | HOST = 'www.google.com' 10 | GET = '/' 11 | 12 | try: 13 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 14 | except socket.error, msg: 15 | sys.stderr.write("[ERROR] %s\n" % msg[1]) 16 | sys.exit(1) 17 | 18 | try: 19 | sock.connect((PROXY_HOST, PROXY_PORT)) 20 | except socket.error, msg: 21 | sys.stderr.write("[ERROR] %s\n" % msg[1]) 22 | sys.exit(2) 23 | 24 | sock.send("GET %s HTTP/1.0\r\nHost: %s\r\nVia: 1.0 a-proxy, 1.0\r\n\r\n" % (GET, HOST)) 25 | 26 | data = sock.recv(1024) 27 | string = "" 28 | while len(data): 29 | string = string + data 30 | data = sock.recv(1024) 31 | sock.close() 32 | 33 | print string 34 | 35 | sys.exit(0) 36 | 37 | -------------------------------------------------------------------------------- /QA/test/get-not-here: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import socket 4 | import sys 5 | 6 | PROXY_HOST = '127.0.0.1' 7 | PROXY_PORT = 3128 8 | 9 | HOST = 'www.does-not.exists' 10 | GET = '/' 11 | 12 | try: 13 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 14 | except socket.error, msg: 15 | sys.stderr.write("[ERROR] %s\n" % msg[1]) 16 | sys.exit(1) 17 | 18 | try: 19 | sock.connect((PROXY_HOST, PROXY_PORT)) 20 | except socket.error, msg: 21 | sys.stderr.write("[ERROR] %s\n" % msg[1]) 22 | sys.exit(2) 23 | 24 | sock.send("GET %s HTTP/1.0\r\nHost: %s\r\n\r\n" % (GET, HOST)) 25 | 26 | data = sock.recv(1024) 27 | string = "" 28 | while len(data): 29 | string = string + data 30 | data = sock.recv(1024) 31 | sock.close() 32 | 33 | print string 34 | 35 | sys.exit(0) 36 | 37 | -------------------------------------------------------------------------------- /QA/test/kill-stuck: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ps x | grep -- "-m exaproxy.util.debug" | grep -v grep | awk '{ print $1 }' | xargs kill -9 3 | 4 | -------------------------------------------------------------------------------- /QA/test/loop-google: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import socket 4 | import sys 5 | 6 | PROXY_HOST = '127.0.0.1' 7 | PROXY_PORT = 3128 8 | 9 | HOST = 'www.google.com' 10 | GET = '/' 11 | 12 | while True: 13 | try: 14 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 15 | except socket.error, msg: 16 | sys.stderr.write("[ERROR] %s\n" % msg[1]) 17 | sys.exit(1) 18 | 19 | try: 20 | sock.connect((PROXY_HOST, PROXY_PORT)) 21 | except socket.error, msg: 22 | sys.stderr.write("[ERROR] %s\n" % msg[1]) 23 | sys.exit(2) 24 | 25 | sock.send("GET %s HTTP/1.0\r\nHost: %s\r\n\r\n" % (GET, HOST)) 26 | 27 | data = sock.recv(1024) 28 | string = "" 29 | while len(data): 30 | string = string + data 31 | data = sock.recv(1024) 32 | sock.close() 33 | 34 | print string 35 | 36 | sys.exit(0) 37 | 38 | -------------------------------------------------------------------------------- /QA/test/options-http-apache: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ( \ 4 | echo "OPTIONS / HTTP/1.1" ; 5 | echo "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:8.0.1) Gecko/20100101 Firefox/8.0.1"; 6 | echo "Host: www.apache.org"; 7 | echo; 8 | sleep 100) | telnet 127.0.0.1 3128 9 | -------------------------------------------------------------------------------- /QA/test/options-http-apache-max-forwards: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ( \ 4 | echo "OPTIONS / HTTP/1.1" ; 5 | echo "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:8.0.1) Gecko/20100101 Firefox/8.0.1"; 6 | echo "Host: www.apache.org"; 7 | echo "Max-Forwards: 1"; 8 | echo; 9 | sleep 100) | telnet 127.0.0.1 3128 10 | -------------------------------------------------------------------------------- /QA/test/options-http-local: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ( \ 4 | echo "OPTIONS / HTTP/1.1" ; 5 | echo "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:8.0.1) Gecko/20100101 Firefox/8.0.1"; 6 | echo "Host: www.apache.org"; 7 | echo "Max-Forwards: 0"; 8 | echo; 9 | sleep 100) | telnet 127.0.0.1 3128 10 | -------------------------------------------------------------------------------- /QA/test/pipeline: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | (echo "GET / HTTP/1.1"; echo "Host: 127.0.0.1" ; echo "X-Forwarded-For: 127.0.0.1"; echo "Connection: close"; echo; sleep 100) | telnet 127.0.0.1 3128 3 | -------------------------------------------------------------------------------- /QA/test/pipeline-google: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import socket 4 | import sys 5 | 6 | PROXY_HOST = '127.0.0.1' 7 | PROXY_PORT = 3128 8 | 9 | HOST = 'www.google.com' 10 | GET = '/' 11 | 12 | try: 13 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 14 | except socket.error, msg: 15 | sys.stderr.write("[ERROR] %s\n" % msg[1]) 16 | sys.exit(1) 17 | 18 | try: 19 | sock.connect((PROXY_HOST, PROXY_PORT)) 20 | except socket.error, msg: 21 | sys.stderr.write("[ERROR] %s\n" % msg[1]) 22 | sys.exit(2) 23 | 24 | sock.send("GET %s HTTP/1.1\r\nHost: %s\r\n\r\n" % (GET, HOST)) 25 | sock.send("GET %s HTTP/1.1\r\nHost: %s\r\n\r\n" % (GET, HOST)) 26 | 27 | data = sock.recv(1024) 28 | string = "" 29 | while len(data): 30 | string = string + data 31 | data = sock.recv(1024) 32 | sock.close() 33 | 34 | print string 35 | 36 | sys.exit(0) 37 | 38 | -------------------------------------------------------------------------------- /QA/test/proxy/chunk.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import socket 5 | 6 | pattern = ''.join(chr(_) for _ in range(0,256) if chr(_).isalpha()) 7 | 8 | for sizes in ( 9 | [str(_) for _ in range(1,10)], 10 | [str(1) for _ in range(0,0xff)], 11 | ['ffff',], 12 | ['20000',], 13 | (), 14 | ): 15 | for te_key in ('Transfer-Encoding', 'TE'): 16 | for te_value in ('chunked', 'chunked, trailer','chunked,trailer','trailers, deflate;q=0.5', 'something;q=1.5, trailer, chunked,else;q=0.5', 'chunked, something;q=1.5'): 17 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 18 | s.connect(('127.0.0.1', 3128)) 19 | 20 | request = \ 21 | 'POST /empty/ HTTTP/1.1\r\n' \ 22 | 'Host: 127.0.0.1\r\n' \ 23 | '%s' \ 24 | 'Connection: close\r\n\r\n' % ('%s: %s\r\n' % (te_key,te_value) if te_value else '') 25 | 26 | s.send(request) 27 | 28 | debug = ['\n'] 29 | debug.append('=' * 80) 30 | debug.append('sending chunk sizes (%s: %s) :' % (te_key,te_value)) 31 | debug.append('%s' % ' '.join(sizes)) 32 | debug.append('\n') 33 | 34 | if 'chunk' in request and not ';q=1.5' in request: 35 | sys.stdout.write('%-8s chunk size %-6d %-65s' % ('single' if len(sizes) == 1 else 'multiple', sum(int(_,16) for _ in sizes),'-' if not te_key else '%s: %s' % (te_key,te_value))) 36 | 37 | for size in sizes: 38 | sys.stdout.flush() 39 | length = int(size,16) 40 | repeat = length/len(pattern)+1 41 | chunk = (pattern * repeat)[:length] 42 | sent = '%s\r\n%s\r\n' % (size,chunk) 43 | s.send(sent) 44 | request += sent.replace('\t', '\\t').replace('\r', '\\r').replace('\n', '\\n\n') 45 | 46 | request += '0\\r\\n\\r\\n\n' 47 | s.send('0\r\n\r\n') 48 | else: 49 | sys.stdout.write('no chunk %-65s' % ('-' if not te_key else '%s: %s' % (te_key,te_value))) 50 | 51 | 52 | debug.append('[[%s]]' % request.replace('\t', '\\t').replace('\r', '\\r').replace('\n', '\\n\n')) 53 | debug.append('\n') 54 | 55 | try: 56 | data = s.recv(0x20000) 57 | except KeyboardInterrupt: 58 | print '\n'.join(debug) 59 | sys.exit(1) 60 | s.close() 61 | 62 | if '200' in data: 63 | sys.stdout.write('page received\n') 64 | sys.stdout.flush() 65 | elif '501 Method Not Implemented' in data: 66 | sys.stdout.write('not implemented\n') 67 | sys.stdout.flush() 68 | else: 69 | debug.append('[[%s]]' % data.replace('\t', '\\t').replace('\r', '\\r').replace('\n', '\\n\n')) 70 | print '\n'.join(debug) 71 | sys.stdout.flush() 72 | sys.exit(1) 73 | -------------------------------------------------------------------------------- /QA/test/reqmod.data: -------------------------------------------------------------------------------- 1 | REQMOD icap://surfprotect-demonstration ICAP/1.0 2 | Host: surfprotect-demonstration 3 | Encapsulated: req-hdr=0, null-body=127 4 | 5 | GET http://www.bbc.co.uk/ HTTP/1.1 6 | Host: www.bbc.co.uk 7 | User-Agent: curl/7.43.0 8 | Accept: */* 9 | Proxy-Connection: Keep-Alive 10 | 11 | -------------------------------------------------------------------------------- /QA/test/request.data: -------------------------------------------------------------------------------- 1 | GET / HTTP/1.1 2 | Host: www.exa.net.uk 3 | X-Forwarded-For: 127.0.0.1 4 | 5 | -------------------------------------------------------------------------------- /QA/test/run: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | env PDB=1 ./sbin/exaproxy ./redirector/parrot 3 | 4 | -------------------------------------------------------------------------------- /README.FIRST: -------------------------------------------------------------------------------- 1 | ExaProxy is a small and fast proxy. It uses select on Unix like systems and 2 | epoll on linux to provide the best performance possible. 3 | 4 | It can forward HTTPS request using the CONNECT directive. 5 | 6 | ExaProxy was developed to be a transparent proxy, and can be used without 7 | client side configuration. Thanks to its url rewriting features, compatible 8 | with SQUID rewriters, it can as well be used as reverse proxy where complex URL 9 | rewriting are required. 10 | 11 | The code is modular and we do welcome contribution from other parties. 12 | ExaProxy can server files locally, for example displaying "denied" pages, 13 | without requiring the installation of a third party web server. 14 | 15 | ExaProxy does not require any root privileges. 16 | 17 | Features 18 | * Non-caching HTTP Proxy 19 | - HTTP (1.0 / 1.1) 20 | - HTTPS (supports CONNECT) 21 | * Polyvalent 22 | - forward, reverse or transparent proxy 23 | - IPv6 and/or IPv4 support 24 | + for incoming connection (we are in 2012) 25 | + for outgoing connection (so it can provide IPv6 access to IPv4 only clients or vice versa) 26 | * High Performance 27 | - non-blocking event based network loop 28 | - using epoll on Linux, select on other OSes 29 | - use of cheap co-routine to handle clients connections 30 | - automatically adding threads to handle load peaks (threads are forking the redirectors programs) 31 | - internal use of sockets as message parsing technique with the threads 32 | - internal non-blocking DNS resolver (UDP and TCP messages integrated in the main loop) 33 | - optional local DNS caching of records 34 | * Traffic interception 35 | - forks helper redirector programs able to perform traffic modification 36 | - SQUID compatible interface 37 | + provides a new file:// interface, to serve local files as answer 38 | + provides a new intercept:// interface, to force the connection to a particular backend host 39 | + provides a new redirect:// interface allowing to return a 302 redirect to the host 40 | - ICAP compatible interface via local program 41 | + message are in ICAP format (to make it easy to write a real ICAP helper later on) 42 | + parse ICAP response (only accept "Encapsulated: res-hdr=0, null-body=" replies) 43 | + accept CONNECT with HTTP request, modified replies are then redirected. 44 | + accept "GET file:// HTTP/1.1" reply to serve local files 45 | + 204 No Content support (content is not modified by the ICAP server) 46 | - many examples of helper programs in the repository 47 | * Built-in web servers to monitor the proxy via local webpage ( default http://127.0.0.1:8080 ) 48 | 49 | The SQUID redirector API was improved and we accept more than the usual http:// answer. New options such as file:// to serve a local file, or intercept:// to redirect the page are as well available or redirect:// to serve a 302 redirection page. 50 | 51 | Just browse the etc/exaproxy/redirector folder for many examples of what can be done: 52 | * force Youtube safesearch changing the cookies 53 | * send requests to https://www.google.com to https://www.wolframalpha.com/ - just for fun :) 54 | 55 | This programs does not have any dependences on any third party libraries. 56 | Development is done on python 2.7 57 | 58 | The documentation is at http://code.google.com/p/exaproxy/w/list 59 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## ExaProxy 2 | 3 | ExaProxy is a a high-performance non-caching proxy. It is able to filter HTTP traffic using your favorite programming language. 4 | 5 | It was part of Exa Networks' [SurfProtect](http://www.surfprotect.co.uk/) solution, but has been replaced by a new codebase in Golang. 6 | 7 | Exaproxy is used in production since early 2013, and proxies millions of URL per day, one installation sees Gb/s of HTTP traffic, with hundreds of Mb/s per server, and several tens of thousands of connections per machine, but this does not mean our work is finished. We continue to improve it. 8 | 9 | ## News 10 | 11 | August 21st 2014, released ExaProxy 1.2.1 12 | 13 | ## Features 14 | 15 | * Non-caching HTTP/HTTPS (with CONNECT) Proxy 16 | * forward, reverse or transparent proxy 17 | * IPv6 and/or IPv4 support (can act as a 4to6 or 6to4 gateway) 18 | * High Performance 19 | * Working with the "upcoming" web services 20 | * support for unknown HTTP versions 21 | * websocket and TLS support (Upgrade header support) 22 | * Traffic interception 23 | * [SQUID compatible](http://www.squid-cache.org/Doc/config/url_rewrite_program/) interface 24 | * [ICAP like](http://www.faqs.org/rfcs/rfc3507.html) interface via local program 25 | * Support for [HAProxy proxy protocol](http://haproxy.1wt.eu/download/1.5/doc/proxy-protocol.txt) 26 | * Built-in web servers to monitor the proxy via local webpage ( default http://127.0.0.1:8080 ) 27 | * dynamic configuration change 28 | * running information in json format (/json) 29 | 30 | ## Usage 31 | 32 | Start ExaProxy on your local machine and point your browser to 127.0.0.1 port 8000 33 | 34 | ## More Information 35 | 36 | Keep up to date, follow *[twitter](https://twitter.com/#!/search/exaproxy)* or the *[google community](https://plus.google.com/u/0/communities/100651429598143540706)* 37 | 38 | ExaProxy was born out by necessity. No other open source proxy has the same features [RFC compliance](https://github.com/Exa-Networks/exaproxy/wiki/RFC) 39 | 40 | This [presentation](http://www.uknof.org.uk/uknof22/Mangin-ExaProxy.pdf) explains why other solutions were not suitable. 41 | 42 | Development is done on python 2.7. This program has no dependencies on third party libraries and will run out of the box on any Unix system. 43 | 44 | Tested with [Co-Advisor](http://coad.measurement-factory.com/). We are failing HTTP/1.0 conversion requirement and responses modifications (which we do not support as we assume that both the client and server are valid HTTP/1.1 implementation). 45 | 46 | 47 | ## Get it 48 | 49 | ```sh 50 | > wget https://codeload.github.com/Exa-Networks/exaproxy/tar.gz/1.2.1 51 | > tar xzvf 1.2.1 52 | > cd exaproxy-1.2.1 53 | > ./sbin/exaproxy 54 | ``` 55 | 56 | will give you a working proxy on port 3128 57 | 58 | ```sh 59 | > ./sbin/exaproxy -h 60 | ``` 61 | 62 | will give you a definition of all the configuration options 63 | 64 | ```sh 65 | > env exaproxy.tcp4.port=8088./sbin/exaproxy -de 66 | 67 | exaproxy.tcp4.port='8088' 68 | ``` 69 | or 70 | ```sh 71 | > export exaproxy_tcp4_port=8088 72 | > ./sbin/exaproxy -de 73 | 74 | exaproxy.tcp4.port='8088' 75 | ``` 76 | 77 | To change from the command line and see what options were changed from their default configuration values in the configuration file. 78 | -------------------------------------------------------------------------------- /ROADMAP: -------------------------------------------------------------------------------- 1 | BUG Fixes: 2 | - fix all XXX: 3 | 4 | Reliablity: 5 | - look at tox: http://tox.testrun.org/latest/ 6 | - implement the timeout of client sockets. 7 | - warning when we are reaching 90% connection limit 8 | - do not rely on remote Google JS code should the proxy not have no net access ( !!! .... ) 9 | 10 | RFC Compliance 11 | - consider adding a VIA header to the answer from the server (not done to save CPU) 12 | - need to serve a HTTP/1.0 via the built-in webserver if the request is 1.0 13 | - need to remove connection: header if HTTP is 1/0 14 | - support 100-continue ? (Section 8.2.3) 15 | - do not forward 100 response due to Expect:100 if client is HTTP/1.0 (Section 8.2.3) 16 | - pass the REQUEST HTTP version all the way to the HTTP page generated in response 17 | - 414 Request-URI Too Long instead of aborting the request 18 | 19 | Protection 20 | - incomming IP ACL (for allowed users) - for the moment the firewall of the machine will do .. 21 | 22 | Performance 23 | - use ctypes and the linux splice system call for better performance 24 | - remove a double dict lookup in the main loop 25 | - look at python Buffer and Array API 26 | - look at python 2.7 performance for list.join vs string appending 27 | - investigate regex for header parsing performance 28 | - investigate why pypy takes more CPU than cPython in some cases 29 | - split the worker forking to another process as the process can use several GB of memory when it happens 30 | 31 | Cleanup 32 | - refactor the communication over pipe to have the serialisation/deserialisation in one place 33 | - use some structure when passing message for readability 34 | - configuration should not be stored in object as it prevents reload (unless required) 35 | 36 | Feature 37 | - offer an option for happy Eyeballs (RFC 6555) when it comes to IPv6 (http://tools.ietf.org/html/rfc6555) 38 | - fetch the SSL X.509 certificate from IPs to extract its hostname and pass it to the thread when using CONNECT 39 | - more monitoring (self-updating page with realtime rotating logs) 40 | - a redirector which connects to our ICAP service 41 | - option to prever IPv6 DNS answers over IPv4 or vice-versa 42 | - record data transfered ? 43 | - show number of connections per ip on the webpage 44 | - ability for the process to request a recording of the steam 45 | 46 | Fun 47 | - make a redirector which blocks all old version of IE 48 | 49 | -------------------------------------------------------------------------------- /etc/exaproxy/dns/types: -------------------------------------------------------------------------------- 1 | A 1 ipv4 2 | NS 2 string 3 | MD 3 unimplemented 4 | MF 4 unimplemented 5 | CNAME 5 string 6 | SOA 6 string 7 | MB 7 unimplemented 8 | MG 8 unimplemented 9 | MR 9 unimplemented 10 | NULL 10 unimplemented 11 | WKS 11 unimplemented 12 | PTR 12 unimplemented 13 | HINFO 13 unimplemented 14 | MINFO 14 unimplemented 15 | MX 15 unimplemented 16 | TXT 16 unimplemented 17 | RP 17 unimplemented 18 | AFSDB 18 unimplemented 19 | X25 19 unimplemented 20 | ISDN 20 unimplemented 21 | RT 21 unimplemented 22 | NSAP 22 unimplemented 23 | NSAPPTR 23 unimplemented 24 | SIG 24 unimplemented 25 | KEY 25 unimplemented 26 | PX 26 unimplemented 27 | GPOS 27 unimplemented 28 | AAAA 28 ipv6 29 | LOC 29 unimplemented 30 | NXT 30 unimplemented 31 | EID 31 unimplemented 32 | NIMLOC 32 unimplemented 33 | SRV 33 unimplemented 34 | ATMA 34 unimplemented 35 | NAPTR 35 unimplemented 36 | KX 36 unimplemented 37 | CERT 37 unimplemented 38 | A6 38 unimplemented 39 | DNAME 39 unimplemented 40 | SINK 40 unimplemented 41 | OPT 41 unimplemented 42 | APL 42 unimplemented 43 | DS 43 unimplemented 44 | SSHFP 44 unimplemented 45 | IPSECKEY 45 unimplemented 46 | RRSIG 46 unimplemented 47 | NSEC 47 unimplemented 48 | DNSKEY 48 unimplemented 49 | DHCID 49 unimplemented 50 | NSEC3 50 unimplemented 51 | NSEC3PARAM 51 unimplemented 52 | HIP 55 unimplemented 53 | NINFO 56 unimplemented 54 | RKEY 57 unimplemented 55 | TALINK 58 unimplemented 56 | SPF 99 unimplemented 57 | UINFO 100 unimplemented 58 | UID 101 unimplemented 59 | GID 102 unimplemented 60 | UNSPEC 103 unimplemented 61 | TKEY 249 unimplemented 62 | TSIG 250 unimplemented 63 | IXFR 251 unimplemented 64 | AXFR 252 unimplemented 65 | MAILB 253 unimplemented 66 | MAILA 254 unimplemented 67 | STAR 255 unimplemented 68 | DNSSECAUTHORITIES 32768 unimplemented 69 | DNSSECLOOKASIDE 32769 unimplemented 70 | -------------------------------------------------------------------------------- /etc/exaproxy/exaproxy.conf: -------------------------------------------------------------------------------- 1 | 2 | [daemon] 3 | daemonize = false 4 | identifier = ExaProxy 5 | pidfile = '' 6 | poll-interfaces = true 7 | reactor = 'best' 8 | speed = 2 9 | user = 'nobody' 10 | 11 | [dns] 12 | definitions = 'etc/exaproxy/dns/types' 13 | fqdn = true 14 | resolver = '/etc/resolv.conf' 15 | retries = 10 16 | timeout = 2 17 | ttl = 900 18 | 19 | [http] 20 | connect = true 21 | connections = 32768 22 | expect = false 23 | extensions = '' 24 | forward = '' 25 | header-size = 65536 26 | idle-connect = 300 27 | proxied = false 28 | transparent = false 29 | mask = false 30 | 31 | [log] 32 | client = true 33 | configuration = true 34 | daemon = true 35 | destination = 'stdout' 36 | download = true 37 | enable = true 38 | header = true 39 | http = true 40 | level = ERROR 41 | manager = true 42 | resolver = true 43 | server = true 44 | signal = true 45 | supervisor = true 46 | web = true 47 | worker = true 48 | 49 | [profile] 50 | destination = 'stdout' 51 | enable = false 52 | 53 | [redirector] 54 | enable = false 55 | maximum = 25 56 | minimum = 5 57 | program = 'etc/exaproxy/redirector/url-allow' 58 | protocol = 'url' 59 | 60 | [security] 61 | connect = '443 981 7000' 62 | local = '' 63 | 64 | [tcp4] 65 | backlog = 200 66 | bind = '0.0.0.0' 67 | host = '127.0.0.1' 68 | listen = true 69 | out = true 70 | port = 3128 71 | timeout = 5 72 | 73 | [tcp6] 74 | backlog = 200 75 | bind = '::' 76 | host = '::1' 77 | listen = false 78 | out = true 79 | port = 3128 80 | timeout = 5 81 | 82 | [usage] 83 | destination = 'stdout' 84 | enable = false 85 | 86 | [web] 87 | connections = 100 88 | debug = false 89 | enable = true 90 | host = '127.0.0.1' 91 | html = 'etc/exaproxy/html' 92 | port = 8080 93 | -------------------------------------------------------------------------------- /etc/exaproxy/html/images/deny.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Exa-Networks/exaproxy/8b7291b79c1cd6542213a5e7d8dda3cf5a676166/etc/exaproxy/html/images/deny.gif -------------------------------------------------------------------------------- /etc/exaproxy/html/images/deny.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Exa-Networks/exaproxy/8b7291b79c1cd6542213a5e7d8dda3cf5a676166/etc/exaproxy/html/images/deny.jpg -------------------------------------------------------------------------------- /etc/exaproxy/html/images/deny.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Exa-Networks/exaproxy/8b7291b79c1cd6542213a5e7d8dda3cf5a676166/etc/exaproxy/html/images/deny.png -------------------------------------------------------------------------------- /etc/exaproxy/redirector/icap-allow-204: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | icap-allow-204.py 5 | 6 | Created by Thomas Mangin on 2011-11-29. 7 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 8 | """ 9 | 10 | import os 11 | import sys 12 | import time 13 | 14 | 15 | 16 | def debug(s, message=''): 17 | with open('/tmp/debug', 'a+') as w_fd: 18 | w_fd.write(str(time.strftime('%Y-%m-%d %H:%M:%S')) + '\n') 19 | if message: 20 | w_fd.write(message + '\n') 21 | 22 | w_fd.write(str(type(s)) + '\n') 23 | w_fd.write(str(len(str(s))) + ': ' + str(s) + '\n\n') 24 | 25 | 26 | 27 | try: 28 | pid = os.getpid() 29 | 30 | while True: 31 | debug(True, 'new request') 32 | #with open(os.path.join('/tmp/debugging/', str(pid)), 'a+') as w_fd: 33 | #w_fd.write('READING:\n') 34 | line = sys.stdin.readline() 35 | debug(line, 'first line') 36 | #w_fd.write(line) 37 | method,uri,version = line.rstrip().split() 38 | if method != 'REQMOD': 39 | sys.stderr.write('ICAP METHOD not supported %s\n' % method) 40 | sys.stderr.flush() 41 | if version != 'ICAP/1.0': 42 | sys.stderr.write('ICAP version not supported %s\n' % version) 43 | sys.stderr.flush() 44 | 45 | 46 | content_length = 0 47 | host = '' 48 | 49 | while True: 50 | line = sys.stdin.readline() 51 | 52 | if 'null-body=' in line: 53 | debug(line, 'line') 54 | content_length=int(line.rsplit('=', 1)[1]) 55 | debug(content_length, 'content length') 56 | 57 | if line.startswith('Pragma: host='): 58 | host = line.split('=', 1)[1].strip() 59 | 60 | if not line.strip(): 61 | break 62 | 63 | if not content_length: 64 | raise Exception, 'could not extract body length' 65 | 66 | while content_length > 0: 67 | data = sys.stdin.read(content_length) 68 | content_length = content_length - len(data) 69 | 70 | if not data: 71 | raise Exception, 'file closed before all data was read' 72 | 73 | host = host.strip() 74 | debug(host, 'HOST') 75 | 76 | 77 | 78 | if host == 'www.google.com': 79 | headers = """\ 80 | GET / redirect.php HTTP/1.1 81 | Host: www.surfprotect.co.uk 82 | Connection: close 83 | 84 | """ 85 | reply = """\ 86 | ICAP/1.0 302 OK 87 | Encapsulated: res-hdr=0, null-body=%d 88 | 89 | %s""" % (len(headers),headers) 90 | 91 | 92 | sys.stdout.write(reply) 93 | sys.stdout.flush() 94 | 95 | else: 96 | sys.stdout.write("ICAP/1.0 204 OK\n\n") 97 | sys.stdout.flush() 98 | except KeyboardInterrupt, e: 99 | debug(e, 'keyboard interrupt') 100 | sys.stderr.write('^C keyboard interrupt. exiting.\n') 101 | sys.stderr.flush() 102 | 103 | except Exception, e: 104 | debug(e, 'exception') 105 | sys.stderr.write('CHILD FAILED %s\n' % str(e)) 106 | sys.stderr.flush() 107 | -------------------------------------------------------------------------------- /etc/exaproxy/redirector/icap-allow-304: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | icap-allow-304.py 5 | 6 | Created by Thomas Mangin on 2011-11-29. 7 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 8 | """ 9 | 10 | import sys 11 | 12 | try: 13 | while True: 14 | line = sys.stdin.readline() 15 | method,uri,version = line.rstrip().split() 16 | if method != 'REQMOD': 17 | sys.stderr.write('ICAP METHOD not supported %s\n' % method) 18 | sys.stderr.flush() 19 | if version != 'ICAP/1.0': 20 | sys.stderr.write('ICAP version not supported %s\n' % version) 21 | sys.stderr.flush() 22 | 23 | # consume the message 24 | empty = 2 25 | while empty: 26 | line = sys.stdin.readline() 27 | if not line.rstrip(): 28 | empty -= 1 29 | 30 | sys.stdout.write("ICAP/1.0 304 OK\n\n") 31 | sys.stdout.flush() 32 | except KeyboardInterrupt, e: 33 | sys.stderr.write('^C keyboard interrupt. exiting.\n') 34 | sys.stderr.flush() 35 | except Exception, e: 36 | sys.stderr.write('CHILD FAILED %s\n' % str(e)) 37 | sys.stderr.flush() 38 | -------------------------------------------------------------------------------- /etc/exaproxy/redirector/icap-deny: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | icap-deny.py 5 | 6 | Created by Thomas Mangin on 2011-11-29. 7 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 8 | """ 9 | 10 | # redirect.php is a simple to force hostname : 11 | # 14 | # ?> 15 | 16 | import sys 17 | 18 | try: 19 | while True: 20 | line = sys.stdin.readline() 21 | method,uri,version = line.rstrip().split() 22 | if method != 'REQMOD': 23 | sys.stderr.write('ICAP METHOD not supported %s\n' % method) 24 | sys.stderr.flush() 25 | if version != 'ICAP/1.0': 26 | sys.stderr.write('ICAP version not supported %s\n' % version) 27 | sys.stderr.flush() 28 | 29 | client = '' 30 | while True: 31 | line = sys.stdin.readline().rstrip() 32 | if not line: 33 | break 34 | if line.startswith('Pragma: client='): 35 | client = line[15] 36 | last = line 37 | 38 | # we know ExaProxy send the Encapsulation as last parameter .. yes we do :) 39 | length = int(last.rstrip().split('=')[-1]) 40 | _ = sys.stdin.read(length) 41 | 42 | if _[:7].upper() == 'CONNECT': 43 | headers = _ if 'exa-networks.co.uk' in _ else """\ 44 | CONNECT www.exa-networks.co.uk:443 HTTP/1.1 45 | Proxy-Connection: keep-alive 46 | Host: www.exa-networks.co.uk 47 | 48 | """ 49 | else: 50 | headers = _ if 'surfprotect.co.uk' in _ else """\ 51 | GET /redirect.php HTTP/1.1 52 | Host: www.surfprotect.co.uk 53 | Connection: close 54 | 55 | """ 56 | # Always return the Encapsulated size as last header and with a null-body 57 | reply = """\ 58 | ICAP/1.0 302 OK 59 | Encapsulated: res-hdr=0, null-body=%d 60 | 61 | %s""" % (len(headers),headers) 62 | sys.stdout.write(reply) 63 | sys.stdout.flush() 64 | except KeyboardInterrupt, e: 65 | sys.stderr.write('^C keyboard interrupt. exiting.\n') 66 | sys.stderr.flush() 67 | except Exception, e: 68 | sys.stderr.write('CHILD FAILED %s\n' % str(e)) 69 | sys.stderr.flush() 70 | -------------------------------------------------------------------------------- /etc/exaproxy/redirector/icap-deny-proxy-explanation: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | icap-deny.py 5 | 6 | Created by Thomas Mangin on 2011-11-29. 7 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 8 | """ 9 | 10 | # redirect.php is a simple to force hostname : 11 | # 14 | # ?> 15 | 16 | import sys 17 | 18 | try: 19 | while True: 20 | line = sys.stdin.readline() 21 | method,uri,version = line.rstrip().split() 22 | if method != 'REQMOD': 23 | sys.stderr.write('ICAP METHOD not supported %s\n' % method) 24 | sys.stderr.flush() 25 | if version != 'ICAP/1.0': 26 | sys.stderr.write('ICAP version not supported %s\n' % version) 27 | sys.stderr.flush() 28 | 29 | client = '' 30 | while True: 31 | line = sys.stdin.readline().rstrip() 32 | if not line: 33 | break 34 | if line.startswith('Pragma: client='): 35 | client = line[15] 36 | last = line 37 | 38 | # we know ExaProxy send the Encapsulation as last parameter .. yes we do :) 39 | length = int(last.rstrip().split('=')[-1]) 40 | _ = sys.stdin.read(length) 41 | 42 | headers = """\ 43 | HTTP/1.1 403 Forbidden 44 | Content-Type: application/proxy-explanation+json 45 | Cache-Control: no-cache 46 | Connection: close 47 | 48 | { 49 | "name": "Exa Networks" 50 | "title": "Policy Violation" 51 | "description": "This content is _way_ above your pay grade." 52 | "moreinfo": "https://exa.net.uk/" 53 | } 54 | 55 | """ 56 | # Always return the Encapsulated size as last header and with a null-body 57 | reply = """\ 58 | ICAP/1.0 200 OK 59 | Encapsulated: res-hdr=0, null-body=%d 60 | 61 | %s""" % (len(headers),headers) 62 | sys.stdout.write(reply) 63 | sys.stdout.flush() 64 | except KeyboardInterrupt, e: 65 | sys.stderr.write('^C keyboard interrupt. exiting.\n') 66 | sys.stderr.flush() 67 | except Exception, e: 68 | sys.stderr.write('CHILD FAILED %s\n' % str(e)) 69 | sys.stderr.flush() 70 | -------------------------------------------------------------------------------- /etc/exaproxy/redirector/icap-google-no-ssl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | icap-deny.py 5 | 6 | Created by Thomas Mangin on 2011-11-29. 7 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 8 | """ 9 | 10 | # redirect.php is a simple to force hostname : 11 | # 14 | # ?> 15 | 16 | import sys 17 | 18 | try: 19 | while True: 20 | line = sys.stdin.readline() 21 | method,uri,version = line.rstrip().split() 22 | if method != 'REQMOD': 23 | sys.stderr.write('ICAP METHOD not supported %s\n' % method) 24 | sys.stderr.flush() 25 | if version != 'ICAP/1.0': 26 | sys.stderr.write('ICAP version not supported %s\n' % version) 27 | sys.stderr.flush() 28 | 29 | client = '' 30 | while True: 31 | line = sys.stdin.readline().rstrip() 32 | if not line: 33 | break 34 | if line.startswith('Pragma: client='): 35 | client = line[15] 36 | last = line 37 | 38 | # we know ExaProxy send the Encapsulation as last parameter .. yes we do :) 39 | length = int(last.rstrip().split('=')[-1]) 40 | _ = sys.stdin.read(length) 41 | 42 | if _.startswith('CONNECT www.google.'): 43 | headers = """\ 44 | CONNECT nosslsearch.google.com:443 HTTP/1.1 45 | Proxy-Connection: keep-alive 46 | Host: nosslsearch.google.com 47 | 48 | """ 49 | else: 50 | headers = _ 51 | # Always return the Encapsulated size as last header and with a null-body 52 | reply = """\ 53 | ICAP/1.0 200 OK 54 | Encapsulated: req-hdr=0, null-body=%d 55 | 56 | %s""" % (len(headers),headers) 57 | #print >> sys.stderr, reply 58 | sys.stdout.write(reply) 59 | sys.stdout.flush() 60 | except KeyboardInterrupt, e: 61 | sys.stderr.write('^C keyboard interrupt. exiting.\n') 62 | sys.stderr.flush() 63 | except Exception, e: 64 | sys.stderr.write('CHILD FAILED %s\n' % str(e)) 65 | sys.stderr.flush() 66 | -------------------------------------------------------------------------------- /etc/exaproxy/redirector/icap-google-to-others: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | icap-deny.py 5 | 6 | Created by Thomas Mangin on 2011-11-29. 7 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 8 | """ 9 | 10 | # redirect.php is a simple to force hostname : 11 | # 14 | # ?> 15 | 16 | import sys 17 | 18 | try: 19 | while True: 20 | line = sys.stdin.readline() 21 | method,uri,version = line.rstrip().split() 22 | if method != 'REQMOD': 23 | sys.stderr.write('ICAP METHOD not supported %s\n' % method) 24 | sys.stderr.flush() 25 | if version != 'ICAP/1.0': 26 | sys.stderr.write('ICAP version not supported %s\n' % version) 27 | sys.stderr.flush() 28 | 29 | client = '' 30 | while True: 31 | line = sys.stdin.readline().rstrip() 32 | if not line: 33 | break 34 | if line.startswith('Pragma: client='): 35 | client = line[15] 36 | last = line 37 | 38 | # we know ExaProxy send the Encapsulation as last parameter .. yes we do :) 39 | length = int(last.rstrip().split('=')[-1]) 40 | _ = sys.stdin.read(length) 41 | 42 | request = _.split('\n')[0] 43 | if 'GET' in request and 'www.google.' in request: 44 | headers = """\ 45 | CONNECT www.wolframalpha.com:80 HTTP/1.1 46 | Proxy-Connection: keep-alive 47 | Host: www.wolframalpha.com 48 | 49 | %s""" % _ 50 | else: 51 | headers = _ 52 | # Always return the Encapsulated size as last header and with a null-body 53 | reply = """\ 54 | ICAP/1.0 200 OK 55 | Encapsulated: res-hdr=0, null-body=%d 56 | 57 | %s""" % (len(headers),headers) 58 | sys.stdout.write(reply) 59 | sys.stdout.flush() 60 | except KeyboardInterrupt, e: 61 | sys.stderr.write('^C keyboard interrupt. exiting.\n') 62 | sys.stderr.flush() 63 | except Exception, e: 64 | sys.stderr.write('CHILD FAILED %s\n' % str(e)) 65 | sys.stderr.flush() 66 | -------------------------------------------------------------------------------- /etc/exaproxy/redirector/icap-safe-youtube: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | icap-safe-youtube.py 5 | 6 | Created by Thomas Mangin on 2011-11-29. 7 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 8 | """ 9 | 10 | import re 11 | import sys 12 | 13 | try: 14 | while True: 15 | line = sys.stdin.readline() 16 | method,uri,version = line.rstrip().split() 17 | if method != 'REQMOD': 18 | sys.stderr.write('ICAP METHOD not supported %s\n' % method) 19 | sys.stderr.flush() 20 | if version != 'ICAP/1.0': 21 | sys.stderr.write('ICAP version not supported %s\n' % version) 22 | sys.stderr.flush() 23 | 24 | client = '' 25 | while True: 26 | line = sys.stdin.readline().rstrip() 27 | if not line: 28 | break 29 | if line.startswith('Pragma: client='): 30 | client = line[15] 31 | last = line 32 | 33 | # we know ExaProxy send the Encapsulation as last parameter .. yes we do :) 34 | length = int(last.rstrip().split('=')[-1]) 35 | _ = sys.stdin.read(length) 36 | upper = _.upper() 37 | 38 | if 'CONNECT ' in upper: 39 | headers = _ 40 | elif 'youtube.' in _: 41 | headers = re.sub("([Cc]ookie:.*[;\\s])PREF=([^;]*)(.*)","\\1PREF=f2=8000000&\\2; \\3",_) 42 | else: 43 | headers = _ 44 | 45 | reply = """\ 46 | ICAP/1.0 204 No Content 47 | Encapsulated: res-hdr=0, null-body=%d 48 | 49 | %s""" % (len(headers),headers) 50 | sys.stdout.write(reply) 51 | sys.stdout.flush() 52 | except KeyboardInterrupt, e: 53 | sys.stderr.write('^C keyboard interrupt. exiting.\n') 54 | sys.stderr.flush() 55 | except Exception, e: 56 | sys.stderr.write('CHILD FAILED %s\n' % str(e)) 57 | sys.stderr.flush() 58 | -------------------------------------------------------------------------------- /etc/exaproxy/redirector/icap-serve: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | icap-deny.py 5 | 6 | Created by Thomas Mangin on 2011-11-29. 7 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 8 | """ 9 | 10 | # redirect.php is a simple to force hostname : 11 | # 14 | # ?> 15 | 16 | import sys 17 | 18 | try: 19 | while True: 20 | line = sys.stdin.readline() 21 | method,uri,version = line.rstrip().split() 22 | if method != 'REQMOD': 23 | sys.stderr.write('ICAP METHOD not supported %s\n' % method) 24 | sys.stderr.flush() 25 | if version != 'ICAP/1.0': 26 | sys.stderr.write('ICAP version not supported %s\n' % version) 27 | sys.stderr.flush() 28 | 29 | client = '' 30 | while True: 31 | line = sys.stdin.readline().rstrip() 32 | if not line: 33 | break 34 | if line.startswith('Pragma: client='): 35 | client = line[15] 36 | last = line 37 | 38 | # we know ExaProxy send the Encapsulation as last parameter .. yes we do :) 39 | length = int(last.rstrip().split('=')[-1]) 40 | _ = sys.stdin.read(length) 41 | 42 | headers = """\ 43 | HTTP/1.1 200 OK 44 | 45 | """ 46 | 47 | 48 | body = """\ 49 | 50 | 51 | 52 | 53 | no, you are stuck on this dummy page
54 | this dummy page is your life ... 55 | 56 | 57 | """ 58 | # Always return the Encapsulated size as last header and with a null-body 59 | reply = """\ 60 | ICAP/1.0 200 OK 61 | Encapsulated: res-hdr=0, res-body=%d 62 | 63 | %s""" % (len(headers),headers + ('%x\n' % len(body)) + body + '0\n') 64 | sys.stdout.write(reply) 65 | sys.stdout.flush() 66 | except KeyboardInterrupt, e: 67 | sys.stderr.write('^C keyboard interrupt. exiting.\n') 68 | sys.stderr.flush() 69 | except Exception, e: 70 | sys.stderr.write('CHILD FAILED %s\n' % str(e)) 71 | sys.stderr.flush() 72 | -------------------------------------------------------------------------------- /etc/exaproxy/redirector/icap-serve-file: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | icap-deny.py 5 | 6 | Created by Thomas Mangin on 2011-11-29. 7 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 8 | """ 9 | 10 | # redirect.php is a simple to force hostname : 11 | # 14 | # ?> 15 | 16 | import sys 17 | 18 | try: 19 | while True: 20 | line = sys.stdin.readline() 21 | method,uri,version = line.rstrip().split() 22 | if method != 'REQMOD': 23 | sys.stderr.write('ICAP METHOD not supported %s\n' % method) 24 | sys.stderr.flush() 25 | if version != 'ICAP/1.0': 26 | sys.stderr.write('ICAP version not supported %s\n' % version) 27 | sys.stderr.flush() 28 | 29 | client = '' 30 | while True: 31 | line = sys.stdin.readline().rstrip() 32 | if not line: 33 | break 34 | if line.startswith('Pragma: client='): 35 | client = line[15] 36 | last = line 37 | 38 | # we know ExaProxy send the Encapsulation as last parameter .. yes we do :) 39 | length = int(last.rstrip().split('=')[-1]) 40 | _ = sys.stdin.read(length) 41 | 42 | headers = """\ 43 | GET file://success.html HTTP/1.1 44 | 45 | """ 46 | # Always return the Encapsulated size as last header and with a null-body 47 | reply = """\ 48 | ICAP/1.0 200 OK 49 | Encapsulated: res-hdr=0, null-body=%d 50 | 51 | %s""" % (len(headers),headers) 52 | sys.stdout.write(reply) 53 | sys.stdout.flush() 54 | except KeyboardInterrupt, e: 55 | sys.stderr.write('^C keyboard interrupt. exiting.\n') 56 | sys.stderr.flush() 57 | except Exception, e: 58 | sys.stderr.write('CHILD FAILED %s\n' % str(e)) 59 | sys.stderr.flush() 60 | -------------------------------------------------------------------------------- /etc/exaproxy/redirector/url-allow: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | allow.py 5 | 6 | Created by Thomas Mangin on 2011-11-29. 7 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 8 | """ 9 | 10 | import sys 11 | 12 | try: 13 | while True: 14 | data = sys.stdin.readline() 15 | sys.stdout.write('\n') 16 | sys.stdout.flush() 17 | except KeyboardInterrupt, e: 18 | sys.stderr.write('^C keyboard interrupt. exiting.\n') 19 | sys.stderr.flush() 20 | except Exception, e: 21 | sys.stderr.write('CHILD FAILED %s\n' % str(e)) 22 | sys.stderr.flush() 23 | -------------------------------------------------------------------------------- /etc/exaproxy/redirector/url-allow-net: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | allow.py 5 | 6 | Created by Thomas Mangin on 2011-11-29. 7 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 8 | """ 9 | 10 | import sys 11 | 12 | def reply (response): 13 | sys.stdout.write(response + '\n') 14 | sys.stdout.flush() 15 | 16 | try: 17 | while True: 18 | data = sys.stdin.readline().strip() 19 | 20 | url, ip, _, method, __ = data.split() 21 | 22 | if method == 'CONNECT': 23 | reply('http://www.surfprotect.co.uk/') 24 | continue 25 | if url.endswith('.net'): 26 | reply('') 27 | continue 28 | reply('http://www.surfprotect.co.uk/') 29 | 30 | except KeyboardInterrupt, e: 31 | sys.stderr.write('^C keyboard interrupt. exiting.\n') 32 | sys.stderr.flush() 33 | except Exception, e: 34 | sys.stderr.write('CHILD FAILED %s\n' % str(e)) 35 | sys.stderr.flush() 36 | -------------------------------------------------------------------------------- /etc/exaproxy/redirector/url-deny: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | deny.py 5 | 6 | Created by Thomas Mangin on 2012-01-14. 7 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 8 | """ 9 | 10 | import os 11 | import sys 12 | 13 | cwd = os.getcwd() 14 | prg = sys.argv[0] 15 | 16 | if not prg.startswith('/'): # Unix only - we are bad .. 17 | location = os.path.normpath(os.path.join(cwd,prg)) 18 | else: 19 | location = os.path.normpath(prg) 20 | 21 | deny = os.path.join(os.path.join(os.sep,*os.path.join(location.split(os.sep)[:-2])),'html','deny.html') 22 | 23 | try: 24 | while True: 25 | data = sys.stdin.readline() 26 | url = data.split(' ')[0] 27 | sys.stdout.write('file://%s\r\n' % deny) 28 | sys.stdout.flush() 29 | except KeyboardInterrupt, e: 30 | sys.stderr.write('^C keyboard interrupt. exiting.\n') 31 | sys.stderr.flush() 32 | except Exception, e: 33 | sys.stderr.write('CHILD FAILED %s\n' % str(e)) 34 | sys.stderr.flush() 35 | -------------------------------------------------------------------------------- /etc/exaproxy/redirector/url-google-no-ssl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | url-goole-no-ssl.py 5 | 6 | Created by Thomas Mangin on 2012-01-14. 7 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 8 | """ 9 | 10 | import os 11 | import sys 12 | 13 | cwd = os.getcwd() 14 | prg = sys.argv[0] 15 | 16 | try: 17 | while True: 18 | data = sys.stdin.readline() 19 | if 'CONNECT' in data and 'www.google.' in data: 20 | sys.stdout.write('intercept://%s' % 'nosslsearch.google.com\n') 21 | sys.stdout.flush() 22 | continue 23 | sys.stdout.write('\n') 24 | sys.stdout.flush() 25 | except KeyboardInterrupt, e: 26 | sys.stderr.write('^C keyboard interrupt. exiting.\n') 27 | sys.stderr.flush() 28 | except Exception, e: 29 | sys.stderr.write('CHILD FAILED %s\n' % str(e)) 30 | sys.stderr.flush() 31 | -------------------------------------------------------------------------------- /etc/exaproxy/redirector/url-google-to-others: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | url-goole-no-ssl.py 5 | 6 | Created by Thomas Mangin on 2012-01-14. 7 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 8 | """ 9 | 10 | import os 11 | import sys 12 | 13 | cwd = os.getcwd() 14 | prg = sys.argv[0] 15 | 16 | try: 17 | while True: 18 | data = sys.stdin.readline() 19 | if 'CONNECT' not in data and 'www.google.' in data: 20 | sys.stdout.write('redirect://%s' % 'http://bing.com\n') 21 | sys.stdout.flush() 22 | continue 23 | # This is scandalous that browser let you do that ! 24 | if 'CONNECT' in data and 'www.google.' in data: 25 | sys.stdout.write('redirect://%s' % 'https://www.wolframalpha.com\n') 26 | sys.stdout.flush() 27 | continue 28 | sys.stdout.write('\n') 29 | sys.stdout.flush() 30 | except KeyboardInterrupt, e: 31 | sys.stderr.write('^C keyboard interrupt. exiting.\n') 32 | sys.stderr.flush() 33 | except Exception, e: 34 | sys.stderr.write('CHILD FAILED %s\n' % str(e)) 35 | sys.stderr.flush() 36 | -------------------------------------------------------------------------------- /etc/exaproxy/redirector/url-parrot: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | parrot.py 5 | 6 | Created by Thomas Mangin on 2011-11-29. 7 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 8 | """ 9 | 10 | import sys 11 | 12 | try: 13 | while True: 14 | data = sys.stdin.readline() 15 | url = data.split(' ')[0] 16 | sys.stdout.write('%s\r\n' % url) 17 | sys.stdout.flush() 18 | except KeyboardInterrupt, e: 19 | sys.stderr.write('^C keyboard interrupt. exiting.\n') 20 | sys.stderr.flush() 21 | except Exception, e: 22 | sys.stderr.write('CHILD FAILED %s\n' % str(e)) 23 | sys.stderr.flush() 24 | -------------------------------------------------------------------------------- /etc/exaproxy/redirector/url-serve-file: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | parrot.py 5 | 6 | Created by Thomas Mangin on 2011-11-29. 7 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 8 | """ 9 | 10 | import sys 11 | 12 | try: 13 | while True: 14 | data = sys.stdin.readline() 15 | sys.stdout.write('file://success.html\r\n') 16 | sys.stdout.flush() 17 | except KeyboardInterrupt, e: 18 | sys.stderr.write('^C keyboard interrupt. exiting.\n') 19 | sys.stderr.flush() 20 | except Exception, e: 21 | sys.stderr.write('CHILD FAILED %s\n' % str(e)) 22 | sys.stderr.flush() 23 | -------------------------------------------------------------------------------- /etc/systemd/exaproxy.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=ExaProxy 3 | After=network.target 4 | 5 | [Service] 6 | Environment=exaproxy.daemon.daemonize=false 7 | Environment=exaproxy.log.destination=stdout 8 | ExecStart=/usr/bin/exaproxy 9 | 10 | [Install] 11 | WantedBy=multi-user.target 12 | -------------------------------------------------------------------------------- /lib/exaproxy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Exa-Networks/exaproxy/8b7291b79c1cd6542213a5e7d8dda3cf5a676166/lib/exaproxy/__init__.py -------------------------------------------------------------------------------- /lib/exaproxy/dns/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | __init__.py 4 | 5 | Created by David Farrar on 2012-02-08. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | -------------------------------------------------------------------------------- /lib/exaproxy/dns/convert.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | convert.py 4 | 5 | Created by David Farrar on 2012-02-08. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | from struct import unpack 10 | import socket 11 | #import array 12 | 13 | def u8(s): 14 | return ord(s) 15 | 16 | def u16(s): 17 | return unpack('>H', s)[0] 18 | 19 | def u32(s): 20 | return unpack('>I', s)[0] 21 | 22 | def dns_string(s): 23 | parts = [] 24 | ptr = None 25 | remaining = len(s) 26 | bytes_read = 0 27 | 28 | while s: 29 | length = u8(s[0]) 30 | remaining -= length + 1 31 | 32 | if length >= 0xc0: 33 | ptr = ((length - 0xc0)<<8) + u8(s[1]) 34 | bytes_read += 2 35 | break 36 | 37 | if length == 0: 38 | bytes_read += 1 39 | break 40 | 41 | if remaining <= 0: 42 | bytes_read = None 43 | parts = [] 44 | break 45 | 46 | bytes_read += length + 1 47 | 48 | parts.append(s[1:1+length]) 49 | s = s[1+length:] 50 | else: 51 | parts = [] 52 | ptr = None 53 | 54 | return bytes_read, '.'.join(parts) if parts is not None else None, ptr 55 | 56 | def dns_to_ipv4(ip, packet_s): 57 | return socket.inet_ntoa(ip) 58 | 59 | def ipv4_to_dns(ip, packet_s): 60 | return socket.inet_aton(ip) 61 | 62 | def dns_to_ipv6(ip, packet_s): 63 | return socket.inet_ntop(socket.AF_INET6, ip) 64 | 65 | def ipv6_to_dns(s, packet_s): 66 | return socket.inet_pton(socket.AF_INET6, s) 67 | 68 | def dns_to_string_info (s, packet_s): 69 | bytes_read, value, ptr = dns_string(s) 70 | 71 | parts = [value] if value else [] 72 | while ptr is not None: 73 | _, value, ptr = dns_string(packet_s[ptr:]) 74 | 75 | if value: 76 | parts.append(value) 77 | 78 | elif value is None: 79 | parts = None 80 | break 81 | 82 | if sum(map(len, parts)) > 500: 83 | parts = None 84 | break 85 | 86 | return bytes_read, '.'.join(parts) if parts is not None else None 87 | 88 | def dns_to_string (s, packet_s): 89 | _, value_s = dns_to_string_info(s, packet_s) 90 | return value_s 91 | 92 | def string_to_dns(s, packet_s=None): 93 | try: 94 | parts = (s.rstrip('.') + '.').split('.') 95 | res = ''.join('%c%s' % (len(p), p) for p in parts) 96 | except OverflowError: 97 | res = None 98 | 99 | return res 100 | -------------------------------------------------------------------------------- /lib/exaproxy/dns/definition.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | definition.py 4 | 5 | Created by David Farrar on 2012-02-08. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | import random 10 | import dnstype 11 | 12 | # OPCODE: Operation Type, 4 bits 13 | # 0: QUERY, Standary query RFC 1035 14 | # 1: IQUERY, Inverse query RFC 1035, RFC 3425 15 | # 2: STATUS, Server status request RFC 1035 16 | # 3: 17 | # 4: Notify RFC 1996 18 | # 5: Update RFC 2136 19 | # 6: RESERVED 20 | # ... RESERVED 21 | # 15: RESERVED 22 | 23 | 24 | #OPCODE, AA, TC, RD, RA, Z, AD, CD, RCODE 25 | 26 | 27 | class DNSBaseType: 28 | QR = None 29 | OPCODE = 0 # Operation type 30 | 31 | def __init__(self, identifier): 32 | self.identifier = identifier 33 | 34 | 35 | class DNSRequestType(DNSBaseType): 36 | QR = 0 # Query 37 | OPCODE = 0 # Query 38 | 39 | resource_factory = dnstype.DNSTypeFactory() 40 | 41 | @property 42 | def query_len(self): 43 | return len(self.queries) 44 | 45 | def __init__(self, identifier, queries=[]): 46 | self.identifier = identifier 47 | self.queries = queries or [] 48 | self.flags = 256 # recursion desired 49 | 50 | def addQuestion(self, querytype, question): 51 | q = self.resource_factory.createQuery(querytype, question) 52 | self.queries.append(q) 53 | 54 | def __str__(self): 55 | query_s = "\n\t ".join(str(q) for q in self.queries) 56 | 57 | return """DNS REQUEST %(id)s 58 | QUERIES: %(queries)s""" % {'id':self.identifier, 'queries':query_s} 59 | 60 | 61 | 62 | class DNSResponseType(DNSBaseType): 63 | QR = 1 # Response 64 | OPCODE = 0 65 | 66 | def __init__(self, identifier, complete, queries=[], responses=[], authorities=[], additionals=[]): 67 | ok = complete is True and None not in (identifier, queries, responses, authorities, additionals) 68 | 69 | self.identifier = identifier 70 | self.complete = bool(complete) 71 | self.queries = (queries or []) if ok else [] 72 | self.responses = (responses or []) if ok else [] 73 | self.authorities = (authorities or []) if ok else [] 74 | self.additionals = (additionals or []) if ok else [] 75 | 76 | if self.queries: 77 | query = self.queries[0] 78 | self.qtype = query.querytype 79 | self.qhost = query.question 80 | else: 81 | self.qtype = None 82 | self.qhost = None 83 | 84 | @property 85 | def query_len (self): 86 | return len(self.queries) 87 | 88 | @property 89 | def response_len (self): 90 | return len(self.responses) 91 | 92 | @property 93 | def authority_len (self): 94 | return len(self.authorities) 95 | 96 | @property 97 | def additional_len (self): 98 | return len(self.additionals) 99 | 100 | @property 101 | def resources (self): 102 | for resource in self.responses: 103 | yield resource 104 | 105 | for resource in self.authorities: 106 | yield resource 107 | 108 | for resource in self.additionals: 109 | yield resource 110 | 111 | def getResponse(self): 112 | info = {} 113 | 114 | for response in self.responses: 115 | info.setdefault(response.question, {}).setdefault(response.querytype, []).append(response.response) 116 | 117 | for response in self.authorities: 118 | info.setdefault(response.question, {}).setdefault(response.querytype, []).append(response.response) 119 | 120 | for response in self.additionals: 121 | info.setdefault(response.question, {}).setdefault(response.querytype, []).append(response.response) 122 | 123 | return info 124 | 125 | def extract(self, hostname, rdtype, info, seen=[]): 126 | data = info.get(hostname) 127 | 128 | if data: 129 | if rdtype in data: 130 | value = random.choice(data[rdtype]) 131 | else: 132 | value = None 133 | else: 134 | value = None 135 | 136 | return value 137 | 138 | def getValue(self, question=None, qtype=None): 139 | if question is None or qtype is None: 140 | if self.queries: 141 | query = self.queries[0] 142 | 143 | if question is None: 144 | question = query.question 145 | 146 | if qtype is None: 147 | qtype = query.querytype 148 | 149 | info = self.getResponse() 150 | return qtype, self.extract(question, qtype, info) 151 | 152 | def getChainedValue(self): 153 | cname = None 154 | 155 | if self.queries: 156 | qtype = 'CNAME' 157 | question = self.queries[0].question 158 | 159 | while question is not None and qtype == 'CNAME': 160 | cname = question 161 | qtype, question = self.getValue(question, qtype) 162 | 163 | return self.getValue(cname) 164 | 165 | def getRelated (self): 166 | for response in self.responses: 167 | if response.querytype == 'CNAME': 168 | related = response.response 169 | break 170 | else: 171 | related = None 172 | 173 | return related 174 | 175 | def isComplete(self): 176 | return self.complete 177 | 178 | def __str__(self): 179 | query_s = "\n".join('\t' + str(q) for q in self.queries) 180 | response_s = "\n\t".join('\t' + str(r) for r in self.responses) 181 | authority_s = "\n\t".join('\t' + str(r) for r in self.authorities) 182 | additional_s = "\n\t".join('\t' + str(r) for r in self.additionals) 183 | 184 | return """DNS RESPONSE %(id)s 185 | QUERIES: %(queries)s 186 | RESPONSES: %(response)s 187 | AUTHORITIES: %(authorities)s 188 | ADDITIONAL: %(additional)s""" % {'id':self.identifier, 'queries':query_s, 'authorities':authority_s, 'additional':additional_s, 'response':response_s} 189 | -------------------------------------------------------------------------------- /lib/exaproxy/dns/dnstype.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | querytype.py 4 | 5 | Created by David Farrar on 2012-02-08. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | import convert 10 | 11 | class DNSType: 12 | dnsclass = None 13 | querytype = None 14 | question = None 15 | 16 | def __str__(self): 17 | return str(self.question) 18 | 19 | class DNSQueryType(DNSType): 20 | def __init__(self, querytype, question, dnsclass=1): 21 | self.dnsclass = dnsclass 22 | self.querytype = querytype 23 | self.question = question 24 | 25 | def __str__(self): 26 | return "Query of type %s for %s" % (self.querytype, self.question) 27 | 28 | class DNSResourceType(DNSType): 29 | def __init__(self, querytype, question, response, ttl, dnsclass=1): 30 | self.dnsclass = dnsclass 31 | self.querytype = querytype 32 | self.question = question 33 | self.response = response 34 | self.ttl = ttl 35 | 36 | def __str__(self): 37 | return "Resource of type %s for %s: %s" % (self.querytype, self.question, self.response) 38 | 39 | 40 | conversion = { 41 | 'ipv4' : (convert.ipv4_to_dns, convert.dns_to_ipv4), 42 | 'ipv6' : (convert.ipv6_to_dns, convert.dns_to_ipv6), 43 | 'string' : (convert.string_to_dns, convert.dns_to_string), 44 | 'unimplemented' : (None, None), 45 | } 46 | 47 | 48 | 49 | class DNSTypeFactory: 50 | CLASS = 1 # Internet 51 | 52 | @staticmethod 53 | def createQuery(name, question): 54 | return DNSQueryType(name, question) 55 | 56 | @staticmethod 57 | def createResource(name, question, response, ttl): 58 | return DNSResourceType(name, question, response, ttl) 59 | 60 | 61 | 62 | class DNSTypeCodec: 63 | CLASS = 1 # Internet 64 | 65 | def __init__(self, definitions): 66 | self.byname = {} 67 | self.byvalue = {} 68 | 69 | self.parseConfiguration(definitions) 70 | 71 | def parseConfiguration(self, filename): 72 | try: 73 | with open(filename) as fd: 74 | for line in fd: 75 | line = line.strip().split('#', 1)[0] 76 | if line.startswith('#'): 77 | continue 78 | 79 | name, value, querytype = line.split() 80 | value = int(value) 81 | 82 | if name in self.byname: 83 | raise ValueError('Configuration file defines record of type %s more than once' % name) 84 | 85 | if value in self.byvalue: 86 | raise ValueError('Configuration file defines record with value %s more than once' % value) 87 | 88 | if querytype not in conversion: 89 | raise ValueError('Configuration file uses undefined type: %s' % querytype) 90 | 91 | encoder, decoder = conversion[querytype] 92 | 93 | self.byname[name] = value, encoder 94 | self.byvalue[value] = name, decoder 95 | 96 | except (IndexError, ValueError): 97 | raise RuntimeError('Corrupt DNS type definition') 98 | except TypeError: 99 | raise RuntimeError('Corrupt DNS type definition') 100 | except IOError: 101 | raise RuntimeError('Cannot read DNS type definition file: %s' % filename) 102 | 103 | def decodeQuery (self, value, question, data_s=''): 104 | name, decoder = self.byvalue.get(value, (None, None)) 105 | return DNSQueryType(name, question) 106 | 107 | def encodeQuery (self, query, data_s=''): 108 | value, encoder = self.byname.get(query.querytype, (None, None)) 109 | return value, query.question if value is not None else None 110 | 111 | def decodeResource (self, value, question, response, ttl, data_s=''): 112 | name, decoder = self.byvalue.get(value, (None, None)) 113 | if name is not None and decoder is not None: 114 | decoded = decoder(response, data_s) 115 | else: 116 | decoded = None 117 | 118 | return DNSResourceType(name, question, decoded, ttl) 119 | 120 | def encodeResource (self, resource, data_s=''): 121 | value, encoder = self.byname.get(resource.querytype, (None, None)) 122 | if value is not None: 123 | encoded = encoder(resource.response, data_s) 124 | else: 125 | encoded = None 126 | 127 | return value, resource.question, encoded, resource.ttl 128 | -------------------------------------------------------------------------------- /lib/exaproxy/dns/factory.py: -------------------------------------------------------------------------------- 1 | from codec import DNSCodec 2 | from codec import DNSRequestType 3 | from codec import DNSResponseType 4 | 5 | import struct 6 | 7 | class DNSPacketFactory: 8 | request_factory = DNSRequestType 9 | response_factory = DNSResponseType 10 | 11 | def __init__(self, definitions): 12 | self.codec = DNSCodec(definitions) 13 | 14 | def serializeRequest(self, request, extended=False): 15 | try: 16 | encoded = self.codec.encodeRequest(request) 17 | except OverflowError: 18 | encoded = None 19 | 20 | if encoded is not None and extended: 21 | encoded = struct.pack('>H', len(encoded)) + encoded 22 | 23 | return encoded 24 | 25 | def normalizeRequest(self, request_s, extended=False): 26 | if extended: 27 | (length,) = struct.unpack('>H', (request_s + '\0\0')[:2]) 28 | request_s = request_s[2:] 29 | 30 | if length != len(request_s): 31 | request_s = '' 32 | 33 | if request_s: 34 | res = True, self.codec.decodeRequest(request_s) 35 | else: 36 | res = False, None 37 | 38 | return res 39 | 40 | def createRequestString(self, identifier, request_type, request_name, extended=False): 41 | request = self.request_factory(identifier) 42 | request.addQuestion(request_type, request_name) 43 | 44 | try: 45 | encoded = self.codec.encodeRequest(request) 46 | except OverflowError: 47 | encoded = None 48 | 49 | if encoded is not None and extended: 50 | encoded = struct.pack('>H', len(encoded)) + encoded 51 | 52 | return encoded 53 | 54 | def serializeResponse(self, response, extended=False): 55 | encoded = self.codec.encodeResponse(response) 56 | if extended: 57 | encoded = struct.pack('>H', len(encoded)) + encoded 58 | 59 | return encoded 60 | 61 | def normalizeResponse(self, response_s, extended=False): 62 | if extended: 63 | (length,) = struct.unpack('>H', (response_s + '\0\0')[:2]) 64 | response_s = response_s[2:] 65 | 66 | if length != len(response_s): 67 | response_s = '' 68 | 69 | if response_s: 70 | res = True, self.codec.decodeResponse(response_s) 71 | else: 72 | res = False, None 73 | 74 | return res 75 | -------------------------------------------------------------------------------- /lib/exaproxy/html/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Exa-Networks/exaproxy/8b7291b79c1cd6542213a5e7d8dda3cf5a676166/lib/exaproxy/html/__init__.py -------------------------------------------------------------------------------- /lib/exaproxy/html/graph.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | graph.py 4 | 5 | Created by Thomas Mangin on 2012-02-25. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | 10 | _chart_header = """\ 11 | 12 | 16 | """ 17 | 18 | _chart = """\ 19 | 41 | """ 42 | 43 | def _nothing (value): return value 44 | 45 | def graph (monitor,title,reload_time,_keys,cumulative=False,split=False,adaptor=_nothing): 46 | page = _chart_header % reload_time + '
' 47 | keys = [[_] for _ in _keys] if split else [_keys] 48 | 49 | for ident, interval,ratio in (('seconds','Seconds',1),('minutes','Minutes',60)): 50 | data = [] 51 | datasource = getattr(monitor,ident) 52 | for k in keys: 53 | columns = "data.addColumn('number', '%s');\n" % interval + '\n'.join(["data.addColumn('number', '%s');" % _ for _ in k]) 54 | nb_records = len(datasource) 55 | last = [0]*len(k) 56 | 57 | chart = [] 58 | index = monitor.nb_recorded - nb_records 59 | for values in datasource: 60 | if cumulative: 61 | new = [values[_] for _ in k] 62 | chart.append("[ %d, %s]" % (index, ','.join([str(max(0,adaptor(n-l)/ratio)).rstrip('L') for (n,l) in zip(new,last)]))) 63 | last = new 64 | else: 65 | chart.append("[ %d, %s]" % (index, ','.join([str(adaptor(values[_])) for _ in k]))) 66 | index += 1 67 | 68 | if cumulative and chart: 69 | chart.pop(0) 70 | 71 | padding = [] 72 | index = 0 73 | top = monitor.nb_recorded - nb_records 74 | while index < top: 75 | padding.append("[ %d, %s ]" % (index, ','.join(['0']*len(k)))) 76 | index += 1 77 | values = ',\n'.join(padding + chart) 78 | 79 | data.append(_chart % { 80 | 'ident' : ident, 81 | 'title' : title + ' (%s)' % interval, 82 | 'columns': columns, 83 | 'values' : values, 84 | }) 85 | page += '\n'.join(data) + '
' % ident 86 | 87 | return page + '
' 88 | -------------------------------------------------------------------------------- /lib/exaproxy/html/humans.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | humans.py 4 | 5 | Created by Thomas Mangin on 2012-02-25. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | from .images import thomas,david 10 | 11 | class humans: 12 | txt = """\ 13 | /* TEAM */ 14 | 15 | Slave Driver / Grand Visionary: Thomas Mangin 16 | Google+: https://plus.google.com/104241996506596749840 17 | Github: https://github.com/thomas-mangin 18 | 19 | Engineer Extraordinaire: David Farrar 20 | Google+: https://plus.google.com/108845019528954357090 21 | Github: https://github.com/david-farrar 22 | 23 | /* Other contributors */ 24 | Marek Obuchowicz (kqueue reactor) 25 | Github: https://github.com/marek-obuchowicz 26 | 27 | """ 28 | 29 | html = """\ 30 |
31 | /* TEAM */
32 |
33 |
34 | 35 |
36 |
37 | Slave Driver / Grand Visionary
38 | Thomas Mangin
39 |
40 | 41 |
42 | 43 |
44 |
45 | Engineer Extraordinaire
46 | David Farrar
47 |
48 |
49 | /* Other contributors */ 50 |
51 | Marek Obuchowicz (kqueue reactor) 52 |
53 |
54 | """ % (thomas,david) 55 | -------------------------------------------------------------------------------- /lib/exaproxy/html/img.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | img.py 4 | 5 | Created by Thomas Mangin on 2012-02-25. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | def png (base64): 10 | return '' % base64 11 | 12 | def jpg (base64): 13 | return '' % base64 14 | -------------------------------------------------------------------------------- /lib/exaproxy/html/index.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | index.py 4 | 5 | Created by Thomas Mangin on 2012-02-25. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | index = """\ 10 | """ 11 | -------------------------------------------------------------------------------- /lib/exaproxy/html/licence.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | licence.py 4 | 5 | Created by Thomas Mangin on 2012-02-25. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | from datetime import datetime 10 | 11 | year = datetime.now().year 12 | 13 | licence = """\ 14 | 
15 | ExaProxy
16 | 
17 | Copyright (c) 2011-%d, Exa Networks Limited
18 | Copyright (c) 2011-%d, Thomas Mangin
19 | Copyright (c) 2011-%d, David Farrar
20 | 
21 | All rights reserved.
22 | 
23 | Redistribution and use in source and binary forms, with or without
24 | modification, are permitted provided that the following conditions are met:
25 | 
26 | 1. Redistributions of source code must retain the above copyright notice, this
27 |    list of conditions and the following disclaimer.
28 | 2. Redistributions in binary form must reproduce the above copyright notice,
29 |    this list of conditions and the following disclaimer in the documentation
30 |    and/or other materials provided with the distribution.
31 | 
32 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
33 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
34 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
35 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
36 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
37 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
38 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
39 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
41 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 |
43 | """ % (year,year,year) 44 | -------------------------------------------------------------------------------- /lib/exaproxy/html/mail.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | email.py 4 | 5 | Created by Thomas Mangin on 2012-02-25. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | import urlparse 10 | import time 11 | from email.mime.text import MIMEText 12 | from email.Utils import formatdate 13 | from smtplib import SMTP,SMTPException 14 | 15 | 16 | class mail (object): 17 | form = """\ 18 | 66 | 67 |
68 |
69 |
70 | 71 |
77 | 78 | 79 |
80 | 81 | 82 |
83 | 84 | 85 |
86 | 87 | 88 |
89 | 90 |
105 |
106 | 107 |
108 |
109 |
110 |
111 | """ 112 | 113 | @staticmethod 114 | def send (args): 115 | answers = urlparse.parse_qs(args) 116 | 117 | _from = answers.get('email',[None,])[0] 118 | _to = 'The ExaProxy Team ' 119 | 120 | if not _from: 121 | return False,'
A email address is required to make sure our mail server let this mail through
(press back on your browser)
' 122 | if '@' not in _from: 123 | return False,'
A valid email address is required to make sure our mail server let this mail through
(press back on your browser)
' 124 | 125 | formated = dict((k,"%s" % ','.join(v)) for (k,v) in answers.items()) 126 | 127 | employer = formated.pop('employer','') 128 | 129 | message = "" 130 | message += "%s %s %s" % (formated.pop('title',''), formated.pop('firstname','').capitalize(), formated.pop('lastname','-').upper()) 131 | message += " from %s" % employer if employer else "" 132 | message += " said\n\n%s\n" % formated.pop('message','') 133 | 134 | msg = MIMEText(message) 135 | msg['Subject'] = 'ExaProxy Message' 136 | msg['From'] = _from 137 | msg['To'] = _to 138 | msg['Message-ID'] = 'DO-NOT-HAVE-ONE-AND-SPAMASSASSIN-COMPLAINS-%s' % time.time() 139 | msg['Date'] = formatdate(localtime=True) 140 | msg.preamble = 'ExaProxy Message' 141 | 142 | try: 143 | s = SMTP('mx.exa-networks.co.uk') 144 | s.sendmail(_from, [_to,], msg.as_string()) 145 | s.quit() 146 | return True,'
Email sent, thank you
' 147 | except (SMTPException,Exception),e: 148 | return False,'
Could not send email

%s' % str(e) 149 | -------------------------------------------------------------------------------- /lib/exaproxy/html/menu.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | menu.py 4 | 5 | Created by Thomas Mangin on 2012-02-25. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | import sys 10 | import time 11 | 12 | from .img import png 13 | from .images import logo 14 | 15 | python = '%s %s' % ('Pypy' if 'PyPy' in sys.version else 'Python', sys.version.split()[0]) 16 | 17 | def html (title,header,color='#FF0000',image=png(logo)): 18 | if header: header += '
' 19 | return """\ 20 | 21 | 22 | %s 23 | 24 | 25 | 26 | 74 | 75 |
76 |
77 | %s 78 | %s 79 |
80 |
81 |
82 | 83 | %s 84 | 85 | 86 | %s 87 | 88 | 89 | *time* 90 | 91 |
92 |
93 | *menu* 94 | *text* 95 |
96 |
97 | 98 | 99 | 100 | 101 | """ % (title,color,header,image,time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()),python) 102 | 103 | _title = 'ExaProxy Monitoring' 104 | _image = '%s' % png(logo) 105 | 106 | 107 | def Menu (options): 108 | menu = '
\n' 109 | menu += '\t

Home

\n' 110 | 111 | for name, url, section in options: 112 | menu += '\t

%s

\n' % name 113 | 114 | if section: 115 | menu += '\t
    \n' 116 | for name, url, new in section: 117 | if new: 118 | menu += '\t\t
  • %s
    • \n' % (url,name,name) 119 | else: 120 | menu += '\t\t
  • %s
    • \n' % (url,name) 121 | menu += '\t
\n' 122 | 123 | menu += '
\n' 124 | 125 | _html = html(_title,'','#9999FF',_image).replace('*text*','%s').replace('*menu*',menu) 126 | 127 | def _lambda (page): 128 | return _html.replace('*time*',time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())) % page 129 | 130 | return _lambda 131 | -------------------------------------------------------------------------------- /lib/exaproxy/http/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | http.py 4 | 5 | Created by Thomas Mangin on 2011-12-02. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | # destination = re.compile("(GET|POST|PUT|HEAD|DELETE|OPTIONS|TRACE|CONNECT)\s+(http://[^/]*|)(/?[^ \r]*)\s+(HTTP/.*\r?\nHost\s*:\s*)([^\r]*)(|\r?\n)", re.IGNORECASE) 10 | # x_forwarded_for = re.compile("(|\n)X-Forwarded-For: ?(((1?\d?\d)|(2([0-4]\d|5[0-5])))\.)(((1?\d?\d)|(2([0-4]\d|5[0-5])))\.)(((1?\d?\d)|(2([0-4]\d|5[0-5])))\.)((2([0-4]\d|5[0-5]))|(1?\d?\d))", re.IGNORECASE) 11 | 12 | #450 Blocked by Windows Parental Controls 13 | #A Microsoft extension. This error is given when Windows Parental Controls are turned on and are blocking access to the given webpage. 14 | #598 Network read timeout error 15 | #This status code is not specified in any RFCs, but is used by some[which?] HTTP proxies to signal a network read timeout behind the proxy to a client in front of the proxy. 16 | #599 Network connect timeout error 17 | #This status code is not specified in any RFCs, but is used by some[which?] HTTP proxies to signal a network connect timeout behind the proxy to a client in front of the proxy. 18 | -------------------------------------------------------------------------------- /lib/exaproxy/http/factory.py: -------------------------------------------------------------------------------- 1 | from .message import HTTP 2 | 3 | class HTTPRequestFactory: 4 | def __init__ (self, configuration): 5 | self.configuration = configuration 6 | 7 | def parseRequest (self, peer, request_string): 8 | request = HTTP(self.configuration, request_string, peer) 9 | request.parse(True) 10 | return request 11 | -------------------------------------------------------------------------------- /lib/exaproxy/http/headers.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | http.py 4 | 5 | Created by Thomas Mangin on 2011-12-02. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | #import re 10 | #reg=re.compile('(\w+)[:=] ?"?(\w+)"?') 11 | #reg=re.compile('(\w+)[:=] ?"?([^" ,]+)"?') 12 | #dict(reg.findall(headers)) 13 | 14 | class ExpectationFailed (Exception): 15 | pass 16 | 17 | class InvalidRequest (Exception): 18 | pass 19 | 20 | class Headers (object): 21 | def __init__ (self,http_version,separator, expect=True): 22 | self._order = [] 23 | self._data = {} 24 | self.http_version = http_version 25 | self.separator = separator 26 | self.expect = expect 27 | 28 | def get (self,key,default): 29 | return self._data.get(key,default) 30 | 31 | def set (self,key,value): 32 | if key not in self._order: 33 | self._order.append(key) 34 | 35 | self._data[key] = [value] 36 | return self 37 | 38 | def replace (self,key,value): 39 | self._data[key] = [value] 40 | 41 | def default (self,key,value): 42 | if key not in self._data: 43 | self._data[key] = [value] 44 | 45 | def extend (self,key,value): 46 | if key in self._order: 47 | self._data[key].append(value) 48 | return self 49 | self._order.append(key) 50 | self._data[key] = [value] 51 | return self 52 | 53 | def pop (self, key, default=None): 54 | if key in self._data: 55 | value = self._data.pop(key) 56 | self._order.remove(key) 57 | return value 58 | else: 59 | return default 60 | 61 | def count_quotes (self, line): 62 | return line.count('"') - line.count('\\"') 63 | 64 | def parse (self, transparent, lines): 65 | # HTTP/1.0 can validly send no headers, and we must not error 66 | if lines.strip() and lines[0].isspace(): 67 | raise InvalidRequest('malformed headers, headers starts with a white space') 68 | 69 | key = '' 70 | quoted = False 71 | 72 | try: 73 | for line in lines.split('\n'): 74 | line = line.strip('\r') 75 | 76 | if quoted: 77 | self.extend(key, line) 78 | if self.count_quotes(line) % 2: 79 | quoted = False 80 | continue 81 | 82 | if not line: break 83 | 84 | if self.count_quotes(line) % 2: 85 | quoted = True 86 | 87 | if line[0].isspace(): 88 | # ValueError if key is not already there 89 | # IndexError the list is empty 90 | self.extend(key,line) 91 | continue 92 | 93 | # KeyError if split does not return two elements 94 | key, value = line.split(':', 1) 95 | key = key.strip().lower() 96 | self.extend(key,line) 97 | 98 | except (KeyError,TypeError,IndexError): 99 | raise InvalidRequest('malformed headers (line : %s) headers %s' % (line,lines.replace('\t','\\t').replace('\r','\\r').replace('\n','\\n'))) 100 | 101 | if quoted: 102 | raise InvalidRequest('end of headers reached while in quoted content') 103 | 104 | if not transparent: 105 | try: 106 | # follow rules about not forwarding connection header as set in section s14.10 107 | if self.http_version == '1.1': 108 | upgrades = self.get('upgrade',[]) 109 | for upgrade in upgrades[:]: 110 | key, value = upgrade.split(':', 1) 111 | value = value.strip().lower() 112 | if not (value == 'websocket' or value.startswith('tls/')): 113 | upgrades.remove(upgrade) 114 | self.pop(value.lower()) 115 | # we modified the list in data directly 116 | if not upgrades: 117 | self.pop('upgrade') 118 | 119 | connections = self.get('connection',[]) 120 | for connection in connections[:]: 121 | key, value = connection.split(':', 1) 122 | value = value.strip().lower() 123 | # we keep connection: close 124 | if value == 'close': 125 | continue 126 | # we have a websocket in upgrade 127 | if value == 'upgrade' and upgrades: 128 | continue 129 | # otherwise we remove the unknown upgrade 130 | connections.remove(connection) 131 | # we modified the list in data directly 132 | if not connections: 133 | self.pop('connection') 134 | 135 | # remove keep-alive header for http/1.0 136 | if self.http_version == '1.0': 137 | self.pop('keep-alive') 138 | 139 | # this proxy can not honour expect, so we are returning a 417 through the use of an exception 140 | expect = self.get('expect',None) 141 | if expect and self.expect: 142 | raise ExpectationFailed() 143 | 144 | except (KeyError,TypeError,IndexError): 145 | raise InvalidRequest('can not remove connection tokens from headers') 146 | 147 | 148 | # we got a line starting with a : 149 | if '' in self._data: 150 | raise InvalidRequest('malformed headers, line starts with colon (:)') 151 | 152 | return self 153 | 154 | def __str__ (self): 155 | return self.separator.join([self.separator.join(self._data[key]) for key in self._order]) 156 | -------------------------------------------------------------------------------- /lib/exaproxy/http/request.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | request.py 4 | 5 | Created by Thomas Mangin on 2012-02-27. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | class Request (object): 10 | def __init__ (self,data): 11 | request, remaining = data.split('\n',1) 12 | parts = request.split() 13 | if len(parts) == 3: 14 | self.raw = request.rstrip('\r') 15 | method, self.uri, version = parts 16 | self.use_raw = False 17 | self.remaining = remaining 18 | elif len(parts) == 2: 19 | http,rest = remaining.split('\n',1) 20 | if http.upper()[:5].startswith('HTTP/'): 21 | version = http.strip().split('/',1)[-1] 22 | self.raw = '%s\n%s' % (request,http) 23 | self.remaining = rest 24 | else: 25 | version = '1.0' 26 | self.raw = request 27 | self.remaining = remaining 28 | method, self.uri = parts 29 | self.use_raw = True 30 | else: 31 | raise ValueError('Malformed request') 32 | 33 | self.method = method.upper() 34 | 35 | version = version.split('/')[-1] 36 | if version not in ('1.1', '1.0') and '.' in version: 37 | major, minor = version.split('.', 1) 38 | if major.isdigit() and minor.isdigit(): 39 | version = str(int(major)) + '.' + str(int(minor)) 40 | 41 | self.version = version 42 | 43 | def parse (self): 44 | # protocol 45 | if '://' in self.uri: 46 | protocol, uri = self.uri.split('://', 1) 47 | # we have :// in the path 48 | if '/' in protocol: 49 | self.protocol = 'http' 50 | uri = self.uri 51 | else: 52 | self.protocol = protocol 53 | else: 54 | self.protocol = 'http' 55 | uri = self.uri 56 | 57 | # ipv6 host 58 | if uri.startswith('[') and ']' in uri: 59 | self.host, remaining = uri[1:].split(']', 1) 60 | if '/' in remaining: 61 | port,path = remaining.split('/') 62 | self.path = '/' + path 63 | if port: 64 | if not port.startswith(':'): 65 | raise ValueError('Malformed headers, ipv6 address was followed by an invalid port') 66 | 67 | self.port = self._checkport(port[1:]) 68 | else: 69 | self.port = 80 70 | 71 | return self 72 | else: 73 | self.port = self._checkport(remaining) 74 | self.path = '' 75 | return self 76 | 77 | # split on path 78 | if '/' in uri: 79 | # we have a path 80 | host, path = uri.split('/', 1) 81 | self.path = '/' + path 82 | if ':' in host: 83 | self.host,port = host.split(':',1) 84 | self.port = self._checkport(port) 85 | else: 86 | self.host = host 87 | self.port = 80 88 | else: 89 | self.path = '/' 90 | if ':' in uri: 91 | self.host,port = uri.split(':',1) 92 | self.port = self._checkport(port) 93 | else: 94 | self.host = uri 95 | self.port = 80 96 | return self 97 | 98 | def _checkport (self,port): 99 | if port and not port.isdigit(): 100 | raise ValueError('Malformed headers') 101 | return int(port) 102 | 103 | def __str__ (self): 104 | if self.use_raw or self.protocol != 'http': 105 | return self.raw 106 | 107 | if self.method == 'CONNECT': 108 | host = self.host + ((':' + str(self.port)) if self.port else '') 109 | return self.method + ' ' + host + ' HTTP/' + self.version 110 | 111 | return self.method + ' ' + self.path + ' HTTP/' + self.version 112 | -------------------------------------------------------------------------------- /lib/exaproxy/http/response.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | http.py 4 | 5 | Created by Thomas Mangin on 2011-12-02. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | import sys 10 | import time 11 | 12 | #from exaproxy.html.images import logo 13 | #from exaproxy.html.img import png 14 | 15 | from exaproxy.configuration import load 16 | 17 | _HTTP_NAMES = { 18 | '100': 'CONTINUE', 19 | '101': 'SWITCHING PROTOCOLS', 20 | '200': 'OK', 21 | '201': 'CREATED', 22 | '202': 'ACCEPTED', 23 | '203': 'NON-AUTHORITATIVE INFORMATION', 24 | '204': 'NO CONTENT', 25 | '205': 'RESET CONTENT', 26 | '206': 'PARTIAL CONTENT', 27 | '226': 'IM USED', 28 | '300': 'MULTIPLE CHOICES', 29 | '301': 'MOVED PERMANENTLY', 30 | '302': 'FOUND', 31 | '303': 'SEE OTHER', 32 | '304': 'NOT MODIFIED', 33 | '305': 'USE PROXY', 34 | '306': 'RESERVED', 35 | '307': 'TEMPORARY REDIRECT', 36 | '400': 'BAD REQUEST', 37 | '401': 'UNAUTHORIZED', 38 | '402': 'PAYMENT REQUIRED', 39 | '403': 'FORBIDDEN', 40 | '404': 'NOT FOUND', 41 | '405': 'METHOD NOT ALLOWED', 42 | '406': 'NOT ACCEPTABLE', 43 | '407': 'PROXY AUTHENTICATION REQUIRED', 44 | '408': 'REQUEST TIMEOUT', 45 | '409': 'CONFLICT', 46 | '410': 'GONE', 47 | '411': 'LENGTH REQUIRED', 48 | '412': 'PRECONDITION FAILED', 49 | '413': 'REQUEST ENTITY TOO LARGE', 50 | '414': 'REQUEST-URI TOO LONG', 51 | '415': 'UNSUPPORTED MEDIA TYPE', 52 | '416': 'REQUESTED RANGE NOT SATISFIABLE', 53 | '417': 'EXPECTATION FAILED', 54 | '500': 'INTERNAL SERVER ERROR', 55 | '501': 'NOT IMPLEMENTED', 56 | '502': 'BAD GATEWAY', 57 | '503': 'SERVICE UNAVAILABLE', 58 | '504': 'GATEWAY TIMEOUT', 59 | '505': 'HTTP VERSION NOT SUPPORTED', 60 | } 61 | 62 | 63 | def file_header (code, size, protocol='1.1'): 64 | date = time.strftime('%c %Z') 65 | version = load().proxy.version 66 | 67 | return '\r\n'.join([ 68 | 'HTTP/%s %s %s' % (protocol, str(code), _HTTP_NAMES.get(code,'-')), 69 | 'Date: %s' % date, 70 | 'Server: exaproxy/%s (%s)' % (str(version), str(sys.platform)), 71 | 'Content-Length: %d' % size, 72 | 'Connection: close', 73 | 'Content-Type: text/html', 74 | 'Cache-Control: no-store', 75 | 'Pragma: no-cache', 76 | '' 77 | ]) 78 | 79 | def http (code,message, protocol='1.1'): 80 | encoding = 'html' if message[:5].lower().startswith(' 0 20 | 21 | @property 22 | def contains_body (self): 23 | return 'req-body' in self.offsets 24 | 25 | 26 | class ICAPRequestFactory: 27 | def __init__ (self, configuration): 28 | self.configuration = configuration 29 | 30 | def create (self, method, url, version, headers, icap_header, http_header, offsets, content_length, complete): 31 | return ICAPRequest(method, url, version, headers, icap_header, http_header, offsets, content_length, complete) 32 | -------------------------------------------------------------------------------- /lib/exaproxy/icap/response.py: -------------------------------------------------------------------------------- 1 | 2 | class ICAPResponse (object): 3 | def __init__ (self, version, code, status, headers, icap_header, http_header, http_body): 4 | self.version = version 5 | self.code = code 6 | self.status = status 7 | self.headers = headers 8 | 9 | icap_len = len(icap_header) 10 | http_len = len(http_header) 11 | 12 | icap_end = icap_len 13 | 14 | if http_header: 15 | http_string = http_header 16 | 17 | if http_body: 18 | http_len_string = '%x\r\n' % len(http_body) 19 | http_string += http_len_string + http_body + '\r\n0\r\n' 20 | 21 | elif http_body is not None: 22 | http_len_string = '0\r\n' 23 | http_string += http_len_string 24 | 25 | else: 26 | http_len_string = '' 27 | http_body = '' 28 | 29 | http_header_offset = icap_end 30 | http_header_end = http_header_offset + len(http_header) 31 | 32 | http_body_offset = http_header_end + len(http_len_string) 33 | http_body_end = http_body_offset + len(http_body) 34 | 35 | else: 36 | http_string = http_header 37 | http_header_offset = icap_end 38 | http_header_end = icap_end 39 | http_body_offset = icap_end 40 | http_body_end = icap_end 41 | 42 | self.response_view = memoryview(icap_header + http_string) 43 | self.icap_view = self.response_view[:icap_end] 44 | self.http_header_view = self.response_view[http_header_offset:http_header_end] 45 | self.http_body_view = self.response_view[http_body_offset:http_body_end] 46 | 47 | @property 48 | def response_string (self): 49 | return self.response_view.tobytes() 50 | 51 | @property 52 | def icap_header (self): 53 | return self.icap_view.tobytes() 54 | 55 | @property 56 | def http_response (self): 57 | return self.http_header_view.tobytes() + self.http_body_view.tobytes() 58 | 59 | @property 60 | def pragma (self): 61 | return self.headers.get('pragma', {}) 62 | 63 | @property 64 | def is_permit (self): 65 | return False 66 | 67 | @property 68 | def is_modify (self): 69 | return False 70 | 71 | @property 72 | def is_content (self): 73 | return False 74 | 75 | @property 76 | def is_intercept (self): 77 | return False 78 | 79 | 80 | class ICAPRequestModification (ICAPResponse): 81 | def __init__ (self, version, code, status, headers, icap_header, http_header, http_body, intercept_header=None): 82 | ICAPResponse.__init__(self, version, code, status, headers, icap_header, http_header, http_body) 83 | self.intercept_header = intercept_header 84 | 85 | @property 86 | def is_permit (self): 87 | return self.code == 204 88 | 89 | @property 90 | def is_modify (self): 91 | return self.code == 200 and self.intercept_header is None 92 | 93 | @property 94 | def is_intercept (self): 95 | return self.code == 200 and self.intercept_header is not None 96 | 97 | 98 | class ICAPResponseModification (ICAPResponse): 99 | @property 100 | def is_content (self): 101 | return self.code == 200 102 | 103 | 104 | class ICAPResponseFactory: 105 | def __init__ (self, configuration): 106 | self.configuration = configuration 107 | 108 | def create (self, version, code, status, headers, icap_header, request_header, response_header, body_string, intercept_header=None): 109 | if response_header: 110 | response = ICAPResponseModification(version, code, status, headers, icap_header, response_header, body_string) 111 | 112 | else: 113 | response = ICAPRequestModification(version, code, status, headers, icap_header, request_header, body_string, intercept_header=intercept_header) 114 | 115 | return response 116 | -------------------------------------------------------------------------------- /lib/exaproxy/leak/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Exa-Networks/exaproxy/8b7291b79c1cd6542213a5e7d8dda3cf5a676166/lib/exaproxy/leak/__init__.py -------------------------------------------------------------------------------- /lib/exaproxy/leak/gcdump.py: -------------------------------------------------------------------------------- 1 | # http://teethgrinder.co.uk/perm.php?a=Python-memory-leak-detector 2 | import gc 3 | import inspect 4 | 5 | def dump(): 6 | # force collection 7 | print "\nCollecting GARBAGE:" 8 | gc.collect() 9 | # prove they have been collected 10 | print "\nCollecting GARBAGE:" 11 | gc.collect() 12 | 13 | print "\nGARBAGE OBJECTS:" 14 | for x in gc.garbage: 15 | s = str(x) 16 | if len(s) > 80: s = "%s..." % s[:80] 17 | 18 | print "::", s 19 | print " type:", type(x) 20 | print " referrers:", len(gc.get_referrers(x)) 21 | try: 22 | print " is class:", inspect.isclass(type(x)) 23 | print " module:", inspect.getmodule(x) 24 | 25 | lines, line_num = inspect.getsourcelines(type(x)) 26 | print " line num:", line_num 27 | for l in lines: 28 | print " line:", l.rstrip("\n") 29 | except: 30 | pass 31 | 32 | print 33 | 34 | class tmp(object): 35 | def __init__(self): 36 | a = 0 37 | 38 | if __name__=="__main__": 39 | import gc 40 | gc.enable() 41 | gc.set_debug(gc.DEBUG_LEAK) 42 | 43 | # make a leak 44 | l = [tmp()] 45 | l.append(l) 46 | del l 47 | 48 | dump_garbage() 49 | 50 | 51 | -------------------------------------------------------------------------------- /lib/exaproxy/monitor.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | monitor.py 4 | 5 | Created by Thomas Mangin on 2012-02-05. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | from collections import deque 10 | 11 | class _Container (object): 12 | def __init__ (self,supervisor): 13 | self.supervisor = supervisor 14 | 15 | class Monitor (object): 16 | nb_recorded = 60 17 | 18 | def __init__(self,supervisor): 19 | self._supervisor = supervisor 20 | self._container = _Container(supervisor) 21 | self.seconds = deque() 22 | self.minutes = deque() 23 | 24 | def zero (self, stats): 25 | if stats: 26 | self.seconds.append(stats) 27 | self.minutes.append(stats) 28 | 29 | return bool(stats) 30 | 31 | def introspection (self,objects): 32 | obj = self._container 33 | ks = [_ for _ in dir(obj) if not _.startswith('__') and not _.endswith('__')] 34 | 35 | for key in objects: 36 | if not key in ks: 37 | raise StopIteration() 38 | obj = getattr(obj,key) 39 | ks = [_ for _ in dir(obj) if not _.startswith('__') and not _.endswith('__')] 40 | 41 | for k in ks: 42 | value = str(getattr(obj,k)) 43 | if value.startswith(' self.nb_recorded: 157 | self.seconds.popleft() 158 | 159 | return True 160 | 161 | def minute (self, stats): 162 | self.minutes.append(stats) 163 | 164 | if len(self.minutes) > self.nb_recorded: 165 | self.minutes.popleft() 166 | 167 | return True 168 | -------------------------------------------------------------------------------- /lib/exaproxy/network/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Exa-Networks/exaproxy/8b7291b79c1cd6542213a5e7d8dda3cf5a676166/lib/exaproxy/network/__init__.py -------------------------------------------------------------------------------- /lib/exaproxy/network/async/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | async/__init__.py 4 | 5 | Created by David Farrar on 2012-01-31. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | import sys 10 | import select 11 | 12 | from exaproxy.util.log.logger import Logger 13 | from exaproxy.configuration import load 14 | 15 | configuration = load() 16 | log = Logger('supervisor', configuration.log.supervisor) 17 | 18 | def Poller (configuration, speed=None): 19 | reactor = configuration.reactor 20 | 21 | if reactor == 'best': 22 | if sys.platform.startswith('linux'): 23 | configuration.reactor = 'epoll' 24 | elif sys.platform.startswith('freebsd'): 25 | configuration.reactor = 'kqueue' 26 | elif sys.platform.startswith('darwin'): 27 | configuration.reactor = 'kqueue' 28 | else: 29 | log.error('we could not autodetect an high performance reactor for your OS') 30 | log.error('as the "select" reactor is not suitable for production,') 31 | log.error('please consider changing reactor by hand') 32 | configuration.reactor = 'select' 33 | 34 | reactor = configuration.reactor 35 | log.info('the chosen polling reactor was %s' % reactor) 36 | 37 | if reactor not in ('epoll','kqueue','select'): 38 | log.error('invalid reactor name: "%s"' % reactor) 39 | sys.exit(1) 40 | 41 | timeout = speed if speed is not None else configuration.speed 42 | 43 | if reactor == 'epoll' and hasattr(select, 'epoll'): 44 | from epoll import EPoller as Poller 45 | return Poller(timeout) 46 | 47 | if reactor == 'kqueue' and hasattr(select, 'kqueue'): 48 | from kqueue import KQueuePoller as Poller 49 | return Poller(timeout) 50 | 51 | if hasattr(select, 'select'): 52 | from selectpoll import SelectPoller as Poller 53 | return Poller(timeout) 54 | 55 | log.error('this version of python does not have the requested reactor %s or select' % reactor) 56 | sys.exit(1) 57 | -------------------------------------------------------------------------------- /lib/exaproxy/network/async/interface.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | poller.py 4 | 5 | Created by David Farrar on 2012-01-31. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | class IPoller: 10 | """Interface for pollers""" 11 | 12 | def addReadSocket(self, name, socket): 13 | """Start watching for data in the socket's recvbuf""" 14 | raise NotImplementedError 15 | 16 | def removeReadSocket(self, name, socket): 17 | """Stop watching the socket for incoming data""" 18 | raise NotImplementedError 19 | 20 | def corkReadSocket(self, name, socket): 21 | """Stop watching the socket for incoming data. The socket is 22 | still in use at this point and we expect this operation to 23 | be reversed so don't clean up data we will need again""" 24 | raise NotImplementedError 25 | 26 | def uncorkReadSocket(self, name, socket): 27 | """Start watching the socket for incoming data again""" 28 | raise NotImplementedError 29 | 30 | def setupRead(self, name): 31 | """Define a new event that sockets can subscribe to""" 32 | raise NotImplementedError 33 | 34 | def clearRead(self, name): 35 | """Flush all sockets currently watched for the event""" 36 | raise NotImplementedError 37 | 38 | def addWriteSocket(self, name, socket): 39 | """Start watching for space in the socket's sendbuf""" 40 | raise NotImplementedError 41 | 42 | def removeWriteSocket(self, name, socket): 43 | """Stop watching for space in the socket's sendbuf""" 44 | raise NotImplementedError 45 | 46 | def corkWriteSocket(self, name, socket): 47 | """Stop waiting for space in the socket's sendbuf. The socket is 48 | still in use at this point and we expect this operation to 49 | be reversed so don't clean up data we will need again""" 50 | raise NotImplementedError 51 | 52 | def uncorkWriteSocket(self, name, socket): 53 | """Start watching for space in the socket's sendbuf again""" 54 | raise NotImplementedError 55 | 56 | def setupWrite(self, name): 57 | """Define a new event that sockets can subscribe to""" 58 | raise NotImplementedError 59 | 60 | def clearWrite(self, name): 61 | """Flush all sockets currently watched for the event""" 62 | raise NotImplementedError 63 | 64 | def poll(self): 65 | """Wait for events we're watching for to occur and 66 | return a list of each eventful socket per event""" 67 | raise NotImplementedError 68 | -------------------------------------------------------------------------------- /lib/exaproxy/network/async/selectpoll.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | poller.py 4 | 5 | Created by Thomas Mangin on 2011-11-30. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | # http://code.google.com/speed/articles/web-metrics.html 10 | 11 | import select 12 | import socket 13 | import errno 14 | 15 | from exaproxy.network.errno_list import errno_block, errno_fatal 16 | from interface import IPoller 17 | 18 | from exaproxy.util.log.logger import Logger 19 | from exaproxy.configuration import load 20 | 21 | configuration = load() 22 | log = Logger('select', configuration.log.server) 23 | 24 | def poll_select(read, write, timeout=None): 25 | try: 26 | r, w, x = select.select(read, write, read + write, timeout) 27 | except socket.error, e: 28 | if e.args[0] in errno_block: 29 | log.error('select not ready, errno %d: %s' % (e.args[0], errno.errorcode.get(e.args[0], ''))) 30 | return [], [], [] 31 | 32 | if e.args[0] in errno_fatal: 33 | log.error('select problem, errno %d: %s' % (e.args[0], errno.errorcode.get(e.args[0], ''))) 34 | log.error('poller read : %s' % str(read)) 35 | log.error('poller write : %s' % str(write)) 36 | log.error('read : %s' % str(read)) 37 | else: 38 | log.critical('select problem, debug it. errno %d: %s' % (e.args[0], errno.errorcode.get(e.args[0], ''))) 39 | 40 | raise e 41 | except (ValueError, AttributeError, TypeError), e: 42 | log.error("fatal error encountered during select - %s %s" % (type(e),str(e))) 43 | raise e 44 | except select.error, e: 45 | if e.args[0] in errno_block: 46 | return [], [], [] 47 | log.error("fatal error encountered during select - %s %s" % (type(e),str(e))) 48 | raise e 49 | except KeyboardInterrupt,e: 50 | raise e 51 | except Exception, e: 52 | log.critical("fatal error encountered during select - %s %s" % (type(e),str(e))) 53 | raise e 54 | 55 | return r, w, x 56 | 57 | 58 | class SelectPoller (IPoller): 59 | poller = staticmethod(poll_select) 60 | 61 | def __init__(self, speed): 62 | self.speed = speed 63 | 64 | self.read_sockets = {} 65 | self.write_sockets = {} 66 | 67 | self.read_modified = {} 68 | self.write_modified = {} 69 | 70 | self.read_all = [] 71 | self.write_all = [] 72 | 73 | def addReadSocket(self, name, socket): 74 | sockets = self.read_sockets[name] 75 | if socket not in sockets: 76 | sockets.append(socket) 77 | self.read_modified[name] = True 78 | 79 | def removeReadSocket(self, name, socket): 80 | sockets = self.read_sockets[name] 81 | if socket in sockets: 82 | sockets.remove(socket) 83 | self.read_modified[name] = True 84 | 85 | def setupRead(self, name): 86 | if name not in self.read_sockets: 87 | self.read_sockets[name] = [] 88 | 89 | def clearRead(self, name): 90 | had_sockets = bool(self.read_sockets[name]) 91 | self.read_sockets[name] = {} 92 | 93 | if had_sockets: 94 | self.read_modified[name] = True 95 | 96 | def addWriteSocket(self, name, socket): 97 | sockets = self.write_sockets[name] 98 | if socket not in sockets: 99 | sockets.append(socket) 100 | self.write_modified[name] = True 101 | 102 | def removeWriteSocket(self, name, socket): 103 | sockets = self.write_sockets[name] 104 | if socket in sockets: 105 | sockets.remove(socket) 106 | self.write_modified[name] = True 107 | 108 | def setupWrite(self, name): 109 | if name not in self.write_sockets: 110 | self.write_sockets[name] = [] 111 | 112 | def clearWrite(self, name): 113 | had_sockets = bool(self.write_sockets[name]) 114 | self.write_sockets[name] = {} 115 | 116 | if had_sockets: 117 | self.write_modified[name] = True 118 | 119 | corkReadSocket = removeReadSocket 120 | uncorkReadSocket = addReadSocket 121 | 122 | corkWriteSocket = removeWriteSocket 123 | uncorkWriteSocket = addWriteSocket 124 | 125 | def poll(self): 126 | all_socks = {} 127 | 128 | for name, socks in self.read_sockets.items(): 129 | socks, _, __ = self.poller(socks, [], 0) 130 | if socks: 131 | all_socks[name] = socks 132 | 133 | for name, socks in self.write_sockets.items(): 134 | _, socks, __ = self.poller([], socks, 0) 135 | if socks: 136 | all_socks[name] = socks 137 | 138 | if all_socks: 139 | return all_socks 140 | 141 | if self.read_modified: 142 | self.read_all = sum(self.read_sockets.values(), []) 143 | self.read_modified = {} 144 | 145 | if self.write_modified: 146 | self.write_all = sum(self.write_sockets.values(), []) 147 | self.write_modified = {} 148 | 149 | r, w, x = self.poller(self.read_all, self.write_all, self.speed) 150 | 151 | for name, socks in self.read_sockets.items(): 152 | polled, _, __ = self.poller(socks, [], 0) 153 | if polled: all_socks[name] = polled 154 | 155 | for name, socks in self.write_sockets.items(): 156 | polled, all_socks[name], __ = self.poller([], socks, 0) 157 | if polled: all_socks[name] = polled 158 | 159 | return all_socks 160 | -------------------------------------------------------------------------------- /lib/exaproxy/network/errno_list.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | server.py 4 | 5 | Created by Thomas Mangin on 2011-11-30. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | # http://code.google.com/speed/articles/web-metrics.html 10 | 11 | import errno 12 | 13 | errno_block = { 14 | errno.EINPROGRESS, errno.EALREADY, 15 | errno.EAGAIN, errno.EWOULDBLOCK, 16 | errno.EINTR, errno.EDEADLK, 17 | errno.EBUSY, errno.ENOBUFS, 18 | errno.ENOMEM, 19 | } 20 | 21 | errno_fatal = { 22 | errno.ECONNABORTED, errno.EPIPE, 23 | errno.ECONNREFUSED, errno.EBADF, 24 | errno.ESHUTDOWN, errno.ENOTCONN, 25 | errno.ECONNRESET, errno.ETIMEDOUT, 26 | errno.EINVAL, 27 | } 28 | 29 | errno_unavailable = { 30 | errno.ECONNREFUSED, errno.EHOSTUNREACH, 31 | } 32 | -------------------------------------------------------------------------------- /lib/exaproxy/network/functions.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | nettools.py 4 | 5 | Created by Thomas Mangin on 2011-11-30. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | import socket 10 | import errno 11 | 12 | from exaproxy.util.log.logger import Logger 13 | from exaproxy.network.errno_list import errno_block 14 | from exaproxy.configuration import load 15 | 16 | IP_TRANSPARENT = 19 17 | 18 | configuration = load() 19 | log = Logger('server', configuration.log.server) 20 | 21 | def isipv4(address): 22 | try: 23 | socket.inet_pton(socket.AF_INET, address) 24 | return True 25 | except socket.error: 26 | return False 27 | 28 | def isipv6(address): 29 | try: 30 | socket.inet_pton(socket.AF_INET6, address) 31 | return True 32 | except socket.error: 33 | return False 34 | 35 | def isip(address): 36 | return isipv4(address) or isipv6(address) 37 | 38 | def listen (ip,port,timeout=None,backlog=0): 39 | try: 40 | if isipv6(ip): 41 | s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM, socket.IPPROTO_TCP) 42 | try: 43 | s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 44 | s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) 45 | except (AttributeError, socket.error): 46 | pass 47 | s.bind((ip,port,0,0)) 48 | 49 | elif isipv4(ip): 50 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) 51 | try: 52 | s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 53 | s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) 54 | except (AttributeError, socket.error): 55 | pass 56 | s.bind((ip,port)) 57 | 58 | else: 59 | return None 60 | 61 | if timeout: 62 | s.settimeout(timeout) 63 | 64 | ##s.setblocking(0) 65 | s.listen(backlog) 66 | return s 67 | except socket.error, e: 68 | if e.args[0] == errno.EADDRINUSE: 69 | log.debug('could not listen, port already in use %s:%d' % (ip,port)) 70 | elif e.args[0] == errno.EADDRNOTAVAIL: 71 | log.debug('could not listen, invalid address %s:%d' % (ip,port)) 72 | else: 73 | log.debug('could not listen on %s:%d - %s' % (ip,port,str(e))) 74 | return None 75 | 76 | def listen_intercept (ip,port,timeout=None,backlog=0): 77 | try: 78 | if isipv6(ip): 79 | s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM, socket.IPPROTO_TCP) 80 | try: 81 | s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 82 | s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) 83 | s.setsockopt(socket.SOL_IP, IP_TRANSPARENT, 1) 84 | except (AttributeError, socket.error): 85 | pass 86 | 87 | s.bind((ip,port,0,0)) 88 | 89 | elif isipv4(ip): 90 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) 91 | try: 92 | s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 93 | s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) 94 | s.setsockopt(socket.SOL_IP, IP_TRANSPARENT, 1) 95 | except (AttributeError, socket.error): 96 | pass 97 | 98 | s.bind((ip,port)) 99 | 100 | else: 101 | return None 102 | 103 | if timeout: 104 | s.settimeout(timeout) 105 | 106 | ##s.setblocking(0) 107 | s.listen(backlog) 108 | return s 109 | except socket.error, e: 110 | if e.args[0] == errno.EADDRINUSE: 111 | log.debug('could not listen, port already in use %s:%d' % (ip,port)) 112 | elif e.args[0] == errno.EADDRNOTAVAIL: 113 | log.debug('could not listen, invalid address %s:%d' % (ip,port)) 114 | else: 115 | log.debug('could not listen on %s:%d - %s' % (ip,port,str(e))) 116 | return None 117 | 118 | 119 | def connect (ip,port,bind,immediate=True): 120 | try: 121 | if isipv6(ip): 122 | s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM, socket.IPPROTO_TCP) 123 | elif isipv4(ip): 124 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) 125 | else: 126 | return None 127 | except socket.error: 128 | return None 129 | 130 | # try: 131 | # s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 64*1024) 132 | # except socket.error, e: 133 | # print "CANNOT SET RCVBUF" 134 | # log.debug('server','could not set sock rcvbuf size') 135 | # except Exception,e: 136 | # print "*"*10 137 | # print type(e),str(e) 138 | # raise 139 | 140 | if immediate: 141 | try: 142 | # diable Nagle's algorithm (no grouping of packets) 143 | s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) 144 | except AttributeError: 145 | pass 146 | 147 | if bind not in ('0.0.0.0','::'): 148 | try: 149 | s.bind((bind,0)) 150 | except socket.error: 151 | log.critical('could not bind to the requested ip "%s" - using OS default' % bind) 152 | 153 | try: 154 | s.setblocking(0) 155 | s.connect((ip, port)) 156 | except socket.error,e: 157 | if e.args[0] == errno.EINPROGRESS: 158 | pass 159 | 160 | elif e.args[0] in errno_block: 161 | pass 162 | 163 | else: 164 | s = None 165 | 166 | return s 167 | -------------------------------------------------------------------------------- /lib/exaproxy/network/server.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | server.py 4 | 5 | Created by Thomas Mangin on 2011-11-30. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | # http://code.google.com/speed/articles/web-metrics.html 10 | # http://itamarst.org/writings/pycon05/fast.html 11 | 12 | from .functions import listen 13 | from .functions import listen_intercept 14 | import socket 15 | 16 | from exaproxy.util.log.logger import Logger 17 | from exaproxy.configuration import load 18 | 19 | configuration = load() 20 | 21 | class Server(object): 22 | _listen = staticmethod(listen) 23 | 24 | def __init__(self, name, poller, read_name, config): 25 | self.socks = {} 26 | self.name = name 27 | self.poller = poller 28 | self.read_name = read_name 29 | self.max_clients = config.connections 30 | self.client_count = 0 31 | self.saturated = False # we are receiving more connections than we can handle 32 | self.binding = set() 33 | self.log = Logger('server', configuration.log.server) 34 | self.serving = config.enable # We are currenrly listening 35 | if self.serving: 36 | self.log.info('server [%s] accepting up to %d clients' % (name, self.max_clients)) 37 | 38 | def accepting (self): 39 | if self.serving: 40 | return True 41 | 42 | for ip, port, timeout, backlog in self.binding: 43 | try: 44 | self.log.critical('re-listening on %s:%d' % (ip,port)) 45 | self.listen(ip,port,timeout,backlog) 46 | except socket.error,e: 47 | self.log.critical('could not re-listen on %s:%d : %s' % (ip,port,str(e))) 48 | return False 49 | self.serving = True 50 | return True 51 | 52 | def rejecting (self): 53 | if self.serving: 54 | for sock,(ip,port) in self.socks.items(): 55 | self.log.critical('stop listening on %s:%d' % (ip,port)) 56 | self.poller.removeReadSocket(self.read_name,sock) 57 | sock.close() 58 | self.socks = {} 59 | self.serving = False 60 | 61 | def saturation (self): 62 | if not self.saturated: 63 | return 64 | self.saturated = False 65 | self.log.error('we received more %s connections that we could handle' % self.name) 66 | self.log.error('we current have %s client(s) out of a maximum of %s' % (self.client_count, self.max_clients)) 67 | 68 | def listen(self, ip, port, timeout, backlog): 69 | s = self._listen(ip, port,timeout,backlog) 70 | if s: 71 | self.binding.add((ip,port,timeout,backlog)) 72 | self.socks[s] = (ip,port) 73 | 74 | # register the socket with the poller 75 | if self.client_count < self.max_clients: 76 | self.poller.addReadSocket(self.read_name, s) 77 | 78 | return s 79 | 80 | def accept(self, sock): 81 | try: 82 | # should we check to make sure it's a socket we provided 83 | s, a = sock.accept() 84 | s.setblocking(0) 85 | # NOTE: we really should try to handle the entire queue at once 86 | yield s, a[0] 87 | except socket.error, e: 88 | # It doesn't really matter if accept fails temporarily. We will 89 | # try again next loop 90 | self.log.debug('%s could not accept a new connection %s' % (self.name,str(e))) 91 | else: 92 | self.client_count += 1 93 | finally: 94 | if self.client_count >= self.max_clients: 95 | self.saturated = True 96 | 97 | for listening_sock in self.socks: 98 | self.poller.removeReadSocket(self.read_name, listening_sock) 99 | 100 | def notifyClose (self, client, count=1): 101 | paused = self.client_count >= self.max_clients 102 | self.client_count -= count 103 | 104 | if paused and self.client_count < self.max_clients: 105 | for listening_sock in self.socks: 106 | self.poller.addReadSocket(self.read_name, listening_sock) 107 | 108 | def stop(self): 109 | for sock in self.socks: 110 | try: 111 | sock.close() 112 | except socket.error: 113 | pass 114 | 115 | self.socks = {} 116 | self.poller.clearRead(self.read_name) 117 | 118 | 119 | class InterceptServer (Server): 120 | _listen = staticmethod(listen_intercept) 121 | -------------------------------------------------------------------------------- /lib/exaproxy/network/splice.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | # from https://gist.github.com/NicolasT/4519146 4 | 5 | """ 6 | Demonstration of using splice from Python 7 | 8 | This code starts a TCP/IP server, waits for a connection, and once a connection 9 | has been made, launches a subprocess ('cat' of this file). Then, it transfers 10 | everything this subprocess outputs on stdout to the socket client. When no more 11 | data is available, everything is shut down. 12 | 13 | The server is fully blocking etc. etc. etc. even though splice(2) supports 14 | non-blocking execution. You should set any pipes to non-blocking mode (using 15 | fcntl or whatever) and call splice with the `SPLICE_F_NONBLOCK` flag set, then 16 | integrate FD read/write'ability with your mainloop and select/poll/epoll/... 17 | calls. This is very application/framework/library-specific, so I don't bother 18 | with it in this code. Notice you might need to wrap calls to splice in an 19 | exception handler to catch EWOULDBLOCK, EAGAIN,... The lot. 20 | 21 | Bindings to splice(2) are made using ctypes. 22 | 23 | This code is public domain as fully as possible in any applicable law, etc. etc. 24 | etc. 25 | 26 | It comes without warranty blah blah blah do whatever you want with it but don't 27 | blame me if anything breaks. 28 | 29 | If you find any errors, please let me know! 30 | """ 31 | 32 | import os 33 | import os.path 34 | import errno 35 | import socket 36 | import subprocess 37 | 38 | import ctypes 39 | import ctypes.util 40 | 41 | def make_splice(): 42 | """Set up a splice(2) wrapper""" 43 | 44 | # Load libc 45 | libc_name = ctypes.util.find_library('c') 46 | libc = ctypes.CDLL(libc_name, use_errno=True) 47 | 48 | # Get a handle to the 'splice' call 49 | c_splice = libc.splice 50 | 51 | # These should match for x86_64, might need some tweaking for other 52 | # platforms... 53 | c_loff_t = ctypes.c_uint64 54 | c_loff_t_p = ctypes.POINTER(c_loff_t) 55 | 56 | # ssize_t splice(int fd_in, loff_t *off_in, int fd_out, 57 | # loff_t *off_out, size_t len, unsigned int flags) 58 | c_splice.argtypes = [ 59 | ctypes.c_int, c_loff_t_p, 60 | ctypes.c_int, c_loff_t_p, 61 | ctypes.c_size_t, 62 | ctypes.c_uint 63 | ] 64 | c_splice.restype = ctypes.c_ssize_t 65 | 66 | # Clean-up closure names. Yup, useless nit-picking. 67 | del libc 68 | del libc_name 69 | del c_loff_t_p 70 | 71 | # pylint: disable-msg=W0621,R0913 72 | def splice(fd_in, off_in, fd_out, off_out, len_, flags): 73 | '''Wrapper for splice(2) 74 | 75 | See the syscall documentation ('man 2 splice') for more information 76 | about the arguments and return value. 77 | 78 | `off_in` and `off_out` can be `None`, which is equivalent to `NULL`. 79 | 80 | If the call to `splice` fails (i.e. returns -1), an `OSError` is raised 81 | with the appropriate `errno`, unless the error is `EINTR`, which results 82 | in the call to be retried. 83 | ''' 84 | 85 | c_off_in = \ 86 | ctypes.byref(c_loff_t(off_in)) if off_in is not None else None 87 | c_off_out = \ 88 | ctypes.byref(c_loff_t(off_out)) if off_out is not None else None 89 | 90 | # For handling EINTR... 91 | while True: 92 | res = c_splice(fd_in, c_off_in, fd_out, c_off_out, len_, flags) 93 | 94 | if res == -1: 95 | errno_ = ctypes.get_errno() 96 | 97 | # Try again on EINTR 98 | if errno_ == errno.EINTR: 99 | continue 100 | 101 | raise IOError(errno_, os.strerror(errno_)) 102 | 103 | return res 104 | 105 | return splice 106 | 107 | # Build and export wrapper 108 | splice = make_splice() # pylint: disable-msg=C0103 109 | del make_splice 110 | 111 | 112 | # From bits/fcntl.h 113 | # Values for 'flags', can be OR'ed together 114 | SPLICE_F_MOVE = 1 115 | SPLICE_F_NONBLOCK = 2 116 | SPLICE_F_MORE = 4 117 | SPLICE_F_GIFT = 8 118 | 119 | 120 | def main(host, port, path): 121 | """Server implementation""" 122 | 123 | # Set up a simple server socket 124 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 125 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 126 | sock.bind((host, port)) 127 | sock.listen(1) 128 | 129 | # Single accept, we'll clean up once this one connection has been handled. 130 | # Yes, this is a very stupid server indeed. 131 | conn, addr = sock.accept() 132 | print 'Connection from:', addr 133 | 134 | # Set up some subprocess which produces some output which should be 135 | # transferred to the client. 136 | # In this case, we just 'cat' this source file. 137 | argv = ['cat', path] 138 | # Might want to do something about stdin and stdout as well in a serious 139 | # application 140 | proc = subprocess.Popen(argv, close_fds=True, stdout=subprocess.PIPE) 141 | 142 | # We need the integer FDs for splice to work 143 | pipe_fd = proc.stdout.fileno() 144 | conn_fd = conn.fileno() # pylint: disable-msg=E1101 145 | print 'Will splice data from FD', pipe_fd, 'to', conn_fd 146 | 147 | transferred = 0 148 | 149 | # 32MB chunks 150 | chunksize = 32 * 1024 * 1024 151 | 152 | # If you know the number of bytes to be transferred upfront, you could 153 | # change this into a 'while todo > 0', pass 'todo' to splice instead of the 154 | # arbitrary 'chunksize', and error out if splice returns 0 before all bytes 155 | # are transferred. 156 | # In this example, we just transfer as much as possible until the write-end 157 | # closes the pipe. 158 | while True: 159 | done = splice(pipe_fd, None, conn_fd, None, chunksize, 160 | SPLICE_F_MOVE | SPLICE_F_MORE) 161 | 162 | if done == 0: 163 | # Write-end of the source pipe has gone, no more data will be 164 | # available 165 | break 166 | 167 | transferred += done 168 | 169 | print 'Bytes transferred:', transferred 170 | 171 | # Close client and server socket 172 | conn.close() 173 | sock.close() 174 | 175 | # Wait for subprocess to finish (it should be finished by now anyway...) 176 | proc.wait() 177 | 178 | 179 | if __name__ == '__main__': 180 | main('', 9009, os.path.abspath(__file__)) 181 | -------------------------------------------------------------------------------- /lib/exaproxy/reactor/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | __init__.py 4 | 5 | Created by Thomas Mangin on 2012-02-24. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | from .reactor import * 10 | -------------------------------------------------------------------------------- /lib/exaproxy/reactor/client/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Exa-Networks/exaproxy/8b7291b79c1cd6542213a5e7d8dda3cf5a676166/lib/exaproxy/reactor/client/__init__.py -------------------------------------------------------------------------------- /lib/exaproxy/reactor/content/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Exa-Networks/exaproxy/8b7291b79c1cd6542213a5e7d8dda3cf5a676166/lib/exaproxy/reactor/content/__init__.py -------------------------------------------------------------------------------- /lib/exaproxy/reactor/content/worker.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | worker.py 4 | 5 | Created by Thomas Mangin on 2011-12-01. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | from exaproxy.network.functions import connect,isipv4 10 | from exaproxy.network.errno_list import errno_block 11 | from exaproxy.network.errno_list import errno_unavailable 12 | 13 | import socket 14 | import errno 15 | 16 | # http://tools.ietf.org/html/rfc2616#section-8.2.3 17 | # Says we SHOULD keep track of the server version and deal with 100-continue 18 | # I say I am too lazy - and if you want the feature use this software as as rev-proxy :D 19 | 20 | DEFAULT_READ_BUFFER_SIZE = 64*1024 21 | 22 | 23 | class Content (object): 24 | _connect = staticmethod(connect) 25 | 26 | __slots__ = ['client', 'sock', 'host', 'port', 'method', 'w_buffer', 'log', 'ipv4'] 27 | 28 | def __init__(self, client, host, port, bind, method, request, logger): 29 | self.client = client 30 | self.sock = self._connect(host, port, bind) 31 | self.host = host 32 | self.port = port 33 | self.method = method 34 | self.w_buffer = request 35 | self.log = logger 36 | self.ipv4 = isipv4(host) 37 | 38 | def startConversation(self): 39 | """Send our buffered request to get the conversation flowing 40 | Don't send anything yet if the client sent a CONNECT - instead, 41 | we respond with our own HTTP header indicating that we connected""" 42 | 43 | #self.log.info('download socket is now open for client %s %s' % (self.client, self.sock)) 44 | 45 | res,sent4,sent6 = self.writeData('') 46 | # XXX: Are we not accounting this data transfer ! ? 47 | response='HTTP/1.1 200 Connection Established\r\n\r\n' if self.method == 'connect' else '' 48 | return self.client, res is not None, response 49 | 50 | def readData(self, buflen=DEFAULT_READ_BUFFER_SIZE): 51 | """Read data that we have already received from the remote server""" 52 | 53 | try: 54 | # without this the exception can barf with data not defined on error 55 | data = '' 56 | data = self.sock.recv(buflen) or None 57 | #if data: 58 | # self.log.debug("<< [%s]" % data.replace('\t','\\t').replace('\r','\\r').replace('\n','\\n')) 59 | except socket.error, e: 60 | if e.args[0] in errno_block: 61 | self.log.info('interrupted when trying to read, will retry: got %s bytes' % len(data)) 62 | self.log.info('reason, errno %d: %s' % (e.args[0], errno.errorcode.get(e.args[0], ''))) 63 | data = '' 64 | else: 65 | #self.log.info('unexpected error reading on socket') 66 | #self.log.info('reason, errno %d: %s' % (e.args[0], errno.errorcode.get(e.args[0], ''))) 67 | data = None 68 | 69 | return data 70 | 71 | def writeData(self, data): 72 | """Write data to the remote server""" 73 | 74 | w_buffer = self.w_buffer + data 75 | 76 | try: 77 | sent = self.sock.send(w_buffer) 78 | #if sent: 79 | # self.log.debug(">> [%s]" % w_buffer[:sent].replace('\t','\\t').replace('\r','\\r').replace('\n','\\n')) 80 | #self.log.info('sent %s of %s bytes of data. %s bytes were unbuffered : %s' % (sent, len(w_buffer), len(data), self.sock)) 81 | self.w_buffer = w_buffer[sent:] 82 | res = bool(self.w_buffer) 83 | 84 | except socket.error, e: 85 | sent = 0 86 | self.w_buffer = w_buffer 87 | 88 | if e.args[0] in errno_block: 89 | #self.log.error('Write failed as it would have blocked. Why were we woken up? Error %d: %s' % (e.args[0], errno.errorcode.get(e.args[0], ''))) 90 | res = bool(self.w_buffer) 91 | elif e.args[0] in errno_unavailable: 92 | res = None 93 | else: 94 | res = None 95 | 96 | return res,sent if self.ipv4 else 0, 0 if self.ipv4 else sent 97 | 98 | def bufferData(self, data): 99 | """Buffer data to be sent later""" 100 | self.w_buffer += data 101 | return bool(self.w_buffer) 102 | 103 | def shutdown(self): 104 | try: 105 | self.sock.shutdown(socket.SHUT_RDWR) 106 | except socket.error: 107 | pass 108 | finally: 109 | self.sock.close() 110 | -------------------------------------------------------------------------------- /lib/exaproxy/reactor/redirector/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from exaproxy.util.control import SlaveBox 5 | 6 | from .supervisor import RedirectorSupervisor 7 | from .messagebox import ProxyToRedirectorMessageBox, RedirectorToProxyMessageBox 8 | from .dispatch import RedirectorDispatcher 9 | 10 | def redirector_message_thread (message_box): 11 | dispatcher = RedirectorDispatcher(message_box) 12 | dispatcher.start() 13 | 14 | return dispatcher 15 | 16 | 17 | def fork_redirector (poller, configuration): 18 | r1, w1 = os.pipe() 19 | r2, w2 = os.pipe() 20 | 21 | cr1, cw1 = os.pipe() 22 | cr2, cw2 = os.pipe() 23 | 24 | pid = os.fork() 25 | 26 | if pid == 0: # the child process 27 | os.close(r1) 28 | os.close(w2) 29 | os.close(cr1) 30 | os.close(cw2) 31 | 32 | messagebox = RedirectorToProxyMessageBox(r2, w1) 33 | controlbox = SlaveBox(cr2, cw1) 34 | supervisor = RedirectorSupervisor(configuration, messagebox, controlbox) 35 | 36 | # run forever 37 | try: 38 | supervisor.run() 39 | 40 | except KeyboardInterrupt: 41 | pass 42 | 43 | except IOError: 44 | pass 45 | 46 | # unless we don't 47 | messagebox.close() 48 | sys.exit() 49 | 50 | else: 51 | os.close(w1) 52 | os.close(r2) 53 | os.close(cw1) 54 | os.close(cr2) 55 | 56 | redirector = ProxyToRedirectorMessageBox(pid, r1, w2, cr1, cw2) 57 | poller.addReadSocket('read_redirector', redirector.box.pipe_in) 58 | poller.addReadSocket('read_control', redirector.control.box.pipe_in) 59 | 60 | return redirector 61 | -------------------------------------------------------------------------------- /lib/exaproxy/reactor/redirector/child.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | child.py 4 | 5 | Created by David Farrar on 2014-04-21. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | import subprocess 10 | import fcntl 11 | import os 12 | import errno 13 | 14 | from exaproxy.util.log.logger import Logger 15 | 16 | class ChildFactory: 17 | def preExec (self): 18 | os.setpgrp() 19 | 20 | def __init__ (self, configuration, name): 21 | self.log = Logger('worker ' + str(name), configuration.log.worker) 22 | 23 | def createProcess (self, program, universal=False): 24 | try: 25 | process = subprocess.Popen(program.split(' '), 26 | stdin=subprocess.PIPE, 27 | stdout=subprocess.PIPE, 28 | stderr=subprocess.PIPE, 29 | universal_newlines=universal, 30 | preexec_fn=self.preExec, 31 | ) 32 | 33 | self.log.debug('spawn process %s' % program) 34 | 35 | except KeyboardInterrupt: 36 | process = None 37 | 38 | except (subprocess.CalledProcessError,OSError,ValueError): 39 | self.log.error('could not spawn process %s' % program) 40 | process = None 41 | 42 | if process: 43 | try: 44 | fcntl.fcntl(process.stderr, fcntl.F_SETFL, os.O_NONBLOCK) 45 | except IOError: 46 | self.destroyProcess(process) 47 | process = None 48 | 49 | return process 50 | 51 | def destroyProcess (self, process): 52 | try: 53 | process.terminate() 54 | process.wait() 55 | self.log.info('terminated process PID %s' % process.pid) 56 | 57 | except OSError, e: 58 | # No such processs 59 | if e[0] != errno.ESRCH: 60 | self.log.error('PID %s died' % process.pid) 61 | -------------------------------------------------------------------------------- /lib/exaproxy/reactor/redirector/dispatch.py: -------------------------------------------------------------------------------- 1 | from threading import Thread 2 | from exaproxy.util.messagequeue import Queue 3 | 4 | 5 | class DispatcherThread (Thread): 6 | def __init__ (self, messagebox, queue): 7 | self.messagebox = messagebox 8 | self.queue = queue 9 | Thread.__init__(self) 10 | 11 | def run (self): 12 | while True: 13 | command, message = self.queue.get() 14 | 15 | if command == 'REQUEST': 16 | self.messagebox.box.put(message) 17 | continue 18 | 19 | if command == 'STOP': 20 | break 21 | 22 | 23 | class RedirectorDispatcher (object): 24 | dispatcher_factory = DispatcherThread 25 | 26 | def __init__ (self, messagebox): 27 | self.messagebox = messagebox 28 | self.queue = Queue() 29 | self.thread = self.dispatcher_factory(messagebox, self.queue) 30 | 31 | def start (self): 32 | self.thread.start() 33 | 34 | def stop (self): 35 | self.queue.put(('STOP', '')) 36 | self.thread.join() 37 | try: 38 | res = self.messagebox.stop() 39 | except IOError: 40 | res = None 41 | 42 | return res 43 | 44 | def sendRequest (self, client_id, accept_addr, accept_port, peer, request, subrequest, source): 45 | message = client_id, accept_addr, accept_port, peer, request, subrequest, source 46 | return self.queue.put(('REQUEST', message)) 47 | 48 | def getDecision (self): 49 | return self.messagebox.getDecision() 50 | 51 | def respawn (self): 52 | return self.messagebox.respawn() 53 | 54 | def decreaseSpawnLimit (self, *args): 55 | return self.messagebox.decreaseSpawnLimit(*args) 56 | 57 | def increaseSpawnLimit (self, *args): 58 | return self.messagebox.increaseSpawnLimit(*args) 59 | 60 | def requestStats (self): 61 | return self.messagebox.requestStats() 62 | 63 | def readResponse (self): 64 | command, [data] = self.messagebox.readResponse() 65 | return command, data 66 | -------------------------------------------------------------------------------- /lib/exaproxy/reactor/redirector/messagebox.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | messagebox.py 4 | 5 | Created by David Farrar on 2014-06-10 (or earlier). 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | from exaproxy.util.messagebox import MessageBox 10 | from exaproxy.util.control import ControlBox 11 | 12 | class ProxyToRedirectorMessageBox: 13 | def __init__ (self, pid, pipe_in, pipe_out, control_in, control_out): 14 | self.pid = pid 15 | self.box = MessageBox(pipe_in, pipe_out) 16 | self.control = ControlBox(control_in, control_out) 17 | 18 | def close (self): 19 | return self.box.close() 20 | 21 | def sendRequest (self, client_id, accept_addr, accept_port, peer, request, subrequest, source): 22 | message = client_id, accept_addr, accept_port, peer, request, subrequest, source 23 | return self.box.put(message) 24 | 25 | def getDecision (self): 26 | message = self.box.get() 27 | if message is not None: 28 | client_id, command, decision = message 29 | 30 | else: 31 | client_id, command, decision = None, None, None 32 | 33 | return client_id, command, decision 34 | 35 | def stop (self): 36 | self.control.send('STOP') 37 | return self.control.wait_stop() 38 | 39 | def respawn (self): 40 | self.control.send('RESPAWN') 41 | 42 | def decreaseSpawnLimit (self, count=1): 43 | self.control.send('DECREASE', count) 44 | 45 | def increaseSpawnLimit (self, count=1): 46 | self.control.send('INCREASE', count) 47 | 48 | def requestStats (self): 49 | identifier = self.control.send('STATS') 50 | 51 | def readResponse (self): 52 | return self.control.receive() 53 | 54 | 55 | 56 | class RedirectorToProxyMessageBox: 57 | def __init__ (self, pipe_in, pipe_out): 58 | self.box = MessageBox(pipe_in, pipe_out) 59 | 60 | def close (self): 61 | return self.box.close() 62 | 63 | def isClosed (self): 64 | return self.box.pipe_in.closed 65 | 66 | def getRequest (self): 67 | return self.box.get() 68 | 69 | def sendResponse (self, client_id, command, decision): 70 | message = client_id, command, decision 71 | return self.box.put(message) 72 | -------------------------------------------------------------------------------- /lib/exaproxy/reactor/redirector/reactor.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | reactor.py 4 | 5 | Created by David Farrar on 2014-03-17. 6 | Copyright (c) 2011-2014 Exa Networks. All rights reserved. 7 | """ 8 | 9 | from exaproxy.util.log.logger import Logger 10 | 11 | 12 | def debug (item, message): 13 | print '*' * 40 + ' ' + str(message) 14 | print str(item) 15 | print '*' * 40 16 | print 17 | 18 | class RedirectorReactor (object): 19 | def __init__ (self, configuration, querier, decider, logger, usage, poller): 20 | self.querier = querier # Incoming requests from the proxy 21 | self.decider = decider # Decides how each request should be handled 22 | self.logger = logger # Log writing interfaces 23 | self.usage = usage 24 | self.poller = poller 25 | 26 | # NOT the same logger the rest of the proxy uses since we're running in a different process 27 | self.log = Logger('redirector', configuration.log.supervisor) 28 | self.running = True 29 | 30 | def run(self): 31 | self.running = True 32 | 33 | # we may have new workers to deal with queued requests 34 | client_id, command, decision = self.decider.doqueue() 35 | while command is not None: 36 | self.querier.sendResponse(client_id, command, decision) 37 | client_id, command, decision = self.decider.doqueue() 38 | 39 | while self.running: 40 | # wait until we have something to do 41 | events = self.poller.poll() 42 | 43 | if events.get('control'): 44 | break 45 | 46 | # new requests 47 | if events.get('read_request'): 48 | try: 49 | message = self.querier.getRequest() 50 | except Exception: 51 | message = None 52 | 53 | if message is None: 54 | return False 55 | 56 | client_id, accept_addr, accept_port, peer, header, subheader, source = message 57 | _, command, decision = self.decider.request(client_id, accept_addr, accept_port, peer, header, subheader, source) 58 | 59 | if command is not None: 60 | self.querier.sendResponse(client_id, command, decision) 61 | 62 | # decisions made by the child processes 63 | for worker in events.get('read_workers', []): 64 | client_id, command, decision = self.decider.getDecision(worker) 65 | if command is not None: 66 | self.querier.sendResponse(client_id, command, decision) 67 | 68 | # we should have available workers now so check for queued requests 69 | for worker in events.get('read_workers', []): 70 | client_id, command, decision = self.decider.doqueue() 71 | 72 | if command is not None: 73 | self.querier.sendResponse(client_id, command, decision) 74 | 75 | self.logger.writeMessages() 76 | self.usage.writeMessages() 77 | 78 | return True 79 | -------------------------------------------------------------------------------- /lib/exaproxy/reactor/redirector/redirector.py: -------------------------------------------------------------------------------- 1 | from .worker import Redirector 2 | from .icap import ICAPRedirector 3 | 4 | class RedirectorFactory (object): 5 | def __init__ (self, configuration, program, protocol): 6 | self.configuration = configuration 7 | self.program = program 8 | self.protocol = protocol 9 | 10 | def create (self, wid): 11 | if self.protocol == 'url': 12 | redirector = Redirector(self.configuration, wid, self.program, self.protocol) 13 | 14 | elif self.protocol.startswith('icap://'): 15 | redirector = ICAPRedirector(self.configuration, wid, self.program, self.protocol) 16 | 17 | else: 18 | redirector = None 19 | 20 | return redirector 21 | -------------------------------------------------------------------------------- /lib/exaproxy/reactor/redirector/response.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | 4 | class ResponseEncoder (object): 5 | @staticmethod 6 | def icap (client_id, response, length): 7 | return client_id, 'icap', (response, str(length)) 8 | 9 | @staticmethod 10 | def download (client_id, ip, port, upgrade, length, message): 11 | return client_id, 'download', (ip, str(port), upgrade, str(length), str(message)) 12 | 13 | @staticmethod 14 | def connect (client_id, host, port, message): 15 | return client_id, 'connect', (host, str(port), str(message)) 16 | 17 | @staticmethod 18 | def intercept (client_id, host, port, message): 19 | return client_id, 'intercept', (host, str(port), str(message)) 20 | 21 | @staticmethod 22 | def file (client_id, code, reason): 23 | return client_id, 'file', (str(code), reason) 24 | 25 | @staticmethod 26 | def rewrite (client_id, code, reason, comment, message): 27 | return client_id, 'rewrite', (code, reason, comment, message.request.protocol, message.url, message.host, str(message.client)) 28 | 29 | @staticmethod 30 | def http (client_id, data): 31 | return client_id, 'http', data 32 | 33 | @staticmethod 34 | def monitor (client_id, path): 35 | return client_id, 'monitor', path 36 | 37 | @staticmethod 38 | def redirect (client_id, url): 39 | return client_id, 'redirect', url 40 | 41 | @staticmethod 42 | def stats (wid, timestamp, stats): 43 | return wid, 'stats', (timestamp, stats) 44 | 45 | @staticmethod 46 | def requeue (client_id, peer, header, subheader, source): 47 | # header and source are flipped to make it easier to split the values 48 | return client_id, 'requeue', (peer, source, header, subheader) 49 | 50 | @staticmethod 51 | def hangup (wid): 52 | return '', 'close', wid 53 | 54 | @staticmethod 55 | def close (client_id): 56 | return client_id, 'close', '' 57 | 58 | @staticmethod 59 | def defer (client_id, message): 60 | return client_id, 'defer', message 61 | 62 | @staticmethod 63 | def error (client_id): 64 | return client_id, None, None 65 | 66 | 67 | 68 | def splithost (data, default_port): 69 | if ':' in data: 70 | host, port = data.split(':', 1) 71 | 72 | else: 73 | host, port = data, None 74 | 75 | if port is None or not port.isdigit(): 76 | port = default_port 77 | 78 | return host, port 79 | 80 | 81 | class ResponseFactory (object): 82 | encoder = ResponseEncoder 83 | 84 | def contentResponse (self, client_id, message, classification, data, comment): 85 | if classification == 'permit': 86 | return ('PERMIT', message.host), self.encoder.download(client_id, message.host, message.port, message.upgrade, message.content_length, message) 87 | 88 | if classification == 'rewrite': 89 | message.redirect(None, data) 90 | return ('REWRITE', data), self.encoder.download(client_id, message.host, message.port, '', message.content_length, message) 91 | 92 | if classification == 'file': 93 | return ('FILE', data), self.encoder.rewrite(client_id, '200', data, comment, message) 94 | 95 | if classification == 'redirect': 96 | return ('REDIRECT', data), self.encoder.redirect(client_id, data) 97 | 98 | if classification == 'intercept': 99 | host, port = splithost(data, message.port) 100 | return ('INTERCEPT', data), self.encoder.download(client_id, host, port, '', message.content_length, message) 101 | 102 | if classification == 'http': 103 | return ('LOCAL', ''), self.encoder.http(client_id, data) 104 | 105 | return ('PERMIT', message.host), self.encoder.download(client_id, message.host, message.port, message.upgrade, message.content_length, message) 106 | 107 | def connectResponse (self, client_id, message, classification, data, comment): 108 | if classification == 'permit': 109 | return ('PERMIT', message.host), self.encoder.connect(client_id, message.host, message.port, '') 110 | 111 | if classification == 'redirect': 112 | return ('REDIRECT', data), self.encoder.redirect(client_id, data) 113 | 114 | if classification == 'intercept': 115 | host, port = splithost(data, message.port) 116 | return ('INTERCEPT', data), self.encoder.intercept(client_id, host, port, message) 117 | 118 | if classification == 'file': 119 | return ('FILE', data), self.encoder.rewrite(client_id, '200', data, comment, message) 120 | 121 | if classification == 'http': 122 | return ('LOCAL', ''), self.encoder.http(client_id, data) 123 | 124 | return ('PERMIT', message.host), self.encoder.connect(client_id, message.host, message.port, '') 125 | -------------------------------------------------------------------------------- /lib/exaproxy/reactor/redirector/serialize/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Exa-Networks/exaproxy/8b7291b79c1cd6542213a5e7d8dda3cf5a676166/lib/exaproxy/reactor/redirector/serialize/__init__.py -------------------------------------------------------------------------------- /lib/exaproxy/reactor/redirector/serialize/icap.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | class ICAPSerializer (object): 4 | def __init__ (self, configuration, protocol): 5 | self.configuration = configuration 6 | self.protocol = protocol 7 | 8 | def serialize (self, accept_addr, accept_port, peer, message, icap_message, http_header, path, icap_host): 9 | if icap_message is not None and icap_message.method == 'OPTIONS': 10 | res = self.createOptionsRequest(accept_addr, accept_port, peer, icap_message, path) 11 | return res 12 | 13 | return self.createRequest(accept_addr, accept_port, peer, message, icap_message, http_header, path, icap_host) 14 | 15 | def createOptionsRequest (self, accept_addr, accept_port, peer, icap_message, path): 16 | return """\ 17 | OPTIONS %s ICAP/1.0 18 | Pragma: transport= 19 | Pragma: proxy=test 20 | Pragma: scheme= 21 | Pragma: accept=%s 22 | Pragma: accept-port=%s 23 | Pragma: client=%s 24 | Pragma: host= 25 | Pragma: path= 26 | Pragma: method= 27 | Encapsulated: req-hdr=0, null-body=0 28 | 29 | 30 | """ % (path, accept_addr, accept_port, peer) 31 | 32 | def createRequest (self, accept_addr, accept_port, peer, message, icap_message, http_header, path, icap_host): 33 | username = icap_message.headers.get('x-authenticated-user', '').strip() if icap_message else None 34 | groups = icap_message.headers.get('x-authenticated-groups', '').strip() if icap_message else None 35 | ip_addr = icap_message.headers.get('x-client-ip', '').strip() if icap_message else None 36 | customer = icap_message.headers.get('x-customer-name', '').strip() if icap_message else None 37 | allow = icap_message.headers.get('allow', '').strip() if icap_message else None 38 | 39 | icap_request = """\ 40 | REQMOD %s ICAP/1.0 41 | Host: %s 42 | Pragma: transport=%s 43 | Pragma: proxy=test 44 | Pragma: scheme=http 45 | Pragma: accept=%s 46 | Pragma: accept-port=%s 47 | Pragma: client=%s 48 | Pragma: host=%s 49 | Pragma: path=%s 50 | Pragma: method=%s""" % ( 51 | 52 | path, icap_host, message.request.protocol, 53 | accept_addr, accept_port, peer, message.host, message.request.path, message.request.method, 54 | ) 55 | 56 | if ip_addr: 57 | icap_request += """ 58 | X-Client-IP: %s""" % ip_addr 59 | 60 | if username: 61 | icap_request += """ 62 | X-Authenticated-User: %s""" % username 63 | 64 | if groups: 65 | icap_request += """ 66 | X-Authenticated-Groups: %s""" % groups 67 | 68 | if customer: 69 | icap_request += """ 70 | X-Customer-Name: %s""" % customer 71 | 72 | if allow: 73 | icap_request += """ 74 | Allow: %s""" % allow 75 | 76 | return icap_request + """ 77 | Encapsulated: req-hdr=0, null-body=%d 78 | 79 | %s""" % (len(http_header), http_header) 80 | -------------------------------------------------------------------------------- /lib/exaproxy/reactor/redirector/serialize/passthrough.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | 4 | 5 | class PassthroughSerializer (object): 6 | def __init__ (self, configuration, protocol): 7 | self.configuration = configuration 8 | self.protocol = protocol 9 | 10 | def serialize (self, accept_addr, accept_port, peer, path, icap_host): 11 | icap_request = """\ 12 | REQMOD %s ICAP/1.0 13 | Host: %s 14 | Pragma: transport=%s 15 | Pragma: accept=%s 16 | Pragma: accept-port=%s 17 | Pragma: client=%s 18 | Pragma: method=%s""" % ( 19 | 20 | path, icap_host, 'tcp', 21 | accept_addr, accept_port, peer, 'TCP', 22 | ) 23 | 24 | return icap_request + """ 25 | Encapsulated: req-hdr=0, null-body=0 26 | 27 | """ 28 | -------------------------------------------------------------------------------- /lib/exaproxy/reactor/redirector/serialize/tls.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | 4 | 5 | class TLSSerializer (object): 6 | def __init__ (self, configuration, protocol): 7 | self.configuration = configuration 8 | self.protocol = protocol 9 | 10 | def serialize (self, accept_addr, accept_port, peer, message, tls_header, path, icap_host): 11 | icap_request = """\ 12 | REQMOD %s ICAP/1.0 13 | Host: %s 14 | Pragma: transport=%s 15 | Pragma: proxy=test 16 | Pragma: scheme=http 17 | Pragma: accept=%s 18 | Pragma: accept-port=%s 19 | Pragma: client=%s 20 | Pragma: host=%s 21 | Pragma: path=%s 22 | Pragma: method=%s""" % ( 23 | 24 | path, icap_host, 'tls', 25 | accept_addr, accept_port, peer, message.hostname if message else '', '', 'TLS', 26 | ) 27 | 28 | return icap_request + """ 29 | Encapsulated: req-hdr=0, null-body=%d 30 | 31 | %s""" % (len(tls_header), tls_header) 32 | -------------------------------------------------------------------------------- /lib/exaproxy/reactor/redirector/supervisor.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | supervisor.py 4 | 5 | Created by Thomas Mangin on 2011-11-29. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | import signal 10 | 11 | from exaproxy.network.async import Poller 12 | from exaproxy.util.log.writer import UsageWriter 13 | from exaproxy.util.log.writer import SysLogWriter 14 | from .manager import RedirectorManager 15 | from .reactor import RedirectorReactor 16 | 17 | 18 | class RedirectorSupervisor (object): 19 | alarm_time = 1 # regular backend work 20 | increase_frequency = int(5/alarm_time) # when we add workers 21 | decrease_frequency = int(60/alarm_time) # when we remove workers 22 | 23 | def __init__ (self, configuration, messagebox, controlbox): 24 | self.configuration = configuration 25 | self.log_writer = SysLogWriter('log', configuration.log.destination, configuration.log.enable, level=configuration.log.level) 26 | self.usage_writer = UsageWriter('usage', configuration.usage.destination, configuration.usage.enable) 27 | 28 | if configuration.debug.log: 29 | self.usage_writer.toggleDebug() 30 | 31 | self.messagebox = messagebox 32 | self.controlbox = controlbox 33 | self.poller = Poller(self.configuration.daemon, speed=0) 34 | self.poller.setupRead('control') # control messages from the main process 35 | self.poller.addReadSocket('control', controlbox.box.pipe_in) 36 | 37 | signal.signal(signal.SIGALRM, self.sigalrm) 38 | self._respawn = False 39 | 40 | # poller for the reactor 41 | poller = Poller(self.configuration.daemon) 42 | poller.setupRead('read_request') # requests passed from the main process 43 | poller.setupRead('read_workers') # responses from the child processes 44 | poller.setupRead('control') # the reactor needs to yield to the supervisor 45 | poller.addReadSocket('read_request', messagebox.box.pipe_in) 46 | poller.addReadSocket('control', controlbox.box.pipe_in) 47 | 48 | self.manager = RedirectorManager(configuration, poller) 49 | 50 | # start the child processes 51 | self.manager.startup() 52 | 53 | self.reactor = RedirectorReactor(self.configuration, self.messagebox, self.manager, self.log_writer, self.usage_writer, poller) 54 | self.running = True 55 | 56 | def sigalrm (self, signum, frame): 57 | self.reactor.running = False 58 | signal.setitimer(signal.ITIMER_REAL, self.alarm_time, self.alarm_time) 59 | 60 | 61 | def increase_spawn_limit (self,count): 62 | self.manager.low += count 63 | self.manager.high = max(self.manager.low, self.manager.high) 64 | self.manager.increase(count) 65 | 66 | def decrease_spawn_limit (self,count): 67 | self.manager.high = max(1, self.manager.high - count) 68 | self.manager.low = min(self.manager.high, self.manager.low) 69 | self.manager.decrease(count) 70 | 71 | def sendStats (self, identifier): 72 | self.controlbox.respond(identifier, 'STATS', { 73 | 'forked' : len(self.manager.worker), 74 | 'min' : self.manager.low, 75 | 'max' : self.manager.high, 76 | 'queue' : self.manager.queue.qsize(), 77 | }) 78 | 79 | def control (self): 80 | status = True 81 | identifier, command, data = self.controlbox.receive() 82 | 83 | if command == 'STATS': 84 | self.sendStats(identifier) 85 | 86 | elif command == 'INCREASE': 87 | self.increase_spawn_limit(data[0]) 88 | 89 | elif command == 'DECREASE': 90 | self.decrease_spawn_limit(data[0]) 91 | 92 | elif command == 'RESPAWN': 93 | self._respawn = True 94 | 95 | if command == 'STOP' or not command: 96 | status = False 97 | 98 | return status 99 | 100 | def run (self): 101 | signal.setitimer(signal.ITIMER_REAL,self.alarm_time,self.alarm_time) 102 | 103 | count_increase = 0 104 | count_decrease = 0 105 | 106 | try: 107 | while self.running is True: 108 | count_increase = (count_increase + 1) % self.increase_frequency 109 | count_decrease = (count_decrease + 1) % self.decrease_frequency 110 | 111 | # check for IO change with select 112 | status = self.reactor.run() 113 | if status is False: 114 | break 115 | 116 | 117 | # make sure we have enough workers 118 | if count_increase == 0: 119 | self.manager.provision() 120 | 121 | # and every so often remove useless workers 122 | if count_decrease == 0: 123 | self.manager.deprovision() 124 | 125 | # check to respawn command 126 | if self._respawn is True: 127 | self._respawn = False 128 | self.manager.respawn() 129 | 130 | 131 | events = self.poller.poll() 132 | while events.get('control'): 133 | if not self.control(): 134 | self.running = False 135 | break 136 | 137 | events = self.poller.poll() 138 | 139 | except KeyboardInterrupt: 140 | pass 141 | 142 | self.manager.kill_workers() 143 | -------------------------------------------------------------------------------- /lib/exaproxy/reactor/resolver/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Exa-Networks/exaproxy/8b7291b79c1cd6542213a5e7d8dda3cf5a676166/lib/exaproxy/reactor/resolver/__init__.py -------------------------------------------------------------------------------- /lib/exaproxy/tls/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Exa-Networks/exaproxy/8b7291b79c1cd6542213a5e7d8dda3cf5a676166/lib/exaproxy/tls/__init__.py -------------------------------------------------------------------------------- /lib/exaproxy/tls/decode.py: -------------------------------------------------------------------------------- 1 | from struct import unpack 2 | 3 | from .header import TLS_HEADER_LEN 4 | from .header import TLS_HANDSHAKE_CONTENT_TYPE 5 | from .header import TLS_HANDSHAKE_TYPE_CLIENT_HELLO 6 | 7 | 8 | # inspired from https://raw.githubusercontent.com/dlundquist/sniproxy/master/src/tls.c 9 | 10 | ALERT = [ 11 | 0x15, # TLS Alert 12 | 0x03, 0x01, # TLS version 13 | 0x00, 0x02, # Payload length 14 | 0x02, 0x28, # Fatal, handshake failure 15 | ] 16 | 17 | def short (data): 18 | return unpack('!H',data)[0] 19 | 20 | 21 | def get_tls_hello_size (data): 22 | return short(data[3:5]) + TLS_HEADER_LEN 23 | 24 | 25 | def parse_hello (data): 26 | try: 27 | # Check that our TCP payload is at least large enough for a TLS header 28 | 29 | if len(data) < TLS_HEADER_LEN+1: # HEADER + HELLO 30 | # Not enough data to be a TLS header 31 | return None 32 | 33 | # SSL 2.0 compatible Client Hello 34 | # High bit of first byte (length) and content type is Client Hello 35 | # See RFC5246 Appendix E.2 36 | 37 | tls_content_type = ord(data[0]) 38 | 39 | if tls_content_type != TLS_HANDSHAKE_CONTENT_TYPE: 40 | # Request did not begin with TLS handshake 41 | return None 42 | 43 | tls_version_major = ord(data[1]) 44 | tls_version_minor = ord(data[2]) 45 | 46 | if tls_content_type & 0x80 and tls_version_minor == 1: 47 | # Received SSL 2.0 Client Hello which can not support SNI') 48 | return None 49 | 50 | if tls_version_major < 3: 51 | # "Received SSL %d.%d handshake which which can not support SNI." % (tls_version_major, tls_version_minor) 52 | return None 53 | 54 | # TLS record length 55 | length = short(data[3:5]) + TLS_HEADER_LEN 56 | data = data[TLS_HEADER_LEN:] 57 | data = data[:length] 58 | 59 | # Handshake 60 | if ord(data[0]) != TLS_HANDSHAKE_TYPE_CLIENT_HELLO: 61 | # Not a TLS Client HELLO 62 | return None 63 | 64 | # Skip past fixed length records: 65 | # 1 Handshake Type 66 | # 3 Length 67 | # 2 Version (again) 68 | # 32 Random 69 | # to Session ID Length 70 | 71 | data = data[38:] 72 | 73 | # Session ID 74 | length = ord(data[0]) 75 | data = data[length+1:] 76 | 77 | # Cipher Suites 78 | length = short(data[:2]) 79 | data = data[length+2:] 80 | 81 | # Compression methods 82 | length = ord(data[0]) 83 | data = data[length+1:] 84 | 85 | if not data and tls_version_major == 3 and tls_version_minor == 0: 86 | # Received SSL 3.0 handshake without extensions 87 | return None 88 | 89 | if not data: 90 | return None 91 | 92 | # Extensions 93 | length = short(data[:2]) 94 | data = data[2:] 95 | data = data[:length] 96 | 97 | # parse extension 98 | while data: 99 | length = short(data[2:4]) 100 | if short(data[0:2]) == 0x0000: 101 | data = data[4+2:length+4:] # +2 to eat the outter length 102 | break 103 | data = data[length+4:] 104 | 105 | # name extension not found 106 | if not data: 107 | return None 108 | 109 | while data: 110 | length = short(data[1:3]) 111 | # found name 112 | if ord(data[0]) == 0x00: 113 | return data[3:length+3] 114 | data = data[length+3:] 115 | 116 | return None 117 | 118 | except IndexError: 119 | # Buffer Underflow 120 | return None 121 | 122 | if __name__ == '__main__': 123 | # Google TLS HELLO 124 | raw = ''.join([ chr(_) for _ in [ 125 | 0x16, 0x03, 0x01, 0x00, 0xb9, 0x01, 0x00, 0x00, 126 | 0xb5, 0x03, 0x03, 0xdc, 0xb1, 0xcb, 0xde, 0x25, 127 | 0x3b, 0xc4, 0x7c, 0x10, 0xb0, 0x23, 0xee, 0xed, 128 | 0x47, 0xeb, 0xa1, 0xa1, 0x43, 0xbb, 0x39, 0x87, 129 | 0x0b, 0x5f, 0x64, 0x6a, 0xd1, 0x1c, 0x5c, 0xe9, 130 | 0x2d, 0x9f, 0xc9, 0x00, 0x00, 0x16, 0xc0, 0x2b, 131 | 0xc0, 0x2f, 0xc0, 0x0a, 0xc0, 0x09, 0xc0, 0x13, 132 | 0xc0, 0x14, 0x00, 0x33, 0x00, 0x39, 0x00, 0x2f, 133 | 0x00, 0x35, 0x00, 0x0a, 0x01, 0x00, 0x00, 0x76, 134 | 0x00, 0x00, 0x00, 0x15, 0x00, 0x13, 0x00, 0x00, 135 | 0x10, 0x77, 0x77, 0x77, 0x2e, 0x67, 0x6f, 0x6f, 136 | 0x67, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x2e, 0x75, 137 | 0x6b, 0xff, 0x01, 0x00, 0x01, 0x00, 0x00, 0x0a, 138 | 0x00, 0x08, 0x00, 0x06, 0x00, 0x17, 0x00, 0x18, 139 | 0x00, 0x19, 0x00, 0x0b, 0x00, 0x02, 0x01, 0x00, 140 | 0x00, 0x23, 0x00, 0x00, 0x33, 0x74, 0x00, 0x00, 141 | 0x00, 0x10, 0x00, 0x17, 0x00, 0x15, 0x02, 0x68, 142 | 0x32, 0x08, 0x73, 0x70, 0x64, 0x79, 0x2f, 0x33, 143 | 0x2e, 0x31, 0x08, 0x68, 0x74, 0x74, 0x70, 0x2f, 144 | 0x31, 0x2e, 0x31, 0x00, 0x05, 0x00, 0x05, 0x01, 145 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x16, 146 | 0x00, 0x14, 0x04, 0x01, 0x05, 0x01, 0x06, 0x01, 147 | 0x02, 0x01, 0x04, 0x03, 0x05, 0x03, 0x06, 0x03, 148 | 0x02, 0x03, 0x04, 0x02, 0x02, 0x02 149 | ]]) 150 | 151 | print get_tls_hello_size(raw), '==', len(raw) 152 | print parse_hello(raw) 153 | -------------------------------------------------------------------------------- /lib/exaproxy/tls/header.py: -------------------------------------------------------------------------------- 1 | TLS_HEADER_LEN = 0x5 2 | TLS_HANDSHAKE_CONTENT_TYPE = 0x16 3 | TLS_HANDSHAKE_TYPE_CLIENT_HELLO = 0x01 4 | 5 | -------------------------------------------------------------------------------- /lib/exaproxy/tls/parser.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from .request import TLSRequestFactory 3 | from .response import TLSResponseFactory 4 | 5 | from .decode import parse_hello 6 | 7 | class TLSParser (object): 8 | TLSRequestFactory = TLSRequestFactory 9 | TLSResponseFactory = TLSResponseFactory 10 | 11 | def __init__ (self, configuration): 12 | self.configuration = configuration 13 | self.request_factory = self.TLSRequestFactory(configuration) 14 | self.response_factory = self.TLSResponseFactory(configuration) 15 | 16 | def parseClientHello (self, tls_header): 17 | hostname = parse_hello(tls_header) 18 | 19 | if not hostname: 20 | return None 21 | 22 | return self.request_factory.createClientHello(hostname) 23 | 24 | 25 | def encodeFailureResponse (self, response): 26 | return self.response_factory.createResponse(response.version, response.content_type, response.response, response.reason) 27 | -------------------------------------------------------------------------------- /lib/exaproxy/tls/request.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | class TLSRequest(object): 4 | __slots__ = ('hostname',) 5 | 6 | def __init__ (self, hostname): 7 | self.hostname = hostname 8 | 9 | 10 | class TLSRequestFactory: 11 | def __init__ (self, configuration): 12 | self.configuration = configuration 13 | 14 | def createClientHello (self, hostname): 15 | return TLSRequest(hostname) 16 | -------------------------------------------------------------------------------- /lib/exaproxy/tls/response.py: -------------------------------------------------------------------------------- 1 | class TLSResponse (object): 2 | @property 3 | def is_handshake (self): 4 | return False 5 | 6 | @property 7 | def is_failure (self): 8 | return False 9 | 10 | 11 | 12 | class TLSFailureResponse (TLSResponse): 13 | def __init__ (self, version, reason): 14 | self.version = version 15 | self.reason = reason 16 | 17 | @property 18 | def is_failure (self): 19 | return True 20 | 21 | 22 | 23 | 24 | 25 | 26 | TLS_CONTENT_ALERT = 15 27 | TLS_RESPONSE_FATAL = 2 28 | 29 | TLS_FAILURE_HANDSHAKE = 40 30 | 31 | 32 | class TLSResponseFactory (object): 33 | version = 3 34 | 35 | def __init__ (self, configuration): 36 | self.configuration = configuration 37 | 38 | def getHandshakeFailure (self): 39 | return TLSFailureResponse(self.version, TLS_FAILURE_HANDSHAKE) 40 | 41 | 42 | def encodeFailureResponse (reason): 43 | return '\x15\x03' + chr(self.version) + '\x00\x02\x02' + chr(reason) 44 | -------------------------------------------------------------------------------- /lib/exaproxy/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Exa-Networks/exaproxy/8b7291b79c1cd6542213a5e7d8dda3cf5a676166/lib/exaproxy/util/__init__.py -------------------------------------------------------------------------------- /lib/exaproxy/util/alarm.py: -------------------------------------------------------------------------------- 1 | import os 2 | from threading import Thread 3 | 4 | from .messagequeue import Queue, Empty 5 | from .messagebox import MessageBox 6 | 7 | class AlarmThread (Thread): 8 | def __init__ (self, messagebox, queue, period): 9 | self.messagebox = messagebox 10 | self.queue = queue 11 | self.period = period 12 | Thread.__init__(self) 13 | 14 | def run (self): 15 | while True: 16 | command, message = self.queue.get() 17 | 18 | while True: 19 | if command == 'STOP': 20 | break 21 | 22 | if command == 'ALARM': 23 | try: 24 | command, message = self.queue.get(timeout=self.period) 25 | 26 | except Empty: pass 27 | 28 | else: 29 | continue 30 | 31 | self.messagebox.put('alarm') 32 | 33 | 34 | if command == 'STOP': 35 | break 36 | 37 | 38 | class AlarmDispatcher (object): 39 | dispatcher_factory = AlarmThread 40 | 41 | def __init__ (self, messagebox, period): 42 | self.messagebox = messagebox 43 | self.queue = Queue() 44 | self.thread = self.dispatcher_factory(messagebox, self.queue, period) 45 | 46 | def start (self): 47 | self.thread.start() 48 | 49 | def stop (self): 50 | self.queue.put(('STOP', '')) 51 | self.thread.join() 52 | return True 53 | 54 | def setAlarm (self): 55 | # NOTE: could dynamically set the delay by passing it through the pipe 56 | self.queue.put(('ALARM', '')) 57 | 58 | def acknowledgeAlarm (self): 59 | return self.messagebox.get() 60 | 61 | 62 | 63 | 64 | def alarm_thread (poller, period): 65 | r, w = os.pipe() 66 | message_box = MessageBox(r, w) 67 | 68 | dispatcher = AlarmDispatcher(message_box, period) 69 | dispatcher.start() 70 | 71 | poller.addReadSocket('read_interrupt', dispatcher.messagebox.pipe_in) 72 | return dispatcher 73 | -------------------------------------------------------------------------------- /lib/exaproxy/util/cache.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | try: 4 | from collections import OrderedDict 5 | except ImportError: 6 | # support installable ordereddict module in older python versions 7 | from ordereddict import OrderedDict 8 | 9 | from time import time 10 | 11 | class TimeCache (dict): 12 | __default = object() 13 | 14 | def __init__ (self,timeout): 15 | self.timeout = timeout 16 | self.last = None 17 | self.time = OrderedDict() 18 | dict.__init__(self) 19 | 20 | def __setitem__ (self,key,value): 21 | dict.__setitem__(self,key,value) 22 | if self.timeout > 0: 23 | self.time[key] = time() 24 | 25 | def __delitem__ (self,key): 26 | if key in self.time: 27 | del self.time[key] 28 | dict.__delitem__(self,key) 29 | 30 | # Cpython implementation of dict.pop does not call __delitem__ - sigh ! 31 | def pop (self,key,default=__default): 32 | if key in self.time: 33 | del self.time[key] 34 | if default is self.__default: 35 | return dict.pop(self,key) 36 | return dict.pop(self,key,default) 37 | 38 | def expired (self,maximum): 39 | expire = time() - self.timeout 40 | 41 | if self.last: 42 | k,t = self.last 43 | if t > expire: 44 | return 45 | if k in self: 46 | maximum -= 1 47 | yield k 48 | self.last = None 49 | 50 | while self.time and maximum: 51 | k,t = self.time.popitem(False) 52 | if t > expire: 53 | self.last = k,t 54 | break 55 | if k in self: 56 | maximum -= 1 57 | yield k 58 | -------------------------------------------------------------------------------- /lib/exaproxy/util/control.py: -------------------------------------------------------------------------------- 1 | from .messagebox import MessageBox 2 | 3 | def cycler(): 4 | def cycle_identifiers(): 5 | while True: 6 | for identifier in xrange(0xffff): 7 | yield identifier 8 | 9 | return cycle_identifiers().next 10 | 11 | 12 | 13 | class ControlBox: 14 | def __init__ (self, channel_in, channel_out): 15 | self.box = MessageBox(channel_in, channel_out) 16 | self.next_identifier = cycler() 17 | 18 | def send (self, command, *args): 19 | identifier = self.next_identifier() 20 | self.box.put((identifier, command, args)) 21 | return identifier 22 | 23 | def receive (self, identifier=None): 24 | message = self.box.get() 25 | 26 | if message is not None: 27 | ack, command, data = message 28 | 29 | else: 30 | ack, command, data = None, None, [None] 31 | 32 | if identifier is not None and ack != identifier: 33 | data = None 34 | 35 | return command, data 36 | 37 | def wait_stop (self): 38 | message = self.box.get() 39 | 40 | if message is None: 41 | return True 42 | 43 | raise RuntimeError, 'got data from a process that should have stopped' 44 | 45 | 46 | class SlaveBox: 47 | def __init__ (self, channel_in, channel_out): 48 | self.box = MessageBox(channel_in, channel_out) 49 | 50 | def receive (self): 51 | message = self.box.get() 52 | 53 | if message is not None: 54 | identifier, command, data = message 55 | 56 | else: 57 | identifier, command, data = None, None, None 58 | 59 | return identifier, command, data 60 | 61 | def respond (self, identifier, command, *args): 62 | self.box.put((identifier, command, args)) 63 | -------------------------------------------------------------------------------- /lib/exaproxy/util/daemon.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | daemon.py 4 | 5 | Created by Thomas Mangin on 2011-11-29. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | import os 10 | import sys 11 | import pwd 12 | import errno 13 | import socket 14 | import resource 15 | 16 | from .log.logger import Logger 17 | 18 | def signed (value): 19 | if value == sys.maxint: 20 | return -1 21 | return value 22 | 23 | class Daemon (object): 24 | def __init__ (self,configuration): 25 | self.filemax = 0 26 | self.daemonize = configuration.daemon.daemonize 27 | self.user = configuration.daemon.user 28 | self.log = Logger('daemon', configuration.log.daemon) 29 | #mask = os.umask(0137) 30 | 31 | if configuration.web.debug: 32 | self.log.critical('WARNING: python remote execution via the web server is enabled') 33 | 34 | if configuration.daemon.reactor == 'epoll' and not sys.platform.startswith('linux'): 35 | self.log.error('exaproxy.daemon.reactor can only be epoll only on Linux') 36 | sys.exit(1) 37 | 38 | if configuration.daemon.reactor == 'kqueue' and not sys.platform.startswith('freebsd') and not sys.platform.startswith('darwin'): 39 | self.log.error('exaproxy.daemon.reactor can only be kqueue only on FreeBSD or OS X') 40 | sys.exit(1) 41 | 42 | self.nb_descriptors = 40 # some to be safe ... 43 | self.nb_descriptors += configuration.http.connections*2 # one socket for client and server connection 44 | self.nb_descriptors += configuration.web.connections # one socket per web client connection 45 | self.nb_descriptors += configuration.redirector.maximum*2 # one socket per pipe to the thread and one for the forked process 46 | self.nb_descriptors += configuration.dns.retries*10 # some sockets for the DNS 47 | 48 | if configuration.daemon.reactor == 'select': 49 | if self.nb_descriptors > 1024: 50 | self.log.critical('the select reactor is not very scalable, and can only handle 1024 simultaneous descriptors') 51 | self.log.critical('your configuration requires %d file descriptors' % self.nb_descriptors) 52 | self.log.critical('please increase your system maximum limit, alternatively you can reduce') 53 | self.log.critical('exaproxy.http.connections, exaproxy.web.connections and/or configuration.redirector.maximum') 54 | self.log.critical('shutting down, we can not safely run with these settings') 55 | sys.exit(1) 56 | 57 | soft,hard = resource.getrlimit(resource.RLIMIT_NOFILE) 58 | 59 | if soft < self.nb_descriptors: 60 | try: 61 | self.log.warning('not enough file descriptor available, increasing the limit from %d to %d' % (soft,self.nb_descriptors)) 62 | soft_limit,hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) 63 | wanted_limit = min(self.nb_descriptors, hard_limit if hard_limit > 0 else self.nb_descriptors) 64 | # default on mac are (256,-1) 65 | resource.setrlimit(resource.RLIMIT_NOFILE, (wanted_limit, hard_limit)) 66 | 67 | except (resource.error,ValueError),e: 68 | self.log.warning('problem when trying to increase resource limit : %s' % str(e)) 69 | 70 | soft,hard = resource.getrlimit(resource.RLIMIT_NOFILE) 71 | if soft < self.nb_descriptors: 72 | self.log.critical('could not increase file descriptor limit to %d, limit is still %d' % (self.nb_descriptors,signed(soft))) 73 | self.log.critical('on Linux you may want to increase the value in /proc/sys/fs/file-max') 74 | self.log.critical('please increase your system maximum limit, alternatively you can reduce') 75 | self.log.critical('exaproxy.http.connections, exaproxy.web.connections and/or configuration.redirector.maximum') 76 | return 77 | 78 | # on linux : 79 | # need to get page size, and box memory, read /proc/sys/net/ipv4/tcp_mem and make sure the values are same. 80 | # look at /proc/sys/net/ipv4/tcp_max_orphans and make sure the value is high enough for the number of FD 81 | # > free -m 82 | # > getconf PAGESIZE 83 | # we should monitor and graph some of the values of /proc/net/sockstat and add it to our web page 84 | # like TCP inuse, orphan, etc. 85 | 86 | 87 | self.log.info('for information, your configuration requires %d available file descriptors' % self.nb_descriptors) 88 | self.filemax = self.nb_descriptors 89 | 90 | def drop_privileges (self): 91 | """returns true if we are left with insecure privileges""" 92 | # os.name can be ['posix', 'nt', 'os2', 'ce', 'java', 'riscos'] 93 | if os.name not in ['posix',]: 94 | return False 95 | 96 | uid = os.getuid() 97 | gid = os.getgid() 98 | 99 | if uid and gid: 100 | return False 101 | 102 | try: 103 | user = pwd.getpwnam(self.user) 104 | nuid = int(user.pw_uid) 105 | ngid = int(user.pw_uid) 106 | except KeyError: 107 | return True 108 | 109 | # not sure you can change your gid if you do not have a pid of zero 110 | try: 111 | if not gid: 112 | os.setgid(ngid) 113 | if not uid: 114 | os.setuid(nuid) 115 | return False 116 | except OSError: 117 | return True 118 | 119 | def _is_socket (self,fd): 120 | try: 121 | s = socket.fromfd(fd, socket.AF_INET, socket.SOCK_RAW) 122 | except ValueError: 123 | # The file descriptor is closed 124 | return False 125 | try: 126 | s.getsockopt(socket.SOL_SOCKET, socket.SO_TYPE) 127 | except socket.error, e: 128 | # It is look like one but it is not a socket ... 129 | if e.args[0] == errno.ENOTSOCK: 130 | return False 131 | return True 132 | 133 | def daemonise (self): 134 | if not self.daemonize: 135 | return 136 | 137 | def fork_exit (): 138 | try: 139 | pid = os.fork() 140 | if pid > 0: 141 | os._exit(0) 142 | except OSError, e: 143 | self.log.debug('Can not fork, errno %d : %s' % (e.errno,e.strerror)) 144 | 145 | # do not detach if we are already supervised or run by init like process 146 | if self._is_socket(sys.__stdin__.fileno()) or os.getppid() == 1: 147 | return 148 | 149 | fork_exit() 150 | os.setsid() 151 | fork_exit() 152 | self.silence() 153 | 154 | def silence (self): 155 | maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1] 156 | if maxfd == resource.RLIM_INFINITY: 157 | maxfd = 1024 158 | 159 | for fd in range(0, maxfd): 160 | try: 161 | os.close(fd) 162 | except OSError: 163 | pass 164 | 165 | os.open("/dev/null", os.O_RDWR) 166 | os.dup2(0, 1) 167 | os.dup2(0, 2) 168 | -------------------------------------------------------------------------------- /lib/exaproxy/util/debug.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | debug.py 4 | 5 | Created by Thomas Mangin on 2011-11-29. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | import os 10 | import sys 11 | 12 | import traceback 13 | 14 | from exaproxy.util.log.history import History 15 | 16 | 17 | def bug_report (exc_type, value, trace): 18 | print >> sys.stderr, '' 19 | print >> sys.stderr, '' 20 | print >> sys.stderr, '-'*80 21 | print >> sys.stderr, '-- Please provide the information below on :' 22 | print >> sys.stderr, '-- http://code.google.com/p/exaproxy/issues/entry' 23 | print >> sys.stderr, '-'*80 24 | print >> sys.stderr, '' 25 | print >> sys.stderr, '' 26 | print >> sys.stderr, '-- Version' 27 | print >> sys.stderr, '' 28 | print >> sys.stderr, sys.version 29 | print >> sys.stderr, '' 30 | print >> sys.stderr, '' 31 | print >> sys.stderr, '-- Logging History' 32 | print >> sys.stderr, '' 33 | print >> sys.stderr, '' 34 | for line in History().formated(): 35 | print >> sys.stderr, line 36 | print >> sys.stderr, '' 37 | print >> sys.stderr, '' 38 | print >> sys.stderr, '-- Traceback' 39 | print >> sys.stderr, '' 40 | print >> sys.stderr, '' 41 | traceback.print_exception(exc_type,value,trace) 42 | print >> sys.stderr, '' 43 | print >> sys.stderr, '' 44 | print >> sys.stderr, '-'*80 45 | print >> sys.stderr, '-- Please provide the information above on :' 46 | print >> sys.stderr, '-- http://code.google.com/p/exaproxy/issues/entry' 47 | print >> sys.stderr, '-'*80 48 | print >> sys.stderr, '' 49 | print >> sys.stderr, '' 50 | 51 | #print >> sys.stderr, 'the program failed with message :', value 52 | 53 | def intercept (exc_type, value, trace): 54 | interactive = os.environ.get('PDB',None) 55 | 56 | if interactive in ['0','']: 57 | # PDB was set to 0 or '' which is undocumented, and we do nothing 58 | pass 59 | else: 60 | bug_report(exc_type, value, trace) 61 | if interactive == 'true': 62 | import pdb 63 | pdb.pm() 64 | 65 | sys.excepthook = intercept 66 | 67 | if sys.argv: 68 | del sys.argv[0] 69 | __file__ = os.path.abspath(sys.argv[0]) 70 | __name__ = '__main__' 71 | execfile(sys.argv[0]) 72 | -------------------------------------------------------------------------------- /lib/exaproxy/util/log/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Exa-Networks/exaproxy/8b7291b79c1cd6542213a5e7d8dda3cf5a676166/lib/exaproxy/util/log/__init__.py -------------------------------------------------------------------------------- /lib/exaproxy/util/log/history.py: -------------------------------------------------------------------------------- 1 | 2 | import time 3 | import logging 4 | from collections import deque 5 | 6 | class Level: 7 | class value: 8 | DEBUG = logging.DEBUG 9 | INFO = logging.INFO 10 | WARNING = logging.WARNING 11 | ERROR = logging.ERROR 12 | CRITICAL = logging.CRITICAL 13 | 14 | string = { 15 | value.DEBUG : 'debug', 16 | value.INFO : 'info', 17 | value.WARNING : 'warning', 18 | value.ERROR : 'error', 19 | value.CRITICAL : 'critical', 20 | } 21 | 22 | @staticmethod 23 | def name (level): 24 | return Level.string.get(level, 'UNKNOWN') 25 | 26 | 27 | class _History: 28 | _log = None 29 | _err = None 30 | 31 | def __init__ (self, size): 32 | self.size = size 33 | self.messages = deque() 34 | 35 | def record (self, timestamp, name, level, text): 36 | message = timestamp, name, level, text 37 | self.messages.append(message) 38 | if len(self.messages) > self.size: 39 | self.messages.popleft() 40 | 41 | def snapshot (self): 42 | return list(self.messages) 43 | 44 | def formated (self): 45 | for timestamp, name, level, text in self.snapshot(): 46 | date = time.strftime('%a, %d %b %Y %H:%M:%S',timestamp) 47 | yield '%s %s %-13s %s' % (date, name, Level.name(level), text) 48 | 49 | def History (size=1000): 50 | if not _History._log: 51 | _History._log = _History(size) 52 | return _History._log 53 | 54 | def Errors (size=1000): 55 | if not _History._err: 56 | _History._err = _History(size) 57 | return _History._err 58 | -------------------------------------------------------------------------------- /lib/exaproxy/util/log/logger.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | log.py 4 | 5 | Created by Thomas Mangin on 2011-11-29. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | import time 10 | import logging 11 | 12 | from .message import message_store 13 | from .message import usage_store 14 | from .history import History 15 | 16 | 17 | class Logger: 18 | mailbox = message_store 19 | history = History() 20 | 21 | def __init__ (self, name, active=True, loglevel=logging.DEBUG): 22 | self.name = str(name) 23 | self.active = active 24 | self.loglevel = loglevel 25 | 26 | def log (self, text, loglevel): 27 | now = time.localtime() 28 | self.history.record(now, self.name, loglevel, text) 29 | 30 | if self.active is True and loglevel >= self.loglevel: 31 | self.mailbox.addMessage((self.name, loglevel, now, text)) 32 | res = True 33 | else: 34 | res = None 35 | 36 | return res 37 | 38 | def stdout (self, message): 39 | print message 40 | 41 | def debug (self, message): 42 | self.log(message, logging.DEBUG) 43 | 44 | def info (self, message): 45 | self.log(message, logging.INFO) 46 | 47 | def notice (self, message): 48 | self.log(message, logging.INFO) 49 | 50 | def warning (self, message): 51 | self.log(message, logging.WARNING) 52 | 53 | def error (self, message): 54 | self.log(message, logging.ERROR) 55 | 56 | def critical (self, message): 57 | self.log(message, logging.CRITICAL) 58 | 59 | def alert (self, message): 60 | self.log(message, logging.WARNING) 61 | 62 | def emergency (self, message): 63 | self.log(message, logging.CRITICAL) 64 | 65 | 66 | class UsageLogger (Logger): 67 | mailbox = usage_store 68 | 69 | def logRequest (self, client_id, client_port, accept_ip, client_ip, command, url, status, destination): 70 | if self.active: 71 | now = time.time() 72 | line = '%s %.02f %s %s %s %s %s %s/%s' % ( 73 | time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(now)), 74 | now, client_id, accept_ip, client_ip, command, url, status, destination 75 | ) 76 | 77 | res = self.log(line, logging.INFO) 78 | else: 79 | res = None 80 | 81 | return res 82 | -------------------------------------------------------------------------------- /lib/exaproxy/util/log/message.py: -------------------------------------------------------------------------------- 1 | class MessageStore: 2 | def __init__ (self, name): 3 | self.name = name 4 | self.queue = [] 5 | 6 | def addMessage (self, message): 7 | return self.queue.append(message) 8 | 9 | def readMessages (self): 10 | self.queue, messages = [], self.queue 11 | return messages 12 | 13 | 14 | message_store = MessageStore('messages') 15 | usage_store = MessageStore('usage') 16 | -------------------------------------------------------------------------------- /lib/exaproxy/util/log/writer.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | log.py 4 | 5 | Created by Thomas Mangin on 2011-11-29. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | import os 10 | import sys 11 | import time 12 | import logging 13 | import logging.handlers 14 | 15 | from .history import History,Level 16 | from .message import message_store 17 | from .message import usage_store 18 | 19 | class RecordedLog (object): 20 | history = History() 21 | 22 | 23 | class LogWriter (RecordedLog): 24 | gidentifier = ['ExaProxy'] 25 | mailbox = None 26 | debug_level = Level.value.DEBUG 27 | 28 | level = None 29 | active = None 30 | backup = None 31 | 32 | def writeMessages (self): 33 | messages = self.mailbox.readMessages() if self.mailbox is not None else [] 34 | messages = ((n,l,t,m) for (n,l,t,m) in messages if l >= self.level) if self.active else [] 35 | for name, level, timestamp, message in messages: 36 | text = self.formatMessage(name, level, timestamp, message) 37 | self.writeMessage(level, text) 38 | 39 | self.finishWriting() 40 | 41 | def formatMessage (self, name, level, timestamp, message): 42 | return message 43 | 44 | def writeMessage (self, level, message): 45 | raise NotImplementedError 46 | 47 | def finishWriting (self): 48 | pass 49 | 50 | def setIdentifier (self, identifier): 51 | self.gidentifier.append(identifier) 52 | self.gidentifier.pop(0) 53 | 54 | def getIdentifier (self): 55 | return self.gidentifier[0] 56 | 57 | def toggleDebug (self): 58 | if self.backup is not None: 59 | (active, level) = self.backup 60 | self.backup = None 61 | self.active = active 62 | self.level = level 63 | else: 64 | self.backup = (self.active, self.level) 65 | self.active = True 66 | self.level = self.debug_level 67 | 68 | 69 | class DebugLogWriter(LogWriter): 70 | mailbox = message_store 71 | 72 | def __init__ (self, active=True, fd=sys.stdout, level=Level.value.WARNING): 73 | self.pid = os.getpid() 74 | self.active = active 75 | self.level = level 76 | self.fd = fd 77 | 78 | def formatMessage (self, name, level, timestamp, message): 79 | identifier = self.getIdentifier() 80 | loglevel = Level.name(level) 81 | date_string = time.strftime('%a, %d %b %Y %H:%M:%S',timestamp) 82 | template = '%s %s %-6d %-10s %-13s %%s' % (date_string, identifier, self.pid, loglevel, name) 83 | 84 | return '\n'.join(template % line for line in message.split('\n')) 85 | 86 | def writeMessage (self, level, message): 87 | self.fd.write('%s\n' % message) 88 | 89 | def finishWriting (self): 90 | self.fd.flush() 91 | 92 | 93 | class SysLogWriter(LogWriter): 94 | mailbox = message_store 95 | 96 | # changing the default level from level=logging.INFO to anything less verbose 97 | # is likely to break UDP sysloging which sends its message a INFO level 98 | def __init__ (self, name, destination, active=True, level=Level.value.INFO): 99 | self.backup = None 100 | self.pid = os.getpid() 101 | self.active = active 102 | self.level = level 103 | 104 | _syslog = logging.getLogger(name) 105 | for handler in _syslog.handlers: 106 | _syslog.removeHandler(handler) 107 | _handler = self.getHandler(destination) 108 | _syslog.addHandler(_handler) 109 | _syslog.setLevel(level) 110 | 111 | self._syslog = _syslog 112 | 113 | def formatMessage (self, name, level, timestamp, message): 114 | identifier = self.getIdentifier() 115 | return '%s %-6d %-13s %s' % (identifier, self.pid, name, message) 116 | 117 | def writeMessage (self, level, message): 118 | self._syslog.log(level, message) 119 | 120 | def getHandler (self, destination): 121 | if destination in ('stdout', 'stderr'): 122 | handler = logging.StreamHandler() 123 | 124 | elif destination == '': 125 | if sys.platform == 'darwin': 126 | address = '/var/run/syslog' 127 | else: 128 | address = '/dev/log' 129 | 130 | if not os.path.exists(address): 131 | address = 'localhost', 514 132 | 133 | handler = logging.handlers.SysLogHandler(address) 134 | 135 | elif destination.lower().startswith('host:'): 136 | if ':' in destination[5:]: 137 | _host,_port = destination.strip().split(':',1) 138 | host = _host.rstrip() 139 | _port = _port.strip() 140 | if _port.isdigit(): 141 | address = host, int(_port) 142 | else: 143 | address = destination[5:].strip(), 514 144 | handler = logging.handlers.SysLogHandler(address) 145 | 146 | else: 147 | handler = logging.handlers.RotatingFileHandler(destination, maxBytes=5*1024*1024, backupCount=5) 148 | 149 | return handler 150 | 151 | 152 | class UsageWriter(SysLogWriter): 153 | mailbox = usage_store 154 | -------------------------------------------------------------------------------- /lib/exaproxy/util/messagebox.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | 4 | from exaproxy.network.errno_list import errno_block 5 | 6 | class MessageReader: 7 | delimiter = ':' 8 | eom = ',' 9 | 10 | def read (self, pipe_in): 11 | # NOTE: we may block here if badly formatted data is sent 12 | 13 | while True: 14 | try: 15 | message_s = pipe_in.read(3) 16 | break 17 | except IOError: 18 | pass 19 | 20 | while message_s.isdigit(): 21 | try: 22 | message_s += pipe_in.read(1) 23 | except IOError: 24 | pass 25 | 26 | try: 27 | if self.delimiter in message_s: 28 | pickled_size, pickled = message_s.split(self.delimiter, 1) 29 | 30 | else: 31 | pickled_size, pickled = None, None 32 | 33 | if pickled_size is not None and pickled_size.isdigit(): 34 | pickled_len = int(pickled_size) 35 | 36 | else: 37 | pickled_len, pickled = None, None 38 | 39 | if pickled_len is not None: 40 | remaining = pickled_len + 1 - len(pickled) 41 | 42 | else: 43 | remaining = 0 44 | 45 | while remaining > 0: 46 | data = pipe_in.read(remaining) 47 | remaining = remaining - len(data) 48 | pickled += data 49 | 50 | except (ValueError, TypeError): # I/O operation on closed file 51 | pickled = None 52 | 53 | if pickled is not None and pickled.endswith(self.eom): 54 | pickled = pickled[:-1] 55 | 56 | else: 57 | pickled = None 58 | 59 | return pickled 60 | 61 | def get (self, pipe_in): 62 | pickled = self.read(pipe_in) 63 | 64 | if pickled is not None: 65 | try: 66 | message = pickle.loads(pickled) 67 | except (TypeError, IndexError, ValueError, EOFError): 68 | message = None 69 | 70 | else: 71 | message = None 72 | 73 | return message 74 | 75 | 76 | class MessageBox (MessageReader): 77 | def __init__ (self, pipe_in, pipe_out): 78 | self.pipe_in = os.fdopen(pipe_in, 'r', 0) 79 | self.pipe_out = os.fdopen(pipe_out, 'w', 0) 80 | 81 | def close (self): 82 | if self.pipe_in is not None: 83 | try: 84 | self.pipe_in.close() 85 | except IOError: 86 | pass 87 | 88 | if self.pipe_out is not None: 89 | try: 90 | self.pipe_out.close() 91 | except IOError: 92 | pass 93 | 94 | def put (self, message): 95 | pickled = pickle.dumps(message) 96 | message_s = str(len(pickled)) + self.delimiter + str(pickled) + self.eom 97 | 98 | while True: 99 | try: 100 | return self.pipe_out.write(message_s) 101 | except IOError, e: 102 | if e.errno not in errno_block: 103 | raise e 104 | 105 | def get (self): 106 | while True: 107 | try: 108 | pickled = self.read(self.pipe_in) 109 | break 110 | except IOError, e: 111 | if e.errno not in errno_block: 112 | raise e 113 | 114 | if pickled is not None: 115 | try: 116 | message = pickle.loads(pickled) 117 | except (TypeError, IndexError, ValueError, EOFError): 118 | message = None 119 | 120 | else: 121 | message = None 122 | 123 | return message 124 | 125 | -------------------------------------------------------------------------------- /lib/exaproxy/util/messagequeue.py: -------------------------------------------------------------------------------- 1 | from collections import deque 2 | import time 3 | 4 | class Empty (Exception): 5 | pass 6 | 7 | class Queue(): 8 | def __init__ (self): 9 | self.queue = deque() 10 | 11 | def qsize (self): 12 | return len(self.queue) 13 | 14 | def isempty (self): 15 | return len(self.queue) == 0 16 | 17 | def put (self, message): 18 | self.queue.append(message) 19 | 20 | def get (self, timeout=None): 21 | try: 22 | if self.queue: 23 | return self.queue.popleft() 24 | except IndexError: 25 | pass 26 | 27 | delay = 0.0005 28 | start = time.time() 29 | 30 | running = True 31 | 32 | while running: 33 | try: 34 | while True: 35 | if timeout: 36 | if time.time() > start + timeout: 37 | running = False 38 | break 39 | 40 | delay = min(0.05, 2*delay) 41 | time.sleep(delay) 42 | 43 | if self.queue: 44 | return self.queue.popleft() 45 | except IndexError: 46 | pass 47 | 48 | raise Empty 49 | 50 | 51 | if __name__ == '__main__': 52 | q = Queue() 53 | q.put('foo') 54 | q.put('bar') 55 | print q.get(1) 56 | print q.get(1) 57 | try: 58 | q.put('yay') 59 | print q.get(1) 60 | print q.get(2) 61 | except: 62 | print 'forever - print ^C' 63 | print q.get() 64 | -------------------------------------------------------------------------------- /lib/exaproxy/util/pid.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | """ 3 | pid.py 4 | 5 | Created by Thomas Mangin on 2011-11-29. 6 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 7 | """ 8 | 9 | # encoding: utf-8 10 | """ 11 | daemon.py 12 | 13 | Created by Thomas Mangin on 2011-11-29. 14 | Copyright (c) 2011-2013 Exa Networks. All rights reserved. 15 | """ 16 | 17 | import os 18 | import errno 19 | 20 | from .log.logger import Logger 21 | 22 | class PID (object): 23 | def __init__ (self, configuration): 24 | self.log = Logger('daemon', configuration.log.daemon) 25 | self.pid_file = configuration.daemon.pidfile 26 | self._saved_pid = False 27 | #mask = os.umask(0137) 28 | 29 | def save (self): 30 | self._saved_pid = False 31 | 32 | if not self.pid_file: 33 | return 34 | 35 | try: 36 | with open(self.pid_file,"r") as f: 37 | running_pid = int(f.read() or 0) 38 | if running_pid <= 0: 39 | return 40 | except IOError, e: 41 | # No such file or directory 42 | if e[0] == errno.ENOENT: 43 | pass 44 | if e[0] in (errno.EPERM,errno.EACCES): 45 | self.log.warning("PIDfile already exists, not updated %s" % self.pid_file) 46 | return 47 | raise 48 | except ValueError: 49 | # Non numeric data in PID file 50 | pass 51 | 52 | try: 53 | os.kill(running_pid, 0) 54 | except OSError, e: 55 | # No such processs 56 | if e[0] != errno.ESRCH: 57 | self.log.warning("PID %s is still running" % self.pid_file) 58 | return 59 | 60 | ownid = os.getpid() 61 | 62 | flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY 63 | mode = ((os.R_OK | os.W_OK) << 6) | (os.R_OK << 3) | os.R_OK 64 | 65 | try: 66 | fd = os.open(self.pid_file,flags,mode) 67 | except OSError: 68 | self.log.warning("PIDfile already exists, not updated %s" % self.pid_file) 69 | return 70 | 71 | try: 72 | f = os.fdopen(fd,'w') 73 | line = "%d\n" % ownid 74 | f.write(line) 75 | f.close() 76 | self._saved_pid = True 77 | except IOError: 78 | self.log.warning("Can not create PIDfile %s" % self.pid_file) 79 | return 80 | self.log.info("Created PIDfile %s with value %d" % (self.pid_file,ownid)) 81 | 82 | def remove (self): 83 | if not self.pid_file or not self._saved_pid: 84 | return 85 | try: 86 | os.remove(self.pid_file) 87 | except OSError, e: 88 | if e.errno == errno.ENOENT: 89 | pass 90 | else: 91 | self.log.warning("Can not remove PIDfile %s" % self.pid_file) 92 | return 93 | self.log.info("Removed PIDfile %s" % self.pid_file) 94 | -------------------------------------------------------------------------------- /lib/exaproxy/util/proxy.py: -------------------------------------------------------------------------------- 1 | import socket 2 | 3 | def validate_ip4 (address): 4 | try: 5 | socket.inet_aton(address) 6 | ip4_address = address 7 | except (socket.error, TypeError): 8 | ip4_address = None 9 | 10 | return ip4_address 11 | 12 | def validate_ip6 (address): 13 | try: 14 | socket.inet_pton(socket.AF_INET6, address) 15 | ip6_address = address 16 | except (socket.error, TypeError): 17 | ip6_address = None 18 | 19 | return ip6_address 20 | 21 | def invalidate (address): 22 | return None 23 | 24 | 25 | class ProxyProtocol: 26 | ip_validators = { 27 | 'TCP4' : validate_ip4, 28 | 'TCP6' : validate_ip6, 29 | 'UNKNOWN' : invalidate 30 | } 31 | 32 | def parse (self, header): 33 | if '\r\n' in header: 34 | proxy_line, request = header.split('\r\n', 1) 35 | else: 36 | proxy_line, request = '', None 37 | 38 | 39 | try: 40 | magic, fproto, source, destination, sport, dport = proxy_line.split(' ') 41 | except ValueError: 42 | proxy_line, request = '', None 43 | magic, fproto, source, destination, sport, dport = None, None, None, None, None, None 44 | 45 | if magic != 'PROXY': 46 | # We don't care about parsing the source or destination ports 47 | request = None 48 | source, destination = None, None 49 | 50 | validate = self.ip_validators.get(fproto, invalidate) 51 | source_addr = validate(source) 52 | dest_addr = validate(destination) # pylint: disable=W0612 53 | 54 | return source_addr, request 55 | -------------------------------------------------------------------------------- /sbin/exaproxy: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | dirname=`dirname $0` 4 | 5 | case $dirname in 6 | /*) 7 | path=$dirname/.. 8 | ;; 9 | *) 10 | cd `pwd`/$dirname/.. > /dev/null 11 | path=`pwd` 12 | cd - > /dev/null 13 | ;; 14 | esac 15 | 16 | export PYTHONPATH=$path/lib 17 | 18 | if [ "$INTERPRETER" != "" ] 19 | then 20 | INTERPRETER=`which $INTERPRETER` 21 | fi 22 | 23 | PYPY=`which pypy 2> /dev/null` 24 | PYTHON27=`which python2.7 2> /dev/null` 25 | PYTHON26=`which python2.6 2> /dev/null` 26 | PYTHON25=`which python2.5 2> /dev/null` 27 | PYTHON24=`which python2.4 2> /dev/null` 28 | PYTHON2=`which python2 2> /dev/null` 29 | PYTHON=`which python 2> /dev/null` 30 | 31 | if [ -f "$INTERPRETER" ] 32 | then 33 | DUMMY='need this for the then' 34 | elif [ -f "$PYPY" ] 35 | then 36 | INTERPRETER=$PYPY 37 | elif [ -f "$PYTHON27" ] 38 | then 39 | INTERPRETER=$PYTHON27 40 | elif [ -f "$PYTHON26" ] 41 | then 42 | INTERPRETER=$PYTHON26 43 | elif [ -f "$PYTHON25" ] 44 | then 45 | INTERPRETER=$PYTHON25 46 | elif [ -f "$PYTHON24" ] 47 | then 48 | INTERPRETER=$PYTHON24 49 | elif [ -f "$PYTHON2" ] 50 | then 51 | INTERPRETER=$PYTHON2 52 | elif [ -f "$PYTHON" ] 53 | then 54 | INTERPRETER=$PYTHON 55 | else 56 | INTERPRETER=python 57 | fi 58 | 59 | APPLICATIONS=`$INTERPRETER -c "import sys,os; print ' '.join(os.path.join(_,'exaproxy','application.py') for _ in sys.path if os.path.isfile('/'.join((_,'exaproxy/application.py'))))"` 60 | APPLICATION=`echo $APPLICATIONS | awk '{ print $1; }'` 61 | 62 | # 3x (2^16) = 196608 (should be enough for everyone as otherwise we will run of TCP port for outgoing connections anyway) 63 | ulimit -n 196608 > /dev/null 2> /dev/null 64 | 65 | exec $INTERPRETER -m exaproxy.util.debug $APPLICATION $* 66 | -------------------------------------------------------------------------------- /sbin/icap-connector: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import sys 5 | import socket 6 | 7 | 8 | def log(prefix, string): 9 | location = os.environ.get('LOG',None) 10 | if prefix: 11 | with open(location + ('-%s' % str(os.getpid())), 'a+') as fd: 12 | fd.write('%-6s %s\n' % (prefix,string.replace('\r','\\r').replace('\n','\\n'))) 13 | 14 | 15 | class NET (object): 16 | def __init__ (self,host,port): 17 | self.host = host 18 | self.port = port 19 | self.socket = None 20 | self.buffered = '' 21 | 22 | def connect (self): 23 | try: 24 | self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 25 | self.socket.connect((self.host,self.port)) 26 | return True 27 | except KeyboardInterrupt: 28 | sys.exit(0) 29 | except (socket.gaierror,socket.herror): 30 | return False 31 | 32 | def read (self,size=1,end=None): 33 | buf_data = self.buffered 34 | 35 | if end is not None: 36 | if end in buf_data: 37 | returned,buf_data = buf_data.split(end,1) 38 | self.buffered = buf_data 39 | return returned + end 40 | else: 41 | while end not in buf_data: 42 | extra = self.socket.recv(4016) 43 | buf_data += extra 44 | if not extra: 45 | log('NO-DATA','') 46 | sys.exit(1) 47 | self.buffered = buf_data 48 | return self.read(end=end) 49 | 50 | buf_size = len(buf_data) 51 | 52 | if size <= buf_size: 53 | buf_size -= size 54 | returned, buf_data = buf_data[:size],buf_data[size:] 55 | self.buffered = buf_data 56 | return returned 57 | 58 | read_size = size - len(buf_data) 59 | while read_size > 0: 60 | extra = self.socket.recv(read_size) 61 | read_size -= len(extra) 62 | buf_data += extra 63 | self.buffered = '' 64 | return buf_data 65 | 66 | def read_size (self): 67 | total = 0 68 | string = '' 69 | while True: 70 | byte = self.read(1) 71 | string += byte 72 | # this allows '\r\n' to be zero which is wrong but simple 73 | if byte == '\r': 74 | continue 75 | if byte == '\n': 76 | return total, string 77 | total = total * 16 78 | total = total + int(byte,16) 79 | 80 | def sendall (self,buffer,reconnect=True): 81 | if reconnect: 82 | try: 83 | return self.socket.sendall(buffer) 84 | except (IOError, socket.error, socket.timeout, ): 85 | if self.connect(): 86 | return self.socket.sendall(buffer) 87 | raise 88 | 89 | def recv (self,number): 90 | return self.socket.recv(number) 91 | 92 | def close (self): 93 | if self.socket: 94 | try: 95 | self.socket.close() 96 | except Exception: 97 | pass 98 | finally: 99 | self.sock = None 100 | 101 | def __del__ (self): 102 | self.close() 103 | 104 | 105 | def main (host, port): 106 | sock = None 107 | while True: 108 | try: 109 | data = '' 110 | 111 | net = NET(host,port) 112 | if not net.connect(): 113 | sys.exit(1) 114 | 115 | # ExaProxy has/had a bug where it will first look for \r\n 116 | # and then \n\n for the end of headers. 117 | # replacing the ICAP lines to end with \r\ is harmless 118 | # and is a good work around the issue 119 | 120 | while True: 121 | # firewalls can kill long inactive connections, 122 | # make sure to reconnect if it happens when reading the REQMOD line 123 | line = sys.stdin.readline() 124 | log('STDIN-FIRST',line) 125 | log('SEND--FIRST',line.strip() + '\r\n') 126 | net.sendall(line.strip() + '\r\n',reconnect=True) 127 | 128 | looping = 2 129 | while looping: 130 | line = sys.stdin.readline() 131 | log('STDIN-LOOPS',line) 132 | if looping == 2: 133 | if line.startswith('Pragma:'): 134 | continue 135 | log('SEND--LOOP2',line.strip() + '\r\n') 136 | net.sendall(line.strip() + '\r\n') 137 | else: 138 | log('SEND--LOOP1',line) 139 | net.sendall(line) 140 | if not line.strip(): 141 | looping -= 1 142 | 143 | log('SENT-ICAP','----') 144 | 145 | while True: 146 | recv = net.read(end='\n') 147 | log('RECV---ICAP',recv) 148 | data += recv 149 | if '\r\n\r\n' in data: 150 | break 151 | 152 | response,data = data.split('\r\n\r\n',1) 153 | 154 | log('STDOUT-ICAP',response + '\r\n\r\n') 155 | sys.stdout.write(response + '\r\n\r\n') 156 | sys.stdout.flush() 157 | 158 | # ExaProxy always put the encapsulation last, this is not a RFC behaviour 159 | encapsulation = response.split('\r\n')[-1] 160 | 161 | # Encapsulated: req-hdr=0, null-body=135 162 | # Encapsulated: res-hdr=0, res-body=226 163 | headers = dict((k.strip(),int(v)) for k,v in (_.split('=') for _ in encapsulation.split(':')[-1].split(','))) 164 | # this is wrong in theory .. in practice it works 165 | 166 | if 'req-hdr' in headers: 167 | response = net.read(headers['null-body']) 168 | # chunk encoded 169 | elif 'res-hdr' in headers: 170 | response = net.read(headers['res-body']) 171 | 172 | chunk_size = -1 173 | while True: 174 | chunk_size, chunk_data = net.read_size() 175 | log('RECV--SIZE',chunk_data) 176 | response += chunk_data 177 | chunk = net.read(chunk_size) 178 | log('RECV-CHUNK',chunk) 179 | response += chunk 180 | if chunk_size == 0: 181 | break 182 | eol = net.read(2) 183 | log('RECV---EOL',eol) 184 | response += eol 185 | else: 186 | log('FATAL', 'unknown encapsulation [%s]' % encapsulation) 187 | 188 | log('STDOUT-HTTP',response) 189 | sys.stdout.write(response) 190 | sys.stdout.flush() 191 | 192 | except KeyboardInterrupt: 193 | log('KEYBOARD','INTERRUPT') 194 | sys.exit(0) 195 | 196 | except Exception, e: 197 | net.close() 198 | log('CRASH','exaproxy surfprotect connector failed\n') 199 | log('CRASH',str(e)) 200 | sys.exit(1) 201 | 202 | 203 | if __name__ == '__main__': 204 | main( 205 | sys.argv[1], 206 | 1344 if len(sys.argv) < 3 else int(sys.argv[2]) 207 | ) 208 | -------------------------------------------------------------------------------- /scripts/exaproxy: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from exaproxy.application import main 4 | main() 5 | -------------------------------------------------------------------------------- /service/exaproxy/log/run: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | exec multilog s2500000 n20 ./main 3 | -------------------------------------------------------------------------------- /service/exaproxy/run: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | dirname=`dirname $0` 4 | 5 | case $dirname in 6 | /*) 7 | path=$dirname/../.. 8 | ;; 9 | *) 10 | cd `pwd`/$dirname/../.. > /dev/null 11 | path=`pwd` 12 | cd - > /dev/null 13 | ;; 14 | esac 15 | 16 | exec env PYTHONPATH=$path/lib $path/sbin/exaproxy 2>&1 17 | -------------------------------------------------------------------------------- /service/surfprotect/log/run: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | exec multilog s2500000 n20 ./main 3 | -------------------------------------------------------------------------------- /service/surfprotect/run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This requires your IP address to be enabled 4 | NAME=surfprotect-demonstration 5 | 6 | # This is the IP of an ExaProxy ICAP server 7 | ICAP_SERVER=icap.exa-networks.co.uk 8 | 9 | LISTEN_IP=0.0.0.0 10 | ADMIN_IP=127.0.0.1 11 | 12 | HTTP_PORT=3128 13 | ADMIN_PORT=8080 14 | 15 | # this depends on the numbers of clients using the proxy 16 | SURFPROTECT_CONNECTIONS=10 17 | 18 | # HIGH >>> EMERG, ALERT, CRIT, ERR, WARNING, NOTICE, INFO, DEBUG <<< LOW 19 | LOG_LEVEL=INFO 20 | 21 | # DO NOT EDIT BELOW THIS LINE 22 | 23 | dirname=`dirname $0` 24 | 25 | case $dirname in 26 | /*) 27 | path=$dirname/../.. 28 | ;; 29 | *) 30 | cd `pwd`/$dirname/../.. > /dev/null 31 | path=`pwd` 32 | cd - > /dev/null 33 | ;; 34 | esac 35 | 36 | PROXYPATH=$path 37 | REDIRECTOR=$path/sbin/icap-connector 38 | 39 | cd $PROXYPATH 40 | 41 | # ulimit -n 196608 42 | export PYTHONPATH=$PYTHONPATH 43 | export exaproxy_daemon_identifier=$NAME 44 | export exaproxy_daemon_user=nobody 45 | export exaproxy_log_level=$LOG_LEVEL 46 | export exaproxy_http_connections=1000 47 | export exaproxy_tcp4_host=$LISTEN_IP 48 | export exaproxy_tcp4_port=$HTTP_PORT 49 | # export exaproxy_tls_enable=false 50 | export exaproxy_web_host=$ADMIN_IP 51 | export exaproxy_web_port=$ADMIN_PORT 52 | export exaproxy_redirector_enable=true 53 | export exaproxy_redirector_minimum=$SURFPROTECT_CONNECTIONS 54 | export exaproxy_redirector_maximum=$SURFPROTECT_CONNECTIONS 55 | export exaproxy_redirector_program="$REDIRECTOR $ICAP_SERVER" 56 | export exaproxy_redirector_protocol=icap://$NAME 57 | export exaproxy_usage_enable=false 58 | 59 | exec \ 60 | $PROXYPATH/sbin/exaproxy 2>&1 61 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | setup.py 5 | 6 | Created by Thomas Mangin on 2011-01-24. 7 | Copyright (c) 2011-2012 Exa Networks. All rights reserved. 8 | """ 9 | 10 | import os 11 | import sys 12 | from distutils.core import setup 13 | from distutils.util import get_platform 14 | from setuptools.command.install import install 15 | from setuptools.command.easy_install import chmod 16 | 17 | try: 18 | version = None 19 | root = os.path.dirname(os.path.join(os.getcwd(),sys.argv[0])) 20 | with open(os.path.join(root,'lib/exaproxy/application.py'),'r') as stream: 21 | for line in stream: 22 | if line.startswith("version = '"): 23 | _,version,_ = line.split("'") 24 | break 25 | if version is None: 26 | raise Exception() 27 | if False in [ _.isdigit() for _ in version.split('.')]: 28 | raise Exception() 29 | print 'exaproxy version', version 30 | except Exception,e: 31 | print "can not extract exaproxy version" 32 | sys.exit(1) 33 | 34 | def packages (lib): 35 | def dirs (*path): 36 | for location,_,_ in os.walk(os.path.join(*path)): 37 | yield location 38 | def modules (lib): 39 | return os.walk(lib).next()[1] 40 | r = [] 41 | for module in modules(lib): 42 | for d in dirs(lib,module): 43 | r.append(d.replace('/','.').replace('\\','.')[len(lib)+1:]) 44 | return r 45 | 46 | def configuration (etc): 47 | etcs = [] 48 | for l,d,fs in os.walk(etc): 49 | e = [] 50 | for f in fs: 51 | e.append(os.path.join(l,f)) 52 | etcs.append((l,e)) 53 | return etcs 54 | 55 | def files (): 56 | r = [] 57 | if 'linux' in sys.platform: 58 | r.append(('/usr/lib/systemd/system',['etc/systemd/exaproxy.service',])) 59 | r.extend(configuration('etc/exaproxy')) 60 | return r 61 | 62 | class custom_install (install): 63 | def run (self): 64 | install.run(self) 65 | 66 | if not self.dry_run: 67 | for names in ( names for section,names in files() if section == 'etc/exaproxy/redirector'): 68 | for name in names: 69 | location = os.path.join(self.install_data,name) 70 | chmod(location, 0755) # the 0 make the value octal 71 | install.run(self) 72 | 73 | 74 | 75 | setup(name='exaproxy', 76 | version=version, 77 | description='non-caching http/https proxy', 78 | long_description="ExaProxy is a forward (or reverse) non-caching proxy. It can be used transparently and/or to rewrite HTTP requests.", 79 | author='Thomas Mangin, David Farrar', 80 | author_email='thomas.mangin@exa-networks.co.uk, david.farrar@exa-networks.co.uk', 81 | url='https://github.com/Exa-Networks/exaproxy', 82 | license="BSD", 83 | platforms=[get_platform(),], 84 | package_dir = {'': 'lib'}, 85 | packages=packages('lib'), 86 | scripts=['scripts/exaproxy',], 87 | # entry_points = {'console_scripts': ['exaproxy = exaproxy.application:main',]}, 88 | download_url='https://codeload.github.com/Exa-Networks/exaproxy/tar.gz/%s' % version, 89 | data_files=files(), 90 | classifiers=[ 91 | 'Development Status :: 4 - Beta', 92 | 'Environment :: Console', 93 | 'Intended Audience :: System Administrators', 94 | 'License :: OSI Approved :: BSD License', 95 | 'Operating System :: POSIX', 96 | 'Operating System :: MacOS :: MacOS X', 97 | 'Programming Language :: Python', 98 | 'Topic :: Internet :: WWW/HTTP', 99 | ], 100 | cmdclass={'install': custom_install}, 101 | ) 102 | 103 | --------------------------------------------------------------------------------