├── .gitattributes ├── callback.py ├── docs ├── Automatic Protocol Format Reverse Engineering.pdf ├── Finding contributors to Great Firewall by their papers.pdf ├── doc_to_download.txt ├── 一种接入网的流量识别方法.pdf ├── 基于SVM的网络流量监测.pdf ├── 基于会话的应用特征自适应提取.pdf ├── 基于加权累积和检验的加密流量盲识别算法.pdf ├── 基于字节熵矢量加权指纹的二进制协议识别.pdf └── 网络流量分类研究进展与展望.pdf ├── local.py ├── mqttclient.py ├── server.py ├── server.py.lprof ├── socks5client.py ├── socks5proxy.py ├── socks5server.py └── test.py /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | 7 | # Standard to msysgit 8 | *.doc diff=astextplain 9 | *.DOC diff=astextplain 10 | *.docx diff=astextplain 11 | *.DOCX diff=astextplain 12 | *.dot diff=astextplain 13 | *.DOT diff=astextplain 14 | *.pdf diff=astextplain 15 | *.PDF diff=astextplain 16 | *.rtf diff=astextplain 17 | *.RTF diff=astextplain 18 | -------------------------------------------------------------------------------- /callback.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module provides some helper functions to allow straightforward subscribing 3 | to topics and retrieving messages. The two functions are simple(), which 4 | returns one or messages matching a set of topics, and callback() which allows 5 | you to pass a callback for processing of messages. 6 | """ 7 | 8 | import paho.mqtt.client as paho 9 | import paho.mqtt as mqtt 10 | import ssl 11 | 12 | 13 | def _on_connect(c, userdata, flags, rc): 14 | """Internal callback""" 15 | if rc != 0: 16 | raise mqtt.MQTTException(paho.connack_string(rc)) 17 | 18 | if type(userdata['topics']) is list: 19 | for t in userdata['topics']: 20 | c.subscribe(t, userdata['qos']) 21 | else: 22 | c.subscribe(userdata['topics'], userdata['qos']) 23 | 24 | 25 | def _on_message_callback(c, userdata, message): 26 | """Internal callback""" 27 | userdata['callback'](c, userdata['userdata'], message) 28 | 29 | 30 | def _on_message_simple(c, userdata, message): 31 | """Internal callback""" 32 | 33 | if userdata['msg_count'] == 0: 34 | return 35 | 36 | # Don't process stale retained messages if 'retained' was false 37 | if userdata['retained'] == False and message.retain == True: 38 | return 39 | 40 | userdata['msg_count'] = userdata['msg_count'] - 1 41 | 42 | if userdata['messages'] is None and userdata['msg_count'] == 0: 43 | userdata['messages'] = message 44 | c.disconnect() 45 | return 46 | 47 | userdata['messages'].append(message) 48 | if userdata['msg_count'] == 0: 49 | c.disconnect() 50 | 51 | 52 | def callback(callback, topics, qos=0, userdata=None, hostname="localhost", 53 | port=1883, client_id="", keepalive=60, will=None, auth=None, tls=None, 54 | protocol=paho.MQTTv311, transport="tcp"): 55 | """Subscribe to a list of topics and process them in a callback function. 56 | This function creates an MQTT client, connects to a broker and subscribes 57 | to a list of topics. Incoming messages are processed by the user provided 58 | callback. This is a blocking function and will never return. 59 | callback : function of the form "on_message(client, userdata, message)" for 60 | processing the messages received. 61 | topics : either a string containing a single topic to subscribe to, or a 62 | list of topics to subscribe to. 63 | qos : the qos to use when subscribing. This is applied to all topics. 64 | userdata : passed to the callback 65 | hostname : a string containing the address of the broker to connect to. 66 | Defaults to localhost. 67 | port : the port to connect to the broker on. Defaults to 1883. 68 | client_id : the MQTT client id to use. If "" or None, the Paho library will 69 | generate a client id automatically. 70 | keepalive : the keepalive timeout value for the client. Defaults to 60 71 | seconds. 72 | will : a dict containing will parameters for the client: will = {'topic': 73 | "", 'payload':", 'qos':, 'retain':}. 74 | Topic is required, all other parameters are optional and will 75 | default to None, 0 and False respectively. 76 | Defaults to None, which indicates no will should be used. 77 | auth : a dict containing authentication parameters for the client: 78 | auth = {'username':"", 'password':""} 79 | Username is required, password is optional and will default to None 80 | if not provided. 81 | Defaults to None, which indicates no authentication is to be used. 82 | tls : a dict containing TLS configuration parameters for the client: 83 | dict = {'ca_certs':"", 'certfile':"", 84 | 'keyfile':"", 'tls_version':"", 85 | 'ciphers':"} 86 | ca_certs is required, all other parameters are optional and will 87 | default to None if not provided, which results in the client using 88 | the default behaviour - see the paho.mqtt.client documentation. 89 | Defaults to None, which indicates that TLS should not be used. 90 | transport : set to "tcp" to use the default setting of transport which is 91 | raw TCP. Set to "websockets" to use WebSockets as the transport. 92 | """ 93 | 94 | if qos < 0 or qos > 2: 95 | raise ValueError('qos must be in the range 0-2') 96 | 97 | callback_userdata = { 98 | 'callback':callback, 99 | 'topics':topics, 100 | 'qos':qos, 101 | 'userdata':userdata} 102 | 103 | client = paho.Client(client_id=client_id, 104 | userdata=callback_userdata, protocol=protocol, transport=transport) 105 | client.on_message = _on_message_callback 106 | client.on_connect = _on_connect 107 | 108 | if auth is not None: 109 | username = auth['username'] 110 | try: 111 | password = auth['password'] 112 | except KeyError: 113 | password = None 114 | client.username_pw_set(username, password) 115 | 116 | if will is not None: 117 | will_topic = will['topic'] 118 | try: 119 | will_payload = will['payload'] 120 | except KeyError: 121 | will_payload = None 122 | try: 123 | will_qos = will['qos'] 124 | except KeyError: 125 | will_qos = 0 126 | try: 127 | will_retain = will['retain'] 128 | except KeyError: 129 | will_retain = False 130 | 131 | client.will_set(will_topic, will_payload, will_qos, will_retain) 132 | 133 | if tls is not None: 134 | ca_certs = tls['ca_certs'] 135 | try: 136 | certfile = tls['certfile'] 137 | except KeyError: 138 | certfile = None 139 | try: 140 | keyfile = tls['keyfile'] 141 | except KeyError: 142 | keyfile = None 143 | try: 144 | tls_version = tls['tls_version'] 145 | except KeyError: 146 | tls_version = ssl.PROTOCOL_SSLv23; 147 | try: 148 | ciphers = tls['ciphers'] 149 | except KeyError: 150 | ciphers = None 151 | client.tls_set(ca_certs, certfile, keyfile, tls_version=tls_version, 152 | ciphers=ciphers) 153 | 154 | client.connect(hostname, port, keepalive) 155 | client.loop_forever() 156 | 157 | 158 | def simple(topics, qos=0, msg_count=1, retained=True, hostname="localhost", port=1883, 159 | client_id="", keepalive=60, will=None, auth=None, tls=None, 160 | protocol=paho.MQTTv311, transport="tcp"): 161 | """Subscribe to a list of topics and return msg_count messages. 162 | This function creates an MQTT client, connects to a broker and subscribes 163 | to a list of topics. Once "msg_count" messages have been received, it 164 | disconnects cleanly from the broker and returns the messages. 165 | topics : either a string containing a single topic to subscribe to, or a 166 | list of topics to subscribe to. 167 | qos : the qos to use when subscribing. This is applied to all topics. 168 | msg_count : the number of messages to retrieve from the broker. 169 | if msg_count == 1 then a single MQTTMessage will be returned. 170 | if msg_count > 1 then a list of MQTTMessages will be returned. 171 | retained : If set to True, retained messages will be processed the same as 172 | non-retained messages. If set to False, retained messages will 173 | be ignored. This means that with retained=False and msg_count=1, 174 | the function will return the first message received that does 175 | not have the retained flag set. 176 | hostname : a string containing the address of the broker to connect to. 177 | Defaults to localhost. 178 | port : the port to connect to the broker on. Defaults to 1883. 179 | client_id : the MQTT client id to use. If "" or None, the Paho library will 180 | generate a client id automatically. 181 | keepalive : the keepalive timeout value for the client. Defaults to 60 182 | seconds. 183 | will : a dict containing will parameters for the client: will = {'topic': 184 | "", 'payload':", 'qos':, 'retain':}. 185 | Topic is required, all other parameters are optional and will 186 | default to None, 0 and False respectively. 187 | Defaults to None, which indicates no will should be used. 188 | auth : a dict containing authentication parameters for the client: 189 | auth = {'username':"", 'password':""} 190 | Username is required, password is optional and will default to None 191 | if not provided. 192 | Defaults to None, which indicates no authentication is to be used. 193 | tls : a dict containing TLS configuration parameters for the client: 194 | dict = {'ca_certs':"", 'certfile':"", 195 | 'keyfile':"", 'tls_version':"", 196 | 'ciphers':"} 197 | ca_certs is required, all other parameters are optional and will 198 | default to None if not provided, which results in the client using 199 | the default behaviour - see the paho.mqtt.client documentation. 200 | Defaults to None, which indicates that TLS should not be used. 201 | transport : set to "tcp" to use the default setting of transport which is 202 | raw TCP. Set to "websockets" to use WebSockets as the transport. 203 | """ 204 | 205 | if msg_count < 1: 206 | raise ValueError('msg_count must be > 0') 207 | 208 | # Set ourselves up to return a single message if msg_count == 1, or a list 209 | # if > 1. 210 | if msg_count == 1: 211 | messages = None 212 | else: 213 | messages = [] 214 | 215 | userdata = {'retained':retained, 'msg_count':msg_count, 'messages':messages} 216 | 217 | callback(_on_message_simple, topics, qos, userdata, hostname, port, 218 | client_id, keepalive, will, auth, tls, protocol, transport) 219 | 220 | return userdata['messages'] 221 | -------------------------------------------------------------------------------- /docs/Automatic Protocol Format Reverse Engineering.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PengFoo/BostonIvy/06d996d9988e0f886faa3440a533839df4c25f2d/docs/Automatic Protocol Format Reverse Engineering.pdf -------------------------------------------------------------------------------- /docs/Finding contributors to Great Firewall by their papers.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PengFoo/BostonIvy/06d996d9988e0f886faa3440a533839df4c25f2d/docs/Finding contributors to Great Firewall by their papers.pdf -------------------------------------------------------------------------------- /docs/doc_to_download.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PengFoo/BostonIvy/06d996d9988e0f886faa3440a533839df4c25f2d/docs/doc_to_download.txt -------------------------------------------------------------------------------- /docs/一种接入网的流量识别方法.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PengFoo/BostonIvy/06d996d9988e0f886faa3440a533839df4c25f2d/docs/一种接入网的流量识别方法.pdf -------------------------------------------------------------------------------- /docs/基于SVM的网络流量监测.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PengFoo/BostonIvy/06d996d9988e0f886faa3440a533839df4c25f2d/docs/基于SVM的网络流量监测.pdf -------------------------------------------------------------------------------- /docs/基于会话的应用特征自适应提取.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PengFoo/BostonIvy/06d996d9988e0f886faa3440a533839df4c25f2d/docs/基于会话的应用特征自适应提取.pdf -------------------------------------------------------------------------------- /docs/基于加权累积和检验的加密流量盲识别算法.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PengFoo/BostonIvy/06d996d9988e0f886faa3440a533839df4c25f2d/docs/基于加权累积和检验的加密流量盲识别算法.pdf -------------------------------------------------------------------------------- /docs/基于字节熵矢量加权指纹的二进制协议识别.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PengFoo/BostonIvy/06d996d9988e0f886faa3440a533839df4c25f2d/docs/基于字节熵矢量加权指纹的二进制协议识别.pdf -------------------------------------------------------------------------------- /docs/网络流量分类研究进展与展望.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PengFoo/BostonIvy/06d996d9988e0f886faa3440a533839df4c25f2d/docs/网络流量分类研究进展与展望.pdf -------------------------------------------------------------------------------- /local.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/python 2 | # -*- coding: UTF-8 -*- 3 | __author__ = 'fp' 4 | 5 | ''' 6 | local.py is actually a socks5 server to send data 7 | to remote server via MQTT protocal 8 | ''' 9 | 10 | import socket 11 | import sys 12 | import select 13 | import SocketServer 14 | import struct 15 | import time, datetime 16 | import mqttclient 17 | import thread 18 | import select 19 | 20 | c = mqttclient.MQTTClient() 21 | conns = [] 22 | dict_sock = {} 23 | class Server(SocketServer.ThreadingMixIn, SocketServer.TCPServer): 24 | pass 25 | 26 | 27 | class Socks5Handler(SocketServer.StreamRequestHandler): 28 | # is it OK? 29 | global c, dict_conn 30 | def handle_and_send(self, sock, data): 31 | # TODO ENCRYPTION 32 | bytes_sent = 0 33 | # print [ord(i) for i in data] 34 | # print len(data) 35 | data = bytearray(data) 36 | # print 'req:', data 37 | c.client.publish('c2s',data) 38 | 39 | 40 | def handle_tcp(self, sock, remote, localport, remoteAddr, remotePort): 41 | fdset = [sock] 42 | while 1: 43 | r, w, e = select.select(fdset, [], []) 44 | 45 | if sock in r: 46 | data = sock.recv(2048) 47 | if len(data) <= 0: 48 | return 49 | lpStr = str(hex(localport))[2:] 50 | raStr = str(remoteAddr) 51 | rpStr = str(hex(remotePort))[2:] 52 | 53 | lpStr = '0' * (4 - len(lpStr)) + lpStr 54 | rpStr = '0' * (4 - len(rpStr)) + rpStr 55 | lenRaStr = str(hex(len(raStr)))[2:] 56 | raStr = '0' * (2- len(lenRaStr)) + lenRaStr + raStr 57 | 58 | data = lpStr + raStr + rpStr + data 59 | self.handle_and_send(remote, data) 60 | ''' 61 | ''' 62 | dict_sock[localport] = sock 63 | if localport not in conns: 64 | conns.append(localport) 65 | while 1: 66 | if localport not in conns: 67 | time.sleep(0.1) 68 | return 69 | return 70 | ''' 71 | ''' 72 | timeout = 60 73 | i = time.time() + 60 74 | while 1: 75 | if time.time()>i: 76 | del dict_conn[localport] 77 | conns.remove(localport) 78 | break 79 | if localport in dict_conn.keys(): 80 | bytes_sent = 0 81 | dataReply = dict_conn[localport] 82 | while 1: 83 | r = sock.send(dataReply[bytes_sent:]) 84 | if r < 0: 85 | del dict_conn[localport] 86 | break 87 | bytes_sent += r 88 | if bytes_sent == len(dataReply): 89 | del dict_conn[localport] 90 | break 91 | 92 | 93 | 94 | def handle(self): 95 | try: 96 | conn = self.connection 97 | rf = self.rfile 98 | # print 'connection from ', self.client_address 99 | 100 | # version 0x05, method 0x00, see socks5client and socks5 server for socks5 more info 101 | 102 | # should be client socks5 header 0x05 0x00 0x01 103 | rf.read(3) # or conn.recv(1024) 104 | conn.send(b'\x05\x00') 105 | 106 | data = rf.read(4) 107 | 108 | # socks version, 0x05 for socks v5 109 | ver = ord(data[0]) 110 | 111 | # cmd 112 | # 0x01 for connect 113 | # 0x02 for bind 114 | # 0x03 for udp associate 115 | cmd = ord(data[1]) 116 | 117 | # rsv no use 118 | rsv = ord(data[2]) 119 | 120 | # address type 121 | # 0x01 for ipv4 addr 122 | # 0x03 for domainname 123 | # 0x04 for ipv6 addr 124 | atyp = ord(data[3]) 125 | 126 | # print ver,cmd,rsv,atyp 127 | addr = 0 128 | if atyp == 1: 129 | # ipv4, next 4 bytes shall be the ip addr 130 | addr = socket.inet_ntoa(rf.read(4)) 131 | # print addr 132 | elif atyp == 3: 133 | # domainname 134 | # first byte for the length of domain, no \0 character 135 | length = ord(rf.read(1)) 136 | addr = rf.read(length) 137 | # print addr 138 | elif atyp == 4: 139 | # ipv6 140 | # TODO handle the ipv6 address 141 | addr = 0 142 | pass 143 | port = struct.unpack('>H', rf.read(2))[0] 144 | 145 | # reply ver, rep, rsv, atyp, wait to add other info 146 | reply = '\x05\x00\x00\x01' 147 | 148 | remote = None 149 | # start connection to remote server 150 | try: 151 | if cmd == 1: 152 | # tcp connect 153 | # TODO REPLACE THE SOCKET CODE TO THE MQTT CODE 154 | remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 155 | remote.connect((addr, port)) 156 | 157 | # print 'Tcp connect to', addr, port 158 | else: 159 | # Command not supported 160 | reply = b"\x05\x07\x00\x01" 161 | local = remote.getsockname() 162 | reply += socket.inet_aton(local[0]) + struct.pack(">H", local[1]) 163 | except Exception, e: 164 | # connection refused 165 | reply = '\x05\x05\x00\x01\x00\x00\x00\x00\x00\x00' 166 | print e 167 | 168 | conn.send(reply) 169 | # 3. Transfering 170 | if reply[1] == '\x00': # Success 171 | if cmd == 1: # 1. Tcp connect 172 | self.handle_tcp(conn, remote, self.client_address[1], addr, port) 173 | 174 | except socket.error, e: 175 | print 'socket error!', e 176 | 177 | pass 178 | i = 0 179 | def on_message(client, userdata, msg): 180 | ''' 181 | when mqtt message comes 182 | do: 183 | handle the payload and form the socket payload 184 | set a socket 185 | send the payload via socket. 186 | ''' 187 | try: 188 | payload = msg.payload 189 | 190 | localPort = int(payload[0:4], 16) 191 | remoteAddrLen = int(payload[4:6],16) 192 | remoteAddr = payload[6:6+remoteAddrLen] 193 | remotePort = int(payload[6+remoteAddrLen:10+remoteAddrLen],16) 194 | data = payload[10+remoteAddrLen:] 195 | print 'localport:',localPort,'len', len(payload),'connection', conns 196 | dataReply = data 197 | if dataReply == 'cometoend': 198 | conns.remove(localPort) 199 | # print 'end!' 200 | return 201 | bytes_sent = 0 202 | sock = dict_sock[localPort] 203 | global i 204 | i += 1 205 | # print 'sent' , i 206 | while 1: 207 | r = sock.send(dataReply[bytes_sent:]) 208 | if r < 0: 209 | break 210 | bytes_sent += r 211 | if bytes_sent == len(dataReply): 212 | break 213 | except Exception,e: 214 | print e 215 | return 216 | 217 | 218 | if __name__ == '__main__': 219 | server = Server(('', 8765), Socks5Handler) 220 | c.client.on_message = on_message 221 | c.client.subscribe('s2c') 222 | thread.start_new_thread(c.client.loop_forever,()) 223 | server.serve_forever() 224 | 225 | -------------------------------------------------------------------------------- /mqttclient.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/python 2 | # -*- coding: UTF-8 -*- 3 | 4 | __author__ = 'fp' 5 | 6 | import paho.mqtt.client as mqtt 7 | 8 | 9 | class MQTTClient(object): 10 | 11 | client = None 12 | 13 | def __init__(self): 14 | self.client = mqtt.Client() 15 | self.client.on_connect = self.on_connect 16 | self.client.on_message = self.on_message 17 | #self.client.connect('120.24.59.185',1883,60) 18 | self.client.connect('127.0.0.1',1883,60) 19 | pass 20 | 21 | # callback of function connect() 22 | def on_connect(self,client, userdata, flags, rc): 23 | print 'userdata', userdata 24 | print 'flag', flags 25 | print 'rc', str(rc) 26 | ''' 27 | userdata None 28 | flag {'session present': 0} 29 | rc 0 30 | ''' 31 | 32 | # subscribe the topic s2c (server to client) 33 | # self.client.subscribe('c2s') 34 | pass 35 | 36 | # callback of coming message 37 | def on_message(self, client, userdata, msg): 38 | print msg.payload 39 | pass 40 | 41 | # connect the 42 | def publish(self, topic, payload): 43 | self.client.publish(topic, payload) 44 | 45 | if __name__ == '__main__': 46 | c = MQTTClient() 47 | c.client.subscribe('c2s') 48 | while 1: 49 | # c.client.publish('c2s','test') 50 | c.client.loop() 51 | 52 | 53 | -------------------------------------------------------------------------------- /server.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/python 2 | # -*- coding: UTF-8 -*- 3 | 4 | __author__ = 'fp' 5 | 6 | ''' 7 | subscribe the topic, 8 | when message received, 9 | handle the message, get 1.local port 2.remote ip 3.remote port 10 | 2 and 3 make a socket from the server to the website 11 | 127.0.0.1 and 1 make a socket from local to the browser 12 | between the server and local, use the MQTT protocol v3.1 13 | ''' 14 | 15 | import socket 16 | import sys 17 | import select 18 | import SocketServer 19 | import struct 20 | import time 21 | import mqttclient 22 | import threadpool 23 | import datetime 24 | import gc 25 | 26 | 27 | pool = threadpool.ThreadPool(300) 28 | i = 0 29 | # callback of coming message 30 | def on_message(client, userdata, msg): 31 | payload = msg.payload 32 | # print payload 33 | req = threadpool.WorkRequest(reply,[client, payload]) 34 | pool.putRequest(req) 35 | 36 | def getHeaderAndLength(socket): 37 | header = [] 38 | line = '' 39 | length = 0 40 | while 1: 41 | tmp = socket.recv(1) 42 | # print tmp 43 | if tmp == '\r': 44 | tmp += socket.recv(1) 45 | # print tmp 46 | line += tmp 47 | if tmp == '\r\n': 48 | header.append(line) 49 | line = '' 50 | tmp = socket.recv(2) 51 | # print tmp 52 | line += tmp 53 | if tmp == '\r\n': 54 | header.append(line) 55 | length = getLength(header) 56 | # print length 57 | break 58 | else: 59 | continue 60 | else: 61 | continue 62 | else: 63 | line += tmp 64 | continue 65 | return ''.join(header),length 66 | 67 | ''' 68 | according to afc 2616 69 | chapter 4.4 :: message length 70 | http://www.w3.org/Protocols/rfc2616/rfc2616.html 71 | ''' 72 | # need socket param because if the head is chunked, 73 | # need recv more data to decide the length 74 | def getLength(header): 75 | length = -2 76 | h = {} 77 | # print '\n'.join(header) 78 | for line in header: 79 | if line == '\r\n': 80 | continue 81 | #print line 82 | # value has a space in the front 83 | line = line[:-2] 84 | if 'HTTP/1.' in line: 85 | # Any response message which "MUST NOT" include a message-body 86 | # (such as the 1xx, 204, and 304 responses and any response to a HEAD request) 87 | # is always terminated by the first empty line after the header fields, regardless 88 | # of the entity-header fields present in the message. 89 | if line.split()[1] == '304' or line.split()[1] == '204' or line.split()[1][0]=='1': 90 | return 0 91 | else: 92 | 93 | key,value = line.split(':')[0],line.split(':')[1][1:] 94 | #print key,value 95 | h[key] = value 96 | #print h 97 | if 'Transfer-Encoding' in h.keys() and h['Transfer-Encoding'] == 'chunked': 98 | # chunked 99 | return -1 100 | elif 'Content-Length'in h.keys(): 101 | # content-length 102 | # print h['Content-Length'] 103 | return int(h['Content-Length']) 104 | return length 105 | 106 | def recvBySize(socket, size, buff): 107 | data = '' 108 | _size = size 109 | while size >= buff: 110 | data += socket.recv(buff) 111 | size -= buff 112 | data += socket.recv(size) 113 | while len(data) < _size: 114 | data += socket.recv(_size-len(data)) 115 | 116 | return data 117 | 118 | # @profile 119 | def recv(the_socket, header, client): 120 | data='' 121 | contentLength = 0 122 | t1 = time.time() 123 | headerStr,length = getHeaderAndLength(the_socket) 124 | 125 | data = headerStr 126 | client.publish('s2c',bytearray(header+data)) 127 | data = '' 128 | t2 = time.time() 129 | # print 'get header and length using' , str(t2-t1) 130 | # 1024 per message 131 | # print length 132 | # print headerStr 133 | length = - 100 134 | if length > 0: 135 | while length > 2048: 136 | data += the_socket.recv(2048) 137 | client.publish('s2c',bytearray(header+data)) 138 | data = '' 139 | length -= 2048 140 | t1 = time.time() 141 | data += the_socket.recv(length) 142 | t2 = time.time() 143 | client.publish('s2c',bytearray(header+data)) 144 | 145 | elif length == -2: 146 | _len = 0 147 | while 1: 148 | tmp = the_socket.recv(1) 149 | data += tmp 150 | if tmp == '\r': 151 | tmp = the_socket.recv(1) 152 | data += tmp 153 | if tmp == '\n': 154 | tmp = the_socket.recv(2) 155 | data += tmp 156 | if tmp == '\r\n': 157 | client.publish('s2c',bytearray(header+data)) 158 | _len = 0 159 | break 160 | _len += 1 161 | if _len >= 2048: 162 | client.publish('s2c',bytearray(header+data)) 163 | data = '' 164 | _len = 0 165 | elif length == -1: 166 | # TODO support chunk with format, now just support chunk with size 167 | # chunked 168 | # chunk --> 169 | # (size)\r\n(data of size)\r\n 170 | 171 | ''' 172 | while 1: 173 | data = the_socket.recv(3) 174 | while data[-2:] != '\r\n': 175 | data += the_socket.recv(1) 176 | if len(data)>1024: 177 | pass 178 | 179 | #print len(data) 180 | #print [hex(ord(i)) for i in data] 181 | if data == '0\r\n': 182 | client.publish('s2c',bytearray(header+data)) 183 | return 184 | 185 | ''' 186 | while 1: 187 | # the smallest chunk is 3 bytes : 188 | # 0\r\n 189 | data = the_socket.recv(3) 190 | 191 | # recv till got the \r\n 192 | while data[-2:] != '\r\n': 193 | data += the_socket.recv(1) 194 | _len = len(data) 195 | 196 | # chunk size 197 | size = int(data[:-2],16) 198 | 199 | # chunk size == 0 means last chunk 200 | if size == 0: 201 | data = '0' + the_socket.recv(2) 202 | print 'end!', [hex(ord(i)) for i in data] 203 | client.publish('s2c',bytearray(header+data)) 204 | break 205 | 206 | 207 | 208 | data += recvBySize(the_socket, size, 1024) 209 | 210 | print len(data),size+ _len 211 | 212 | data += the_socket.recv(2) # \r\n 213 | client.publish('s2c',bytearray(header+data)) 214 | # print [hex(ord(i)) for i in data] 215 | 216 | 217 | # print size 218 | 219 | 220 | elif length == 0: 221 | client.publish('s2c',bytearray(header+data)) 222 | else: 223 | while True: 224 | data = the_socket.recv(2048) 225 | if len(data) <= 0: 226 | break 227 | client.publish('s2c',bytearray(header+data)) 228 | print 'published!' 229 | 230 | return 231 | 232 | sentnum = 0 233 | def reply(client,payload): 234 | 235 | global sentnum 236 | localPort = int(payload[0:4], 16) 237 | remoteAddrLen = int(payload[4:6],16) 238 | remoteAddr = payload[6:6+remoteAddrLen] 239 | remotePort = int(payload[6+remoteAddrLen:10+remoteAddrLen],16) 240 | header = payload[:10+remoteAddrLen] 241 | data = payload[10+remoteAddrLen:] 242 | remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 243 | remote.connect((remoteAddr, remotePort)) 244 | err = remote.sendall(data) 245 | t1 = datetime.datetime.now() 246 | # data2reply = recv_basic(remote) 247 | # remote.setblocking(0) 248 | fdset = [remote] 249 | while 1: 250 | r, w, e = select.select(fdset, [], []) 251 | if remote in r: 252 | data = remote.recv(4096) 253 | if len(data) <= 0: 254 | client.publish('s2c',header + 'cometoend') 255 | sentnum += 1 256 | # print 'send an end!',sentnum 257 | break 258 | print 'len:',len(header+data),'no.',sentnum 259 | sentnum += 1 260 | client.publish('s2c',bytearray(header+data)) 261 | 262 | # print 'send',sentnum 263 | 264 | return 265 | 266 | 267 | if __name__ == '__main__': 268 | 269 | server = mqttclient.MQTTClient() 270 | server.client.on_message = on_message 271 | server.client.subscribe('c2s') 272 | t = time.time() + 90 273 | while 1: 274 | ''' 275 | if time.time() > t: 276 | break 277 | ''' 278 | server.client.loop() 279 | 280 | -------------------------------------------------------------------------------- /server.py.lprof: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PengFoo/BostonIvy/06d996d9988e0f886faa3440a533839df4c25f2d/server.py.lprof -------------------------------------------------------------------------------- /socks5client.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/python 2 | # -*- coding: UTF-8 -*- 3 | 4 | __author__ = 'fp' 5 | 6 | import socket 7 | 8 | 9 | # socks5 rfc 1928 at http://www.openssh.com/txt/rfc1928.txt 10 | # socks5 rfc 1928 chinese version at http://blog.chinaunix.net/uid-26548237-id-3434356.html 11 | 12 | 13 | def client(): 14 | server = '127.0.0.1' 15 | port = 7654 16 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 17 | 18 | # establish socket 19 | s.connect((server, port)) 20 | 21 | ''' 22 | The client connects to the server, and sends a version 23 | identifier/method selection message: 24 | 25 | +----+----------+----------+ 26 | |VER | NMETHODS | METHODS | 27 | +----+----------+----------+ 28 | | 1 | 1 | 1 to 255 | 29 | +----+----------+----------+ 30 | 31 | The VER field is set to X'05' for this version of the protocol. The 32 | NMETHODS field contains the number of method identifier octets that 33 | appear in the METHODS field. 34 | ''' 35 | # sent sokcs5 header 36 | # ver = 0x05, nmethods = 0x01, methods = 0x00 37 | s.sendall('\x05\x01\x00') 38 | 39 | print repr(s.recv(1024)) 40 | 41 | ''' 42 | The SOCKS request is formed as follows: 43 | 44 | +----+-----+-------+------+----------+----------+ 45 | |VER | CMD | RSV | ATYP | DST.ADDR | DST.PORT | 46 | +----+-----+-------+------+----------+----------+ 47 | | 1 | 1 | X'00' | 1 | Variable | 2 | 48 | +----+-----+-------+------+----------+----------+ 49 | 50 | Where: 51 | 52 | o VER protocol version: X'05' 53 | o CMD 54 | o CONNECT X'01' 55 | o BIND X'02' 56 | o UDP ASSOCIATE X'03' 57 | o RSV RESERVED 58 | o ATYP address type of following address 59 | o IP V4 address: X'01' 60 | o DOMAINNAME: X'03' 61 | o IP V6 address: X'04' 62 | o DST.ADDR desired destination address 63 | o DST.PORT desired destination port in network octet 64 | order 65 | 66 | ''' 67 | # send request header 68 | # ver = 0x05, cmd = 0x01, rsv = 0x00, atvp = 0x01, 69 | # dst.addr = 0x7f.0x00.0x00.0x01 (127.0.0.1), dst.port =0x50 (80) 70 | # 71 | # addr and port currently unused 72 | # 73 | s.sendall("\x05\x01\x00\x01\x7f\x00\x00\x01\x00\x50") 74 | print repr(s.recv(1024)) 75 | 76 | # http request 77 | s.sendall("this is a request via socks5:)") 78 | print repr(s.recv(1024)) 79 | 80 | if __name__ == '__main__': 81 | client() -------------------------------------------------------------------------------- /socks5proxy.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/python 2 | # -*- coding: UTF-8 -*- 3 | 4 | __author__ = 'fp' 5 | 6 | import socket 7 | import select 8 | import SocketServer 9 | import struct 10 | 11 | 12 | 13 | 14 | class Server(SocketServer.ThreadingMixIn, SocketServer.TCPServer): 15 | pass 16 | 17 | 18 | class Socks5Handler(SocketServer.StreamRequestHandler): 19 | # send the data in order and in pieces 20 | def handle_and_send(self, sock, data): 21 | # TODO ENCRYPTION AND SEND VIA MQTT 22 | bytes_sent = 0 23 | while True: 24 | r = sock.send(data[bytes_sent:]) 25 | if r < 0: 26 | return r 27 | bytes_sent += r 28 | if bytes_sent == len(data): 29 | return bytes_sent 30 | 31 | def handle_tcp(self, sock, remote): 32 | fdset = [sock, remote] 33 | while True: 34 | r, w, e = select.select(fdset, [], []) 35 | 36 | if sock in r: 37 | data = sock.recv(4096) 38 | if len(data) <= 0: 39 | break 40 | result = self.handle_and_send(remote, data) 41 | if result < len(data): 42 | raise Exception('failed to send all data') 43 | 44 | if remote in r: 45 | data = remote.recv(4096) 46 | if len(data) <= 0: 47 | break 48 | result = self.handle_and_send(sock, data) 49 | if result < len(data): 50 | raise Exception('failed to send all data') 51 | 52 | def handle(self): 53 | try: 54 | conn = self.connection 55 | rf = self.rfile 56 | print 'connection from ', self.client_address 57 | # version 0x05, method 0x00, see socks5client and socks5 server for socks5 more info 58 | 59 | # should be client socks5 header 0x05 0x00 0x01 60 | rf.read(3) # or conn.recv(1024) 61 | conn.send(b'\x05\x00') 62 | 63 | data = rf.read(4) 64 | 65 | # socks version, 0x05 for socks v5 66 | ver = ord(data[0]) 67 | 68 | # cmd 69 | # 0x01 for connect 70 | # 0x02 for bind 71 | # 0x03 for udp associate 72 | cmd = ord(data[1]) 73 | 74 | # rsv no use 75 | rsv = ord(data[2]) 76 | 77 | # address type 78 | # 0x01 for ipv4 addr 79 | # 0x03 for domainname 80 | # 0x04 for ipv6 addr 81 | atyp = ord(data[3]) 82 | 83 | # print ver,cmd,rsv,atyp 84 | addr = 0 85 | if atyp == 1: 86 | # ipv4, next 4 bytes shall be the ip addr 87 | addr = socket.inet_ntoa(rf.read(4)) 88 | # print addr 89 | elif atyp == 3: 90 | # domainname 91 | # first byte for the length of domain, no \0 character 92 | length = ord(rf.read(1)) 93 | addr = rf.read(length) 94 | # print addr 95 | elif atyp == 4: 96 | # ipv6 97 | # TODO handle the ipv6 address 98 | addr = 0 99 | pass 100 | port = struct.unpack('>H', rf.read(2))[0] 101 | 102 | # reply ver, rep, rsv, atyp, wait to add other info 103 | reply = '\x05\x00\x00\x01' 104 | 105 | remote = None 106 | # start connection to remote server 107 | try: 108 | if cmd == 1: 109 | # tcp connect 110 | # TODO REPLACE THE SOCKET CODE TO THE MQTT CODE 111 | remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 112 | remote.connect((addr, port)) 113 | print 'Tcp connect to', addr, port 114 | else: 115 | # Command not supported 116 | reply = b"\x05\x07\x00\x01" 117 | local = remote.getsockname() 118 | reply += socket.inet_aton(local[0]) + struct.pack(">H", local[1]) 119 | except Exception, e: 120 | # connection refused 121 | reply = '\x05\x05\x00\x01\x00\x00\x00\x00\x00\x00' 122 | print e 123 | 124 | conn.send(reply) 125 | # 3. Transfering 126 | if reply[1] == '\x00': # Success 127 | if cmd == 1: # 1. Tcp connect 128 | self.handle_tcp(conn, remote) 129 | 130 | except socket.error, e: 131 | print 'socket error!', e 132 | 133 | pass 134 | 135 | if __name__ == '__main__': 136 | server = Server(('', 8765), Socks5Handler) 137 | server.serve_forever() -------------------------------------------------------------------------------- /socks5server.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/python 2 | # -*- coding: UTF-8 -*- 3 | 4 | __author__ = 'fp' 5 | 6 | import socket 7 | 8 | 9 | # socks5 rfc 1928 at http://www.openssh.com/txt/rfc1928.txt 10 | # socks5 rfc 1928 chinese version at http://blog.chinaunix.net/uid-26548237-id-3434356.html 11 | 12 | 13 | def server(): 14 | HOST = '0.0.0.0' 15 | PORT = 7654 16 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 17 | s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 18 | s.bind((HOST, PORT)) 19 | s.listen(1024) 20 | conn, addr = s.accept() 21 | print repr(conn.recv(1024)) 22 | 23 | ''' 24 | The server selects from one of the methods given in METHODS, and 25 | sends a METHOD selection message: 26 | 27 | +----+--------+ 28 | |VER | METHOD | 29 | +----+--------+ 30 | | 1 | 1 | 31 | +----+--------+ 32 | 33 | If the selected METHOD is X'FF', none of the methods listed by the 34 | client are acceptable, and the client MUST close the connection. 35 | 36 | The values currently defined for METHOD are: 37 | 38 | o X'00' NO AUTHENTICATION REQUIRED 39 | o X'01' GSSAPI 40 | o X'02' USERNAME/PASSWORD 41 | o X'03' to X'7F' IANA ASSIGNED 42 | o X'80' to X'FE' RESERVED FOR PRIVATE METHODS 43 | o X'FF' NO ACCEPTABLE METHODS 44 | 45 | The client and server then enter a method-specific sub-negotiation. 46 | ''' 47 | conn.send("\x05\x00") 48 | print repr(conn.recv(1024)) 49 | 50 | ''' 51 | The SOCKS request information is sent by the client as soon as it has 52 | established a connection to the SOCKS server, and completed the 53 | authentication negotiations. The server evaluates the request, and 54 | returns a reply formed as follows: 55 | 56 | +----+-----+-------+------+----------+----------+ 57 | |VER | REP | RSV | ATYP | BND.ADDR | BND.PORT | 58 | +----+-----+-------+------+----------+----------+ 59 | | 1 | 1 | X'00' | 1 | Variable | 2 | 60 | +----+-----+-------+------+----------+----------+ 61 | 62 | Where: 63 | 64 | o VER protocol version: X'05' 65 | o REP Reply field: 66 | o X'00' succeeded 67 | o X'01' general SOCKS server failure 68 | o X'02' connection not allowed by ruleset 69 | o X'03' Network unreachable 70 | o X'04' Host unreachable 71 | o X'05' Connection refused 72 | o X'06' TTL expired 73 | o X'07' Command not supported 74 | o X'08' Address type not supported 75 | o X'09' to X'FF' unassigned 76 | o RSV RESERVED 77 | o ATYP address type of following address 78 | ''' 79 | 80 | # ver=5, Reply=0(succeeded), reserved=0, atype=1(ip), host=127.0.0.1 + port=80 81 | # 82 | # host and port currently unused 83 | # 84 | conn.send("\x05\x00\x00" + "\x01\x7f\x00\x00\x01" + "\x00\x50") 85 | 86 | # http-request from client 87 | x = conn.recv(4096) 88 | print x 89 | 90 | # http-response 91 | conn.send("this is a response via socks5:)") 92 | 93 | if __name__ == '__main__': 94 | server() -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/python 2 | # -*- coding: UTF-8 -*- 3 | 4 | __author__ = 'fp' 5 | 6 | ''' 7 | subscribe the topic, 8 | when message received, 9 | handle the message, get 1.local port 2.remote ip 3.remote port 10 | 2 and 3 make a socket from the server to the website 11 | 127.0.0.1 and 1 make a socket from local to the browser 12 | between the server and local, use the MQTT protocol v3.1 13 | ''' 14 | 15 | import socket 16 | import sys 17 | import select 18 | import SocketServer 19 | import struct 20 | import time 21 | import mqttclient 22 | 23 | i = 0 24 | # callback of coming message 25 | def on_message(client, userdata, msg): 26 | payload = msg.payload 27 | print payload 28 | 29 | 30 | 31 | if __name__ == '__main__': 32 | server = mqttclient.MQTTClient() 33 | server.client.on_message = on_message 34 | server.client.subscribe('s2c') 35 | for i in range(2000): 36 | server.client.publish('c2s',bytearray(str(i)),qos=1) 37 | server.client.loop_forever() 38 | 39 | --------------------------------------------------------------------------------