├── .style.yapf ├── LICENSE ├── MANIFEST.in ├── README.md ├── doc ├── COMMANDS.md └── FOR_DEBUG.md ├── p2p_python ├── __init__.py ├── config.py ├── core.py ├── serializer.py ├── server.py ├── tool │ ├── __init__.py │ ├── traffic.py │ ├── upnpc.py │ └── utils.py ├── user.py └── utils.py ├── requirements.txt └── setup.py /.style.yapf: -------------------------------------------------------------------------------- 1 | [style] 2 | based_on_style = google 3 | ARITHMETIC_PRECEDENCE_INDICATION = true 4 | NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS = true 5 | SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED = true 6 | COLUMN_LIMIT = 110 7 | INDENT_DICTIONARY_VALUE = true 8 | 9 | # create patch: yapf -d -r -p p2p_python > yapf.patch 10 | # affect patch: patch -p0 < yapf.patch 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 namuyang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt 2 | exclude test/* 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | p2p-python 2 | ========== 3 | I seek a library that can make a simple P2P network. 4 | This library enables you create P2P application. 5 | 6 | ## IMPORTANT 7 | **THIS IS OLD, newly rewritten version 4.0.0 will be released.** 8 | 9 | ## Specification 10 | * Asynchronous IO 11 | * Pure Python code 12 | * TCP and UDP connection 13 | * Automatic network build 14 | * Python**3.6+** 15 | 16 | ## How to install 17 | warning: **Destructive change from 3.0.0** 18 | ```commandline 19 | pip3 install --user p2p-python>=3.0.0 20 | ``` 21 | 22 | ## How to use 23 | basic usage with debug tool `aiomonitor`. 24 | install by `pip3 install --user aiomonitor`. 25 | ```python 26 | from p2p_python.utils import setup_p2p_params, setup_logger 27 | from p2p_python.server import Peer2Peer, Peer2PeerCmd 28 | import logging 29 | import asyncio 30 | import aiomonitor 31 | import time 32 | 33 | loop = asyncio.get_event_loop() 34 | log = logging.getLogger(__name__) 35 | 36 | setup_logger(logging.INFO) 37 | 38 | # setup Peer2Peer 39 | setup_p2p_params( 40 | network_ver=11111, # (int) identify other network 41 | p2p_port=2000, # (int) P2P listen port 42 | p2p_accept=True, # (bool) switch on TCP server 43 | p2p_udp_accept=True, # (bool) switch on UDP server 44 | ) 45 | p2p = Peer2Peer(listen=100) # allow 100 connection 46 | p2p.setup() 47 | 48 | # close method example 49 | def close(): 50 | p2p.close() 51 | loop.call_later(1.0, loop.stop) 52 | 53 | # You can setup DirectDmd method 54 | class DirectCmd(object): 55 | 56 | @staticmethod 57 | async def what_is_your_name(user, data): 58 | print("what_is_your_name", user, data) 59 | return {"you return": time.time()} 60 | 61 | @staticmethod 62 | async def get_time_now(user, data): 63 | print("get_time_now", user, data) 64 | return {"get time now": time.time()} 65 | 66 | # register methods for DirectCmd 67 | p2p.event.setup_events_from_class(DirectCmd) 68 | # throw cmd by `await p2p.send_direct_cmd(DirectCmd.what_is_your_name, 'kelly')` 69 | # or `await p2p.send_direct_cmd('what_is_your_name', 'kelly')` 70 | 71 | # You can setup broadcast policy (default disabled) 72 | # WARNING: You must set strict policy or will be broken by users 73 | async def broadcast_check_normal(user, data): 74 | return True 75 | 76 | # overwrite method 77 | p2p.broadcast_check = broadcast_check_normal 78 | 79 | # setup netcat monitor 80 | local = locals().copy() 81 | local.update({k: v for k, v in globals().items() if not k.startswith('__')}) 82 | log.info('local', list(local.keys())) 83 | aiomonitor.start_monitor(loop, port=3000, locals=local) 84 | log.info(f"you can connect by `nc 127.0.0.1 3000`") 85 | 86 | # start event loop 87 | # close by `close()` on netcat console 88 | try: 89 | loop.run_forever() 90 | except KeyboardInterrupt: 91 | log.info("closing") 92 | loop.close() 93 | ``` 94 | 95 | ## Documents 96 | * [about inner commands](doc/COMMANDS.md) 97 | * [for debug](doc/FOR_DEBUG.md) 98 | * [how to work p2p-python? (OLD and JP)](https://ameblo.jp/namuyan/entry-12398575560.html) 99 | 100 | ## Author 101 | [@namuyan_mine](http://twitter.com/namuyan_mine/) 102 | 103 | ## Licence 104 | [MIT](LICENSE) 105 | -------------------------------------------------------------------------------- /doc/COMMANDS.md: -------------------------------------------------------------------------------- 1 | Another commands 2 | ================ 3 | Used for network stabilize. 4 | Please import commands by `from p2p_python.server import Peer2PeerCmd`. 5 | 6 | commands 7 | ---------------- 8 | **ping-pong** 9 | ```text 10 | # check delay your node to a peer 11 | 12 | <<< await p2p.send_command(Peer2PeerCmd.PING_PONG, data=time.time()) 13 | 14 | >>> (, {'ping': None, 'pong': 1561109423.2927444}) 15 | ``` 16 | 17 | **get-peer-info** 18 | ```text 19 | # find a peer's peer list 20 | 21 | <<< await p2p.send_command(Peer2PeerCmd.GET_PEER_INFO) 22 | 23 | >>> (, [[['127.0.0.1', 2000], {'name': 'Army:54510', 'client_ver': '3.0.0', 'network_ver': 12345, 'p2p_accept': True, 'p2p_udp_accept': True, 'p2p_port': 2000, 'start_time': 1561107522, 'last_seen': 1561109616}]]) 24 | ``` 25 | 26 | **get-nears** 27 | ```text 28 | # for : get peer's connection info 29 | 30 | <<< await p2p.send_command(Peer2PeerCmd.GET_NEARS) 31 | 32 | >>> (, [[['127.0.0.1', 2000], {'name': 'Army:54510', 'client_ver': '3.0.0', 'network_ver': 12345, 'p2p_accept': True, 'p2p_udp_accept': True, 'p2p_port': 2000, 'start_time': 1561107522, 'last_seen': 1561109473}]]) 33 | ``` 34 | 35 | **check-reachable** 36 | ```text 37 | # check PORT connection reachable from outside 38 | 39 | <<< await p2p.send_command(Peer2PeerCmd.CHECK_REACHABLE, data={'port': 1000}) 40 | 41 | >>> (, False) 42 | ``` 43 | 44 | **direct-cmd** 45 | ```text 46 | # You can define any command like README.md 47 | # warning: do not forget check data format is can convert json 48 | 49 | <<< await p2p.send_direct_cmd(DirectCmd.what_is_your_name, data='kelly') 50 | 51 | >>> (, {"you return": 1561110.0}) 52 | ``` 53 | 54 | note 55 | ---- 56 | I checked by netcat console. 57 | 58 | -------------------------------------------------------------------------------- /doc/FOR_DEBUG.md: -------------------------------------------------------------------------------- 1 | for debugging 2 | ==== 3 | Good option for debuggers. 4 | 5 | ```python 6 | from p2p_python.config import Debug 7 | 8 | Debug.P_PRINT_EXCEPTION = False # print full exception info 9 | Debug.P_SEND_RECEIVE_DETAIL = False # print send/receive msg info 10 | Debug.F_RECODE_TRAFFIC = False # recode traffic recode to file 11 | ``` 12 | 13 | You can switch debug option online. 14 | -------------------------------------------------------------------------------- /p2p_python/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '3.0.5' 2 | -------------------------------------------------------------------------------- /p2p_python/config.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | class V: 4 | # path 5 | DATA_PATH = None 6 | 7 | # info 8 | CLIENT_VER = None 9 | SERVER_NAME = None 10 | NETWORK_VER = None 11 | P2P_PORT = None 12 | P2P_ACCEPT = None 13 | P2P_UDP_ACCEPT = None 14 | 15 | # setting 16 | TOR_CONNECTION = None # proxy (host, port) 17 | MY_HOST_NAME = None # optional: example.com 18 | 19 | 20 | class Debug: 21 | P_PRINT_EXCEPTION = False # print exception info 22 | P_SEND_RECEIVE_DETAIL = False # print receive msg info 23 | F_RECODE_TRAFFIC = False # recode traffic to file 24 | 25 | 26 | class PeerToPeerError(Exception): 27 | pass 28 | 29 | 30 | __all__ = [ 31 | "V", 32 | "Debug", 33 | "PeerToPeerError", 34 | ] 35 | -------------------------------------------------------------------------------- /p2p_python/core.py: -------------------------------------------------------------------------------- 1 | from p2p_python.config import V, Debug, PeerToPeerError 2 | from p2p_python.user import UserHeader, User 3 | from p2p_python.serializer import dumps 4 | from p2p_python.tool.traffic import Traffic 5 | from p2p_python.tool.utils import AESCipher 6 | from ecdsa.keys import SigningKey, VerifyingKey 7 | from ecdsa.curves import NIST256p 8 | from typing import Optional, Dict, List 9 | from logging import getLogger 10 | from binascii import a2b_hex 11 | from time import time 12 | from io import BytesIO 13 | from hashlib import sha256 14 | from expiringdict import ExpiringDict 15 | from asyncio.streams import StreamWriter, StreamReader 16 | import asyncio 17 | import json 18 | import random 19 | import socket 20 | import socks 21 | import zlib 22 | 23 | 24 | # socket direction 25 | INBOUND = 'inbound' 26 | OUTBOUND = 'outbound' 27 | 28 | log = getLogger(__name__) 29 | loop = asyncio.get_event_loop() 30 | tcp_servers: List[asyncio.AbstractServer] = list() 31 | udp_servers: List[socket.socket] = list() 32 | ban_address = list() # deny connection address 33 | BUFFER_SIZE = 8192 34 | socket2name = { 35 | socket.AF_INET: "ipv4", 36 | socket.AF_INET6: "ipv6", 37 | socket.AF_UNSPEC: "ipv4/6", 38 | } 39 | 40 | 41 | class Core(object): 42 | 43 | def __init__(self, host=None, listen=15): 44 | assert V.DATA_PATH is not None, 'Setup p2p params before CoreClass init.' 45 | assert host is None or host == 'localhost' 46 | # status params 47 | self.f_stop = False 48 | self.f_finish = False 49 | self.f_running = False 50 | # working info 51 | self.start_time = int(time()) 52 | self.number = 0 53 | self.user: List[User] = list() 54 | self.user_lock = asyncio.Lock() 55 | self.host = host # local=>'localhost', 'global'=>None 56 | self.core_que = asyncio.Queue() 57 | self.backlog = listen 58 | self.traffic = Traffic() 59 | self.ping_status: Dict[int, asyncio.Event] = ExpiringDict(max_len=5000, max_age_seconds=900) 60 | 61 | def close(self): 62 | if not self.f_running: 63 | raise Exception('Core is not running') 64 | self.traffic.close() 65 | for user in self.user.copy(): 66 | self.remove_connection(user, 'manual closing') 67 | for sock in tcp_servers: 68 | sock.close() 69 | asyncio.ensure_future(sock.wait_closed()) 70 | for sock in udp_servers: 71 | loop.remove_reader(sock.fileno()) 72 | sock.close() 73 | self.f_stop = True 74 | 75 | async def ping(self, user: User, f_udp=False): 76 | uuid = random.randint(1000000000, 4294967295) 77 | try: 78 | # prepare Event 79 | event = asyncio.Event() 80 | self.ping_status[uuid] = event 81 | # send ping 82 | msg_body = b'Ping:' + str(uuid).encode() 83 | await self.send_msg_body(msg_body=msg_body, user=user, allow_udp=f_udp, f_pro_force=True) 84 | # wait for event set (5s) 85 | await asyncio.wait_for(event.wait(), 5.0) 86 | return True 87 | except asyncio.TimeoutError: 88 | log.debug(f"failed to udp ping {user}") 89 | except ConnectionError as e: 90 | log.debug(f"socket error on ping by {e}") 91 | except Exception: 92 | log.error("ping exception", exc_info=True) 93 | # failed 94 | return False 95 | 96 | def start(self, s_family=socket.AF_UNSPEC): 97 | assert s_family in (socket.AF_INET, socket.AF_INET6, socket.AF_UNSPEC) 98 | # setup TCP/UDP socket server 99 | setup_all_socket_server(core=self, s_family=s_family) 100 | # listen socket ipv4/ipv6 101 | log.info(f"setup socket server " 102 | f"tcp{len(tcp_servers)}={V.P2P_ACCEPT} udp{len(udp_servers)}={V.P2P_UDP_ACCEPT}") 103 | self.f_running = True 104 | 105 | def get_my_user_header(self): 106 | """return my UserHeader format dict""" 107 | return { 108 | 'name': V.SERVER_NAME, 109 | 'client_ver': V.CLIENT_VER, 110 | 'network_ver': V.NETWORK_VER, 111 | 'my_host_name': V.MY_HOST_NAME, 112 | 'p2p_accept': V.P2P_ACCEPT, 113 | 'p2p_udp_accept': V.P2P_UDP_ACCEPT, 114 | 'p2p_port': V.P2P_PORT, 115 | 'start_time': self.start_time, 116 | 'last_seen': int(time()), 117 | } 118 | 119 | async def create_connection(self, host, port) -> bool: 120 | """create connection without exception""" 121 | if self.f_stop: 122 | return False 123 | # get connection list 124 | future: asyncio.Future = loop.run_in_executor( 125 | None, socket.getaddrinfo, host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) 126 | try: 127 | await asyncio.wait_for(future, 10.0) 128 | address_infos = future.result() 129 | except (asyncio.TimeoutError, socket.gaierror): 130 | return False 131 | # try to connect one by one 132 | for af, socktype, proto, canonname, host_port in address_infos: 133 | if host_port[0] in ban_address: 134 | return False # baned address 135 | try: 136 | if V.TOR_CONNECTION: 137 | if af != socket.AF_INET: 138 | continue 139 | sock = socks.socksocket() 140 | sock.setproxy(socks.PROXY_TYPE_SOCKS5, V.TOR_CONNECTION[0], V.TOR_CONNECTION[1]) 141 | else: 142 | sock = socket.socket(af, socktype, proto) 143 | future: asyncio.Future = loop.run_in_executor( 144 | None, sock.connect, host_port) 145 | await asyncio.wait_for(future, 10.0) 146 | future.result() # raised exception of socket 147 | sock.setblocking(False) 148 | reader, writer = await asyncio.open_connection(sock=sock, loop=loop) 149 | break 150 | except asyncio.TimeoutError: 151 | continue # try to connect but do not reach 152 | except ConnectionRefusedError: 153 | continue # try to connect closed socket 154 | except OSError as e: 155 | log.debug(f"socket creation error by {str(e)}") 156 | continue 157 | else: 158 | # create no connection 159 | return False 160 | log.debug(f"success create connection to {host_port}") 161 | 162 | try: 163 | # 1. receive plain message 164 | try: 165 | msg = await asyncio.wait_for(reader.read(BUFFER_SIZE), 5.0) 166 | if msg != b'hello': 167 | raise PeerToPeerError('first plain msg not correct? {}'.format(msg)) 168 | except asyncio.TimeoutError: 169 | raise PeerToPeerError('timeout on first plain msg receive') 170 | 171 | # 2. send my header 172 | send = json.dumps(self.get_my_user_header()).encode() 173 | writer.write(send) 174 | await writer.drain() 175 | self.traffic.put_traffic_up(send) 176 | 177 | # 3. receive public key 178 | try: 179 | my_sec, my_pub = generate_keypair() 180 | receive = await asyncio.wait_for(reader.read(BUFFER_SIZE), 5.0) 181 | self.traffic.put_traffic_down(receive) 182 | msg = json.loads(receive.decode()) 183 | except asyncio.TimeoutError: 184 | raise PeerToPeerError('timeout on public key receive') 185 | except json.JSONDecodeError: 186 | raise PeerToPeerError('json decode error on public key receive') 187 | 188 | # 4. send public key 189 | send = json.dumps({'public-key': my_pub}).encode() 190 | writer.write(send) 191 | await writer.drain() 192 | self.traffic.put_traffic_up(send) 193 | 194 | # 5. Get AES key and header and decrypt 195 | try: 196 | receive = await asyncio.wait_for(reader.read(BUFFER_SIZE), 5.0) 197 | self.traffic.put_traffic_down(receive) 198 | key = generate_shared_key(my_sec, msg['public-key']) 199 | dec = AESCipher.decrypt(key, receive) 200 | data = json.loads(dec.decode()) 201 | except asyncio.TimeoutError: 202 | raise PeerToPeerError('timeout on AES key and header receive') 203 | except json.JSONDecodeError: 204 | raise PeerToPeerError('json decode error on AES key and header receive') 205 | aeskey, header = data['aes-key'], data['header'] 206 | 207 | # 6. generate new user 208 | user_header = UserHeader(**header) 209 | new_user = User(user_header, self.number, reader, writer, host_port, aeskey, OUTBOUND) 210 | 211 | # 7. check header 212 | if new_user.header.network_ver != V.NETWORK_VER: 213 | raise PeerToPeerError('Don\'t same network version [{}!={}]'.format( 214 | new_user.header.network_ver, V.NETWORK_VER)) 215 | self.number += 1 216 | 217 | # 8. send accept signal 218 | encrypted = AESCipher.encrypt(new_user.aeskey, b'accept') 219 | await new_user.send(encrypted) 220 | self.traffic.put_traffic_up(encrypted) 221 | 222 | # 9. accept connection 223 | log.info(f"established connection as client to {new_user.header.name} {new_user.get_host_port()}") 224 | asyncio.ensure_future(self.receive_loop(new_user)) 225 | # server port's reachable check 226 | asyncio.ensure_future(self.check_reachable(new_user)) 227 | return True 228 | except PeerToPeerError as e: 229 | msg = "peer2peer error, {} ({})".format(e, host) 230 | except ConnectionRefusedError as e: 231 | msg = "connection refused error, {} ({})".format(e, host) 232 | except ValueError as e: 233 | msg = "ValueError: {} {}".format(host, e) 234 | except Exception as e: 235 | log.error("NewConnectionError", exc_info=True) 236 | msg = "NewConnectionError {} {}".format(host, e) 237 | 238 | # close socket 239 | log.debug(msg) 240 | if not writer.transport.is_closing(): 241 | writer.close() 242 | return False 243 | 244 | def remove_connection(self, user: User, reason: str) -> bool: 245 | if user is None: 246 | return False 247 | user.close() 248 | if user in self.user: 249 | self.user.remove(user) 250 | if 0 < user.score: 251 | log.info(f"remove connection of {user} by '{reason}'") 252 | else: 253 | log.debug(f"remove connection of {user} by '{reason}'") 254 | return True 255 | else: 256 | return False 257 | 258 | async def send_msg_body(self, msg_body, user: Optional[User] = None, allow_udp=False, f_pro_force=False): 259 | assert isinstance(msg_body, bytes), 'msg_body is bytes' 260 | 261 | # check user existence 262 | if len(self.user) == 0: 263 | raise PeerToPeerError('there is no user connection') 264 | # select random user 265 | if user is None: 266 | user = random.choice(self.user) 267 | 268 | # send message 269 | if allow_udp and f_pro_force: 270 | loop.run_in_executor(None, self.send_udp_body, msg_body, user) 271 | elif allow_udp and user.header.p2p_udp_accept and len(msg_body) < 1400: 272 | loop.run_in_executor(None, self.send_udp_body, msg_body, user) 273 | else: 274 | msg_body = zlib.compress(msg_body) 275 | msg_body = AESCipher.encrypt(key=user.aeskey, raw=msg_body) 276 | msg_len = len(msg_body).to_bytes(4, 'big') 277 | send_data = msg_len + msg_body 278 | await user.send(send_data) 279 | self.traffic.put_traffic_up(send_data) 280 | return user 281 | 282 | def send_udp_body(self, msg_body, user): 283 | """send UDP message""" 284 | name_len = len(V.SERVER_NAME.encode()).to_bytes(1, 'big') 285 | msg_body = AESCipher.encrypt(key=user.aeskey, raw=msg_body) 286 | send_data = name_len + V.SERVER_NAME.encode() + msg_body 287 | host_port = user.get_host_port() 288 | sock_family = socket.AF_INET if len(host_port) == 2 else socket.AF_INET6 289 | # warning: may block this closure, use run_in_executor 290 | with socket.socket(sock_family, socket.SOCK_DGRAM) as sock: 291 | sock.sendto(send_data, host_port) 292 | self.traffic.put_traffic_up(send_data) 293 | 294 | async def initial_connection_check(self, reader: StreamReader, writer: StreamWriter): 295 | host_port = writer.get_extra_info('peername') 296 | new_user: Optional[User] = None 297 | try: 298 | # 1. send plain message 299 | writer.write(b'hello') 300 | await writer.drain() 301 | 302 | # 2. receive other's header 303 | try: 304 | received = await asyncio.wait_for(reader.read(BUFFER_SIZE), 5.0) 305 | if len(received) == 0: 306 | raise PeerToPeerError('empty msg receive') 307 | header = json.loads(received.decode()) 308 | except asyncio.TimeoutError: 309 | raise PeerToPeerError('timeout on other\'s header receive') 310 | except json.JSONDecodeError: 311 | raise PeerToPeerError('json decode error on other\'s header receive') 312 | 313 | # 3. generate new user 314 | user_header = UserHeader(**header) 315 | new_user = User(user_header, self.number, reader, writer, host_port, AESCipher.create_key(), INBOUND) 316 | self.number += 1 317 | if new_user.header.name == V.SERVER_NAME: 318 | raise ConnectionAbortedError('Same origin connection') 319 | 320 | # 4. send my public key 321 | my_sec, my_pub = generate_keypair() 322 | send = json.dumps({'public-key': my_pub}).encode() 323 | await new_user.send(send) 324 | self.traffic.put_traffic_up(send) 325 | 326 | # 5. receive public key 327 | try: 328 | receive = await new_user.recv() 329 | self.traffic.put_traffic_down(receive) 330 | if len(receive) == 0: 331 | raise ConnectionAbortedError('received msg is zero.') 332 | data = json.loads(receive.decode()) 333 | except asyncio.TimeoutError: 334 | raise PeerToPeerError('timeout on public key receive') 335 | except json.JSONDecodeError: 336 | raise PeerToPeerError('json decode error on public key receive') 337 | 338 | # 6. encrypt and send AES key and header 339 | send = json.dumps({ 340 | 'aes-key': new_user.aeskey, 341 | 'header': self.get_my_user_header(), 342 | }) 343 | key = generate_shared_key(my_sec, data['public-key']) 344 | encrypted = AESCipher.encrypt(key, send.encode()) 345 | await new_user.send(encrypted) 346 | self.traffic.put_traffic_up(encrypted) 347 | 348 | # 7. receive accept signal 349 | try: 350 | encrypted = await new_user.recv() 351 | self.traffic.put_traffic_down(encrypted) 352 | except asyncio.TimeoutError: 353 | raise PeerToPeerError('timeout on accept signal receive') 354 | receive = AESCipher.decrypt(new_user.aeskey, encrypted) 355 | if receive != b'accept': 356 | raise PeerToPeerError(f"Not accept signal! {receive}") 357 | 358 | # 8. accept connection 359 | log.info(f"established connection as server from {new_user.header.name} {new_user.get_host_port()}") 360 | asyncio.ensure_future(self.receive_loop(new_user)) 361 | # server port's reachable check 362 | asyncio.ensure_future(self.check_reachable(new_user)) 363 | return 364 | except (ConnectionAbortedError, ConnectionResetError) as e: 365 | msg = f"disconnect error {host_port} {e}" 366 | except PeerToPeerError as e: 367 | msg = f"peer2peer error {host_port} {e}" 368 | except Exception as e: 369 | msg = "InitialConnCheck: {}".format(e) 370 | log.error(msg, exc_info=True) 371 | 372 | # EXCEPTION! 373 | if new_user: 374 | # remove user 375 | self.remove_connection(new_user, msg) 376 | else: 377 | # close socket 378 | log.debug(msg) 379 | try: 380 | writer.write(msg.encode()) 381 | await writer.drain() 382 | except Exception: 383 | pass 384 | try: 385 | writer.close() 386 | except Exception: 387 | pass 388 | 389 | async def receive_loop(self, user: User): 390 | # Accept connection 391 | for check_user in self.user.copy(): 392 | if check_user.header.name != user.header.name: 393 | continue 394 | elif await self.ping(check_user): 395 | error = f"same origin found and ping success, remove new connection" 396 | self.remove_connection(user, error) 397 | return 398 | else: 399 | error = f"same origin found but ping failed, remove old connection" 400 | self.remove_connection(check_user, error) 401 | self.user.append(user) 402 | log.info(f"check success and go into loop {user}") 403 | 404 | bio = BytesIO() # Warning: don't use initial_bytes, same duplicate ID used? 405 | bio_length = 0 406 | msg_length = 0 407 | f_raise_timeout = False 408 | error = None 409 | while not self.f_stop: 410 | try: 411 | get_msg = await user.recv() 412 | if len(get_msg) == 0: 413 | error = "Fall in loop, socket closed." 414 | break 415 | 416 | # check message params init 417 | bio_length += bio.write(get_msg) 418 | if msg_length == 0: 419 | # init message params 420 | msg_bytes = bio.getvalue() 421 | msg_length, initial_bytes = int.from_bytes(msg_bytes[:4], 'big'), msg_bytes[4:] 422 | bio.truncate(0) 423 | bio.seek(0) 424 | bio_length = bio.write(initial_bytes) 425 | elif bio_length == 0: 426 | error = "Why bio_length is zero?, msg_length={}".format(msg_length, bio_length) 427 | break 428 | else: 429 | pass 430 | 431 | # check complete message receive 432 | if bio_length >= msg_length: 433 | # success, get all message 434 | msg_bytes = bio.getvalue() 435 | msg_body, initial_bytes = msg_bytes[:msg_length], msg_bytes[msg_length:] 436 | if len(initial_bytes) == 0: 437 | # no another message 438 | msg_length = 0 439 | f_raise_timeout = False 440 | elif len(initial_bytes) < 4: 441 | error = "Failed to get message length? {}".format(initial_bytes) 442 | break 443 | else: 444 | # another message pushing 445 | msg_length, initial_bytes = int.from_bytes(initial_bytes[:4], 446 | 'big'), initial_bytes[4:] 447 | f_raise_timeout = True 448 | bio.truncate(0) 449 | bio.seek(0) 450 | bio_length = bio.write(initial_bytes) 451 | else: 452 | # continue getting message 453 | f_raise_timeout = True 454 | continue 455 | 456 | # continue to process msg_body 457 | self.traffic.put_traffic_down(msg_body) 458 | msg_body = AESCipher.decrypt(key=user.aeskey, enc=msg_body) 459 | msg_body = zlib.decompress(msg_body) 460 | if msg_body.startswith(b'Ping:'): 461 | uuid_bytes = msg_body.split(b':')[1] 462 | log.debug(f"receive Ping from {user.header.name}") 463 | await self.send_msg_body(b'Pong:' + uuid_bytes, user) 464 | elif msg_body.startswith(b'Pong:'): 465 | uuid_int = int(msg_body.decode().split(':')[1]) 466 | if uuid_int in self.ping_status: 467 | log.debug(f"receive Pong from {user.header.name}") 468 | self.ping_status[uuid_int].set() 469 | else: 470 | await self.core_que.put((user, msg_body, time())) 471 | f_raise_timeout = False 472 | 473 | except asyncio.TimeoutError: 474 | if f_raise_timeout: 475 | error = "Timeout: Not allowed timeout when getting message!" 476 | break 477 | except ConnectionError as e: 478 | error = "ConnectionError: " + str(e) 479 | break 480 | except OSError as e: 481 | error = "OSError: " + str(e) 482 | break 483 | except Exception: 484 | import traceback 485 | error = "Exception: " + str(traceback.format_exc()) 486 | break 487 | 488 | # After exit from loop, close socket 489 | if not bio.closed: 490 | bio.close() 491 | self.remove_connection(user, error) 492 | 493 | async def check_reachable(self, new_user: User): 494 | """check TCP/UDP port is opened""" 495 | try: 496 | # wait for accept or reject as user 497 | while new_user not in self.user: 498 | if new_user.closed: 499 | log.debug(f"user connection closed on check_reachable {new_user}") 500 | return 501 | await asyncio.sleep(1.0) 502 | # try to check TCP 503 | f_tcp = True 504 | host_port = new_user.get_host_port() 505 | af = socket.AF_INET if len(host_port) == 2 else socket.AF_INET6 506 | sock = socket.socket(af, socket.SOCK_STREAM) 507 | sock.settimeout(3.0) 508 | try: 509 | future: asyncio.Future = loop.run_in_executor( 510 | None, sock.connect_ex, host_port) 511 | await asyncio.wait_for(future, 10.0) 512 | result = future.result() 513 | if result != 0: 514 | f_tcp = False 515 | except (OSError, asyncio.TimeoutError): 516 | f_tcp = False 517 | loop.run_in_executor(None, sock.close) 518 | # try to check UDP 519 | f_udp = await self.ping(user=new_user, f_udp=True) 520 | f_changed = False 521 | # reflect user status 522 | if f_tcp is not new_user.header.p2p_accept: 523 | # log.debug(f"{new_user} Update TCP accept status {new_user.header.p2p_accept}=>{f_tcp}") 524 | new_user.header.p2p_accept = f_tcp 525 | f_changed = True 526 | if f_udp is not new_user.header.p2p_udp_accept: 527 | # log.debug(f"{new_user} Update UDP accept status {new_user.header.p2p_udp_accept}=>{f_udp}") 528 | new_user.header.p2p_udp_accept = f_udp 529 | f_changed = True 530 | if f_changed: 531 | log.debug(f"{new_user} Change socket status tcp={f_tcp} udp={f_udp}") 532 | except Exception: 533 | log.error("check_reachable exception", exc_info=True) 534 | 535 | async def try_reconnect(self, user: User, reason: str): 536 | self.remove_connection(user, reason) 537 | host_port = user.get_host_port() 538 | if self.f_stop: 539 | return False 540 | elif not user.header.p2p_accept: 541 | return False 542 | elif await self.create_connection(host=host_port[0], port=host_port[1]): 543 | log.debug(f"reconnect success {user}") 544 | new_user = self.host_port2user(host_port) 545 | if new_user: 546 | new_user.neers = user.neers 547 | new_user.score = user.score 548 | new_user.warn = user.warn 549 | return True 550 | else: 551 | log.warning(f"reconnect failed {user}") 552 | return False 553 | 554 | def name2user(self, name) -> Optional[User]: 555 | for user in self.user: 556 | if user.header.name == name: 557 | return user 558 | return None 559 | 560 | def host_port2user(self, host_port) -> Optional[User]: 561 | for user in self.user: 562 | if host_port == user.get_host_port(): 563 | return user 564 | return None 565 | 566 | 567 | """ECDH functions 568 | """ 569 | 570 | 571 | def generate_shared_key(sk, vk_str) -> bytes: 572 | vk = VerifyingKey.from_string(a2b_hex(vk_str), NIST256p) 573 | point = sk.privkey.secret_multiplier * vk.pubkey.point 574 | return sha256(point.x().to_bytes(32, 'big')).digest() 575 | 576 | 577 | def generate_keypair() -> (SigningKey, str): 578 | sk = SigningKey.generate(NIST256p) 579 | vk = sk.get_verifying_key() 580 | return sk, vk.to_string().hex() 581 | 582 | 583 | """socket connection functions 584 | """ 585 | 586 | 587 | async def udp_server_handle(msg, addr, core: Core): 588 | msg_body = None 589 | try: 590 | msg_len = msg[0] 591 | msg_name, msg_body = msg[1:msg_len + 1], msg[msg_len + 1:] 592 | user = core.name2user(msg_name.decode()) 593 | if user is None: 594 | return 595 | core.traffic.put_traffic_down(msg_body) 596 | msg_body = AESCipher.decrypt(key=user.aeskey, enc=msg_body) 597 | if msg_body.startswith(b'Ping:'): 598 | log.info(f"get udp ping from {user}") 599 | uuid_bytes = msg_body.split(b':')[1] 600 | await core.send_msg_body(msg_body=b'Pong:' + uuid_bytes, user=user) 601 | else: 602 | log.debug(f"get udp packet from {user}") 603 | await core.core_que.put((user, msg_body, time())) 604 | except ValueError as e: 605 | log.debug(f"maybe decrypt failed by {e} {msg_body}") 606 | except OSError as e: 607 | log.debug(f"OSError on udp listen by {str(e)}") 608 | except Exception as e: 609 | log.debug("UDP handle exception", exc_info=Debug.P_PRINT_EXCEPTION) 610 | 611 | 612 | def create_tcp_server(core: Core, family, host_port): 613 | assert family == socket.AF_INET or family == socket.AF_INET6 614 | coroutine = asyncio.start_server( 615 | core.initial_connection_check, host_port[0], host_port[1], 616 | family=family, backlog=core.backlog, loop=loop) 617 | abstract_server = loop.run_until_complete(coroutine) 618 | for sock in abstract_server.sockets: 619 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 620 | return abstract_server 621 | 622 | 623 | def create_udp_server(core: Core, family, host_port): 624 | assert family == socket.AF_INET or family == socket.AF_INET6 625 | sock = socket.socket(family, socket.SOCK_DGRAM, 0) 626 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 627 | sock.setblocking(False) 628 | sock.bind(host_port) 629 | fd = sock.fileno() 630 | 631 | def listen(): 632 | try: 633 | data, addr = sock.recvfrom(BUFFER_SIZE) 634 | asyncio.ensure_future(udp_server_handle(data, addr, core)) 635 | except (BlockingIOError, InterruptedError): 636 | pass 637 | except Exception: 638 | log.warning("UDP server exception", exc_info=True) 639 | # UDP server is not stream 640 | loop.add_reader(fd, listen) 641 | return sock 642 | 643 | 644 | def setup_all_socket_server(core: Core, s_family): 645 | # create new TCP socket server 646 | log.info(f"try to setup server socket {core.host}:{V.P2P_PORT}") 647 | if V.P2P_ACCEPT: 648 | V.P2P_ACCEPT = False 649 | for res in socket.getaddrinfo(core.host, V.P2P_PORT, s_family, socket.SOCK_STREAM, 0, socket.AI_PASSIVE): 650 | af, sock_type, proto, canon_name, sa = res 651 | try: 652 | sock = create_tcp_server(core, af, sa) 653 | log.debug(f"success tcp server creation af={socket2name.get(af)}") 654 | tcp_servers.append(sock) 655 | V.P2P_ACCEPT = True 656 | except Exception: 657 | log.debug("create tcp server exception", exc_info=True) 658 | # create new UDP socket server 659 | if V.P2P_UDP_ACCEPT: 660 | V.P2P_UDP_ACCEPT = False 661 | for res in socket.getaddrinfo(core.host, V.P2P_PORT, s_family, socket.SOCK_DGRAM, 0, socket.AI_PASSIVE): 662 | af, sock_type, proto, canon_name, sa = res 663 | try: 664 | sock = create_udp_server(core, af, sa) 665 | log.debug(f"success udp server creation af={socket2name.get(af)}") 666 | udp_servers.append(sock) 667 | V.P2P_UDP_ACCEPT = True 668 | except Exception: 669 | log.debug("create udp server exception", exc_info=True) 670 | 671 | 672 | __all__ = [ 673 | "INBOUND", 674 | "OUTBOUND", 675 | "ban_address", 676 | "Core", 677 | ] 678 | -------------------------------------------------------------------------------- /p2p_python/serializer.py: -------------------------------------------------------------------------------- 1 | import msgpack 2 | 3 | 4 | def dump(obj, fp, default=None): 5 | msgpack.pack(obj, fp, use_bin_type=True, default=default) 6 | 7 | 8 | def dumps(obj, default=None): 9 | return msgpack.packb(obj, use_bin_type=True, default=default) 10 | 11 | 12 | def load(fp, object_hook=None): 13 | return msgpack.unpack(fp, object_hook=object_hook, encoding='utf8') 14 | 15 | 16 | def loads(b, object_hook=None): 17 | return msgpack.unpackb(b, object_hook=object_hook, encoding='utf8') 18 | 19 | 20 | def stream_unpacker(fp, object_hook=None): 21 | return msgpack.Unpacker(fp, object_hook=object_hook, encoding='utf8') 22 | 23 | 24 | __all__ = [ 25 | "dump", 26 | "dumps", 27 | "load", 28 | "loads", 29 | "stream_unpacker", 30 | ] 31 | -------------------------------------------------------------------------------- /p2p_python/server.py: -------------------------------------------------------------------------------- 1 | from p2p_python.tool.utils import * 2 | from p2p_python.tool.upnpc import * 3 | from p2p_python.config import V, Debug, PeerToPeerError 4 | from p2p_python.core import Core, ban_address 5 | from p2p_python.utils import is_reachable 6 | from p2p_python.user import User 7 | from p2p_python.serializer import * 8 | from expiringdict import ExpiringDict 9 | from time import time 10 | from logging import getLogger 11 | from typing import Dict, List, Set, Optional 12 | import asyncio 13 | import os.path 14 | import random 15 | import socket 16 | 17 | 18 | log = getLogger(__name__) 19 | loop = asyncio.get_event_loop() 20 | loop_futures: List[asyncio.Future] = list() 21 | LOCAL_IP = get_localhost_ip() 22 | GLOBAL_IPV4 = get_global_ip() 23 | GLOBAL_IPV6 = get_global_ip_ipv6() 24 | STICKY_LIMIT = 2 25 | TIMEOUT = 10.0 26 | 27 | # Constant type 28 | T_REQUEST = 'request' 29 | T_RESPONSE = 'response' 30 | T_ACK = 'ack' 31 | 32 | # stabilize objects 33 | user_score: Dict[tuple, int] = dict() 34 | sticky_peers: Set[tuple] = set() 35 | ignore_peers = set() 36 | 37 | 38 | class Peer2PeerCmd: 39 | # ノード間で内部的に用いるコマンド 40 | PING_PONG = 'ping-pong' # ping-pong 41 | BROADCAST = 'broadcast' # 全ノードに伝播 42 | GET_PEER_INFO = 'get-peer-info' # 隣接ノードの情報を取得 43 | GET_NEARS = 'get-nears' # ピアリストを取得 44 | CHECK_REACHABLE = 'check-reachable' # 外部からServerに到達できるかチェック 45 | DIRECT_CMD = 'direct-cmd' # 隣接ノードに直接CMDを打つ 46 | 47 | 48 | class Peer2Peer(object): 49 | 50 | def __init__(self, listen=15, f_local=False, default_hook=None, object_hook=None): 51 | assert V.DATA_PATH is not None, 'Setup p2p params before PeerClientClass init.' 52 | 53 | # object control params 54 | self.f_stop = False 55 | self.f_finish = False 56 | self.f_running = False 57 | 58 | # co-objects 59 | self.core = Core(host='localhost' if f_local else None, listen=listen) 60 | self.peers = PeerData(os.path.join(V.DATA_PATH, 'peer.dat')) # {(host, port): header,..} 61 | self.event = EventIgnition() # DirectCmdを受け付ける窓口 62 | 63 | # data status control 64 | self.broadcast_status: Dict[int, asyncio.Future] = ExpiringDict(max_len=5000, max_age_seconds=90) 65 | self.result_futures: Dict[int, asyncio.Future] = ExpiringDict(max_len=5000, max_age_seconds=90) 66 | 67 | # recode traffic if f_debug true 68 | if Debug.F_RECODE_TRAFFIC: 69 | self.core.traffic.recode_dir = V.DATA_PATH 70 | 71 | # serializer/deserializer hook 72 | self.default_hook = default_hook 73 | self.object_hook = object_hook 74 | 75 | def close(self): 76 | self.f_stop = True 77 | self.core.close() 78 | 79 | def setup(self, s_family=socket.AF_UNSPEC, f_stabilize=True): 80 | async def inner_loop(): 81 | log.info("start P2P inner loop") 82 | while not self.f_stop: 83 | try: 84 | user, msg_body, push_time = await asyncio.wait_for(self.core.core_que.get(), 1.0) 85 | item = loads(b=msg_body, object_hook=self.object_hook) 86 | except asyncio.TimeoutError: 87 | continue 88 | except Exception: 89 | log.debug(f"core que getting exception", exc_info=True) 90 | continue 91 | 92 | if Debug.P_SEND_RECEIVE_DETAIL: 93 | log.debug(f"receive {int(time()-push_time)}s => {item}") 94 | 95 | try: 96 | if not isinstance(item, dict): 97 | log.debug("unrecognized message receive") 98 | elif item['type'] == T_REQUEST: 99 | # process request asynchronously 100 | asyncio.ensure_future(self.type_request(user, item, push_time)) 101 | elif item['type'] == T_RESPONSE: 102 | await self.type_response(user, item) 103 | user.header.update_last_seen() 104 | elif item['type'] == T_ACK: 105 | await self.type_ack(user, item) 106 | else: 107 | log.debug(f"unknown type={item['type']}") 108 | except asyncio.TimeoutError: 109 | log.warning(f"timeout on broadcast and cancel task") 110 | broadcast_task = None 111 | except Exception: 112 | log.debug(f"core que processing exception of {user}", exc_info=True) 113 | self.f_finish = True 114 | self.f_running = False 115 | log.info("close inner_loop process") 116 | 117 | assert not loop.is_running(), "setup before event loop start!" 118 | self.core.start(s_family=s_family) 119 | if f_stabilize: 120 | loop_futures.append(asyncio.ensure_future(auto_stabilize_network(self))) 121 | # Processing 122 | loop_futures.append(asyncio.ensure_future(inner_loop())) 123 | log.info(f"start user, name={V.SERVER_NAME} port={V.P2P_PORT}") 124 | self.f_running = True 125 | 126 | async def type_request(self, user: User, item: dict, push_time: float): 127 | temperate = { 128 | 'type': T_RESPONSE, 129 | 'cmd': item['cmd'], 130 | 'data': None, 131 | 'time': None, 132 | 'received': push_time, 133 | 'uuid': item['uuid'] 134 | } 135 | allows: List[User] = list() 136 | denys: List[User] = list() 137 | ack_list: List[User] = list() 138 | ack_status: Optional[bool] = None 139 | allow_udp = False 140 | 141 | if item['cmd'] == Peer2PeerCmd.PING_PONG: 142 | temperate['data'] = { 143 | 'ping': item['data'], 144 | 'pong': time(), 145 | } 146 | allows.append(user) 147 | 148 | elif item['cmd'] == Peer2PeerCmd.BROADCAST: 149 | if item['uuid'] in self.broadcast_status: 150 | # already get broadcast data, only send ACK 151 | future = self.broadcast_status[item['uuid']] 152 | # send ACK after broadcast_check finish 153 | await asyncio.wait_for(future, TIMEOUT) 154 | ack_status = future.result() 155 | ack_list.append(user) 156 | 157 | elif item['uuid'] in self.result_futures: 158 | # I'm broadcaster, get from ack 159 | ack_status = True 160 | ack_list.append(user) 161 | else: 162 | # set future 163 | future = asyncio.Future() 164 | self.broadcast_status[item['uuid']] = future 165 | # try to check broadcast data 166 | if asyncio.iscoroutinefunction(self.broadcast_check): 167 | broadcast_result = await asyncio.wait_for( 168 | self.broadcast_check(user, item['data']), TIMEOUT) 169 | else: 170 | broadcast_result = self.broadcast_check(user, item['data']) 171 | # set broadcast result 172 | future.set_result(broadcast_result) 173 | # prepare response 174 | if broadcast_result: 175 | user.score += 1 176 | # send ACK 177 | ack_status = True 178 | ack_list.append(user) 179 | # broadcast to all 180 | allows = self.core.user.copy() 181 | denys.append(user) 182 | temperate['type'] = T_REQUEST 183 | temperate['data'] = item['data'] 184 | allow_udp = True 185 | else: 186 | user.warn += 1 187 | # send ACK 188 | ack_status = False 189 | ack_list.append(user) 190 | 191 | elif item['cmd'] == Peer2PeerCmd.GET_PEER_INFO: 192 | # [[(host,port), header],..] 193 | temperate['data'] = [(host_port, header.getinfo()) for host_port, header in self.peers.copy().items()] 194 | allows.append(user) 195 | 196 | elif item['cmd'] == Peer2PeerCmd.GET_NEARS: 197 | # [[(host,port), header],..] 198 | temperate['data'] = [(user.get_host_port(), user.header.getinfo()) for user in self.core.user] 199 | allows.append(user) 200 | 201 | elif item['cmd'] == Peer2PeerCmd.CHECK_REACHABLE: 202 | try: 203 | port = item['data']['port'] 204 | except Exception: 205 | port = user.header.p2p_port 206 | try: 207 | temperate['data'] = await asyncio.wait_for( 208 | is_reachable(host=user.host_port[0], port=port), TIMEOUT) 209 | except Exception: 210 | temperate['data'] = False 211 | allows.append(user) 212 | 213 | elif item['cmd'] == Peer2PeerCmd.DIRECT_CMD: 214 | data = item['data'] 215 | if self.event.have_event(data['cmd']): 216 | allows.append(user) 217 | temperate['data'] = await asyncio.wait_for( 218 | self.event.ignition(user, data['cmd'], data['data']), TIMEOUT) 219 | else: 220 | log.debug(f"not found request cmd '{item['cmd']}'") 221 | 222 | # send message 223 | temperate['time'] = time() 224 | send_count = await self._send_many_users(item=temperate, allows=allows, denys=denys, allow_udp=allow_udp) 225 | # send ack 226 | ack_count = 0 227 | if len(ack_list) > 0: 228 | assert ack_status is not None 229 | ack_temperate = temperate.copy() 230 | ack_temperate['type'] = T_ACK 231 | ack_temperate['data'] = ack_status 232 | ack_count = await self._send_many_users(item=ack_temperate, allows=ack_list, denys=[]) 233 | # debug 234 | if Debug.P_SEND_RECEIVE_DETAIL: 235 | log.debug(f"reply => {temperate}") 236 | log.debug(f"status => all={len(self.core.user)} send={send_count} ack={ack_count}") 237 | 238 | async def type_response(self, user: User, item: dict): 239 | # cmd = item['cmd'] 240 | data = item['data'] 241 | uuid = item['uuid'] 242 | if uuid in self.result_futures: 243 | future = self.result_futures[uuid] 244 | if not future.done(): 245 | future.set_result((user, data)) 246 | elif future.cancelled(): 247 | log.debug(f"uuid={uuid} type_response failed, already future canceled") 248 | else: 249 | pass 250 | else: 251 | log.debug(f"uuid={uuid} type_response failed, not found uuid") 252 | 253 | async def type_ack(self, user: User, item: dict): 254 | # cmd = item['cmd'] 255 | ack_status = bool(item['data']) 256 | uuid = item['uuid'] 257 | 258 | if uuid in self.result_futures: 259 | future = self.result_futures[uuid] 260 | if ack_status and not future.done(): 261 | future.set_result((user, ack_status)) 262 | 263 | async def _send_many_users(self, item, allows: List[User], denys: List[User], allow_udp=False) -> int: 264 | """send to many user and return how many send""" 265 | msg_body = dumps(obj=item, default=self.default_hook) 266 | count = 0 267 | for user in allows: 268 | if user not in denys: 269 | try: 270 | await self.core.send_msg_body(msg_body=msg_body, user=user, allow_udp=allow_udp) 271 | count += 1 272 | except Exception as e: 273 | user.warn += 1 274 | log.debug(f"failed send msg to {user} by {str(e)}") 275 | return count 276 | 277 | async def send_command(self, cmd, data=None, user=None, timeout=10.0, retry=2) -> (User, dict): 278 | assert 0.0 < timeout and 0 < retry 279 | 280 | if self.f_stop: 281 | raise PeerToPeerError('already p2p-python closed') 282 | 283 | uuid = random.randint(10, 0xffffffff) 284 | # 1. Make template 285 | temperate = { 286 | 'type': T_REQUEST, 287 | 'cmd': cmd, 288 | 'data': data, 289 | 'time': time(), 290 | 'uuid': uuid, 291 | } 292 | f_udp = False 293 | 294 | # 2. Setup allows to send nodes 295 | if len(self.core.user) == 0: 296 | raise PeerToPeerError('no client connection found') 297 | elif cmd == Peer2PeerCmd.BROADCAST: 298 | allows = self.core.user.copy() 299 | f_udp = True 300 | elif user is None: 301 | user = random.choice(self.core.user) 302 | allows = [user] 303 | elif user in self.core.user: 304 | allows = [user] 305 | else: 306 | raise PeerToPeerError("Not found user in list") 307 | 308 | # 3. Send message to a node or some nodes 309 | start = time() 310 | future = asyncio.Future() 311 | self.result_futures[uuid] = future 312 | 313 | # get best timeout 314 | if user is None: 315 | # broadcast-cmd 316 | best_timeout = timeout / retry 317 | else: 318 | # inner-cmd/direct-cmd 319 | average = user.average_process_time() 320 | if average is None: 321 | best_timeout = timeout / retry 322 | else: 323 | best_timeout = min(5.0, max(1.0, average * 10)) 324 | 325 | f_timeout = False 326 | for _ in range(retry): 327 | send_num = await self._send_many_users(item=temperate, allows=allows, denys=[], allow_udp=f_udp) 328 | send_time = time() 329 | if send_num == 0: 330 | raise PeerToPeerError(f"We try to send no users? {len(self.core.user)}user connected") 331 | if Debug.P_SEND_RECEIVE_DETAIL: 332 | log.debug(f"send({send_num}) => {temperate}") 333 | 334 | # 4. Get response 335 | try: 336 | # avoid future canceled by wait_for 337 | await asyncio.wait_for(asyncio.shield(future), best_timeout) 338 | if 5.0 < time() - start: 339 | log.debug(f"id={uuid}, command {int(time()-start)}s blocked by {user}") 340 | if user is not None: 341 | user.process_time.append(time() - send_time) 342 | break 343 | except (asyncio.TimeoutError, asyncio.CancelledError): 344 | log.debug(f"id={uuid}, timeout now, cmd({cmd}) to {user}") 345 | except Exception: 346 | log.debug("send_command exception", exc_info=True) 347 | 348 | # 5. will lost packet 349 | log.debug(f"id={uuid}, will lost packet and retry") 350 | 351 | else: 352 | f_timeout = True 353 | 354 | # 6. timeout 355 | if f_timeout and user: 356 | if user.closed or not await self.core.ping(user): 357 | # already closed or ping failed -> reconnect 358 | await self.core.try_reconnect(user, reason="ping failed on send_command") 359 | else: 360 | log.debug("timeout and retry but ping success") 361 | 362 | # 7. return result 363 | if future.done(): 364 | return future.result() 365 | else: 366 | future.cancel() 367 | raise asyncio.TimeoutError("timeout cmd") 368 | 369 | async def send_direct_cmd(self, cmd, data, user=None) -> (User, dict): 370 | if len(self.core.user) == 0: 371 | raise PeerToPeerError('not found peers') 372 | if callable(cmd): 373 | cmd = cmd.__name__ 374 | assert isinstance(cmd, str) 375 | user = user if user else random.choice(self.core.user) 376 | send_data = {'cmd': cmd, 'data': data} 377 | receive_user, item = await self.send_command(Peer2PeerCmd.DIRECT_CMD, send_data, user) 378 | if user != receive_user: 379 | log.warning(f"do not match sender and receiver {user} != {receive_user}") 380 | return user, item 381 | 382 | @staticmethod 383 | def broadcast_check(user: User, data): 384 | """return true if spread to all connections""" 385 | return False # overwrite 386 | 387 | 388 | async def auto_stabilize_network( 389 | p2p: Peer2Peer, 390 | auto_reset_sticky=True, 391 | self_disconnect=False, 392 | ): 393 | """ 394 | automatic stabilize p2p network 395 | params: 396 | auto_reset_sticky: (bool) You know connections but can not connect again and again, 397 | stabilizer mark "sticky" and ignore forever. This flag enable auto reset the mark. 398 | self_disconnect: (bool) stabilizer keep a number of connection same with listen/2. 399 | self disconnection avoid overflow backlog but will make unstable network. 400 | """ 401 | # update ignore peers 402 | ignore_peers.update({ 403 | # ipv4 404 | (GLOBAL_IPV4, V.P2P_PORT), 405 | (LOCAL_IP, V.P2P_PORT), 406 | ('127.0.0.1', V.P2P_PORT), 407 | # ipv6 408 | (GLOBAL_IPV6, V.P2P_PORT, 0, 0), 409 | ('::1', V.P2P_PORT, 0, 0), 410 | }) 411 | 412 | # wait for P2P running 413 | while not p2p.f_running: 414 | await asyncio.sleep(1.0) 415 | log.info(f"start auto stabilize loop known={len(p2p.peers)}") 416 | 417 | # show info 418 | if len(p2p.peers) == 0: 419 | log.info("peer list is zero, need bootnode") 420 | 421 | # start stabilize connection 422 | count = 0 423 | need_connection = 3 424 | while p2p.f_running: 425 | count += 1 426 | 427 | # decide wait time 428 | if len(p2p.core.user) <= need_connection: 429 | wait_time = 3.0 430 | else: 431 | wait_time = 4.0 * (4.5 + random.random()) # wait 18s~20s 432 | 433 | # waiting 434 | while p2p.f_running and 0.0 < wait_time: 435 | await asyncio.sleep(0.1) 436 | wait_time -= 0.1 437 | 438 | # clear sticky 439 | if count % 13 == 0 and len(sticky_peers) > 0: 440 | if auto_reset_sticky: 441 | log.debug(f"clean sticky_nodes num={len(sticky_peers)}") 442 | sticky_peers.clear() 443 | 444 | try: 445 | # no connection and try to connect from peer list 446 | if 0 == len(p2p.core.user): 447 | if 0 < len(p2p.peers): 448 | peers = list(p2p.peers.keys()) 449 | while 0 < len(peers) and len(p2p.core.user) < need_connection: 450 | host_port = peers.pop() 451 | if host_port in ignore_peers: 452 | p2p.peers.remove_from_memory(host_port) 453 | elif await p2p.core.create_connection(host_port[0], host_port[1]): 454 | pass 455 | else: 456 | sticky_peers.add(host_port) 457 | log.info(f"init connection num={len(p2p.core.user)}") 458 | # wait when disconnected from network 459 | if len(p2p.core.user) == 0: 460 | wait_time = 15.0 461 | else: 462 | log.info("no peer info and no connections, wait 5s") 463 | wait_time = 5.0 464 | 465 | # waiting if required 466 | while p2p.f_running and 0.0 < wait_time: 467 | await asyncio.sleep(0.1) 468 | wait_time -= 0.1 469 | 470 | # update 1 user's neer info one by one 471 | if 0 < len(p2p.core.user): 472 | update_user = p2p.core.user[count % len(p2p.core.user)] 473 | _, item = await p2p.send_command(cmd=Peer2PeerCmd.GET_NEARS, user=update_user) 474 | update_user.update_neers(item) 475 | # peer list update 476 | p2p.peers.add(update_user) 477 | 478 | # Calculate score (高ければ優先度が高い) 479 | search = set(p2p.peers.keys()) 480 | for user in p2p.core.user: 481 | for host_port in user.neers.keys(): 482 | search.add(host_port) 483 | search.difference_update(ignore_peers) 484 | search.difference_update(sticky_peers) 485 | for host_port in search: # 第一・二層を含む 486 | score = 0 487 | score += sum(1 for user in p2p.core.user if host_port in user.neers) # 第二層は加点 488 | score -= sum(1 for user in p2p.core.user if host_port == user.get_host_port()) # 第一層は減点 489 | user_score[host_port] = max(-20, min(20, score)) 490 | if len(user_score) == 0: 491 | continue 492 | 493 | # Action join or remove or nothing 494 | if len(p2p.core.user) > p2p.core.backlog * 2 // 3: # Remove 495 | if not self_disconnect: 496 | continue 497 | # スコアの下位半分を取得 498 | sorted_score = sorted(user_score.items(), key=lambda x: x[1])[:len(user_score) // 3] 499 | # 既接続のもののみを取得 500 | already_connected = tuple(user.get_host_port() for user in p2p.core.user) 501 | sorted_score = list(filter(lambda x: x[0] in already_connected, sorted_score)) 502 | if len(sorted_score) == 0: 503 | continue 504 | log.debug(f"try to remove score {sorted_score}") 505 | host_port, score = random.choice(sorted_score) 506 | user = p2p.core.host_port2user(host_port) 507 | if user is None: 508 | pass # 既接続でない 509 | elif len(user.neers) < need_connection: 510 | pass # 接続数が少なすぎるノード 511 | elif p2p.core.remove_connection(user, 'low score user'): 512 | log.debug(f"remove connection {score} {user} {host_port}") 513 | else: 514 | log.warning("failed remove connection. Already disconnected?") 515 | sticky_peers.add(host_port) 516 | if p2p.peers.remove_from_memory(host_port): 517 | del user_score[host_port] 518 | 519 | elif len(p2p.core.user) < p2p.core.backlog * 2 // 3: # Join 520 | # スコア上位半分を取得 521 | sorted_score = sorted( 522 | user_score.items(), key=lambda x: x[1], reverse=True)[:len(user_score) // 3] 523 | # 既接続を除く 524 | already_connected = tuple(user.get_host_port() for user in p2p.core.user) 525 | sorted_score = list(filter(lambda x: 526 | x[0] not in already_connected and 527 | x[0] not in sticky_peers, 528 | sorted_score)) 529 | if len(sorted_score) == 0: 530 | continue 531 | log.debug(f"join score {sorted_score}") 532 | host_port, score = random.choice(sorted_score) 533 | if p2p.core.host_port2user(host_port): 534 | continue # 既に接続済み 535 | elif host_port in sticky_peers: 536 | continue # 接続不能回数大杉 537 | elif host_port in ignore_peers: 538 | p2p.peers.remove_from_memory(host_port) 539 | continue 540 | elif host_port[0] in ban_address: 541 | continue # BAN address 542 | elif await p2p.core.create_connection(host=host_port[0], port=host_port[1]): 543 | log.debug(f"new connection {host_port}") 544 | else: 545 | log.debug(f"failed connect try, remove {host_port}") 546 | sticky_peers.add(host_port) 547 | if p2p.peers.remove_from_memory(host_port): 548 | del user_score[host_port] 549 | else: 550 | pass 551 | 552 | except asyncio.TimeoutError as e: 553 | log.info(f"stabilize {str(e)}") 554 | except PeerToPeerError as e: 555 | log.debug(f"Peer2PeerError: {str(e)}") 556 | except Exception: 557 | log.debug("stabilize exception", exc_info=True) 558 | log.info("auto stabilize closed") 559 | 560 | 561 | __all__ = [ 562 | "LOCAL_IP", 563 | "GLOBAL_IPV4", 564 | "GLOBAL_IPV6", 565 | "T_REQUEST", 566 | "T_RESPONSE", 567 | "T_ACK", 568 | "Peer2PeerCmd", 569 | "Peer2Peer", 570 | "auto_stabilize_network", 571 | "user_score", 572 | "sticky_peers", 573 | ] 574 | -------------------------------------------------------------------------------- /p2p_python/tool/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/namuyan/p2p-python/d67982f10d7e2506fbdacce933ce8ce3c8771909/p2p_python/tool/__init__.py -------------------------------------------------------------------------------- /p2p_python/tool/traffic.py: -------------------------------------------------------------------------------- 1 | from logging import getLogger 2 | import collections 3 | import os.path 4 | import asyncio 5 | import time 6 | 7 | loop = asyncio.get_event_loop() 8 | log = getLogger(__name__) 9 | 10 | 11 | class Traffic(object): 12 | f_stop = False 13 | f_finish = False 14 | 15 | def __init__(self, recode_dir=None, span=300, max_hours=24): 16 | self.data = collections.deque(maxlen=int(3600 * max_hours // span)) 17 | self.recode_dir = recode_dir if recode_dir and os.path.exists(recode_dir) else None 18 | self.span = span # 5min 19 | self.traffic_up = list() 20 | self.traffic_down = list() 21 | self._future = asyncio.ensure_future(self.loop()) 22 | 23 | def close(self): 24 | self.f_stop = True 25 | self._future.cancel() 26 | log.debug("traffic recoder close") 27 | 28 | async def loop(self): 29 | count = 0 30 | while True: 31 | try: 32 | wait_time = self.span 33 | while not self.f_stop and 0.0 < wait_time: 34 | await asyncio.sleep(0.5) 35 | wait_time -= 0.5 36 | 37 | count += 1 38 | ntime, up, down = int(time.time()), sum(self.traffic_up), sum(self.traffic_down) 39 | self.data.append((ntime, up, down)) 40 | self.traffic_up.clear() 41 | self.traffic_down.clear() 42 | # recode 43 | if self.recode_dir is None: 44 | continue 45 | date = time.strftime('%y-%m-%d') 46 | recode_path = os.path.join(self.recode_dir, 'traffic.%s.csv' % date) 47 | 48 | f_first = os.path.exists(recode_path) 49 | with open(recode_path, mode='a') as f: 50 | if not f_first: 51 | f.write("unix time,date,up (kb),down (kb)\n") 52 | f.write("{},{},{},{}\n".format(ntime, time.strftime('%Hh%Mm', time.gmtime(ntime)), 53 | round(up / 1000, 3), round(down / 1000, 3))) 54 | except asyncio.CancelledError: 55 | break 56 | except Exception as e: 57 | log.debug(e) 58 | self.f_finish = True 59 | 60 | def put_traffic_up(self, b): 61 | self.traffic_up.append(len(b)) 62 | 63 | def put_traffic_down(self, b): 64 | self.traffic_down.append(len(b)) 65 | 66 | 67 | __all__ = [ 68 | "Traffic", 69 | ] 70 | -------------------------------------------------------------------------------- /p2p_python/tool/upnpc.py: -------------------------------------------------------------------------------- 1 | from urllib.request import Request, urlopen 2 | from urllib.parse import urlparse 3 | from xml.etree import ElementTree 4 | from collections import namedtuple 5 | from logging import getLogger 6 | from typing import Optional, List 7 | import xmltodict 8 | import requests 9 | import socket 10 | import random 11 | 12 | 13 | log = getLogger(__name__) 14 | NAME_SERVER = '8.8.8.8' 15 | Mapping = namedtuple('Mapping', [ 16 | 'enabled', # int: 1 17 | 'external_port', # int: 38008 18 | 'external_client', # str: '192.168.1.1' 19 | 'internal_port', # int: 38008 20 | 'lease_duration', # int: 0 21 | 'description', # str: 'Apple' 22 | 'protocol', # str: 'TCP' 23 | 'remote_host', # None 24 | ]) 25 | 26 | """how to use example 27 | request_url = cast_rooter_request() 28 | soap_url = get_soap_url(request_url) 29 | print(soap_get_mapping(soap_url)) 30 | internal_client = get_localhost_ip() 31 | soap_add_mapping(soap_url, 5000, 5000, internal_client) 32 | print(soap_get_mapping(soap_url)) 33 | soap_delete_mapping(soap_url, 5000) 34 | """ 35 | 36 | 37 | def check_and_open_port_by_upnp(external_port, internal_port, protocol): 38 | """open the router's port to enable external connection""" 39 | request_url = cast_rooter_request() 40 | if request_url is None: 41 | log.debug("node is not in local network protected by a router") 42 | return 43 | soap_url = get_soap_url(request_url) 44 | internal_client = get_localhost_ip() 45 | # check existence 46 | for mapping in soap_get_mapping(soap_url): 47 | if mapping.enabled == 1 and \ 48 | mapping.external_port == external_port and \ 49 | mapping.internal_port == internal_port and \ 50 | mapping.protocol == protocol and \ 51 | mapping.external_client == internal_client: 52 | return 53 | # open port 54 | soap_add_mapping(soap_url, external_port, internal_port, internal_client, protocol) 55 | log.info(f"open port by upnp {internal_port} -> {external_port}") 56 | 57 | 58 | def cast_rooter_request(host='239.255.255.250', port=1900) -> Optional[str]: 59 | try: 60 | messages = [ 61 | 'M-SEARCH * HTTP/1.1', 62 | 'MX: 3', 63 | 'HOST: 239.255.255.250:1900', 64 | 'MAN: "ssdp:discover"', 65 | 'ST: urn:schemas-upnp-org:service:WANIPConnection:1', 66 | ] 67 | message = '\r\n'.join(messages) 68 | message += '\r\n\r\n' 69 | with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: 70 | sock.settimeout(5) 71 | sock.sendto(message.encode('utf8'), (host, port)) 72 | res = sock.recv(1024) 73 | res_list = res.decode().replace("\r\n", "\n").split("\n") 74 | for e in res_list: 75 | if 'LOCATION: ' in e: 76 | return e[10:] 77 | except socket.timeout: 78 | pass 79 | except Exception: 80 | log.debug("cast_rooter_request exception", exc_info=True) 81 | return None 82 | 83 | 84 | def get_soap_url(request_url) -> Optional[str]: 85 | """get soap url""" 86 | try: 87 | xml_string = requests.get(url=request_url).text 88 | xml = ElementTree.fromstring(xml_string) 89 | ns = {'ns': 'urn:schemas-upnp-org:device-1-0'} 90 | for child in xml.findall(".//ns:service", ns): 91 | if child.find('ns:serviceType', ns).text == 'urn:schemas-upnp-org:service:WANIPConnection:1': 92 | control_url = child.find('ns:controlURL', ns).text 93 | parse = urlparse(request_url) 94 | return "{0}://{1}/{2}".format(parse.scheme, parse.netloc, control_url) 95 | except Exception: 96 | log.debug("get_soap_url exception", exc_info=True) 97 | return None 98 | 99 | 100 | def soap_get_mapping(soap_url) -> List[Mapping]: 101 | """get upnp mapping""" 102 | i_d = 0 103 | ports = list() 104 | while True: 105 | soap = '\r\n' 106 | soap += '\r\n' 108 | soap += '\r\n' 109 | soap += '\r\n' 110 | soap += '' + str(i_d) + '\r\n' 111 | soap += '\r\n' 112 | soap += '\r\n' 113 | soap += '\r\n' 114 | 115 | try: 116 | req = Request(soap_url) 117 | req.add_header('Content-Type', 'text/xml; charset="utf-8"') 118 | req.add_header('SOAPACTION', 119 | '"urn:schemas-upnp-org:service:WANPPPConnection:1#GetGenericPortMappingEntry"') 120 | req.data = soap.encode('utf8') 121 | 122 | result = xmltodict.parse(urlopen(req).read().decode()) 123 | data = dict(result['s:Envelope']['s:Body']['u:GetGenericPortMappingEntryResponse']) 124 | ports.append(Mapping( 125 | int(data['NewEnabled']), 126 | int(data['NewExternalPort']), 127 | data['NewInternalClient'], 128 | int(data['NewInternalPort']), 129 | int(data['NewLeaseDuration']), 130 | data['NewPortMappingDescription'], 131 | data['NewProtocol'], 132 | data['NewRemoteHost'], 133 | )) 134 | except Exception as e: 135 | if '500' not in str(e): 136 | log.debug("soap_get_mapping exception", exc_info=True) 137 | break 138 | i_d += 1 139 | return ports 140 | 141 | 142 | def soap_add_mapping(soap_url, external_port, internal_port, internal_client, 143 | protocol='TCP', duration=0, description='') -> Optional[Mapping]: 144 | """ 145 | add to upnp mapping 146 | add_setting: 147 | external_port: WAN側のポート番号 148 | internal_port: 転送先ホストのポート番号 149 | internal_client: 転送先ホストのIPアドレス 150 | protocol: 'TCP' or 'UDP' 151 | duration: 設定の有効期間(秒)。0のときは無期限 152 | description: 'test' 153 | """ 154 | soap = '\r\n' 155 | soap += '\r\n' 157 | soap += '\r\n' 158 | soap += '\r\n' 159 | soap += '\r\n' 160 | soap += '' + str(external_port) + '\r\n' 161 | soap += '' + protocol + '\r\n' 162 | soap += '' + str(internal_port) + '\r\n' 163 | soap += '' + internal_client + '\r\n' 164 | soap += '1\r\n' 165 | soap += '' + str(description) + '\r\n' 166 | soap += '' + str(duration) + '\r\n' 167 | soap += '\r\n' 168 | soap += '\r\n' 169 | soap += '\r\n' 170 | 171 | try: 172 | req = Request(soap_url) 173 | req.add_header('Content-Type', 'text/xml; charset="utf-8"') 174 | req.add_header('SOAPACTION', '"urn:schemas-upnp-org:service:WANPPPConnection:1#AddPortMapping"') 175 | req.data = soap.encode('utf8') 176 | 177 | result = xmltodict.parse(urlopen(req).read().decode()) 178 | if "@xmlns:u" in result["s:Envelope"]["s:Body"]["u:AddPortMappingResponse"]: 179 | return Mapping(1, external_port, internal_client, internal_port, duration, description, protocol, None) 180 | except Exception: 181 | log.error("soap_add_mapping exception", exc_info=True) 182 | return None 183 | 184 | 185 | def soap_delete_mapping(soap_url, external_port, protocol='TCP') -> bool: 186 | """ 187 | delete from upnp mapping 188 | external_port: WAN側のポート番号 189 | protocol: 'TCP' or 'UDP' 190 | """ 191 | soap = '\r\n' 192 | soap += '\r\n' 194 | soap += '\r\n' 195 | soap += '\r\n' 196 | soap += '\r\n' 197 | soap += '' + str(external_port) + '\r\n' 198 | soap += '' + protocol + '\r\n' 199 | soap += '\r\n' 200 | soap += '\r\n' 201 | soap += '\r\n' 202 | 203 | try: 204 | req = Request(soap_url) 205 | req.add_header('Content-Type', 'text/xml; charset="utf-8"') 206 | req.add_header('SOAPACTION', '"urn:schemas-upnp-org:service:WANPPPConnection:1#DeletePortMapping"') 207 | req.data = soap.encode('utf8') 208 | 209 | result = xmltodict.parse(urlopen(req).read().decode()) 210 | if "@xmlns:u" in result["s:Envelope"]["s:Body"]["u:DeletePortMappingResponse"]: 211 | return True 212 | except Exception: 213 | log.error("soap_delete_mapping exception", exc_info=True) 214 | return False 215 | 216 | 217 | def get_external_ip(soap_url) -> str: 218 | """get external ip address""" 219 | s_o_a_p = '\r\n' 220 | s_o_a_p += '\r\n' 222 | s_o_a_p += '\r\n' 223 | s_o_a_p += '\r\n' 224 | s_o_a_p += '\r\n' 225 | s_o_a_p += '\r\n' 226 | s_o_a_p += '\r\n' 227 | 228 | try: 229 | req = Request(soap_url) 230 | req.add_header('Content-Type', 'text/xml; charset="utf-8"') 231 | req.add_header('SOAPACTION', '"urn:schemas-upnp-org:service:WANPPPConnection:1#GetExternalIPAddress"') 232 | req.data = s_o_a_p.encode('utf8') 233 | result = xmltodict.parse(urlopen(req).read().decode()) 234 | return result['s:Envelope']['s:Body']['u:GetExternalIPAddressResponse']['NewExternalIPAddress'] 235 | except Exception: 236 | log.debug("get_external_ip exception", exc_info=True) 237 | 238 | 239 | def get_localhost_ip(): 240 | """get local ip address""" 241 | try: 242 | return [ 243 | (s.connect((NAME_SERVER, 80)), s.getsockname()[0], s.close()) 244 | for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)] 245 | ][0][1] 246 | except Exception: 247 | return '127.0.0.1' 248 | 249 | 250 | def get_global_ip(): 251 | """get global ip address""" 252 | network_info_providers = [ 253 | 'http://api.ipify.org/', 254 | 'http://myip.dnsomatic.com', 255 | 'http://inet-ip.info/ip', 256 | 'http://v4.ident.me/', 257 | ] 258 | random.shuffle(network_info_providers) 259 | for url in network_info_providers: 260 | try: 261 | return requests.get(url).text.lstrip().rstrip() 262 | except Exception: 263 | continue 264 | else: 265 | log.info('cannot find global ip') 266 | return "" 267 | 268 | 269 | def get_global_ip_ipv6(): 270 | """get global ipv6 address""" 271 | network_info_providers = [ 272 | 'http://v6.ipv6-test.com/api/myip.php', 273 | 'http://v6.ident.me/', 274 | ] 275 | random.shuffle(network_info_providers) 276 | for url in network_info_providers: 277 | try: 278 | return requests.get(url).text.lstrip().rstrip() 279 | except Exception: 280 | continue 281 | else: 282 | log.info('cannot find global ipv6 ip') 283 | return "" 284 | 285 | 286 | __all__ = [ 287 | "NAME_SERVER", 288 | "Mapping", 289 | "check_and_open_port_by_upnp", 290 | "cast_rooter_request", 291 | "get_soap_url", 292 | "soap_get_mapping", 293 | "soap_add_mapping", 294 | "soap_delete_mapping", 295 | "get_external_ip", 296 | "get_localhost_ip", 297 | "get_global_ip", 298 | "get_global_ip_ipv6", 299 | ] 300 | -------------------------------------------------------------------------------- /p2p_python/tool/utils.py: -------------------------------------------------------------------------------- 1 | from p2p_python.serializer import stream_unpacker, dump 2 | from p2p_python.user import UserHeader, User 3 | from logging import getLogger 4 | from typing import Dict, Optional 5 | from time import time 6 | import asyncio 7 | import os 8 | 9 | # For AES 10 | from Cryptodome.Cipher import AES 11 | from Cryptodome.Util.Padding import pad, unpad 12 | from Cryptodome import Random 13 | from base64 import b64encode, b64decode 14 | 15 | loop = asyncio.get_event_loop() 16 | log = getLogger(__name__) 17 | 18 | 19 | class EventIgnition(object): 20 | 21 | def __init__(self): 22 | self.event = dict() 23 | 24 | def setup_events_from_class(self, klass): 25 | for cmd, fnc in klass.__dict__.items(): 26 | if cmd.startswith('_'): 27 | continue 28 | if isinstance(fnc, staticmethod): 29 | fnc = fnc.__func__ 30 | self.add_event(cmd, fnc) 31 | 32 | def add_event(self, cmd, fnc): 33 | assert fnc.__code__.co_argcount == 2 34 | if cmd in self.event: 35 | raise Exception('already registered cmd') 36 | self.event[cmd] = fnc 37 | log.info(f"add DirectCmd event '{cmd}'") 38 | 39 | def remove_event(self, cmd): 40 | if cmd in self.event: 41 | del self.event[cmd] 42 | 43 | def have_event(self, cmd): 44 | return cmd in self.event 45 | 46 | async def ignition(self, user, cmd, data): 47 | if cmd in self.event: 48 | fnc = self.event[cmd] 49 | if asyncio.iscoroutinefunction(fnc): 50 | return await fnc(user, data) 51 | else: 52 | return fnc(user, data) 53 | else: 54 | raise KeyError('Not found cmd "{}"'.format(cmd)) 55 | 56 | 57 | class AESCipher: 58 | 59 | @staticmethod 60 | def create_key(): 61 | return b64encode(os.urandom(AES.block_size)).decode() 62 | 63 | @staticmethod 64 | def is_aes_key(key): 65 | try: 66 | return len(b64decode(key.encode())) == AES.block_size 67 | except Exception as e: 68 | return False 69 | 70 | @staticmethod 71 | def encrypt(key, raw): 72 | assert type(raw) == bytes, "input data is bytes" 73 | if isinstance(key, str): 74 | key = b64decode(key.encode()) 75 | raw = pad(raw, AES.block_size) 76 | iv = Random.new().read(AES.block_size) 77 | cipher = AES.new(key, AES.MODE_CBC, iv) 78 | return iv + cipher.encrypt(raw) 79 | 80 | @staticmethod 81 | def decrypt(key, enc): 82 | assert type(enc) == bytes, 'Encrypt data is bytes' 83 | if isinstance(key, str): 84 | key = b64decode(key.encode()) 85 | iv = enc[:AES.block_size] 86 | cipher = AES.new(key, AES.MODE_CBC, iv) 87 | raw = cipher.decrypt(enc[AES.block_size:]) 88 | raw = unpad(raw, AES.block_size) 89 | if len(raw) == 0: 90 | raise ValueError("AES decryption error, not correct key.") 91 | return raw 92 | 93 | 94 | class PeerData(object): 95 | 96 | def __init__(self, path): 97 | """recode all node, don't remove""" 98 | self._peer: Dict[(str, int), UserHeader] = dict() # {(host, port): header,..} 99 | self.path = path 100 | self.init_cleanup() 101 | 102 | def get(self, host_port) -> Optional[UserHeader]: 103 | return self._peer.get(tuple(host_port)) 104 | 105 | def remove_from_memory(self, host_port): 106 | host_port = tuple(host_port) 107 | if host_port in self._peer: 108 | del self._peer[tuple(host_port)] 109 | return True 110 | return False 111 | 112 | def __contains__(self, item): 113 | return tuple(item) in self._peer 114 | 115 | def __len__(self): 116 | return len(self._peer) 117 | 118 | def keys(self): 119 | yield from self._peer.keys() 120 | 121 | def items(self): 122 | yield from self._peer.items() 123 | 124 | def copy(self): 125 | return self._peer.copy() 126 | 127 | def add(self, user: User): 128 | host_port = user.get_host_port() 129 | self._peer[host_port] = user.header 130 | with open(self.path, mode='ba') as fp: 131 | header = user.header.getinfo() 132 | dump((host_port, header), fp) 133 | 134 | def init_cleanup(self): 135 | time_limit = int(time() - 3600*24*30) 136 | try: 137 | with open(self.path, mode='br') as fp: 138 | for host_port, header_dict in stream_unpacker(fp): 139 | header = UserHeader(**header_dict) 140 | if time_limit < header.last_seen: 141 | self._peer[tuple(host_port)] = header 142 | # re recode 143 | with open(self.path, mode='bw') as fp: 144 | for host_port, header in self._peer.items(): 145 | dump((host_port, header.getinfo()), fp) 146 | except Exception: 147 | pass 148 | 149 | 150 | __all__ = [ 151 | "EventIgnition", 152 | "AESCipher", 153 | "PeerData", 154 | ] 155 | -------------------------------------------------------------------------------- /p2p_python/user.py: -------------------------------------------------------------------------------- 1 | from asyncio.streams import StreamReader, StreamWriter 2 | from logging import getLogger 3 | from time import time 4 | from typing import Dict 5 | from collections import deque 6 | import asyncio 7 | 8 | 9 | log = getLogger(__name__) 10 | 11 | 12 | class UserHeader(object): 13 | """user shared data on network""" 14 | __slots__ = ( 15 | "name", # (str) Name randomly chosen by name_list.txt 16 | "client_ver", # (str) __version__ of __init__.py 17 | "network_ver", # (int) Random number assigned to each P2P net 18 | "my_host_name", # (str) user's optional hostname (higher priority than peername) 19 | "p2p_accept", # (bool) flag of accept TCP connection 20 | "p2p_udp_accept", # (bool) flag of accept UDP packet 21 | "p2p_port", # (int) P2P port 22 | "start_time", # (int) start UNIX time 23 | "last_seen", # (int) last time we get socket data 24 | ) 25 | 26 | def __init__(self, **kwargs): 27 | self.name: str = kwargs['name'] 28 | self.client_ver: int = kwargs['client_ver'] 29 | self.network_ver: int = kwargs['network_ver'] 30 | self.my_host_name: str = kwargs.get('my_host_name') 31 | self.p2p_accept: bool = kwargs['p2p_accept'] 32 | self.p2p_udp_accept: bool = kwargs['p2p_udp_accept'] 33 | self.p2p_port: int = kwargs['p2p_port'] 34 | self.start_time: int = kwargs['start_time'] 35 | self.last_seen = kwargs.get('last_seen', int(time())) 36 | 37 | def __repr__(self): 38 | return f"" 39 | 40 | def getinfo(self): 41 | return { 42 | 'name': self.name, 43 | 'client_ver': self.client_ver, 44 | 'network_ver': self.network_ver, 45 | 'my_host_name': self.my_host_name, 46 | 'p2p_accept': self.p2p_accept, 47 | 'p2p_udp_accept': self.p2p_udp_accept, 48 | 'p2p_port': self.p2p_port, 49 | 'start_time': self.start_time, 50 | 'last_seen': self.last_seen, 51 | } 52 | 53 | def update_last_seen(self): 54 | self.last_seen = int(time()) 55 | 56 | 57 | class User(object): 58 | __slots__ = ( 59 | "header", # (UserHeader) 60 | "number", # (int) unique number assigned to each User object 61 | "_reader", # (StreamReader) TCP socket reader 62 | "_writer", # (StreamWriter) TCP socket writer 63 | "host_port", # ([str, int]) Interface used on our PC 64 | "aeskey", # (str) Common key 65 | "direction", # (str) We are as server or client side 66 | "neers", # ({host_port: header}) Neer clients info 67 | "score", # (int )User score 68 | "warn", # (int) User warning score 69 | "create_time", # (int) User object creation time 70 | "process_time", # list of time used for process 71 | ) 72 | 73 | def __init__(self, header, number, reader, writer, host_port, aeskey, direction): 74 | self.header: UserHeader = header 75 | self.number = number 76 | self._reader: StreamReader = reader 77 | self._writer: StreamWriter = writer 78 | self.host_port = host_port 79 | self.aeskey = aeskey 80 | self.direction = direction 81 | self.neers: Dict[(str, int), UserHeader] = dict() 82 | # user experience 83 | self.score = 0 84 | self.warn = 0 85 | self.create_time = int(time()) 86 | self.process_time = deque(maxlen=10) 87 | 88 | def __repr__(self): 89 | age = time2string(time() - self.header.start_time) 90 | passed = time2string(time() - self.create_time) 91 | host_port = self.host_port[0] + ":" + str(self.header.p2p_port) 92 | if self.closed: 93 | status = 'close' 94 | else: 95 | status = 'open' 96 | return f"" 97 | 98 | def __del__(self): 99 | self.close() 100 | 101 | @property 102 | def closed(self): 103 | return self._writer.transport.is_closing() 104 | 105 | def close(self): 106 | if not self.closed: 107 | self._writer.close() 108 | 109 | async def send(self, msg): 110 | self._writer.write(msg) 111 | await self._writer.drain() 112 | 113 | async def recv(self, timeout=1.0): 114 | return await asyncio.wait_for(self._reader.read(8192), timeout) 115 | 116 | def getinfo(self): 117 | return { 118 | 'number': self.number, 119 | 'object': repr(self), 120 | 'header': self.header.getinfo(), 121 | 'neers': [stringify_host_port(*host_port) for host_port in self.neers.keys()], 122 | 'host_port': stringify_host_port(*self.get_host_port()), 123 | 'direction': self.direction, 124 | 'score': self.score, 125 | 'warn': self.warn, 126 | 'average_process_time': self.average_process_time(), 127 | } 128 | 129 | def get_host_port(self) -> tuple: 130 | # connection先 131 | host_port = list(self.host_port) 132 | if self.header.my_host_name: 133 | host_port[0] = self.header.my_host_name 134 | host_port[1] = self.header.p2p_port 135 | return tuple(host_port) 136 | 137 | def update_neers(self, items): 138 | # [[(host,port), header],..] 139 | for host_port, header in items: 140 | self.neers[tuple(host_port)] = UserHeader(**header) 141 | 142 | def average_process_time(self): 143 | if len(self.process_time) == 0: 144 | return None 145 | else: 146 | return sum(self.process_time) / len(self.process_time) 147 | 148 | 149 | def stringify_host_port(*args): 150 | if len(args) == 2: 151 | return "{}:{}".format(args[0], args[1]) 152 | elif len(args) == 4: 153 | return "[{}]:{}".format(args[0], args[1]) 154 | else: 155 | return str(args) 156 | 157 | 158 | def time2string(ntime: float) -> str: 159 | if ntime < 120.0: # 2m 160 | return str(round(ntime, 1)) + "s" 161 | elif ntime < 7200.0: # 2h 162 | return str(round(ntime/60.0, 1)) + "m" 163 | elif ntime < 172800.0: # 2d 164 | return str(round(ntime/3600.0, 1)) + "h" 165 | else: 166 | return str(round(ntime/86400.0, 1)) + "d" 167 | 168 | 169 | __all__ = [ 170 | "UserHeader", 171 | "User", 172 | ] 173 | -------------------------------------------------------------------------------- /p2p_python/utils.py: -------------------------------------------------------------------------------- 1 | from p2p_python.config import V, Debug 2 | import logging 3 | import socket 4 | import random 5 | import asyncio 6 | import os 7 | 8 | 9 | loop = asyncio.get_event_loop() 10 | 11 | 12 | NAMES = ( 13 | "Angle", "Ant", "Apple", "Arch", "Arm", "Army", "Baby", "Bag", "Ball", "Band", "Basin", "Bath", "Bed", 14 | "Bee", "Bell", "Berry", "Bird", "Blade", "Board", "Boat", "Bone", "Book", "Boot", "Box", "Boy", "Brain", 15 | "Brake", "Brick", "Brush", "Bulb", "Cake", "Card", "Cart", "Cat", "Chain", "Chest", "Chin", "Clock", 16 | "Cloud", "Coat", "Comb", "Cord", "Cow", "Cup", "Dog", "Door", "Drain", "Dress", "Drop", "Ear", "Egg", 17 | "Eye", "Face", "Farm", "Fish", "Flag", "Floor", "Fly", "Foot", "Fork", "Fowl", "Frame", "Girl", "Glove", 18 | "Goat", "Gun", "Hair", "Hand", "Hat", "Head", "Heart", "Hook", "Horn", "Horse", "House", "Jewel", "Key", 19 | "Knee", "Knife", "Knot", "Leaf", "Leg", "Line", "Lip", "Lock", "Map", "Match", "Moon", "Mouth", "Nail", 20 | "Neck", "Nerve", "Net", "Nose", "Nut", "Oven", "Pen", "Pig", "Pin", "Pipe", "Plane", "Plate", "Pot", 21 | "Pump", "Rail", "Rat", "Ring", "Rod", "Roof", "Root", "Sail", "Screw", "Seed", "Sheep", "Shelf", "Ship", 22 | "Shirt", "Shoe", "Skin", "Skirt", "Snake", "Sock", "Spade", "Spoon", "Stamp", "Star", "Stem", "Stick", 23 | "Store", "Sun", "Table", "Tail", "Thumb", "Toe", "Tooth", "Town", "Train", "Tray", "Tree", "Wall", 24 | "Watch", "Wheel", "Whip", "Wing", "Wire", "Worm" 25 | ) 26 | 27 | 28 | def get_version(): 29 | """get program version string""" 30 | if Debug.P_PRINT_EXCEPTION: 31 | return 'debug' 32 | 33 | # read version from code 34 | try: 35 | from p2p_python import __version__ 36 | return __version__ 37 | except Exception: 38 | pass 39 | 40 | # read version from file 41 | try: 42 | hear = os.path.dirname(os.path.abspath(__file__)) 43 | with open(os.path.join(hear, '__init__.py'), mode='r') as fp: 44 | for word in fp.readlines(): 45 | if word.startswith('__version__'): 46 | return word.replace('"', "'").split("'")[-2] 47 | except Exception: 48 | pass 49 | return 'unknown' 50 | 51 | 52 | def get_name(): 53 | """get random name for identify from others""" 54 | return "{}:{}".format(random.choice(NAMES), random.randint(10000, 99999)) 55 | 56 | 57 | def setup_p2p_params(network_ver, p2p_port, p2p_accept=False, p2p_udp_accept=False, sub_dir=None): 58 | """ setup general connection setting """ 59 | # directory params 60 | if V.DATA_PATH is not None: 61 | raise Exception('Already setup params.') 62 | root_data_dir = os.path.join(os.path.expanduser('~'), 'p2p-python') 63 | if not os.path.exists(root_data_dir): 64 | os.makedirs(root_data_dir) 65 | V.DATA_PATH = os.path.join(root_data_dir, str(p2p_port)) 66 | if not os.path.exists(V.DATA_PATH): 67 | os.makedirs(V.DATA_PATH) 68 | if sub_dir: 69 | V.DATA_PATH = os.path.join(V.DATA_PATH, sub_dir) 70 | if not os.path.exists(V.DATA_PATH): 71 | os.makedirs(V.DATA_PATH) 72 | # network params 73 | V.CLIENT_VER = get_version() 74 | V.SERVER_NAME = get_name() 75 | V.NETWORK_VER = network_ver 76 | V.P2P_PORT = p2p_port 77 | V.P2P_ACCEPT = p2p_accept 78 | V.P2P_UDP_ACCEPT = p2p_udp_accept 79 | 80 | 81 | def setup_tor_connection(proxy_host='127.0.0.1', port=9150, f_raise_error=True): 82 | """ client connection to onion router """ 83 | # Typically, Tor listens for SOCKS connections on port 9050. 84 | # Tor-browser listens on port 9150. 85 | host_port = (proxy_host, port) 86 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 87 | if V.P2P_ACCEPT or V.P2P_UDP_ACCEPT: 88 | raise ConnectionError('P2P socket accept enable? tcp={} udp={}'.format( 89 | V.P2P_ACCEPT, V.P2P_UDP_ACCEPT)) 90 | if 0 != sock.connect_ex(host_port): 91 | if f_raise_error: 92 | raise ConnectionError('Cannot connect proxy by test.') 93 | else: 94 | V.TOR_CONNECTION = host_port 95 | sock.close() 96 | 97 | 98 | def setup_server_hostname(hostname: str = None): 99 | """ 100 | hostname displayed for others 101 | This is useful when proxy provide different ip address 102 | """ 103 | V.MY_HOST_NAME = hostname 104 | 105 | 106 | async def is_reachable(host, port): 107 | """check a port is opened, finish in 2s""" 108 | future: asyncio.Future = loop.run_in_executor( 109 | None, socket.getaddrinfo, host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) 110 | try: 111 | await asyncio.wait_for(future, 10.0) 112 | addrs = future.result() 113 | except (socket.gaierror, asyncio.TimeoutError): 114 | return False 115 | for af, socktype, proto, canonname, host_port in addrs: 116 | try: 117 | sock = socket.socket(af, socktype, proto) 118 | except OSError: 119 | continue 120 | sock.settimeout(2.0) 121 | future: asyncio.Future = loop.run_in_executor( 122 | None, sock.connect_ex, host_port) 123 | await asyncio.wait_for(future, 3.0) 124 | result = future.result() 125 | loop.run_in_executor(None, sock.close) 126 | if result == 0: 127 | return True 128 | else: 129 | # create no connection 130 | return False 131 | 132 | 133 | def is_unbind_port(port, family=socket.AF_INET, protocol=socket.SOCK_STREAM): 134 | """check is bind port by server""" 135 | try: 136 | with socket.socket(family, protocol) as sock: 137 | sock.bind(("127.0.0.1", port)) 138 | return True 139 | except socket.error: 140 | return False 141 | 142 | 143 | def setup_logger(level=logging.DEBUG, format_str='[%(levelname)-6s] [%(threadName)-10s] [%(asctime)-24s] %(message)s'): 144 | """setup basic logging handler""" 145 | logger = logging.getLogger() 146 | for sh in logger.handlers: 147 | logger.removeHandler(sh) 148 | logger.setLevel(logging.DEBUG) 149 | formatter = logging.Formatter(format_str) 150 | sh = logging.StreamHandler() 151 | sh.setLevel(level) 152 | sh.setFormatter(formatter) 153 | logger.addHandler(sh) 154 | 155 | 156 | __all__ = [ 157 | "NAMES", 158 | "get_version", 159 | "get_name", 160 | "setup_server_hostname", 161 | "is_reachable", 162 | "is_unbind_port", 163 | "setup_tor_connection", 164 | "setup_p2p_params", 165 | "setup_logger", 166 | ] 167 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | expiringdict 2 | pycryptodomex 3 | xmltodict 4 | requests 5 | pysocks 6 | ecdsa>=0.13 7 | msgpack>=0.5.6 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/user/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | from setuptools import setup, find_packages 5 | import os 6 | 7 | try: 8 | with open('README.md') as f: 9 | readme = f.read() 10 | except (IOError, UnicodeError): 11 | readme = '' 12 | 13 | # version 14 | here = os.path.dirname(os.path.abspath(__file__)) 15 | ver_path = os.path.join(here, 'p2p_python', '__init__.py') 16 | version = next((line.split('=')[1].strip().replace("'", '') 17 | for line in open(ver_path) 18 | if line.startswith('__version__ = ')), 19 | '0.0.dev0') 20 | 21 | # requirements.txt 22 | # https://github.com/pypa/setuptools/issues/1080 23 | with open(os.path.join(here, 'requirements.txt')) as fp: 24 | install_requires = fp.read() 25 | 26 | 27 | setup( 28 | name="p2p_python", 29 | version=version, 30 | url='https://github.com/namuyan/p2p-python', 31 | author='namuyan', 32 | description='Simple peer2peer library.', 33 | long_description=readme, 34 | long_description_content_type='text/markdown', 35 | packages=find_packages(), 36 | install_requires=install_requires, 37 | include_package_data=True, 38 | python_requires=">=3.6", 39 | license="MIT Licence", 40 | classifiers=[ 41 | 'Programming Language :: Python :: 3', 42 | 'Programming Language :: Python :: 3.6', 43 | 'License :: OSI Approved :: MIT License', 44 | ], 45 | ) 46 | --------------------------------------------------------------------------------