├── xwing ├── __init__.py ├── network │ ├── transport │ │ ├── __init__.py │ │ ├── socket │ │ │ ├── backend │ │ │ │ ├── __init__.py │ │ │ │ └── rfc1078.py │ │ │ ├── __init__.py │ │ │ ├── client.py │ │ │ └── server.py │ │ └── stream │ │ │ ├── server.py │ │ │ ├── client.py │ │ │ └── __init__.py │ ├── handshake.py │ ├── inbound.py │ ├── outbound.py │ ├── controller.py │ └── connection.py ├── exceptions.py ├── mailbox │ └── __init__.py └── hub.py ├── setup.cfg ├── .coveragerc ├── AUTHORS ├── .travis.yml ├── .gitignore ├── bench └── echo_server │ ├── mailbox │ ├── server.py │ └── run.py │ ├── server.py │ └── run.py ├── examples ├── mailbox │ ├── distributed │ │ ├── ping.py │ │ └── pong.py │ ├── ping_pong.py │ └── rpc.py └── socket │ └── echo │ ├── client.py │ └── server.py ├── tests ├── integration │ ├── run_server.py │ └── test_integration.py ├── helpers.py ├── test_controller.py ├── test_inbound.py ├── test_outbound.py ├── test_handshake.py └── test_connection.py ├── LICENSE ├── bin └── xwing ├── setup.py └── README.rst /xwing/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /xwing/network/transport/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test=pytest 3 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | concurrency = thread 3 | -------------------------------------------------------------------------------- /xwing/network/transport/socket/backend/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Mauricio de Abreu Antunes 2 | Victor Poluceno 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.5" 4 | 5 | install: 6 | - pip install pytest 7 | 8 | script: 9 | - pip install -e . 10 | - py.test tests 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | env 3 | experiments 4 | .cache/ 5 | .coverage 6 | htmlcov 7 | build 8 | dist 9 | *.egg-info 10 | *.eggs 11 | .vagrant 12 | Vagrantfile 13 | env_linux 14 | -------------------------------------------------------------------------------- /xwing/exceptions.py: -------------------------------------------------------------------------------- 1 | class HandshakeError(Exception): 2 | pass 3 | 4 | 5 | class HandshakeTimeoutError(HandshakeError): 6 | pass 7 | 8 | 9 | class HandshakeProtocolError(HandshakeError): 10 | pass 11 | 12 | 13 | class MaxRetriesExceededError(Exception): 14 | pass 15 | 16 | 17 | class HeartbeatFailureError(Exception): 18 | pass 19 | 20 | 21 | class ConnectionAlreadyExists(Exception): 22 | pass 23 | -------------------------------------------------------------------------------- /bench/echo_server/mailbox/server.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from xwing.mailbox import initialize, spawn, start 4 | initialize() 5 | 6 | logging.basicConfig(level='INFO') 7 | 8 | 9 | async def run_server(mailbox): 10 | while True: 11 | data = await mailbox.recv() 12 | if not data: 13 | break 14 | 15 | sender, message = data 16 | await mailbox.send(sender, message) 17 | 18 | 19 | if __name__ == '__main__': 20 | spawn(run_server, name='server') 21 | start() 22 | -------------------------------------------------------------------------------- /examples/mailbox/distributed/ping.py: -------------------------------------------------------------------------------- 1 | from xwing.mailbox import init_node, start_node, spawn 2 | 3 | 4 | async def ping(mailbox, n, pong_node): 5 | for _ in range(n): 6 | await mailbox.send(pong_node, 'ping', mailbox.pid) 7 | message = await mailbox.recv() 8 | if message[0] == 'pong': 9 | print('Ping received pong') 10 | 11 | await mailbox.send('pong', 'finished') 12 | 13 | 14 | if __name__ == '__main__': 15 | init_node() 16 | spawn(ping, 3, 'pong@127.0.0.1') 17 | start_node() 18 | -------------------------------------------------------------------------------- /examples/mailbox/distributed/pong.py: -------------------------------------------------------------------------------- 1 | from xwing.mailbox import init_node, start_node, spawn 2 | 3 | 4 | async def pong(mailbox): 5 | while True: 6 | data = await mailbox.recv() 7 | if len(data) == 1 and data[0] == 'finished': 8 | print('Pong finished') 9 | break 10 | 11 | print('Pong received ping') 12 | message, pid = data 13 | await mailbox.send(pid, 'pong') 14 | 15 | 16 | if __name__ == '__main__': 17 | init_node() 18 | spawn(pong, name='pong') 19 | start_node() 20 | -------------------------------------------------------------------------------- /examples/socket/echo/client.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from xwing.socket.client import Client 4 | 5 | 6 | async def main(loop, endpoint): 7 | client = Client(loop, endpoint) 8 | conn = await client.connect('server0') 9 | 10 | for i in range(100): 11 | await conn.send(b'x') 12 | print('Echo received: %r' % await conn.recv()) 13 | 14 | 15 | if __name__ == '__main__': 16 | # python examples/socket/echo/client.py 17 | loop = asyncio.get_event_loop() 18 | loop.run_until_complete(main(loop, "localhost:5555")) 19 | loop.close() 20 | -------------------------------------------------------------------------------- /tests/integration/run_server.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from xwing.socket.server import Server 4 | 5 | BACKEND_ADDRESS = '/var/tmp/xwing.socket' 6 | 7 | 8 | async def start_server(loop): 9 | server = Server(loop, BACKEND_ADDRESS, 'server0') 10 | await server.listen() 11 | 12 | conn = await server.accept() 13 | while True: 14 | data = await conn.recv() 15 | if not data: 16 | break 17 | 18 | await conn.send(data) 19 | 20 | conn.close() 21 | 22 | 23 | loop = asyncio.get_event_loop() 24 | loop.run_until_complete(start_server(loop)) 25 | loop.close() 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, Victor Godoy Poluceno 2 | 3 | Permission to use, copy, modify, and/or distribute this software for any 4 | purpose with or without fee is hereby granted, provided that the above 5 | copyright notice and this permission notice appear in all copies. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -------------------------------------------------------------------------------- /examples/socket/echo/server.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import asyncio 3 | 4 | from xwing.socket.server import Server 5 | 6 | 7 | async def run(loop, hub_endpoint, identity): 8 | socket_server = Server(loop, hub_endpoint, identity) 9 | await socket_server.listen() 10 | conn = await socket_server.accept() 11 | 12 | while True: 13 | data = await conn.recv() 14 | if not data: 15 | break 16 | 17 | print('Echoing ...') 18 | await conn.send(data) 19 | 20 | 21 | if __name__ == '__main__': 22 | # python examples/socket/echo/server.py /var/tmp/xwing.socket server0 23 | print(sys.argv) 24 | hub_endpoint, identity = sys.argv[1:] 25 | loop = asyncio.get_event_loop() 26 | loop.run_until_complete(run(loop, hub_endpoint, identity)) 27 | loop.close() 28 | -------------------------------------------------------------------------------- /bench/echo_server/server.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | 4 | from xwing.socket.server import Server 5 | 6 | logging.basicConfig(level='INFO') 7 | 8 | 9 | async def handle_client(loop, conn): 10 | while True: 11 | data = await conn.recv() 12 | if not data: 13 | break 14 | 15 | await conn.send(data) 16 | 17 | conn.close() 18 | 19 | 20 | async def run(loop, hub_endpoint, identity): 21 | server = Server(loop, hub_endpoint, identity) 22 | await server.listen() 23 | 24 | while True: 25 | conn = await server.accept() 26 | loop.create_task(handle_client(loop, conn)) 27 | 28 | server.close() 29 | 30 | 31 | if __name__ == '__main__': 32 | hub_endpoint, identity = '/var/tmp/xwing.socket', 'server0' 33 | loop = asyncio.get_event_loop() 34 | loop.run_until_complete(run(loop, hub_endpoint, identity)) 35 | loop.close() 36 | -------------------------------------------------------------------------------- /examples/mailbox/ping_pong.py: -------------------------------------------------------------------------------- 1 | from xwing.mailbox import init_node, start_node, spawn 2 | 3 | 4 | async def pong(mailbox): 5 | while True: 6 | data = await mailbox.recv() 7 | if len(data) == 1 and data[0] == 'finished': 8 | print('Pong finished') 9 | break 10 | 11 | print('Pong received ping') 12 | message, pid = data 13 | await mailbox.send(pid, 'pong') 14 | 15 | 16 | async def ping(mailbox, n): 17 | for _ in range(n): 18 | await mailbox.send('pong', 'ping', mailbox.pid) 19 | message = await mailbox.recv() 20 | if message[0] == 'pong': 21 | print('Ping received pong') 22 | 23 | await mailbox.send('pong', 'finished') 24 | 25 | 26 | if __name__ == '__main__': 27 | # python examples/mailbox/ping_pong.py 28 | init_node() 29 | spawn(pong, name='pong') 30 | spawn(ping, 3) 31 | start_node() 32 | -------------------------------------------------------------------------------- /bin/xwing: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import argparse 3 | import logging 4 | 5 | from xwing.hub import Hub 6 | 7 | 8 | logging.basicConfig() 9 | log = logging.getLogger() 10 | 11 | 12 | if __name__ == '__main__': 13 | parser = argparse.ArgumentParser(description='Xwing hub daemon') 14 | parser.add_argument("--address", help="frontend address", 15 | default='127.0.0.1:5555') 16 | parser.add_argument("--backend_address", help="backend address", 17 | default='/var/tmp/xwing.socket') 18 | parser.add_argument("--loglevel", help="set loglevel", default='INFO', 19 | choices=['DEBUG', 'INFO', 'WARNING', 'ERROR']) 20 | args = parser.parse_args() 21 | log.setLevel(args.loglevel) 22 | 23 | hub = Hub(args.address, args.backend_address) 24 | 25 | try: 26 | hub.run() 27 | except KeyboardInterrupt: 28 | log.info('Terminated by user.') 29 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | 4 | setup( 5 | name='xwing', 6 | version='0.0.1.dev0', 7 | url='https://github.com/victorpoluceno/xwing', 8 | license='ISC', 9 | description='Xwing is a Python library writen using that help ' 10 | 'to distribute connect to a single port to other process', 11 | author='Victor Poluceno', 12 | author_email='victorpoluceno@gmail.com', 13 | packages=['xwing'], 14 | setup_requires=['pytest-runner'], 15 | tests_require=['pytest'], 16 | install_requires=['attrs==16.2.0'], 17 | classifiers=[ 18 | 'Development Status :: 4 - Beta', 19 | 'Intended Audience :: Developers', 20 | 'Programming Language :: Python :: 3.4', 21 | 'Programming Language :: Python :: 3.5', 22 | 'Topic :: Software Development :: Libraries :: Python Modules', 23 | 'Topic :: System :: Distributed Computing', 24 | 'Topic :: System :: Networking', 25 | ], 26 | scripts=['bin/xwing'] 27 | ) 28 | -------------------------------------------------------------------------------- /xwing/network/transport/stream/server.py: -------------------------------------------------------------------------------- 1 | from xwing.network.transport.socket.server import Server 2 | from xwing.network.transport.stream import ( 3 | StreamConnection, DummyStreamConnection) 4 | 5 | 6 | class StreamServer(Server): 7 | 8 | async def accept(self): 9 | stream_connection = StreamConnection(self.loop, await super( 10 | StreamServer, self).accept()) 11 | await stream_connection.initialize() 12 | return stream_connection 13 | 14 | 15 | class DummyStreamServer: 16 | 17 | def __init__(self, loop, settings): 18 | self.loop = loop 19 | self.settings = settings 20 | 21 | async def listen(self): 22 | return True 23 | 24 | async def accept(self): 25 | stream_connection = DummyStreamConnection(self.loop, None) 26 | await stream_connection.initialize() 27 | return stream_connection 28 | 29 | 30 | kind_map = { 31 | 'real': StreamServer, 32 | 'dummy': DummyStreamServer, 33 | } 34 | 35 | 36 | def get_stream_server(kind): 37 | return kind_map[kind] 38 | -------------------------------------------------------------------------------- /xwing/network/transport/stream/client.py: -------------------------------------------------------------------------------- 1 | from xwing.network.transport.socket.client import Client 2 | from xwing.network.transport.stream import ( 3 | StreamConnection, DummyStreamConnection) 4 | 5 | 6 | class StreamClient(Client): 7 | 8 | async def connect(self, service): 9 | stream_connection = StreamConnection(self.loop, await super( 10 | StreamClient, self).connect(service)) 11 | await stream_connection.initialize() 12 | return stream_connection 13 | 14 | 15 | class DummyStreamClient: 16 | 17 | def __init__(self, loop, remote_hub_frontend, identity): 18 | self.loop = loop 19 | self.remote_hub_frontend = remote_hub_frontend 20 | self.identity = identity 21 | 22 | async def connect(self, service): 23 | stream_connection = DummyStreamConnection(self.loop, None) 24 | await stream_connection.initialize() 25 | return stream_connection 26 | 27 | kind_map = { 28 | 'real': StreamClient, 29 | 'dummy': DummyStreamClient, 30 | } 31 | 32 | 33 | def get_stream_client(kind): 34 | return kind_map[kind] 35 | -------------------------------------------------------------------------------- /xwing/network/transport/socket/__init__.py: -------------------------------------------------------------------------------- 1 | from xwing.network.transport.socket.backend.rfc1078 import send, recv 2 | 3 | 4 | class Connection: 5 | 6 | def __init__(self, loop, sock): 7 | self.loop = loop 8 | self.sock = sock 9 | 10 | async def recv(self): 11 | '''Try to recv data. If not data is recv NoData exception will 12 | raise. 13 | 14 | :param timeout: Timeout in seconds. `None` meaning forever. 15 | ''' 16 | return await recv(self.loop, self.sock) 17 | 18 | async def recv_str(self, encoding='utf-8'): 19 | data = await self.recv() 20 | if encoding: 21 | data = data.decode(encoding) 22 | 23 | return data 24 | 25 | async def send(self, data): 26 | '''Send data to connected client. 27 | 28 | :param data: Data to send. 29 | ''' 30 | return await send(self.loop, self.sock, data) 31 | 32 | async def send_str(self, data, encoding='utf-8'): 33 | if encoding: 34 | data = bytes(data, encoding) 35 | 36 | return await self.send(data) 37 | 38 | def close(self): 39 | self.sock.close() 40 | -------------------------------------------------------------------------------- /tests/helpers.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from unittest import mock 3 | 4 | 5 | def run(loop, coro_or_future): 6 | return loop.run_until_complete(coro_or_future) 7 | 8 | 9 | def run_until_complete(f): 10 | def wrap(*args, **kwargs): 11 | return run(asyncio.get_event_loop(), f(*args, **kwargs)) 12 | return wrap 13 | 14 | 15 | def make_coro_mock(): 16 | coro = mock.Mock(name="CoroutineResult") 17 | corofunc = mock.Mock(name="CoroutineFunction", 18 | side_effect=asyncio.coroutine(coro)) 19 | corofunc.coro = coro 20 | return corofunc 21 | 22 | 23 | def run_once(f, return_value=None): 24 | def wrapper(*args, **kwargs): 25 | if not wrapper.has_run: 26 | wrapper.has_run = True 27 | return f(*args, **kwargs) 28 | return return_value 29 | wrapper.has_run = False 30 | return wrapper 31 | 32 | 33 | class SynteticBuffer: 34 | 35 | def __init__(self): 36 | self.buffer = [] 37 | 38 | def put(self, data): 39 | self.buffer.append(data) 40 | 41 | def pop(self): 42 | return self.buffer.pop() 43 | 44 | syntetic_buffer = SynteticBuffer() 45 | -------------------------------------------------------------------------------- /examples/mailbox/rpc.py: -------------------------------------------------------------------------------- 1 | from xwing.mailbox import init_node, start_node, spawn 2 | 3 | 4 | class Server(object): 5 | 6 | def hello_world(self): 7 | return 'Hello World!' 8 | 9 | def run(self): 10 | async def rpc_server(mailbox, server): 11 | while True: 12 | function, pid = await mailbox.recv() 13 | print('Got call from: ', pid) 14 | 15 | result = getattr(server, function)() 16 | await mailbox.send(pid, result) 17 | 18 | spawn(rpc_server, self, name='rpc_server') 19 | 20 | 21 | class Client(object): 22 | 23 | def __init__(self, server_pid): 24 | self.server_pid = server_pid 25 | 26 | def call(self, function): 27 | async def dispatch(mailbox, function): 28 | await mailbox.send(self.server_pid, function, mailbox.pid) 29 | result = await mailbox.recv() 30 | print('Got result: ', result) 31 | 32 | spawn(dispatch, function) 33 | 34 | 35 | if __name__ == '__main__': 36 | # python examples/mailbox/rpc.py 37 | init_node() 38 | 39 | server = Server() 40 | server.run() 41 | 42 | client = Client('rpc_server@127.0.0.1') 43 | client.call('hello_world') 44 | 45 | start_node() 46 | -------------------------------------------------------------------------------- /bench/echo_server/run.py: -------------------------------------------------------------------------------- 1 | import time 2 | import asyncio 3 | import logging 4 | from concurrent import futures 5 | 6 | from xwing.socket.client import Client 7 | 8 | logging.basicConfig(level='INFO') 9 | 10 | 11 | async def connect_and_send(loop, endpoint, payload, start, duration): 12 | client = Client(loop, endpoint) 13 | conn = await client.connect('server0') 14 | 15 | n = 0 16 | while time.monotonic() - start < duration: 17 | await conn.send(payload) 18 | await conn.recv() 19 | n += 1 20 | 21 | return n 22 | 23 | 24 | def run(start, duration, data=b'x'): 25 | loop = asyncio.get_event_loop() 26 | requests = loop.run_until_complete(connect_and_send( 27 | loop, "localhost:5555", b'x', start, duration)) 28 | loop.close() 29 | return requests 30 | 31 | 32 | def main(number_of_workers=10, duration=30): 33 | start = time.monotonic() 34 | with futures.ProcessPoolExecutor(max_workers=number_of_workers) as \ 35 | executor: 36 | fs = [executor.submit(run, start, duration) for i in range(number_of_workers)] 37 | reqs_per_second = sum([f.result() for f in futures.wait(fs).done]) / duration 38 | print('Requests per second w=%d: ' % number_of_workers, 39 | reqs_per_second) 40 | 41 | 42 | if __name__ == '__main__': 43 | main(number_of_workers=4) 44 | -------------------------------------------------------------------------------- /xwing/network/transport/socket/client.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from xwing.network.transport.socket import Connection 4 | from xwing.network.transport.socket.backend.rfc1078 import connect 5 | 6 | log = logging.getLogger(__name__) 7 | 8 | 9 | class Client(object): 10 | '''The Socket Client implementation. 11 | 12 | Provide a Client that knowns how to connect to a Proxy service 13 | send and recv data. 14 | 15 | :param multiplex_endpoint: Mutliplex service address to connect. 16 | :type multiplex_endpoint: str 17 | :param identity: Unique client identification. If not set uuid1 will be 18 | used. 19 | :type identity: str 20 | 21 | Usage:: 22 | 23 | >>> from xwing.socket.client import Client 24 | >>> client = Client('localhost:5555', 'client1') 25 | >>> conn = client.connect('server0') 26 | >>> conn.send(b'ping') 27 | >>> conn.recv() 28 | ''' 29 | 30 | def __init__(self, loop, remote_hub_frontend, identity): 31 | self.loop = loop 32 | self.remote_hub_frontend = remote_hub_frontend 33 | self.identity = identity 34 | 35 | async def connect(self, remote_identity): 36 | address, port = self.remote_hub_frontend.split(':') 37 | return Connection(self.loop, await connect( 38 | self.loop, (address, int(port)), remote_identity)) 39 | -------------------------------------------------------------------------------- /xwing/network/handshake.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from xwing.exceptions import HandshakeTimeoutError, HandshakeProtocolError 4 | 5 | HANDSHAKE_SIGNAL = b'HANDSHAKE' 6 | HANDSHAKE_ACK_SIGNAL = b'HANDSHAKE_ACK' 7 | 8 | 9 | async def connect_handshake(connection, local_identity, timeout=10): 10 | connection.send(b''.join([HANDSHAKE_SIGNAL, b';', 11 | local_identity.encode('utf-8')])) 12 | await connection.stream.drain() 13 | 14 | try: 15 | handshake_ack = await asyncio.wait_for(connection.recv(), timeout) 16 | except asyncio.TimeoutError: 17 | raise HandshakeTimeoutError 18 | 19 | if not handshake_ack.startswith(HANDSHAKE_ACK_SIGNAL): 20 | raise HandshakeProtocolError 21 | 22 | # TODO implement a identity check and seconde handshake ack. 23 | return True 24 | 25 | 26 | async def accept_handshake(connection, local_identity, timeout=10): 27 | try: 28 | handshake = await asyncio.wait_for(connection.recv(), timeout) 29 | except asyncio.TimeoutError: 30 | raise HandshakeTimeoutError 31 | 32 | if not handshake.startswith(HANDSHAKE_SIGNAL): 33 | raise HandshakeProtocolError 34 | 35 | remote_identity = handshake.split(b';')[1].strip() 36 | connection.send(b''.join([HANDSHAKE_ACK_SIGNAL, b';', 37 | local_identity.encode('utf-8')])) 38 | await connection.stream.drain() 39 | 40 | # TODO implement a wait for second hanshake ack here. 41 | return remote_identity.decode('utf-8') 42 | -------------------------------------------------------------------------------- /xwing/network/inbound.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | log = logging.getLogger(__name__) 4 | 5 | 6 | class Inbound: 7 | 8 | def __init__(self, loop, settings): 9 | self.loop = loop 10 | self.settings = settings 11 | self.inbox = asyncio.Queue(loop=self.loop) 12 | self.stop_event = asyncio.Event() 13 | 14 | def stop(self): 15 | self.stop_event.set() 16 | 17 | async def get(self, timeout=None): 18 | try: 19 | data = await asyncio.wait_for(self.inbox.get(), timeout) 20 | except asyncio.TimeoutError: 21 | return None 22 | return data 23 | 24 | async def run_recv_loop(self, connection, timeout=None): 25 | while not self.stop_event.is_set(): 26 | try: 27 | data = await asyncio.wait_for(connection.recv(), timeout) 28 | except asyncio.TimeoutError: 29 | continue 30 | 31 | # FIXME shouldn't this raise an exception? 32 | # As it is right now, it will stop looking for data as recv loop 33 | # will stop. But the hearbeat loop of connection will keep running 34 | # and eventually will raise an exception. 35 | # 36 | # If we raise an exception on recv loop, we wil known of a 37 | # connection error sonner and have a more responsive system. 38 | if not data: # connection is closed 39 | break 40 | 41 | await self.inbox.put(data) 42 | -------------------------------------------------------------------------------- /bench/echo_server/mailbox/run.py: -------------------------------------------------------------------------------- 1 | import time 2 | import logging 3 | from concurrent import futures 4 | 5 | from xwing.mailbox import initialize, spawn, run, get_node_instance 6 | initialize() 7 | 8 | logging.basicConfig(level='INFO') 9 | 10 | 11 | async def send(mailbox, start, duration, payload, target_pid): 12 | n = 0 13 | while time.monotonic() - start < duration: 14 | await mailbox.send(target_pid, mailbox.pid, payload) 15 | await mailbox.recv() 16 | n += 1 17 | 18 | await mailbox.send('collector', n) 19 | 20 | 21 | def run_bench(start, duration, data=b'x'): 22 | initialize() # initialize a new node with a new event loop 23 | spawn(send, start, duration, b'x', 'server') 24 | run() 25 | 26 | 27 | async def collector(mailbox, wait_for): 28 | total = 0 29 | for i in range(wait_for): 30 | r = await mailbox.recv() 31 | total += r[0] 32 | 33 | print('Requests per second w=%d' % wait_for, total) 34 | 35 | 36 | async def dispatch(loop, executor, start, duration): 37 | await loop.run_in_executor(executor, run_bench, start, duration) 38 | 39 | 40 | def main(number_of_workers=4, duration=30): 41 | start = time.monotonic() 42 | spawn(collector, number_of_workers, name='collector') 43 | executor = futures.ProcessPoolExecutor(max_workers=number_of_workers) 44 | for i in range(number_of_workers): 45 | node = get_node_instance() 46 | node.loop.create_task(dispatch(node.loop, executor, start, duration)) 47 | 48 | run() 49 | 50 | 51 | if __name__ == '__main__': 52 | main(number_of_workers=4) 53 | -------------------------------------------------------------------------------- /xwing/network/transport/stream/__init__.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from tests.helpers import syntetic_buffer 4 | 5 | 6 | async def connection_to_stream(connection, loop): 7 | return await asyncio.open_connection(sock=connection.sock, 8 | loop=loop) 9 | 10 | 11 | class StreamConnection: 12 | 13 | def __init__(self, loop, connection): 14 | self.loop = loop 15 | self.connection = connection 16 | 17 | async def initialize(self): 18 | self.reader, self.writer = await connection_to_stream( 19 | self.connection, self.loop) 20 | return True 21 | 22 | async def readline(self): 23 | return await self.reader.readline() 24 | 25 | def write(self, data): 26 | return self.writer.write(data) 27 | 28 | async def drain(self): 29 | return await self.writer.drain() 30 | 31 | 32 | class DummyStreamConnection: 33 | 34 | def __init__(self, loop, connection): 35 | self.loop = loop 36 | self.connection = connection 37 | 38 | async def initialize(self): 39 | return True 40 | 41 | async def readline(self): 42 | data = syntetic_buffer.pop() 43 | if isinstance(data, int) or isinstance(data, float): 44 | await asyncio.sleep(data) 45 | return None 46 | 47 | return data 48 | 49 | def write(self, data): 50 | return True 51 | 52 | async def drain(self): 53 | return True 54 | 55 | 56 | kind_map = { 57 | 'real': StreamConnection, 58 | 'dummy': DummyStreamConnection, 59 | } 60 | 61 | 62 | def get_stream_connection(kind): 63 | return kind_map[kind] 64 | -------------------------------------------------------------------------------- /tests/test_controller.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from xwing.network.controller import Controller 4 | from xwing.network.transport.stream.client import get_stream_client 5 | from xwing.network.transport.stream.server import get_stream_server 6 | from xwing.mailbox import TaskPool, Settings 7 | from xwing.network.handshake import HANDSHAKE_ACK_SIGNAL 8 | from tests.helpers import run_until_complete, run_once, syntetic_buffer 9 | 10 | 11 | class TestController: 12 | 13 | def setup_method(self, method): 14 | self.loop = asyncio.get_event_loop() 15 | self.settings = Settings() 16 | self.task_pool = TaskPool(self.loop) 17 | self.controller = Controller( 18 | self.loop, self.settings, self.task_pool, 19 | get_stream_client('dummy'), 20 | get_stream_server('dummy')) 21 | self.controller.stop_event.is_set = run_once( 22 | self.controller.stop_event.is_set, return_value=True) 23 | 24 | @run_until_complete 25 | async def test_run_outbound(self): 26 | syntetic_buffer.put(HANDSHAKE_ACK_SIGNAL + b'\n') 27 | await self.controller.put_outbound(('127.0.0.1', 'foo'), b'bar') 28 | await self.controller.run_outbound(timeout=0.1) 29 | 30 | @run_until_complete 31 | async def test_run_outbound_with_not_data(self): 32 | await self.controller.run_outbound(timeout=0.1) 33 | 34 | @run_until_complete 35 | async def test_run_inbound(self): 36 | syntetic_buffer.put(b'data\n') 37 | syntetic_buffer.put(HANDSHAKE_ACK_SIGNAL + b';bar\n') 38 | await self.controller.run_inbound() 39 | assert await self.controller.get_inbound() == b'data' 40 | 41 | @run_until_complete 42 | async def test_run_and_stop(self): 43 | self.controller.start() 44 | self.controller.stop() 45 | -------------------------------------------------------------------------------- /tests/test_inbound.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from xwing.mailbox import Settings 4 | from xwing.network.connection import get_connection 5 | from xwing.network.transport.stream import get_stream_connection 6 | from xwing.network.inbound import Inbound 7 | from xwing.mailbox import TaskPool 8 | from tests.helpers import run_once, run_until_complete, syntetic_buffer 9 | 10 | 11 | class TestInbound: 12 | 13 | def setup_method(self, method): 14 | self.loop = asyncio.get_event_loop() 15 | settings = Settings() 16 | self.inbound = Inbound(self.loop, settings) 17 | self.inbound.stop_event.is_set = run_once( 18 | self.inbound.stop_event.is_set, return_value=True) 19 | 20 | self.stream_connection = get_stream_connection('dummy')( 21 | self.loop, None) 22 | self.task_pool = TaskPool(self.loop) 23 | self.connection = get_connection('real')( 24 | self.loop, self.stream_connection, self.task_pool) 25 | 26 | @run_until_complete 27 | async def test_get_with_timeout_error(self): 28 | assert await self.inbound.get(timeout=0.1) is None 29 | 30 | @run_until_complete 31 | async def test_get_return_put_value(self): 32 | await self.inbound.inbox.put('foo') 33 | assert await self.inbound.get() == 'foo' 34 | 35 | @run_until_complete 36 | async def test_run_recv_loop(self): 37 | syntetic_buffer.put(b'foo\n') 38 | await self.inbound.run_recv_loop(self.connection) 39 | assert await self.inbound.get() == b'foo' 40 | 41 | @run_until_complete 42 | async def test_stop_nothing_gets_done(self): 43 | self.inbound.stop() 44 | await self.inbound.run_recv_loop(self.connection) 45 | assert await self.inbound.get(0.1) is None 46 | 47 | @run_until_complete 48 | async def test_run_recv_loop_may_raise_timeout_error(self): 49 | syntetic_buffer.put(0.2) 50 | await self.inbound.run_recv_loop(self.connection, timeout=0.1) 51 | 52 | @run_until_complete 53 | async def test_run_recv_loop_break_connection(self): 54 | syntetic_buffer.put(None) 55 | await self.inbound.run_recv_loop(self.connection) 56 | -------------------------------------------------------------------------------- /tests/test_outbound.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | import pytest 4 | 5 | from xwing.exceptions import MaxRetriesExceededError 6 | from xwing.mailbox import Settings 7 | from xwing.network.outbound import Outbound, Connector 8 | from xwing.network.transport.stream.client import get_stream_client 9 | from tests.helpers import run_until_complete, make_coro_mock 10 | 11 | 12 | class TestConnector: 13 | 14 | def setup_method(self, method): 15 | self.loop = asyncio.get_event_loop() 16 | self.settings = Settings() 17 | client_factory = get_stream_client('dummy') 18 | self.connector = Connector(self.loop, self.settings, 19 | client_factory) 20 | self.stream = client_factory(self.loop, 'localhost:5555', 'foo') 21 | 22 | def test_create_client(self): 23 | stream = self.connector.create_client('localhost') 24 | assert stream.remote_hub_frontend == 'localhost:5555' 25 | assert stream.identity == self.settings.identity 26 | 27 | @run_until_complete 28 | async def test_connect_client(self): 29 | await self.connector.connect_client(self.stream, 'bar') 30 | 31 | @run_until_complete 32 | async def test_connect_client_raise_error(self): 33 | self.stream.connect = make_coro_mock() 34 | self.stream.connect.side_effect = ConnectionError 35 | with pytest.raises(MaxRetriesExceededError): 36 | await self.connector.connect_client( 37 | self.stream, 'bar', max_retries=1) 38 | 39 | @run_until_complete 40 | async def test_connect_return_connection(self): 41 | assert await self.connector.connect(('localhost', 'foo')) 42 | 43 | 44 | class TestOutbound: 45 | 46 | def setup_method(self, method): 47 | self.loop = asyncio.get_event_loop() 48 | settings = Settings() 49 | self.outbound = Outbound(self.loop, settings) 50 | 51 | @run_until_complete 52 | async def test_get_with_timeout_error(self): 53 | assert await self.outbound.get(timeout=0.1) is None 54 | 55 | @run_until_complete 56 | async def test_get_after_put(self): 57 | await self.outbound.put(*('foo', 'bar')) is None 58 | assert await self.outbound.get() == ('foo', 'bar') 59 | -------------------------------------------------------------------------------- /xwing/network/transport/socket/server.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import asyncio 3 | 4 | from xwing.network.transport.socket import Connection 5 | from xwing.network.transport.socket.backend.rfc1078 import accept, listen 6 | 7 | log = logging.getLogger(__name__) 8 | 9 | 10 | class Server(object): 11 | '''The Socket Server implementation. 12 | 13 | Provides an socket that knows how to connect to a proxy 14 | and receive data from clients. 15 | 16 | :param multiplex_endpoint: Multiplex proxy address to connect. 17 | :type multiplex_endpoint: str 18 | :param identity: Unique server identification. If not set uuid1 will be 19 | used. 20 | :type identity: str 21 | 22 | Usage:: 23 | 24 | >>> from xwing.socket.server import Server 25 | >>> socket_server = Server('/var/run/xwing.socket', 'server0') 26 | >>> socket_server.listen() 27 | >>> conn = socket_server.accept() 28 | >>> data = conn.recv() 29 | >>> conn.send(data) 30 | ''' 31 | 32 | def __init__(self, loop, settings): 33 | self.loop = loop 34 | self.settings = settings 35 | self.reconnecting = False 36 | 37 | async def listen(self): 38 | self.sock = await listen(self.loop, self.settings.hub_backend, 39 | self.settings.identity) 40 | log.info('%s is listening.' % self.settings.identity) 41 | return True 42 | 43 | async def reconnect(self): 44 | self.reconnecting = True 45 | while True: 46 | try: 47 | await self.listen() 48 | except ConnectionRefusedError: 49 | await asyncio.sleep(0.1) 50 | else: 51 | log.debug('Connection to Hub estabilished.') 52 | self.reconnecting = False 53 | break 54 | 55 | async def accept(self): 56 | if self.reconnecting: 57 | return None 58 | 59 | conn = await accept(self.loop, self.sock) 60 | if conn is None: 61 | log.debug( 62 | 'Connection to Hub lost, starting reconnecting task.') 63 | self.loop.create_task(self.reconnect()) 64 | return None 65 | 66 | return Connection(self.loop, conn) 67 | 68 | def close(self): 69 | self.sock.close() 70 | -------------------------------------------------------------------------------- /tests/test_handshake.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | import pytest 4 | 5 | from xwing.exceptions import HandshakeTimeoutError, HandshakeProtocolError 6 | from xwing.network.handshake import ( 7 | connect_handshake, accept_handshake, HANDSHAKE_ACK_SIGNAL) 8 | from xwing.network.transport.stream import get_stream_connection 9 | from xwing.mailbox import TaskPool 10 | from xwing.network.connection import get_connection 11 | from tests.helpers import run_until_complete, syntetic_buffer 12 | 13 | 14 | class TestHandshake: 15 | 16 | def setup_method(self, method): 17 | self.loop = asyncio.get_event_loop() 18 | self.stream_connection = get_stream_connection('dummy')( 19 | self.loop, None) 20 | self.task_pool = TaskPool(self.loop) 21 | self.connection = get_connection('real')(self.loop, 22 | self.stream_connection, 23 | self.task_pool) 24 | 25 | @run_until_complete 26 | async def test_connect_handshake(self): 27 | syntetic_buffer.put(HANDSHAKE_ACK_SIGNAL + b'\n') 28 | assert await connect_handshake(self.connection, 'foo') 29 | 30 | @run_until_complete 31 | async def test_connect_handshake_protocol_error(self): 32 | syntetic_buffer.put(b'garbage\n') 33 | with pytest.raises(HandshakeProtocolError): 34 | await connect_handshake(self.connection, 'foo') 35 | 36 | @run_until_complete 37 | async def test_connect_handshake_timeout_error(self): 38 | syntetic_buffer.put(0.2) 39 | with pytest.raises(HandshakeTimeoutError): 40 | await connect_handshake(self.connection, 'foo', 0.1) 41 | 42 | @run_until_complete 43 | async def test_accept_handshake(self): 44 | syntetic_buffer.put(HANDSHAKE_ACK_SIGNAL + b';bar\n') 45 | assert await accept_handshake(self.connection, 'foo') 46 | 47 | @run_until_complete 48 | async def test_accept_handshake_protocol_error(self): 49 | syntetic_buffer.put(b'garbage\n') 50 | with pytest.raises(HandshakeProtocolError): 51 | await accept_handshake(self.connection, 'foo') 52 | 53 | @run_until_complete 54 | async def test_accept_handshake_timeout_error(self): 55 | syntetic_buffer.put(0.2) 56 | with pytest.raises(HandshakeTimeoutError): 57 | await accept_handshake(self.connection, 'foo', 0.1) 58 | -------------------------------------------------------------------------------- /tests/test_connection.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | import pytest 4 | 5 | from xwing.exceptions import HeartbeatFailureError, ConnectionAlreadyExists 6 | from xwing.mailbox import TaskPool 7 | from xwing.network.connection import Connection, Repository, HEARTBEAT_SIGNAL 8 | from xwing.network.transport.stream import get_stream_connection 9 | from tests.helpers import run_until_complete, syntetic_buffer 10 | 11 | 12 | class TestRepository: 13 | 14 | def setup_method(self, method): 15 | self.repository = Repository() 16 | 17 | def test_add_get(self): 18 | self.repository.add(1, 'foo') 19 | assert self.repository.get('foo') == 1 20 | 21 | def test_add_existing_connection_fails(self): 22 | self.repository.add(1, 'foo') 23 | with pytest.raises(ConnectionAlreadyExists): 24 | self.repository.add(1, 'foo') 25 | 26 | def test_contains(self): 27 | self.repository.add(1, 'foo') 28 | assert 'foo' in self.repository 29 | assert 'bar' not in self.repository 30 | 31 | def test_identity_must_be_string(self): 32 | with pytest.raises(TypeError): 33 | b'foo' in self.repository 34 | 35 | 36 | class TestConnection: 37 | 38 | def setup_method(self, method): 39 | self.loop = asyncio.get_event_loop() 40 | self.stream_connection = get_stream_connection('dummy')( 41 | self.loop, None) 42 | self.task_pool = TaskPool(self.loop) 43 | self.connection = Connection(self.loop, self.stream_connection, 44 | self.task_pool) 45 | 46 | def test_send(self): 47 | assert self.connection.send(b'foo') == b'foo' 48 | 49 | @run_until_complete 50 | async def test_recv(self): 51 | syntetic_buffer.put(b'foo\n') 52 | assert await self.connection.recv() == b'foo' 53 | 54 | @run_until_complete 55 | async def test_recv_data_and_heartbeat(self): 56 | syntetic_buffer.put(b'foo\n') 57 | syntetic_buffer.put(HEARTBEAT_SIGNAL + b'\n') 58 | assert await self.connection.recv() == b'foo' 59 | 60 | @run_until_complete 61 | async def test_recv_partial_data(self): 62 | syntetic_buffer.put(b'foo') 63 | assert await self.connection.recv() is None 64 | 65 | @run_until_complete 66 | async def test_liveness_zero_raises_error(self): 67 | self.connection.liveness = 1 68 | with pytest.raises(HeartbeatFailureError): 69 | await self.connection.run_heartbeat_loop(heartbeat_interval=0.1) 70 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | xwing 2 | ===== 3 | 4 | .. image:: https://travis-ci.org/victorpoluceno/xwing.svg?branch=development 5 | :target: https://travis-ci.org/victorpoluceno/xwing 6 | 7 | | 8 | 9 | **xwing** is Actor based Python concurrency framework that uses *asyncio* and Concurrent Oriented Programming ideas, inspired by Erlang's design. 10 | 11 | Features 12 | -------- 13 | 14 | * Simple. Ships with a minimal for humans Actor API. 15 | * Fast. Message multiplexing implemented by routing connections, not data. 16 | * Powerful. Based and inspired by the battle tested Erlang's actor design. 17 | * High level. Allow developers to write applications without worrying too much about low level details. 18 | * Interoperable. Can live along side other async libraries or applications. 19 | * Rich. Packed with features that are essential to write distributed applications. 20 | 21 | Sample 22 | ------ 23 | 24 | .. code-block:: python 25 | 26 | from xwing.mailbox import init_node, start_node, spawn 27 | 28 | async def pong(mailbox): 29 | message, pid = await mailbox.recv() 30 | await mailbox.send(pid, 'pong') 31 | 32 | async def ping(mailbox, pong_pid): 33 | await mailbox.send(pong_pid, 'ping', mailbox.pid) 34 | print(await mailbox.recv()) 35 | 36 | init_node() 37 | pong_pid = spawn(pong) 38 | spawn(ping, pong_pid) 39 | start_node() 40 | 41 | Status 42 | ------ 43 | 44 | Not released, under heavy development. **Unstable API**. 45 | 46 | Requirements 47 | ------------ 48 | 49 | Xwing requires Python 3.5+. 50 | 51 | Roadmap 52 | ------- 53 | 54 | Features planned to *0.1* release: 55 | 56 | * Basic Actor functions: spawn, send and recv [DONE]. 57 | * Heartbeat for connection check and keepalive [DONE]. 58 | * Full remote communication support with multiplexing and port mapper. [DONE] 59 | * One connection per connected actor. [DONE] 60 | * Auto start of Xwing Hub on Node start. [NOT STARTED] 61 | * Fault isolation between actors. [NOT STARTED] 62 | * Task Scheduler with SMP support. [NOT STARTED] 63 | * Full tested and benchmarked uvloop support. [IN PROGRESS] 64 | 65 | Development 66 | ---------- 67 | 68 | Bootstraping 69 | ~~~~~~~~~~~~ 70 | 71 | Create a venv and install development requirements:: 72 | 73 | pyvenv env && source env/bin/activate 74 | pip install -e . 75 | 76 | Testing 77 | ~~~~~~~ 78 | 79 | Run using `py.test`:: 80 | 81 | py.test tests 82 | 83 | Or if you want to see coverage report:: 84 | 85 | pip install pytest-cov 86 | py.test --cov=xwing --cov-report html tests/ 87 | open htmlcov/index.html 88 | 89 | License 90 | ------- 91 | 92 | The software is licensed under ISC license. 93 | -------------------------------------------------------------------------------- /xwing/network/outbound.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | log = logging.getLogger(__name__) 4 | 5 | from xwing.exceptions import MaxRetriesExceededError 6 | 7 | DEFAULT_FRONTEND_PORT = 5555 8 | 9 | 10 | class Connector: 11 | 12 | def __init__(self, loop, settings, client_factory): 13 | self.loop = loop 14 | self.settings = settings 15 | self.client_factory = client_factory 16 | self.clients = {} 17 | 18 | async def connect(self, pid): 19 | hub_frontend, identity = pid 20 | if hub_frontend not in self.clients: 21 | self.clients[hub_frontend] = self.create_client(hub_frontend) 22 | 23 | client = self.clients[hub_frontend] 24 | return await self.connect_client(client, identity) 25 | 26 | async def connect_client(self, client, remote_identity, max_retries=30, 27 | retry_sleep=0.1): 28 | log.info('Creating connection to %s' % remote_identity) 29 | connection = None 30 | number_of_retries = 0 31 | while number_of_retries < max_retries: 32 | try: 33 | connection = await client.connect(remote_identity) 34 | except ConnectionError: 35 | log.info('Retrying connection to %s...' % remote_identity) 36 | number_of_retries += 1 37 | await asyncio.sleep(retry_sleep) 38 | continue 39 | else: 40 | break 41 | 42 | if not connection: 43 | raise MaxRetriesExceededError 44 | 45 | return connection 46 | 47 | def create_client(self, remote_hub_frontend): 48 | log.info('Creating client to %s' % remote_hub_frontend) 49 | remote_address = remote_hub_frontend 50 | if ':' not in remote_hub_frontend: 51 | remote_address = '%s:%d' % ( 52 | remote_hub_frontend, DEFAULT_FRONTEND_PORT) 53 | 54 | return self.client_factory( 55 | self.loop, remote_hub_frontend=remote_address, 56 | identity=self.settings.identity) 57 | 58 | 59 | class Outbound(object): 60 | 61 | def __init__(self, loop, settings): 62 | self.loop = loop 63 | self.outbox = asyncio.Queue(loop=self.loop) 64 | 65 | async def put(self, pid, data): 66 | await self.outbox.put((pid, data)) 67 | return data 68 | 69 | async def get(self, timeout=None): 70 | try: 71 | item = await asyncio.wait_for(self.outbox.get(), 72 | timeout=timeout) 73 | except asyncio.TimeoutError: 74 | return None 75 | 76 | return item 77 | -------------------------------------------------------------------------------- /tests/integration/test_integration.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path.append('.') 3 | 4 | import asyncio 5 | import time 6 | import subprocess 7 | import logging 8 | logging.basicConfig(level=logging.DEBUG) 9 | 10 | import pytest 11 | 12 | from xwing.mailbox import init_node, start_node, spawn 13 | from xwing.network.transport.socket.client import Client 14 | 15 | FRONTEND_ADDRESS = '127.0.0.1:5555' 16 | 17 | 18 | def setup_module(module): 19 | module.hub_process = subprocess.Popen('bin/xwing') 20 | time.sleep(1) 21 | module.server_process = subprocess.Popen( 22 | ['python', 'tests/integration/run_server.py']) 23 | 24 | 25 | def teardown_module(module): 26 | module.hub_process.kill() 27 | module.server_process.kill() 28 | 29 | 30 | @pytest.mark.skip() 31 | class TestSocket: 32 | 33 | @classmethod 34 | def setup_class(cls): 35 | cls.loop = asyncio.get_event_loop() 36 | cls.client = Client(cls.loop, FRONTEND_ADDRESS) 37 | 38 | async def connect(cls): 39 | while True: 40 | try: 41 | cls.connection = await cls.client.connect('server0') 42 | except ConnectionError: 43 | await asyncio.sleep(1) 44 | continue 45 | else: 46 | break 47 | 48 | cls.loop.run_until_complete(asyncio.wait_for(connect(cls), 30)) 49 | 50 | @classmethod 51 | def teardown_class(cls): 52 | cls.connection.close() 53 | 54 | def test_send_and_recv_str(self): 55 | async def run(self): 56 | data = 'ping' 57 | await self.connection.send_str(data) 58 | await self.connection.recv_str() 59 | return True 60 | 61 | event_loop = asyncio.get_event_loop() 62 | assert event_loop.run_until_complete(run(self)) 63 | 64 | def test_send_and_recv(self): 65 | async def run(self): 66 | data = b'ping' 67 | await self.connection.send(data) 68 | await self.connection.recv() 69 | return True 70 | 71 | event_loop = asyncio.get_event_loop() 72 | assert event_loop.run_until_complete(run(self)) 73 | 74 | 75 | @pytest.mark.skip() 76 | class TestMailbox(object): 77 | 78 | def setup_class(self): 79 | init_node() 80 | 81 | def test_send_and_recv(self): 82 | async def echo_server(mailbox): 83 | message, pid = await mailbox.recv() 84 | await mailbox.send(pid, message) 85 | 86 | async def echo_client(mailbox, pid_server): 87 | await mailbox.send(pid_server, 'hello', mailbox.pid) 88 | await mailbox.recv() 89 | 90 | pid = spawn(echo_server) 91 | spawn(echo_client, pid) 92 | start_node() 93 | -------------------------------------------------------------------------------- /xwing/network/controller.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from xwing.network.inbound import Inbound 4 | from xwing.network.outbound import Outbound, Connector 5 | from xwing.network.handshake import connect_handshake, accept_handshake 6 | from xwing.network.connection import Connection, Repository 7 | 8 | 9 | class Controller: 10 | 11 | def __init__(self, loop, settings, task_pool, client_factory, 12 | server_factory): 13 | self.loop = loop 14 | self.task_pool = task_pool 15 | self.settings = settings 16 | self.repository = Repository() 17 | self.inbound = Inbound(self.loop, settings) 18 | self.outbound = Outbound(self.loop, settings) 19 | self.client_factory = client_factory 20 | self.server_factory = server_factory 21 | self.stop_event = asyncio.Event() 22 | 23 | def start(self, timeout=None): 24 | self.task_pool.create_task(self.run_inbound()) 25 | self.task_pool.create_task(self.run_outbound(timeout=timeout)) 26 | 27 | def stop(self): 28 | self.stop_event.set() 29 | self.inbound.stop() 30 | 31 | async def get_inbound(self, *args, **kwargs): 32 | return await self.inbound.get(*args, **kwargs) 33 | 34 | async def put_outbound(self, *args, **kwargs): 35 | return await self.outbound.put(*args, **kwargs) 36 | 37 | async def run_outbound(self, timeout=None): 38 | connector = Connector(self.loop, self.settings, 39 | self.client_factory) 40 | while not self.stop_event.is_set(): 41 | item = await self.outbound.get(timeout=timeout) 42 | if not item: 43 | continue 44 | 45 | pid, data = item 46 | hub_frontend, remote_identity = pid 47 | if remote_identity not in self.repository: 48 | stream = await connector.connect(pid) 49 | connection = Connection(self.loop, stream, self.task_pool) 50 | await connect_handshake( 51 | connection, local_identity=self.settings.identity) 52 | 53 | connection.start() 54 | self.repository.add(connection, remote_identity) 55 | self.start_receiver(connection) 56 | 57 | connection = self.repository.get(remote_identity) 58 | connection.send(data) 59 | 60 | async def run_inbound(self): 61 | stream_server = self.server_factory(self.loop, self.settings) 62 | await stream_server.listen() 63 | while not self.stop_event.is_set(): 64 | stream = await stream_server.accept() 65 | connection = Connection(self.loop, stream, self.task_pool) 66 | remote_identity = await accept_handshake( 67 | connection, local_identity=self.settings.identity) 68 | 69 | connection.start() 70 | self.repository.add(connection, remote_identity) 71 | self.start_receiver(connection) 72 | 73 | def start_receiver(self, connection, timeout=5.0): 74 | self.task_pool.create_task(self.inbound.run_recv_loop( 75 | connection, timeout / 5)) 76 | -------------------------------------------------------------------------------- /xwing/network/transport/socket/backend/rfc1078.py: -------------------------------------------------------------------------------- 1 | import array 2 | import socket 3 | import asyncio 4 | 5 | BUFFER_SIZE = 4096 6 | SERVICE_POSITIVE_ANSWER = b'+' 7 | SERVICE_PING = b'!' 8 | 9 | # FIXME implement a shared buffer beteween listening and 10 | # accept, this way listening may get pings and accept 11 | # may get its file descriptors, avoiding this way the 12 | # problem that a socket not issuing accepts may hold 13 | # a service id forever. 14 | 15 | 16 | async def listen(loop, unix_address, service): 17 | sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 18 | sock.setblocking(False) 19 | await loop.sock_connect(sock, unix_address) 20 | await loop.sock_sendall(sock, bytes(service, encoding='utf-8')) 21 | response = await loop.sock_recv(sock, BUFFER_SIZE) 22 | if not response.startswith(SERVICE_POSITIVE_ANSWER): 23 | raise ConnectionError(response.decode().strip()) # NOQA 24 | 25 | return sock 26 | 27 | 28 | async def accept(loop, sock): 29 | while True: 30 | try: 31 | data = sock.recvmsg(1, BUFFER_SIZE) 32 | # In order to be able to tell if a server is alive or not 33 | # the hub send a ping message to server in case some one 34 | # try to connect using same service name. In this case 35 | # We can just ignore this data. 36 | # 37 | # Be aware that current implementation will only allow 38 | # this ping mechancis to work after user start accepting 39 | # connections, so a Server listening without accepting 40 | # will hold this service forever. 41 | if data[0] == SERVICE_PING: 42 | continue 43 | _, ancdata, flags, addr = data 44 | if not ancdata: 45 | # Hub is gone, return None will signal that listen must run 46 | # again. We also close the socket, this way any further 47 | # operations on this socket will raise OSError 48 | sock.close() 49 | return None 50 | 51 | cmsg_level, cmsg_type, cmsg_data = ancdata[0] 52 | except BlockingIOError: 53 | await asyncio.sleep(0.1) 54 | continue 55 | else: 56 | break 57 | 58 | fda = array.array('I') 59 | fda.frombytes(cmsg_data) 60 | 61 | client = socket.fromfd(fda[0], socket.AF_INET, socket.SOCK_STREAM) 62 | client.setblocking(False) 63 | 64 | await loop.sock_sendall(client, SERVICE_POSITIVE_ANSWER) 65 | return client 66 | 67 | 68 | async def connect(loop, tcp_address, service, tcp_nodelay=True): 69 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 70 | if tcp_nodelay: 71 | sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) 72 | 73 | sock.setblocking(False) 74 | await loop.sock_connect(sock, tcp_address) 75 | await loop.sock_sendall(sock, bytes(service, encoding='utf-8')) 76 | response = await loop.sock_recv(sock, BUFFER_SIZE) 77 | if response != SERVICE_POSITIVE_ANSWER: 78 | raise ConnectionError(response.decode().strip()) # NOQA 79 | 80 | return sock 81 | 82 | 83 | async def send(loop, sock, data): 84 | ret = await loop.sock_sendall(sock, data) 85 | return True if ret is None else False 86 | 87 | async def recv(loop, sock): 88 | return await loop.sock_recv(sock, BUFFER_SIZE) 89 | -------------------------------------------------------------------------------- /xwing/network/connection.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import time 3 | import logging 4 | log = logging.getLogger(__name__) 5 | 6 | from xwing.exceptions import HeartbeatFailureError, ConnectionAlreadyExists 7 | 8 | EOL = b'\n' 9 | HEARTBEAT = b'HEARTBEAT' 10 | HEARTBEAT_SIGNAL = b'HEARTBEAT_SIGNAL' 11 | HEARTBEAT_ACK = b'HEARTBEAT_ACK' 12 | 13 | INITIAL_HEARBEAT_LIVENESS = 3 14 | 15 | 16 | class Repository: 17 | 18 | def __init__(self): 19 | self.connections = {} 20 | 21 | def get(self, identity): 22 | return self[identity] 23 | 24 | def add(self, connection, identity): 25 | log.debug('Adding new connection to {0}'.format(identity)) 26 | if self.connections.get(identity): 27 | raise ConnectionAlreadyExists 28 | 29 | self.connections[identity] = connection 30 | 31 | def __getitem__(self, item): 32 | if not isinstance(item, str): 33 | raise TypeError('Item must be of str type') 34 | 35 | return self.connections[item] 36 | 37 | def __contains__(self, item): 38 | try: 39 | self.__getitem__(item) 40 | except KeyError: 41 | return False 42 | 43 | return True 44 | 45 | 46 | class Connection: 47 | 48 | def __init__(self, loop, stream, task_pool): 49 | self.loop = loop 50 | self.stream = stream 51 | self.task_pool = task_pool 52 | self.liveness = INITIAL_HEARBEAT_LIVENESS 53 | self.stop_event = asyncio.Event() 54 | 55 | def start(self): 56 | self.task_pool.create_task(self.run_heartbeat_loop()) 57 | 58 | async def run_heartbeat_loop(self, heartbeat_interval=5): 59 | self.start_time = time.time() 60 | 61 | while not self.stop_event.is_set(): 62 | if self.liveness <= 0: 63 | # TODO now we need to decide how this is going 64 | # to work. Following this exception, we need to kill 65 | # this close this connection, delete the mailbox 66 | # and kill the process 67 | raise HeartbeatFailureError() 68 | 69 | if time.time() - self.start_time > heartbeat_interval: 70 | self.liveness -= 1 71 | self.start_time = time.time() 72 | self.send(HEARTBEAT_SIGNAL) 73 | log.debug('Sent hearbeat signal message.') 74 | 75 | await asyncio.sleep(0.1) 76 | 77 | def send(self, data): 78 | self.stream.write(data + EOL) 79 | # TODO think more about this, not sure if should be the default case. 80 | # self.stream.writer.drain() 81 | return data 82 | 83 | async def recv(self): 84 | data = await self.stream.readline() 85 | if data is None: 86 | return data 87 | 88 | # TODO may be we can reduce this to one set, just time? 89 | self.liveness = INITIAL_HEARBEAT_LIVENESS 90 | self.start_time = time.time() 91 | 92 | while True: 93 | if not data.startswith(HEARTBEAT): 94 | break 95 | 96 | if data.startswith(HEARTBEAT_SIGNAL): 97 | log.debug('Sending heartbeat ack message.') 98 | self.send(HEARTBEAT_ACK) 99 | 100 | data = await self.stream.readline() 101 | 102 | if data and not data.endswith(EOL): 103 | log.warning('Received a partial message. ' 104 | 'This may indicate a broken pipe.') 105 | # TODO may be we need to raise an exception here 106 | # and only return None when connection is really closed? 107 | return None 108 | 109 | return data[:-1] 110 | 111 | 112 | connection_map = { 113 | 'real': Connection, 114 | } 115 | 116 | 117 | def get_connection(kind): 118 | return connection_map[kind] 119 | -------------------------------------------------------------------------------- /xwing/mailbox/__init__.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import uuid 3 | import asyncio 4 | from functools import partial 5 | 6 | import attr 7 | 8 | from xwing.network.controller import Controller 9 | from xwing.network.transport.stream.client import get_stream_client 10 | from xwing.network.transport.stream.server import get_stream_server 11 | 12 | 13 | def resolve(name_or_pid): 14 | if isinstance(name_or_pid, str): 15 | if '@' in name_or_pid: 16 | name, hub_address = name_or_pid.split('@') 17 | return hub_address, name 18 | else: 19 | return '127.0.0.1', name_or_pid 20 | 21 | return name_or_pid 22 | 23 | 24 | class TaskPool: 25 | 26 | def __init__(self, loop): 27 | self.loop = loop 28 | self.callbacks = [] 29 | 30 | def create_task(self, fn): 31 | fut = self.loop.create_task(fn) 32 | fut.add_done_callback(self.done_callback) 33 | 34 | def done_callback(self, fut): 35 | if not fut.cancelled() and fut.exception: 36 | for callback in self.callbacks: 37 | callback(fut) 38 | 39 | def add_exception_callback(self, callback): 40 | self.callbacks.append(callback) 41 | 42 | 43 | class Mailbox(object): 44 | 45 | def __init__(self, loop, settings): 46 | self.loop = loop 47 | self.settings = settings 48 | self.task_pool = TaskPool(loop) 49 | self.controller = Controller(loop, settings, self.task_pool, 50 | get_stream_client('real'), 51 | get_stream_server('real')) 52 | 53 | def start(self): 54 | self.controller.start() 55 | 56 | def stop(self): 57 | self.controller.stop() 58 | 59 | @property 60 | def pid(self): 61 | return self.settings.hub_frontend, self.settings.identity 62 | 63 | async def recv(self, timeout=None): 64 | payload = await self.controller.get_inbound(timeout=timeout) 65 | return pickle.loads(payload) 66 | 67 | async def send(self, name_or_pid, *args): 68 | payload = pickle.dumps(args) 69 | pid = resolve(name_or_pid) 70 | await self.controller.put_outbound(pid, payload) 71 | 72 | 73 | @attr.s 74 | class Settings(object): 75 | hub_frontend = attr.ib(default='127.0.0.1:5555') 76 | hub_backend = attr.ib(default='/var/tmp/xwing.socket') 77 | identity = attr.ib(default=attr.Factory(uuid.uuid1), convert=str) 78 | 79 | 80 | class Node(object): 81 | 82 | def __init__(self, loop=None, settings={}): 83 | self.loop = asyncio.new_event_loop() 84 | asyncio.set_event_loop(self.loop) 85 | self.settings = Settings(**settings) 86 | self.tasks = [] 87 | 88 | 89 | node_ref = None 90 | 91 | 92 | def get_node_instance(): 93 | global node_ref 94 | return node_ref 95 | 96 | 97 | def init_node(): 98 | global node_ref 99 | node_ref = Node() 100 | 101 | 102 | def spawn(fn, *args, name=None, node=None): 103 | if not node: 104 | node = get_node_instance() 105 | 106 | if name: 107 | node.settings.identity = name 108 | 109 | mailbox = Mailbox(node.loop, node.settings) 110 | mailbox.start() 111 | 112 | # FIXME right now we need this to make sure that 113 | # the finished messages is sent before the actor 114 | # exit and its mailbox gets garbaged. How does 115 | # Erlang fix this problem? 116 | 117 | async def wrap(fn, mailbox, *args): 118 | ret = await fn(mailbox, *args) 119 | await asyncio.sleep(0.1) 120 | return ret 121 | 122 | task = node.loop.create_task(wrap(fn, mailbox, *args)) 123 | node.tasks.append(task) 124 | 125 | def finish(process, fut): 126 | process.set_exception(fut.exception()) 127 | 128 | mailbox.task_pool.add_exception_callback(partial(finish, task)) 129 | return mailbox.pid 130 | 131 | 132 | def start_node(node=None): 133 | '''Start node loop, by running all actors.''' 134 | if not node: 135 | node = get_node_instance() 136 | 137 | try: 138 | done, pending = node.loop.run_until_complete(asyncio.wait( 139 | node.tasks, return_when=asyncio.FIRST_EXCEPTION)) 140 | 141 | # If a exception happened on any of waited tasks 142 | # this forces the exception to buble up 143 | for future in done: 144 | future.result() 145 | finally: 146 | stop_node() 147 | 148 | 149 | def stop_node(node=None): 150 | '''Graceful node stop. 151 | 152 | Cancel all running actors and wait for them to finish before 153 | stopping.''' 154 | if not node: 155 | node = get_node_instance() 156 | 157 | for task in asyncio.Task.all_tasks(): 158 | task.cancel() 159 | 160 | try: 161 | pending = asyncio.Task.all_tasks() 162 | node.loop.run_until_complete(asyncio.wait(pending)) 163 | except RuntimeError: 164 | # Ignore RuntimeErrors like loop is already closed. 165 | # It may happens in KeyboardInterrupt exception for 166 | # example, as asyncio already killed the event loop. 167 | pass 168 | -------------------------------------------------------------------------------- /xwing/hub.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | import socket 4 | import array 5 | import asyncio 6 | 7 | SERVICE_POSITIVE_ANSWER = b'+' 8 | SERVICE_PING = b'!' 9 | BUFFER_SIZE = 4096 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | 14 | class Hub: 15 | '''The Socket Hub implementation. 16 | 17 | Provides a Hub that known how to connect sockets 18 | between clients and servers. 19 | 20 | :param frontend_endpoint: Endpoint where clients will connect. 21 | :type frontend_endpoint: str 22 | :param backend_endpoint: Endpoint where servers will connect. 23 | :type frontend_endpoint: str 24 | :param polling_interval: Interval used on polling socket in seconds. 25 | 26 | Usage:: 27 | 28 | >>> from xwing.hub import Hub 29 | >>> hub = Hub('0.0.0.0:5555', '/var/run/xwing.socket') 30 | >>> hub.run() 31 | ''' 32 | 33 | def __init__(self, frontend_endpoint, backend_endpoint): 34 | self.frontend_endpoint = frontend_endpoint 35 | self.backend_endpoint = backend_endpoint 36 | self.loop = asyncio.get_event_loop() 37 | self.stop_event = asyncio.Event() 38 | self.services = {} 39 | 40 | def run(self): 41 | '''Run the server loop''' 42 | tasks = [ 43 | asyncio.ensure_future(self.run_frontend(self.frontend_endpoint)), 44 | asyncio.ensure_future(self.run_backend(self.backend_endpoint)) 45 | ] 46 | 47 | try: 48 | done, pending = self.loop.run_until_complete(asyncio.wait( 49 | tasks, return_when=asyncio.FIRST_EXCEPTION)) 50 | 51 | # If a exception happned on any of waited tasks 52 | # this forces the exception to buble up 53 | for future in done: 54 | future.result() 55 | except KeyboardInterrupt: 56 | self.stop() 57 | 58 | def stop(self): 59 | '''Loop stop.''' 60 | self.stop_event.set() 61 | for task in asyncio.Task.all_tasks(): 62 | task.cancel() 63 | 64 | self.loop.run_forever() 65 | self.loop.close() 66 | 67 | async def run_frontend(self, tcp_address, backlog=10, timeout=0.1): 68 | log.info('Running frontend loop') 69 | address, port = tcp_address.split(':') 70 | with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: 71 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 72 | sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) 73 | sock.bind((address, int(port))) 74 | sock.listen(backlog) 75 | sock.settimeout(timeout) 76 | 77 | while not self.stop_event.is_set(): 78 | try: 79 | conn, address = await self.loop.sock_accept(sock) 80 | except socket.timeout: 81 | await asyncio.sleep(timeout) 82 | continue 83 | 84 | service = await self.loop.sock_recv(conn, BUFFER_SIZE) 85 | if not service: 86 | break 87 | 88 | if service not in self.services: 89 | self.loop.sock_sendall(conn, b'-Service not found\r\n') 90 | continue 91 | 92 | # detach and pack FD into a array 93 | fd = conn.detach() 94 | fds = array.array("I", [fd]) 95 | 96 | try: 97 | # Send FD to server connection 98 | server_conn = self.services[service] 99 | server_conn.sendmsg([b'1'], [(socket.SOL_SOCKET, 100 | socket.SCM_RIGHTS, fds)]) 101 | except BrokenPipeError: # NOQA 102 | # If connections is broken, the server is gone 103 | # so we need to remove it from services 104 | del self.services[service] 105 | conn = socket.fromfd(fd, socket.AF_INET, 106 | socket.SOCK_STREAM) 107 | conn.sendall(b'-Service not found\r\n') 108 | conn.close() 109 | 110 | async def run_backend(self, unix_address, backlog=10, timeout=0.1): 111 | log.info('Running backend loop') 112 | try: 113 | # Make sure that there is no zombie socket 114 | os.unlink(unix_address) 115 | except OSError: 116 | pass 117 | 118 | with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: 119 | sock.bind(unix_address) 120 | sock.listen(backlog) 121 | sock.settimeout(timeout) 122 | 123 | while not self.stop_event.is_set(): 124 | try: 125 | conn, address = await self.loop.sock_accept(sock) 126 | except socket.timeout: 127 | await asyncio.sleep(timeout) 128 | continue 129 | 130 | service = await self.loop.sock_recv(conn, BUFFER_SIZE) 131 | if not service: # connection was closed 132 | break 133 | 134 | server_conn = self.services.get(service) 135 | if server_conn: 136 | try: 137 | server_conn.sendall(SERVICE_PING) 138 | except BrokenPipeError: # NOQA 139 | # If connections is broken, the server is gone 140 | # so we need to remove it from services 141 | del self.services[service] 142 | else: 143 | conn.sendall(b'-Service already exists\r\n') 144 | continue 145 | 146 | # TODO we should detach the fd from connection 147 | # can it be that conn variable will be collected 148 | # and the connection will be closed? 149 | self.services[service] = conn 150 | await self.loop.sock_sendall(conn, SERVICE_POSITIVE_ANSWER) 151 | --------------------------------------------------------------------------------