├── tests ├── __init__.py ├── python-var │ └── app.py ├── .DS_Store ├── example01 │ ├── run.py │ ├── app01.py │ └── friend01.py ├── example02 │ ├── run.py │ ├── test.py │ ├── app02.py │ └── friend02.py ├── example03 │ ├── run.py │ ├── test.py │ ├── app02.py │ └── friend02.py ├── example00 │ ├── app2.py │ └── app1.py ├── example05 │ ├── clients2.py │ ├── clients.py │ ├── server.py │ ├── url_parser │ │ └── url_namespace.py │ └── router.py ├── example04 │ ├── clients.py │ ├── test_async_thread.py │ ├── server.py │ ├── async_loop.py │ ├── test_distributed_threads.py │ ├── test_concurrency.py │ └── spawn.py └── libs.py ├── microservices_connector ├── url_parser │ ├── __init__.py │ └── url_namespace.py ├── __init__.py ├── minisocket.py ├── spawn.py └── Interservices.py ├── runtest.py ├── .DS_Store ├── images └── Distributed-system.jpeg ├── LICENSE ├── .gitignore ├── setup.py ├── README.md └── README.rst /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/python-var/app.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /microservices_connector/url_parser/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /runtest.py: -------------------------------------------------------------------------------- 1 | # from Microservices_connector.microservices_connector import Interservices -------------------------------------------------------------------------------- /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/minhtuan221/Microservices-connector/HEAD/.DS_Store -------------------------------------------------------------------------------- /tests/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/minhtuan221/Microservices-connector/HEAD/tests/.DS_Store -------------------------------------------------------------------------------- /tests/example01/run.py: -------------------------------------------------------------------------------- 1 | from app01 import Micro 2 | 3 | if __name__ == '__main__': 4 | Micro.run(debug=True) 5 | -------------------------------------------------------------------------------- /images/Distributed-system.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/minhtuan221/Microservices-connector/HEAD/images/Distributed-system.jpeg -------------------------------------------------------------------------------- /tests/example02/run.py: -------------------------------------------------------------------------------- 1 | from app02 import Micro 2 | 3 | if __name__ == '__main__': 4 | Micro.run(port=5000, host='0.0.0.0', debug=True) 5 | -------------------------------------------------------------------------------- /tests/example03/run.py: -------------------------------------------------------------------------------- 1 | from app02 import Micro 2 | 3 | if __name__ == '__main__': 4 | Micro.run(port=5000, host='0.0.0.0', debug=True) 5 | -------------------------------------------------------------------------------- /tests/example00/app2.py: -------------------------------------------------------------------------------- 1 | from microservices_connector.Interservices import Friend 2 | 3 | aFriend= Friend('app1', 'http://0.0.0.0:5000') 4 | message = aFriend.send('/helloworld','Mr. Developer') 5 | print(message) 6 | -------------------------------------------------------------------------------- /tests/example00/app1.py: -------------------------------------------------------------------------------- 1 | from microservices_connector.Interservices import Microservice 2 | 3 | M = Microservice(__name__) 4 | 5 | @M.typing('/helloworld') 6 | @M.reply 7 | def helloworld(name): 8 | return 'Welcome %s' % (name) 9 | 10 | if __name__ == '__main__': 11 | M.run() -------------------------------------------------------------------------------- /tests/example02/test.py: -------------------------------------------------------------------------------- 1 | from sanic import Sanic 2 | from sanic.response import json 3 | 4 | app = Sanic() 5 | 6 | r = frozenset({'GET'}) 7 | print(r) 8 | 9 | @app.route("/") 10 | async def test(request): 11 | return json({"hello": "world"}) 12 | 13 | if __name__ == "__main__": 14 | app.run(host="0.0.0.0", port=8000) 15 | -------------------------------------------------------------------------------- /tests/example03/test.py: -------------------------------------------------------------------------------- 1 | from sanic import Sanic 2 | from sanic.response import json 3 | 4 | app = Sanic() 5 | 6 | r = frozenset({'GET'}) 7 | print(r) 8 | 9 | @app.route("/") 10 | async def test(request): 11 | return json({"hello": "world"}) 12 | 13 | if __name__ == "__main__": 14 | app.run(host="0.0.0.0", port=8000) 15 | -------------------------------------------------------------------------------- /microservices_connector/__init__.py: -------------------------------------------------------------------------------- 1 | from . import Interservices, spawn, minisocket 2 | 3 | __all__ = ['Interservices','spawn','minisocket'] 4 | __author__ = 'Minh Tuan Nguyen' 5 | __version__ = '0.1.0' 6 | __copyright__ = "Copyright 2018, Minh Tuan Nguyen" 7 | __credits__ = ["Minh Tuan Nguyen", ] 8 | 9 | # python setup.py bdist_wheel 10 | # python setup.py sdist bdist_wheel 11 | # twine upload dist/* 12 | -------------------------------------------------------------------------------- /tests/example05/clients2.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import websockets 3 | 4 | 5 | async def hello(): 6 | async with websockets.connect('ws://localhost:8765/hello') as websocket: 7 | greeting = '?' 8 | while greeting != 'exit': 9 | name = input("What's your name? ") 10 | await websocket.send(name) 11 | greeting = await websocket.recv() 12 | print(f"< {greeting}") 13 | 14 | asyncio.get_event_loop().run_until_complete(hello()) 15 | -------------------------------------------------------------------------------- /tests/example04/clients.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import websockets 3 | 4 | 5 | async def hello(): 6 | async with websockets.connect('ws://localhost:8765/abcdzyx') as websocket: 7 | name = input("What's your name? ") 8 | print(f"> {name}") 9 | greeting = await websocket.recv() 10 | print(f"< {greeting}") 11 | while greeting != 'close': 12 | await websocket.send(name) 13 | greeting = await websocket.recv() 14 | print(f"< {greeting}") 15 | name = input("What's your name? ") 16 | 17 | asyncio.get_event_loop().run_until_complete(hello()) 18 | -------------------------------------------------------------------------------- /tests/example05/clients.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import websockets 3 | 4 | from cython_npm.cythoncompile import require 5 | minisocket = require('../../microservices_connector/minisocket') 6 | 7 | # ws = minisocket.SocketClient(host='localhost:8765',url='/hello/minhtuan') 8 | def hello(ws): 9 | greeting = '?' 10 | while greeting != 'exit': 11 | name = input("What's your name? ") 12 | ws.send(name) 13 | greeting = ws.recv() 14 | print(f"< {greeting}") 15 | if name == 'exit': 16 | break 17 | 18 | # hello(ws) 19 | 20 | ws = minisocket.SocketClient(host='localhost:8765', url='/render') 21 | ws.send('test render socket') 22 | x = ws.recv() 23 | # ws.send('other things') 24 | # ws.send('other things') 25 | ws.send(None) 26 | print(x) 27 | -------------------------------------------------------------------------------- /tests/example04/test_async_thread.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import time 3 | import queue 4 | 5 | import random 6 | 7 | 8 | class AsyncThread(threading.Thread): 9 | """Threaded website reader""" 10 | 11 | def __init__(self, queue, f): 12 | threading.Thread.__init__(self) 13 | self.queue = queue 14 | self.f = f 15 | 16 | def run(self): 17 | while True: 18 | # Grabs host from queue 19 | host = self.queue.get() 20 | 21 | # Grabs urls of hosts and then grabs chunk of webpage 22 | self.f(host) 23 | print("Reading: %s" % host) 24 | 25 | # Signals to queue job is done 26 | self.queue.task_done() 27 | 28 | 29 | queue = queue.Queue() 30 | threads =[] 31 | def test(item): 32 | print('item is:',item) 33 | for i in range(5): 34 | t = AsyncThread(queue, test) 35 | t.daemon = True 36 | threads.append(t) 37 | t.start() 38 | hosts = [i for i in range(20)] 39 | # Populate queue with data 40 | for host in hosts: 41 | queue.put(host) 42 | # Wait on the queue until everything has been processed 43 | queue.join() 44 | -------------------------------------------------------------------------------- /tests/example05/server.py: -------------------------------------------------------------------------------- 1 | # from minisocket import SocketServer 2 | from microservices_connector.Interservices import Microservice 3 | import threading, time 4 | from cython_npm.cythoncompile import require 5 | SocketServer = require('../../microservices_connector/minisocket').SocketServer 6 | 7 | sk = SocketServer(__name__) 8 | app = Microservice('Flask_app').app 9 | 10 | @app.route('/') 11 | def helloworld(): 12 | time.sleep(2) 13 | return 'Sleep 2s before response' 14 | 15 | 16 | @sk.route('/hello/') 17 | def test(message, name): 18 | print(message,'hello:'+name) 19 | return 'hello:'+message 20 | 21 | 22 | @sk.route('/xinchao') 23 | def test2(message): 24 | print(message, 'End:') 25 | return 'xinchao:'+message 26 | 27 | 28 | @sk.render('/render') 29 | async def test3(ws, message): 30 | print(message, 'received') 31 | await ws.send('Render 1:'+message) 32 | x = await ws.recv() 33 | print(x, type(x)) 34 | ws.close() 35 | 36 | def socket_runner(): 37 | sk.run() 38 | 39 | def main(): 40 | socket_runner() 41 | print('start web framework') 42 | app.run() 43 | 44 | 45 | if __name__ == '__main__': 46 | main() 47 | -------------------------------------------------------------------------------- /tests/example04/server.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import websockets 3 | import spawn 4 | import threading, time 5 | from microservices_connector.Interservices import Microservice 6 | Micro = Microservice(__name__) 7 | 8 | async def hello(websocket, path): 9 | name = await websocket.recv() 10 | print(f"< {name}") 11 | 12 | greeting = f"Hello {name}!" 13 | 14 | await websocket.send(greeting) 15 | print(f"> {greeting}") 16 | 17 | def socket_run(): 18 | print('websocket is starting') 19 | loop = asyncio.new_event_loop() 20 | asyncio.set_event_loop(loop) 21 | start_server = websockets.serve(hello, 'localhost', 8765) 22 | loop.run_until_complete(start_server) 23 | loop.run_forever() 24 | 25 | @Micro.typing('/', methods=['GET']) 26 | def string1(): 27 | time.sleep(2) 28 | return 'Sleep 2s before response' 29 | 30 | def flask_app(): 31 | Micro.run(port=5000, host='0.0.0.0') 32 | 33 | if __name__ == '__main__': 34 | # t = threading.Thread(target=flask_app) 35 | # t.start() 36 | s = threading.Thread(target=socket_run) 37 | s.start() 38 | flask_app() 39 | # asyncio.get_event_loop().run_until_complete(start_server) 40 | # asyncio.get_event_loop().run_forever() 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The 3-Clause BSD License 2 | 3 | Copyright (c) 2018 Minh Tuan Nguyen 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | .vscode/ -------------------------------------------------------------------------------- /tests/example04/async_loop.py: -------------------------------------------------------------------------------- 1 | from microservices_connector import spawn, Interservices as conn 2 | import time 3 | import asyncio 4 | from datetime import datetime 5 | friend = conn.Friend('1','10.26.53.142:5010') 6 | import requests 7 | import concurrent.futures as fn 8 | 9 | async def myTask(i): 10 | time.sleep(1) 11 | print("Processing Task %s" % i) 12 | 13 | 14 | @spawn.async_to_sync 15 | async def myTaskGenerator(): 16 | for i in range(5): 17 | asyncio.ensure_future(myTask(i)) 18 | 19 | # myTaskGenerator() 20 | async def call_api(): 21 | r = requests.get('http://example.com/') 22 | print('request status is %s' % r.text) 23 | 24 | # @spawn.async_to_sync 25 | async def low_long_job(param): 26 | print('Long job %s begin' % param) 27 | await call_api() 28 | print('end %s'%param) 29 | return 'end' 30 | 31 | # for i in range(100): 32 | # low_long_job(i) 33 | 34 | # print('End of 100 long job') 35 | tasks=[] 36 | start = time.time() 37 | 38 | # @spawn.sync_to_async 39 | def do_async(): 40 | try: 41 | loop = asyncio.new_event_loop() 42 | asyncio.set_event_loop(loop) 43 | for i in range(10): 44 | tasks.append(asyncio.ensure_future(low_long_job(i))) 45 | loop.run_until_complete(asyncio.wait(tasks)) 46 | except KeyboardInterrupt: 47 | pass 48 | finally: 49 | loop.close() 50 | do_async() 51 | 52 | print('This will out first') 53 | 54 | 55 | # async def custom_sleep(): 56 | # print('SLEEP {}\n'.format(datetime.now())) 57 | # await asyncio.sleep(1) 58 | 59 | 60 | # async def factorial(name, number): 61 | # f = 1 62 | # for i in range(2, number+1): 63 | # print('Task {}: Compute factorial({})'.format(name, i)) 64 | # await custom_sleep() 65 | # f *= i 66 | # print('Task {}: factorial({}) is {}\n'.format(name, number, f)) 67 | 68 | 69 | # loop = asyncio.get_event_loop() 70 | # tasks = [ 71 | # asyncio.ensure_future(factorial("A", 3)), 72 | # asyncio.ensure_future(factorial("B", 4)), 73 | # ] 74 | # loop.run_until_complete(asyncio.wait(tasks)) 75 | # loop.close() 76 | end = time.time() 77 | print("Total time: {}".format(end - start)) 78 | 79 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import microservices_connector 3 | from setuptools import setup, find_packages 4 | 5 | # python setup.py register -r pypi 6 | # python setup.py sdist bdist_wheel 7 | # twine upload dist/* 8 | 9 | # I really prefer Markdown to reStructuredText. PyPi does not. This allows me 10 | # to have things how I'd like, but not throw complaints when people are trying 11 | # to install the package and they don't have pypandoc or the README in the 12 | # right place. 13 | readme = '' 14 | try: 15 | import pypandoc 16 | readme = pypandoc.convert('README.md', 'rst', format='md') 17 | with open('README.rst', 'w') as out: 18 | out.write(readme) 19 | except (IOError, ImportError): 20 | readme = '' 21 | 22 | 23 | setup( 24 | name='microservices_connector', 25 | version='0.3.7', 26 | description='Inter-Service communication framework, support for microservice architecture and distributed system via http', 27 | long_description='README.rst', 28 | author='Minh Tuan Nguyen', 29 | author_email='ntuan221@gmail.com', 30 | license='BSD', 31 | platforms='any', 32 | url='https://github.com/minhtuan221/Microservices-connector', 33 | classifiers=[ 34 | 'Development Status :: 3 - Alpha', 35 | 'License :: OSI Approved :: BSD License', 36 | 'Environment :: Web Environment', 37 | 'Operating System :: OS Independent', 38 | 'Programming Language :: Python', 39 | 'Programming Language :: Python :: 3.5', 40 | 'Programming Language :: Python :: 3.6', 41 | 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', 42 | 'Topic :: Internet :: WWW/HTTP :: WSGI :: Application', 43 | 'Topic :: Software Development :: Libraries :: Application Frameworks', 44 | 'Topic :: Software Development :: Libraries :: Python Modules', 45 | ], 46 | keywords=['microservice', 'http', 'flask', 'sanic', 'websockets'], 47 | # entry_points={'console_scripts': [ 48 | # 'microservices_connector = microservices_connector:main', 49 | # ]}, 50 | packages=find_packages(exclude=('test*', 'testpandoc*','image*','runtest*')), 51 | include_package_data=False, 52 | install_requires=['flask', 'requests', 53 | 'sanic', 'websockets', 'websocket_client'], 54 | ) 55 | -------------------------------------------------------------------------------- /tests/example03/app02.py: -------------------------------------------------------------------------------- 1 | from cython_npm.cythoncompile import require 2 | from functools import wraps 3 | # from Interservices import SanicApp as Microservice 4 | 5 | module = require('../../microservices_connector/Interservices') 6 | Microservice = module.AioSocket 7 | timeit = module.timeit 8 | 9 | Micro = Microservice(__name__) 10 | 11 | # run a normal function in python 12 | print('one cat here') 13 | 14 | # test return string 15 | 16 | 17 | @Micro.route('/str') 18 | @Micro.async_json 19 | async def string1(key): 20 | return '-'+key 21 | 22 | # test return multiple string 23 | 24 | 25 | @Micro.route('/str2') 26 | @Micro.async_json 27 | async def string2(a, b, key): 28 | return a, key, b+'-'+key 29 | 30 | # test return Integer and float 31 | 32 | 33 | @Micro.typing('/int') 34 | @Micro.async_json 35 | async def int1(a, key): 36 | return a+key 37 | 38 | 39 | @Micro.typing('/float') 40 | @Micro.async_json 41 | async def float2(a, key): 42 | return a+key 43 | 44 | 45 | @Micro.typing('/int3') 46 | @Micro.async_json 47 | async def int3(a, key, key2): 48 | return a+key2, key*key, a*a 49 | 50 | # test return list and dict 51 | 52 | 53 | @Micro.typing('/list') 54 | @Micro.async_json 55 | async def list1(a, key): 56 | a.extend(key) 57 | return a 58 | 59 | 60 | @Micro.typing('/dict') 61 | @Micro.async_json 62 | async def dict1(a, key): 63 | key['dict'] = a 64 | return key 65 | 66 | 67 | @Micro.typing('/list3') 68 | @Micro.async_json 69 | async def list3(a, key): 70 | key.append('other value') 71 | c = None 72 | return a, key, c 73 | 74 | # return None, class Object 75 | 76 | 77 | @Micro.typing('/None') 78 | @Micro.async_json 79 | async def TestNoneValue(a, key): 80 | key.append('Do something in the server') 81 | 82 | 83 | class testservice(object): 84 | name = 'test' 85 | Purpose = 'For test only' 86 | empty = None 87 | 88 | def __init__(self, value): 89 | self.value = value 90 | 91 | def onemethod(self): 92 | print('This is test class') 93 | 94 | 95 | @Micro.typing('/class', token='123456') 96 | @Micro.async_json 97 | async def TestClass(a, key): 98 | t = testservice(a) 99 | return t 100 | 101 | 102 | @Micro.typing('/class2', token='123456') 103 | @Micro.async_json 104 | async def TestClass2(a, key): 105 | t = testservice(key) 106 | return t, a, None 107 | 108 | 109 | @Micro.typing('/class3', token='123456') 110 | @Micro.async_json 111 | @timeit 112 | async def TestClass3(a, key): 113 | x = testservice(key) 114 | y = testservice(a) 115 | z = [y, x] 116 | return x, y, z 117 | 118 | 119 | @Micro.typing('/json') 120 | @Micro.async_json 121 | @timeit 122 | async def TestReceiveJson(a=1, b='string', c=None): 123 | return {'1':a,'2':b,'3':c} 124 | 125 | 126 | @Micro.route('/json1') 127 | @Micro.async_json 128 | async def TestReceiveJson2(a=None): 129 | return a 130 | 131 | 132 | # Option 1: run Microservice within file it's created 133 | if __name__ == '__main__': 134 | Micro.run(port=5000, host='0.0.0.0') 135 | -------------------------------------------------------------------------------- /tests/example04/test_distributed_threads.py: -------------------------------------------------------------------------------- 1 | # import time 2 | from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor 3 | from collections import deque 4 | import queue 5 | import threading 6 | import multiprocessing 7 | import random 8 | import time 9 | import spawn 10 | 11 | def wait_on_b(b): 12 | # print('working on b=%s' % b) 13 | time.sleep(random.random()) 14 | return 'working on b=%s' % b 15 | # return 5 16 | 17 | 18 | def wait_on_a(a): 19 | # print('working on a=%s' % a) 20 | time.sleep(1) 21 | return a 22 | # return 6 23 | 24 | 25 | poll = [ 26 | {'id': 1, 'x': 'Nguyen'}, 27 | {'id': 1, 'x': 'Minh'}, 28 | {'id': 1, 'x': 'Tuan'}, 29 | {'id': 2, 'x': 'Vu'}, 30 | {'id': 3, 'x': 'Ai do khac'}, 31 | {'id': 2, 'x': 'Kim'}, 32 | {'id': 2, 'x': 'Oanh'}, 33 | {'id': 4, 'x': '1'}, 34 | {'id': 4, 'x': '2'}, 35 | {'id': 4, 'x': '3'}, 36 | {'id': 4, 'x': '4'}, 37 | {'id': 4, 'x': '5'}, 38 | {'id': 4, 'x': '6'}, 39 | {'id': 4, 'x': '7'}, 40 | {'id': 4, 'x': '8'}, 41 | {'id': 5, 'x': '101'}, 42 | {'id': 5, 'x': '102'}, 43 | {'id': 5, 'x': '103'}, 44 | {'id': 5, 'x': '104'}, 45 | {'id': 5, 'x': '105'}, 46 | {'id': 6, 'x': 'Test watching'}, 47 | {'id': 6, 'x': 'Test watching'}, 48 | {'id': 7, 'x': 'Test watching'}, 49 | {'id': 8, 'x': 'Test watching'}, 50 | {'id': 9, 'x': 'Test watching'}, 51 | {'id': 10, 'x': 'Test watching'}, 52 | {'id': 11, 'x': 'Test watching'}, 53 | {'id': 12, 'x': 'Test watching'}, 54 | {'id': 13, 'x': 'Test watching'}, 55 | {'id': 14, 'x': 'Test watching'}, 56 | {'id': 15, 'x': 'Test watching'}, 57 | {'id': 16, 'x': 'Test watching'}, 58 | {'id': 17, 'x': 'Test watching'}, 59 | {'id': 18, 'x': 'Test watching'}, 60 | {'id': 19, 'x': 'Test watching'}, 61 | {'id': 20, 'x': 'Test watching'}, 62 | {'id': 21, 'x': 'Test watching'}, 63 | {'id': 22, 'x': 'Test watching'}, 64 | ] 65 | 66 | def Print_out(q): 67 | while q: 68 | obj = q.get() 69 | print(obj) 70 | 71 | 72 | def main(): 73 | 74 | thread_out_queue = queue.Queue() 75 | pool = spawn.DistributedThreads( 76 | max_workers=4, max_watching=100, out_queue=thread_out_queue) 77 | 78 | start = time.time() 79 | for item in poll: 80 | pool.submit_id(item['id'], wait_on_a, item) 81 | # print(thread_out_queue.get()) 82 | t = spawn.Worker(thread_out_queue, print) 83 | t.daemon = True 84 | t.start() 85 | for w in pool.worker_list: 86 | print(w.name) 87 | pool.shutdown() 88 | print('Finish after: ', time.time()-start, 'seconds') 89 | 90 | print("========= End of threads ==============") 91 | 92 | # process_out_queue = multiprocessing.Queue() 93 | # pool2 = spawn.DistributedProcess( 94 | # max_workers=4, max_watching=100, out_queue=process_out_queue) 95 | # for item in poll: 96 | # pool2.submit_id(item['id'], wait_on_a, item) 97 | # # process = Worker(process_out_queue, print) 98 | # # process.daemon = True 99 | # # process.start() 100 | # pool2.shutdown() 101 | 102 | print('Finish after: ', time.time()-start, 'seconds') 103 | 104 | 105 | if __name__ == '__main__': 106 | main() 107 | -------------------------------------------------------------------------------- /tests/example02/app02.py: -------------------------------------------------------------------------------- 1 | from cython_npm.cythoncompile import require 2 | from functools import wraps 3 | # from Interservices import SanicApp as Microservice 4 | 5 | module = require('../../microservices_connector/Interservices') 6 | Microservice = module.SanicApp 7 | timeit = module.timeit 8 | 9 | Micro = Microservice(__name__) 10 | 11 | # run a normal function in python 12 | print('one cat here') 13 | 14 | # test return string 15 | 16 | 17 | @Micro.route('/str') 18 | @Micro.reply 19 | def string1(a, key): 20 | return a+'-'+key 21 | 22 | # test return multiple string 23 | 24 | 25 | @Micro.route('/str2') 26 | @Micro.reply 27 | def string2(a, b, key): 28 | return a, key, b+'-'+key 29 | 30 | # test return Integer and float 31 | 32 | 33 | @Micro.typing('/int') 34 | @Micro.reply 35 | def int1(a, key): 36 | return a+key 37 | 38 | 39 | @Micro.typing('/float') 40 | @Micro.reply 41 | def float2(a, key): 42 | return a+key 43 | 44 | 45 | @Micro.typing('/int3') 46 | @Micro.reply 47 | def int3(a, key, key2): 48 | return a+key2, key*key, a*a 49 | 50 | # test return list and dict 51 | 52 | 53 | @Micro.typing('/list') 54 | @Micro.reply 55 | def list1(a, key): 56 | a.extend(key) 57 | return a 58 | 59 | 60 | @Micro.typing('/dict') 61 | @Micro.reply 62 | def dict1(a, key): 63 | key['dict'] = a 64 | return key 65 | 66 | 67 | @Micro.typing('/list3') 68 | @Micro.reply 69 | def list3(a, key): 70 | key.append('other value') 71 | c = None 72 | return a, key, c 73 | 74 | # return None, class Object 75 | 76 | 77 | @Micro.typing('/None') 78 | @Micro.reply 79 | def TestNoneValue(a, key): 80 | key.append('Do something in the server') 81 | 82 | 83 | class testservice(object): 84 | name = 'test' 85 | Purpose = 'For test only' 86 | empty = None 87 | 88 | def __init__(self, value): 89 | self.value = value 90 | 91 | def onemethod(self): 92 | print('This is test class') 93 | 94 | 95 | @Micro.typing('/class', token='123456') 96 | @Micro.reply 97 | def TestClass(a, key): 98 | t = testservice(a) 99 | return t 100 | 101 | 102 | @Micro.typing('/class2', token='123456') 103 | @Micro.reply 104 | def TestClass2(a, key): 105 | t = testservice(key) 106 | return t, a, None 107 | 108 | 109 | @Micro.typing('/class3', token='123456') 110 | @Micro.reply 111 | @timeit 112 | def TestClass3(a, key): 113 | x = testservice(key) 114 | y = testservice(a) 115 | z = [y, x] 116 | return x, y, z 117 | 118 | 119 | @Micro.typing('/json') 120 | @Micro.json 121 | @timeit 122 | def TestReceiveJson(a=1, b='string',c=None): 123 | return {'1':a,'2':b,'3':c} 124 | 125 | 126 | @Micro.route('/json1', methods=['GET','POST']) 127 | @Micro.async_json 128 | async def TestReceiveJson2(a=None): 129 | return a 130 | 131 | 132 | @Micro.typing('/get/none', methods=['GET', 'POST']) 133 | @Micro.json 134 | def TestReceiveJsonGet(): 135 | return {'1': 'a', '2': 'b', '3': 'c'} 136 | 137 | 138 | @Micro.typing('/post/none', methods=['GET', 'POST']) 139 | @Micro.json 140 | def TestReceiveJsonNone(): 141 | return {'1': 'a', '2': 'b', '3': 'c'} 142 | 143 | 144 | @Micro.route('/one', methods=['POST', 'GET']) 145 | @Micro.dict 146 | def TestOne(): 147 | return [12121212] 148 | 149 | 150 | @Micro.route('/one2', methods=['POST', 'GET']) 151 | @Micro.dict 152 | def TestOne2(): 153 | return {'data': 'something'} 154 | 155 | # Option 1: run Microservice within file it's created 156 | if __name__ == '__main__': 157 | Micro.run(port=5000, host='0.0.0.0', debug=True) 158 | -------------------------------------------------------------------------------- /microservices_connector/minisocket.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import os 3 | import traceback 4 | import threading 5 | import websockets 6 | import websocket 7 | import uvloop 8 | import time 9 | import threading 10 | from microservices_connector.url_parser.url_namespace import ArgsParse 11 | 12 | 13 | def SocketClient(host='localhost:8765', url='/'): 14 | return websocket.create_connection(f'ws://{host}{url}') 15 | 16 | 17 | async def raw_middleware(websocket, handler, *args): 18 | message = await websocket.recv() 19 | await handler(websocket, message, *args) 20 | 21 | 22 | class SocketServer(threading.Thread): 23 | def __init__(self, name=__file__): 24 | threading.Thread.__init__(self) 25 | self.name = name 26 | self.url = {} 27 | self.url_args = {} 28 | self.timeout = 1000 * 60 29 | 30 | def router(self, rule): 31 | def response(handler): 32 | self.add_route(rule, handler) 33 | return handler 34 | return response 35 | 36 | route = router 37 | 38 | def render(self, rule): 39 | def response(handler): 40 | self.add_route(rule, handler, middleware=raw_middleware) 41 | return handler 42 | return response 43 | 44 | def add_route(self, rule, handler, middleware=None): 45 | if not middleware: 46 | print(rule, 'middelware is None') 47 | middleware = self.basic_middleware 48 | args = ArgsParse(rule) 49 | if args.is_hashable(): 50 | self.url[rule] = handler, middleware 51 | else: 52 | self.url_args[rule] = handler, middleware 53 | 54 | async def basic_middleware(self, websocket, handler, *args): 55 | message = '?' 56 | while message != 'exit': 57 | try: 58 | message = await websocket.recv() 59 | reply = handler(message, *args) 60 | if reply is not None: 61 | await websocket.send(reply) 62 | except: 63 | traceback.print_exc() 64 | 65 | async def handle_immutalble_route(self, websocket, path, *args): 66 | handler, middleware = self.url[path] 67 | await middleware(websocket, handler, *args) 68 | 69 | async def handle_mutalble_route(self, websocket, path, *args): 70 | handler, middleware = self.url_args[path] 71 | await middleware(websocket, handler, *args) 72 | 73 | async def connect(self, websocket, path): 74 | # check if url is immutalble or contain args 75 | if path in self.url: 76 | await self.handle_immutalble_route(websocket, path) 77 | else: 78 | matched_rule = None 79 | for rule in self.url_args: 80 | args = ArgsParse(rule) 81 | if args.parse(path) is not None: 82 | matched_rule = rule 83 | break 84 | if matched_rule: 85 | await self.handle_mutalble_route(websocket, rule, *args.parse(path)) 86 | else: 87 | await websocket.send('Websocket close: path does not exist') 88 | 89 | def server(self, host='127.0.0.1', port=8765): 90 | print("Starting socket in %s:%s" % (host, port)) 91 | loop = uvloop.new_event_loop() 92 | asyncio.set_event_loop(loop) 93 | start_server = websockets.serve(self.connect, host, port) 94 | loop.run_until_complete(start_server) 95 | loop.run_forever() 96 | 97 | def run(self, host='127.0.0.1', port=8765): 98 | s = threading.Thread(target=self.server, args=(host, port)) 99 | s.daemon = True 100 | s.start() 101 | 102 | 103 | def main(): 104 | sk = SocketServer(__name__) 105 | 106 | @sk.router('/hello') 107 | def test(message): 108 | print(message) 109 | return 'ok:'+message 110 | sk.run() 111 | 112 | 113 | if __name__ == '__main__': 114 | main() 115 | -------------------------------------------------------------------------------- /tests/libs.py: -------------------------------------------------------------------------------- 1 | 2 | # send json by request and received 3 | import requests 4 | from flask import request, Response 5 | r = requests.post('http://httpbin.org/post', json={"key": "value"}) 6 | r.status_code 7 | r.json() 8 | 9 | # receive json in flask 10 | @app.route('/api/add_message/', methods=['GET', 'POST']) 11 | def add_message(uuid): 12 | if request.is_json(): 13 | content = request.json 14 | print(content) 15 | return uuid 16 | 17 | # calculate response time in flask 18 | @app.after_request 19 | def after_request(response): 20 | diff = time.time() - g.start 21 | if (response.response): 22 | response.response[0] = response.response[0].replace( 23 | '__EXECUTION_TIME__', str(diff)) 24 | return response 25 | 26 | 27 | # response is a WSGI object, and that means the body of the response must be an iterable. For jsonify() responses that's just a list with just one string in it. 28 | 29 | # However, you should use the response.get_data() method here to retrieve the response body, as that'll flatten the response iterable for you. 30 | """process_response(response) 31 | Can be overridden in order to modify the response object before it’s sent to the WSGI server. By default this will call all the after_request() decorated functions. 32 | 33 | Changed in version 0.5: As of Flask 0.5 the functions registered for after request execution are called in reverse order of registration. 34 | 35 | Parameters: response – a response_class object. 36 | Returns: a new response object or the same, has to be an instance of response_class.""" 37 | # The following should work: 38 | 39 | 40 | @app.after_request 41 | def after(response): 42 | d = json.loads(response.get_data()) 43 | d['altered'] = 'this has been altered...GOOD!' 44 | response.set_data(json.dumps(d)) 45 | return response 46 | # Don't use jsonify() again here 47 | # that returns a full new response object 48 | # all you want is the JSON response body here. 49 | 50 | # Do use response.set_data() as that'll also adjust the Content-Length header to reflect the altered response size. 51 | 52 | 53 | @app.route('/data') 54 | def get_data(): 55 | return {'foo': 'bar'} 56 | 57 | 58 | # Here is a custom response class that supports the above syntax, without affecting how other routes that do not return JSON work in any way: 59 | 60 | 61 | class MyResponse(Response): 62 | @classmethod 63 | def force_type(cls, rv, environ=None): 64 | if isinstance(rv, dict): 65 | rv = jsonify(rv) 66 | return super(MyResponse, cls).force_type(rv, environ) 67 | 68 | 69 | # Using a Custom Response Class 70 | # By now I'm sure you agree that there are some interesting use cases that can benefit from using a custom response class. Before I show you some actual examples, let me tell you how simple it is to configure a Flask application to use a custom response class. Take a look at the following example: 71 | 72 | from flask import Flask, Response 73 | 74 | 75 | class MyResponse1(Response): 76 | pass 77 | 78 | 79 | app = Flask(__name__) 80 | app.response_class = MyResponse1 81 | 82 | # ... 83 | from flask import Flask, Response 84 | 85 | 86 | class MyResponse2(Response): 87 | pass 88 | 89 | 90 | class MyFlask(Flask): 91 | response_class = MyResponse2 92 | 93 | 94 | app = MyFlask(__name__) 95 | 96 | # ... 97 | 98 | # Changing Response Defaults 99 | 100 | 101 | class MyResponse3(Response): 102 | default_mimetype = 'application/xml' 103 | 104 | 105 | # Determining Content Type Automatically 106 | class MyResponse4(Response): 107 | def __init__(self, response, **kwargs): 108 | if 'mimetype' not in kwargs and 'contenttype' not in kwargs: 109 | if response.startswith('' #'/event/' 17 | 18 | class ArgsParse(object): 19 | 20 | properties = {"unhashable": None} 21 | 22 | def __init__(self, url_pattern): 23 | self.url_pattern = url_pattern 24 | self.parameter_pattern = re.compile(r'<(.+?)>') 25 | pattern_string = re.sub(self.parameter_pattern, 26 | self.add_parameter, self.url_pattern) 27 | self.pattern = re.compile(r'^{}$'.format(pattern_string)) 28 | 29 | def parse_parameter_string(self, parameter_string): 30 | """Parse a parameter string into its constituent name, type, and 31 | pattern 32 | For example:: 33 | parse_parameter_string('')` -> 34 | ('param_one', str, '[A-z]') 35 | :param parameter_string: String to parse 36 | :return: tuple containing 37 | (parameter_name, parameter_type, parameter_pattern) 38 | """ 39 | # We could receive NAME or NAME:PATTERN 40 | name = parameter_string 41 | pattern = 'string' 42 | if ':' in parameter_string: 43 | name, pattern = parameter_string.split(':', 1) 44 | if not name: 45 | raise ValueError( 46 | "Invalid parameter syntax: {}".format(parameter_string) 47 | ) 48 | 49 | default = (str, pattern) 50 | # Pull from pre-configured types 51 | _type, pattern = REGEX_TYPES.get(pattern, default) 52 | 53 | return name, _type, pattern 54 | 55 | def add_parameter(self, match): 56 | name = match.group(1) 57 | name, _type, pattern = self.parse_parameter_string(name) 58 | 59 | # parameter = Parameter( 60 | # name=name, cast=_type) 61 | # parameters.append(parameter) 62 | 63 | # Mark the whole route as unhashable if it has the hash key in it 64 | if re.search(r'(^|[^^]){1}/', pattern): 65 | self.properties['unhashable'] = True 66 | # Mark the route as unhashable if it matches the hash key 67 | elif re.search(r'/', pattern): 68 | self.properties['unhashable'] = True 69 | 70 | return '({})'.format(pattern) # return '(?P{})'.format(pattern) 71 | 72 | def parse(self, url): 73 | res = self.pattern.search(url) 74 | if res is None: 75 | return None 76 | else: 77 | return self.pattern.search(url).groups() 78 | 79 | def is_hashable(self): 80 | if self.properties["unhashable"] is True: 81 | return False 82 | else: 83 | return True 84 | 85 | def main(): 86 | 87 | #======================================= 88 | # Test parse 2 args 89 | url_pattern = '/event' 90 | a = ArgsParse(url_pattern) 91 | 92 | # check propertise 93 | print(a.is_hashable()) 94 | 95 | 96 | #======================================= 97 | # Test parse 1 arg 98 | url_pattern = '/event/' 99 | a = ArgsParse(url_pattern) 100 | 101 | # basic test 102 | url_request = '/event/tuan' 103 | print(a.parse(url_request)) 104 | # test a normal simillar url 105 | url_request = '/event' 106 | print(a.parse(url_request)) 107 | # test a missing parse 108 | url_request = '/event/' 109 | print(a.parse(url_request)) 110 | 111 | # check propertise 112 | print(a.is_hashable()) 113 | 114 | #======================================= 115 | # Test parse 2 args 116 | url_pattern = '/event//' 117 | a = ArgsParse(url_pattern) 118 | 119 | # basic test 120 | url_request = '/event/tuan/minh' 121 | print(a.parse(url_request)) 122 | 123 | url_request = '/event' 124 | print(a.parse(url_request)) 125 | # test a missing arg 126 | url_request = '/event/' 127 | print(a.parse(url_request)) 128 | 129 | # test a missing 1 arg 130 | url_request = '/event/tuan' 131 | print(a.parse(url_request)) 132 | # test a missing 1 arg 133 | url_request = '/event//minh' 134 | print(a.parse(url_request)) 135 | 136 | # test a missing 1 arg 137 | url_request = '/eventFalse/tuan/minh' 138 | print(a.parse(url_request)) 139 | # test a missing 1 arg 140 | url_request = '/something/tuan/minh' 141 | print(a.parse(url_request)) 142 | 143 | # check propertise 144 | print(a.properties) 145 | print(a.pattern) 146 | 147 | 148 | if __name__ == '__main__': 149 | main() 150 | -------------------------------------------------------------------------------- /microservices_connector/url_parser/url_namespace.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import re 3 | import uuid 4 | from urllib.parse import urlparse 5 | 6 | REGEX_TYPES = { 7 | 'string': (str, r'[^/]+'), 8 | 'int': (int, r'\d+'), 9 | 'number': (float, r'[0-9\\.]+'), 10 | 'alpha': (str, r'[A-Za-z]+'), 11 | 'path': (str, r'[^/].*?'), 12 | 'uuid': (uuid.UUID, r'[A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}') 13 | } 14 | 15 | 16 | # pattern = r'<.*?>' #'/event/' 17 | 18 | class ArgsParse(object): 19 | 20 | properties = {"unhashable": None} 21 | 22 | def __init__(self, url_pattern): 23 | self.url_pattern = url_pattern 24 | self.parameter_pattern = re.compile(r'<(.+?)>') 25 | pattern_string = re.sub(self.parameter_pattern, 26 | self.add_parameter, self.url_pattern) 27 | self.pattern = re.compile(r'^{}$'.format(pattern_string)) 28 | 29 | def parse_parameter_string(self, parameter_string): 30 | """Parse a parameter string into its constituent name, type, and 31 | pattern 32 | For example:: 33 | parse_parameter_string('')` -> 34 | ('param_one', str, '[A-z]') 35 | :param parameter_string: String to parse 36 | :return: tuple containing 37 | (parameter_name, parameter_type, parameter_pattern) 38 | """ 39 | # We could receive NAME or NAME:PATTERN 40 | name = parameter_string 41 | pattern = 'string' 42 | if ':' in parameter_string: 43 | name, pattern = parameter_string.split(':', 1) 44 | if not name: 45 | raise ValueError( 46 | "Invalid parameter syntax: {}".format(parameter_string) 47 | ) 48 | 49 | default = (str, pattern) 50 | # Pull from pre-configured types 51 | _type, pattern = REGEX_TYPES.get(pattern, default) 52 | 53 | return name, _type, pattern 54 | 55 | def add_parameter(self, match): 56 | name = match.group(1) 57 | name, _type, pattern = self.parse_parameter_string(name) 58 | 59 | # parameter = Parameter( 60 | # name=name, cast=_type) 61 | # parameters.append(parameter) 62 | 63 | # Mark the whole route as unhashable if it has the hash key in it 64 | if re.search(r'(^|[^^]){1}/', pattern): 65 | self.properties['unhashable'] = True 66 | # Mark the route as unhashable if it matches the hash key 67 | elif re.search(r'/', pattern): 68 | self.properties['unhashable'] = True 69 | 70 | return '({})'.format(pattern) # return '(?P{})'.format(pattern) 71 | 72 | def parse(self, url): 73 | res = self.pattern.search(url) 74 | if res is None: 75 | return None 76 | else: 77 | return self.pattern.search(url).groups() 78 | 79 | def is_hashable(self): 80 | if self.properties["unhashable"] is True: 81 | return False 82 | else: 83 | return True 84 | 85 | def main2(): 86 | 87 | #======================================= 88 | # Test parse 2 args 89 | url_pattern = '/event' 90 | a = ArgsParse(url_pattern) 91 | 92 | # check propertise 93 | print(a.is_hashable()) 94 | 95 | 96 | #======================================= 97 | # Test parse 1 arg 98 | url_pattern = '/event/' 99 | a = ArgsParse(url_pattern) 100 | 101 | # basic test 102 | url_request = '/event/tuan' 103 | print(a.parse(url_request)) 104 | # test a normal simillar url 105 | url_request = '/event' 106 | print(a.parse(url_request)) 107 | # test a missing parse 108 | url_request = '/event/' 109 | print(a.parse(url_request)) 110 | 111 | # check propertise 112 | print(a.is_hashable()) 113 | 114 | #======================================= 115 | # Test parse 2 args 116 | url_pattern = '/event//' 117 | a = ArgsParse(url_pattern) 118 | 119 | # basic test 120 | url_request = '/event/tuan/minh' 121 | print(a.parse(url_request)) 122 | 123 | url_request = '/event' 124 | print(a.parse(url_request)) 125 | # test a missing arg 126 | url_request = '/event/' 127 | print(a.parse(url_request)) 128 | 129 | # test a missing 1 arg 130 | url_request = '/event/tuan' 131 | print(a.parse(url_request)) 132 | # test a missing 1 arg 133 | url_request = '/event//minh' 134 | print(a.parse(url_request)) 135 | 136 | # test a missing 1 arg 137 | url_request = '/eventFalse/tuan/minh' 138 | print(a.parse(url_request)) 139 | # test a missing 1 arg 140 | url_request = '/something/tuan/minh' 141 | print(a.parse(url_request)) 142 | 143 | # check propertise 144 | print(a.properties) 145 | print(a.pattern) 146 | 147 | 148 | if __name__ == '__main__': 149 | main2() 150 | -------------------------------------------------------------------------------- /tests/example04/test_concurrency.py: -------------------------------------------------------------------------------- 1 | import time 2 | from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor 3 | from collections import deque 4 | from queue import Queue 5 | import threading 6 | import random 7 | 8 | def wait_on_b(b): 9 | time.sleep(random.random()) 10 | print('working on b=%s'%b) # b will never complete because it is waiting on a. 11 | return 5 12 | 13 | 14 | def wait_on_a(a): 15 | time.sleep(1) 16 | print('working on a=%s'%a) # a will never complete because it is waiting on b. 17 | return 6 18 | 19 | 20 | poll = [ 21 | {'id': 1, 'x': 'Nguyen'}, 22 | {'id': 1, 'x': 'Minh'}, 23 | {'id': 1, 'x': 'Tuan'}, 24 | {'id': 2, 'x': 'Vu'}, 25 | {'id': 3, 'x': 'Ai do khac'}, 26 | {'id': 2, 'x': 'Ngoc'}, 27 | {'id': 2, 'x': 'Anh'}, 28 | {'id': 4, 'x': '1'}, 29 | {'id': 4, 'x': '2'}, 30 | {'id': 4, 'x': '3'}, 31 | {'id': 4, 'x': '4'}, 32 | {'id': 3, 'x': '5'}, 33 | {'id': 3, 'x': '6'}, 34 | {'id': 3, 'x': '7'}, 35 | {'id': 3, 'x': '8'}, 36 | {'id': 3, 'x': '9'}, 37 | ] 38 | 39 | 40 | class AsyncThread(threading.Thread): 41 | """Threaded website reader""" 42 | 43 | def __init__(self, queue): 44 | threading.Thread.__init__(self) 45 | self.queue = queue 46 | 47 | def input_function(self, parameter_list): 48 | pass 49 | 50 | def run(self): 51 | while True: 52 | # Grabs host from queue 53 | item = self.queue.get() 54 | 55 | # Grabs item and put to input_function 56 | self.input_function(item) 57 | 58 | # Signals to queue job is done 59 | self.queue.task_done() 60 | 61 | class WaitGroup(object): 62 | def __init__(self, max_workers=4, max_watching=4): 63 | self.max_workers = max_workers 64 | self.max_watching = max_watching 65 | # 1=> create a watching list 66 | self.pending = deque([]) 67 | self.watching = deque([]) 68 | self.executor = ThreadPoolExecutor(max_workers=max_workers) 69 | # self.f = f 70 | # create a thread for doing sychonous job 71 | self.cleaner = ThreadPoolExecutor(max_workers=1) 72 | # self.cleaner = AsyncThread(self.pending) 73 | # self.cleaner.input_function = f 74 | # self.cleaner.daemon = True 75 | # self.cleaner.start() 76 | 77 | def clear_pending(self, f): 78 | if len(self.pending) > 0: 79 | for item in self.pending: 80 | if item['id'] not in self.watching: 81 | f(item) 82 | self.pending.remove(item) 83 | # self.pending.clear() 84 | # pass 85 | 86 | # def clean(self, item): 87 | # self.pending.put(item) 88 | 89 | def execute_next(self, f, key, item): 90 | self.watching.append(key) # append item['id'] to watching 91 | future = self.executor.submit(f, item) 92 | # remove one old item from watching if it's full 93 | 94 | # if future.done(): 95 | if len(self.watching) > self.max_watching: 96 | self.watching.popleft() 97 | 98 | def submit(self, f, key, item): 99 | # forever going execute pool 100 | # 2=> check if the next item have same id in watching list 101 | print('watching', self.watching, (self.max_watching)) 102 | if key in self.watching: 103 | # if yes => put this item in to a pending queue 104 | self.pending.append(item) 105 | # print('Synchonous', item, self.pending) 106 | # pending queue will be process in a single process with order and concurrent 107 | else: 108 | # then execute for the next item 109 | self.clear_pending(f) 110 | # print('==> Asynchonous', item, self.watching) 111 | self.execute_next(f, key, item) 112 | 113 | def clear(self, f): 114 | if len(self.pending) > 0: 115 | for item in self.pending: 116 | f(item) 117 | # self.pending.clear() # clear pending after do 118 | 119 | def shutdown(self): 120 | # self.cleaner.join() 121 | self.cleaner.shutdown() 122 | self.executor.shutdown() 123 | 124 | # start = time.time() 125 | # print('Apply with normal ThreadPoolExecutor:') 126 | # with ThreadPoolExecutor(max_workers=4) as e: 127 | # for item in poll: 128 | # e.submit(wait_on_b, item) 129 | # print('Finish after: ',time.time()-start, 'seconds') 130 | 131 | start = time.time() 132 | print('Apply with WaitGroup:', len(poll)) 133 | e = WaitGroup(max_watching=2) 134 | for item in poll: 135 | e.submit(wait_on_b, item['id'], item) 136 | e.clear(wait_on_b) 137 | # e.submit(wait_on_b,'0','end') 138 | print(e.pending) 139 | e.shutdown() 140 | print('Finish after: ', time.time()-start, 'seconds') 141 | 142 | a=deque([1,2,3,4]) 143 | a.popleft() 144 | print(a) 145 | 146 | # print('assign to %s'%i) 147 | # # e.map(wait_on_a,[i for i in range(10,20)]) 148 | 149 | # never shutdown thread pool 150 | # executor = ThreadPoolExecutor(max_workers=3) 151 | # task1 = executor.submit(wait_on_b, 100) 152 | # task2 = executor.submit(wait_on_b, 200) 153 | -------------------------------------------------------------------------------- /tests/example03/friend02.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import time 3 | import json 4 | from cython_npm.cythoncompile import require 5 | # from Interservices import Friend 6 | # from ../ 7 | Interservices = require('../../microservices_connector/Interservices') 8 | Friend = Interservices.Friend 9 | # test sanic app 10 | 11 | timeit = Interservices.timeit 12 | # post is the fastest way 13 | @timeit 14 | def doget(): 15 | r = requests.get('http://0.0.0.0:5000/hello', json={"key": "value"}) 16 | r.status_code 17 | r.json() 18 | print('port', r.json()) 19 | 20 | 21 | @timeit 22 | def dopost(): 23 | r = requests.post('http://localhost:5000/hello', 24 | json={"args": ["value", ], 'kwargs': {'onekey': 'value of key'}}) 25 | r.status_code 26 | r.json() 27 | print('get', r.json()) 28 | 29 | 30 | @timeit 31 | def doput(): 32 | r = requests.put('http://localhost:5000/hello', json={"key": "value"}) 33 | r.status_code 34 | r.json() 35 | print('put', r.json()) 36 | 37 | 38 | @timeit 39 | def dodelete(): 40 | r = requests.delete('http://localhost:5000/hello', json={"key": "value"}) 41 | r.status_code 42 | r.json() 43 | print('delete', r.json()) 44 | 45 | # doget() 46 | # dopost() 47 | # doput() 48 | # dodelete() 49 | 50 | 51 | # @timeit 52 | # def test(): 53 | # aFriend= Friend('app1', 'http://localhost:5000') 54 | # aFriend.setRule('/hello') 55 | # r = aFriend.send('/hello', 'A variable value', onekey='A keyword variable value') 56 | # return r 57 | 58 | # python run.py 59 | # test return a string 60 | # @timeit 61 | def testStr(): 62 | print( 63 | """############################## 64 | Test return string 65 | """) 66 | aFriend= Friend('app1', 'localhost:5000') 67 | print('Test: return a simple string') 68 | x = aFriend.send('/str', key='A keyword variable value') 69 | print('x=', x, type(x)) 70 | print('==========================') 71 | print('Test: return multiple string') 72 | x, y, z = aFriend.send('/str2', 'A variable value','second Variable', 73 | key='A keyword variable value') 74 | print('x=' ,x, type(x)) 75 | print('y=', y, type(y)) 76 | print('z=', z, type(z)) 77 | 78 | testStr() 79 | """[Result] 80 | Test: return a simple string 81 | x= A variable value-A keyword variable value 82 | ========================== 83 | Test: return multiple string 84 | x= A variable value 85 | y= A keyword variable value 86 | z= A variable value-A keyword variable value 87 | 'testStr' 23.17 ms 88 | """ 89 | 90 | 91 | # @timeit 92 | def testInt(): 93 | print( 94 | """############################## 95 | Test return a int, float 96 | """) 97 | aFriend= Friend('app1', 'localhost:5000') 98 | print('Test: return a simple Value') 99 | x = aFriend.send('/int', 2018, key=312) 100 | print('x=', x, type(x)) 101 | print('==========================') 102 | print('Test: return a simple Value') 103 | x = aFriend.send('/float', 2.018, key=3.12) 104 | print('x=', x, type(x)) 105 | print('==========================') 106 | print('Test: return multiple Value') 107 | x, y, z = aFriend.send('/int3', 3.1427, 108 | key=1000000000, key2=2.71230) 109 | print('x=', x, type(x)) 110 | print('y=', y, type(y)) 111 | print('z=', z, type(z)) 112 | testInt() 113 | """[result] 114 | Test: return a simple Value 115 | x= 2330 116 | ========================== 117 | Test: return a simple Value 118 | x= 5.138 119 | ========================== 120 | Test: return multiple Value 121 | x= 1000000003.1427 122 | y= 1000000000000000000 123 | z= 9.87656329 124 | """ 125 | 126 | 127 | # @timeit 128 | def testListDict(): 129 | print( 130 | """############################## 131 | Test return a list, dict 132 | """) 133 | aFriend= Friend('app1', 'localhost:5000') 134 | print('Test: return a simple Value') 135 | x = aFriend.send('/list', [12,34,45], key=['abc','zyz']) 136 | print('x=', x, type(x)) 137 | print('==========================') 138 | print('Test: return a simple Value') 139 | x = aFriend.send('/dict', {'keyword':['anything']}, key={'int':20,'str':'adfafsa','float':0.2323}) 140 | print('x=', x, type(x)) 141 | print('==========================') 142 | print('Test: return multiple Value') 143 | x, y, z = aFriend.send('/list3', {'keyword': ['anything']}, 144 | key=['abc', 'zyz']) 145 | print('x=', x, type(x)) 146 | print('y=', y, type(y)) 147 | print('z=', z, type(z)) 148 | 149 | 150 | testListDict() 151 | """[Result] 152 | Test: return a simple Value 153 | x= [12, 34, 45, 'abc', 'zyz'] 154 | ========================== 155 | Test: return a simple Value 156 | x= {'dict': {'keyword': ['anything']}, 'float': 0.2323, 'int': 20, 'str': 'adfafsa'} 157 | ========================== 158 | Test: return multiple Value 159 | x= {'keyword': ['anything']} 160 | y= ['abc', 'zyz', 'other value'] 161 | z= None 162 | 'testListDict' 22.19 ms 163 | """ 164 | 165 | 166 | class testservice(object): 167 | name = 'test' 168 | Purpose = 'For test only' 169 | empty = None 170 | 171 | def __init__(self, value): 172 | self.value = value 173 | 174 | def onemethod(self): 175 | print('This is test class') 176 | 177 | 178 | # @timeit 179 | def testClassType(): 180 | print( 181 | """############################## 182 | Test return NoneType, Class, use of Token 183 | """) 184 | aFriend= Friend('app1', 'localhost:5000') 185 | print('Test: return a simple Value') 186 | x = aFriend.send('/None', [12, 34, 45], key=['abc', 'zyz']) 187 | print('x=', x, type(x)) 188 | print('==========================') 189 | print('Test: return a simple Value with token') 190 | aFriend.setRule('/class', token='123456') 191 | x = aFriend.send('/class', {'keyword': ['anything']}, 192 | key={'int': 20, 'str': 'adfafsa', 'float': 0.2323}) 193 | print('x=', x, type(x)) 194 | print('==========================') 195 | print('Test: return multiple Value') 196 | aFriend.setRule('/class2', token='123456') 197 | x,y,z = aFriend.send('/class2', {'keyword': ['anything']}, 198 | key={'int': 20, 'str': 'adfafsa', 'float': 0.2323}) 199 | print('x=', x, type(x)) 200 | print('y=', y, type(y)) 201 | print('z=', z, type(z)) 202 | 203 | # Test send class and list of class object 204 | print('Test: send class and list of class object') 205 | aFriend.setRule('/class3', token='123456') 206 | t1 = testservice('value1') 207 | t2 = testservice('value2') 208 | x, y, z = aFriend.send('/class3', [t1,t2], 209 | key={'t1': t1, 't2': t2, 'list': [t1, t2]}) 210 | print('x=', x, type(x)) 211 | print('y=', y, type(y)) 212 | print('z=', z, type(z)) 213 | x = aFriend.json('/json', a=12,b='This is a text',c={'dict':'a dict'}) 214 | print(x) 215 | y = aFriend.json('/json1', a={'dict': 'a only dict'}) 216 | print(y) 217 | 218 | 219 | 220 | testClassType() 221 | """[Results] 222 | ############################## 223 | Test return NoneType, Class, use of Token 224 | 225 | Test: return a simple Value 226 | x= None 227 | ========================== 228 | Test: return a simple Value with token 229 | x= {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': {'keyword': ['anything']}} 230 | ========================== 231 | Test: return multiple Value 232 | x= {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': {'float': 0.2323, 'int': 20, 'str': 'adfafsa'}} 233 | y= {'keyword': ['anything']} 234 | z= None 235 | 'testClassType' 19.20 ms 236 | """ 237 | -------------------------------------------------------------------------------- /tests/example01/friend01.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import time 3 | import json 4 | from cython_npm.cythoncompile import require 5 | # from ../ 6 | Interservices = require('../../microservices_connector/Interservices') 7 | Friend = Interservices.Friend 8 | 9 | def timeit(method): 10 | 11 | def timed(*args, **kw): 12 | ts = time.time() 13 | result = method(*args, **kw) 14 | te = time.time() 15 | 16 | print('%r %2.2f ms' % 17 | (method.__name__, (te - ts) * 1000)) 18 | return result 19 | 20 | return timed 21 | 22 | # post is the fastest way 23 | @timeit 24 | def doget(): 25 | r = requests.get('http://0.0.0.0:5000/hello', json={"key": "value"}) 26 | r.status_code 27 | r.json() 28 | print('port', r.json()) 29 | 30 | 31 | @timeit 32 | def dopost(): 33 | r = requests.post('http://localhost:5010/hello', 34 | json={"args": ["value", ], 'kwargs': {'onekey': 'value of key'}}) 35 | r.status_code 36 | r.json() 37 | print('get', r.json()) 38 | 39 | 40 | @timeit 41 | def doput(): 42 | r = requests.put('http://localhost:5010/hello', json={"key": "value"}) 43 | r.status_code 44 | r.json() 45 | print('put', r.json()) 46 | 47 | 48 | @timeit 49 | def dodelete(): 50 | r = requests.delete('http://localhost:5010/hello', json={"key": "value"}) 51 | r.status_code 52 | r.json() 53 | print('delete', r.json()) 54 | 55 | # doget() 56 | # dopost() 57 | # doput() 58 | # dodelete() 59 | 60 | 61 | # @timeit 62 | # def test(): 63 | # aFriend= Friend('app1', 'http://localhost:5010') 64 | # aFriend.setRule('/hello') 65 | # r = aFriend.send('/hello', 'A variable value', onekey='A keyword variable value') 66 | # return r 67 | 68 | # python run.py 69 | # test return a string 70 | @timeit 71 | def testStr(): 72 | print( 73 | """############################## 74 | Test return string 75 | """) 76 | aFriend= Friend('app1', 'localhost:5010') 77 | print('Test: return a simple string') 78 | x = aFriend.send('/str', 'A variable value', key='A keyword variable value') 79 | print('x=', x, type(x)) 80 | print('==========================') 81 | print('Test: return multiple string') 82 | x, y, z = aFriend.send('/str2', 'A variable value','second Variable', 83 | key='A keyword variable value') 84 | print('x=' ,x, type(x)) 85 | print('y=', y, type(y)) 86 | print('z=', z, type(z)) 87 | 88 | testStr() 89 | """[Result] 90 | Test: return a simple string 91 | x= A variable value-A keyword variable value 92 | ========================== 93 | Test: return multiple string 94 | x= A variable value 95 | y= A keyword variable value 96 | z= A variable value-A keyword variable value 97 | 'testStr' 23.17 ms 98 | """ 99 | 100 | 101 | @timeit 102 | def testInt(): 103 | print( 104 | """############################## 105 | Test return a int, float 106 | """) 107 | aFriend= Friend('app1', 'localhost:5010') 108 | print('Test: return a simple Value') 109 | x = aFriend.send('/int', 2018, key=312) 110 | print('x=', x, type(x)) 111 | print('==========================') 112 | print('Test: return a simple Value') 113 | x = aFriend.send('/float', 2.018, key=3.12) 114 | print('x=', x, type(x)) 115 | print('==========================') 116 | print('Test: return multiple Value') 117 | x, y, z = aFriend.send('/int3', 3.1427, 118 | key=1000000000, key2=2.71230) 119 | print('x=', x, type(x)) 120 | print('y=', y, type(y)) 121 | print('z=', z, type(z)) 122 | testInt() 123 | """[result] 124 | Test: return a simple Value 125 | x= 2330 126 | ========================== 127 | Test: return a simple Value 128 | x= 5.138 129 | ========================== 130 | Test: return multiple Value 131 | x= 1000000003.1427 132 | y= 1000000000000000000 133 | z= 9.87656329 134 | """ 135 | 136 | 137 | @timeit 138 | def testListDict(): 139 | print( 140 | """############################## 141 | Test return a list, dict 142 | """) 143 | aFriend= Friend('app1', 'localhost:5010') 144 | print('Test: return a simple Value') 145 | x = aFriend.send('/list', [12,34,45], key=['abc','zyz']) 146 | print('x=', x, type(x)) 147 | print('==========================') 148 | print('Test: return a simple Value') 149 | x = aFriend.send('/dict', {'keyword':['anything']}, key={'int':20,'str':'adfafsa','float':0.2323}) 150 | print('x=', x, type(x)) 151 | print('==========================') 152 | print('Test: return multiple Value') 153 | x, y, z = aFriend.send('/list3', {'keyword': ['anything']}, 154 | key=['abc', 'zyz']) 155 | print('x=', x, type(x)) 156 | print('y=', y, type(y)) 157 | print('z=', z, type(z)) 158 | 159 | 160 | testListDict() 161 | """[Result] 162 | Test: return a simple Value 163 | x= [12, 34, 45, 'abc', 'zyz'] 164 | ========================== 165 | Test: return a simple Value 166 | x= {'dict': {'keyword': ['anything']}, 'float': 0.2323, 'int': 20, 'str': 'adfafsa'} 167 | ========================== 168 | Test: return multiple Value 169 | x= {'keyword': ['anything']} 170 | y= ['abc', 'zyz', 'other value'] 171 | z= None 172 | 'testListDict' 22.19 ms 173 | """ 174 | 175 | 176 | class testservice(object): 177 | name = 'test' 178 | Purpose = 'For test only' 179 | empty = None 180 | 181 | def __init__(self, value): 182 | self.value = value 183 | 184 | def onemethod(self): 185 | print('This is test class') 186 | 187 | 188 | @timeit 189 | def testClassType(): 190 | print( 191 | """############################## 192 | Test return NoneType, Class, use of Token 193 | """) 194 | aFriend= Friend('app1', 'localhost:5010') 195 | print('Test: return a simple Value') 196 | x = aFriend.send('/None', [12, 34, 45], key=['abc', 'zyz']) 197 | print('x=', x, type(x)) 198 | print('==========================') 199 | print('Test: return a simple Value with token') 200 | aFriend.setRule('/class', token='123456') 201 | x = aFriend.send('/class', {'keyword': ['anything']}, 202 | key={'int': 20, 'str': 'adfafsa', 'float': 0.2323}) 203 | print('x=', x, type(x)) 204 | print('==========================') 205 | print('Test: return multiple Value') 206 | aFriend.setRule('/class2', token='123456') 207 | x,y,z = aFriend.send('/class2', {'keyword': ['anything']}, 208 | key={'int': 20, 'str': 'adfafsa', 'float': 0.2323}) 209 | print('x=', x, type(x)) 210 | print('y=', y, type(y)) 211 | print('z=', z, type(z)) 212 | 213 | # Test send class and list of class object 214 | print('Test: send class and list of class object') 215 | aFriend.setRule('/class3', token='123456') 216 | t1 = testservice('value1') 217 | t2 = testservice('value2') 218 | x, y, z = aFriend.send('/class3', [t1,t2], 219 | key={'t1': t1, 't2': t2, 'list': [t1, t2]}) 220 | print('x=', x, type(x)) 221 | print('y=', y, type(y)) 222 | print('z=', z, type(z)) 223 | 224 | 225 | 226 | testClassType() 227 | """[Results] 228 | ############################## 229 | Test return NoneType, Class, use of Token 230 | 231 | Test: return a simple Value 232 | x= None 233 | ========================== 234 | Test: return a simple Value with token 235 | x= {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': {'keyword': ['anything']}} 236 | ========================== 237 | Test: return multiple Value 238 | x= {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': {'float': 0.2323, 'int': 20, 'str': 'adfafsa'}} 239 | y= {'keyword': ['anything']} 240 | z= None 241 | 'testClassType' 19.20 ms 242 | """ 243 | -------------------------------------------------------------------------------- /tests/example02/friend02.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import time 3 | import json 4 | from cython_npm.cythoncompile import require 5 | # from Interservices import Friend 6 | # from ../ 7 | Interservices = require('../../microservices_connector/Interservices') 8 | Friend = Interservices.Friend 9 | # test sanic app 10 | 11 | timeit = Interservices.timeit 12 | # post is the fastest way 13 | @timeit 14 | def doget(): 15 | r = requests.get('http://0.0.0.0:5000/hello', json={"key": "value"}) 16 | r.status_code 17 | r.json() 18 | print('port', r.json()) 19 | 20 | 21 | @timeit 22 | def dopost(): 23 | r = requests.post('http://localhost:5000/hello', 24 | json={"args": ["value", ], 'kwargs': {'onekey': 'value of key'}}) 25 | r.status_code 26 | r.json() 27 | print('get', r.json()) 28 | 29 | 30 | @timeit 31 | def doput(): 32 | r = requests.put('http://localhost:5000/hello', json={"key": "value"}) 33 | r.status_code 34 | r.json() 35 | print('put', r.json()) 36 | 37 | 38 | @timeit 39 | def dodelete(): 40 | r = requests.delete('http://localhost:5000/hello', json={"key": "value"}) 41 | r.status_code 42 | r.json() 43 | print('delete', r.json()) 44 | 45 | # doget() 46 | # dopost() 47 | # doput() 48 | # dodelete() 49 | 50 | 51 | # @timeit 52 | # def test(): 53 | # aFriend= Friend('app1', 'http://localhost:5000') 54 | # aFriend.setRule('/hello') 55 | # r = aFriend.send('/hello', 'A variable value', onekey='A keyword variable value') 56 | # return r 57 | 58 | # python run.py 59 | # test return a string 60 | @timeit 61 | def testStr(): 62 | print( 63 | """############################## 64 | Test return string 65 | """) 66 | aFriend= Friend('app1', 'localhost:5000') 67 | print('Test: return a simple string') 68 | x = aFriend.send('/str', 'A variable value', key='A keyword variable value') 69 | print('x=', x, type(x)) 70 | print('==========================') 71 | print('Test: return multiple string') 72 | x, y, z = aFriend.send('/str2', 'A variable value','second Variable', 73 | key='A keyword variable value') 74 | print('x=' ,x, type(x)) 75 | print('y=', y, type(y)) 76 | print('z=', z, type(z)) 77 | 78 | testStr() 79 | """[Result] 80 | Test: return a simple string 81 | x= A variable value-A keyword variable value 82 | ========================== 83 | Test: return multiple string 84 | x= A variable value 85 | y= A keyword variable value 86 | z= A variable value-A keyword variable value 87 | 'testStr' 23.17 ms 88 | """ 89 | 90 | 91 | @timeit 92 | def testInt(): 93 | print( 94 | """############################## 95 | Test return a int, float 96 | """) 97 | aFriend= Friend('app1', 'localhost:5000') 98 | print('Test: return a simple Value') 99 | x = aFriend.send('/int', 2018, key=312) 100 | print('x=', x, type(x)) 101 | print('==========================') 102 | print('Test: return a simple Value') 103 | x = aFriend.send('/float', 2.018, key=3.12) 104 | print('x=', x, type(x)) 105 | print('==========================') 106 | print('Test: return multiple Value') 107 | x, y, z = aFriend.send('/int3', 3.1427, 108 | key=1000000000, key2=2.71230) 109 | print('x=', x, type(x)) 110 | print('y=', y, type(y)) 111 | print('z=', z, type(z)) 112 | testInt() 113 | """[result] 114 | Test: return a simple Value 115 | x= 2330 116 | ========================== 117 | Test: return a simple Value 118 | x= 5.138 119 | ========================== 120 | Test: return multiple Value 121 | x= 1000000003.1427 122 | y= 1000000000000000000 123 | z= 9.87656329 124 | """ 125 | 126 | 127 | @timeit 128 | def testListDict(): 129 | print( 130 | """############################## 131 | Test return a list, dict 132 | """) 133 | aFriend= Friend('app1', 'localhost:5000') 134 | print('Test: return a simple Value') 135 | x = aFriend.send('/list', [12,34,45], key=['abc','zyz']) 136 | print('x=', x, type(x)) 137 | print('==========================') 138 | print('Test: return a simple Value') 139 | x = aFriend.send('/dict', {'keyword':['anything']}, key={'int':20,'str':'adfafsa','float':0.2323}) 140 | print('x=', x, type(x)) 141 | print('==========================') 142 | print('Test: return multiple Value') 143 | x, y, z = aFriend.send('/list3', {'keyword': ['anything']}, 144 | key=['abc', 'zyz']) 145 | print('x=', x, type(x)) 146 | print('y=', y, type(y)) 147 | print('z=', z, type(z)) 148 | 149 | 150 | testListDict() 151 | """[Result] 152 | Test: return a simple Value 153 | x= [12, 34, 45, 'abc', 'zyz'] 154 | ========================== 155 | Test: return a simple Value 156 | x= {'dict': {'keyword': ['anything']}, 'float': 0.2323, 'int': 20, 'str': 'adfafsa'} 157 | ========================== 158 | Test: return multiple Value 159 | x= {'keyword': ['anything']} 160 | y= ['abc', 'zyz', 'other value'] 161 | z= None 162 | 'testListDict' 22.19 ms 163 | """ 164 | 165 | 166 | class testservice(object): 167 | name = 'test' 168 | Purpose = 'For test only' 169 | empty = None 170 | 171 | def __init__(self, value): 172 | self.value = value 173 | 174 | def onemethod(self): 175 | print('This is test class') 176 | 177 | 178 | @timeit 179 | def testClassType(): 180 | print( 181 | """############################## 182 | Test return NoneType, Class, use of Token 183 | """) 184 | aFriend= Friend('app1', 'localhost:5000') 185 | print('Test: return a simple Value') 186 | x = aFriend.send('/None', [12, 34, 45], key=['abc', 'zyz']) 187 | print('x=', x, type(x)) 188 | print('==========================') 189 | print('Test: return a simple Value with token') 190 | aFriend.setRule('/class', token='123456') 191 | x = aFriend.send('/class', {'keyword': ['anything']}, 192 | key={'int': 20, 'str': 'adfafsa', 'float': 0.2323}) 193 | print('x=', x, type(x)) 194 | print('==========================') 195 | print('Test: return multiple Value') 196 | aFriend.setRule('/class2', token='123456') 197 | x,y,z = aFriend.send('/class2', {'keyword': ['anything']}, 198 | key={'int': 20, 'str': 'adfafsa', 'float': 0.2323}) 199 | print('x=', x, type(x)) 200 | print('y=', y, type(y)) 201 | print('z=', z, type(z)) 202 | 203 | # Test send class and list of class object 204 | print('Test: send class and list of class object') 205 | aFriend.setRule('/class3', token='123456') 206 | t1 = testservice('value1') 207 | t2 = testservice('value2') 208 | x, y, z = aFriend.send('/class3', [t1,t2], 209 | key={'t1': t1, 't2': t2, 'list': [t1, t2]}) 210 | print('x=', x, type(x)) 211 | print('y=', y, type(y)) 212 | print('z=', z, type(z)) 213 | 214 | print('=================Response json===============') 215 | x = aFriend.json('/json', a=12,b='This is a text',c={'dict':'a dict'}) 216 | print('Synchonous POST:', x) 217 | y = aFriend.json('/json1', method='GET' , a={'dict': 'a only dict'}) 218 | print('Asynchonous GET:', y) 219 | z = aFriend.json('/json1', a={'dict': 'a only dict'}) 220 | print('Asynchonous POST:', z) 221 | 222 | t = aFriend.json('/get/none', method='GET') 223 | print('Synchonous GET:', t) 224 | q = aFriend.json('/post/none', method='POST') 225 | print('Synchonous POST:', q) 226 | 227 | 228 | 229 | testClassType() 230 | """[Results] 231 | ############################## 232 | Test return NoneType, Class, use of Token 233 | 234 | Test: return a simple Value 235 | x= None 236 | ========================== 237 | Test: return a simple Value with token 238 | x= {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': {'keyword': ['anything']}} 239 | ========================== 240 | Test: return multiple Value 241 | x= {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': {'float': 0.2323, 'int': 20, 'str': 'adfafsa'}} 242 | y= {'keyword': ['anything']} 243 | z= None 244 | 'testClassType' 19.20 ms 245 | """ 246 | -------------------------------------------------------------------------------- /microservices_connector/spawn.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import functools 3 | import os 4 | import threading 5 | from concurrent.futures import Future, ThreadPoolExecutor 6 | 7 | # Alternatively, you can create an instance of the loop manually, using: 8 | 9 | import uvloop 10 | # loop = uvloop.new_event_loop() 11 | # asyncio.set_event_loop(loop) 12 | import queue 13 | import time 14 | from collections import deque 15 | import queue 16 | import threading 17 | import random 18 | import multiprocessing 19 | 20 | 21 | class SyncThread(threading.Thread): 22 | """Threaded Sync reader, read data from queue""" 23 | 24 | def __init__(self, in_queue, out_queue=None, name=None, watching: deque=None): 25 | """Threaded Sync reader, read data from queue 26 | 27 | Arguments: 28 | queue {[type]} -- queue or deque 29 | 30 | Keyword Arguments: 31 | out_queue {[type]} -- queue receive result (default: {None}) 32 | """ 33 | 34 | threading.Thread.__init__(self, name=name) 35 | self.in_queue = in_queue 36 | self.out_queue = out_queue 37 | self.watching = watching 38 | 39 | def main_process(self, f, *args, **kwargs): 40 | return f(*args, **kwargs) 41 | 42 | def run(self): 43 | while True: 44 | # Grabs host from queue 45 | f, args, kwargs = self.in_queue.get() 46 | 47 | # Grabs item and put to input_function 48 | result = self.main_process(f, *args, *kwargs), None 49 | result = result[:-1] 50 | 51 | if self.out_queue is not None: 52 | self.out_queue.put(*result) 53 | 54 | if self.watching is not None: 55 | self.watching.popleft() 56 | # print('Watching list is:', self.watching) 57 | 58 | # Signals to queue job is done 59 | self.in_queue.task_done() 60 | 61 | 62 | class AsyncProcess(multiprocessing.Process): 63 | 64 | def __init__(self, in_queue, out_queue, name=None, watching: deque=None): 65 | multiprocessing.Process.__init__(self, name=name) 66 | self.in_queue = in_queue 67 | self.out_queue = out_queue 68 | self.stop_event = multiprocessing.Event() 69 | self.watching = watching 70 | 71 | def stop(self): 72 | self.stop_event.set() 73 | 74 | def main_process(self, f, *args, **kwargs): 75 | return f(*args, **kwargs) 76 | 77 | def run(self): 78 | while not self.stop_event.is_set(): 79 | # Grabs host from queue 80 | f, args, kwargs = self.in_queue.get() 81 | 82 | # Grabs item and put to input_function 83 | result = self.main_process(f, *args, *kwargs), None 84 | result = result[:-1] 85 | 86 | if self.out_queue is not None: 87 | self.out_queue.put(*result) 88 | 89 | if self.watching is not None: 90 | self.watching.popleft() 91 | 92 | # Signals to queue job is done 93 | self.in_queue.task_done() 94 | if self.stop_event.is_set(): 95 | print('Process %s is stopping' % self.pid) 96 | break 97 | 98 | 99 | class DistributedThreads(object): 100 | 101 | def __init__(self, out_queue=None, max_workers=4, max_watching=100, worker=None, maxsize=0, delay=1): 102 | self.out_queue = out_queue 103 | self.max_workers = max_workers 104 | self.max_watching = max_watching 105 | self.current_id = 0 106 | self.max_qsize = maxsize 107 | self.delay = delay 108 | if worker is None: 109 | self.init_worker() 110 | else: 111 | self.init_worker(worker=worker) 112 | 113 | def init_worker(self, worker=SyncThread): 114 | # create list of queue 115 | self.queue_list = [queue.Queue(maxsize=self.max_qsize) 116 | for i in range(self.max_workers)] 117 | 118 | # create list of watching queue 119 | self.watching_list = [deque() for i in range(self.max_workers)] 120 | 121 | # create list of threads: 122 | self.worker_list = [] 123 | for i in range(self.max_workers): 124 | one_worker = worker( 125 | self.queue_list[i], out_queue=self.out_queue, name=str(i), watching=self.watching_list[i]) 126 | one_worker.daemon = True 127 | self.worker_list.append(one_worker) 128 | one_worker.start() 129 | 130 | def iterate_queue(self, watching: list, key): 131 | if key is not None: 132 | watching.append(key) 133 | if len(watching) > self.max_watching: 134 | time.sleep(self.delay) 135 | 136 | def next_worker(self, last_id): 137 | return (last_id+1) % self.max_workers 138 | 139 | def check_next_queue(self, current_queue_size, last_id): 140 | next_id = self.next_worker(last_id) 141 | if current_queue_size >= self.queue_list[next_id].qsize(): 142 | return next_id 143 | else: 144 | return self.check_next_queue(current_queue_size, next_id) 145 | 146 | def choose_worker(self): 147 | current_queue_size = self.queue_list[self.current_id].qsize() 148 | return self.check_next_queue(current_queue_size, self.current_id) 149 | # return (self.current_id+1) % self.max_workers 150 | 151 | def submit(self, f, *args, **kwargs): 152 | return self.submit_id(None, f, *args, **kwargs) 153 | 154 | def submit_id(self, key, f, *args, **kwargs): 155 | worker_id = None 156 | # check if key belong to any worker 157 | if key is not None: 158 | for i in range(self.max_workers): 159 | if key in self.watching_list[i]: 160 | if worker_id is not None: 161 | raise ValueError("Key belong to more than one worker") 162 | worker_id = i 163 | self.current_id = worker_id 164 | break 165 | # choosing a work_id if not 166 | if worker_id is None: 167 | worker_id = self.choose_worker() 168 | # print('choose queue =>', worker_id) 169 | self.current_id = worker_id 170 | # assign to worker and watching list 171 | worker = self.queue_list[worker_id] 172 | watching = self.watching_list[worker_id] 173 | 174 | # add key to a watching 175 | self.iterate_queue(watching, key) 176 | # print(worker_id, watching) 177 | # add function to queue 178 | worker.put((f, args, kwargs)) 179 | 180 | def shutdown(self): 181 | for q in self.queue_list: 182 | q.join() 183 | 184 | 185 | class DistributedProcess(DistributedThreads): 186 | def init_worker(self, worker=AsyncProcess): 187 | # create list of queue 188 | self.queue_list = [multiprocessing.JoinableQueue() 189 | for i in range(self.max_workers)] 190 | 191 | # create list of watching queue 192 | self.watching_list = [deque() for i in range(self.max_workers)] 193 | 194 | # create list of threads: 195 | self.worker_list = [] 196 | for i in range(self.max_workers): 197 | one_worker = worker( 198 | self.queue_list[i], out_queue=self.out_queue, name=str(i)) 199 | self.worker_list.append(one_worker) 200 | one_worker.start() 201 | 202 | def iterate_queue(self, watching: list, key): 203 | if key is not None: 204 | watching.append(key) 205 | if len(watching) > self.max_watching: 206 | watching.popleft() 207 | 208 | def choose_worker(self): 209 | return (self.current_id+1) % self.max_workers 210 | 211 | def shutdown(self): 212 | for q in self.queue_list: 213 | q.join() 214 | for process in self.worker_list: 215 | print('Distributed Process %s is stopping' % process.pid) 216 | process.terminate() 217 | 218 | 219 | class Worker(threading.Thread): 220 | """Threaded title parser""" 221 | 222 | def __init__(self, out_queue, f): 223 | threading.Thread.__init__(self) 224 | self.out_queue = out_queue 225 | self.f = f 226 | 227 | def run(self): 228 | while True: 229 | # Grabs chunk from queue 230 | args = self.out_queue.get(), None 231 | 232 | # Parse the chunk 233 | args = args[:-1] 234 | self.f(*args) 235 | 236 | # Signals to queue job is done 237 | self.out_queue.task_done() 238 | 239 | 240 | class AsyncToSync: 241 | """ 242 | Utility class which turns an awaitable that only works on the thread with 243 | the event loop into a synchronous callable that works in a subthread. 244 | Must be initialised from the main thread. 245 | """ 246 | 247 | def __init__(self, awaitable): 248 | self.awaitable = awaitable 249 | self.uvloop = False 250 | try: 251 | self.main_event_loop = asyncio.get_event_loop() 252 | except RuntimeError: 253 | # There's no event loop in this thread. Look for the threadlocal if 254 | # we're inside SyncToAsync 255 | self.main_event_loop = getattr( 256 | SyncToAsync.threadlocal, "main_event_loop", None) 257 | 258 | def __call__(self, *args, **kwargs): 259 | # You can't call AsyncToSync from a thread with a running event loop 260 | try: 261 | event_loop = asyncio.get_event_loop() 262 | except RuntimeError: 263 | pass 264 | else: 265 | if event_loop.is_running(): 266 | raise RuntimeError( 267 | "You cannot use AsyncToSync in the same thread as an async event loop - " 268 | "just await the async function directly." 269 | ) 270 | # Make a future for the return information 271 | call_result = Future() 272 | # Use call_soon_threadsafe to schedule a synchronous callback on the 273 | # main event loop's thread 274 | if not (self.main_event_loop and self.main_event_loop.is_running()): 275 | # Make our own event loop and run inside that. 276 | if self.uvloop: 277 | loop = uvloop.new_event_loop() 278 | else: 279 | loop = asyncio.new_event_loop() 280 | asyncio.set_event_loop(loop) 281 | try: 282 | loop.run_until_complete( 283 | self.main_wrap(args, kwargs, call_result)) 284 | finally: 285 | try: 286 | if hasattr(loop, "shutdown_asyncgens"): 287 | loop.run_until_complete(loop.shutdown_asyncgens()) 288 | finally: 289 | loop.close() 290 | asyncio.set_event_loop(self.main_event_loop) 291 | else: 292 | self.main_event_loop.call_soon_threadsafe( 293 | self.main_event_loop.create_task, 294 | self.main_wrap( 295 | args, 296 | kwargs, 297 | call_result, 298 | ), 299 | ) 300 | # Wait for results from the future. 301 | return call_result.result() 302 | 303 | def __get__(self, parent, objtype): 304 | """ 305 | Include self for methods 306 | """ 307 | return functools.partial(self.__call__, parent) 308 | 309 | async def main_wrap(self, args, kwargs, call_result): 310 | """ 311 | Wraps the awaitable with something that puts the result into the 312 | result/exception future. 313 | """ 314 | try: 315 | result = await self.awaitable(*args, **kwargs) 316 | except Exception as e: 317 | call_result.set_exception(e) 318 | else: 319 | call_result.set_result(result) 320 | 321 | 322 | class SyncToAsync: 323 | """ 324 | Utility class which turns a synchronous callable into an awaitable that 325 | runs in a threadpool. It also sets a threadlocal inside the thread so 326 | calls to AsyncToSync can escape it. 327 | """ 328 | 329 | threadpool = ThreadPoolExecutor( 330 | max_workers=( 331 | int(os.environ["ASGI_THREADS"]) 332 | if "ASGI_THREADS" in os.environ 333 | else None 334 | ) 335 | ) 336 | threadlocal = threading.local() 337 | 338 | def __init__(self, func): 339 | self.func = func 340 | 341 | async def __call__(self, *args, **kwargs): 342 | loop = asyncio.get_event_loop() 343 | future = loop.run_in_executor( 344 | self.threadpool, 345 | functools.partial(self.thread_handler, loop, *args, **kwargs), 346 | ) 347 | return await asyncio.wait_for(future, timeout=None) 348 | 349 | def __get__(self, parent, objtype): 350 | """ 351 | Include self for methods 352 | """ 353 | return functools.partial(self.__call__, parent) 354 | 355 | def thread_handler(self, loop, *args, **kwargs): 356 | """ 357 | Wraps the sync application with exception handling. 358 | """ 359 | # Set the threadlocal for AsyncToSync 360 | self.threadlocal.main_event_loop = loop 361 | # Run the function 362 | return self.func(*args, **kwargs) 363 | 364 | 365 | # Lowercase is more sensible for most things 366 | sync_to_async = SyncToAsync 367 | async_to_sync = AsyncToSync 368 | -------------------------------------------------------------------------------- /tests/example04/spawn.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import functools 3 | import os 4 | import threading 5 | from concurrent.futures import Future, ThreadPoolExecutor 6 | 7 | # Alternatively, you can create an instance of the loop manually, using: 8 | 9 | import uvloop 10 | # loop = uvloop.new_event_loop() 11 | # asyncio.set_event_loop(loop) 12 | import queue 13 | import time 14 | from collections import deque 15 | import queue 16 | import threading 17 | import random 18 | import multiprocessing 19 | 20 | 21 | class SyncThread(threading.Thread): 22 | """Threaded Sync reader, read data from queue""" 23 | 24 | def __init__(self, in_queue, out_queue=None, name=None, watching:deque=None): 25 | """Threaded Sync reader, read data from queue 26 | 27 | Arguments: 28 | queue {[type]} -- queue or deque 29 | 30 | Keyword Arguments: 31 | out_queue {[type]} -- queue receive result (default: {None}) 32 | """ 33 | 34 | threading.Thread.__init__(self, name=name) 35 | self.in_queue = in_queue 36 | self.out_queue = out_queue 37 | self.watching = watching 38 | 39 | def main_process(self, f, *args, **kwargs): 40 | return f(*args, **kwargs) 41 | 42 | def run(self): 43 | while True: 44 | # Grabs host from queue 45 | f, args, kwargs = self.in_queue.get() 46 | 47 | # Grabs item and put to input_function 48 | result = self.main_process(f, *args, *kwargs), None 49 | result = result[:-1] 50 | 51 | if self.out_queue is not None: 52 | self.out_queue.put(*result) 53 | 54 | if self.watching is not None: 55 | self.watching.popleft() 56 | # print('Watching list is:', self.watching) 57 | 58 | # Signals to queue job is done 59 | self.in_queue.task_done() 60 | 61 | 62 | class AsyncProcess(multiprocessing.Process): 63 | 64 | def __init__(self, in_queue, out_queue, name=None, watching: deque=None): 65 | multiprocessing.Process.__init__(self, name=name) 66 | self.in_queue = in_queue 67 | self.out_queue = out_queue 68 | self.stop_event = multiprocessing.Event() 69 | self.watching = watching 70 | 71 | def stop(self): 72 | self.stop_event.set() 73 | 74 | def main_process(self, f, *args, **kwargs): 75 | return f(*args, **kwargs) 76 | 77 | def run(self): 78 | while not self.stop_event.is_set(): 79 | # Grabs host from queue 80 | f, args, kwargs = self.in_queue.get() 81 | 82 | # Grabs item and put to input_function 83 | result = self.main_process(f, *args, *kwargs), None 84 | result = result[:-1] 85 | 86 | if self.out_queue is not None: 87 | self.out_queue.put(*result) 88 | 89 | if self.watching is not None: 90 | self.watching.popleft() 91 | 92 | # Signals to queue job is done 93 | self.in_queue.task_done() 94 | if self.stop_event.is_set(): 95 | print('Process %s is stopping' % self.pid) 96 | break 97 | 98 | 99 | class DistributedThreads(object): 100 | 101 | def __init__(self, out_queue=None, max_workers=4, max_watching=100, worker=None, maxsize=0, delay=1): 102 | self.out_queue = out_queue 103 | self.max_workers = max_workers 104 | self.max_watching = max_watching 105 | self.current_id = 0 106 | self.max_qsize = maxsize 107 | self.delay = delay 108 | if worker is None: 109 | self.init_worker() 110 | else: 111 | self.init_worker(worker=worker) 112 | 113 | def init_worker(self, worker=SyncThread): 114 | # create list of queue 115 | self.queue_list = [queue.Queue(maxsize=self.max_qsize) 116 | for i in range(self.max_workers)] 117 | 118 | # create list of watching queue 119 | self.watching_list = [deque() for i in range(self.max_workers)] 120 | 121 | # create list of threads: 122 | self.worker_list = [] 123 | for i in range(self.max_workers): 124 | one_worker = worker( 125 | self.queue_list[i], out_queue=self.out_queue, name=str(i), watching=self.watching_list[i]) 126 | one_worker.daemon = True 127 | self.worker_list.append(one_worker) 128 | one_worker.start() 129 | 130 | def iterate_queue(self, watching: list, key): 131 | if key is not None: 132 | watching.append(key) 133 | if len(watching) > self.max_watching: 134 | time.sleep(self.delay) 135 | 136 | def next_worker(self, last_id): 137 | return (last_id+1) % self.max_workers 138 | 139 | def check_next_queue(self, current_queue_size, last_id): 140 | next_id = self.next_worker(last_id) 141 | if current_queue_size >= self.queue_list[next_id].qsize(): 142 | return next_id 143 | else: 144 | return self.check_next_queue(current_queue_size, next_id) 145 | 146 | def choose_worker(self): 147 | current_queue_size = self.queue_list[self.current_id].qsize() 148 | return self.check_next_queue(current_queue_size, self.current_id) 149 | # return (self.current_id+1) % self.max_workers 150 | 151 | def submit(self, f, *args, **kwargs): 152 | return self.submit_id(None, f, *args, **kwargs) 153 | 154 | def submit_id(self, key, f, *args, **kwargs): 155 | worker_id = None 156 | # check if key belong to any worker 157 | if key is not None: 158 | for i in range(self.max_workers): 159 | if key in self.watching_list[i]: 160 | if worker_id is not None: 161 | raise ValueError("Key belong to more than one worker") 162 | worker_id = i 163 | self.current_id = worker_id 164 | break 165 | # choosing a work_id if not 166 | if worker_id is None: 167 | worker_id = self.choose_worker() 168 | # print('choose queue =>', worker_id) 169 | self.current_id = worker_id 170 | # assign to worker and watching list 171 | worker = self.queue_list[worker_id] 172 | watching = self.watching_list[worker_id] 173 | 174 | # add key to a watching 175 | self.iterate_queue(watching, key) 176 | # print(worker_id, watching) 177 | # add function to queue 178 | worker.put((f, args, kwargs)) 179 | 180 | def shutdown(self): 181 | for q in self.queue_list: 182 | q.join() 183 | 184 | 185 | class DistributedProcess(DistributedThreads): 186 | def init_worker(self, worker=AsyncProcess): 187 | # create list of queue 188 | self.queue_list = [multiprocessing.JoinableQueue() 189 | for i in range(self.max_workers)] 190 | 191 | # create list of watching queue 192 | self.watching_list = [deque() for i in range(self.max_workers)] 193 | 194 | # create list of threads: 195 | self.worker_list = [] 196 | for i in range(self.max_workers): 197 | one_worker = worker( 198 | self.queue_list[i], out_queue=self.out_queue, name=str(i)) 199 | self.worker_list.append(one_worker) 200 | one_worker.start() 201 | 202 | def iterate_queue(self, watching: list, key): 203 | if key is not None: 204 | watching.append(key) 205 | if len(watching) > self.max_watching: 206 | watching.popleft() 207 | 208 | def choose_worker(self): 209 | return (self.current_id+1) % self.max_workers 210 | 211 | def shutdown(self): 212 | for q in self.queue_list: 213 | q.join() 214 | for process in self.worker_list: 215 | print('Distributed Process %s is stopping' % process.pid) 216 | process.terminate() 217 | 218 | 219 | class Worker(threading.Thread): 220 | """Threaded title parser""" 221 | 222 | def __init__(self, out_queue, f): 223 | threading.Thread.__init__(self) 224 | self.out_queue = out_queue 225 | self.f = f 226 | 227 | def run(self): 228 | while True: 229 | # Grabs chunk from queue 230 | args = self.out_queue.get(), None 231 | 232 | # Parse the chunk 233 | args = args[:-1] 234 | self.f(*args) 235 | 236 | # Signals to queue job is done 237 | self.out_queue.task_done() 238 | 239 | 240 | class AsyncToSync: 241 | """ 242 | Utility class which turns an awaitable that only works on the thread with 243 | the event loop into a synchronous callable that works in a subthread. 244 | Must be initialised from the main thread. 245 | """ 246 | 247 | def __init__(self, awaitable): 248 | self.awaitable = awaitable 249 | self.uvloop = False 250 | try: 251 | self.main_event_loop = asyncio.get_event_loop() 252 | except RuntimeError: 253 | # There's no event loop in this thread. Look for the threadlocal if 254 | # we're inside SyncToAsync 255 | self.main_event_loop = getattr( 256 | SyncToAsync.threadlocal, "main_event_loop", None) 257 | 258 | def __call__(self, *args, **kwargs): 259 | # You can't call AsyncToSync from a thread with a running event loop 260 | try: 261 | event_loop = asyncio.get_event_loop() 262 | except RuntimeError: 263 | pass 264 | else: 265 | if event_loop.is_running(): 266 | raise RuntimeError( 267 | "You cannot use AsyncToSync in the same thread as an async event loop - " 268 | "just await the async function directly." 269 | ) 270 | # Make a future for the return information 271 | call_result = Future() 272 | # Use call_soon_threadsafe to schedule a synchronous callback on the 273 | # main event loop's thread 274 | if not (self.main_event_loop and self.main_event_loop.is_running()): 275 | # Make our own event loop and run inside that. 276 | if self.uvloop: 277 | loop = uvloop.new_event_loop() 278 | else: 279 | loop = asyncio.new_event_loop() 280 | asyncio.set_event_loop(loop) 281 | try: 282 | loop.run_until_complete( 283 | self.main_wrap(args, kwargs, call_result)) 284 | finally: 285 | try: 286 | if hasattr(loop, "shutdown_asyncgens"): 287 | loop.run_until_complete(loop.shutdown_asyncgens()) 288 | finally: 289 | loop.close() 290 | asyncio.set_event_loop(self.main_event_loop) 291 | else: 292 | self.main_event_loop.call_soon_threadsafe( 293 | self.main_event_loop.create_task, 294 | self.main_wrap( 295 | args, 296 | kwargs, 297 | call_result, 298 | ), 299 | ) 300 | # Wait for results from the future. 301 | return call_result.result() 302 | 303 | def __get__(self, parent, objtype): 304 | """ 305 | Include self for methods 306 | """ 307 | return functools.partial(self.__call__, parent) 308 | 309 | async def main_wrap(self, args, kwargs, call_result): 310 | """ 311 | Wraps the awaitable with something that puts the result into the 312 | result/exception future. 313 | """ 314 | try: 315 | result = await self.awaitable(*args, **kwargs) 316 | except Exception as e: 317 | call_result.set_exception(e) 318 | else: 319 | call_result.set_result(result) 320 | 321 | 322 | class SyncToAsync: 323 | """ 324 | Utility class which turns a synchronous callable into an awaitable that 325 | runs in a threadpool. It also sets a threadlocal inside the thread so 326 | calls to AsyncToSync can escape it. 327 | """ 328 | 329 | threadpool = ThreadPoolExecutor( 330 | max_workers=( 331 | int(os.environ["ASGI_THREADS"]) 332 | if "ASGI_THREADS" in os.environ 333 | else None 334 | ) 335 | ) 336 | threadlocal = threading.local() 337 | 338 | def __init__(self, func): 339 | self.func = func 340 | 341 | async def __call__(self, *args, **kwargs): 342 | loop = asyncio.get_event_loop() 343 | future = loop.run_in_executor( 344 | self.threadpool, 345 | functools.partial(self.thread_handler, loop, *args, **kwargs), 346 | ) 347 | return await asyncio.wait_for(future, timeout=None) 348 | 349 | def __get__(self, parent, objtype): 350 | """ 351 | Include self for methods 352 | """ 353 | return functools.partial(self.__call__, parent) 354 | 355 | def thread_handler(self, loop, *args, **kwargs): 356 | """ 357 | Wraps the sync application with exception handling. 358 | """ 359 | # Set the threadlocal for AsyncToSync 360 | self.threadlocal.main_event_loop = loop 361 | # Run the function 362 | return self.func(*args, **kwargs) 363 | 364 | 365 | # Lowercase is more sensible for most things 366 | sync_to_async = SyncToAsync 367 | async_to_sync = AsyncToSync 368 | -------------------------------------------------------------------------------- /tests/example05/router.py: -------------------------------------------------------------------------------- 1 | import re 2 | import uuid 3 | from collections import defaultdict, namedtuple 4 | from collections.abc import Iterable 5 | from functools import lru_cache 6 | from urllib.parse import unquote 7 | 8 | from sanic.exceptions import NotFound, MethodNotSupported 9 | from sanic.views import CompositionView 10 | 11 | Route = namedtuple( 12 | 'Route', 13 | ['handler', 'methods', 'pattern', 'parameters', 'name', 'uri']) 14 | Parameter = namedtuple('Parameter', ['name', 'cast']) 15 | 16 | REGEX_TYPES = { 17 | 'string': (str, r'[^/]+'), 18 | 'int': (int, r'\d+'), 19 | 'number': (float, r'[0-9\\.]+'), 20 | 'alpha': (str, r'[A-Za-z]+'), 21 | 'path': (str, r'[^/].*?'), 22 | 'uuid': (uuid.UUID, r'[A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-' 23 | r'[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}') 24 | } 25 | 26 | ROUTER_CACHE_SIZE = 1024 27 | 28 | 29 | def url_hash(url): 30 | return url.count('/') 31 | 32 | 33 | class RouteExists(Exception): 34 | pass 35 | 36 | 37 | class RouteDoesNotExist(Exception): 38 | pass 39 | 40 | 41 | class Router: 42 | """Router supports basic routing with parameters and method checks 43 | Usage: 44 | .. code-block:: python 45 | @sanic.route('/my/url/', methods=['GET', 'POST', ...]) 46 | def my_route(request, my_param): 47 | do stuff... 48 | or 49 | .. code-block:: python 50 | @sanic.route('/my/url/', methods['GET', 'POST', ...]) 51 | def my_route_with_type(request, my_param: my_type): 52 | do stuff... 53 | Parameters will be passed as keyword arguments to the request handling 54 | function. Provided parameters can also have a type by appending :type to 55 | the . Given parameter must be able to be type-casted to this. 56 | If no type is provided, a string is expected. A regular expression can 57 | also be passed in as the type. The argument given to the function will 58 | always be a string, independent of the type. 59 | """ 60 | routes_static = None 61 | routes_dynamic = None 62 | routes_always_check = None 63 | parameter_pattern = re.compile(r'<(.+?)>') 64 | 65 | def __init__(self): 66 | self.routes_all = {} 67 | self.routes_names = {} 68 | self.routes_static_files = {} 69 | self.routes_static = {} 70 | self.routes_dynamic = defaultdict(list) 71 | self.routes_always_check = [] 72 | self.hosts = set() 73 | 74 | @classmethod 75 | def parse_parameter_string(cls, parameter_string): 76 | """Parse a parameter string into its constituent name, type, and 77 | pattern 78 | For example:: 79 | parse_parameter_string('')` -> 80 | ('param_one', str, '[A-z]') 81 | :param parameter_string: String to parse 82 | :return: tuple containing 83 | (parameter_name, parameter_type, parameter_pattern) 84 | """ 85 | # We could receive NAME or NAME:PATTERN 86 | name = parameter_string 87 | pattern = 'string' 88 | if ':' in parameter_string: 89 | name, pattern = parameter_string.split(':', 1) 90 | if not name: 91 | raise ValueError( 92 | "Invalid parameter syntax: {}".format(parameter_string) 93 | ) 94 | 95 | default = (str, pattern) 96 | # Pull from pre-configured types 97 | _type, pattern = REGEX_TYPES.get(pattern, default) 98 | 99 | return name, _type, pattern 100 | 101 | def add(self, uri, methods, handler, host=None, strict_slashes=False, 102 | version=None, name=None): 103 | """Add a handler to the route list 104 | :param uri: path to match 105 | :param methods: sequence of accepted method names. If none are 106 | provided, any method is allowed 107 | :param handler: request handler function. 108 | When executed, it should provide a response object. 109 | :param strict_slashes: strict to trailing slash 110 | :param version: current version of the route or blueprint. See 111 | docs for further details. 112 | :return: Nothing 113 | """ 114 | if version is not None: 115 | version = re.escape(str(version).strip('/').lstrip('v')) 116 | uri = "/".join(["/v{}".format(version), uri.lstrip('/')]) 117 | # add regular version 118 | self._add(uri, methods, handler, host, name) 119 | 120 | if strict_slashes: 121 | return 122 | 123 | if not isinstance(host, str) and host is not None: 124 | # we have gotten back to the top of the recursion tree where the 125 | # host was originally a list. By now, we've processed the strict 126 | # slashes logic on the leaf nodes (the individual host strings in 127 | # the list of host) 128 | return 129 | 130 | # Add versions with and without trailing / 131 | slashed_methods = self.routes_all.get(uri + '/', frozenset({})) 132 | unslashed_methods = self.routes_all.get(uri[:-1], frozenset({})) 133 | if isinstance(methods, Iterable): 134 | _slash_is_missing = all(method in slashed_methods for 135 | method in methods) 136 | _without_slash_is_missing = all(method in unslashed_methods for 137 | method in methods) 138 | else: 139 | _slash_is_missing = methods in slashed_methods 140 | _without_slash_is_missing = methods in unslashed_methods 141 | 142 | slash_is_missing = ( 143 | not uri[-1] == '/' and not _slash_is_missing 144 | ) 145 | without_slash_is_missing = ( 146 | uri[-1] == '/' and not 147 | _without_slash_is_missing and not 148 | uri == '/' 149 | ) 150 | # add version with trailing slash 151 | if slash_is_missing: 152 | self._add(uri + '/', methods, handler, host, name) 153 | # add version without trailing slash 154 | elif without_slash_is_missing: 155 | self._add(uri[:-1], methods, handler, host, name) 156 | 157 | def _add(self, uri, methods, handler, host=None, name=None): 158 | """Add a handler to the route list 159 | :param uri: path to match 160 | :param methods: sequence of accepted method names. If none are 161 | provided, any method is allowed 162 | :param handler: request handler function. 163 | When executed, it should provide a response object. 164 | :param name: user defined route name for url_for 165 | :return: Nothing 166 | """ 167 | if host is not None: 168 | if isinstance(host, str): 169 | uri = host + uri 170 | self.hosts.add(host) 171 | 172 | else: 173 | if not isinstance(host, Iterable): 174 | raise ValueError("Expected either string or Iterable of " 175 | "host strings, not {!r}".format(host)) 176 | 177 | for host_ in host: 178 | self.add(uri, methods, handler, host_, name) 179 | return 180 | 181 | # Dict for faster lookups of if method allowed 182 | if methods: 183 | methods = frozenset(methods) 184 | 185 | parameters = [] 186 | properties = {"unhashable": None} 187 | 188 | def add_parameter(match): 189 | name = match.group(1) 190 | name, _type, pattern = self.parse_parameter_string(name) 191 | 192 | parameter = Parameter( 193 | name=name, cast=_type) 194 | parameters.append(parameter) 195 | 196 | # Mark the whole route as unhashable if it has the hash key in it 197 | if re.search(r'(^|[^^]){1}/', pattern): 198 | properties['unhashable'] = True 199 | # Mark the route as unhashable if it matches the hash key 200 | elif re.search(r'/', pattern): 201 | properties['unhashable'] = True 202 | 203 | return '({})'.format(pattern) 204 | 205 | pattern_string = re.sub(self.parameter_pattern, add_parameter, uri) 206 | pattern = re.compile(r'^{}$'.format(pattern_string)) 207 | 208 | def merge_route(route, methods, handler): 209 | # merge to the existing route when possible. 210 | if not route.methods or not methods: 211 | # method-unspecified routes are not mergeable. 212 | raise RouteExists( 213 | "Route already registered: {}".format(uri)) 214 | elif route.methods.intersection(methods): 215 | # already existing method is not overloadable. 216 | duplicated = methods.intersection(route.methods) 217 | raise RouteExists( 218 | "Route already registered: {} [{}]".format( 219 | uri, ','.join(list(duplicated)))) 220 | if isinstance(route.handler, CompositionView): 221 | view = route.handler 222 | else: 223 | view = CompositionView() 224 | view.add(route.methods, route.handler) 225 | view.add(methods, handler) 226 | route = route._replace( 227 | handler=view, methods=methods.union(route.methods)) 228 | return route 229 | 230 | if parameters: 231 | # TODO: This is too complex, we need to reduce the complexity 232 | if properties['unhashable']: 233 | routes_to_check = self.routes_always_check 234 | ndx, route = self.check_dynamic_route_exists( 235 | pattern, routes_to_check, parameters) 236 | else: 237 | routes_to_check = self.routes_dynamic[url_hash(uri)] 238 | ndx, route = self.check_dynamic_route_exists( 239 | pattern, routes_to_check, parameters) 240 | if ndx != -1: 241 | # Pop the ndx of the route, no dups of the same route 242 | routes_to_check.pop(ndx) 243 | else: 244 | route = self.routes_all.get(uri) 245 | 246 | # prefix the handler name with the blueprint name 247 | # if available 248 | # special prefix for static files 249 | is_static = False 250 | if name and name.startswith('_static_'): 251 | is_static = True 252 | name = name.split('_static_', 1)[-1] 253 | 254 | if hasattr(handler, '__blueprintname__'): 255 | handler_name = '{}.{}'.format( 256 | handler.__blueprintname__, name or handler.__name__) 257 | else: 258 | handler_name = name or getattr(handler, '__name__', None) 259 | 260 | if route: 261 | route = merge_route(route, methods, handler) 262 | else: 263 | route = Route( 264 | handler=handler, methods=methods, pattern=pattern, 265 | parameters=parameters, name=handler_name, uri=uri) 266 | 267 | self.routes_all[uri] = route 268 | if is_static: 269 | pair = self.routes_static_files.get(handler_name) 270 | if not (pair and (pair[0] + '/' == uri or uri + '/' == pair[0])): 271 | self.routes_static_files[handler_name] = (uri, route) 272 | 273 | else: 274 | pair = self.routes_names.get(handler_name) 275 | if not (pair and (pair[0] + '/' == uri or uri + '/' == pair[0])): 276 | self.routes_names[handler_name] = (uri, route) 277 | 278 | if properties['unhashable']: 279 | self.routes_always_check.append(route) 280 | elif parameters: 281 | self.routes_dynamic[url_hash(uri)].append(route) 282 | else: 283 | self.routes_static[uri] = route 284 | 285 | @staticmethod 286 | def check_dynamic_route_exists(pattern, routes_to_check, parameters): 287 | for ndx, route in enumerate(routes_to_check): 288 | if route.pattern == pattern and route.parameters == parameters: 289 | return ndx, route 290 | else: 291 | return -1, None 292 | 293 | def remove(self, uri, clean_cache=True, host=None): 294 | if host is not None: 295 | uri = host + uri 296 | try: 297 | route = self.routes_all.pop(uri) 298 | for handler_name, pairs in self.routes_names.items(): 299 | if pairs[0] == uri: 300 | self.routes_names.pop(handler_name) 301 | break 302 | 303 | for handler_name, pairs in self.routes_static_files.items(): 304 | if pairs[0] == uri: 305 | self.routes_static_files.pop(handler_name) 306 | break 307 | 308 | except KeyError: 309 | raise RouteDoesNotExist("Route was not registered: {}".format(uri)) 310 | 311 | if route in self.routes_always_check: 312 | self.routes_always_check.remove(route) 313 | elif url_hash(uri) in self.routes_dynamic \ 314 | and route in self.routes_dynamic[url_hash(uri)]: 315 | self.routes_dynamic[url_hash(uri)].remove(route) 316 | else: 317 | self.routes_static.pop(uri) 318 | 319 | if clean_cache: 320 | self._get.cache_clear() 321 | 322 | @lru_cache(maxsize=ROUTER_CACHE_SIZE) 323 | def find_route_by_view_name(self, view_name, name=None): 324 | """Find a route in the router based on the specified view name. 325 | :param view_name: string of view name to search by 326 | :param kwargs: additional params, usually for static files 327 | :return: tuple containing (uri, Route) 328 | """ 329 | if not view_name: 330 | return (None, None) 331 | 332 | if view_name == 'static' or view_name.endswith('.static'): 333 | return self.routes_static_files.get(name, (None, None)) 334 | 335 | return self.routes_names.get(view_name, (None, None)) 336 | 337 | def get(self, request): 338 | """Get a request handler based on the URL of the request, or raises an 339 | error 340 | :param request: Request object 341 | :return: handler, arguments, keyword arguments 342 | """ 343 | # No virtual hosts specified; default behavior 344 | if not self.hosts: 345 | return self._get(request.path, request.method, '') 346 | # virtual hosts specified; try to match route to the host header 347 | try: 348 | return self._get(request.path, request.method, 349 | request.headers.get("Host", '')) 350 | # try default hosts 351 | except NotFound: 352 | return self._get(request.path, request.method, '') 353 | 354 | def get_supported_methods(self, url): 355 | """Get a list of supported methods for a url and optional host. 356 | :param url: URL string (including host) 357 | :return: frozenset of supported methods 358 | """ 359 | route = self.routes_all.get(url) 360 | # if methods are None then this logic will prevent an error 361 | return getattr(route, 'methods', None) or frozenset() 362 | 363 | @lru_cache(maxsize=ROUTER_CACHE_SIZE) 364 | def _get(self, url, method, host): 365 | """Get a request handler based on the URL of the request, or raises an 366 | error. Internal method for caching. 367 | :param url: request URL 368 | :param method: request method 369 | :return: handler, arguments, keyword arguments 370 | """ 371 | url = unquote(host + url) 372 | # Check against known static routes 373 | route = self.routes_static.get(url) 374 | method_not_supported = MethodNotSupported( 375 | 'Method {} not allowed for URL {}'.format(method, url), 376 | method=method, 377 | allowed_methods=self.get_supported_methods(url)) 378 | if route: 379 | if route.methods and method not in route.methods: 380 | raise method_not_supported 381 | match = route.pattern.match(url) 382 | else: 383 | route_found = False 384 | # Move on to testing all regex routes 385 | for route in self.routes_dynamic[url_hash(url)]: 386 | match = route.pattern.match(url) 387 | route_found |= match is not None 388 | # Do early method checking 389 | if match and method in route.methods: 390 | break 391 | else: 392 | # Lastly, check against all regex routes that cannot be hashed 393 | for route in self.routes_always_check: 394 | match = route.pattern.match(url) 395 | route_found |= match is not None 396 | # Do early method checking 397 | if match and method in route.methods: 398 | break 399 | else: 400 | # Route was found but the methods didn't match 401 | if route_found: 402 | raise method_not_supported 403 | raise NotFound('Requested URL {} not found'.format(url)) 404 | 405 | kwargs = {p.name: p.cast(value) 406 | for value, p 407 | in zip(match.groups(1), route.parameters)} 408 | route_handler = route.handler 409 | if hasattr(route_handler, 'handlers'): 410 | route_handler = route_handler.handlers[method] 411 | return route_handler, [], kwargs, route.uri 412 | 413 | def is_stream_handler(self, request): 414 | """ Handler for request is stream or not. 415 | :param request: Request object 416 | :return: bool 417 | """ 418 | try: 419 | handler = self.get(request)[0] 420 | except (NotFound, MethodNotSupported): 421 | return False 422 | if (hasattr(handler, 'view_class') and 423 | hasattr(handler.view_class, request.method.lower())): 424 | handler = getattr(handler.view_class, request.method.lower()) 425 | return hasattr(handler, 'is_stream') 426 | -------------------------------------------------------------------------------- /microservices_connector/Interservices.py: -------------------------------------------------------------------------------- 1 | from flask import Flask, stream_with_context 2 | from flask import request, jsonify, Response 3 | import inspect 4 | import json 5 | from functools import wraps 6 | import requests 7 | import time 8 | import traceback 9 | import asyncio 10 | import os 11 | # import aiohttp.web 12 | # import aiohttp_debugtoolbar 13 | # from aiohttp_debugtoolbar import toolbar_middleware_factory 14 | 15 | def split_to_equal_text(text, lenght=25000): 16 | return [text[start:start+lenght] for start in range(0, len(text), lenght)] 17 | 18 | @stream_with_context 19 | def stream_generate(splited_text): 20 | for text in splited_text: 21 | yield text 22 | 23 | def timeit(method): 24 | 25 | def timed(*args, **kw): 26 | ts = time.time() 27 | result = method(*args, **kw) 28 | te = time.time() 29 | 30 | print('%r %2.4f ms' % 31 | (method.__name__, (te - ts) * 1000)) 32 | return result 33 | 34 | return timed 35 | 36 | 37 | class Microservice(object): 38 | 39 | def __init__(self, name, port: int=5000, host: str='0.0.0.0', debug=None, token = {}, secretKey=None, **kwargs): 40 | """Microservice(name, port: int=5000, host: str='0.0.0.0', debug=None, token = {}, secretKey=None) 41 | 42 | Arguments: 43 | name {str} -- Require a name for your app, recommend put __name__ for it 44 | port {int} -- Choose from 3000 to 9000, default to 5000 45 | host {str} -- Host ip, Default 0.0.0.0 for localhost 46 | debug {boolean} -- True for development, False/None for production 47 | token {dict} -- A dict contain all rule and its token. It can be set later 48 | """ 49 | self.port = port 50 | self.host = host 51 | self.debug = debug 52 | self.token = token 53 | self.secretKey = secretKey 54 | self.init_app(name, **kwargs) 55 | 56 | def init_app(self, name, **kwargs): 57 | self.app = Flask(name) 58 | 59 | def removeToken(self, token: str): 60 | return self.token.pop(token, None) 61 | 62 | def typing(self, rule: str, **options): 63 | if not rule.startswith('/'): 64 | rule = '/' + rule 65 | 66 | def decorator(f): 67 | endpoint = options.pop('endpoint', None) 68 | methods = options.pop('methods', None) 69 | token = options.pop('token', None) 70 | try: 71 | self.token[rule] = str(token) 72 | except: 73 | traceback.print_exc() 74 | # if the methods are not given and the view_func object knows its 75 | # methods we can use that instead. If neither exists, we go with 76 | # a tuple of only ``POST`` as default. 77 | if methods is None: 78 | options['methods'] = ('POST',) 79 | self.app.add_url_rule(rule, endpoint, f, **options) 80 | return f 81 | return decorator 82 | 83 | route = typing 84 | 85 | def reply(self, f): 86 | @wraps(f) 87 | def wrapper(*args, **kwargs): 88 | content = request.get_json(silent=True) 89 | if content is not None: 90 | if 'token' in content: 91 | # print(request.path) 92 | if self.token[request.path] != content['token'] and self.token[request.path] == None: 93 | # print(self.token[request.path] is not None) will return true !! 94 | return {'type': 'error', 'obj': 'Token is wrong'} 95 | # check token 96 | if args is not None: 97 | for arg in content['args']: 98 | args += (arg,) 99 | if kwargs is not None: 100 | for key in content['kwargs']: 101 | kwargs[key] = content['kwargs'][key] 102 | # else: 103 | # raise ValueError('Request contain no json') 104 | # print(request.headers) 105 | return self.microResponse(f(*args, **kwargs)) 106 | return wrapper 107 | 108 | def json(self, f): 109 | @wraps(f) 110 | def wrapper(*args, **kwargs): 111 | content = request.get_json(silent=True) 112 | if content is not None: 113 | for key in content: 114 | kwargs[key] = content[key] 115 | return self.microResponse(f(*args, **kwargs)) 116 | return wrapper 117 | 118 | def dict(self, f): 119 | @wraps(f) 120 | def wrapper(*args, **kwargs): 121 | content = request.get_json(silent=True) 122 | if content is not None: 123 | for key in content: 124 | kwargs[key] = content[key] 125 | return jsonify(oneResponse(f(*args, **kwargs))) 126 | return wrapper 127 | 128 | def stream(self, f): 129 | @wraps(f) 130 | def wrapper(*args, **kwargs): 131 | content = request.get_json(silent=True) 132 | if content is not None: 133 | for key in content: 134 | kwargs[key] = content[key] 135 | return Response(stream_generate(json.dumps(f(*args, **kwargs))), mimetype='application/json') 136 | return wrapper 137 | 138 | def run(self, port=None, host=None, debug=None): 139 | if port is None: 140 | port = self.port 141 | if host is None: 142 | host = self.host 143 | if debug is None: 144 | debug = self.debug 145 | self.app.run(port=port, host=host, debug=debug) 146 | 147 | def microResponse(self, *args): 148 | final = [] 149 | # print(len(args), type(args)) 150 | if len(args) == 0: 151 | return final 152 | else: 153 | args = list(args,) 154 | # print(args) 155 | for arg in args: 156 | if isinstance(arg, tuple): 157 | arg = list(arg) 158 | # print(arg) 159 | for i in arg: 160 | final.append(oneResponse(i)) 161 | else: 162 | final.append(oneResponse(arg)) 163 | return jsonify(final) 164 | 165 | 166 | def oneResponse(res): 167 | if res is None: 168 | return None 169 | elif isinstance(res, (float, int, str)): 170 | return res 171 | elif isinstance(res, (list, tuple)): 172 | return [oneResponse(i) for i in res] 173 | elif isinstance(res, (dict, set)): 174 | resDict = {} 175 | for i in res: 176 | resDict[i] = oneResponse(res[i]) 177 | return resDict 178 | elif isinstance(res, object): 179 | try: 180 | return propsOBJ(res) 181 | except Exception: 182 | traceback.print_exc() 183 | return 'Error: Object type is not supported!' 184 | else: 185 | return 'Error: Object type is not supported!' 186 | 187 | # send mess to a microservices. It's a friend 188 | 189 | 190 | class Friend(object): 191 | def __init__(self, name: str, address: str, token: dict = {}, ruleMethods: dict = {}): 192 | self.name = name 193 | if not address.startswith('http://') and not address.startswith('https://'): 194 | address = 'http://'+address 195 | self.address = address 196 | self.token = token 197 | self.lastMessage = None 198 | self.lastreply = None 199 | self.ruleMethods = ruleMethods 200 | 201 | def setRule(self, rule: str, method: str = None, token: str = None): 202 | self.ruleMethods[rule] = method 203 | self.token[rule] = token 204 | 205 | def send(self, rule: str, *args, **kwargs): 206 | listargs = [] 207 | if len(args) > 0: 208 | for i in list(args): 209 | listargs.append(oneResponse(i)) 210 | dictkwargs = dict() 211 | if len(kwargs) > 0: 212 | kwargs = dict(kwargs) 213 | for i in kwargs: 214 | dictkwargs[i] = oneResponse(kwargs[i]) 215 | if rule in self.token: 216 | token = self.token[rule] 217 | else: 218 | token = None 219 | jsonsend = {"args": listargs, 'kwargs': dictkwargs, 'token': token} 220 | 221 | if rule in self.ruleMethods: 222 | ruleMethods = self.ruleMethods 223 | elif 'methods' in kwargs: 224 | methods = kwargs.pop('methods', None) 225 | ruleMethods = {rule: methods} 226 | else: 227 | ruleMethods = {} 228 | if rule in ruleMethods: 229 | method = ruleMethods[rule] 230 | r = self.http_request(rule, method, jsonsend) 231 | else: 232 | r = requests.post(self.address+rule, 233 | json=jsonsend) 234 | # print(r.headers) 235 | self.lastreply = r 236 | # print(r.text) 237 | if r.status_code == 200: 238 | # print(r.headers['Content-Type'] == 'application/json') 239 | if r.headers['Content-Type'] == 'application/json': 240 | # print(r.text) 241 | res = r.json() 242 | try: 243 | # res = json.loads(res['res']) 244 | self.lastMessage = res 245 | if isinstance(res, list): 246 | final = [] 247 | for arg in res: 248 | final.append(arg) 249 | if len(final) <= 1: 250 | return final[0] 251 | return final 252 | else: 253 | final = res 254 | except Exception: 255 | traceback.print_exc() 256 | final = r.text 257 | self.lastMessage = res 258 | return final 259 | self.lastMessage = r.text 260 | return r.text 261 | self.lastMessage = None 262 | return None 263 | 264 | def http_request(self, rule, method, jsonsend): 265 | if method == 'GET': 266 | r = requests.get(self.address+rule, 267 | json=jsonsend) 268 | elif method == 'PUT': 269 | r = requests.put(self.address+rule, 270 | json=jsonsend) 271 | elif method == 'DELETE': 272 | r = requests.delete(self.address+rule, 273 | json=jsonsend) 274 | elif method == 'PATCH': 275 | r = requests.patch(self.address+rule, 276 | json=jsonsend) 277 | elif method == 'HEAD': 278 | r = requests.head(self.address+rule, 279 | json=jsonsend) 280 | elif method == 'OPTIONS': 281 | r = requests.options(self.address+rule, 282 | json=jsonsend) 283 | else: 284 | r = requests.post(self.address+rule, 285 | json=jsonsend) 286 | return r 287 | 288 | def json(self, rule: str, method='POST', **kwargs): 289 | jsonsend = {} 290 | for key in kwargs: 291 | jsonsend[key] = kwargs[key] 292 | r = self.http_request(rule, method, jsonsend) 293 | self.lastreply = r 294 | # print(r.text) 295 | if r.status_code == 200: 296 | # print(r.headers['Content-Type'] == 'application/json') 297 | if r.headers['Content-Type'] == 'application/json': 298 | # print(r.text) 299 | res = r.json() 300 | try: 301 | # res = json.loads(res['res']) 302 | self.lastMessage = res 303 | if isinstance(res, list): 304 | final = [] 305 | for arg in res: 306 | final.append(arg) 307 | if len(final) <= 1: 308 | return final[0] 309 | return final 310 | else: 311 | final = res 312 | except Exception: 313 | traceback.print_exc() 314 | final = r.text 315 | self.lastMessage = res 316 | return final 317 | self.lastMessage = r.text 318 | return r.text 319 | self.lastMessage = None 320 | return None 321 | 322 | 323 | def propsOBJ(obj): 324 | pr = {} 325 | for name in dir(obj): 326 | # print(dir(obj)) 327 | value = getattr(obj, name) 328 | if not name.startswith('__') and not inspect.ismethod(value): 329 | pr[name] = value 330 | return pr 331 | 332 | 333 | from sanic import Sanic 334 | from sanic import response 335 | 336 | 337 | class SanicApp(Microservice): 338 | def __init__(self, name=None, port: int=5000, host: str='0.0.0.0', debug=None, token = {}, secretKey=None, **kwargs): 339 | """SanicApp(name, port: int=5000, host: str='0.0.0.0', debug=None, token: dict = {}, secretKey=None) 340 | 341 | Arguments: 342 | name {str} -- Require a name for your app, recommend put __name__ for it 343 | port {int} -- Choose from 3000 to 9000, default to 5000 344 | host {str} -- Host ip, Default 0.0.0.0 for localhost 345 | debug {boolean} -- True for development, False/None for production 346 | token {dict} -- A dict contain all rule and its token. It can be set later 347 | """ 348 | super().__init__(name, port, host, debug, token, secretKey, **kwargs) 349 | 350 | def init_app(self, name, **kwargs): 351 | self.app = Sanic(name) 352 | 353 | def typing(self, uri, methods=['POST'], token=None, host=None, 354 | strict_slashes=None, stream=False, version=None, name=None): 355 | """Decorate a function to be registered as a route 356 | 357 | :param uri: path of the URL 358 | :param methods: list or tuple of methods allowed 359 | :param host: 360 | :param strict_slashes: 361 | :param stream: 362 | :param version: 363 | :param name: user defined route name for url_for 364 | :return: decorated function 365 | """ 366 | 367 | # Fix case where the user did not prefix the URL with a / 368 | # and will probably get confused as to why it's not working 369 | if not uri.startswith('/'): 370 | uri = '/' + uri 371 | 372 | if stream: 373 | self.app.is_request_stream = True 374 | 375 | if strict_slashes is None: 376 | strict_slashes = self.app.strict_slashes 377 | 378 | try: 379 | self.token[uri] = str(token) 380 | except: 381 | traceback.print_exc() 382 | 383 | def response(handler): 384 | if stream: 385 | handler.is_stream = stream 386 | self.app.router.add(uri=uri, methods=methods, handler=handler, 387 | host=host, strict_slashes=strict_slashes, 388 | version=version, name=name) 389 | return handler 390 | 391 | return response 392 | 393 | route = typing 394 | 395 | def reply(self, f): 396 | @wraps(f) 397 | def wrapper(sanicRequest, *args, **kwargs): 398 | content = sanicRequest.json 399 | if content is not None: 400 | if 'token' in content: 401 | # print(request.path) 402 | if self.token[sanicRequest.path] != content['token'] and self.token[sanicRequest.path] == None: 403 | # print(self.token[request.path] is not None) will return true !! 404 | return {'type': 'error', 'obj': 'Token is wrong'} 405 | # check token 406 | if args is not None: 407 | for arg in content['args']: 408 | args += (arg,) 409 | if kwargs is not None: 410 | for key in content['kwargs']: 411 | kwargs[key] = content['kwargs'][key] 412 | # else: 413 | # raise ValueError('Request contain no json') 414 | # print(request.headers) 415 | return self.microResponse(f(*args, **kwargs)) 416 | return wrapper 417 | 418 | def json(self, f): 419 | @wraps(f) 420 | def wrapper(sanicRequest, *args, **kwargs): 421 | content = sanicRequest.json 422 | for key in content: 423 | kwargs[key] = content[key] 424 | return self.microResponse(f(*args, **kwargs)) 425 | return wrapper 426 | 427 | def dict(self, f): 428 | @wraps(f) 429 | def wrapper(sanicRequest, *args, **kwargs): 430 | content = sanicRequest.json or [] 431 | for key in content: 432 | kwargs[key] = content[key] 433 | return response.json(oneResponse(f(*args, **kwargs))) 434 | return wrapper 435 | 436 | def async_json(self, f): 437 | @wraps(f) 438 | async def wrapper(sanicRequest, *args, **kwargs): 439 | content = sanicRequest.json or [] 440 | for key in content: 441 | kwargs[key] = content[key] 442 | return self.microResponse(await f(*args, **kwargs)) 443 | return wrapper 444 | 445 | def microResponse(self, *args): 446 | final = [] 447 | # print(len(args), type(args)) 448 | if len(args) == 0: 449 | return final 450 | else: 451 | args = list(args,) 452 | # print(args) 453 | for arg in args: 454 | if isinstance(arg, tuple): 455 | arg = list(arg) 456 | # print(arg) 457 | for i in arg: 458 | final.append(oneResponse(i)) 459 | else: 460 | final.append(oneResponse(arg)) 461 | return response.json(final) 462 | 463 | 464 | # class AioSocket(object): 465 | # def __init__(self, name=None, port: int=5000, host: str='0.0.0.0', debug=None, **kwargs): 466 | # self.port = port 467 | # self.host = host 468 | # self.debug = debug 469 | # self.init_app(name, **kwargs) 470 | 471 | # def init_app(self, name, **kwargs): 472 | # loop = asyncio.get_event_loop() 473 | # self.app = aiohttp.web.Application(loop=loop) 474 | # aiohttp_debugtoolbar.setup(self.app) 475 | 476 | # def typing(self, rule: str, methods='POST', **options): 477 | # if not rule.startswith('/'): 478 | # rule = '/' + rule 479 | 480 | # def decorator(f): 481 | # token = options.pop('token', None) 482 | # # if the methods are not given and the view_func object knows its 483 | # # methods we can use that instead. If neither exists, we go with 484 | # # a tuple of only ``POST`` as default. 485 | # for method in methods: 486 | # self.app.router.add_route(method, rule, f, **options) 487 | # return f 488 | # return decorator 489 | # route = typing 490 | 491 | # def async_json(self, f): 492 | # @wraps(f) 493 | # async def wrapper(aio_request, *args, **kwargs): 494 | # print(await aio_request.text(), type(await aio_request.text())) 495 | # # print(aio_request.json) 496 | # content = json.loads(await aio_request.text()) 497 | # for key in content: 498 | # kwargs[key] = content[key] 499 | # return self.microResponse(await f(*args, **kwargs)) 500 | # return wrapper 501 | 502 | # def microResponse(self, *args): 503 | # final = [] 504 | # # print(len(args), type(args)) 505 | # if len(args) == 0: 506 | # return final 507 | # else: 508 | # args = list(args,) 509 | # # print(args) 510 | # for arg in args: 511 | # if isinstance(arg, tuple): 512 | # arg = list(arg) 513 | # # print(arg) 514 | # for i in arg: 515 | # final.append(oneResponse(i)) 516 | # else: 517 | # final.append(oneResponse(arg)) 518 | # return aiohttp.web.json_response(final) 519 | 520 | # def run(self, host, port, **kwargs): 521 | # aiohttp.web.run_app(self.app, host=host, port=port, **kwargs) 522 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Microservices Connector 2 | Microservices Connector is a Inter-Service communication framework, support for microservice architecture and distributed system. 3 | 4 | ## Getting Started 5 | 6 | __Microservices__ is a way of breaking large software projects into loosely coupled modules, which communicate with each other through simple APIs. The advantages of microservices are improves fault isolation, scalability, Ease of development. It convinced some big enterprise players – like Amazon, Netflix, and eBay – to begin their transitions. 7 | 8 | _Microservices Connector_ is a Inter-Service communication framework written in python, support for microservice architecture and distributed system. Its features contain: 9 | * Transfering data as returning a function results or quering data. 10 | * Support transfer multiple values with many type as string, int, float, list, dict, class attribute 11 | * Do not require knowledge about web/http connection or touch to them 12 | * Distributed system 13 | 14 | ![alt text](images/Distributed-system.jpeg "Illustration of network system. Source:medium.com") 15 | 16 | Illustration of network system. Source:medium.com 17 | 18 | As illustration, distributed systems are very stable and Infinite scalability. But distributed systems are the most difficult to maintain. 19 | 20 | _Microservices Connector_ supports communication by the following framework: 21 | * flask - A micro web framework 22 | * Sanic - Gotta go fast (A async web framework) 23 | ### Quickstart: 24 | 25 | Microservices Connector is available on pip. You can install via pip (require python>=3.5): 26 | 27 | ``` 28 | pip install microservices_connector 29 | ``` 30 | Start project by a minimum example: 31 | 32 | * Step 1: Create 1 file name app1.py, write and save with the code below 33 | 34 | ```python 35 | from microservices_connector.Interservices import Microservice 36 | 37 | Micro = Microservice(__name__) 38 | 39 | @Micro.typing('/helloworld') 40 | @Micro.reply 41 | def helloworld(name): 42 | return 'Welcome %s' % (name) 43 | 44 | if __name__ == '__main__': 45 | Micro.run() 46 | ``` 47 | 48 | * Step 2: Create 1 file name app2.py, write and save with the code below 49 | 50 | ```python 51 | from microservices_connector.Interservices import Friend 52 | 53 | aFriend= Friend('app1', 'http://0.0.0.0:5000') 54 | message = aFriend.send('/helloworld','Mr. Developer') 55 | print(message) 56 | ``` 57 | * Step 3: Run app1.py and app2.py together. 58 | 59 | Open a terminal in the same folder with them and run: 60 | `python app1.py`. 61 | 62 | Open another terminal in the same folder and run: 63 | `python app2.py` 64 | 65 | You get the result: `Welcome Mr. Developer` in terminal of app2.py. This is no difference if you do `from .app1 import helloword; message = helloworld('Mr. Developer'); print(message)`. Note: To stop app1, open its terminal and Ctrl + C. The example can be found in `test/example00` folder 66 | 67 | **Explanation**: App1 and app2 are small example of microservice system. In the example, **M** in app1 is listener, a http server while __F__ in app2 is sender. Listener and sender are isolated design, they can work seperately or together in an app/service. A microservice can be a listener of many other microservices or sender of many other microservices or both of them. A standard microservice mention in this project contain both listener and sender. 68 | 69 | ## Tutorial 70 | 71 | ### 1. Start your first app 72 | 73 | In the tutorial, we assume you have this code in the top of your file. 74 | 75 | ```python 76 | from microservices_connector.Interservices import Microservice 77 | 78 | Micro = Microservice(__name__) 79 | 80 | # for Sanic framework if you want to use sanic instead of flask 81 | from microservices_connector.Interservices import SanicApp as Microservice 82 | 83 | Micro = Microservice(__name__) 84 | ``` 85 | 86 | Now, look closer at `Microservice(__name__)`, it actually look like this: 87 | 88 | Microservice(name, port: int=5000, host: str='0.0.0.0', debug=None, token: dict = {}, secretKey=None) 89 | 90 | Arguments: 91 | * name {str} -- Require a name for your app, recommend put `__name__` for it 92 | * port {int} -- Choose from 3000 to 9000, default to 5000 93 | * host {str} -- Host ip, Default 0.0.0.0 for localhost 94 | * debug {boolean} -- True for development, False/None for production 95 | * token {dict} -- A dict contain all rule and its token. It can be set later 96 | 97 | The class Microservice is used to create a listener/a microservice. In a file/app, you should only have one listener. About parameters, If you aren't familiar with http server, you only need remember: 98 | * One app should have only one listener 99 | * Should use `__name__` for name and name need to be unique 100 | * If you run multiple listener, use only one unique port for each listener. for example: 101 | 102 | ```python 103 | M1 = Microservice(__name__, port=5010) # in file app1 104 | M2 = Microservice(__name__, port=5020) # in file app2 105 | ``` 106 | Note: You should be carefully if there are other web applications running in your port/server. 107 | 108 | ### 2. Way of running an app 109 | 110 | A sender is a python def, so you can put it anywhere in your app. Listener is a http server so it's a bit difference from other. 111 | 112 | Option 1: Use if/main in the end of startup file (file that you start your project by `python `). Add the following code the end: 113 | ```python 114 | # Micro is your Microservice Object 115 | if __name__ == '__main__': 116 | Micro.run() 117 | ``` 118 | 119 | Option 2: Create a file name run.py and run your app from this file. For example, we create a run.py in the same folder of app1.py in the first example. It will be like this: 120 | ```python 121 | from app1 import Micro 122 | 123 | if __name__ == '__main__': 124 | Micro.run(port=5000, host='0.0.0.0', debug=True) 125 | ``` 126 | Option 2 is more appreciated. It avoid the app looping from them self, so get away of stunning your app. If you have 2 app in a server/computer, you should create 2 run file for it. Don't for get `Ctrl + C` to stop your app. 127 | 128 | Note: _We assume you already use one of the options above for your code_. This tutorial focuses on communication between 'service-to-service' as def function, not http connect. 129 | 130 | ### 3. Send, Typing and reply 131 | 132 | Think like a human, if you want to communicate with some friend in facebook, you will open *messenger*, find your friend and send a message to them. It's a way of sending message to each other. Then, your friend will type a message and reply you. The process is similar here. See the code: 133 | ```python 134 | aFriend= Friend('Corgi', 'http://0.0.0.0:5000') # this is: you're finding friend in your head. 135 | # You can call him with a cute name like 'Puppy','Teddy' or 'Corgi'. 136 | # But you must always remember his real-name is 'http://0.0.0.0:5000' to know actually who he is 137 | 138 | message = aFriend.send('/helloworld','Mr. Close friend') # then you can send him a message 139 | ``` 140 | 141 | `/helloworld` is the rule/topic you say/ask to a friend or the route in http. It need to start with `/`. The rule must match with the rule of `Typing` to be replied. `Mr. Close friend` is what you are talking about, which can be string, integer, float, list, dict or class. For example: `aFriend.send('/topic',variable1, variable2, keyword1='secret key')` 142 | 143 | In other side, your friend or a microservice or a listener has the following process: 144 | ```python 145 | @Micro.typing('/helloworld') # this is the rule/topic he knows. If he don't know, he cannot reply 146 | @Micro.reply # he is replying 147 | def helloworld(name): # this is the process in side his head 148 | return 'Welcome %s' % (name) # the answer 149 | ``` 150 | 151 | `@Micro.typing` - The rule/topic must exactly match with the topic was sent and should startwith "/". The `@Micro.reply` must come before def. Then, Microservice handles the remain. Next chapter is about returning data 152 | 153 | ### 4. Send and reply string, integer, float 154 | 155 | In the sender side, you can send data type as the code below: 156 | ```python 157 | print( 158 | """############################## 159 | Test return string 160 | """) 161 | aFriend= Friend('app1', 'http://localhost:5000') 162 | print('Test: return a simple string') 163 | x = aFriend.send('/str', 'A variable value', key='A keyword variable value') 164 | print('x=', x, type(x)) 165 | print('==========================') 166 | print('Test: return multiple string') 167 | x, y, z = aFriend.send('/str2', 'A variable value', 168 | key='A keyword variable value') 169 | print('x=' ,x, type(x)) 170 | print('y=', y, type(y)) 171 | print('z=', z, type(z)) 172 | 173 | print( 174 | """############################## 175 | Test return a int, float 176 | """) 177 | aFriend= Friend('app1', 'http://localhost:5000') 178 | print('Test: return a simple Value') 179 | x = aFriend.send('/int', 2018, key=312) 180 | print('x=', x, type(x)) 181 | print('==========================') 182 | print('Test: return a simple Value') 183 | x = aFriend.send('/float', 2.018, key=3.12) 184 | print('x=', x, type(x)) 185 | print('==========================') 186 | print('Test: return multiple Value') 187 | x, y, z = aFriend.send('/int3', 3.1427, 188 | key=1000000000) 189 | print('x=', x, type(x)) 190 | print('y=', y, type(y)) 191 | print('z=', z, type(z)) 192 | ``` 193 | In the listener, you can reply/return data type as string, integer, float as below: 194 | ```python 195 | # run a normal function in python 196 | print('one cat here') 197 | 198 | # return string 199 | @Micro.typing('/str') 200 | @Micro.reply 201 | def string1(a,key): 202 | return a+'-'+key 203 | 204 | # return multiple string 205 | @Micro.typing('/str2') 206 | @Micro.reply 207 | def string2(a, key): 208 | return a, key, a+'-'+key 209 | 210 | # return Integer and float 211 | @Micro.typing('/int') 212 | @Micro.reply 213 | def int1(a, key): 214 | return a+key 215 | 216 | @Micro.typing('/float') 217 | @Micro.reply 218 | def float2(a, key): 219 | return a+key 220 | 221 | 222 | @Micro.typing('/int3') 223 | @Micro.reply 224 | def int3(a, key): 225 | return a+key, key*key, a*a 226 | ``` 227 | After that, first run listener then run sender. We have results (see example01): 228 | ``` 229 | Test: return a simple string 230 | x= A variable value-A keyword variable value 231 | ========================== 232 | Test: return multiple string 233 | x= A variable value 234 | y= A keyword variable value 235 | z= A variable value-A keyword variable value 236 | 'testStr' 23.17 ms 237 | Test: return a simple Value 238 | x= 2330 239 | ========================== 240 | Test: return a simple Value 241 | x= 5.138 242 | ========================== 243 | Test: return multiple Value 244 | x= 1000000003.1427 245 | y= 1000000000000000000 246 | z= 9.87656329 247 | ``` 248 | Note: print('one cat here') print in the screen of listener. You can run any other python function, python code as normal in listener. 249 | 250 | ### 5. Send and reply list, dict 251 | 252 | In the sender side, you can send data type as the code below: 253 | ```python 254 | print( 255 | """############################## 256 | Test return a list, dict 257 | """) 258 | aFriend= Friend('app1', 'http://localhost:5000') 259 | print('Test: return a simple Value') 260 | x = aFriend.send('/list', [12,34,45], key=['abc','zyz']) 261 | print('x=', x, type(x)) 262 | print('==========================') 263 | print('Test: return a simple Value') 264 | x = aFriend.send('/dict', {'keyword':['anything']}, key={'int':20,'str':'adfafsa','float':0.2323}) 265 | print('x=', x, type(x)) 266 | print('==========================') 267 | print('Test: return multiple Value') 268 | x, y, z = aFriend.send('/list3', {'keyword': ['anything']}, 269 | key=['abc', 'zyz']) 270 | print('x=', x, type(x)) 271 | print('y=', y, type(y)) 272 | print('z=', z, type(z)) 273 | ``` 274 | In the listener, you can reply/return data type as string, integer, float as below: 275 | ```python 276 | # test return list and dict 277 | @Micro.typing('/list') 278 | @Micro.reply 279 | def list1(a, key): 280 | a.extend(key) 281 | return a 282 | 283 | 284 | @Micro.typing('/dict') 285 | @Micro.reply 286 | def dict1(a, key): 287 | key['dict'] = a 288 | return key 289 | 290 | 291 | @Micro.typing('/list3') 292 | @Micro.reply 293 | def list3(a, key): 294 | key.append('other value') 295 | c = None 296 | return a, key, c 297 | ``` 298 | After that, first run listener then run sender. We have results (for full example see tests/example01): 299 | ``` 300 | Test: return a simple Value 301 | x= [12, 34, 45, 'abc', 'zyz'] 302 | ========================== 303 | Test: return a simple Value 304 | x= {'dict': {'keyword': ['anything']}, 'float': 0.2323, 'int': 20, 'str': 'adfafsa'} 305 | ========================== 306 | Test: return multiple Value 307 | x= {'keyword': ['anything']} 308 | y= ['abc', 'zyz', 'other value'] 309 | z= None 310 | 'testListDict' 22.19 ms 311 | ``` 312 | 313 | ### 6. Send and reply void, Nonetype, class attributes and use of token 314 | In the sender side, you can send data type as the code below: 315 | 316 | ```python 317 | print( 318 | """############################## 319 | Test return NoneType, Class, use of Token 320 | """) 321 | aFriend= Friend('app1', 'http://localhost:5000') 322 | print('Test: return a simple Value') 323 | x = aFriend.send('/None', [12, 34, 45], key=['abc', 'zyz']) 324 | print('x=', x, type(x)) 325 | print('==========================') 326 | print('Test: return a simple Value with token') 327 | aFriend.setRule('/class', token='123456') 328 | x = aFriend.send('/class', {'keyword': ['anything']}, 329 | key={'int': 20, 'str': 'adfafsa', 'float': 0.2323}) 330 | print('x=', x, type(x)) 331 | print('==========================') 332 | print('Test: return multiple Value') 333 | aFriend.setRule('/class2', token='123456') 334 | x,y,z = aFriend.send('/class2', {'keyword': ['anything']}, 335 | key={'int': 20, 'str': 'adfafsa', 'float': 0.2323}) 336 | print('x=', x, type(x)) 337 | print('y=', y, type(y)) 338 | print('z=', z, type(z)) 339 | 340 | # Test send class and list of class object 341 | print('\n Test: send class and list of class object') 342 | aFriend.setRule('/class3', token='123456') 343 | t1 = testservice('value1') 344 | t2 = testservice('value2') 345 | x, y, z = aFriend.send('/class3', [t1,t2], 346 | key={'t1': t1, 't2': t2, 'list': [t1, t2]}) 347 | print('x=', x, type(x)) 348 | print('y=', y, type(y)) 349 | print('z=', z, type(z)) 350 | ``` 351 | 352 | In the listener, you can reply/return data type as string, integer, float as below: 353 | 354 | ```python 355 | # return None, class Object 356 | @Micro.typing('/None') 357 | @Micro.reply 358 | def TestNoneValue(a, key): 359 | key.append('Do something in the server') 360 | 361 | class testservice(object): 362 | name = 'test' 363 | Purpose = 'For test only' 364 | empty = None 365 | def __init__(self, value): 366 | self.value = value 367 | 368 | def onemethod(self): 369 | pass 370 | 371 | 372 | @Micro.typing('/class',token='123456') 373 | @Micro.reply 374 | def TestClass(a, key): 375 | t = testservice(a) 376 | return t 377 | 378 | 379 | @Micro.typing('/class2', token='123456') 380 | @Micro.reply 381 | def TestClass2(a, key): 382 | t = testservice(key) 383 | return t, a, None 384 | 385 | @Micro.typing('/class3', token='123456') 386 | @Micro.reply 387 | def TestClass3(a, key): 388 | x = testservice(key) 389 | y = testservice(a) 390 | z = [y,x] 391 | return x, y, z 392 | ``` 393 | After that, first run listener then run sender. We have results (for full example see tests/example01): 394 | 395 | ``` 396 | ############################## 397 | Test return NoneType, Class, use of Token 398 | 399 | Test: return a simple Value 400 | x= None 401 | ========================== 402 | Test: return a simple Value with token 403 | x= {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': {'keyword': ['anything']}} 404 | ========================== 405 | Test: return multiple Value 406 | x= {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': {'float': 0.2323, 'int': 20, 'str': 'adfafsa'}} 407 | y= {'keyword': ['anything']} 408 | z= None 409 | 410 | Test: send class and list of class object 411 | x= {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': {'list': [{'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value1'}, {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value2'}], 't1': {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value1'}, 't2': {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value2'}}} 412 | y= {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': [{'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value1'}, {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value2'}]} 413 | z= [{'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': [{'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value1'}, {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value2'}]}, {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': {'list': [{'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value1'}, {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value2'}], 't1': {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value1'}, 't2': {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value2'}}}] 414 | 'testClassType' 19.20 ms 415 | ``` 416 | 417 | New feature from 0.2.4, now you can send and receive json similar to dict. It helps more readable response 418 | 419 | ```python 420 | # in client side 421 | print('=================Response json===============') 422 | x = aFriend.json('/json', a=12,b='This is a text',c={'dict':'a dict'}) 423 | print('Synchonous POST:', x) 424 | y = aFriend.json('/json1', method='GET' , a={'dict': 'a only dict'}) 425 | print('Asynchonous GET:', y) 426 | z = aFriend.json('/json1', a={'dict': 'a only dict'}) 427 | print('Asynchonous POST:', z) 428 | ``` 429 | In the server side we have: 430 | 431 | ```python 432 | # in server side 433 | @Micro.typing('/json') 434 | @Micro.json 435 | @timeit 436 | def TestReceiveJson(a=1, b='string',c=None): 437 | return {'1':a,'2':b,'3':c} 438 | 439 | 440 | # for async request (only apply to sanic) 441 | @Micro.route('/json1', methods=['GET','POST']) 442 | @Micro.async_json 443 | async def TestReceiveJson2(a=None): 444 | return a 445 | ``` 446 | 447 | You can response with get, post, put, delete,... as the method above. The result: 448 | 449 | ``` 450 | =================Response json=============== 451 | Synchonous POST: {'1': 12, '2': 'This is a text', '3': {'dict': 'a dict'}} 452 | Asynchonous GET: {'dict': 'a only dict'} 453 | Asynchonous POST: {'dict': 'a only dict'} 454 | ``` 455 | ### 7. From 0.2.7, we support for websocket connection: 456 | In the sender side, we can send data type as the code below: 457 | 458 | ```python 459 | from microservices_connector.minisocket import SocketServer 460 | sk = SocketServer(__name__) 461 | @sk.router('/hello') 462 | def test(message): 463 | print(message) 464 | return 'ok:'+message 465 | 466 | def main(): 467 | sk.run() 468 | # you can put a flask server here 469 | # Socket Server run in a seperate threads, not affect flask server 470 | 471 | if __name__ == '__main__': 472 | main() 473 | ``` 474 | The other option is run minisocket in a different thread, that will alow flask server run seperately. 475 | 476 | ```python 477 | sk = SocketServer(__name__) 478 | app = Microservice('Flask_app').app 479 | 480 | @app.route('/') 481 | def helloworld(): 482 | time.sleep(2) 483 | return 'Sleep 2s before response' 484 | 485 | 486 | @sk.router('/hello') 487 | def test(message): 488 | print(message) 489 | return 'ok:'+message 490 | 491 | def socket_runner(): 492 | sk.run() 493 | 494 | def main(): 495 | socket_runner() 496 | print('start web framework') 497 | app.run() 498 | 499 | 500 | if __name__ == '__main__': 501 | main() 502 | 503 | ``` 504 | 505 | In the client side, we create a socket connection using websocket framework: 506 | 507 | ```python 508 | import asyncio 509 | import websockets 510 | 511 | 512 | async def hello(): 513 | async with websockets.connect('ws://localhost:8765/hello') as websocket: 514 | name = input("What's your name? ") 515 | await websocket.send(name) 516 | print(f"> {name}") 517 | greeting = await websocket.recv() 518 | print(f"< {greeting}") 519 | while greeting != 'close': 520 | name = input("What's your name? ") 521 | await websocket.send(name) 522 | greeting = await websocket.recv() 523 | print(f"< {greeting}") 524 | 525 | asyncio.get_event_loop().run_until_complete(hello()) 526 | ``` 527 | ### 8. From 0.2.7, we support concurrency process with sequences throught DistributedThreads and DistributedProcess : 528 | 529 | This give a solution for the problems of concurrency process that many workers simutanously impact/input to a single database row/table/position or the problem of arbitrary order of data. 530 | First, if you don't need process a data or queue with order or your data do not input to the same database row/table, you should use celery or other framework to scale your project. For that instance, this framework will provide no more efficient than celery which highly use by now. 531 | Second, framework need a key for each row/item in data for knowning which want to be ordered. The guide as example below. You need a output_queue for listening the result and continue other function. 532 | 533 | ```python 534 | from microservices_connector.spawn import DistributedThreads, DistributedProcess, Worker 535 | import random 536 | import time 537 | import queue 538 | import threading 539 | import multiprocessing 540 | 541 | def wait_on_b(b): 542 | time.sleep(random.random()) 543 | # b will never complete because it is waiting on a. 544 | print('working on b=%s' % b) 545 | return 'working on b=%s' % b 546 | # return 5 547 | 548 | 549 | def wait_on_a(a): 550 | time.sleep(1) 551 | # a will never complete because it is waiting on b. 552 | return 'working on a=%s' % a 553 | # return 6 554 | 555 | # example data 556 | poll = [ 557 | {'id': 1, 'x': 'Nguyen'}, 558 | {'id': 1, 'x': 'Minh'}, 559 | {'id': 1, 'x': 'Tuan'}, 560 | {'id': 2, 'x': 'Vu'}, 561 | {'id': 3, 'x': 'Ai do khac'}, 562 | {'id': 2, 'x': 'Kim'}, 563 | {'id': 2, 'x': 'Oanh'}, 564 | {'id': 4, 'x': '1'}, 565 | {'id': 4, 'x': '2'}, 566 | {'id': 4, 'x': '3'}, 567 | {'id': 4, 'x': '4'}, 568 | {'id': 4, 'x': '5'}, 569 | {'id': 4, 'x': '6'}, 570 | {'id': 4, 'x': '7'}, 571 | {'id': 4, 'x': '8'}, 572 | {'id': 5, 'x': '101'}, 573 | {'id': 5, 'x': '102'}, 574 | {'id': 5, 'x': '103'}, 575 | {'id': 5, 'x': '104'}, 576 | {'id': 5, 'x': '105'}, 577 | {'id': 6, 'x': 'Test watching'}, 578 | {'id': 6, 'x': 'Test watching'}, 579 | {'id': 7, 'x': 'Test watching'}, 580 | {'id': 8, 'x': 'Test watching'}, 581 | {'id': 9, 'x': 'Test watching'}, 582 | {'id': 10, 'x': 'Test watching'}, 583 | {'id': 11, 'x': 'Test watching'}, 584 | {'id': 12, 'x': 'Test watching'}, 585 | {'id': 13, 'x': 'Test watching'}, 586 | {'id': 14, 'x': 'Test watching'}, 587 | {'id': 15, 'x': 'Test watching'}, 588 | {'id': 16, 'x': 'Test watching'}, 589 | {'id': 17, 'x': 'Test watching'}, 590 | {'id': 18, 'x': 'Test watching'}, 591 | {'id': 19, 'x': 'Test watching'}, 592 | {'id': 20, 'x': 'Test watching'}, 593 | {'id': 21, 'x': 'Test watching'}, 594 | {'id': 22, 'x': 'Test watching'}, 595 | ] 596 | def main(): 597 | start = time.time() 598 | 599 | thread_out_queue = queue.Queue() 600 | pool = DistributedThreads( 601 | max_workers=4, max_watching=100, out_queue=thread_out_queue) 602 | for item in poll: 603 | pool.submit_id(item['id'], wait_on_a, item) 604 | t = Worker(thread_out_queue, print) 605 | t.daemon = True 606 | t.start() 607 | pool.shutdown() 608 | print('Finish after: ', time.time()-start, 'seconds') 609 | 610 | print("========= End of threads ==============") 611 | 612 | process_out_queue = multiprocessing.Queue() 613 | pool2 = DistributedProcess( 614 | max_workers=4, max_watching=100, out_queue=process_out_queue) 615 | for item in poll: 616 | pool2.submit_id(item['id'], wait_on_b, item) 617 | pool2.shutdown() 618 | 619 | print('Finish after: ', time.time()-start, 'seconds') 620 | 621 | 622 | if __name__ == '__main__': 623 | main() 624 | ``` 625 | 626 | As the result, all name and number will same id will print in the exact order. Max watching should not be lesser than 100. If you put key/id to None or use submit instead of submit_id, it will do no order but faster. 627 | 628 | A Detail User Guide will comming soon... 629 | ## Pros vs Cons and question 630 | From my opinion only, Microservice connector has the following Pros and Cons to improve 631 | ### Pros: 632 | * Ease of use, Ease of development, you don't need to touch on http connection 633 | * Can build decentralize or Distributed system with Infinite scalability 634 | * Send and receive data with many types as string, int, float, list, dict. 635 | * Connect all around the world with internet 636 | 637 | ### Cons: 638 | * Do not support send/receive tuple and set type (because I don't like them). 639 | * Do not support send/receive a whole class, return of decorator and server-side computer 640 | * Is not really fast log-broker server as RabbitMQ, ZeroMQ, kafka: *yes, oneService cannot send 10 million message per second like them, but it has other advance.* 641 | * Do not support Database, user/role management, system manager: *not yet, we are trying to write new feature include them. We welcome any contributor support us.* 642 | 643 | ### Question: 644 | * Why not a load balancer ? 645 | 646 | > *It is out of range. Load balancer cover the other layer. Other package can handle it better. But we consider to add a custom function for it.* 647 | 648 | * What about support more options, async/await ? 649 | 650 | > *We are trying to connect by Sanic and Japronto soon* 651 | 652 | * What about data integrity, blockchain, token ? 653 | 654 | > *We are trying to add them, but cannot be soon* 655 | 656 | ## Authors 657 | 658 | * **Tuan Nguyen Minh** - *Financer and Developer* - email: ntuan221@gmail.com 659 | 660 | Thank for the frameworks and their authors: 661 | * flask - micro webframework 662 | * Sanic - Gotta go fast 663 | * requests 664 | 665 | Favourite idioms: 666 | * Don't repeat your self 667 | * Think like human, make for human 668 | * Simple is stronger 669 | ## License: BDS license -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Microservices Connector 2 | ======================= 3 | 4 | Microservices Connector is a Inter-Service communication framework, 5 | support for microservice architecture and distributed system. 6 | 7 | Getting Started 8 | --------------- 9 | 10 | **Microservices** is a way of breaking large software projects into 11 | loosely coupled modules, which communicate with each other through 12 | simple APIs. The advantages of microservices are improves fault 13 | isolation, scalability, Ease of development. It convinced some big 14 | enterprise players – like Amazon, Netflix, and eBay – to begin their 15 | transitions. 16 | 17 | *Microservices Connector* is a Inter-Service communication framework 18 | written in python, support for microservice architecture and distributed 19 | system. Its features contain: \* Transfering data as returning a 20 | function results or quering data. \* Support transfer multiple values 21 | with many type as string, int, float, list, dict, class attribute \* Do 22 | not require knowledge about web/http connection or touch to them \* 23 | Distributed system 24 | 25 | .. figure:: images/Distributed-system.jpeg 26 | :alt: Illustration of network system. Source:medium.com 27 | 28 | alt text 29 | 30 | Illustration of network system. Source:medium.com 31 | 32 | As illustration, distributed systems are very stable and Infinite 33 | scalability. But distributed systems are the most difficult to maintain. 34 | 35 | *Microservices Connector* supports communication by the following 36 | framework: \* flask - A micro web framework \* Sanic - Gotta go fast (A 37 | async web framework) ### Quickstart: 38 | 39 | Microservices Connector is available on pip. You can install via pip 40 | (require python>=3.5): 41 | 42 | :: 43 | 44 | pip install microservices_connector 45 | 46 | Start project by a minimum example: 47 | 48 | - Step 1: Create 1 file name app1.py, write and save with the code 49 | below 50 | 51 | .. code:: python 52 | 53 | from microservices_connector.Interservices import Microservice 54 | 55 | Micro = Microservice(__name__) 56 | 57 | @Micro.typing('/helloworld') 58 | @Micro.reply 59 | def helloworld(name): 60 | return 'Welcome %s' % (name) 61 | 62 | if __name__ == '__main__': 63 | Micro.run() 64 | 65 | - Step 2: Create 1 file name app2.py, write and save with the code 66 | below 67 | 68 | .. code:: python 69 | 70 | from microservices_connector.Interservices import Friend 71 | 72 | aFriend= Friend('app1', 'http://0.0.0.0:5000') 73 | message = aFriend.send('/helloworld','Mr. Developer') 74 | print(message) 75 | 76 | - Step 3: Run app1.py and app2.py together. 77 | 78 | Open a terminal in the same folder with them and run: 79 | ``python app1.py``. 80 | 81 | Open another terminal in the same folder and run: ``python app2.py`` 82 | 83 | You get the result: ``Welcome Mr. Developer`` in terminal of app2.py. 84 | This is no difference if you do 85 | ``from .app1 import helloword; message = helloworld('Mr. Developer'); print(message)``. 86 | Note: To stop app1, open its terminal and Ctrl + C. The example can be 87 | found in ``test/example00`` folder 88 | 89 | **Explanation**: App1 and app2 are small example of microservice system. 90 | In the example, **M** in app1 is listener, a http server while **F** in 91 | app2 is sender. Listener and sender are isolated design, they can work 92 | seperately or together in an app/service. A microservice can be a 93 | listener of many other microservices or sender of many other 94 | microservices or both of them. A standard microservice mention in this 95 | project contain both listener and sender. 96 | 97 | Tutorial 98 | -------- 99 | 100 | 1. Start your first app 101 | ~~~~~~~~~~~~~~~~~~~~~~~ 102 | 103 | In the tutorial, we assume you have this code in the top of your file. 104 | 105 | .. code:: python 106 | 107 | from microservices_connector.Interservices import Microservice 108 | 109 | Micro = Microservice(__name__) 110 | 111 | # for Sanic framework if you want to use sanic instead of flask 112 | from microservices_connector.Interservices import SanicApp as Microservice 113 | 114 | Micro = Microservice(__name__) 115 | 116 | Now, look closer at ``Microservice(__name__)``, it actually look like 117 | this: 118 | 119 | :: 120 | 121 | Microservice(name, port: int=5000, host: str='0.0.0.0', debug=None, token: dict = {}, secretKey=None) 122 | 123 | Arguments: \* name {str} -- Require a name for your app, recommend put 124 | ``__name__`` for it \* port {int} -- Choose from 3000 to 9000, default 125 | to 5000 \* host {str} -- Host ip, Default 0.0.0.0 for localhost \* debug 126 | {boolean} -- True for development, False/None for production \* token 127 | {dict} -- A dict contain all rule and its token. It can be set later 128 | 129 | The class Microservice is used to create a listener/a microservice. In a 130 | file/app, you should only have one listener. About parameters, If you 131 | aren't familiar with http server, you only need remember: \* One app 132 | should have only one listener \* Should use ``__name__`` for name and 133 | name need to be unique \* If you run multiple listener, use only one 134 | unique port for each listener. for example: 135 | 136 | .. code:: python 137 | 138 | M1 = Microservice(__name__, port=5010) # in file app1 139 | M2 = Microservice(__name__, port=5020) # in file app2 140 | 141 | Note: You should be carefully if there are other web applications 142 | running in your port/server. 143 | 144 | 2. Way of running an app 145 | ~~~~~~~~~~~~~~~~~~~~~~~~ 146 | 147 | A sender is a python def, so you can put it anywhere in your app. 148 | Listener is a http server so it's a bit difference from other. 149 | 150 | Option 1: Use if/main in the end of startup file (file that you start 151 | your project by ``python ``). Add the following code the end: 152 | 153 | .. code:: python 154 | 155 | # Micro is your Microservice Object 156 | if __name__ == '__main__': 157 | Micro.run() 158 | 159 | Option 2: Create a file name run.py and run your app from this file. For 160 | example, we create a run.py in the same folder of app1.py in the first 161 | example. It will be like this: 162 | 163 | .. code:: python 164 | 165 | from app1 import Micro 166 | 167 | if __name__ == '__main__': 168 | Micro.run(port=5000, host='0.0.0.0', debug=True) 169 | 170 | Option 2 is more appreciated. It avoid the app looping from them self, 171 | so get away of stunning your app. If you have 2 app in a 172 | server/computer, you should create 2 run file for it. Don't for get 173 | ``Ctrl + C`` to stop your app. 174 | 175 | Note: *We assume you already use one of the options above for your 176 | code*. This tutorial focuses on communication between 177 | 'service-to-service' as def function, not http connect. 178 | 179 | 3. Send, Typing and reply 180 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 181 | 182 | Think like a human, if you want to communicate with some friend in 183 | facebook, you will open *messenger*, find your friend and send a message 184 | to them. It's a way of sending message to each other. Then, your friend 185 | will type a message and reply you. The process is similar here. See the 186 | code: 187 | 188 | .. code:: python 189 | 190 | aFriend= Friend('Corgi', 'http://0.0.0.0:5000') # this is: you're finding friend in your head. 191 | # You can call him with a cute name like 'Puppy','Teddy' or 'Corgi'. 192 | # But you must always remember his real-name is 'http://0.0.0.0:5000' to know actually who he is 193 | 194 | message = aFriend.send('/helloworld','Mr. Close friend') # then you can send him a message 195 | 196 | ``/helloworld`` is the rule/topic you say/ask to a friend or the route 197 | in http. It need to start with ``/``. The rule must match with the rule 198 | of ``Typing`` to be replied. ``Mr. Close friend`` is what you are 199 | talking about, which can be string, integer, float, list, dict or class. 200 | For example: 201 | ``aFriend.send('/topic',variable1, variable2, keyword1='secret key')`` 202 | 203 | In other side, your friend or a microservice or a listener has the 204 | following process: 205 | 206 | .. code:: python 207 | 208 | @Micro.typing('/helloworld') # this is the rule/topic he knows. If he don't know, he cannot reply 209 | @Micro.reply # he is replying 210 | def helloworld(name): # this is the process in side his head 211 | return 'Welcome %s' % (name) # the answer 212 | 213 | ``@Micro.typing`` - The rule/topic must exactly match with the topic was 214 | sent and should startwith "/". The ``@Micro.reply`` must come before 215 | def. Then, Microservice handles the remain. Next chapter is about 216 | returning data 217 | 218 | 4. Send and reply string, integer, float 219 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 220 | 221 | In the sender side, you can send data type as the code below: 222 | 223 | .. code:: python 224 | 225 | print( 226 | """############################## 227 | Test return string 228 | """) 229 | aFriend= Friend('app1', 'http://localhost:5000') 230 | print('Test: return a simple string') 231 | x = aFriend.send('/str', 'A variable value', key='A keyword variable value') 232 | print('x=', x, type(x)) 233 | print('==========================') 234 | print('Test: return multiple string') 235 | x, y, z = aFriend.send('/str2', 'A variable value', 236 | key='A keyword variable value') 237 | print('x=' ,x, type(x)) 238 | print('y=', y, type(y)) 239 | print('z=', z, type(z)) 240 | 241 | print( 242 | """############################## 243 | Test return a int, float 244 | """) 245 | aFriend= Friend('app1', 'http://localhost:5000') 246 | print('Test: return a simple Value') 247 | x = aFriend.send('/int', 2018, key=312) 248 | print('x=', x, type(x)) 249 | print('==========================') 250 | print('Test: return a simple Value') 251 | x = aFriend.send('/float', 2.018, key=3.12) 252 | print('x=', x, type(x)) 253 | print('==========================') 254 | print('Test: return multiple Value') 255 | x, y, z = aFriend.send('/int3', 3.1427, 256 | key=1000000000) 257 | print('x=', x, type(x)) 258 | print('y=', y, type(y)) 259 | print('z=', z, type(z)) 260 | 261 | In the listener, you can reply/return data type as string, integer, 262 | float as below: 263 | 264 | .. code:: python 265 | 266 | # run a normal function in python 267 | print('one cat here') 268 | 269 | # return string 270 | @Micro.typing('/str') 271 | @Micro.reply 272 | def string1(a,key): 273 | return a+'-'+key 274 | 275 | # return multiple string 276 | @Micro.typing('/str2') 277 | @Micro.reply 278 | def string2(a, key): 279 | return a, key, a+'-'+key 280 | 281 | # return Integer and float 282 | @Micro.typing('/int') 283 | @Micro.reply 284 | def int1(a, key): 285 | return a+key 286 | 287 | @Micro.typing('/float') 288 | @Micro.reply 289 | def float2(a, key): 290 | return a+key 291 | 292 | 293 | @Micro.typing('/int3') 294 | @Micro.reply 295 | def int3(a, key): 296 | return a+key, key*key, a*a 297 | 298 | After that, first run listener then run sender. We have results (see 299 | example01): 300 | 301 | :: 302 | 303 | Test: return a simple string 304 | x= A variable value-A keyword variable value 305 | ========================== 306 | Test: return multiple string 307 | x= A variable value 308 | y= A keyword variable value 309 | z= A variable value-A keyword variable value 310 | 'testStr' 23.17 ms 311 | Test: return a simple Value 312 | x= 2330 313 | ========================== 314 | Test: return a simple Value 315 | x= 5.138 316 | ========================== 317 | Test: return multiple Value 318 | x= 1000000003.1427 319 | y= 1000000000000000000 320 | z= 9.87656329 321 | 322 | Note: print('one cat here') print in the screen of listener. You can run 323 | any other python function, python code as normal in listener. 324 | 325 | 5. Send and reply list, dict 326 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 327 | 328 | In the sender side, you can send data type as the code below: 329 | 330 | .. code:: python 331 | 332 | print( 333 | """############################## 334 | Test return a list, dict 335 | """) 336 | aFriend= Friend('app1', 'http://localhost:5000') 337 | print('Test: return a simple Value') 338 | x = aFriend.send('/list', [12,34,45], key=['abc','zyz']) 339 | print('x=', x, type(x)) 340 | print('==========================') 341 | print('Test: return a simple Value') 342 | x = aFriend.send('/dict', {'keyword':['anything']}, key={'int':20,'str':'adfafsa','float':0.2323}) 343 | print('x=', x, type(x)) 344 | print('==========================') 345 | print('Test: return multiple Value') 346 | x, y, z = aFriend.send('/list3', {'keyword': ['anything']}, 347 | key=['abc', 'zyz']) 348 | print('x=', x, type(x)) 349 | print('y=', y, type(y)) 350 | print('z=', z, type(z)) 351 | 352 | In the listener, you can reply/return data type as string, integer, 353 | float as below: 354 | 355 | .. code:: python 356 | 357 | # test return list and dict 358 | @Micro.typing('/list') 359 | @Micro.reply 360 | def list1(a, key): 361 | a.extend(key) 362 | return a 363 | 364 | 365 | @Micro.typing('/dict') 366 | @Micro.reply 367 | def dict1(a, key): 368 | key['dict'] = a 369 | return key 370 | 371 | 372 | @Micro.typing('/list3') 373 | @Micro.reply 374 | def list3(a, key): 375 | key.append('other value') 376 | c = None 377 | return a, key, c 378 | 379 | After that, first run listener then run sender. We have results (for 380 | full example see tests/example01): 381 | 382 | :: 383 | 384 | Test: return a simple Value 385 | x= [12, 34, 45, 'abc', 'zyz'] 386 | ========================== 387 | Test: return a simple Value 388 | x= {'dict': {'keyword': ['anything']}, 'float': 0.2323, 'int': 20, 'str': 'adfafsa'} 389 | ========================== 390 | Test: return multiple Value 391 | x= {'keyword': ['anything']} 392 | y= ['abc', 'zyz', 'other value'] 393 | z= None 394 | 'testListDict' 22.19 ms 395 | 396 | 6. Send and reply void, Nonetype, class attributes and use of token 397 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 398 | 399 | In the sender side, you can send data type as the code below: 400 | 401 | .. code:: python 402 | 403 | print( 404 | """############################## 405 | Test return NoneType, Class, use of Token 406 | """) 407 | aFriend= Friend('app1', 'http://localhost:5000') 408 | print('Test: return a simple Value') 409 | x = aFriend.send('/None', [12, 34, 45], key=['abc', 'zyz']) 410 | print('x=', x, type(x)) 411 | print('==========================') 412 | print('Test: return a simple Value with token') 413 | aFriend.setRule('/class', token='123456') 414 | x = aFriend.send('/class', {'keyword': ['anything']}, 415 | key={'int': 20, 'str': 'adfafsa', 'float': 0.2323}) 416 | print('x=', x, type(x)) 417 | print('==========================') 418 | print('Test: return multiple Value') 419 | aFriend.setRule('/class2', token='123456') 420 | x,y,z = aFriend.send('/class2', {'keyword': ['anything']}, 421 | key={'int': 20, 'str': 'adfafsa', 'float': 0.2323}) 422 | print('x=', x, type(x)) 423 | print('y=', y, type(y)) 424 | print('z=', z, type(z)) 425 | 426 | # Test send class and list of class object 427 | print('\n Test: send class and list of class object') 428 | aFriend.setRule('/class3', token='123456') 429 | t1 = testservice('value1') 430 | t2 = testservice('value2') 431 | x, y, z = aFriend.send('/class3', [t1,t2], 432 | key={'t1': t1, 't2': t2, 'list': [t1, t2]}) 433 | print('x=', x, type(x)) 434 | print('y=', y, type(y)) 435 | print('z=', z, type(z)) 436 | 437 | In the listener, you can reply/return data type as string, integer, 438 | float as below: 439 | 440 | .. code:: python 441 | 442 | # return None, class Object 443 | @Micro.typing('/None') 444 | @Micro.reply 445 | def TestNoneValue(a, key): 446 | key.append('Do something in the server') 447 | 448 | class testservice(object): 449 | name = 'test' 450 | Purpose = 'For test only' 451 | empty = None 452 | def __init__(self, value): 453 | self.value = value 454 | 455 | def onemethod(self): 456 | pass 457 | 458 | 459 | @Micro.typing('/class',token='123456') 460 | @Micro.reply 461 | def TestClass(a, key): 462 | t = testservice(a) 463 | return t 464 | 465 | 466 | @Micro.typing('/class2', token='123456') 467 | @Micro.reply 468 | def TestClass2(a, key): 469 | t = testservice(key) 470 | return t, a, None 471 | 472 | @Micro.typing('/class3', token='123456') 473 | @Micro.reply 474 | def TestClass3(a, key): 475 | x = testservice(key) 476 | y = testservice(a) 477 | z = [y,x] 478 | return x, y, z 479 | 480 | After that, first run listener then run sender. We have results (for 481 | full example see tests/example01): 482 | 483 | :: 484 | 485 | ############################## 486 | Test return NoneType, Class, use of Token 487 | 488 | Test: return a simple Value 489 | x= None 490 | ========================== 491 | Test: return a simple Value with token 492 | x= {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': {'keyword': ['anything']}} 493 | ========================== 494 | Test: return multiple Value 495 | x= {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': {'float': 0.2323, 'int': 20, 'str': 'adfafsa'}} 496 | y= {'keyword': ['anything']} 497 | z= None 498 | 499 | Test: send class and list of class object 500 | x= {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': {'list': [{'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value1'}, {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value2'}], 't1': {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value1'}, 't2': {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value2'}}} 501 | y= {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': [{'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value1'}, {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value2'}]} 502 | z= [{'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': [{'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value1'}, {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value2'}]}, {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': {'list': [{'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value1'}, {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value2'}], 't1': {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value1'}, 't2': {'Purpose': 'For test only', 'empty': None, 'name': 'test', 'value': 'value2'}}}] 503 | 'testClassType' 19.20 ms 504 | 505 | New feature from 0.2.4, now you can send and receive json similar to 506 | dict. It helps more readable response 507 | 508 | .. code:: python 509 | 510 | # in client side 511 | print('=================Response json===============') 512 | x = aFriend.json('/json', a=12,b='This is a text',c={'dict':'a dict'}) 513 | print('Synchonous POST:', x) 514 | y = aFriend.json('/json1', method='GET' , a={'dict': 'a only dict'}) 515 | print('Asynchonous GET:', y) 516 | z = aFriend.json('/json1', a={'dict': 'a only dict'}) 517 | print('Asynchonous POST:', z) 518 | 519 | In the server side we have: 520 | 521 | .. code:: python 522 | 523 | # in server side 524 | @Micro.typing('/json') 525 | @Micro.json 526 | @timeit 527 | def TestReceiveJson(a=1, b='string',c=None): 528 | return {'1':a,'2':b,'3':c} 529 | 530 | 531 | # for async request (only apply to sanic) 532 | @Micro.route('/json1', methods=['GET','POST']) 533 | @Micro.async_json 534 | async def TestReceiveJson2(a=None): 535 | return a 536 | 537 | You can response with get, post, put, delete,... as the method above. 538 | The result: 539 | 540 | :: 541 | 542 | =================Response json=============== 543 | Synchonous POST: {'1': 12, '2': 'This is a text', '3': {'dict': 'a dict'}} 544 | Asynchonous GET: {'dict': 'a only dict'} 545 | Asynchonous POST: {'dict': 'a only dict'} 546 | 547 | 7. From 0.2.7, we support for websocket connection: 548 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 549 | 550 | In the sender side, we can send data type as the code below: 551 | 552 | .. code:: python 553 | 554 | from microservices_connector.minisocket import SocketServer 555 | sk = SocketServer(__name__) 556 | @sk.router('/hello') 557 | def test(message): 558 | print(message) 559 | return 'ok:'+message 560 | 561 | def main(): 562 | sk.run() 563 | # you can put a flask server here 564 | # Socket Server run in a seperate threads, not affect flask server 565 | 566 | if __name__ == '__main__': 567 | main() 568 | 569 | The other option is run minisocket in a different thread, that will alow 570 | flask server run seperately. 571 | 572 | .. code:: python 573 | 574 | sk = SocketServer(__name__) 575 | app = Microservice('Flask_app').app 576 | 577 | @app.route('/') 578 | def helloworld(): 579 | time.sleep(2) 580 | return 'Sleep 2s before response' 581 | 582 | 583 | @sk.router('/hello') 584 | def test(message): 585 | print(message) 586 | return 'ok:'+message 587 | 588 | def socket_runner(): 589 | sk.run() 590 | 591 | def main(): 592 | socket_runner() 593 | print('start web framework') 594 | app.run() 595 | 596 | 597 | if __name__ == '__main__': 598 | main() 599 | 600 | In the client side, we create a socket connection using websocket 601 | framework: 602 | 603 | .. code:: python 604 | 605 | import asyncio 606 | import websockets 607 | 608 | 609 | async def hello(): 610 | async with websockets.connect('ws://localhost:8765/hello') as websocket: 611 | name = input("What's your name? ") 612 | await websocket.send(name) 613 | print(f"> {name}") 614 | greeting = await websocket.recv() 615 | print(f"< {greeting}") 616 | while greeting != 'close': 617 | name = input("What's your name? ") 618 | await websocket.send(name) 619 | greeting = await websocket.recv() 620 | print(f"< {greeting}") 621 | 622 | asyncio.get_event_loop().run_until_complete(hello()) 623 | 624 | 8. From 0.2.7, we support concurrency process with sequences throught DistributedThreads and DistributedProcess : 625 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 626 | 627 | This give a solution for the problems of concurrency process that many 628 | workers simutanously impact/input to a single database 629 | row/table/position or the problem of arbitrary order of data. First, if 630 | you don't need process a data or queue with order or your data do not 631 | input to the same database row/table, you should use celery or other 632 | framework to scale your project. For that instance, this framework will 633 | provide no more efficient than celery which highly use by now. Second, 634 | framework need a key for each row/item in data for knowning which want 635 | to be ordered. The guide as example below. You need a output\_queue for 636 | listening the result and continue other function. 637 | 638 | .. code:: python 639 | 640 | from microservices_connector.spawn import DistributedThreads, DistributedProcess, Worker 641 | import random 642 | import time 643 | import queue 644 | import threading 645 | import multiprocessing 646 | 647 | def wait_on_b(b): 648 | time.sleep(random.random()) 649 | # b will never complete because it is waiting on a. 650 | print('working on b=%s' % b) 651 | return 'working on b=%s' % b 652 | # return 5 653 | 654 | 655 | def wait_on_a(a): 656 | time.sleep(1) 657 | # a will never complete because it is waiting on b. 658 | return 'working on a=%s' % a 659 | # return 6 660 | 661 | # example data 662 | poll = [ 663 | {'id': 1, 'x': 'Nguyen'}, 664 | {'id': 1, 'x': 'Minh'}, 665 | {'id': 1, 'x': 'Tuan'}, 666 | {'id': 2, 'x': 'Vu'}, 667 | {'id': 3, 'x': 'Ai do khac'}, 668 | {'id': 2, 'x': 'Kim'}, 669 | {'id': 2, 'x': 'Oanh'}, 670 | {'id': 4, 'x': '1'}, 671 | {'id': 4, 'x': '2'}, 672 | {'id': 4, 'x': '3'}, 673 | {'id': 4, 'x': '4'}, 674 | {'id': 4, 'x': '5'}, 675 | {'id': 4, 'x': '6'}, 676 | {'id': 4, 'x': '7'}, 677 | {'id': 4, 'x': '8'}, 678 | {'id': 5, 'x': '101'}, 679 | {'id': 5, 'x': '102'}, 680 | {'id': 5, 'x': '103'}, 681 | {'id': 5, 'x': '104'}, 682 | {'id': 5, 'x': '105'}, 683 | {'id': 6, 'x': 'Test watching'}, 684 | {'id': 6, 'x': 'Test watching'}, 685 | {'id': 7, 'x': 'Test watching'}, 686 | {'id': 8, 'x': 'Test watching'}, 687 | {'id': 9, 'x': 'Test watching'}, 688 | {'id': 10, 'x': 'Test watching'}, 689 | {'id': 11, 'x': 'Test watching'}, 690 | {'id': 12, 'x': 'Test watching'}, 691 | {'id': 13, 'x': 'Test watching'}, 692 | {'id': 14, 'x': 'Test watching'}, 693 | {'id': 15, 'x': 'Test watching'}, 694 | {'id': 16, 'x': 'Test watching'}, 695 | {'id': 17, 'x': 'Test watching'}, 696 | {'id': 18, 'x': 'Test watching'}, 697 | {'id': 19, 'x': 'Test watching'}, 698 | {'id': 20, 'x': 'Test watching'}, 699 | {'id': 21, 'x': 'Test watching'}, 700 | {'id': 22, 'x': 'Test watching'}, 701 | ] 702 | def main(): 703 | start = time.time() 704 | 705 | thread_out_queue = queue.Queue() 706 | pool = DistributedThreads( 707 | max_workers=4, max_watching=100, out_queue=thread_out_queue) 708 | for item in poll: 709 | pool.submit_id(item['id'], wait_on_a, item) 710 | t = Worker(thread_out_queue, print) 711 | t.daemon = True 712 | t.start() 713 | pool.shutdown() 714 | print('Finish after: ', time.time()-start, 'seconds') 715 | 716 | print("========= End of threads ==============") 717 | 718 | process_out_queue = multiprocessing.Queue() 719 | pool2 = DistributedProcess( 720 | max_workers=4, max_watching=100, out_queue=process_out_queue) 721 | for item in poll: 722 | pool2.submit_id(item['id'], wait_on_b, item) 723 | pool2.shutdown() 724 | 725 | print('Finish after: ', time.time()-start, 'seconds') 726 | 727 | 728 | if __name__ == '__main__': 729 | main() 730 | 731 | As the result, all name and number will same id will print in the exact 732 | order. Max watching should not be lesser than 100. If you put key/id to 733 | None or use submit instead of submit\_id, it will do no order but 734 | faster. 735 | 736 | A Detail User Guide will comming soon... ## Pros vs Cons and question 737 | From my opinion only, Microservice connector has the following Pros and 738 | Cons to improve ### Pros: \* Ease of use, Ease of development, you don't 739 | need to touch on http connection \* Can build decentralize or 740 | Distributed system with Infinite scalability \* Send and receive data 741 | with many types as string, int, float, list, dict. \* Connect all around 742 | the world with internet 743 | 744 | Cons: 745 | ~~~~~ 746 | 747 | - Do not support send/receive tuple and set type (because I don't like 748 | them). 749 | - Do not support send/receive a whole class, return of decorator and 750 | server-side computer 751 | - Is not really fast log-broker server as RabbitMQ, ZeroMQ, kafka: 752 | *yes, oneService cannot send 10 million message per second like them, 753 | but it has other advance.* 754 | - Do not support Database, user/role management, system manager: *not 755 | yet, we are trying to write new feature include them. We welcome any 756 | contributor support us.* 757 | 758 | Question: 759 | ~~~~~~~~~ 760 | 761 | - Why not a load balancer ? 762 | 763 | *It is out of range. Load balancer cover the other layer. Other 764 | package can handle it better. But we consider to add a custom 765 | function for it.* 766 | 767 | - What about support more options, async/await ? 768 | 769 | *We are trying to connect by Sanic and Japronto soon* 770 | 771 | - What about data integrity, blockchain, token ? 772 | 773 | *We are trying to add them, but cannot be soon* 774 | 775 | Authors 776 | ------- 777 | 778 | - **Tuan Nguyen Minh** - *Financer and Developer* - email: 779 | ntuan221@gmail.com 780 | 781 | Thank for the frameworks and their authors: \* flask - micro 782 | webframework \* Sanic - Gotta go fast \* requests 783 | 784 | Favourite idioms: \* Don't repeat your self \* Think like human, make 785 | for human \* Simple is stronger ## License: BDS license 786 | --------------------------------------------------------------------------------