├── .gitignore ├── README.md ├── demo1.py ├── demo10.py ├── demo11.py ├── demo12.py ├── demo13.py ├── demo14.py ├── demo15.py ├── demo16.py ├── demo17.py ├── demo18.py ├── demo19.py ├── demo1_1.py ├── demo2.py ├── demo20.py ├── demo21.py ├── demo3.py ├── demo4.py ├── demo5.py ├── demo6.py ├── demo7.py ├── demo8_1.py ├── demo9.py └── demo9_1.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### Python template 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | 8 | /.idea 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | pip-wheel-metadata/ 28 | share/python-wheels/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | MANIFEST 33 | 34 | # PyInstaller 35 | # Usually these files are written by a python script from a template 36 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 37 | *.manifest 38 | *.spec 39 | 40 | # Installer logs 41 | pip-log.txt 42 | pip-delete-this-directory.txt 43 | 44 | # Unit test / coverage reports 45 | htmlcov/ 46 | .tox/ 47 | .nox/ 48 | .coverage 49 | .coverage.* 50 | .cache 51 | nosetests.xml 52 | coverage.xml 53 | *.cover 54 | *.py,cover 55 | .hypothesis/ 56 | .pytest_cache/ 57 | 58 | # Translations 59 | *.mo 60 | *.pot 61 | 62 | # Django stuff: 63 | *.log 64 | local_settings.py 65 | db.sqlite3 66 | db.sqlite3-journal 67 | 68 | # Flask stuff: 69 | instance/ 70 | .webassets-cache 71 | 72 | # Scrapy stuff: 73 | .scrapy 74 | 75 | # Sphinx documentation 76 | docs/_build/ 77 | 78 | # PyBuilder 79 | target/ 80 | 81 | # Jupyter Notebook 82 | .ipynb_checkpoints 83 | 84 | # IPython 85 | profile_default/ 86 | ipython_config.py 87 | 88 | # pyenv 89 | .python-version 90 | 91 | # pipenv 92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 93 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 94 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 95 | # install all needed dependencies. 96 | #Pipfile.lock 97 | 98 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 99 | __pypackages__/ 100 | 101 | # Celery stuff 102 | celerybeat-schedule 103 | celerybeat.pid 104 | 105 | # SageMath parsed files 106 | *.sage.py 107 | 108 | # Environments 109 | .env 110 | .venv 111 | env/ 112 | venv/ 113 | ENV/ 114 | env.bak/ 115 | venv.bak/ 116 | 117 | # Spyder project settings 118 | .spyderproject 119 | .spyproject 120 | 121 | # Rope project settings 122 | .ropeproject 123 | 124 | # mkdocs documentation 125 | /site 126 | 127 | # mypy 128 | .mypy_cache/ 129 | .dmypy.json 130 | dmypy.json 131 | 132 | # Pyre type checker 133 | .pyre/ 134 | 135 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AsyncTest 2 | Async HTTP Demo 3 | -------------------------------------------------------------------------------- /demo1.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import logging 3 | import time 4 | 5 | logging.basicConfig(level=logging.INFO, 6 | format='%(asctime)s - %(levelname)s: %(message)s') 7 | 8 | TOTAL_NUMBER = 100 9 | BASE_URL = 'https://static4.scrape.cuiqingcai.com/detail/{id}' 10 | 11 | start_time = time.time() 12 | for id in range(1, TOTAL_NUMBER + 1): 13 | url = BASE_URL.format(id=id) 14 | logging.info('scraping %s', url) 15 | response = requests.get(url) 16 | end_time = time.time() 17 | logging.info('total time %s seconds', end_time - start_time) 18 | -------------------------------------------------------------------------------- /demo10.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import requests 3 | import time 4 | 5 | start = time.time() 6 | 7 | 8 | async def get(url): 9 | return requests.get(url) 10 | 11 | 12 | async def request(): 13 | url = 'https://static4.scrape.cuiqingcai.com/' 14 | print('Waiting for', url) 15 | response = await get(url) 16 | print('Get response from', url, 'response', response) 17 | 18 | 19 | tasks = [asyncio.ensure_future(request()) for _ in range(10)] 20 | loop = asyncio.get_event_loop() 21 | loop.run_until_complete(asyncio.wait(tasks)) 22 | 23 | end = time.time() 24 | print('Cost time:', end - start) 25 | -------------------------------------------------------------------------------- /demo11.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import aiohttp 3 | import time 4 | 5 | start = time.time() 6 | 7 | async def get(url): 8 | session = aiohttp.ClientSession() 9 | response = await session.get(url) 10 | await response.text() 11 | await session.close() 12 | return response 13 | 14 | async def request(): 15 | url = 'https://static4.scrape.cuiqingcai.com/' 16 | print('Waiting for', url) 17 | response = await get(url) 18 | print('Get response from', url, 'response') 19 | 20 | tasks = [asyncio.ensure_future(request()) for _ in range(100)] 21 | loop = asyncio.get_event_loop() 22 | loop.run_until_complete(asyncio.wait(tasks)) 23 | 24 | end = time.time() 25 | print('Cost time:', end - start) -------------------------------------------------------------------------------- /demo12.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import aiohttp 3 | import time 4 | 5 | 6 | def test(number): 7 | start = time.time() 8 | 9 | async def get(url): 10 | session = aiohttp.ClientSession() 11 | 12 | response = await session.get(url) 13 | await response.text() 14 | await session.close() 15 | return response 16 | 17 | async def request(): 18 | url = 'https://www.baidu.com/' 19 | await get(url) 20 | 21 | tasks = [asyncio.ensure_future(request()) for _ in range(number)] 22 | loop = asyncio.get_event_loop() 23 | loop.run_until_complete(asyncio.wait(tasks)) 24 | 25 | end = time.time() 26 | print('Number:', number, 'Cost time:', end - start) 27 | 28 | for number in [1, 3, 5, 10, 15, 30, 50, 75, 100, 200, 500]: 29 | test(number) 30 | -------------------------------------------------------------------------------- /demo13.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import time 3 | import multiprocessing 4 | 5 | start = time.time() 6 | 7 | def request(_): 8 | url = 'https://static4.scrape.cuiqingcai.com/' 9 | print('Waiting for', url) 10 | result = requests.get(url).text 11 | print('Get response from', url, 'Result:', result) 12 | 13 | cpu_count = multiprocessing.cpu_count() 14 | print('Cpu count:', cpu_count) 15 | pool = multiprocessing.Pool(cpu_count) 16 | pool.map(request, range(100)) 17 | 18 | end = time.time() 19 | print('Cost time:', end - start) -------------------------------------------------------------------------------- /demo14.py: -------------------------------------------------------------------------------- 1 | from aiohttp import web 2 | import asyncio 3 | 4 | async def home(request: web.Request) -> web.Response: 5 | await asyncio.sleep(3) 6 | return web.Response(text="Hi") 7 | 8 | 9 | async def init_app() -> web.Application: 10 | app = web.Application() 11 | app.add_routes([web.get("/", home)]) 12 | return app 13 | 14 | 15 | web.run_app(init_app(), port=5000) 16 | -------------------------------------------------------------------------------- /demo15.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import timeit 3 | from concurrent.futures import ThreadPoolExecutor 4 | import aiohttp 5 | import asyncio 6 | 7 | session = requests.session() 8 | url = "http://127.0.0.1:5000" 9 | 10 | Count = 100 11 | 12 | 13 | def req(url: str): 14 | print('req1', url) 15 | req = requests.get(url) 16 | req.status_code 17 | 18 | 19 | def requests_test(): 20 | """ 21 | 第一组:循环的方式 22 | :return: 23 | """ 24 | for i in range(Count): 25 | req(url) 26 | 27 | 28 | def pool_requests_test(): 29 | """ 30 | 第二组:线程池的方式 31 | :return: 32 | """ 33 | url_list = [url for _ in range(Count)] 34 | with ThreadPoolExecutor(max_workers=20) as pool: 35 | pool.map(req, url_list) 36 | 37 | 38 | async def fetch(url: str): 39 | async with aiohttp.TCPConnector(ssl=False) as tc: 40 | async with aiohttp.ClientSession(connector=tc) as session: 41 | async with session.get(url) as req: 42 | req.status 43 | 44 | 45 | async def start(): 46 | tasks = [asyncio.create_task(fetch(url)) for _ in range(Count)] 47 | await asyncio.wait(tasks) 48 | 49 | 50 | def aiohttp_test(): 51 | """ 52 | 第三组:aiohttp 的方式 53 | :param url: 54 | :return: 55 | """ 56 | asyncio.run(start()) 57 | 58 | 59 | if __name__ == '__main__': 60 | # 循环的 61 | # print(timeit.timeit(stmt=requests_test, number=1)) 62 | # 使用线程池的 63 | # print(timeit.timeit(stmt=pool_requests_test, number=1)) 64 | # 使用 aiohttp 的 65 | print(timeit.timeit(stmt=aiohttp_test, number=1)) 66 | -------------------------------------------------------------------------------- /demo16.py: -------------------------------------------------------------------------------- 1 | import aiohttp 2 | import asyncio 3 | 4 | 5 | async def fetch(session, url): 6 | async with session.get(url) as response: 7 | return await response.text(), response.status 8 | 9 | 10 | async def main(): 11 | async with aiohttp.ClientSession() as session: 12 | html, status = await fetch(session, 'https://cuiqingcai.com') 13 | print(f'html: {html[:100]}...') 14 | print(f'status: {status}') 15 | 16 | 17 | if __name__ == '__main__': 18 | asyncio.run(main()) 19 | -------------------------------------------------------------------------------- /demo17.py: -------------------------------------------------------------------------------- 1 | import aiohttp 2 | import asyncio 3 | 4 | 5 | async def main(): 6 | params = {'name': 'germey', 'age': 25} 7 | async with aiohttp.ClientSession() as session: 8 | async with session.get('https://httpbin.org/get', params=params) as response: 9 | print(await response.text()) 10 | 11 | 12 | if __name__ == '__main__': 13 | asyncio.get_event_loop().run_until_complete(main()) 14 | -------------------------------------------------------------------------------- /demo18.py: -------------------------------------------------------------------------------- 1 | import aiohttp 2 | import asyncio 3 | 4 | 5 | async def main(): 6 | data = {'name': 'germey', 'age': 25} 7 | async with aiohttp.ClientSession() as session: 8 | async with session.post('https://httpbin.org/post', json=data) as response: 9 | print(await response.text()) 10 | 11 | 12 | if __name__ == '__main__': 13 | asyncio.get_event_loop().run_until_complete(main()) 14 | -------------------------------------------------------------------------------- /demo19.py: -------------------------------------------------------------------------------- 1 | import aiohttp 2 | import asyncio 3 | 4 | 5 | async def main(): 6 | data = {'name': 'germey', 'age': 25} 7 | async with aiohttp.ClientSession() as session: 8 | async with session.post('https://httpbin.org/post', data=data) as response: 9 | print(await response.text()) 10 | 11 | 12 | if __name__ == '__main__': 13 | asyncio.get_event_loop().run_until_complete(main()) 14 | -------------------------------------------------------------------------------- /demo1_1.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import logging 3 | import time 4 | 5 | logging.basicConfig(level=logging.INFO, 6 | format='%(asctime)s - %(levelname)s: %(message)s') 7 | 8 | TOTAL_NUMBER = 100 9 | URL = 'https://httpbin.org/delay/5' 10 | 11 | start_time = time.time() 12 | for _ in range(1, TOTAL_NUMBER + 1): 13 | logging.info('scraping %s', URL) 14 | response = requests.get(URL) 15 | end_time = time.time() 16 | logging.info('total time %s seconds', end_time - start_time) 17 | -------------------------------------------------------------------------------- /demo2.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | async def execute(x): 4 | print('Number:', x) 5 | 6 | coroutine = execute(1) 7 | print('Coroutine:', coroutine) 8 | print('After calling execute') 9 | 10 | loop = asyncio.get_event_loop() 11 | loop.run_until_complete(coroutine) 12 | print('After calling loop') -------------------------------------------------------------------------------- /demo20.py: -------------------------------------------------------------------------------- 1 | import aiohttp 2 | import asyncio 3 | 4 | 5 | async def main(): 6 | data = {'name': 'germey', 'age': 25} 7 | async with aiohttp.ClientSession() as session: 8 | async with session.post('https://httpbin.org/post', data=data) as response: 9 | print('status:', response.status) 10 | print('headers:', response.headers) 11 | print('body:', await response.text()) 12 | print('bytes:', await response.read()) 13 | print('json:', await response.json()) 14 | 15 | 16 | if __name__ == '__main__': 17 | asyncio.get_event_loop().run_until_complete(main()) 18 | -------------------------------------------------------------------------------- /demo21.py: -------------------------------------------------------------------------------- 1 | import aiohttp 2 | import asyncio 3 | 4 | 5 | async def main(): 6 | timeout = aiohttp.ClientTimeout(total=0.1) 7 | async with aiohttp.ClientSession(timeout=timeout) as session: 8 | async with session.get('https://httpbin.org/get') as response: 9 | print('status:', response.status) 10 | 11 | 12 | if __name__ == '__main__': 13 | asyncio.get_event_loop().run_until_complete(main()) 14 | -------------------------------------------------------------------------------- /demo3.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | async def execute(x): 4 | print('Number:', x) 5 | return x 6 | 7 | coroutine = execute(1) 8 | print('Coroutine:', coroutine) 9 | print('After calling execute') 10 | 11 | loop = asyncio.get_event_loop() 12 | task = loop.create_task(coroutine) 13 | print('Task:', task) 14 | loop.run_until_complete(task) 15 | print('Task:', task) 16 | print('After calling loop') -------------------------------------------------------------------------------- /demo4.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | async def execute(x): 5 | print('Number:', x) 6 | return x 7 | 8 | 9 | coroutine = execute(1) 10 | print('Coroutine:', coroutine) 11 | print('After calling execute') 12 | 13 | task = asyncio.ensure_future(coroutine) 14 | print('Task:', task) 15 | loop = asyncio.get_event_loop() 16 | loop.run_until_complete(task) 17 | print('Task:', task) 18 | print('After calling loop') 19 | -------------------------------------------------------------------------------- /demo5.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import requests 3 | 4 | 5 | async def request(): 6 | url = 'https://www.baidu.com' 7 | status = requests.get(url) 8 | return status 9 | 10 | 11 | def callback(task): 12 | print('Status:', task.result()) 13 | 14 | 15 | coroutine = request() 16 | task = asyncio.ensure_future(coroutine) 17 | task.add_done_callback(callback) 18 | print('Task:', task) 19 | 20 | loop = asyncio.get_event_loop() 21 | loop.run_until_complete(task) 22 | print('Task:', task) 23 | -------------------------------------------------------------------------------- /demo6.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import requests 3 | 4 | 5 | async def request(): 6 | url = 'https://www.baidu.com' 7 | status = requests.get(url) 8 | return status 9 | 10 | 11 | coroutine = request() 12 | task = asyncio.ensure_future(coroutine) 13 | print('Task:', task) 14 | 15 | loop = asyncio.get_event_loop() 16 | loop.run_until_complete(task) 17 | print('Task:', task) 18 | print('Task Result:', task.result()) 19 | -------------------------------------------------------------------------------- /demo7.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import requests 3 | 4 | async def request(): 5 | url = 'https://www.baidu.com' 6 | status = requests.get(url) 7 | return status 8 | 9 | tasks = [asyncio.ensure_future(request()) for _ in range(5)] 10 | print('Tasks:', tasks) 11 | 12 | loop = asyncio.get_event_loop() 13 | loop.run_until_complete(asyncio.wait(tasks)) 14 | 15 | for task in tasks: 16 | print('Task Result:', task.result()) -------------------------------------------------------------------------------- /demo8_1.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import requests 3 | import time 4 | 5 | start = time.time() 6 | 7 | async def request(): 8 | url = 'https://httpbin.org/delay/5' 9 | print('Waiting for', url) 10 | response = requests.get(url) 11 | print('Get response from', url, 'response', response) 12 | 13 | tasks = [asyncio.ensure_future(request()) for _ in range(10)] 14 | loop = asyncio.get_event_loop() 15 | loop.run_until_complete(asyncio.wait(tasks)) 16 | 17 | end = time.time() 18 | print('Cost time:', end - start) -------------------------------------------------------------------------------- /demo9.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import requests 3 | import time 4 | 5 | start = time.time() 6 | 7 | 8 | async def request(): 9 | url = 'https://static4.scrape.cuiqingcai.com/' 10 | print('Waiting for', url) 11 | response = await requests.get(url) 12 | print('Get response from', url, 'response', response) 13 | 14 | 15 | tasks = [asyncio.ensure_future(request()) for _ in range(10)] 16 | loop = asyncio.get_event_loop() 17 | loop.run_until_complete(asyncio.wait(tasks)) 18 | 19 | end = time.time() 20 | print('Cost time:', end - start) 21 | -------------------------------------------------------------------------------- /demo9_1.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import requests 3 | import time 4 | 5 | start = time.time() 6 | 7 | 8 | async def request(): 9 | url = 'https://httpbin.org/delay/5' 10 | print('Waiting for', url) 11 | response = await requests.get(url) 12 | print('Get response from', url, 'response', response) 13 | 14 | 15 | tasks = [asyncio.ensure_future(request()) for _ in range(10)] 16 | loop = asyncio.get_event_loop() 17 | loop.run_until_complete(asyncio.wait(tasks)) 18 | 19 | end = time.time() 20 | print('Cost time:', end - start) 21 | --------------------------------------------------------------------------------