├── .gitignore ├── LICENSE ├── README.md ├── example ├── __init__.py ├── build.sh ├── controller │ ├── __init__.py │ └── home.py ├── dynamic.sample.conf ├── main.py ├── main_cmd.py ├── main_tcp.py ├── main_test.py ├── model │ └── __init__.py ├── routing.py ├── service │ ├── __init__.py │ └── base │ │ └── __init__.py ├── setting.py ├── static.sample.conf └── view │ ├── __init__.py │ └── home │ ├── __init__.py │ └── default.html ├── hagworm ├── __init__.py ├── extend │ ├── __init__.py │ ├── asyncio │ │ ├── __init__.py │ │ ├── base.py │ │ ├── buffer.py │ │ ├── cache.py │ │ ├── database.py │ │ ├── event.py │ │ ├── file.py │ │ ├── future.py │ │ ├── net.py │ │ ├── ntp.py │ │ ├── task.py │ │ ├── transaction.py │ │ └── zmq.py │ ├── base.py │ ├── cache.py │ ├── compile.py │ ├── crypto.py │ ├── error.py │ ├── event.py │ ├── excel.py │ ├── interface.py │ ├── logging.py │ ├── metaclass.py │ ├── qrcode.py │ ├── struct.py │ └── transaction.py ├── frame │ ├── __init__.py │ ├── stress_tests.py │ └── tornado │ │ ├── __init__.py │ │ ├── base.py │ │ ├── socket.py │ │ └── web.py ├── static │ ├── __init__.py │ └── cacert.pem └── third │ ├── __init__.py │ └── aliyun │ ├── __init__.py │ └── rocketmq.py ├── install.sh ├── setup.cfg ├── setup.py ├── testing ├── main.py ├── test_extend_asyncio_base.py ├── test_extend_asyncio_future.py ├── test_extend_asyncio_net.py ├── test_extend_asyncio_task.py ├── test_extend_base.py ├── test_extend_cache.py └── test_extend_struct.py └── upload.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # Other files and folders 107 | .idea/ 108 | .vscode/ 109 | .settings/ 110 | /example/static.conf 111 | /example/dynamic.conf 112 | 113 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2019 Hagworm of copyright Shaobo.Wang 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 因众所周知的原因，本仓库已经迁移至国内的托管平台，Github的代码将不再继续更新 2 | # https://gitee.com/wsb310/hagworm -------------------------------------------------------------------------------- /example/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wsb310/hagworm/ce69d267cc8f09b903f26238a6b3b3f7d46e0397/example/__init__.py -------------------------------------------------------------------------------- /example/build.sh: -------------------------------------------------------------------------------- 1 | 2 | python3 -m hagworm.extend.compile -i ./ -o ./compiled/ 3 | 4 | cp ./*.conf ./compiled/ 5 | -------------------------------------------------------------------------------- /example/controller/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /example/controller/home.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from hagworm.extend.asyncio.base import Utils 4 | from hagworm.frame.tornado.web import SocketBaseHandler, RequestBaseHandler, DownloadAgent,\ 5 | HttpBasicAuth, LogRequestMixin 6 | 7 | from service.base import DataSource 8 | 9 | 10 | class Default(RequestBaseHandler, LogRequestMixin): 11 | 12 | async def head(self): 13 | 14 | await DataSource().health() 15 | 16 | async def get(self): 17 | 18 | return self.render( 19 | r'home/default.html', 20 | online=DataSource().online 21 | ) 22 | 23 | def delete(self): 24 | 25 | Utils.call_soon(DataSource().reset_mysql_pool) 26 | Utils.call_soon(DataSource().reset_mongo_pool) 27 | 28 | 29 | class Download(DownloadAgent): 30 | 31 | @HttpBasicAuth(r'Test', r'tester', r'123456') 32 | async def get(self): 33 | 34 | await self.transmit(r'https://www.baidu.com/img/bd_logo.png') 35 | 36 | 37 | class Socket(SocketBaseHandler): 38 | 39 | async def open(self, channel): 40 | 41 | self.log.debug(f'WebSocket opened: {channel}') 42 | 43 | async def on_message(self, message): 44 | 45 | self.write_message(message) 46 | 47 | self.log.debug(f'WebSocket message: {message}') 48 | 49 | async def on_ping(self, data): 50 | 51 | self.log.debug(f'WebSocket ping: {data}') 52 | 53 | async def on_close(self): 54 | 55 | self.log.debug(r'WebSocket closed') 56 | -------------------------------------------------------------------------------- /example/dynamic.sample.conf: -------------------------------------------------------------------------------- 1 | 2 | ################################################## 3 | [Base] 4 | 5 | Port = 8080 6 | Debug = True 7 | GZip = False 8 | Secret = c77db6268da111e9ae1b8c85904d8a96 9 | ProcessNum = 1 10 | 11 | ################################################## 12 | [Log] 13 | 14 | LogLevel = debug 15 | LogFilePath = 16 | LogFileSplitSize = 500 17 | LogFileSplitTime = 00:00 18 | LogFileBackups = 7 19 | 20 | ################################################## 21 | [Pool] 22 | 23 | ThreadPoolMaxWorkers = 64 24 | ProcessPoolMaxWorkers = 0 25 | 26 | ################################################## 27 | -------------------------------------------------------------------------------- /example/main.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | import sys 5 | 6 | os.chdir(os.path.dirname(__file__)) 7 | sys.path.insert(0, os.path.abspath(r'../')) 8 | 9 | from hagworm.extend.logging import LogFileRotator 10 | from hagworm.frame.tornado.base import Launcher 11 | 12 | from routing import router 13 | from setting import ConfigStatic, ConfigDynamic 14 | from service.base import GlobalDict, DataSource 15 | 16 | 17 | def main(): 18 | 19 | cluster = os.getenv(r'CLUSTER', None) 20 | 21 | if cluster is None: 22 | ConfigStatic.read(r'./static.conf') 23 | ConfigDynamic.read(r'./dynamic.conf') 24 | else: 25 | ConfigStatic.read(f'./static.{cluster.lower()}.conf') 26 | ConfigDynamic.read(f'./dynamic.{cluster.lower()}.conf') 27 | 28 | # 初始化全局变量字典（进程间同步） 29 | GlobalDict() 30 | 31 | Launcher( 32 | router, 33 | ConfigDynamic.Port, 34 | process_num=ConfigDynamic.ProcessNum, 35 | async_initialize=DataSource.initialize, 36 | debug=ConfigDynamic.Debug, 37 | gzip=ConfigDynamic.GZip, 38 | template_path=r'view', 39 | cookie_secret=ConfigDynamic.Secret, 40 | log_level=ConfigDynamic.LogLevel, 41 | log_file_path=ConfigDynamic.LogFilePath, 42 | log_file_rotation=LogFileRotator.make(ConfigDynamic.LogFileSplitSize, ConfigDynamic.LogFileSplitTime), 43 | log_file_retention=ConfigDynamic.LogFileBackups, 44 | ).start() 45 | 46 | 47 | if __name__ == r'__main__': 48 | 49 | main() 50 | -------------------------------------------------------------------------------- /example/main_cmd.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | import sys 5 | 6 | os.chdir(os.path.dirname(__file__)) 7 | sys.path.insert(0, os.path.abspath(r'../')) 8 | 9 | from hagworm.extend.asyncio.base import Launcher, Utils 10 | 11 | 12 | async def main(): 13 | 14 | Utils.log.info(r'waiting...') 15 | 16 | await Utils.sleep(5) 17 | 18 | Utils.log.info(r'finished') 19 | 20 | 21 | if __name__ == r'__main__': 22 | 23 | Launcher().run(main()) 24 | -------------------------------------------------------------------------------- /example/main_tcp.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | import sys 5 | 6 | os.chdir(os.path.dirname(__file__)) 7 | sys.path.insert(0, os.path.abspath(r'../')) 8 | 9 | from hagworm.frame.tornado.socket import Protocol, Launcher 10 | from hagworm.extend.base import Utils 11 | 12 | from setting import ConfigStatic, ConfigDynamic 13 | from service.base import DataSource 14 | 15 | 16 | class EchoProtocol(Protocol): 17 | 18 | async def connection_made(self): 19 | 20 | Utils.log.info(f'connection made: {self.client_address}') 21 | 22 | async def connection_lost(self): 23 | 24 | Utils.log.info(f'connection lost: {self.client_address}') 25 | 26 | async def data_received(self, chunk): 27 | 28 | await self.data_write(chunk) 29 | 30 | 31 | def main(): 32 | 33 | cluster = os.getenv(r'CLUSTER', None) 34 | 35 | if cluster is None: 36 | ConfigStatic.read(r'./static.conf') 37 | ConfigDynamic.read(r'./dynamic.conf') 38 | else: 39 | ConfigStatic.read(f'./static.{cluster.lower()}.conf') 40 | ConfigDynamic.read(f'./dynamic.{cluster.lower()}.conf') 41 | 42 | Launcher( 43 | EchoProtocol, 44 | ConfigDynamic.Port, 45 | async_initialize=DataSource.initialize, 46 | debug=ConfigDynamic.Debug, 47 | log_level=ConfigDynamic.LogLevel, 48 | log_file_path=ConfigDynamic.LogFilePath, 49 | log_file_num_backups=ConfigDynamic.LogFileBackups, 50 | ).start() 51 | 52 | 53 | if __name__ == r'__main__': 54 | main() 55 | -------------------------------------------------------------------------------- /example/main_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from hagworm.frame.stress_tests import Launcher, Runner, TaskInterface 4 | import os 5 | import sys 6 | 7 | os.chdir(os.path.dirname(__file__)) 8 | sys.path.insert(0, os.path.abspath(r'../')) 9 | 10 | 11 | class Task(TaskInterface): 12 | 13 | async def run(self): 14 | 15 | for index in range(2): 16 | 17 | await self.sleep(0.1) 18 | 19 | resp_time = self.randint(12345, 98765) / 10000 20 | 21 | if self.randhit([True, False], [32, 8]): 22 | self.success(f'Test{index}', resp_time) 23 | else: 24 | self.failure(f'Test{index}', resp_time) 25 | 26 | 27 | if __name__ == r'__main__': 28 | 29 | Launcher(process_number=2).run(Runner(Task).run, 8, 32) 30 | -------------------------------------------------------------------------------- /example/model/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /example/routing.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from controller import home 4 | 5 | 6 | router = [ 7 | 8 | (r'/?', home.Default), 9 | 10 | (r'/download/?', home.Download), 11 | 12 | (r'/socket/(\w+)/?', home.Socket), 13 | 14 | ] 15 | -------------------------------------------------------------------------------- /example/service/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /example/service/base/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from hagworm.extend.base import Ignore, catch_error 4 | from hagworm.extend.metaclass import Singleton 5 | from hagworm.extend.asyncio.base import Utils, FuncCache, ShareFuture, MultiTasks, async_adapter 6 | from hagworm.extend.asyncio.cache import RedisDelegate 7 | from hagworm.extend.asyncio.database import MongoDelegate, MySQLDelegate 8 | from hagworm.extend.asyncio.future import ProcessSyncDict 9 | 10 | from setting import ConfigStatic, ConfigDynamic 11 | 12 | 13 | class GlobalDict(Singleton, ProcessSyncDict): 14 | pass 15 | 16 | 17 | class DataSource(Singleton, RedisDelegate, MongoDelegate, MySQLDelegate): 18 | 19 | def __init__(self): 20 | 21 | RedisDelegate.__init__(self) 22 | 23 | MongoDelegate.__init__( 24 | self, 25 | ConfigStatic.MongoHost, ConfigStatic.MongoUser, ConfigStatic.MongoPasswd, 26 | min_pool_size=ConfigStatic.MongoMinConn, max_pool_size=ConfigStatic.MongoMaxConn, 27 | max_idle_time=3600 28 | ) 29 | 30 | MySQLDelegate.__init__(self) 31 | 32 | self._global_dict = GlobalDict() 33 | 34 | @classmethod 35 | async def initialize(cls): 36 | 37 | inst = cls() 38 | 39 | await inst.async_init_redis( 40 | ConfigStatic.RedisHost, ConfigStatic.RedisPasswd, 41 | minsize=ConfigStatic.RedisMinConn, maxsize=ConfigStatic.RedisMaxConn, 42 | db=ConfigStatic.RedisBase, expire=ConfigStatic.RedisExpire, 43 | key_prefix=ConfigStatic.RedisKeyPrefix 44 | ) 45 | 46 | if ConfigStatic.MySqlMasterServer: 47 | 48 | await inst.async_init_mysql_rw( 49 | ConfigStatic.MySqlMasterServer[0], ConfigStatic.MySqlMasterServer[1], ConfigStatic.MySqlName, 50 | ConfigStatic.MySqlUser, ConfigStatic.MySqlPasswd, 51 | minsize=ConfigStatic.MySqlMasterMinConn, maxsize=ConfigStatic.MySqlMasterMaxConn, 52 | echo=ConfigDynamic.Debug, pool_recycle=21600, conn_life=43200 53 | ) 54 | 55 | if ConfigStatic.MySqlSlaveServer: 56 | 57 | await inst.async_init_mysql_ro( 58 | ConfigStatic.MySqlSlaveServer[0], ConfigStatic.MySqlSlaveServer[1], ConfigStatic.MySqlName, 59 | ConfigStatic.MySqlUser, ConfigStatic.MySqlPasswd, 60 | minsize=ConfigStatic.MySqlSlaveMinConn, maxsize=ConfigStatic.MySqlSlaveMaxConn, 61 | echo=ConfigDynamic.Debug, pool_recycle=21600, readonly=True, conn_life=43200 62 | ) 63 | 64 | @property 65 | def online(self): 66 | 67 | health_refresh_time = self._global_dict.get(r'health_refresh_time', 0) 68 | 69 | if health_refresh_time > 0: 70 | return Utils.loop_time() - health_refresh_time < 60 71 | else: 72 | return False 73 | 74 | def _refresh_online(self): 75 | 76 | self._global_dict[r'health_refresh_time'] = Utils.loop_time() 77 | 78 | @ShareFuture() 79 | @FuncCache(ttl=30) 80 | async def health(self): 81 | 82 | result = False 83 | 84 | with catch_error(): 85 | 86 | tasks = MultiTasks() 87 | 88 | tasks.append(self.cache_health()) 89 | tasks.append(self.mysql_health()) 90 | tasks.append(self.mongo_health()) 91 | 92 | await tasks 93 | 94 | result = all(tasks) 95 | 96 | if result is True: 97 | self._refresh_online() 98 | 99 | return result 100 | 101 | 102 | class ServiceBase(Singleton, Utils): 103 | 104 | def __init__(self): 105 | 106 | self._data_source = DataSource() 107 | 108 | def Break(self, data=None, layers=1): 109 | 110 | raise Ignore(data, layers) 111 | -------------------------------------------------------------------------------- /example/setting.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from hagworm.extend.struct import Configure 4 | 5 | 6 | class _Static(Configure): 7 | 8 | def _init_options(self): 9 | 10 | ################################################## 11 | # MySql数据库 12 | 13 | self.MySqlMasterServer = self._parser.get_split_host(r'MySql', r'MySqlMasterServer') 14 | 15 | self.MySqlSlaveServer = self._parser.get_split_host(r'MySql', r'MySqlSlaveServer') 16 | 17 | self.MySqlName = self._parser.get(r'MySql', r'MySqlName') 18 | 19 | self.MySqlUser = self._parser.get(r'MySql', r'MySqlUser') 20 | 21 | self.MySqlPasswd = self._parser.get(r'MySql', r'MySqlPasswd') 22 | 23 | self.MySqlMasterMinConn = self._parser.getint(r'MySql', r'MySqlMasterMinConn') 24 | 25 | self.MySqlMasterMaxConn = self._parser.getint(r'MySql', r'MySqlMasterMaxConn') 26 | 27 | self.MySqlSlaveMinConn = self._parser.getint(r'MySql', r'MySqlSlaveMinConn') 28 | 29 | self.MySqlSlaveMaxConn = self._parser.getint(r'MySql', r'MySqlSlaveMaxConn') 30 | 31 | ################################################## 32 | # Mongo数据库 33 | 34 | self.MongoHost = self._parser.get_split_str(r'Mongo', r'MongoHost') 35 | 36 | self.MongoName = self._parser.get(r'Mongo', r'MongoName') 37 | 38 | self.MongoUser = self._parser.get(r'Mongo', r'MongoUser') 39 | 40 | self.MongoPasswd = self._parser.get(r'Mongo', r'MongoPasswd') 41 | 42 | self.MongoMinConn = self._parser.getint(r'Mongo', r'MongoMinConn') 43 | 44 | self.MongoMaxConn = self._parser.getint(r'Mongo', r'MongoMaxConn') 45 | 46 | ################################################## 47 | # 缓存 48 | 49 | self.RedisHost = self._parser.get_split_host(r'Redis', r'RedisHost') 50 | 51 | self.RedisBase = self._parser.getint(r'Redis', r'RedisBase') 52 | 53 | self.RedisPasswd = self._parser.getstr(r'Redis', r'RedisPasswd') 54 | 55 | self.RedisMinConn = self._parser.getint(r'Redis', r'RedisMinConn') 56 | 57 | self.RedisMaxConn = self._parser.getint(r'Redis', r'RedisMaxConn') 58 | 59 | self.RedisExpire = self._parser.getint(r'Redis', r'RedisExpire') 60 | 61 | self.RedisKeyPrefix = self._parser.get(r'Redis', r'RedisKeyPrefix') 62 | 63 | ################################################## 64 | 65 | 66 | class _Dynamic(Configure): 67 | 68 | def _init_options(self): 69 | 70 | ################################################## 71 | # 基本 72 | 73 | self.Port = self._parser.getint(r'Base', r'Port') 74 | 75 | self.Debug = self._parser.getboolean(r'Base', r'Debug') 76 | 77 | self.GZip = self._parser.getboolean(r'Base', r'GZip') 78 | 79 | self.Secret = self._parser.get(r'Base', r'Secret') 80 | 81 | self.ProcessNum = self._parser.getint(r'Base', r'ProcessNum') 82 | 83 | ################################################## 84 | # 日志 85 | 86 | self.LogLevel = self._parser.get(r'Log', r'LogLevel') 87 | 88 | self.LogFilePath = self._parser.get(r'Log', r'LogFilePath') 89 | 90 | self.LogFileSplitSize = self._parser.getint(r'Log', r'LogFileSplitSize') 91 | 92 | self.LogFileSplitTime = self._parser.get(r'Log', r'LogFileSplitTime') 93 | 94 | self.LogFileBackups = self._parser.getint(r'Log', r'LogFileBackups') 95 | 96 | ################################################## 97 | # 线程/进程 98 | 99 | self.ThreadPoolMaxWorkers = self._parser.getint(r'Pool', r'ThreadPoolMaxWorkers') 100 | 101 | self.ProcessPoolMaxWorkers = self._parser.getint(r'Pool', r'ProcessPoolMaxWorkers') 102 | 103 | ################################################## 104 | 105 | 106 | ConfigStatic = _Static() 107 | ConfigDynamic = _Dynamic() 108 | -------------------------------------------------------------------------------- /example/static.sample.conf: -------------------------------------------------------------------------------- 1 | 2 | ################################################## 3 | [MySql] 4 | 5 | MySqlMasterServer = 6 | MySqlSlaveServer = 7 | MySqlName = demo 8 | MySqlUser = 9 | MySqlPasswd = 10 | MySqlMasterMinConn = 8 11 | MySqlMasterMaxConn = 16 12 | MySqlSlaveMinConn = 8 13 | MySqlSlaveMaxConn = 16 14 | 15 | ################################################## 16 | [Mongo] 17 | 18 | MongoHost = 19 | MongoName = demo 20 | MongoUser = 21 | MongoPasswd = 22 | MongoMinConn = 8 23 | MongoMaxConn = 16 24 | 25 | ################################################## 26 | [Redis] 27 | 28 | RedisHost = 29 | RedisBase = 0 30 | RedisPasswd = 31 | RedisMinConn = 8 32 | RedisMaxConn = 16 33 | RedisExpire = 3600 34 | RedisKeyPrefix = demo 35 | 36 | ################################################## 37 | -------------------------------------------------------------------------------- /example/view/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /example/view/home/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /example/view/home/default.html: -------------------------------------------------------------------------------- 1 | 2 |

online: {{ online }}

3 | -------------------------------------------------------------------------------- /hagworm/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | __version__ = r'3.11.0' 4 | 5 | package_slogan = r''' 6 | __ __ ______ ______ __ __ ______ ______ __ __ 7 | /\ \_\ \ /\ __ \ /\ ___\ /\ \ _ \ \ /\ __ \ /\ == \ /\ "-./ \ 8 | \ \ __ \\ \ __ \\ \ \__ \\ \ \/ ".\ \\ \ \/\ \\ \ __< \ \ \-./\ \ 9 | \ \_\ \_\\ \_\ \_\\ \_____\\ \__/".~\_\\ \_____\\ \_\ \_\\ \_\ \ \_\ 10 | \/_/\/_/ \/_/\/_/ \/_____/ \/_/ \/_/ \/_____/ \/_/ /_/ \/_/ \/_/ 11 | ''' 12 | -------------------------------------------------------------------------------- /hagworm/extend/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /hagworm/extend/asyncio/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /hagworm/extend/asyncio/base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import types 4 | import weakref 5 | import inspect 6 | import asyncio 7 | 8 | import async_timeout 9 | 10 | from contextlib import asynccontextmanager 11 | from contextvars import Context, ContextVar 12 | 13 | from hagworm import package_slogan 14 | from hagworm import __version__ as package_version 15 | from hagworm.extend import base 16 | from hagworm.extend.logging import DEFAULT_LOG_FILE_ROTATOR 17 | from hagworm.extend.cache import StackCache 18 | from hagworm.extend.interface import RunnableInterface 19 | 20 | 21 | def install_uvloop(): 22 | """尝试安装uvloop 23 | """ 24 | 25 | try: 26 | import uvloop 27 | except ModuleNotFoundError: 28 | Utils.log.warning(f'uvloop is not supported (T＿T)') 29 | else: 30 | uvloop.install() 31 | Utils.log.success(f'uvloop {uvloop.__version__} installed') 32 | 33 | 34 | class Launcher(RunnableInterface): 35 | """异步版本的启动器 36 | 37 | 用于简化和统一程序的启动操作 38 | 39 | """ 40 | 41 | def __init__(self, 42 | log_file_path=None, log_level=r'INFO', 43 | log_file_rotation=DEFAULT_LOG_FILE_ROTATOR, log_file_retention=0xff, 44 | process_number=1, process_guardian=None, 45 | debug=False 46 | ): 47 | 48 | self._process_id = 0 49 | self._process_number = process_number 50 | 51 | if log_file_path: 52 | 53 | _log_file_path = Utils.path.join( 54 | log_file_path, 55 | r'runtime_{time}.log' 56 | ) 57 | 58 | Utils.log.add( 59 | _log_file_path, 60 | level=log_level, 61 | enqueue=True, 62 | backtrace=debug, 63 | rotation=log_file_rotation, 64 | retention=log_file_retention 65 | ) 66 | 67 | else: 68 | 69 | Utils.log.level(log_level) 70 | 71 | environment = Utils.environment() 72 | 73 | Utils.log.info( 74 | f'{package_slogan}' 75 | f'hagworm {package_version}\n' 76 | f'python {environment["python"]}\n' 77 | f'system {" ".join(environment["system"])}' 78 | ) 79 | 80 | install_uvloop() 81 | 82 | if self._process_number > 1: 83 | self._process_id = base.fork_processes(self._process_number, process_guardian) 84 | 85 | self._event_loop = asyncio.get_event_loop() 86 | self._event_loop.set_debug(debug) 87 | 88 | @property 89 | def process_id(self): 90 | 91 | return self._process_id 92 | 93 | def run(self, func, *args, **kwargs): 94 | 95 | Utils.log.success(f'Start process no.{self._process_id}') 96 | 97 | self._event_loop.run_until_complete(func(*args, **kwargs)) 98 | 99 | Utils.log.success(f'Stop process no.{self._process_id}') 100 | 101 | 102 | class Utils(base.Utils): 103 | """异步基础工具类 104 | 105 | 集成常用的异步工具函数 106 | 107 | """ 108 | 109 | sleep = staticmethod(asyncio.sleep) 110 | 111 | @staticmethod 112 | async def awaitable_wrapper(obj): 113 | """自适应awaitable对象 114 | """ 115 | 116 | if inspect.isawaitable(obj): 117 | return await obj 118 | else: 119 | return obj 120 | 121 | @staticmethod 122 | @types.coroutine 123 | def wait_frame(count=10): 124 | """暂停指定帧数 125 | """ 126 | 127 | for _ in range(max(1, count)): 128 | yield 129 | 130 | @staticmethod 131 | def loop_time(): 132 | """获取当前loop内部时钟 133 | """ 134 | 135 | loop = asyncio.events.get_event_loop() 136 | 137 | return loop.time() 138 | 139 | @classmethod 140 | def call_soon(cls, callback, *args, **kwargs): 141 | """延时调用(能隔离上下文) 142 | """ 143 | 144 | loop = asyncio.events.get_event_loop() 145 | 146 | if kwargs: 147 | 148 | return loop.call_soon( 149 | async_adapter( 150 | cls.func_partial( 151 | callback, 152 | *args, 153 | **kwargs 154 | ) 155 | ), 156 | context=Context() 157 | ) 158 | 159 | else: 160 | 161 | return loop.call_soon( 162 | async_adapter(callback), 163 | *args, 164 | context=Context() 165 | ) 166 | 167 | @classmethod 168 | def call_soon_threadsafe(cls, callback, *args, **kwargs): 169 | """延时调用(线程安全，能隔离上下文) 170 | """ 171 | 172 | loop = asyncio.events.get_event_loop() 173 | 174 | if kwargs: 175 | 176 | return loop.call_soon_threadsafe( 177 | async_adapter( 178 | cls.func_partial( 179 | callback, 180 | *args, 181 | **kwargs 182 | ) 183 | ), 184 | context=Context() 185 | ) 186 | 187 | else: 188 | 189 | return loop.call_soon_threadsafe( 190 | async_adapter(callback), 191 | *args, 192 | context=Context() 193 | ) 194 | 195 | @classmethod 196 | def call_later(cls, delay, callback, *args, **kwargs): 197 | """延时指定秒数调用(能隔离上下文) 198 | """ 199 | 200 | loop = asyncio.events.get_event_loop() 201 | 202 | if kwargs: 203 | 204 | return loop.call_later( 205 | delay, 206 | async_adapter( 207 | cls.func_partial( 208 | callback, 209 | *args, 210 | **kwargs 211 | ) 212 | ), 213 | context=Context() 214 | ) 215 | 216 | else: 217 | 218 | return loop.call_later( 219 | delay, 220 | async_adapter(callback), 221 | *args, 222 | context=Context() 223 | ) 224 | 225 | @classmethod 226 | def call_at(cls, when, callback, *args, **kwargs): 227 | """指定时间调用(能隔离上下文) 228 | """ 229 | 230 | loop = asyncio.events.get_event_loop() 231 | 232 | if kwargs: 233 | 234 | return loop.call_at( 235 | when, 236 | async_adapter( 237 | cls.func_partial( 238 | callback, 239 | *args, 240 | **kwargs 241 | ) 242 | ), 243 | context=Context() 244 | ) 245 | 246 | else: 247 | 248 | return loop.call_at( 249 | when, 250 | async_adapter(callback), 251 | *args, 252 | context=Context() 253 | ) 254 | 255 | @staticmethod 256 | def create_task(coro): 257 | """将协程对象包装成task对象(兼容Future接口) 258 | """ 259 | 260 | if asyncio.iscoroutine(coro): 261 | return asyncio.create_task(coro) 262 | else: 263 | return None 264 | 265 | @staticmethod 266 | def run_until_complete(future): 267 | """运行事件循环直到future结束 268 | """ 269 | 270 | loop = asyncio.events.get_event_loop() 271 | 272 | return loop.run_until_complete(future) 273 | 274 | @staticmethod 275 | @asynccontextmanager 276 | async def async_timeout(timeout): 277 | """异步超时等待 278 | 279 | async with timeout(1.5) as res: 280 | pass 281 | print(res.expired) 282 | 283 | """ 284 | 285 | try: 286 | 287 | async with async_timeout.timeout(timeout) as res: 288 | yield res 289 | 290 | except base.Ignore as err: 291 | 292 | if err.throw(): 293 | raise err 294 | 295 | except Exception as err: 296 | 297 | Utils.log.exception(err) 298 | 299 | 300 | def async_adapter(func): 301 | """异步函数适配装饰器 302 | 303 | 使异步函数可以在同步函数中调用，即非阻塞式的启动异步函数，同时会影响上下文资源的生命周期 304 | 305 | """ 306 | 307 | @base.Utils.func_wraps(func) 308 | def _wrapper(*args, **kwargs): 309 | 310 | return Utils.create_task( 311 | func(*args, **kwargs) 312 | ) 313 | 314 | return _wrapper 315 | 316 | 317 | class WeakContextVar: 318 | """弱引用版的上下文资源共享器 319 | """ 320 | 321 | _instances = {} 322 | 323 | def __new__(cls, name): 324 | 325 | if name in cls._instances: 326 | inst = cls._instances[name] 327 | else: 328 | inst = cls._instances[name] = super().__new__(cls) 329 | 330 | return inst 331 | 332 | def __init__(self, name): 333 | 334 | self._context_var = ContextVar(name, default=None) 335 | 336 | def get(self): 337 | 338 | ref = self._context_var.get() 339 | 340 | return None if ref is None else ref() 341 | 342 | def set(self, value): 343 | 344 | return self._context_var.set(weakref.ref(value)) 345 | 346 | 347 | class FutureWithTimeout(asyncio.Future): 348 | """带超时功能的Future 349 | """ 350 | 351 | def __init__(self, delay, default=None): 352 | 353 | super().__init__() 354 | 355 | self._timeout_handle = Utils.call_later( 356 | delay, 357 | self.set_result, 358 | default 359 | ) 360 | 361 | self.add_done_callback(self._clear_timeout) 362 | 363 | def _clear_timeout(self, *_): 364 | 365 | if self._timeout_handle is not None: 366 | self._timeout_handle.cancel() 367 | self._timeout_handle = None 368 | 369 | 370 | class FutureWithTask(asyncio.Future, RunnableInterface): 371 | """Future实例可以被多个协程await，本类实现Future和Task的桥接 372 | """ 373 | 374 | def __init__(self, func, name=None): 375 | 376 | super().__init__() 377 | 378 | self._name = Utils.uuid1() if name is None else name 379 | self._task = None 380 | self._callable = func 381 | self._build_time = Utils.loop_time() 382 | 383 | @property 384 | def name(self): 385 | 386 | return self._name 387 | 388 | @property 389 | def task(self): 390 | 391 | return self._task 392 | 393 | @property 394 | def build_time(self): 395 | 396 | return self._build_time 397 | 398 | def run(self): 399 | 400 | if self._task is None: 401 | self._task = asyncio.create_task(self._run()) 402 | 403 | return self 404 | 405 | async def _run(self): 406 | 407 | result = None 408 | 409 | try: 410 | 411 | result = await self._callable() 412 | 413 | self.set_result(result) 414 | 415 | except Exception as err: 416 | 417 | self.set_exception(err) 418 | 419 | return result 420 | 421 | 422 | class MultiTasks: 423 | """多任务并发管理器 424 | 425 | 提供协程的多任务并发的解决方案 426 | 427 | tasks = MultiTasks() 428 | tasks.append(func1()) 429 | tasks.append(func2()) 430 | ... 431 | tasks.append(funcN()) 432 | await tasks 433 | 434 | 多任务中禁止使用上下文资源共享的对象(如mysql和redis等) 435 | 同时需要注意类似这种不能同时为多个协程提供服务的对象会造成不可预期的问题 436 | 437 | """ 438 | 439 | def __init__(self, *args): 440 | 441 | self._coro_list = list(args) 442 | self._task_list = [] 443 | 444 | def __await__(self): 445 | 446 | if len(self._coro_list) > 0: 447 | self._task_list = [Utils.create_task(coro) for coro in self._coro_list] 448 | self._coro_list.clear() 449 | yield from asyncio.gather(*self._task_list).__await__() 450 | 451 | return [task.result() for task in self._task_list] 452 | 453 | def __len__(self): 454 | 455 | return self._coro_list.__len__() 456 | 457 | def __iter__(self): 458 | 459 | for task in self._task_list: 460 | yield task.result() 461 | 462 | def __getitem__(self, item): 463 | 464 | return self._task_list.__getitem__(item).result() 465 | 466 | def append(self, coro): 467 | 468 | return self._coro_list.append(coro) 469 | 470 | def extend(self, coro_list): 471 | 472 | return self._coro_list.extend(coro_list) 473 | 474 | def clear(self): 475 | 476 | self._coro_list.clear() 477 | self._task_list.clear() 478 | 479 | 480 | class SliceTasks(MultiTasks): 481 | """多任务分片并发管理器 482 | 483 | 继承自MultiTasks类，通过参数slice_num控制并发分片任务数 484 | 485 | """ 486 | 487 | def __init__(self, slice_num, *args): 488 | 489 | super().__init__(*args) 490 | 491 | self._slice_num = max(1, slice_num) 492 | 493 | def __await__(self): 494 | 495 | if len(self._coro_list) > 0: 496 | 497 | for _ in range(Utils.math.ceil(len(self._coro_list) / self._slice_num)): 498 | 499 | tasks = [] 500 | 501 | for _ in range(self._slice_num): 502 | if len(self._coro_list) > 0: 503 | tasks.append(Utils.create_task(self._coro_list.pop(0))) 504 | 505 | if len(tasks) > 0: 506 | self._task_list.extend(tasks) 507 | yield from asyncio.gather(*tasks).__await__() 508 | 509 | return [task.result() for task in self._task_list] 510 | 511 | 512 | class QueueTasks(MultiTasks): 513 | """多任务队列管理器 514 | 515 | 继承自MultiTasks类，通过参数queue_num控制队列长度 516 | 517 | """ 518 | 519 | def __init__(self, queue_num, *args): 520 | 521 | super().__init__(*args) 522 | 523 | self._queue_num = max(1, queue_num) 524 | 525 | self._queue_future = None 526 | 527 | def __await__(self): 528 | 529 | if len(self._coro_list) > 0: 530 | 531 | self._queue_future = asyncio.Future() 532 | 533 | tasks = [] 534 | 535 | for _ in range(self._queue_num): 536 | 537 | if len(self._coro_list) > 0: 538 | 539 | task = Utils.create_task(self._coro_list.pop(0)) 540 | task.add_done_callback(self._done_callback) 541 | 542 | tasks.append(task) 543 | 544 | if len(tasks) > 0: 545 | self._task_list.extend(tasks) 546 | 547 | yield from self._queue_future.__await__() 548 | 549 | return [task.result() for task in self._task_list] 550 | 551 | def _done_callback(self, _): 552 | 553 | if len(self._coro_list) > 0: 554 | 555 | task = Utils.create_task(self._coro_list.pop(0)) 556 | task.add_done_callback(self._done_callback) 557 | 558 | self._task_list.append(task) 559 | 560 | elif self._queue_future is not None and not self._queue_future.done(): 561 | 562 | if all(task.done() for task in self._task_list): 563 | self._queue_future.set_result(True) 564 | 565 | 566 | class AsyncCirculator: 567 | """异步循环器 568 | 569 | 提供一个循环体内的代码重复执行管理逻辑，可控制超时时间、执行间隔(LoopFrame)和最大执行次数 570 | 571 | async for index in AsyncCirculator(): 572 | pass 573 | 574 | 其中index为执行次数，从1开始 575 | 576 | """ 577 | 578 | def __init__(self, timeout=0, interval=0xff, max_times=0): 579 | 580 | if timeout > 0: 581 | self._expire_time = Utils.loop_time() + timeout 582 | else: 583 | self._expire_time = 0 584 | 585 | self._interval = interval 586 | self._max_times = max_times 587 | 588 | self._current = 0 589 | 590 | def __aiter__(self): 591 | 592 | return self 593 | 594 | async def __anext__(self): 595 | 596 | if self._current > 0: 597 | 598 | if (self._max_times > 0) and (self._max_times <= self._current): 599 | raise StopAsyncIteration() 600 | 601 | if (self._expire_time > 0) and (self._expire_time <= Utils.loop_time()): 602 | raise StopAsyncIteration() 603 | 604 | await self._sleep() 605 | 606 | self._current += 1 607 | 608 | return self._current 609 | 610 | async def _sleep(self): 611 | 612 | await Utils.wait_frame(self._interval) 613 | 614 | 615 | class AsyncCirculatorForSecond(AsyncCirculator): 616 | 617 | def __init__(self, timeout=0, interval=1, max_times=0): 618 | 619 | super().__init__(timeout, interval, max_times) 620 | 621 | async def _sleep(self): 622 | 623 | await Utils.sleep(self._interval) 624 | 625 | 626 | class AsyncContextManager: 627 | """异步上下文资源管理器 628 | 629 | 子类通过实现_context_release接口，方便的实现with语句管理上下文资源释放 630 | 631 | """ 632 | 633 | async def __aenter__(self): 634 | 635 | await self._context_initialize() 636 | 637 | return self 638 | 639 | async def __aexit__(self, exc_type, exc_value, traceback): 640 | 641 | await self._context_release() 642 | 643 | if exc_type is base.Ignore: 644 | 645 | return not exc_value.throw() 646 | 647 | elif exc_value: 648 | 649 | Utils.log.exception(exc_value) 650 | 651 | return True 652 | 653 | async def _context_initialize(self): 654 | 655 | pass 656 | 657 | async def _context_release(self): 658 | 659 | raise NotImplementedError() 660 | 661 | 662 | class FuncWrapper(base.FuncWrapper): 663 | """非阻塞异步函数包装器 664 | 665 | 将多个同步或异步函数包装成一个可调用对象 666 | 667 | """ 668 | 669 | def __call__(self, *args, **kwargs): 670 | 671 | for func in self._callables: 672 | Utils.call_soon(func, *args, **kwargs) 673 | 674 | 675 | class AsyncConstructor: 676 | """实现了__ainit__异步构造函数 677 | """ 678 | 679 | def __await__(self): 680 | 681 | yield from self.__ainit__().__await__() 682 | 683 | return self 684 | 685 | async def __ainit__(self): 686 | 687 | raise NotImplementedError() 688 | 689 | 690 | class AsyncFuncWrapper(base.FuncWrapper): 691 | """阻塞式异步函数包装器 692 | 693 | 将多个同步或异步函数包装成一个可调用对象 694 | 695 | """ 696 | 697 | async def __call__(self, *args, **kwargs): 698 | 699 | for func in self._callables: 700 | 701 | try: 702 | 703 | await Utils.awaitable_wrapper( 704 | func(*args, **kwargs) 705 | ) 706 | 707 | except Exception as err: 708 | 709 | Utils.log.error(err) 710 | 711 | 712 | class FuncCache: 713 | """函数缓存 714 | 715 | 使用堆栈缓存实现的函数缓存，在有效期内函数签名一致就会命中缓存 716 | 717 | """ 718 | 719 | def __init__(self, maxsize=0xff, ttl=10): 720 | 721 | self._cache = StackCache(maxsize, ttl) 722 | 723 | def __call__(self, func): 724 | 725 | @base.Utils.func_wraps(func) 726 | async def _wrapper(*args, **kwargs): 727 | 728 | func_sign = Utils.params_sign(func, *args, **kwargs) 729 | 730 | result = self._cache.get(func_sign) 731 | 732 | if result is None: 733 | 734 | result = await Utils.awaitable_wrapper( 735 | func(*args, **kwargs) 736 | ) 737 | 738 | self._cache.set(func_sign, result) 739 | 740 | return result 741 | 742 | return _wrapper 743 | 744 | 745 | class ShareFuture: 746 | """共享Future装饰器 747 | 748 | 同一时刻并发调用函数时，使用该装饰器的函数签名一致的调用，会共享计算结果 749 | 750 | """ 751 | 752 | def __init__(self): 753 | 754 | self._future = {} 755 | 756 | def __call__(self, func): 757 | 758 | @base.Utils.func_wraps(func) 759 | async def _wrapper(*args, **kwargs): 760 | 761 | future = None 762 | 763 | func_sign = Utils.params_sign(func, *args, **kwargs) 764 | 765 | if func_sign in self._future: 766 | 767 | future = asyncio.Future() 768 | 769 | self._future[func_sign].append(future) 770 | 771 | else: 772 | 773 | future = Utils.create_task( 774 | func(*args, **kwargs) 775 | ) 776 | 777 | if future is None: 778 | TypeError(r'Not Coroutine Object') 779 | 780 | self._future[func_sign] = [future] 781 | 782 | future.add_done_callback( 783 | Utils.func_partial(self._clear_future, func_sign) 784 | ) 785 | 786 | return await future 787 | 788 | return _wrapper 789 | 790 | def _clear_future(self, func_sign, _): 791 | 792 | if func_sign not in self._future: 793 | return 794 | 795 | futures = self._future.pop(func_sign) 796 | 797 | result = futures.pop(0).result() 798 | 799 | for future in futures: 800 | future.set_result(Utils.deepcopy(result)) 801 | 802 | 803 | class TimeDiff: 804 | 805 | def __init__(self): 806 | 807 | self._start_time = self._last_check_time = Utils.loop_time() 808 | 809 | def check(self): 810 | 811 | now_time = Utils.loop_time() 812 | 813 | diff_time_1 = now_time - self._start_time 814 | diff_time_2 = now_time - self._last_check_time 815 | 816 | self._last_check_time = now_time 817 | 818 | return diff_time_1, diff_time_2 819 | -------------------------------------------------------------------------------- /hagworm/extend/asyncio/buffer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from tempfile import TemporaryFile 4 | 5 | from .base import Utils 6 | from .task import IntervalTask 7 | 8 | from hagworm.extend.base import ContextManager 9 | 10 | 11 | class QueueBuffer: 12 | 13 | def __init__(self, maxsize, timeout=0): 14 | 15 | self._maxsize = maxsize 16 | 17 | self._timer = IntervalTask.create(timeout, False, self._handle_buffer) if timeout > 0 else None 18 | 19 | self._data_list = [] 20 | 21 | def _handle_buffer(self): 22 | 23 | if len(self._data_list) > 0: 24 | data_list, self._data_list = self._data_list, [] 25 | Utils.call_soon(self._run, data_list) 26 | 27 | async def _run(self, data_list): 28 | raise NotImplementedError() 29 | 30 | def append(self, data): 31 | 32 | self._data_list.append(data) 33 | 34 | if len(self._data_list) >= self._maxsize: 35 | self._handle_buffer() 36 | 37 | def extend(self, data_list): 38 | 39 | self._data_list.extend(data_list) 40 | 41 | if len(self._data_list) >= self._maxsize: 42 | self._handle_buffer() 43 | 44 | 45 | class FileBuffer(ContextManager): 46 | """文件缓存类 47 | """ 48 | 49 | def __init__(self, slice_size=0x1000000): 50 | 51 | self._buffers = [] 52 | 53 | self._slice_size = slice_size 54 | 55 | self._read_offset = 0 56 | 57 | self._append_buffer() 58 | 59 | def _context_release(self): 60 | 61 | self.close() 62 | 63 | def _append_buffer(self): 64 | 65 | self._buffers.append(TemporaryFile()) 66 | 67 | def close(self): 68 | 69 | while len(self._buffers) > 0: 70 | self._buffers.pop(0).close() 71 | 72 | self._read_offset = 0 73 | 74 | def write(self, data): 75 | 76 | buffer = self._buffers[-1] 77 | 78 | buffer.seek(0, 2) 79 | buffer.write(data) 80 | 81 | if buffer.tell() >= self._slice_size: 82 | buffer.flush() 83 | self._append_buffer() 84 | 85 | def read(self, size=None): 86 | 87 | buffer = self._buffers[0] 88 | 89 | buffer.seek(self._read_offset, 0) 90 | 91 | result = buffer.read(size) 92 | 93 | if len(result) == 0 and len(self._buffers) > 1: 94 | self._buffers.pop(0).close() 95 | self._read_offset = 0 96 | else: 97 | self._read_offset = buffer.tell() 98 | 99 | return result 100 | -------------------------------------------------------------------------------- /hagworm/extend/asyncio/database.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import asyncio 4 | import aiomysql 5 | 6 | from aiomysql.sa import SAConnection, Engine 7 | from aiomysql.sa.engine import _dialect as dialect 8 | 9 | from pymysql.err import Warning, DataError, IntegrityError, ProgrammingError 10 | 11 | from sqlalchemy.sql.selectable import Select 12 | from sqlalchemy.sql.dml import Insert, Update, Delete 13 | 14 | from motor.motor_asyncio import AsyncIOMotorClient 15 | 16 | from hagworm.extend.error import MySQLReadOnlyError 17 | 18 | from .base import Utils, WeakContextVar, AsyncContextManager, AsyncCirculator 19 | 20 | 21 | MONGO_POLL_WATER_LEVEL_WARNING_LINE = 0x08 22 | 23 | MYSQL_ERROR_RETRY_COUNT = 0x1f 24 | MYSQL_POLL_WATER_LEVEL_WARNING_LINE = 0x08 25 | 26 | 27 | class MongoPool: 28 | """Mongo连接管理 29 | """ 30 | 31 | def __init__( 32 | self, host, username=None, password=None, 33 | *, name=None, min_pool_size=8, max_pool_size=32, max_idle_time=3600, wait_queue_timeout=10, 34 | compressors=r'zlib', zlib_compression_level=6, 35 | **settings 36 | ): 37 | 38 | self._name = name if name is not None else Utils.uuid1()[:8] 39 | 40 | settings[r'host'] = host 41 | settings[r'minPoolSize'] = min_pool_size 42 | settings[r'maxPoolSize'] = max_pool_size 43 | settings[r'maxIdleTimeMS'] = max_idle_time * 1000 44 | settings[r'waitQueueTimeoutMS'] = wait_queue_timeout * 1000 45 | settings[r'compressors'] = compressors 46 | settings[r'zlibCompressionLevel'] = zlib_compression_level 47 | 48 | if username and password: 49 | settings[r'username'] = username 50 | settings[r'password'] = password 51 | 52 | self._pool = AsyncIOMotorClient(**settings) 53 | 54 | for server in self._servers.values(): 55 | server.pool.remove_stale_sockets() 56 | 57 | Utils.log.info( 58 | f"Mongo {host} ({self._name}) initialized: {self._pool.min_pool_size}/{self._pool.max_pool_size}" 59 | ) 60 | 61 | @property 62 | def _servers(self): 63 | 64 | return self._pool.delegate._topology._servers 65 | 66 | def _echo_pool_info(self): 67 | 68 | global MONGO_POLL_WATER_LEVEL_WARNING_LINE 69 | 70 | for address, server in self._servers.items(): 71 | 72 | poll_size = len(server.pool.sockets) + server.pool.active_sockets 73 | 74 | if (self._pool.max_pool_size - poll_size) < MONGO_POLL_WATER_LEVEL_WARNING_LINE: 75 | Utils.log.warning( 76 | f'Mongo connection pool not enough ({self._name}){address}: ' 77 | f'{poll_size}/{self._pool.max_pool_size}' 78 | ) 79 | else: 80 | Utils.log.debug( 81 | f'Mongo connection pool info ({self._name}){address}: ' 82 | f'{poll_size}/{self._pool.max_pool_size}' 83 | ) 84 | 85 | def reset(self): 86 | 87 | for address, server in self._servers.items(): 88 | 89 | server.pool.reset() 90 | server.pool.remove_stale_sockets() 91 | 92 | Utils.log.info( 93 | f'Mongo connection pool reset {address}: {len(server.pool.sockets)}/{self._pool.max_pool_size}' 94 | ) 95 | 96 | def get_database(self, db_name): 97 | 98 | self._echo_pool_info() 99 | 100 | result = None 101 | 102 | try: 103 | result = self._pool[db_name] 104 | except Exception as err: 105 | Utils.log.exception(err) 106 | 107 | return result 108 | 109 | 110 | class MongoDelegate: 111 | """Mongo功能组件 112 | """ 113 | 114 | def __init__(self, *args, **kwargs): 115 | 116 | self._mongo_pool = MongoPool(*args, **kwargs) 117 | 118 | @property 119 | def mongo_pool(self): 120 | 121 | return self._mongo_pool 122 | 123 | async def mongo_health(self): 124 | 125 | result = False 126 | 127 | try: 128 | result = bool(await self._mongo_pool._pool.server_info()) 129 | except Exception as err: 130 | Utils.log.error(err) 131 | 132 | return result 133 | 134 | def reset_mongo_pool(self): 135 | 136 | self._mongo_pool.reset() 137 | 138 | def get_mongo_database(self, db_name): 139 | 140 | return self._mongo_pool.get_database(db_name) 141 | 142 | def get_mongo_collection(self, db_name, collection): 143 | 144 | return self.get_mongo_database(db_name)[collection] 145 | 146 | 147 | class MySQLPool: 148 | """MySQL连接管理 149 | """ 150 | 151 | class _Connection(SAConnection): 152 | 153 | def __init__(self, connection, engine, compiled_cache=None): 154 | 155 | super().__init__(connection, engine, compiled_cache) 156 | 157 | if not hasattr(connection, r'build_time'): 158 | setattr(connection, r'build_time', Utils.loop_time()) 159 | 160 | @property 161 | def build_time(self): 162 | 163 | return getattr(self._connection, r'build_time', 0) 164 | 165 | async def destroy(self): 166 | 167 | if self._connection is None: 168 | return 169 | 170 | if self._transaction is not None: 171 | await self._transaction.rollback() 172 | self._transaction = None 173 | 174 | self._connection.close() 175 | 176 | self._engine.release(self) 177 | self._connection = None 178 | self._engine = None 179 | 180 | def __init__( 181 | self, host, port, db, user, password, 182 | *, name=None, minsize=8, maxsize=32, echo=False, pool_recycle=21600, 183 | charset=r'utf8', autocommit=True, cursorclass=aiomysql.DictCursor, 184 | readonly=False, conn_life=43200, 185 | **settings 186 | ): 187 | 188 | self._name = name if name is not None else (Utils.uuid1()[:8] + (r'_ro' if readonly else r'_rw')) 189 | self._pool = None 190 | self._engine = None 191 | self._readonly = readonly 192 | self._conn_life = conn_life 193 | 194 | self._settings = settings 195 | 196 | self._settings[r'host'] = host 197 | self._settings[r'port'] = port 198 | self._settings[r'db'] = db 199 | 200 | self._settings[r'user'] = user 201 | self._settings[r'password'] = password 202 | 203 | self._settings[r'minsize'] = minsize 204 | self._settings[r'maxsize'] = maxsize 205 | 206 | self._settings[r'echo'] = echo 207 | self._settings[r'pool_recycle'] = pool_recycle 208 | self._settings[r'charset'] = charset 209 | self._settings[r'autocommit'] = autocommit 210 | self._settings[r'cursorclass'] = cursorclass 211 | 212 | @property 213 | def name(self): 214 | 215 | return self._name 216 | 217 | @property 218 | def readonly(self): 219 | 220 | return self._readonly 221 | 222 | @property 223 | def conn_life(self): 224 | 225 | return self._conn_life 226 | 227 | def __await__(self): 228 | 229 | self._pool = yield from aiomysql.create_pool(**self._settings).__await__() 230 | self._engine = Engine(dialect, self._pool) 231 | 232 | Utils.log.info( 233 | f"MySQL {self._settings[r'host']}:{self._settings[r'port']} {self._settings[r'db']}" 234 | f" ({self._name}) initialized: {self._pool.size}/{self._pool.maxsize}" 235 | ) 236 | 237 | return self 238 | 239 | def _echo_pool_info(self): 240 | 241 | global MYSQL_POLL_WATER_LEVEL_WARNING_LINE 242 | 243 | if (self._pool.maxsize - self._pool.size + self._pool.freesize) < MYSQL_POLL_WATER_LEVEL_WARNING_LINE: 244 | Utils.log.warning( 245 | f'MySQL connection pool not enough ({self._name}): ' 246 | f'{self._pool.freesize}({self._pool.size}/{self._pool.maxsize})' 247 | ) 248 | else: 249 | Utils.log.debug( 250 | f'MySQL connection pool info ({self._name}): ' 251 | f'{self._pool.freesize}({self._pool.size}/{self._pool.maxsize})' 252 | ) 253 | 254 | async def health(self): 255 | 256 | result = False 257 | 258 | async with self.get_client() as client: 259 | 260 | proxy = await client.execute(r'select version();') 261 | await proxy.close() 262 | 263 | result = True 264 | 265 | return result 266 | 267 | async def reset(self): 268 | 269 | if self._pool is not None: 270 | 271 | await self._pool.clear() 272 | 273 | await self.health() 274 | 275 | Utils.log.info( 276 | f'MySQL connection pool reset ({self._name}): {self._pool.size}/{self._pool.maxsize}' 277 | ) 278 | 279 | async def get_sa_conn(self): 280 | 281 | self._echo_pool_info() 282 | 283 | conn = await self._pool.acquire() 284 | 285 | return self._Connection(conn, self._engine) 286 | 287 | def get_client(self): 288 | 289 | result = None 290 | 291 | try: 292 | result = DBClient(self) 293 | except Exception as err: 294 | Utils.log.exception(err) 295 | 296 | return result 297 | 298 | def get_transaction(self): 299 | 300 | result = None 301 | 302 | if self._readonly: 303 | raise MySQLReadOnlyError() 304 | 305 | try: 306 | result = DBTransaction(self) 307 | except Exception as err: 308 | Utils.log.exception(err) 309 | 310 | return result 311 | 312 | 313 | class MySQLDelegate: 314 | """MySQL功能组件 315 | """ 316 | 317 | def __init__(self): 318 | 319 | self._mysql_rw_pool = None 320 | self._mysql_ro_pool = None 321 | 322 | context_uuid = Utils.uuid1() 323 | 324 | self._mysql_rw_client_context = WeakContextVar(f'mysql_rw_client_{context_uuid}') 325 | self._mysql_ro_client_context = WeakContextVar(f'mysql_ro_client_{context_uuid}') 326 | 327 | @property 328 | def mysql_rw_pool(self): 329 | 330 | return self._mysql_rw_pool 331 | 332 | @property 333 | def mysql_ro_pool(self): 334 | 335 | return self._mysql_ro_pool 336 | 337 | async def async_init_mysql_rw(self, *args, **kwargs): 338 | 339 | self._mysql_rw_pool = await MySQLPool(*args, **kwargs) 340 | 341 | async def async_init_mysql_ro(self, *args, **kwargs): 342 | 343 | self._mysql_ro_pool = await MySQLPool(*args, **kwargs) 344 | 345 | async def mysql_health(self): 346 | 347 | result = await self._mysql_rw_pool.health() if self._mysql_rw_pool else True 348 | result &= await self._mysql_ro_pool.health() if self._mysql_ro_pool else True 349 | 350 | return result 351 | 352 | async def reset_mysql_pool(self): 353 | 354 | if self._mysql_rw_pool: 355 | await self._mysql_rw_pool.reset() 356 | 357 | if self._mysql_ro_pool: 358 | await self._mysql_ro_pool.reset() 359 | 360 | def get_db_client(self, readonly=False, *, alone=False): 361 | 362 | client = None 363 | 364 | if alone: 365 | 366 | if readonly: 367 | if self._mysql_ro_pool is not None: 368 | client = self._mysql_ro_pool.get_client() 369 | else: 370 | client = self._mysql_rw_pool.get_client() 371 | client._readonly = True 372 | else: 373 | client = self._mysql_rw_pool.get_client() 374 | 375 | else: 376 | 377 | if readonly: 378 | 379 | _client = self._mysql_rw_client_context.get() 380 | 381 | if _client is not None: 382 | Utils.create_task(_client.release()) 383 | 384 | client = self._mysql_ro_client_context.get() 385 | 386 | if client is None: 387 | 388 | if self._mysql_ro_pool is not None: 389 | client = self._mysql_ro_pool.get_client() 390 | else: 391 | client = self._mysql_rw_pool.get_client() 392 | client._readonly = True 393 | 394 | self._mysql_ro_client_context.set(client) 395 | 396 | else: 397 | 398 | _client = self._mysql_ro_client_context.get() 399 | 400 | if _client is not None: 401 | Utils.create_task(_client.release()) 402 | 403 | client = self._mysql_rw_client_context.get() 404 | 405 | if client is None: 406 | client = self._mysql_rw_pool.get_client() 407 | self._mysql_rw_client_context.set(client) 408 | 409 | return client 410 | 411 | def get_db_transaction(self): 412 | 413 | _client = self._mysql_rw_client_context.get() 414 | 415 | if _client is not None: 416 | Utils.create_task(_client.release()) 417 | 418 | _client = self._mysql_ro_client_context.get() 419 | 420 | if _client is not None: 421 | Utils.create_task(_client.release()) 422 | 423 | return self._mysql_rw_pool.get_transaction() 424 | 425 | 426 | class _ClientBase: 427 | """MySQL客户端基类 428 | """ 429 | 430 | @staticmethod 431 | def safestr(val): 432 | 433 | cls = type(val) 434 | 435 | if cls is str: 436 | val = aiomysql.escape_string(val) 437 | elif cls is dict: 438 | val = aiomysql.escape_dict(val) 439 | else: 440 | val = str(val) 441 | 442 | return val 443 | 444 | def __init__(self, readonly=False): 445 | 446 | self._readonly = readonly 447 | 448 | @property 449 | def readonly(self): 450 | 451 | return self._readonly 452 | 453 | @property 454 | def insert_id(self): 455 | 456 | raise NotImplementedError() 457 | 458 | def _get_conn(self): 459 | 460 | raise NotImplementedError() 461 | 462 | async def execute(self, clause): 463 | 464 | raise NotImplementedError() 465 | 466 | async def select(self, clause): 467 | 468 | result = [] 469 | 470 | if not isinstance(clause, Select): 471 | raise TypeError(r'Not sqlalchemy.sql.selectable.Select object') 472 | 473 | proxy = await self.execute(clause) 474 | 475 | if proxy is not None: 476 | 477 | records = await proxy.cursor.fetchall() 478 | 479 | if records: 480 | result.extend(records) 481 | 482 | if not proxy.closed: 483 | await proxy.close() 484 | 485 | return result 486 | 487 | async def find(self, clause): 488 | 489 | result = None 490 | 491 | if not isinstance(clause, Select): 492 | raise TypeError(r'Not sqlalchemy.sql.selectable.Select object') 493 | 494 | proxy = await self.execute(clause) 495 | 496 | if proxy is not None: 497 | 498 | record = await proxy.cursor.fetchone() 499 | 500 | if record: 501 | result = record 502 | 503 | if not proxy.closed: 504 | await proxy.close() 505 | 506 | return result 507 | 508 | async def insert(self, clause): 509 | 510 | result = 0 511 | 512 | if self._readonly: 513 | raise MySQLReadOnlyError() 514 | 515 | if not isinstance(clause, Insert): 516 | raise TypeError(r'Not sqlalchemy.sql.dml.Insert object') 517 | 518 | proxy = await self.execute(clause) 519 | 520 | if proxy is not None: 521 | 522 | result = self.insert_id 523 | 524 | if not proxy.closed: 525 | await proxy.close() 526 | 527 | return result 528 | 529 | async def update(self, clause): 530 | 531 | result = 0 532 | 533 | if self._readonly: 534 | raise MySQLReadOnlyError() 535 | 536 | if not isinstance(clause, Update): 537 | raise TypeError(r'Not sqlalchemy.sql.dml.Update object') 538 | 539 | proxy = await self.execute(clause) 540 | 541 | if proxy is not None: 542 | 543 | result = proxy.rowcount 544 | 545 | if not proxy.closed: 546 | await proxy.close() 547 | 548 | return result 549 | 550 | async def delete(self, clause): 551 | 552 | result = 0 553 | 554 | if self._readonly: 555 | raise MySQLReadOnlyError() 556 | 557 | if not isinstance(clause, Delete): 558 | raise TypeError(r'Not sqlalchemy.sql.dml.Delete object') 559 | 560 | proxy = await self.execute(clause) 561 | 562 | if proxy is not None: 563 | 564 | result = proxy.rowcount 565 | 566 | if not proxy.closed: 567 | await proxy.close() 568 | 569 | return result 570 | 571 | 572 | class DBClient(_ClientBase, AsyncContextManager): 573 | """MySQL客户端对象，使用with进行上下文管理 574 | 575 | 将连接委托给客户端对象管理，提高了整体连接的使用率 576 | 577 | """ 578 | 579 | def __init__(self, pool): 580 | 581 | super().__init__(pool.readonly) 582 | 583 | self._lock = asyncio.Lock() 584 | 585 | self._pool = pool 586 | self._conn = None 587 | 588 | @property 589 | def insert_id(self): 590 | 591 | return self._conn.connection.insert_id() 592 | 593 | async def _get_conn(self): 594 | 595 | if self._conn is None and self._pool: 596 | self._conn = await self._pool.get_sa_conn() 597 | 598 | return self._conn 599 | 600 | async def _close_conn(self, discard=False): 601 | 602 | if self._conn is not None: 603 | 604 | _conn, self._conn = self._conn, None 605 | 606 | if discard: 607 | await _conn.destroy() 608 | elif (Utils.loop_time() - _conn.build_time) > self._pool.conn_life: 609 | await _conn.destroy() 610 | else: 611 | await _conn.close() 612 | 613 | async def _context_release(self): 614 | 615 | await self._close_conn(self._lock.locked()) 616 | 617 | async def release(self): 618 | 619 | async with self._lock: 620 | 621 | await self._close_conn() 622 | 623 | async def execute(self, clause): 624 | 625 | global MYSQL_ERROR_RETRY_COUNT 626 | 627 | result = None 628 | 629 | async with self._lock: 630 | 631 | async for times in AsyncCirculator(max_times=MYSQL_ERROR_RETRY_COUNT): 632 | 633 | try: 634 | 635 | conn = await self._get_conn() 636 | 637 | result = await conn.execute(clause) 638 | 639 | except (Warning, DataError, IntegrityError, ProgrammingError) as err: 640 | 641 | await self._close_conn(True) 642 | 643 | raise err 644 | 645 | except Exception as err: 646 | 647 | await self._close_conn(True) 648 | 649 | if times < MYSQL_ERROR_RETRY_COUNT: 650 | Utils.log.exception(err) 651 | else: 652 | raise err 653 | 654 | else: 655 | 656 | break 657 | 658 | return result 659 | 660 | 661 | class DBTransaction(DBClient): 662 | """MySQL客户端事务对象，使用with进行上下文管理 663 | 664 | 将连接委托给客户端对象管理，提高了整体连接的使用率 665 | 666 | """ 667 | 668 | def __init__(self, pool): 669 | 670 | super().__init__(pool) 671 | 672 | self._trx = None 673 | 674 | async def _get_conn(self): 675 | 676 | if self._conn is None and self._pool: 677 | self._conn = await self._pool.get_sa_conn() 678 | self._trx = await self._conn.begin() 679 | 680 | return self._conn 681 | 682 | async def _context_release(self): 683 | 684 | await self.rollback() 685 | 686 | async def release(self): 687 | 688 | await self.rollback() 689 | 690 | async def execute(self, clause): 691 | 692 | result = None 693 | 694 | if self._readonly: 695 | raise MySQLReadOnlyError() 696 | 697 | async with self._lock: 698 | 699 | try: 700 | 701 | conn = await self._get_conn() 702 | 703 | result = await conn.execute(clause) 704 | 705 | except Exception as err: 706 | 707 | await self._close_conn(True) 708 | 709 | raise err 710 | 711 | return result 712 | 713 | async def commit(self): 714 | 715 | async with self._lock: 716 | 717 | if self._trx: 718 | _trx, self._trx = self._trx, None 719 | await _trx.commit() 720 | await _trx.close() 721 | 722 | await self._close_conn() 723 | 724 | async def rollback(self): 725 | 726 | async with self._lock: 727 | 728 | if self._trx: 729 | _trx, self._trx = self._trx, None 730 | await _trx.rollback() 731 | await _trx.close() 732 | 733 | await self._close_conn() 734 | -------------------------------------------------------------------------------- /hagworm/extend/asyncio/event.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from aioredis.pubsub import Receiver 4 | 5 | from hagworm.extend.event import EventDispatcher 6 | from hagworm.extend.asyncio.base import Utils, AsyncCirculatorForSecond, FuncWrapper, FutureWithTimeout 7 | 8 | 9 | class DistributedEvent(EventDispatcher): 10 | """Redis实现的消息广播总线 11 | """ 12 | 13 | def __init__(self, redis_pool, channel_name, channel_count): 14 | 15 | super().__init__() 16 | 17 | self._redis_pool = redis_pool 18 | 19 | self._channels = [f'event_bus_{Utils.md5_u32(channel_name)}_{index}' for index in range(channel_count)] 20 | 21 | for channel in self._channels: 22 | Utils.create_task(self._event_listener(channel)) 23 | 24 | async def _event_listener(self, channel): 25 | 26 | async for _ in AsyncCirculatorForSecond(): 27 | 28 | async with self._redis_pool.get_client() as cache: 29 | 30 | receiver, = await cache.subscribe(channel) 31 | 32 | Utils.log.info(f'event bus channel({channel}) receiver created') 33 | 34 | async for message in receiver.iter(): 35 | await self._event_assigner(channel, message) 36 | 37 | async def _event_assigner(self, channel, message): 38 | 39 | message = Utils.pickle_loads(message) 40 | 41 | Utils.log.debug(f'event handling => channel({channel}) message({message})') 42 | 43 | _type = message.get(r'type', r'') 44 | args = message.get(r'args', []) 45 | kwargs = message.get(r'kwargs', {}) 46 | 47 | if _type in self._observers: 48 | self._observers[_type](*args, **kwargs) 49 | 50 | def _gen_observer(self): 51 | 52 | return FuncWrapper() 53 | 54 | async def dispatch(self, _type, *args, **kwargs): 55 | 56 | channel = self._channels[Utils.md5_u32(_type) % len(self._channels)] 57 | 58 | message = { 59 | r'type': _type, 60 | r'args': args, 61 | r'kwargs': kwargs, 62 | } 63 | 64 | Utils.log.debug(f'event dispatch => channel({channel}) message({message})') 65 | 66 | async with self._redis_pool.get_client() as cache: 67 | await cache.publish(channel, Utils.pickle_dumps(message)) 68 | 69 | def gen_event_waiter(self, event_type, delay_time): 70 | 71 | return EventWaiter(self, event_type, delay_time) 72 | 73 | 74 | class EventWaiter(FutureWithTimeout): 75 | """带超时的临时消息接收器 76 | """ 77 | 78 | def __init__(self, dispatcher, event_type, delay_time): 79 | 80 | super().__init__(delay_time) 81 | 82 | self._dispatcher = dispatcher 83 | self._event_type = event_type 84 | 85 | self._dispatcher.add_listener(self._event_type, self._event_handler) 86 | 87 | def set_result(self, result): 88 | 89 | if self.done(): 90 | return 91 | 92 | super().set_result(result) 93 | 94 | self._dispatcher.remove_listener(self._event_type, self._event_handler) 95 | 96 | def _event_handler(self, *args, **kwargs): 97 | 98 | if not self.done(): 99 | self.set_result({r'args': args, r'kwargs': kwargs}) 100 | -------------------------------------------------------------------------------- /hagworm/extend/asyncio/file.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from hagworm.extend.base import Utils, StackCache 4 | 5 | from hagworm.extend.asyncio.net import HTTPClientPool 6 | from hagworm.extend.asyncio.future import ThreadPool 7 | 8 | 9 | class FileLoader: 10 | """带缓存的网络文件加载器 11 | """ 12 | 13 | def __init__(self, maxsize=0xff, ttl=3600, thread=32): 14 | 15 | self._cache = StackCache(maxsize, ttl) 16 | 17 | self._thread_pool = ThreadPool(thread) 18 | self._http_client = HTTPClientPool(limit=thread) 19 | 20 | def _read(self, file): 21 | 22 | with open(file, r'rb') as stream: 23 | return stream.read() 24 | 25 | async def read(self, file): 26 | 27 | result = None 28 | 29 | try: 30 | 31 | if self._cache.exists(file): 32 | 33 | result = self._cache.get(file) 34 | 35 | else: 36 | 37 | result = await self._thread_pool.run(self._read, file) 38 | 39 | self._cache.set(file, result) 40 | 41 | except Exception as err: 42 | 43 | Utils.log.error(err) 44 | 45 | return result 46 | 47 | async def fetch(self, url, params=None, cookies=None, headers=None): 48 | 49 | result = None 50 | 51 | try: 52 | 53 | if self._cache.exists(url): 54 | 55 | result = self._cache.get(url) 56 | 57 | else: 58 | 59 | result = await self._http_client.get(url, params, cookies, headers) 60 | 61 | self._cache.set(url, result) 62 | 63 | except Exception as err: 64 | 65 | Utils.log.error(err) 66 | 67 | return result 68 | -------------------------------------------------------------------------------- /hagworm/extend/asyncio/future.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import asyncio 4 | 5 | from concurrent.futures import ThreadPoolExecutor 6 | from multiprocessing import Lock, Manager 7 | from contextlib import contextmanager 8 | 9 | from hagworm.extend.interface import RunnableInterface, TaskInterface 10 | 11 | from .base import Utils 12 | 13 | 14 | class ThreadPool(RunnableInterface): 15 | """线程池，桥接线程与协程 16 | """ 17 | 18 | def __init__(self, max_workers=None): 19 | 20 | self._executor = ThreadPoolExecutor(max_workers) 21 | 22 | async def run(self, _callable, *args, **kwargs): 23 | """线程转协程，不支持协程函数 24 | """ 25 | 26 | loop = asyncio.events.get_event_loop() 27 | 28 | if kwargs: 29 | 30 | return await loop.run_in_executor( 31 | self._executor, 32 | Utils.func_partial( 33 | _callable, 34 | *args, 35 | **kwargs 36 | ) 37 | ) 38 | 39 | else: 40 | 41 | return await loop.run_in_executor( 42 | self._executor, 43 | _callable, 44 | *args, 45 | ) 46 | 47 | 48 | class ThreadWorker: 49 | """通过线程转协程实现普通函数非阻塞的装饰器 50 | """ 51 | 52 | def __init__(self, max_workers=None): 53 | 54 | self._thread_pool = ThreadPool(max_workers) 55 | 56 | def __call__(self, func): 57 | 58 | @Utils.func_wraps(func) 59 | def _wrapper(*args, **kwargs): 60 | return self._thread_pool.run(func, *args, **kwargs) 61 | 62 | return _wrapper 63 | 64 | 65 | class SubProcess(TaskInterface): 66 | """子进程管理，通过command方式启动子进程 67 | """ 68 | 69 | @classmethod 70 | async def create(cls, program, *args, stdin=None, stdout=None, stderr=None, **kwargs): 71 | 72 | inst = cls(program, *args, stdin, stdout, stderr, **kwargs) 73 | await inst.start() 74 | 75 | return inst 76 | 77 | def __init__(self, program, *args, stdin=None, stdout=None, stderr=None, **kwargs): 78 | 79 | self._program = program 80 | self._args = args 81 | self._kwargs = kwargs 82 | 83 | self._stdin = asyncio.subprocess.DEVNULL if stdin is None else stdin 84 | self._stdout = asyncio.subprocess.DEVNULL if stdout is None else stdout 85 | self._stderr = asyncio.subprocess.DEVNULL if stderr is None else stderr 86 | 87 | self._process = None 88 | self._process_id = None 89 | 90 | @property 91 | def pid(self): 92 | 93 | return self._process_id 94 | 95 | @property 96 | def process(self): 97 | 98 | return self._process 99 | 100 | @property 101 | def stdin(self): 102 | 103 | return self._process.stdin if self._process is not None else None 104 | 105 | @property 106 | def stdout(self): 107 | 108 | return self._process.stdout if self._process is not None else None 109 | 110 | @property 111 | def stderr(self): 112 | 113 | return self._process.stderr if self._process is not None else None 114 | 115 | def is_running(self): 116 | 117 | return self._process is not None and self._process.returncode is None 118 | 119 | async def start(self): 120 | 121 | if self.is_running(): 122 | return False 123 | 124 | self._process = await asyncio.create_subprocess_exec( 125 | self._program, *self._args, 126 | stdin=self._stdin, 127 | stdout=self._stdout, 128 | stderr=self._stderr, 129 | **self._kwargs 130 | ) 131 | 132 | self._process_id = self._process.pid 133 | 134 | return True 135 | 136 | async def stop(self): 137 | 138 | if not self.is_running(): 139 | return False 140 | 141 | self._process.kill() 142 | await self._process.wait() 143 | 144 | return True 145 | 146 | def kill(self): 147 | 148 | if not self.is_running(): 149 | return False 150 | 151 | self._process.kill() 152 | 153 | return True 154 | 155 | async def wait(self, timeout=None): 156 | 157 | if not self.is_running(): 158 | return 159 | 160 | try: 161 | await asyncio.wait_for(self._process.wait(), timeout=timeout) 162 | except Exception as err: 163 | Utils.log.error(err) 164 | finally: 165 | await self.stop() 166 | 167 | 168 | class ProcessSyncDict: 169 | 170 | def __init__(self, lock: Lock = None, manager: Manager = None): 171 | 172 | self._lock = lock if lock is not None else Lock() 173 | self._dict = manager.dict() if manager is not None else Manager().dict() 174 | 175 | @contextmanager 176 | def _locked(self): 177 | 178 | self._lock.acquire() 179 | 180 | try: 181 | yield 182 | except Exception as err: 183 | Utils.log.error(err) 184 | finally: 185 | self._lock.release() 186 | 187 | def __contains__(self, *args, **kwargs): 188 | 189 | result = None 190 | 191 | with self._locked(): 192 | result = self._dict.__contains__(*args, **kwargs) 193 | 194 | return result 195 | 196 | def __delitem__(self, *args, **kwargs): 197 | 198 | with self._locked(): 199 | self._dict.__delitem__(*args, **kwargs) 200 | 201 | def __getitem__(self, *args, **kwargs): 202 | 203 | result = None 204 | 205 | with self._locked(): 206 | result = self._dict.__getitem__(*args, **kwargs) 207 | 208 | return result 209 | 210 | def __setitem__(self, *args, **kwargs): 211 | 212 | with self._locked(): 213 | self._dict.__setitem__(*args, **kwargs) 214 | 215 | def __repr__(self, *args, **kwargs): 216 | 217 | result = None 218 | 219 | with self._locked(): 220 | result = self._dict.__repr__(*args, **kwargs) 221 | 222 | return result 223 | 224 | def __len__(self, *args, **kwargs): 225 | 226 | result = None 227 | 228 | with self._locked(): 229 | result = self._dict.__len__(*args, **kwargs) 230 | 231 | return result 232 | 233 | def __sizeof__(self): 234 | 235 | result = None 236 | 237 | with self._locked(): 238 | result = self._dict.__sizeof__() 239 | 240 | return result 241 | 242 | def clear(self): 243 | 244 | with self._locked(): 245 | self._dict.clear() 246 | 247 | def copy(self): 248 | 249 | result = None 250 | 251 | with self._locked(): 252 | result = self._dict.copy() 253 | 254 | return result 255 | 256 | def get(self, *args, **kwargs): 257 | 258 | result = None 259 | 260 | with self._locked(): 261 | result = self._dict.get(*args, **kwargs) 262 | 263 | return result 264 | 265 | def pop(self, k, d=None): 266 | 267 | result = None 268 | 269 | with self._locked(): 270 | result = self._dict.pop(k, d) 271 | 272 | return result 273 | 274 | def popitem(self): 275 | 276 | result = None 277 | 278 | with self._locked(): 279 | result = self._dict.popitem() 280 | 281 | return result 282 | 283 | def setdefault(self, *args, **kwargs): 284 | 285 | result = None 286 | 287 | with self._locked(): 288 | result = self._dict.setdefault(*args, **kwargs) 289 | 290 | return result 291 | 292 | def update(self, E=None, **F): 293 | 294 | with self._locked(): 295 | self._dict.update(E, **F) 296 | -------------------------------------------------------------------------------- /hagworm/extend/asyncio/net.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | import ssl 5 | import aiohttp 6 | 7 | from enum import Enum 8 | 9 | from hagworm.extend.base import ContextManager 10 | from hagworm.extend.asyncio.base import AsyncCirculatorForSecond 11 | from hagworm.extend.asyncio.buffer import FileBuffer 12 | 13 | from .base import Utils 14 | 15 | 16 | class STATE(Enum): 17 | 18 | PENDING = 0x00 19 | FETCHING = 0x01 20 | SUCCESS = 0x02 21 | FAILURE = 0x03 22 | 23 | 24 | DEFAULT_TIMEOUT = aiohttp.client.ClientTimeout(total=60, connect=10, sock_read=60, sock_connect=10) 25 | DOWNLOAD_TIMEOUT = aiohttp.client.ClientTimeout(total=600, connect=10, sock_read=600, sock_connect=10) 26 | 27 | _CA_FILE = os.path.join( 28 | os.path.split(os.path.abspath(__file__))[0], 29 | r'../../static/cacert.pem' 30 | ) 31 | 32 | 33 | def _json_decoder(val, **kwargs): 34 | 35 | try: 36 | return Utils.json_decode(val, **kwargs) 37 | except Exception as err: 38 | Utils.log.error(f'http client json decode error: {err} => {val}') 39 | 40 | 41 | def create_default_context(): 42 | return ssl.create_default_context(cafile=_CA_FILE) 43 | 44 | 45 | class Result(dict): 46 | 47 | def __init__(self, status, headers, body): 48 | 49 | super().__init__(status=status, headers=headers, body=body) 50 | 51 | def __bool__(self): 52 | 53 | return (self.status >= 200) and (self.status <= 299) 54 | 55 | @property 56 | def status(self): 57 | 58 | return self.get(r'status') 59 | 60 | @property 61 | def headers(self): 62 | 63 | return self.get(r'headers') 64 | 65 | @property 66 | def body(self): 67 | 68 | return self.get(r'body') 69 | 70 | 71 | class _HTTPClient: 72 | """HTTP客户端基类 73 | """ 74 | 75 | def __init__(self, retry_count=5, timeout=None, **kwargs): 76 | 77 | global DEFAULT_TIMEOUT 78 | 79 | self._ssl_context = create_default_context() 80 | 81 | self._retry_count = retry_count 82 | 83 | self._session_config = kwargs 84 | self._session_config[r'timeout'] = timeout if timeout is not None else DEFAULT_TIMEOUT 85 | self._session_config.setdefault(r'raise_for_status', True) 86 | 87 | async def _handle_response(self, response): 88 | 89 | return await response.read() 90 | 91 | def timeout(self, *, total=None, connect=None, sock_read=None, sock_connect=None): 92 | """生成超时配置对象 93 | 94 | Args: 95 | total: 总超时时间 96 | connect: 从连接池中等待获取连接的超时时间 97 | sock_read: Socket数据接收的超时时间 98 | sock_connect: Socket连接的超时时间 99 | 100 | """ 101 | 102 | return aiohttp.client.ClientTimeout( 103 | total=total, connect=connect, 104 | sock_read=sock_read, sock_connect=sock_connect 105 | ) 106 | 107 | async def send_request(self, method, url, data=None, params=None, cookies=None, headers=None, **settings) -> Result: 108 | 109 | response = None 110 | 111 | if headers is None: 112 | headers = {} 113 | 114 | if isinstance(data, dict): 115 | headers.setdefault( 116 | r'Content-Type', 117 | r'application/x-www-form-urlencoded' 118 | ) 119 | 120 | settings[r'data'] = data 121 | settings[r'params'] = params 122 | settings[r'cookies'] = cookies 123 | settings[r'headers'] = headers 124 | 125 | Utils.log.debug( 126 | r'{0} {1} => {2}'.format( 127 | method, 128 | url, 129 | str({key: val for key, val in settings.items() if isinstance(val, (str, list, dict))}) 130 | ) 131 | ) 132 | 133 | settings.setdefault(r'ssl', self._ssl_context) 134 | 135 | async for times in AsyncCirculatorForSecond(max_times=self._retry_count): 136 | 137 | try: 138 | 139 | async with aiohttp.ClientSession(**self._session_config) as _session: 140 | 141 | async with _session.request(method, url, **settings) as _response: 142 | 143 | response = Result( 144 | _response.status, 145 | dict(_response.headers), 146 | await self._handle_response(_response) 147 | ) 148 | 149 | except aiohttp.ClientResponseError as err: 150 | 151 | # 重新尝试的话，会记录异常，否则会继续抛出异常 152 | 153 | if err.status < 500: 154 | raise err 155 | elif times >= self._retry_count: 156 | raise err 157 | else: 158 | Utils.log.error(err) 159 | continue 160 | 161 | except aiohttp.ClientError as err: 162 | 163 | if times >= self._retry_count: 164 | raise err 165 | else: 166 | Utils.log.error(err) 167 | continue 168 | 169 | except Exception as err: 170 | 171 | raise err 172 | 173 | else: 174 | 175 | Utils.log.info(f'{method} {url} => status:{response.status}') 176 | break 177 | 178 | finally: 179 | 180 | if times > 1: 181 | Utils.log.warning(f'{method} {url} => retry:{times}') 182 | 183 | return response 184 | 185 | 186 | class _HTTPTextMixin: 187 | """Text模式混入类 188 | """ 189 | 190 | async def _handle_response(self, response): 191 | 192 | return await response.text() 193 | 194 | 195 | class _HTTPJsonMixin: 196 | """Json模式混入类 197 | """ 198 | 199 | async def _handle_response(self, response): 200 | 201 | return await response.json(encoding=r'utf-8', loads=_json_decoder, content_type=None) 202 | 203 | 204 | class _HTTPTouchMixin: 205 | """Touch模式混入类，不接收body数据 206 | """ 207 | 208 | async def _handle_response(self, response): 209 | 210 | return dict(response.headers) 211 | 212 | 213 | class HTTPClient(_HTTPClient): 214 | """HTTP客户端，普通模式 215 | """ 216 | 217 | async def get(self, url, params=None, *, cookies=None, headers=None): 218 | 219 | result = None 220 | 221 | try: 222 | 223 | resp = await self.send_request(aiohttp.hdrs.METH_GET, url, None, params, cookies=cookies, headers=headers) 224 | 225 | result = resp.body 226 | 227 | except Exception as err: 228 | 229 | Utils.log.error(err) 230 | 231 | return result 232 | 233 | async def options(self, url, params=None, *, cookies=None, headers=None): 234 | 235 | result = None 236 | 237 | try: 238 | 239 | resp = await self.send_request(aiohttp.hdrs.METH_OPTIONS, url, None, params, cookies=cookies, headers=headers) 240 | 241 | result = resp.headers 242 | 243 | except Exception as err: 244 | 245 | Utils.log.error(err) 246 | 247 | return result 248 | 249 | async def head(self, url, params=None, *, cookies=None, headers=None): 250 | 251 | result = None 252 | 253 | try: 254 | 255 | resp = await self.send_request(aiohttp.hdrs.METH_HEAD, url, None, params, cookies=cookies, headers=headers) 256 | 257 | result = resp.headers 258 | 259 | except Exception as err: 260 | 261 | Utils.log.error(err) 262 | 263 | return result 264 | 265 | async def post(self, url, data=None, params=None, *, cookies=None, headers=None): 266 | 267 | result = None 268 | 269 | try: 270 | 271 | resp = await self.send_request(aiohttp.hdrs.METH_POST, url, data, params, cookies=cookies, headers=headers) 272 | 273 | result = resp.body 274 | 275 | except Exception as err: 276 | 277 | Utils.log.error(err) 278 | 279 | return result 280 | 281 | async def put(self, url, data=None, params=None, *, cookies=None, headers=None): 282 | 283 | result = None 284 | 285 | try: 286 | 287 | resp = await self.send_request(aiohttp.hdrs.METH_PUT, url, data, params, cookies=cookies, headers=headers) 288 | 289 | result = resp.body 290 | 291 | except Exception as err: 292 | 293 | Utils.log.error(err) 294 | 295 | return result 296 | 297 | async def patch(self, url, data=None, params=None, *, cookies=None, headers=None): 298 | 299 | result = None 300 | 301 | try: 302 | 303 | resp = await self.send_request(aiohttp.hdrs.METH_PATCH, url, data, params, cookies=cookies, headers=headers) 304 | 305 | result = resp.body 306 | 307 | except Exception as err: 308 | 309 | Utils.log.error(err) 310 | 311 | return result 312 | 313 | async def delete(self, url, params=None, *, cookies=None, headers=None): 314 | 315 | result = None 316 | 317 | try: 318 | 319 | resp = await self.send_request(aiohttp.hdrs.METH_DELETE, url, None, params, cookies=cookies, headers=headers) 320 | 321 | result = resp.body 322 | 323 | except Exception as err: 324 | 325 | Utils.log.error(err) 326 | 327 | return result 328 | 329 | 330 | class HTTPTextClient(_HTTPTextMixin, HTTPClient): 331 | """HTTP客户端，Text模式 332 | """ 333 | pass 334 | 335 | 336 | class HTTPJsonClient(_HTTPJsonMixin, HTTPClient): 337 | """HTTP客户端，Json模式 338 | """ 339 | pass 340 | 341 | 342 | class HTTPTouchClient(_HTTPTouchMixin, HTTPClient): 343 | """HTTP客户端，Touch模式 344 | """ 345 | pass 346 | 347 | 348 | class HTTPClientPool(HTTPClient): 349 | """HTTP带连接池客户端，普通模式 350 | """ 351 | 352 | def __init__(self, 353 | retry_count=5, use_dns_cache=True, ttl_dns_cache=10, 354 | limit=100, limit_per_host=0, timeout=None, 355 | **kwargs 356 | ): 357 | 358 | super().__init__(retry_count, timeout, **kwargs) 359 | 360 | self._tcp_connector = aiohttp.TCPConnector( 361 | use_dns_cache=use_dns_cache, 362 | ttl_dns_cache=ttl_dns_cache, 363 | ssl=self._ssl_context, 364 | limit=limit, 365 | limit_per_host=limit_per_host, 366 | ) 367 | 368 | self._session_config[r'connector'] = self._tcp_connector 369 | self._session_config[r'connector_owner'] = False 370 | 371 | async def close(self): 372 | 373 | if not self._tcp_connector.closed: 374 | await self._tcp_connector.close() 375 | 376 | 377 | class HTTPTextClientPool(_HTTPTextMixin, HTTPClientPool): 378 | """HTTP带连接池客户端，Text模式 379 | """ 380 | pass 381 | 382 | 383 | class HTTPJsonClientPool(_HTTPJsonMixin, HTTPClientPool): 384 | """HTTP带连接池客户端，Json模式 385 | """ 386 | pass 387 | 388 | 389 | class HTTPTouchClientPool(_HTTPTouchMixin, HTTPClientPool): 390 | """HTTP带连接池客户端，Touch模式 391 | """ 392 | pass 393 | 394 | 395 | class Downloader(_HTTPClient): 396 | """HTTP文件下载器 397 | """ 398 | 399 | def __init__(self, file, retry_count=5, timeout=None, **kwargs): 400 | 401 | global DOWNLOAD_TIMEOUT 402 | 403 | super().__init__( 404 | retry_count, 405 | timeout if timeout is not None else DOWNLOAD_TIMEOUT, 406 | **kwargs 407 | ) 408 | 409 | self._file = file 410 | 411 | self._state = STATE.PENDING 412 | 413 | self._response = None 414 | 415 | @property 416 | def file(self): 417 | 418 | return self._file 419 | 420 | @property 421 | def state(self): 422 | 423 | return self._state 424 | 425 | @property 426 | def finished(self): 427 | 428 | return self._state in (STATE.SUCCESS, STATE.FAILURE) 429 | 430 | @property 431 | def response(self): 432 | 433 | return self._response 434 | 435 | async def _handle_response(self, response): 436 | 437 | if self._state != STATE.PENDING: 438 | return 439 | 440 | self._state = STATE.FETCHING 441 | self._response = response 442 | 443 | with open(self._file, r'wb') as stream: 444 | 445 | try: 446 | 447 | while True: 448 | 449 | chunk = await response.content.read(65536) 450 | 451 | if chunk: 452 | stream.write(chunk) 453 | else: 454 | break 455 | 456 | except Exception as err: 457 | 458 | Utils.log.error(err) 459 | 460 | self._state = STATE.FAILURE 461 | 462 | else: 463 | 464 | self._state = STATE.SUCCESS 465 | 466 | if self._state != STATE.SUCCESS and os.path.exists(self._file): 467 | os.remove(self._file) 468 | 469 | async def fetch(self, url, *, params=None, cookies=None, headers=None): 470 | 471 | result = False 472 | 473 | try: 474 | 475 | await self.send_request(aiohttp.hdrs.METH_GET, url, None, params, cookies=cookies, headers=headers) 476 | 477 | result = (self._state == STATE.SUCCESS) 478 | 479 | except Exception as err: 480 | 481 | Utils.log.error(err) 482 | 483 | return result 484 | 485 | 486 | class DownloadBuffer(ContextManager, Downloader): 487 | """HTTP文件下载器(临时文件版) 488 | """ 489 | 490 | def __init__(self, timeout=None, **kwargs): 491 | 492 | global DOWNLOAD_TIMEOUT 493 | 494 | super().__init__( 495 | FileBuffer(), 496 | 1, 497 | timeout if timeout is not None else DOWNLOAD_TIMEOUT, 498 | **kwargs 499 | ) 500 | 501 | def _context_release(self): 502 | 503 | self.close() 504 | 505 | def close(self): 506 | 507 | self._file.close() 508 | 509 | async def _handle_response(self, response): 510 | 511 | if self._state != STATE.PENDING: 512 | return 513 | 514 | self._state = STATE.FETCHING 515 | self._response = response 516 | 517 | try: 518 | 519 | while True: 520 | 521 | chunk = await response.content.read(65536) 522 | 523 | if chunk: 524 | self._file.write(chunk) 525 | else: 526 | break 527 | 528 | except Exception as err: 529 | 530 | Utils.log.error(err) 531 | 532 | self._state = STATE.FAILURE 533 | 534 | else: 535 | 536 | self._state = STATE.SUCCESS 537 | -------------------------------------------------------------------------------- /hagworm/extend/asyncio/ntp.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import time 4 | import numpy 5 | 6 | from ntplib import NTPClient 7 | 8 | from .base import Utils, MultiTasks 9 | from .task import IntervalTask 10 | from .future import ThreadPool 11 | 12 | from hagworm.extend.error import NTPCalibrateError 13 | from hagworm.extend.interface import TaskInterface 14 | 15 | 16 | class _Interface(TaskInterface): 17 | """NTP客户端接口定义 18 | """ 19 | 20 | def start(self): 21 | raise NotImplementedError() 22 | 23 | def stop(self): 24 | raise NotImplementedError() 25 | 26 | def is_running(self): 27 | raise NotImplementedError() 28 | 29 | def calibrate_offset(self): 30 | raise NotImplementedError() 31 | 32 | @property 33 | def offset(self): 34 | raise NotImplementedError() 35 | 36 | @property 37 | def timestamp(self): 38 | raise NotImplementedError() 39 | 40 | 41 | class AsyncNTPClient(_Interface): 42 | """异步NTP客户端类 43 | """ 44 | 45 | @classmethod 46 | async def create(cls, host): 47 | 48 | client = cls(host) 49 | 50 | await client.calibrate_offset() 51 | 52 | client.start() 53 | 54 | return client 55 | 56 | def __init__(self, host, *, version=2, port=r'ntp', timeout=5, interval=3600, sampling=5): 57 | 58 | self._settings = { 59 | r'host': host, 60 | r'version': version, 61 | r'port': port, 62 | r'timeout': timeout, 63 | } 64 | 65 | self._client = NTPClient() 66 | self._offset = 0 67 | 68 | self._thread_pool = ThreadPool(1) 69 | self._sync_task = IntervalTask(self.calibrate_offset, interval) 70 | 71 | self._sampling = sampling 72 | 73 | def start(self): 74 | 75 | return self._sync_task.start() 76 | 77 | def stop(self): 78 | 79 | return self._sync_task.stop() 80 | 81 | def is_running(self): 82 | 83 | return self._sync_task.is_running() 84 | 85 | async def calibrate_offset(self): 86 | 87 | return await self._thread_pool.run(self._calibrate_offset) 88 | 89 | def _calibrate_offset(self): 90 | 91 | samples = [] 92 | host_name = self._settings[r'host'] 93 | 94 | # 多次采样取中位数，减少抖动影响 95 | for _ in range(self._sampling): 96 | try: 97 | resp = self._client.request(**self._settings) 98 | samples.append(resp.offset) 99 | except Exception as err: 100 | Utils.log.error(f'NTP server {host_name} request error: {err}') 101 | 102 | if samples: 103 | self._offset = float(numpy.median(samples)) 104 | Utils.log.debug(f'NTP server {host_name} offset median {self._offset} samples: {samples}') 105 | else: 106 | raise NTPCalibrateError(f'NTP server {host_name} not available, timestamp uncalibrated') 107 | 108 | @property 109 | def offset(self): 110 | 111 | return self._offset 112 | 113 | @property 114 | def timestamp(self): 115 | 116 | return time.time() + self._offset 117 | 118 | 119 | class AsyncNTPClientPool(_Interface): 120 | """异步NTP客户端池，多节点取中位数实现高可用 121 | """ 122 | 123 | @classmethod 124 | async def create(cls, hosts): 125 | 126 | client_pool = cls() 127 | 128 | for host in hosts: 129 | client_pool.append(host) 130 | 131 | await client_pool.calibrate_offset() 132 | 133 | client_pool.start() 134 | 135 | return client_pool 136 | 137 | def __init__(self): 138 | 139 | self._clients = [] 140 | self._running = False 141 | 142 | def append(self, host, *, version=2, port='ntp', timeout=5, interval=3600, sampling=5): 143 | 144 | client = AsyncNTPClient(host, version=version, port=port, timeout=timeout, interval=interval, sampling=sampling) 145 | 146 | self._clients.append(client) 147 | 148 | if self._running: 149 | client.start() 150 | 151 | def start(self): 152 | 153 | for client in self._clients: 154 | client.start() 155 | 156 | self._running = True 157 | 158 | def stop(self): 159 | 160 | for client in self._clients: 161 | client.stop() 162 | 163 | self._running = False 164 | 165 | def is_running(self): 166 | 167 | return self._running 168 | 169 | async def calibrate_offset(self): 170 | 171 | tasks = MultiTasks() 172 | 173 | for client in self._clients: 174 | tasks.append(client.calibrate_offset()) 175 | 176 | await tasks 177 | 178 | @property 179 | def offset(self): 180 | 181 | samples = [] 182 | 183 | for client in self._clients: 184 | samples.append(client.offset) 185 | 186 | return float(numpy.median(samples)) 187 | 188 | @property 189 | def timestamp(self): 190 | 191 | return time.time() + self.offset 192 | -------------------------------------------------------------------------------- /hagworm/extend/asyncio/task.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import asyncio 4 | 5 | from crontab import CronTab 6 | from collections import OrderedDict 7 | 8 | from hagworm.extend.interface import TaskInterface 9 | 10 | from .base import Utils, FutureWithTask 11 | 12 | 13 | class _BaseTask(TaskInterface): 14 | """异步任务基类 15 | """ 16 | 17 | def __init__(self, _callable): 18 | 19 | self._running = False 20 | self._next_timeout = 0 21 | 22 | self._event_loop = None 23 | self._timeout_handle = None 24 | 25 | self._callable = _callable 26 | 27 | def start(self, promptly=False, *, event_loop=None): 28 | 29 | if event_loop: 30 | self._event_loop = event_loop 31 | else: 32 | self._event_loop = asyncio.get_event_loop() 33 | 34 | self._running = True 35 | 36 | if promptly: 37 | self._timeout_handle = Utils.call_soon(self._run) 38 | else: 39 | self._schedule_next() 40 | 41 | def stop(self): 42 | 43 | self._running = False 44 | 45 | if self._timeout_handle is not None: 46 | self._timeout_handle.cancel() 47 | self._timeout_handle = None 48 | 49 | def is_running(self): 50 | 51 | return self._running 52 | 53 | async def _run(self): 54 | 55 | if not self._running: 56 | return 57 | 58 | try: 59 | 60 | await Utils.awaitable_wrapper( 61 | self._callable() 62 | ) 63 | 64 | except Exception as err: 65 | 66 | Utils.log.error(err) 67 | 68 | finally: 69 | 70 | self._schedule_next() 71 | 72 | def _schedule_next(self): 73 | 74 | if self._running: 75 | self._timeout_handle = Utils.call_at(self._update_next(), self._run) 76 | 77 | def _update_next(self): 78 | 79 | self._next_timeout = self._event_loop.time() 80 | 81 | return self._next_timeout 82 | 83 | 84 | class LoopTask(_BaseTask): 85 | """循环任务类 86 | """ 87 | 88 | @classmethod 89 | def create(cls, limit_time, promptly, _callable, *args, **kwargs): 90 | 91 | if args or kwargs: 92 | _callable = Utils.func_partial(_callable, *args, **kwargs) 93 | 94 | task = cls(_callable, limit_time) 95 | 96 | task.start(promptly) 97 | 98 | return task 99 | 100 | def __init__(self, _callable, limit_time): 101 | 102 | super().__init__(_callable) 103 | 104 | self._limit_time = limit_time 105 | 106 | def _update_next(self): 107 | 108 | now_time = self._event_loop.time() 109 | next_timeout = self._next_timeout + self._limit_time 110 | 111 | if next_timeout < now_time: 112 | self._next_timeout = now_time 113 | else: 114 | self._next_timeout = next_timeout 115 | 116 | return self._next_timeout 117 | 118 | 119 | class IntervalTask(_BaseTask): 120 | """间隔任务类 121 | """ 122 | 123 | @classmethod 124 | def create(cls, interval, promptly, _callable, *args, **kwargs): 125 | 126 | if args or kwargs: 127 | _callable = Utils.func_partial(_callable, *args, **kwargs) 128 | 129 | task = cls(_callable, interval) 130 | 131 | task.start(promptly) 132 | 133 | return task 134 | 135 | def __init__(self, _callable, interval): 136 | 137 | super().__init__(_callable) 138 | 139 | self._interval = interval 140 | 141 | def _update_next(self): 142 | 143 | self._next_timeout = self._event_loop.time() + self._interval 144 | 145 | return self._next_timeout 146 | 147 | 148 | class CronTask(_BaseTask, CronTab): 149 | """定时任务类 150 | """ 151 | 152 | @classmethod 153 | def create(cls, crontab, promptly, _callable, *args, **kwargs): 154 | 155 | if args or kwargs: 156 | _callable = Utils.func_partial(_callable, *args, **kwargs) 157 | 158 | task = cls(_callable, crontab) 159 | 160 | task.start(promptly) 161 | 162 | return task 163 | 164 | def __init__(self, _callable, crontab, default_now=None, default_utc=None): 165 | 166 | _BaseTask.__init__(self, _callable) 167 | CronTab.__init__(self, crontab) 168 | 169 | self._default_now = default_now 170 | self._default_utc = default_utc 171 | 172 | def _update_next(self): 173 | 174 | params = {} 175 | 176 | if self._default_now is not None: 177 | params[r'now'] = self._default_now 178 | 179 | if self._default_utc is not None: 180 | params[r'default_utc'] = self._default_utc 181 | 182 | self._next_timeout = self._event_loop.time() + self.next(**params) 183 | 184 | return self._next_timeout 185 | 186 | 187 | class RateLimiter: 188 | """流量控制器，用于对计算资源的保护 189 | 添加任务append函数如果成功会返回Future对象，可以通过await该对象等待执行结果 190 | 进入队列的任务，如果触发限流行为会通过在Future上引发CancelledError传递出来 191 | """ 192 | 193 | def __init__(self, running_limit, waiting_limit=0, timeout=0): 194 | 195 | self._running_limit = running_limit 196 | self._waiting_limit = waiting_limit 197 | 198 | self._timeout = timeout 199 | 200 | self._running_tasks = OrderedDict() 201 | self._waiting_tasks = OrderedDict() 202 | 203 | @property 204 | def running_tasks(self): 205 | 206 | return list(self._running_tasks.values()) 207 | 208 | @property 209 | def running_length(self): 210 | 211 | return len(self._running_tasks) 212 | 213 | @property 214 | def waiting_tasks(self): 215 | 216 | return list(self._waiting_tasks.values()) 217 | 218 | @property 219 | def waiting_length(self): 220 | 221 | return len(self._waiting_tasks) 222 | 223 | def _create_task(self, name, func, *args, **kwargs): 224 | 225 | if len(args) == 0 and len(kwargs) == 0: 226 | return FutureWithTask(func, name) 227 | else: 228 | return FutureWithTask(Utils.func_partial(func, *args, **kwargs), name) 229 | 230 | def append(self, func, *args, **kwargs): 231 | 232 | return self._append(None, func, *args, **kwargs) 233 | 234 | def append_with_name(self, name, func, *args, **kwargs): 235 | 236 | return self._append(name, func, *args, **kwargs) 237 | 238 | def _append(self, name, func, *args, **kwargs): 239 | 240 | task = None 241 | 242 | task_tag = f"{name or r''} {func} {args or r''} {kwargs or r''}" 243 | 244 | if name is None or ((name not in self._running_tasks) and (name not in self._waiting_tasks)): 245 | 246 | if self._check_running_limit(): 247 | 248 | task = self._create_task(name, func, *args, **kwargs) 249 | self._add_running_tasks(task) 250 | 251 | Utils.log.debug(f'rate limit add running tasks: {task_tag}') 252 | 253 | elif self._check_waiting_limit(): 254 | 255 | task = self._create_task(name, func, *args, **kwargs) 256 | self._add_waiting_tasks(task) 257 | 258 | Utils.log.debug(f'rate limit add waiting tasks: {task_tag}') 259 | 260 | else: 261 | 262 | Utils.log.warning( 263 | f'rate limit: {task_tag}\n' 264 | f'running: {self.running_length}/{self.running_limit}\n' 265 | f'waiting: {self.waiting_length}/{self.waiting_limit}' 266 | ) 267 | 268 | else: 269 | 270 | Utils.log.warning(f'rate limit duplicate: {task_tag}') 271 | 272 | return task 273 | 274 | @property 275 | def running_limit(self): 276 | 277 | return self._running_limit 278 | 279 | @running_limit.setter 280 | def running_limit(self, val): 281 | 282 | self._running_limit = val 283 | 284 | self._recover_waiting_tasks() 285 | 286 | def _check_running_limit(self): 287 | 288 | return (self._running_limit <= 0) or (len(self._running_tasks) < self._running_limit) 289 | 290 | @property 291 | def waiting_limit(self): 292 | 293 | return self._waiting_limit 294 | 295 | @waiting_limit.setter 296 | def waiting_limit(self, val): 297 | 298 | self._waiting_limit = val 299 | 300 | if len(self._waiting_tasks) > self._waiting_limit: 301 | self._waiting_tasks = self._waiting_tasks[:self._waiting_limit] 302 | 303 | def _check_waiting_limit(self): 304 | 305 | return (self._waiting_limit <= 0) or (len(self._waiting_tasks) < self._waiting_limit) 306 | 307 | @property 308 | def timeout(self): 309 | 310 | return self._timeout 311 | 312 | @timeout.setter 313 | def timeout(self, val): 314 | 315 | self._timeout = val 316 | 317 | def _check_timeout(self, task): 318 | 319 | return (self._timeout <= 0) or ((Utils.loop_time() - task.build_time) < self._timeout) 320 | 321 | def _add_running_tasks(self, task): 322 | 323 | if not self._check_timeout(task): 324 | task.cancel() 325 | Utils.log.warning(f'rate limit timeout: {task.name} build_time:{task.build_time}') 326 | elif task.name in self._running_tasks: 327 | task.cancel() 328 | Utils.log.warning(f'rate limit duplicate: {task.name}') 329 | else: 330 | task.add_done_callback(self._done_callback) 331 | self._running_tasks[task.name] = task.run() 332 | 333 | def _add_waiting_tasks(self, task): 334 | 335 | if task.name not in self._waiting_tasks: 336 | self._waiting_tasks[task.name] = task 337 | else: 338 | task.cancel() 339 | Utils.log.warning(f'rate limit duplicate: {task.name}') 340 | 341 | def _recover_waiting_tasks(self): 342 | 343 | for _ in range(len(self._waiting_tasks)): 344 | 345 | if self._check_running_limit(): 346 | item = self._waiting_tasks.popitem(False) 347 | self._add_running_tasks(item[1]) 348 | else: 349 | break 350 | 351 | def _done_callback(self, task): 352 | 353 | if task.name in self._running_tasks: 354 | self._running_tasks.pop(task.name) 355 | 356 | self._recover_waiting_tasks() 357 | -------------------------------------------------------------------------------- /hagworm/extend/asyncio/transaction.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .base import Utils, AsyncContextManager 4 | 5 | from hagworm.extend.transaction import TransactionInterface 6 | 7 | 8 | class Transaction(TransactionInterface, AsyncContextManager): 9 | """事务对象 10 | 11 | 使用异步上下文实现的一个事务对象，可以设置commit和rollback回调 12 | 未显示commit的情况下，会自动rollback 13 | 14 | """ 15 | 16 | async def _context_release(self): 17 | 18 | await self.rollback() 19 | 20 | async def commit(self): 21 | 22 | if self._commit_callbacks is None: 23 | return 24 | 25 | callbacks = self._commit_callbacks.copy() 26 | 27 | self._clear_callbacks() 28 | 29 | for _callable in callbacks: 30 | try: 31 | await Utils.awaitable_wrapper( 32 | _callable() 33 | ) 34 | except Exception as err: 35 | Utils.log.critical(f'transaction commit error:\n{err}') 36 | 37 | async def rollback(self): 38 | 39 | if self._rollback_callbacks is None: 40 | return 41 | 42 | callbacks = self._rollback_callbacks.copy() 43 | 44 | self._clear_callbacks() 45 | 46 | for _callable in callbacks: 47 | try: 48 | await Utils.awaitable_wrapper( 49 | _callable() 50 | ) 51 | except Exception as err: 52 | Utils.log.critical(f'transaction rollback error:\n{err}') 53 | -------------------------------------------------------------------------------- /hagworm/extend/asyncio/zmq.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import zmq 4 | 5 | from concurrent.futures import CancelledError 6 | from zmq.asyncio import Context 7 | 8 | from hagworm.extend.base import ContextManager 9 | from hagworm.extend.asyncio.base import Utils, AsyncCirculator 10 | from hagworm.extend.asyncio.buffer import QueueBuffer 11 | 12 | 13 | class _SocketBase(ContextManager): 14 | 15 | def __init__(self, name, socket_type, address, bind_mode): 16 | 17 | self._name = name if name else Utils.uuid1()[:8] 18 | 19 | self._context = Context.instance() 20 | self._socket = self._context.socket(socket_type) 21 | 22 | self._address = address 23 | self._bind_mode = bind_mode 24 | 25 | def _context_initialize(self): 26 | 27 | self.open() 28 | 29 | def _context_release(self): 30 | 31 | self.close() 32 | 33 | def open(self): 34 | 35 | if self._bind_mode: 36 | self._socket.bind(self._address) 37 | else: 38 | self._socket.connect(self._address) 39 | 40 | def close(self): 41 | 42 | if not self._socket.closed: 43 | self._socket.close() 44 | 45 | def set_hwm(self, val): 46 | 47 | self._socket.set_hwm(val) 48 | 49 | 50 | class Subscriber(_SocketBase): 51 | 52 | def __init__(self, address, bind_mode=False, *, name=None, topic=r''): 53 | 54 | super().__init__(name, zmq.SUB, address, bind_mode) 55 | 56 | self._msg_listen_task = None 57 | 58 | self._socket.setsockopt_string(zmq.SUBSCRIBE, topic) 59 | 60 | async def _message_listener(self): 61 | 62 | while not self._socket.closed: 63 | try: 64 | await self._message_handler( 65 | await self._socket.recv_pyobj() 66 | ) 67 | except CancelledError as _: 68 | pass 69 | except Exception as err: 70 | Utils.log.error(err) 71 | 72 | async def _message_handler(self, data): 73 | 74 | raise NotImplementedError() 75 | 76 | def open(self): 77 | 78 | super().open() 79 | 80 | self._msg_listen_task = Utils.create_task(self._message_listener()) 81 | 82 | def close(self): 83 | 84 | super().close() 85 | 86 | if self._msg_listen_task is not None: 87 | self._msg_listen_task.cancel() 88 | 89 | 90 | class Publisher(_SocketBase): 91 | 92 | def __init__(self, address, bind_mode=False, *, name=None): 93 | 94 | super().__init__(name, zmq.PUB, address, bind_mode) 95 | 96 | async def send(self, data): 97 | 98 | await self._socket.send_pyobj(data) 99 | 100 | 101 | class PublisherWithBuffer(_SocketBase, QueueBuffer): 102 | 103 | def __init__(self, address, bind_mode=False, *, name=None, buffer_maxsize=0xffff, buffer_timeout=1): 104 | 105 | _SocketBase.__init__(self, name, zmq.PUB, address, bind_mode) 106 | QueueBuffer.__init__(self, buffer_maxsize, buffer_timeout) 107 | 108 | async def _run(self, data_list): 109 | 110 | await self._socket.send_pyobj(data_list) 111 | 112 | async def safe_close(self, timeout=0): 113 | 114 | async for _ in AsyncCirculator(timeout): 115 | if len(self._data_list) == 0: 116 | super().close() 117 | break 118 | -------------------------------------------------------------------------------- /hagworm/extend/cache.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from cachetools import TTLCache 4 | 5 | from hagworm.extend.base import Utils 6 | from hagworm.extend.transaction import Transaction 7 | 8 | 9 | class StackCache: 10 | """堆栈缓存 11 | 12 | 使用运行内存作为高速缓存，可有效提高并发的处理能力 13 | 14 | """ 15 | 16 | def __init__(self, maxsize=0xff, ttl=60): 17 | 18 | self._cache = TTLCache(maxsize, ttl) 19 | 20 | def has(self, key): 21 | 22 | return key in self._cache 23 | 24 | def get(self, key, default=None): 25 | 26 | return self._cache.get(key, default) 27 | 28 | def set(self, key, val): 29 | 30 | self._cache[key] = val 31 | 32 | def incr(self, key, val=1): 33 | 34 | res = self.get(key, 0) + val 35 | 36 | self.set(key, res) 37 | 38 | return res 39 | 40 | def decr(self, key, val=1): 41 | 42 | res = self.get(key, 0) - val 43 | 44 | self.set(key, res) 45 | 46 | return res 47 | 48 | def delete(self, key): 49 | 50 | del self._cache[key] 51 | 52 | def size(self): 53 | 54 | return len(self._cache) 55 | 56 | def clear(self): 57 | 58 | return self._cache.clear() 59 | 60 | 61 | class PeriodCounter: 62 | 63 | MIN_EXPIRE = 60 64 | 65 | def __init__(self, time_slice, key_prefix=r'', maxsize=0xffff): 66 | 67 | self._time_slice = time_slice 68 | self._key_prefix = key_prefix 69 | 70 | # 缓存对象初始化，key最小过期时间60秒 71 | self._cache = StackCache(maxsize, max(time_slice, self.MIN_EXPIRE)) 72 | 73 | def _get_key(self, key=None): 74 | 75 | time_period = Utils.math.floor(Utils.timestamp() / self._time_slice) 76 | 77 | if key is None: 78 | return f'{self._key_prefix}_{time_period}' 79 | else: 80 | return f'{self._key_prefix}_{key}_{time_period}' 81 | 82 | def get(self, key=None): 83 | 84 | _key = self._get_key(key) 85 | 86 | return self._cache.get(_key, 0) 87 | 88 | def incr(self, val, key=None): 89 | 90 | _key = self._get_key(key) 91 | 92 | return self._cache.incr(_key, val) 93 | 94 | def incr_with_trx(self, val, key=None): 95 | 96 | _key = self._get_key(key) 97 | 98 | trx = Transaction() 99 | trx.add_rollback_callback(self._cache.decr, _key, val) 100 | 101 | return self._cache.incr(_key, val), trx 102 | 103 | def decr(self, val, key=None): 104 | 105 | _key = self._get_key(key) 106 | 107 | return self._cache.decr(_key, val) 108 | 109 | def decr_with_trx(self, val, key=None): 110 | 111 | _key = self._get_key(key) 112 | 113 | trx = Transaction() 114 | trx.add_rollback_callback(self._cache.incr, _key, val) 115 | 116 | return self._cache.decr(_key, val), trx 117 | -------------------------------------------------------------------------------- /hagworm/extend/compile.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | import sys 5 | 6 | import argparse 7 | import py_compile 8 | 9 | from .base import Utils 10 | 11 | 12 | def compile(file_path, cfile_path=None, exclude=None): 13 | """PYC编译函数 14 | """ 15 | 16 | if exclude: 17 | exclude = [os.path.join(file_path, val) for val in Utils.split_str(exclude, r'|')] 18 | else: 19 | exclude = [] 20 | 21 | for root, _, files in os.walk(file_path): 22 | 23 | for _path in exclude: 24 | 25 | if root.find(_path) == 0: 26 | break 27 | 28 | else: 29 | 30 | for _file in files: 31 | 32 | _, ext_name = os.path.splitext(_file) 33 | 34 | if ext_name != r'.py': 35 | continue 36 | 37 | if cfile_path is None: 38 | 39 | dest_path = file_path 40 | 41 | else: 42 | 43 | dest_path = root.replace(file_path, cfile_path) 44 | 45 | if not os.path.exists(dest_path): 46 | os.makedirs(dest_path) 47 | 48 | ori_path = os.path.join(root, _file) 49 | dest_path = os.path.join(dest_path, _file) + r'c' 50 | 51 | py_compile.compile(ori_path, dest_path, optimize=2) 52 | 53 | sys.stdout.write(f'{ori_path} => {dest_path}\n') 54 | 55 | 56 | def main(): 57 | 58 | result = 0 59 | 60 | parser = argparse.ArgumentParser() 61 | 62 | parser.add_argument(r'-i', r'--input', default=r'./', dest=r'input') 63 | parser.add_argument(r'-o', r'--output', default=None, dest=r'output') 64 | parser.add_argument(r'-e', r'--exclude', default=None, dest=r'exclude') 65 | 66 | args = parser.parse_args() 67 | 68 | try: 69 | compile(args.input, args.output, args.exclude) 70 | except Exception as error: 71 | sys.stderr.write(f'{error}\n') 72 | 73 | return result 74 | 75 | 76 | if __name__ == r'__main__': 77 | 78 | sys.exit(main()) 79 | -------------------------------------------------------------------------------- /hagworm/extend/crypto.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import binascii 4 | import base64 5 | import jwt 6 | import textwrap 7 | 8 | from cryptography.hazmat.primitives.serialization import load_pem_public_key 9 | from cryptography.hazmat.primitives import hashes 10 | from cryptography.hazmat.backends import default_backend 11 | 12 | 13 | class RsaUtil: 14 | """Rsa加解密相关工具类 15 | """ 16 | 17 | @classmethod 18 | def gen_rsa_key(cls, rsa_key, private=False): 19 | 20 | if private: 21 | start_line = r'-----BEGIN RSA PRIVATE KEY-----' 22 | end_line = r'-----END RSA PRIVATE KEY-----' 23 | else: 24 | start_line = r'-----BEGIN PUBLIC KEY-----' 25 | end_line = r'-----END PUBLIC KEY-----' 26 | 27 | rsa_key = textwrap.fill(rsa_key, 64) 28 | 29 | return '\n'.join([start_line, rsa_key, end_line]) 30 | 31 | @classmethod 32 | def rsa_sign(cls, rsa_key, sign_data): 33 | 34 | algorithm = jwt.algorithms.RSAAlgorithm(hashes.SHA1) 35 | 36 | key = algorithm.prepare_key(cls.gen_rsa_key(rsa_key, True)) 37 | 38 | signature = algorithm.sign(sign_data.encode(r'utf-8'), key) 39 | 40 | return base64.b64encode(signature).decode() 41 | 42 | @classmethod 43 | def rsa_verity(cls, pubic_key, verity_data, verity_sign): 44 | 45 | algorithm = jwt.algorithms.RSAAlgorithm(hashes.SHA1) 46 | 47 | public_key = load_pem_public_key(cls.gen_rsa_key(pubic_key).encode(r'utf-8'), backend=default_backend()) 48 | 49 | result = algorithm.verify(verity_data.encode(), public_key, binascii.a2b_base64(verity_sign)) 50 | 51 | return result 52 | -------------------------------------------------------------------------------- /hagworm/extend/error.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | # 基础异常 5 | class BaseError(Exception): 6 | 7 | def __init__(self, data=None): 8 | 9 | super().__init__() 10 | 11 | self._data = data 12 | 13 | @property 14 | def data(self): 15 | 16 | return self._data 17 | 18 | def __repr__(self): 19 | 20 | return repr(self._data) 21 | 22 | 23 | # 数据库只读限制异常 24 | class MySQLReadOnlyError(BaseError): 25 | pass 26 | 27 | 28 | # 常量设置异常 29 | class ConstError(BaseError): 30 | pass 31 | 32 | 33 | # NTP校准异常 34 | class NTPCalibrateError(BaseError): 35 | pass 36 | -------------------------------------------------------------------------------- /hagworm/extend/event.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .base import Utils, FuncWrapper 4 | 5 | 6 | class EventDispatcher: 7 | """事件总线 8 | """ 9 | 10 | def __init__(self): 11 | 12 | self._observers = {} 13 | 14 | def _gen_observer(self): 15 | 16 | return FuncWrapper() 17 | 18 | def dispatch(self, _type, *args, **kwargs): 19 | 20 | Utils.log.debug(f'dispatch event {_type} {args} {kwargs}') 21 | 22 | if _type in self._observers: 23 | self._observers[_type](*args, **kwargs) 24 | 25 | def add_listener(self, _type, _callable): 26 | 27 | Utils.log.debug(f'add event listener => type({_type}) function({id(_callable)})') 28 | 29 | result = False 30 | 31 | if _type in self._observers: 32 | result = self._observers[_type].add(_callable) 33 | else: 34 | observer = self._observers[_type] = self._gen_observer() 35 | result = observer.add(_callable) 36 | 37 | return result 38 | 39 | def remove_listener(self, _type, _callable): 40 | 41 | Utils.log.debug(f'remove event listener => type({_type}) function({id(_callable)})') 42 | 43 | result = False 44 | 45 | if _type in self._observers: 46 | 47 | observer = self._observers[_type] 48 | 49 | result = observer.remove(_callable) 50 | 51 | if not observer.is_valid: 52 | del self._observers[_type] 53 | 54 | return result 55 | -------------------------------------------------------------------------------- /hagworm/extend/excel.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import copy 4 | 5 | from io import BytesIO 6 | from datetime import datetime 7 | 8 | from xlwt import Workbook, XFStyle, Borders, Pattern 9 | 10 | 11 | class ExcelWT(Workbook): 12 | """Excel生成工具 13 | """ 14 | 15 | def __init__(self, name, encoding=r'utf-8', style_compression=0): 16 | 17 | super().__init__(encoding, style_compression) 18 | 19 | self._book_name = name 20 | self._current_sheet = None 21 | 22 | self._default_style = XFStyle() 23 | self._default_style.borders.left = Borders.THIN 24 | self._default_style.borders.right = Borders.THIN 25 | self._default_style.borders.top = Borders.THIN 26 | self._default_style.borders.bottom = Borders.THIN 27 | self._default_style.pattern.pattern = Pattern.SOLID_PATTERN 28 | self._default_style.pattern.pattern_fore_colour = 0x01 29 | 30 | self._default_title_style = copy.deepcopy(self._default_style) 31 | self._default_title_style.font.bold = True 32 | self._default_title_style.pattern.pattern_fore_colour = 0x16 33 | 34 | def create_sheet(self, name, titles=[]): 35 | 36 | sheet = self._current_sheet = self.add_sheet(name) 37 | style = self._default_title_style 38 | 39 | for index, title in enumerate(titles): 40 | sheet.write(0, index, title, style) 41 | sheet.col(index).width = 0x1200 42 | 43 | def add_sheet_row(self, *args): 44 | 45 | sheet = self._current_sheet 46 | style = self._default_style 47 | 48 | nrow = len(sheet.rows) 49 | 50 | for index, value in enumerate(args): 51 | sheet.write(nrow, index, value, style) 52 | 53 | def get_file(self): 54 | 55 | result = b'' 56 | 57 | with BytesIO() as stream: 58 | 59 | self.save(stream) 60 | 61 | result = stream.getvalue() 62 | 63 | return result 64 | 65 | def write_request(self, request): 66 | 67 | filename = f"{self._book_name}.{datetime.today().strftime('%y%m%d.%H%M%S')}.xls" 68 | 69 | request.set_header(r'Content-Type', r'application/vnd.ms-excel') 70 | request.set_header(r'Content-Disposition', f'attachment;filename={filename}') 71 | 72 | return request.finish(self.get_file()) 73 | -------------------------------------------------------------------------------- /hagworm/extend/interface.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | class RunnableInterface: 5 | """Runnable接口定义 6 | """ 7 | 8 | def run(self): 9 | raise NotImplementedError() 10 | 11 | 12 | class TaskInterface: 13 | """Task接口定义 14 | """ 15 | 16 | def start(self): 17 | raise NotImplementedError() 18 | 19 | def stop(self): 20 | raise NotImplementedError() 21 | 22 | def is_running(self): 23 | raise NotImplementedError() 24 | 25 | 26 | class ObjectFactoryInterface: 27 | """对象工厂类接口定义 28 | """ 29 | 30 | def create(self): 31 | raise NotImplementedError() 32 | -------------------------------------------------------------------------------- /hagworm/extend/logging.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import logging 4 | 5 | from datetime import timedelta 6 | 7 | from hagworm.extend.base import Utils 8 | 9 | 10 | class LogFileRotator: 11 | 12 | @classmethod 13 | def make(cls, _size=500, _time=r'00:00'): 14 | 15 | return cls(_size, _time).should_rotate 16 | 17 | def __init__(self, _size, _time): 18 | 19 | _size = _size * (1024 ** 2) 20 | _time = Utils.split_int(_time, r':') 21 | 22 | now_time = Utils.today() 23 | 24 | self._size_limit = _size 25 | self._time_limit = now_time.replace(hour=_time[0], minute=_time[1]) 26 | 27 | if now_time >= self._time_limit: 28 | self._time_limit += timedelta(days=1) 29 | 30 | def should_rotate(self, message, file): 31 | 32 | file.seek(0, 2) 33 | 34 | if file.tell() + len(message) > self._size_limit: 35 | return True 36 | 37 | if message.record[r'time'].timestamp() > self._time_limit.timestamp(): 38 | self._time_limit += timedelta(days=1) 39 | return True 40 | 41 | return False 42 | 43 | 44 | DEFAULT_LOG_FILE_ROTATOR = LogFileRotator.make() 45 | 46 | 47 | class LogInterceptor(logging.Handler): 48 | """日志拦截器 49 | """ 50 | 51 | def emit(self, record): 52 | 53 | Utils.log.opt( 54 | depth=6, 55 | exception=record.exc_info 56 | ).log( 57 | record.levelname, 58 | record.getMessage() 59 | ) 60 | 61 | 62 | DEFAULT_LOG_INTERCEPTOR = LogInterceptor() 63 | -------------------------------------------------------------------------------- /hagworm/extend/metaclass.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import threading 4 | import traceback 5 | 6 | 7 | class SafeSingletonMetaclass(type): 8 | """线程安全的单例的元类实现 9 | """ 10 | 11 | def __init__(cls, name, bases, attrs): 12 | 13 | cls._instance = None 14 | 15 | cls._lock = threading.Lock() 16 | 17 | def __call__(cls, *args, **kwargs): 18 | 19 | result = None 20 | 21 | cls._lock.acquire() 22 | 23 | try: 24 | 25 | if cls._instance is not None: 26 | result = cls._instance 27 | else: 28 | result = cls._instance = super().__call__(*args, **kwargs) 29 | 30 | except Exception as _: 31 | 32 | traceback.print_exc() 33 | 34 | finally: 35 | 36 | cls._lock.release() 37 | 38 | return result 39 | 40 | 41 | class SafeSingleton(metaclass=SafeSingletonMetaclass): 42 | """线程安全的单例基类 43 | """ 44 | pass 45 | 46 | 47 | class SingletonMetaclass(type): 48 | """单例的元类实现 49 | """ 50 | 51 | def __init__(cls, name, bases, attrs): 52 | 53 | cls._instance = None 54 | 55 | def __call__(cls, *args, **kwargs): 56 | 57 | result = None 58 | 59 | try: 60 | 61 | if cls._instance is not None: 62 | result = cls._instance 63 | else: 64 | result = cls._instance = super().__call__(*args, **kwargs) 65 | 66 | except Exception as _: 67 | 68 | traceback.print_exc() 69 | 70 | return result 71 | 72 | 73 | class Singleton(metaclass=SingletonMetaclass): 74 | """单例基类 75 | """ 76 | pass 77 | 78 | 79 | class SubclassMetaclass(type): 80 | """子类清单 81 | 82 | 该元类的类，能感知自身被继承，并提供子类清单 83 | 84 | """ 85 | 86 | _baseclasses = [] 87 | 88 | def __new__(mcs, name, bases, attrs): 89 | 90 | subclass = r'.'.join([attrs[r'__module__'], attrs[r'__qualname__']]) 91 | 92 | for base in bases: 93 | 94 | if base in mcs._baseclasses: 95 | getattr(base, r'_subclasses').append(subclass) 96 | else: 97 | mcs._baseclasses.append(base) 98 | setattr(base, r'_subclasses', [subclass]) 99 | 100 | return type.__new__(mcs, name, bases, attrs) 101 | -------------------------------------------------------------------------------- /hagworm/extend/qrcode.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import qrcode 4 | 5 | from io import BytesIO 6 | from qrcode.image.svg import SvgPathImage 7 | 8 | 9 | class QRCode: 10 | 11 | @staticmethod 12 | def make(data): 13 | 14 | code_img = BytesIO() 15 | 16 | qrcode.make(data).save(code_img) 17 | 18 | return code_img.getvalue() 19 | 20 | @staticmethod 21 | def make_svg(data): 22 | 23 | code_img = BytesIO() 24 | 25 | qrcode.make(data, image_factory=SvgPathImage).save(code_img) 26 | 27 | return code_img.getvalue().decode() 28 | -------------------------------------------------------------------------------- /hagworm/extend/struct.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import re 4 | import struct 5 | import threading 6 | 7 | from io import BytesIO 8 | from collections import OrderedDict 9 | from configparser import RawConfigParser 10 | 11 | from .base import Utils 12 | from .error import ConstError 13 | 14 | 15 | class Result(dict): 16 | """返回结果类 17 | """ 18 | 19 | def __init__(self, code=0, data=None, extra=None): 20 | 21 | super().__init__(code=code) 22 | 23 | if data is not None: 24 | self.__setitem__(r'data', data) 25 | 26 | if extra is not None: 27 | self.__setitem__(r'extra', extra) 28 | 29 | def __bool__(self): 30 | 31 | return self.code == 0 32 | 33 | @property 34 | def code(self): 35 | 36 | return self.get(r'code') 37 | 38 | @property 39 | def data(self): 40 | 41 | return self.get(r'data', None) 42 | 43 | @property 44 | def extra(self): 45 | 46 | return self.get(r'extra', None) 47 | 48 | 49 | class NullData: 50 | """NULL类，用于模拟None对象的行为 51 | """ 52 | 53 | def __int__(self): 54 | 55 | return 0 56 | 57 | def __bool__(self): 58 | 59 | return False 60 | 61 | def __float__(self): 62 | 63 | return 0.0 64 | 65 | def __len__(self): 66 | 67 | return 0 68 | 69 | def __repr__(self): 70 | 71 | return r'' 72 | 73 | def __eq__(self, obj): 74 | 75 | return not bool(obj) 76 | 77 | def __nonzero__(self): 78 | 79 | return False 80 | 81 | def __cmp__(self, val): 82 | 83 | if val is None: 84 | return 0 85 | else: 86 | return 1 87 | 88 | 89 | class ThreadList(threading.local): 90 | """多线程安全的列表 91 | """ 92 | 93 | __slots__ = [r'data'] 94 | 95 | def __init__(self): 96 | 97 | self.data = [] 98 | 99 | 100 | class ThreadDict(threading.local): 101 | """多线程安全的字典 102 | """ 103 | 104 | __slots__ = [r'data'] 105 | 106 | def __init__(self): 107 | 108 | self.data = {} 109 | 110 | 111 | class Const(OrderedDict): 112 | """常量类 113 | """ 114 | 115 | def __getattr__(self, key): 116 | 117 | return self.__getitem__(key) 118 | 119 | def __setattr__(self, key, val): 120 | 121 | if key[:1] == r'_': 122 | super().__setattr__(key, val) 123 | else: 124 | self.__setitem__(key, val) 125 | 126 | def __delattr__(self, key): 127 | 128 | if key[:1] == r'_': 129 | super().__delattr__(key) 130 | else: 131 | raise ConstError() 132 | 133 | def __setitem__(self, key, val): 134 | 135 | if key in self: 136 | raise ConstError() 137 | else: 138 | super().__setitem__(key, val) 139 | 140 | def __delitem__(self, key): 141 | 142 | raise ConstError() 143 | 144 | def exist(self, val): 145 | 146 | return val in self.values() 147 | 148 | 149 | class ByteArray(BytesIO): 150 | """扩展的BytesIO类 151 | """ 152 | 153 | NETWORK = r'!' 154 | NATIVE = r'=' 155 | NATIVE_ALIGNMENT = r'@' 156 | LITTLE_ENDIAN = r'<' 157 | BIG_ENDIAN = r'>' 158 | 159 | def __init__(self, *args, **kwargs): 160 | 161 | super().__init__(*args, **kwargs) 162 | 163 | self._endian = self.NETWORK 164 | 165 | def get_endian(self): 166 | 167 | return self._endian 168 | 169 | def set_endian(self, val): 170 | 171 | self._endian = val 172 | 173 | def read_pad_byte(self, _len): 174 | 175 | struct.unpack(f'{self._endian}{_len}x', self.read(_len)) 176 | 177 | def write_pad_byte(self, _len): 178 | 179 | self.write(struct.pack(f'{self._endian}{_len}x')) 180 | 181 | def read_char(self): 182 | 183 | return struct.unpack(f'{self._endian}c', self.read(1))[0] 184 | 185 | def write_char(self, val): 186 | 187 | self.write(struct.pack(f'{self._endian}c', val)) 188 | 189 | def read_signed_char(self): 190 | 191 | return struct.unpack(f'{self._endian}b', self.read(1))[0] 192 | 193 | def write_signed_char(self, val): 194 | 195 | self.write(struct.pack(f'{self._endian}b', val)) 196 | 197 | def read_unsigned_char(self): 198 | 199 | return struct.unpack(f'{self._endian}B', self.read(1))[0] 200 | 201 | def write_unsigned_char(self, val): 202 | 203 | self.write(struct.pack(f'{self._endian}B', val)) 204 | 205 | def read_bool(self): 206 | 207 | return struct.unpack(f'{self._endian}?', self.read(1))[0] 208 | 209 | def write_bool(self, val): 210 | 211 | self.write(struct.pack(f'{self._endian}?', val)) 212 | 213 | def read_short(self): 214 | 215 | return struct.unpack(f'{self._endian}h', self.read(2))[0] 216 | 217 | def write_short(self, val): 218 | 219 | self.write(struct.pack(f'{self._endian}h', val)) 220 | 221 | def read_unsigned_short(self): 222 | 223 | return struct.unpack(f'{self._endian}H', self.read(2))[0] 224 | 225 | def write_unsigned_short(self, val): 226 | 227 | self.write(struct.pack(f'{self._endian}H', val)) 228 | 229 | def read_int(self): 230 | 231 | return struct.unpack(f'{self._endian}i', self.read(4))[0] 232 | 233 | def write_int(self, val): 234 | 235 | self.write(struct.pack(f'{self._endian}i', val)) 236 | 237 | def read_unsigned_int(self): 238 | 239 | return struct.unpack(f'{self._endian}I', self.read(4))[0] 240 | 241 | def write_unsigned_int(self, val): 242 | 243 | self.write(struct.pack(f'{self._endian}I', val)) 244 | 245 | def read_long(self): 246 | 247 | return struct.unpack(f'{self._endian}l', self.read(8))[0] 248 | 249 | def write_long(self, val): 250 | 251 | self.write(struct.pack(f'{self._endian}l', val)) 252 | 253 | def read_unsigned_long(self): 254 | 255 | return struct.unpack(f'{self._endian}L', self.read(8))[0] 256 | 257 | def write_unsigned_long(self, val): 258 | 259 | self.write(struct.pack(f'{self._endian}L', val)) 260 | 261 | def read_long_long(self): 262 | 263 | return struct.unpack(f'{self._endian}q', self.read(8))[0] 264 | 265 | def write_long_long(self, val): 266 | 267 | self.write(struct.pack(f'{self._endian}q', val)) 268 | 269 | def read_unsigned_long_long(self): 270 | 271 | return struct.unpack(f'{self._endian}Q', self.read(8))[0] 272 | 273 | def write_unsigned_long_long(self, val): 274 | 275 | self.write(struct.pack(f'{self._endian}Q', val)) 276 | 277 | def read_float(self): 278 | 279 | return struct.unpack(f'{self._endian}f', self.read(4))[0] 280 | 281 | def write_float(self, val): 282 | 283 | self.write(struct.pack(f'{self._endian}f', val)) 284 | 285 | def read_double(self): 286 | 287 | return struct.unpack(f'{self._endian}d', self.read(8))[0] 288 | 289 | def write_double(self, val): 290 | 291 | self.write(struct.pack(f'{self._endian}d', val)) 292 | 293 | def read_bytes(self, _len): 294 | 295 | return struct.unpack(f'{self._endian}{_len}s', self.read(_len))[0] 296 | 297 | def write_bytes(self, val): 298 | 299 | self.write(struct.pack(f'{self._endian}{len(val)}s', val)) 300 | 301 | def read_string(self, _len): 302 | 303 | return self.read_bytes(_len).decode() 304 | 305 | def write_string(self, val): 306 | 307 | self.write_bytes(val.encode()) 308 | 309 | def read_pascal_bytes(self, _len): 310 | 311 | return struct.unpack(f'{self._endian}{_len}p', self.read(_len))[0] 312 | 313 | def write_pascal_bytes(self, val): 314 | 315 | self.write(struct.pack(f'{self._endian}{len(val)}p', val)) 316 | 317 | def read_pascal_string(self, _len): 318 | 319 | return self.read_pascal_bytes(_len).decode() 320 | 321 | def write_pascal_string(self, val): 322 | 323 | self.write_pascal_bytes(val.encode()) 324 | 325 | def read_python_int(self, _len): 326 | 327 | return struct.unpack(f'{self._endian}{_len}P', self.read(_len))[0] 328 | 329 | def write_python_int(self, val): 330 | 331 | self.write(struct.pack(f'{self._endian}{len(val)}P', val)) 332 | 333 | 334 | class ConfigParser(RawConfigParser): 335 | """配置解析类 336 | """ 337 | 338 | def getstr(self, section, option, default=None, **kwargs): 339 | 340 | val = self.get(section, option, **kwargs) 341 | 342 | return val if val else default 343 | 344 | def getjson(self, section, option, **kwargs): 345 | 346 | val = self.get(section, option, **kwargs) 347 | 348 | result = Utils.json_encode(val) 349 | 350 | return result 351 | 352 | def _split_host(self, val): 353 | 354 | if val.find(r':') > 0: 355 | host, port = val.split(r':', 2) 356 | return host.strip(), int(port.strip()) 357 | else: 358 | return None 359 | 360 | def get_split_host(self, section, option, **kwargs): 361 | 362 | val = self.get(section, option, **kwargs) 363 | 364 | return self._split_host(val) 365 | 366 | def get_split_str(self, section, option, sep=r'|', **kwargs): 367 | 368 | val = self.get(section, option, **kwargs) 369 | 370 | return Utils.split_str(val, sep) 371 | 372 | def get_split_int(self, section, option, sep=r',', **kwargs): 373 | 374 | val = self.get(section, option, **kwargs) 375 | 376 | return Utils.split_int(val, sep) 377 | 378 | def split_float(self, val, sep=r','): 379 | 380 | result = tuple(float(item.strip()) for item in val.split(sep)) 381 | 382 | return result 383 | 384 | def get_split_float(self, section, option, sep=r',', **kwargs): 385 | 386 | val = self.get(section, option, **kwargs) 387 | 388 | return self.split_float(val, sep) 389 | 390 | 391 | class Configure(Const): 392 | """配置类 393 | """ 394 | 395 | def __init__(self): 396 | 397 | super().__init__() 398 | 399 | self._parser = ConfigParser() 400 | 401 | def _init_options(self): 402 | 403 | self.clear() 404 | 405 | def get_option(self, section, option): 406 | 407 | return self._parser.get(section, option) 408 | 409 | def get_options(self, section): 410 | 411 | parser = self._parser 412 | 413 | options = {} 414 | 415 | for option in parser.options(section): 416 | options[option] = parser.get(section, option) 417 | 418 | return options 419 | 420 | def set_options(self, section, **options): 421 | 422 | if not self._parser.has_section(section): 423 | self._parser.add_section(section) 424 | 425 | for option, value in options.items(): 426 | self._parser.set(section, option, value) 427 | 428 | self._init_options() 429 | 430 | def read(self, files): 431 | 432 | self._parser.clear() 433 | self._parser.read(files, r'utf-8') 434 | 435 | self._init_options() 436 | 437 | def read_str(self, val): 438 | 439 | self._parser.clear() 440 | self._parser.read_string(val) 441 | 442 | self._init_options() 443 | 444 | def read_dict(self, val): 445 | 446 | self._parser.clear() 447 | self._parser.read_dict(val) 448 | 449 | self._init_options() 450 | 451 | 452 | class KeyLowerDict(dict): 453 | 454 | _PATTERN = re.compile(r'(?<=[a-z])([A-Z])') 455 | 456 | def __init__(self, _dict): 457 | 458 | super().__init__( 459 | { 460 | KeyLowerDict._PATTERN.sub(r'_\1', key).lower(): KeyLowerDict(val) if isinstance(val, dict) else val 461 | for key, val in _dict.items() 462 | } 463 | ) 464 | -------------------------------------------------------------------------------- /hagworm/extend/transaction.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .base import Utils, ContextManager 4 | 5 | 6 | class TransactionInterface: 7 | """事务接口 8 | """ 9 | 10 | def __init__(self, *, commit_callback=None, rollback_callback=None): 11 | 12 | self._commit_callbacks = [] 13 | self._rollback_callbacks = [] 14 | 15 | if commit_callback is not None: 16 | self._commit_callbacks.append(commit_callback) 17 | 18 | if rollback_callback is not None: 19 | self._rollback_callbacks.append(rollback_callback) 20 | 21 | def _clear_callbacks(self): 22 | 23 | self._commit_callbacks.clear() 24 | self._rollback_callbacks.clear() 25 | 26 | self._commit_callbacks = self._rollback_callbacks = None 27 | 28 | def add_commit_callback(self, _callable, *args, **kwargs): 29 | 30 | if self._commit_callbacks is not None: 31 | if args or kwargs: 32 | self._commit_callbacks.append( 33 | Utils.func_partial(_callable, *args, **kwargs) 34 | ) 35 | else: 36 | self._commit_callbacks.append(_callable) 37 | 38 | def add_rollback_callback(self, _callable, *args, **kwargs): 39 | 40 | if self._rollback_callbacks is not None: 41 | if args or kwargs: 42 | self._rollback_callbacks.append( 43 | Utils.func_partial(_callable, *args, **kwargs) 44 | ) 45 | else: 46 | self._rollback_callbacks.append(_callable) 47 | 48 | def commit(self): 49 | 50 | raise NotImplementedError() 51 | 52 | def rollback(self): 53 | 54 | raise NotImplementedError() 55 | 56 | def bind(self, trx): 57 | 58 | self.add_commit_callback(trx.commit) 59 | self.add_rollback_callback(trx.rollback) 60 | 61 | 62 | class Transaction(TransactionInterface, ContextManager): 63 | """事务对象 64 | 65 | 使用上下文实现的一个事务对象，可以设置commit和rollback回调 66 | 未显示commit的情况下，会自动rollback 67 | 68 | """ 69 | 70 | def _context_release(self): 71 | 72 | self.rollback() 73 | 74 | def commit(self): 75 | 76 | if self._commit_callbacks is None: 77 | return 78 | 79 | callbacks = self._commit_callbacks.copy() 80 | 81 | self._clear_callbacks() 82 | 83 | for _callable in callbacks: 84 | try: 85 | _callable() 86 | except Exception as err: 87 | Utils.log.critical(f'transaction commit error:\n{err}') 88 | 89 | def rollback(self): 90 | 91 | if self._rollback_callbacks is None: 92 | return 93 | 94 | callbacks = self._rollback_callbacks.copy() 95 | 96 | self._clear_callbacks() 97 | 98 | for _callable in callbacks: 99 | try: 100 | _callable() 101 | except Exception as err: 102 | Utils.log.critical(f'transaction rollback error:\n{err}') 103 | -------------------------------------------------------------------------------- /hagworm/frame/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /hagworm/frame/stress_tests.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | 5 | from terminal_table import Table 6 | 7 | from hagworm.extend.logging import DEFAULT_LOG_FILE_ROTATOR 8 | from hagworm.extend.interface import RunnableInterface 9 | from hagworm.extend.asyncio.base import Launcher as _Launcher 10 | from hagworm.extend.asyncio.base import Utils, MultiTasks, AsyncCirculatorForSecond 11 | from hagworm.extend.asyncio.zmq import Subscriber, PublisherWithBuffer 12 | 13 | 14 | SIGNAL_PROTOCOL = r'tcp' 15 | SIGNAL_PORT = 0x310 16 | HIGH_WATER_MARK = 0xffffff 17 | 18 | 19 | class Guardian(RunnableInterface): 20 | 21 | async def _do_polling(self, pids, hwm): 22 | 23 | with Reporter() as reporter: 24 | 25 | reporter.set_hwm(hwm) 26 | 27 | async for _ in AsyncCirculatorForSecond(): 28 | 29 | for pid in pids.copy(): 30 | if os.waitpid(pid, os.WNOHANG)[0] == pid: 31 | pids.remove(pid) 32 | 33 | if len(pids) == 0: 34 | break 35 | 36 | Utils.log.info(f'\n{reporter.get_report_table()}') 37 | 38 | def run(self, pids): 39 | 40 | global HIGH_WATER_MARK 41 | 42 | Utils.run_until_complete(self._do_polling(pids, HIGH_WATER_MARK)) 43 | 44 | 45 | class Launcher(_Launcher): 46 | 47 | def __init__(self, 48 | log_file_path=None, log_level=r'INFO', 49 | log_file_rotation=DEFAULT_LOG_FILE_ROTATOR, log_file_retention=0xff, 50 | process_number=1, process_guardian=None, 51 | debug=False 52 | ): 53 | 54 | if process_guardian is None: 55 | process_guardian = Guardian().run 56 | 57 | super().__init__( 58 | log_file_path, log_level, log_file_rotation, log_file_retention, 59 | process_number, process_guardian, 60 | debug 61 | ) 62 | 63 | def run(self, func, *args, **kwargs): 64 | 65 | if self._process_number > 1: 66 | 67 | super().run(func, *args, **kwargs) 68 | 69 | else: 70 | 71 | async def _func(): 72 | 73 | nonlocal func, args, kwargs 74 | 75 | with Reporter() as reporter: 76 | await func(*args, **kwargs) 77 | Utils.log.info(f'\n{reporter.get_report_table()}') 78 | 79 | super().run(_func) 80 | 81 | 82 | class Reporter(Subscriber): 83 | 84 | class _Report: 85 | 86 | def __init__(self): 87 | self.success = [] 88 | self.failure = [] 89 | 90 | def __init__(self): 91 | 92 | global SIGNAL_PROTOCOL, SIGNAL_PORT, HIGH_WATER_MARK 93 | 94 | super().__init__(f'{SIGNAL_PROTOCOL}://*:{SIGNAL_PORT}', True) 95 | 96 | self.set_hwm(HIGH_WATER_MARK) 97 | 98 | self._reports = {} 99 | 100 | async def _message_handler(self, data): 101 | 102 | for name, result, resp_time in data: 103 | if name and result in (r'success', r'failure'): 104 | getattr(self._get_report(name), result).append(resp_time) 105 | 106 | def _get_report(self, name: str) -> _Report: 107 | 108 | if name not in self._reports: 109 | self._reports[name] = self._Report() 110 | 111 | return self._reports[name] 112 | 113 | def get_report_table(self) -> str: 114 | 115 | reports = [] 116 | 117 | for key, val in self._reports.items(): 118 | reports.append( 119 | ( 120 | key, 121 | len(val.success), 122 | len(val.failure), 123 | r'{:.2%}'.format(len(val.success) / (len(val.success) + len(val.failure))), 124 | r'{:.3f}s'.format(sum(val.success) / len(val.success) if len(val.success) > 0 else 0), 125 | r'{:.3f}s'.format(min(val.success) if len(val.success) > 0 else 0), 126 | r'{:.3f}s'.format(max(val.success) if len(val.success) > 0 else 0), 127 | ) 128 | ) 129 | 130 | return Table.create( 131 | reports, 132 | ( 133 | r'EventName', 134 | r'SuccessTotal', 135 | r'FailureTotal', 136 | r'SuccessRatio', 137 | r'SuccessAveTime', 138 | r'SuccessMinTime', 139 | r'SuccessMaxTime', 140 | ), 141 | use_ansi=False 142 | ) 143 | 144 | 145 | class TaskInterface(Utils, RunnableInterface): 146 | 147 | def __init__(self, publisher: PublisherWithBuffer): 148 | 149 | self._publisher = publisher 150 | 151 | def success(self, name: str, resp_time: int): 152 | 153 | self._publisher.append( 154 | (name, r'success', resp_time,) 155 | ) 156 | 157 | def failure(self, name: str, resp_time: int): 158 | 159 | self._publisher.append( 160 | (name, r'failure', resp_time,) 161 | ) 162 | 163 | async def run(self): 164 | raise NotImplementedError() 165 | 166 | 167 | class Runner(Utils, RunnableInterface): 168 | 169 | def __init__(self, task_cls: TaskInterface): 170 | 171 | global SIGNAL_PROTOCOL, SIGNAL_PORT 172 | 173 | self._task_cls = task_cls 174 | self._publisher = PublisherWithBuffer(f'{SIGNAL_PROTOCOL}://localhost:{SIGNAL_PORT}', False) 175 | 176 | async def run(self, times, task_num): 177 | 178 | self._publisher.open() 179 | 180 | for _ in range(times): 181 | 182 | tasks = MultiTasks() 183 | 184 | for _ in range(task_num): 185 | tasks.append(self._task_cls(self._publisher).run()) 186 | 187 | await tasks 188 | 189 | await self._publisher.safe_close() 190 | -------------------------------------------------------------------------------- /hagworm/frame/tornado/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /hagworm/frame/tornado/base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import signal 4 | import asyncio 5 | import logging 6 | 7 | import jinja2 8 | 9 | from tornado_jinja2 import Jinja2Loader 10 | 11 | from tornado.web import Application 12 | from tornado.options import options 13 | from tornado.process import cpu_count, fork_processes 14 | from tornado.netutil import bind_sockets 15 | from tornado.httpserver import HTTPServer 16 | from tornado.platform.asyncio import AsyncIOMainLoop 17 | 18 | from hagworm import package_slogan 19 | from hagworm import __version__ as package_version 20 | from hagworm.extend.base import Utils 21 | from hagworm.extend.logging import DEFAULT_LOG_FILE_ROTATOR, DEFAULT_LOG_INTERCEPTOR 22 | from hagworm.extend.interface import TaskInterface 23 | from hagworm.extend.asyncio.base import install_uvloop 24 | from hagworm.frame.tornado.web import LogRequestMixin 25 | 26 | 27 | class _LauncherBase(TaskInterface): 28 | """启动器基类 29 | """ 30 | 31 | def __init__(self, **kwargs): 32 | 33 | self._debug = kwargs.get(r'debug', False) 34 | 35 | self._process_num = kwargs.get(r'process_num', 1) 36 | self._async_initialize = kwargs.get(r'async_initialize', None) 37 | 38 | self._background_service = kwargs.get(r'background_service', None) 39 | self._background_process = kwargs.get(r'background_process', None) 40 | 41 | self._process_id = 0 42 | self._process_num = self._process_num if self._process_num > 0 else cpu_count() 43 | 44 | # 后台服务任务对象 45 | if self._background_service is None: 46 | pass 47 | elif not isinstance(self._background_service, TaskInterface): 48 | raise TypeError(r'Background Service Dot Implemented Task Interface') 49 | 50 | # 服务进程任务对象，服务进程不监听端口 51 | if self._background_process is None: 52 | pass 53 | elif isinstance(self._background_process, TaskInterface): 54 | self._process_num += 1 55 | else: 56 | raise TypeError(r'Background Process Dot Implemented Task Interface') 57 | 58 | self._init_logger( 59 | kwargs.get(r'log_level', r'info').upper(), 60 | kwargs.get(r'log_handler', None), 61 | kwargs.get(r'log_file_path', None), 62 | kwargs.get(r'log_file_rotation', DEFAULT_LOG_FILE_ROTATOR), 63 | kwargs.get(r'log_file_retention', 0xff) 64 | ) 65 | 66 | environment = Utils.environment() 67 | 68 | Utils.log.info( 69 | f'{package_slogan}' 70 | f'hagworm {package_version}\n' 71 | f'python {environment["python"]}\n' 72 | f'system {" ".join(environment["system"])}' 73 | ) 74 | 75 | install_uvloop() 76 | 77 | self._event_loop = None 78 | 79 | def _init_logger(self, 80 | log_level, log_handler=None, log_file_path=None, 81 | log_file_rotation=DEFAULT_LOG_FILE_ROTATOR, log_file_retention=0xff 82 | ): 83 | 84 | if log_handler or log_file_path: 85 | 86 | Utils.log.remove() 87 | 88 | if log_handler: 89 | 90 | Utils.log.add( 91 | log_handler, 92 | level=log_level, 93 | enqueue=True, 94 | backtrace=self._debug 95 | ) 96 | 97 | if log_file_path: 98 | 99 | log_file_path = Utils.path.join( 100 | log_file_path, 101 | r'runtime_{time}.log' 102 | ) 103 | 104 | Utils.log.add( 105 | log_file_path, 106 | level=log_level, 107 | enqueue=True, 108 | backtrace=self._debug, 109 | rotation=log_file_rotation, 110 | retention=log_file_retention 111 | ) 112 | 113 | else: 114 | 115 | Utils.log.level(log_level) 116 | 117 | logging.getLogger(None).addHandler(DEFAULT_LOG_INTERCEPTOR) 118 | 119 | @property 120 | def process_id(self): 121 | 122 | return self._process_id 123 | 124 | def start(self): 125 | 126 | if self._background_service is not None: 127 | self._background_service.start() 128 | Utils.log.success(f'Background service no.{self._process_id} running...') 129 | 130 | if self._process_id == 0 and self._background_process is not None: 131 | self._background_process.start() 132 | Utils.log.success(f'Background process no.{self._process_id} running...') 133 | else: 134 | self._server.add_sockets(self._sockets) 135 | 136 | Utils.log.success(f'Startup server no.{self._process_id}') 137 | 138 | self._event_loop.run_forever() 139 | 140 | def stop(self, code=0): 141 | 142 | if self._background_service is not None: 143 | self._background_service.stop() 144 | 145 | if self._process_id == 0 and self._background_process is not None: 146 | self._background_process.stop() 147 | 148 | self._event_loop.stop() 149 | 150 | Utils.log.success(f'Shutdown server no.{self._process_id}: code.{code}') 151 | 152 | def is_running(self): 153 | 154 | return self._event_loop.is_running() 155 | 156 | 157 | class _Application(Application): 158 | 159 | def log_request(self, handler): 160 | 161 | if isinstance(handler, LogRequestMixin): 162 | handler.log_request() 163 | super().log_request(handler) 164 | elif self.settings.get(r'debug') or handler.get_status() >= 400: 165 | super().log_request(handler) 166 | 167 | 168 | class Launcher(_LauncherBase): 169 | """TornadoHttp的启动器 170 | 171 | 用于简化和统一程序的启动操作 172 | 173 | """ 174 | 175 | def __init__(self, router, port=80, **kwargs): 176 | 177 | super().__init__(**kwargs) 178 | 179 | self._settings = { 180 | r'handlers': router, 181 | r'debug': self._debug, 182 | r'gzip': kwargs.get(r'gzip', False), 183 | } 184 | 185 | if r'template_path' in kwargs: 186 | self._settings[r'template_loader'] = Jinja2Loader( 187 | jinja2.Environment( 188 | loader=jinja2.FileSystemLoader(kwargs[r'template_path']) 189 | ) 190 | ) 191 | 192 | if r'static_path' in kwargs: 193 | self._settings[r'static_path'] = kwargs[r'static_path'] 194 | 195 | if r'cookie_secret' in kwargs: 196 | self._settings[r'cookie_secret'] = kwargs[r'cookie_secret'] 197 | 198 | self._sockets = bind_sockets(port) 199 | 200 | if self._process_num > 1: 201 | self._process_id = fork_processes(self._process_num) 202 | 203 | options.parse_command_line() 204 | 205 | AsyncIOMainLoop().install() 206 | 207 | self._event_loop = asyncio.get_event_loop() 208 | self._event_loop.set_debug(self._settings[r'debug']) 209 | 210 | self._event_loop.add_signal_handler(signal.SIGINT, self.stop) 211 | self._event_loop.add_signal_handler(signal.SIGTERM, self.stop) 212 | 213 | self._server = HTTPServer(_Application(**self._settings)) 214 | 215 | if self._async_initialize: 216 | self._event_loop.run_until_complete(self._async_initialize()) 217 | -------------------------------------------------------------------------------- /hagworm/frame/tornado/socket.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import signal 4 | import asyncio 5 | 6 | from tornado.process import fork_processes 7 | from tornado.netutil import bind_sockets 8 | from tornado.tcpserver import TCPServer 9 | from tornado.iostream import StreamClosedError 10 | from tornado.platform.asyncio import AsyncIOMainLoop 11 | 12 | from hagworm.extend.base import Utils 13 | from hagworm.frame.tornado.base import _LauncherBase 14 | 15 | 16 | class _TCPServer(TCPServer): 17 | """TCPServer实现类 18 | """ 19 | 20 | def __init__(self, protocol, *args, **kwargs): 21 | 22 | super().__init__(*args, **kwargs) 23 | 24 | self._protocol = protocol 25 | 26 | async def handle_stream(self, stream, address): 27 | 28 | try: 29 | 30 | await self._protocol(stream, address) 31 | 32 | except Exception as err: 33 | 34 | Utils.log.error(err) 35 | 36 | finally: 37 | 38 | if not stream.closed(): 39 | await stream.close() 40 | 41 | 42 | class Protocol: 43 | """Protocol实现类 44 | """ 45 | 46 | def __init__(self, stream, address): 47 | 48 | self._stream = stream 49 | self._address = address 50 | 51 | def __await__(self): 52 | 53 | try: 54 | 55 | yield from self.connection_made().__await__() 56 | 57 | while True: 58 | 59 | try: 60 | yield from self._read_bytes().__await__() 61 | except StreamClosedError: 62 | break 63 | except Exception as err: 64 | Utils.log.error(err) 65 | 66 | except Exception as err: 67 | 68 | Utils.log.error(err) 69 | 70 | finally: 71 | 72 | yield from self.connection_lost().__await__() 73 | 74 | return self 75 | 76 | async def _read_bytes(self): 77 | 78 | await self.data_received( 79 | await self._stream.read_bytes(65536, True) 80 | ) 81 | 82 | @property 83 | def closed(self): 84 | 85 | self._stream.closed() 86 | 87 | @property 88 | def client_ip(self): 89 | 90 | return self._address[0] 91 | 92 | @property 93 | def client_port(self): 94 | 95 | return self._address[1] 96 | 97 | @property 98 | def client_address(self): 99 | 100 | return Utils.join_str(self._address, r':') 101 | 102 | async def close(self): 103 | 104 | self._stream.close() 105 | 106 | async def connection_made(self): 107 | 108 | raise NotImplementedError() 109 | 110 | async def connection_lost(self): 111 | 112 | raise NotImplementedError() 113 | 114 | async def data_received(self, chunk): 115 | 116 | raise NotImplementedError() 117 | 118 | async def data_write(self, chunk): 119 | 120 | await self._stream.write(chunk) 121 | 122 | 123 | class Launcher(_LauncherBase): 124 | """TornadoTCP的启动器 125 | 126 | 用于简化和统一程序的启动操作 127 | 128 | """ 129 | 130 | def __init__(self, protocol, port, **kwargs): 131 | 132 | super().__init__(**kwargs) 133 | 134 | if not issubclass(protocol, Protocol): 135 | raise TypeError(r'Dot Implemented Protocol Interface') 136 | 137 | self._settings = { 138 | r'ssl_options': kwargs.get(r'ssl_options', None), 139 | r'max_buffer_size': kwargs.get(r'max_buffer_size', None), 140 | r'read_chunk_size': kwargs.get(r'read_chunk_size', None), 141 | } 142 | 143 | self._sockets = bind_sockets(port) 144 | 145 | if self._process_num > 1: 146 | self._process_id = fork_processes(self._process_num) 147 | 148 | AsyncIOMainLoop().install() 149 | 150 | self._event_loop = asyncio.get_event_loop() 151 | self._event_loop.set_debug(self._debug) 152 | 153 | self._event_loop.add_signal_handler(signal.SIGINT, self.stop) 154 | self._event_loop.add_signal_handler(signal.SIGTERM, self.stop) 155 | 156 | self._server = _TCPServer(protocol, **self._settings) 157 | 158 | if self._async_initialize: 159 | self._event_loop.run_until_complete(self._async_initialize()) 160 | -------------------------------------------------------------------------------- /hagworm/frame/tornado/web.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | import aiohttp 5 | 6 | from aiohttp.web_exceptions import HTTPBadGateway 7 | 8 | from tornado.web import RequestHandler 9 | from tornado.websocket import WebSocketHandler 10 | 11 | from hagworm.extend.struct import Result 12 | from hagworm.extend.asyncio.base import Utils 13 | from hagworm.extend.asyncio.net import DownloadBuffer, HTTPClientPool 14 | 15 | from wtforms_tornado import Form 16 | 17 | 18 | PROXY_IGNORE_HEADERS = (r'CONTENT-ENCODING', r'TRANSFER-ENCODING',) 19 | 20 | 21 | def json_wraps(func): 22 | """json装饰器 23 | """ 24 | 25 | @Utils.func_wraps(func) 26 | async def _wrapper(handler, *args, **kwargs): 27 | 28 | resp = await Utils.awaitable_wrapper( 29 | func(handler, *args, **kwargs) 30 | ) 31 | 32 | if isinstance(resp, Result): 33 | return handler.write_json(resp) 34 | 35 | return _wrapper 36 | 37 | 38 | class HttpBasicAuth: 39 | """Http基础认证装饰器 40 | """ 41 | 42 | def __init__(self, realm, username, password): 43 | 44 | self._realm = realm 45 | self._username = username 46 | self._password = password 47 | 48 | def __call__(self, func): 49 | 50 | @Utils.func_wraps(func) 51 | def _wrapper(handler, *args, **kwargs): 52 | 53 | auth_header = handler.get_header(r'Authorization') 54 | 55 | try: 56 | 57 | if auth_header: 58 | 59 | auth_info = Utils.b64_decode(auth_header.split(r' ', 2)[1]) 60 | 61 | if auth_info == f'{self._username}:{self._password}': 62 | return func(handler, *args, **kwargs) 63 | 64 | except Exception as err: 65 | 66 | Utils.log.error(err) 67 | 68 | handler.set_header( 69 | r'WWW-Authenticate', 70 | f'Basic realm="{self._realm}"' 71 | ) 72 | handler.set_status(401) 73 | 74 | return handler.finish() 75 | 76 | return _wrapper 77 | 78 | 79 | class FormInjection: 80 | """表单注入器 81 | """ 82 | 83 | def __init__(self, form_cls=None, err_code: int = -1): 84 | 85 | if not issubclass(form_cls, Form): 86 | raise TypeError(r'Dot Implemented Form Interface') 87 | 88 | self._form_cls = form_cls 89 | self._err_code = err_code 90 | 91 | def __call__(self, func): 92 | 93 | @Utils.func_wraps(func) 94 | async def _wrapper(handler: RequestBaseHandler, *args, **kwargs): 95 | 96 | form = self._form_cls(handler.request.arguments) 97 | 98 | setattr(handler, r'form', form) 99 | setattr(handler, r'data', form.data) 100 | 101 | if form.validate(): 102 | 103 | await json_wraps(func)(handler, *args, **kwargs) 104 | 105 | else: 106 | 107 | return handler.write_json( 108 | Result(self._err_code, extra=form.errors), 109 | 400 110 | ) 111 | 112 | return _wrapper 113 | 114 | 115 | class DebugHeader: 116 | """调试信息注入器 117 | """ 118 | 119 | def __call__(self, func): 120 | 121 | @Utils.func_wraps(func) 122 | async def _wrapper(handler: RequestBaseHandler, *args, **kwargs): 123 | 124 | if self._is_debug() is True: 125 | self._receive_header( 126 | handler.get_header(r'Debug-Data') 127 | ) 128 | 129 | await func(handler, *args, **kwargs) 130 | 131 | if self._is_debug() is True: 132 | handler.set_header( 133 | r'Debug-Data', 134 | self._get_debug_header() 135 | ) 136 | 137 | return _wrapper 138 | 139 | def _is_debug(self) -> bool: 140 | 141 | raise NotImplementedError() 142 | 143 | def _receive_header(self, data: str): 144 | 145 | raise NotImplementedError() 146 | 147 | def _get_debug_header(self) -> str: 148 | 149 | raise NotImplementedError() 150 | 151 | 152 | class LogRequestMixin: 153 | 154 | def log_request(self): 155 | 156 | Utils.log.info( 157 | '\n---------- request arguments ----------\n' + 158 | Utils.json_encode( 159 | { 160 | key: [Utils.basestring(val) for val in items] 161 | for key, items in self.request.arguments.items() 162 | }, 163 | escape_forward_slashes=False, ensure_ascii=False, indent=4 164 | ) 165 | ) 166 | 167 | 168 | class _BaseHandlerMixin(Utils): 169 | """Handler基础工具混入类 170 | """ 171 | 172 | @property 173 | def request_module(self): 174 | 175 | return f'{self.module}.{self.method}' 176 | 177 | @property 178 | def module(self): 179 | 180 | _class = self.__class__ 181 | 182 | return f'{_class.__module__}.{_class.__name__}' 183 | 184 | @property 185 | def method(self): 186 | 187 | return self.request.method.lower() 188 | 189 | @property 190 | def version(self): 191 | 192 | return self.request.version.lower() 193 | 194 | @property 195 | def protocol(self): 196 | 197 | return self.request.protocol 198 | 199 | @property 200 | def host(self): 201 | 202 | return self.request.host 203 | 204 | @property 205 | def path(self): 206 | 207 | return self.request.path 208 | 209 | @property 210 | def query(self): 211 | 212 | return self.request.query 213 | 214 | @property 215 | def body(self): 216 | 217 | return self.request.body 218 | 219 | @property 220 | def files(self): 221 | 222 | return self.request.files 223 | 224 | @property 225 | def referer(self): 226 | 227 | return self.get_header(r'Referer', r'') 228 | 229 | @property 230 | def client_ip(self): 231 | 232 | return self.get_header(r'X-Real-IP', self.request.remote_ip) 233 | 234 | @property 235 | def content_type(self): 236 | 237 | return self.get_header(r'Content-Type', r'') 238 | 239 | @property 240 | def content_length(self): 241 | 242 | result = self.get_header(r'Content-Length', r'') 243 | 244 | return int(result) if result.isdigit() else 0 245 | 246 | @property 247 | def headers(self): 248 | 249 | return self.request.headers 250 | 251 | def get_header(self, name, default=None): 252 | """ 253 | 获取header数据 254 | """ 255 | 256 | return self.request.headers.get(name, default) 257 | 258 | 259 | class SocketBaseHandler(WebSocketHandler, _BaseHandlerMixin): 260 | """WebSocket请求处理类 261 | """ 262 | 263 | @property 264 | def closed(self): 265 | 266 | return self.ws_connection is None 267 | 268 | def check_origin(self, origin): 269 | 270 | return True 271 | 272 | 273 | class RequestBaseHandler(RequestHandler, _BaseHandlerMixin): 274 | """Http请求处理类 275 | """ 276 | 277 | def initialize(self, **kwargs): 278 | 279 | setattr(self, r'_payload', kwargs) 280 | 281 | @property 282 | def payload(self): 283 | 284 | return getattr(self, r'_payload', None) 285 | 286 | @property 287 | def closed(self): 288 | 289 | return self.request.connection.stream.closed() 290 | 291 | def head(self, *_1, **_2): 292 | 293 | self.finish() 294 | 295 | def options(self, *_1, **_2): 296 | 297 | self.finish() 298 | 299 | async def prepare(self): 300 | 301 | self._parse_json_arguments() 302 | 303 | def set_default_headers(self): 304 | 305 | self.set_header(r'Cache-Control', r'no-cache') 306 | 307 | self.set_header(r'X-Timestamp', self.timestamp()) 308 | 309 | payload = self.get_header(r'X-Payload') 310 | 311 | if payload: 312 | self.set_header(r'X-Payload', payload) 313 | 314 | origin = self.get_header(r'Origin') 315 | 316 | if origin: 317 | 318 | self.set_header(r'Access-Control-Allow-Origin', r'*') 319 | 320 | method = self.get_header(r'Access-Control-Request-Method') 321 | if method: 322 | self.set_header(r'Access-Control-Allow-Methods', method) 323 | 324 | headers = self.get_header(r'Access-Control-Request-Headers') 325 | if headers: 326 | self.set_header(r'Access-Control-Allow-Headers', headers) 327 | 328 | self.set_header(r'Access-Control-Max-Age', r'86400') 329 | self.set_header(r'Access-Control-Allow-Credentials', r'true') 330 | 331 | def set_cookie(self, name, value, domain=None, expires=None, path="/", expires_days=None, **kwargs): 332 | 333 | if type(value) not in (str, bytes): 334 | value = str(value) 335 | 336 | return super().set_cookie(name, value, domain, expires, path, expires_days, **kwargs) 337 | 338 | def get_secure_cookie(self, name, value=None, max_age_days=31, min_version=None): 339 | 340 | result = super().get_secure_cookie(name, value, max_age_days, min_version) 341 | 342 | return self.basestring(result) 343 | 344 | def set_secure_cookie(self, name, value, expires_days=30, version=None, **kwargs): 345 | 346 | if type(value) not in (str, bytes): 347 | value = str(value) 348 | 349 | return super().set_secure_cookie(name, value, expires_days, version, **kwargs) 350 | 351 | def get_current_user(self): 352 | 353 | session = self.get_cookie(r'session') 354 | 355 | if not session: 356 | session = self.uuid1() 357 | self.set_cookie(r'session', session) 358 | 359 | self.current_user = session 360 | 361 | return session 362 | 363 | def compute_etag(self): 364 | 365 | return None 366 | 367 | def _parse_json_arguments(self): 368 | 369 | self.request.json_arguments = {} 370 | 371 | content_type = self.content_type 372 | 373 | if content_type and content_type.find(r'application/json') >= 0 and self.body: 374 | 375 | try: 376 | 377 | json_args = self.json_decode(self.body) 378 | 379 | if isinstance(json_args, dict): 380 | 381 | self.request.json_arguments.update(json_args) 382 | 383 | for key, val in self.request.json_arguments.items(): 384 | 385 | if not isinstance(val, str): 386 | val = str(val) 387 | 388 | self.request.arguments.setdefault(key, []).append(val) 389 | 390 | except Exception as _: 391 | 392 | self.log.debug(f'Invalid application/json body: {self.body}') 393 | 394 | def get_files(self, name): 395 | """ 396 | 获取files数据 397 | """ 398 | 399 | result = [] 400 | 401 | file_data = self.files.get(name, None) 402 | 403 | if file_data is not None: 404 | self.list_extend(result, file_data) 405 | 406 | for index in range(len(self.files)): 407 | 408 | file_data = self.files.get(f'{name}[{index}]', None) 409 | 410 | if file_data is not None: 411 | self.list_extend(result, file_data) 412 | 413 | return result 414 | 415 | def get_arg_str(self, name, default=r'', length=0): 416 | """ 417 | 获取str型输入 418 | """ 419 | 420 | result = self.get_argument(name, None, True) 421 | 422 | if result is None: 423 | return default 424 | 425 | if not isinstance(result, str): 426 | result = str(result) 427 | 428 | if (length > 0) and (len(result) > length): 429 | result = result[0:length] 430 | 431 | return result 432 | 433 | def get_arg_bool(self, name, default=False): 434 | """ 435 | 获取bool型输入 436 | """ 437 | 438 | result = self.get_argument(name, None, True) 439 | 440 | if result is None: 441 | return default 442 | 443 | result = self.convert_bool(result) 444 | 445 | return result 446 | 447 | def get_arg_int(self, name, default=0, min_val=None, max_val=None): 448 | """ 449 | 获取int型输入 450 | """ 451 | 452 | result = self.get_argument(name, None, True) 453 | 454 | if result is None: 455 | return default 456 | 457 | result = self.convert_int(result, default) 458 | result = self.interval_limit(result, min_val, max_val) 459 | 460 | return result 461 | 462 | def get_arg_float(self, name, default=0.0, min_val=None, max_val=None): 463 | """ 464 | 获取float型输入 465 | """ 466 | 467 | result = self.get_argument(name, None, True) 468 | 469 | if result is None: 470 | return default 471 | 472 | result = self.convert_float(result, default) 473 | result = self.interval_limit(result, min_val, max_val) 474 | 475 | return result 476 | 477 | def get_arg_json(self, name, default=None): 478 | """ 479 | 获取json型输入 480 | """ 481 | 482 | result = default 483 | 484 | value = self.get_argument(name, None, True) 485 | 486 | if value is not None: 487 | try: 488 | result = self.json_decode(value) 489 | except Exception as _: 490 | self.log.debug(f'Invalid application/json argument({name}): {value}') 491 | 492 | return result 493 | 494 | def get_json_argument(self, name, default=None): 495 | 496 | return self.request.json_arguments.get(name, default) 497 | 498 | def get_json_arguments(self): 499 | 500 | return self.deepcopy(self.request.json_arguments) 501 | 502 | def get_all_arguments(self): 503 | 504 | result = {} 505 | 506 | for key in self.request.arguments.keys(): 507 | result[key] = self.get_argument(key) 508 | 509 | return result 510 | 511 | def write_json(self, chunk, status_code=200): 512 | """ 513 | 输出JSON类型 514 | """ 515 | 516 | self.set_header(r'Content-Type', r'application/json') 517 | 518 | if status_code != 200: 519 | self.set_status(status_code) 520 | 521 | result = None 522 | 523 | try: 524 | result = self.json_encode(chunk) 525 | except Exception as _: 526 | self.log.error(f'json encode error: {chunk}') 527 | 528 | return self.finish(result) 529 | 530 | def write_png(self, chunk): 531 | """ 532 | 输出PNG类型 533 | """ 534 | 535 | self.set_header(r'Content-Type', r'image/png') 536 | 537 | return self.finish(chunk) 538 | 539 | 540 | class DownloadAgent(RequestBaseHandler, DownloadBuffer): 541 | """文件下载代理类 542 | """ 543 | 544 | def __init__(self, *args, **kwargs): 545 | 546 | RequestBaseHandler.__init__(self, *args, **kwargs) 547 | DownloadBuffer.__init__(self) 548 | 549 | def on_finish(self): 550 | 551 | self.close() 552 | 553 | async def _handle_response(self, response): 554 | 555 | for key, val in response.headers.items(): 556 | if key.upper() not in PROXY_IGNORE_HEADERS: 557 | self.set_header(key, val) 558 | 559 | return await DownloadBuffer._handle_response(self, response) 560 | 561 | async def _flush_data(self): 562 | 563 | while True: 564 | 565 | if self.closed and self.response: 566 | self.response.close() 567 | break 568 | 569 | chunk = self._file.read(65536) 570 | 571 | if chunk: 572 | self.write(chunk) 573 | await self.flush() 574 | elif self.finished: 575 | break 576 | else: 577 | await Utils.wait_frame() 578 | 579 | def _get_file_name(self, url): 580 | 581 | result = None 582 | 583 | try: 584 | result = os.path.split(self.urlparse.urlparse(url)[2])[1] 585 | except Exception as _: 586 | self.log.error(f'urlparse error: {url}') 587 | 588 | return result 589 | 590 | async def transmit(self, url, file_name=None, *, params=None, cookies=None, headers=None): 591 | 592 | try: 593 | 594 | _range = self.request.headers.get(r'Range', None) 595 | 596 | if _range is not None: 597 | 598 | if headers is None: 599 | headers = {r'Range': _range} 600 | else: 601 | headers[r'Range'] = _range 602 | 603 | if not file_name: 604 | file_name = self._get_file_name(url) 605 | 606 | if file_name: 607 | self.set_header( 608 | r'Content-Disposition', 609 | f'attachment;filename={file_name}' 610 | ) 611 | 612 | Utils.create_task(self.fetch(url, params=params, cookies=cookies, headers=headers)) 613 | 614 | await self._flush_data() 615 | 616 | except Exception as err: 617 | 618 | self.log.error(err) 619 | 620 | finally: 621 | 622 | self.finish() 623 | 624 | 625 | class HTTPProxy(HTTPClientPool): 626 | """简易的HTTP代理类，带连接池功能 627 | """ 628 | 629 | def __init__(self, 630 | use_dns_cache=True, ttl_dns_cache=10, 631 | limit=100, limit_per_host=0, timeout=None, 632 | **kwargs 633 | ): 634 | 635 | super().__init__(0, use_dns_cache, ttl_dns_cache, limit, limit_per_host, timeout, **kwargs) 636 | 637 | async def send_request(self, method: str, url: str, handler: RequestBaseHandler, **settings) -> int: 638 | 639 | global PROXY_IGNORE_HEADERS 640 | 641 | result = 0 642 | 643 | settings[r'data'] = handler.body 644 | settings[r'params'] = handler.query 645 | 646 | headers = dict(handler.headers) 647 | headers[r'Host'] = Utils.urlparse.urlparse(url).netloc 648 | settings[r'headers'] = headers 649 | 650 | settings.setdefault(r'ssl', self._ssl_context) 651 | 652 | try: 653 | 654 | async with aiohttp.ClientSession(**self._session_config) as _session: 655 | 656 | async with _session.request(method, url, **settings) as _response: 657 | 658 | for key, val in _response.headers.items(): 659 | if key.upper() not in PROXY_IGNORE_HEADERS: 660 | handler.set_header(key, val) 661 | 662 | while True: 663 | 664 | chunk = await _response.content.read(65536) 665 | 666 | if chunk: 667 | handler.write(chunk) 668 | await handler.flush() 669 | else: 670 | break 671 | 672 | handler.finish() 673 | 674 | result = _response.status 675 | 676 | Utils.log.info(f'{method} {url} => status:{_response.status}') 677 | 678 | except aiohttp.ClientResponseError as err: 679 | 680 | Utils.log.error(err) 681 | 682 | handler.send_error(err.status, reason=err.message) 683 | 684 | result = err.status 685 | 686 | except Exception as err: 687 | 688 | Utils.log.error(err) 689 | 690 | handler.send_error(HTTPBadGateway.status_code, reason=r'Proxy Internal Error') 691 | 692 | result = HTTPBadGateway.status_code 693 | 694 | return result 695 | 696 | async def get(self, url: str, handler: RequestBaseHandler) -> int: 697 | 698 | return await self.send_request(aiohttp.hdrs.METH_GET, url, handler) 699 | 700 | async def options(self, url: str, handler: RequestBaseHandler) -> int: 701 | 702 | return await self.send_request(aiohttp.hdrs.METH_OPTIONS, url, handler) 703 | 704 | async def head(self, url: str, handler: RequestBaseHandler) -> int: 705 | 706 | return await self.send_request(aiohttp.hdrs.METH_HEAD, url, handler) 707 | 708 | async def post(self, url: str, handler: RequestBaseHandler) -> int: 709 | 710 | return await self.send_request(aiohttp.hdrs.METH_POST, url, handler) 711 | 712 | async def put(self, url: str, handler: RequestBaseHandler) -> int: 713 | 714 | return await self.send_request(aiohttp.hdrs.METH_PUT, url, handler) 715 | 716 | async def patch(self, url: str, handler: RequestBaseHandler) -> int: 717 | 718 | return await self.send_request(aiohttp.hdrs.METH_PATCH, url, handler) 719 | 720 | async def delete(self, url: str, handler: RequestBaseHandler) -> int: 721 | 722 | return await self.send_request(aiohttp.hdrs.METH_DELETE, url, handler) 723 | -------------------------------------------------------------------------------- /hagworm/static/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /hagworm/third/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /hagworm/third/aliyun/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /hagworm/third/aliyun/rocketmq.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from mq_http_sdk.mq_client import MQClient as _MQClient 4 | from mq_http_sdk.mq_consumer import MQConsumer as _MQConsumer 5 | from mq_http_sdk.mq_producer import MQProducer as _MQProducer, MQTransProducer as _MQTransProducer 6 | from mq_http_sdk.mq_exception import MQServerException 7 | 8 | from hagworm.extend.interface import TaskInterface 9 | from hagworm.extend.asyncio.base import Utils 10 | from hagworm.extend.asyncio.task import LoopTask 11 | from hagworm.extend.asyncio.future import ThreadPool 12 | 13 | 14 | class MQConsumer(_MQConsumer): 15 | 16 | async def consume_message(self, batch_size=1, wait_seconds=-1): 17 | 18 | return await self.mq_client.thread_pool.run(super().consume_message, batch_size, wait_seconds) 19 | 20 | async def ack_message(self, receipt_handle_list): 21 | 22 | return await self.mq_client.thread_pool.run(super().ack_message, receipt_handle_list) 23 | 24 | 25 | class MQProducer(_MQProducer): 26 | 27 | async def publish_message(self, message): 28 | 29 | return await self.mq_client.thread_pool.run(super().publish_message, message) 30 | 31 | 32 | class MQTransProducer(_MQTransProducer): 33 | 34 | async def publish_message(self, message): 35 | 36 | return await self.mq_client.thread_pool.run(super().publish_message, message) 37 | 38 | async def consume_half_message(self, batch_size=1, wait_seconds=-1): 39 | 40 | return await self.mq_client.thread_pool.run(super().consume_half_message, batch_size, wait_seconds) 41 | 42 | async def commit(self, receipt_handle): 43 | 44 | return await self.mq_client.thread_pool.run(super().commit, receipt_handle) 45 | 46 | async def rollback(self, receipt_handle): 47 | 48 | return await self.mq_client.thread_pool.run(super().rollback, receipt_handle) 49 | 50 | 51 | class MQClient(_MQClient): 52 | 53 | def __init__(self, host, access_id, access_key, security_token=r'', debug=False, logger=None): 54 | 55 | super().__init__(host, access_id, access_key, security_token, debug, logger) 56 | 57 | # 原版SDK中的MQClient线程不安全，利用线程池仅有一个线程这种特例，保证线程安全 58 | self._thread_pool = ThreadPool(1) 59 | 60 | @property 61 | def thread_pool(self): 62 | 63 | return self._thread_pool 64 | 65 | def get_consumer(self, instance_id, topic_name, consumer, message_tag=r''): 66 | 67 | return MQConsumer(instance_id, topic_name, consumer, message_tag, self, self.debug) 68 | 69 | def get_producer(self, instance_id, topic_name): 70 | 71 | return MQProducer(instance_id, topic_name, self, self.debug) 72 | 73 | def get_trans_producer(self, instance_id, topic_name, group_id): 74 | 75 | return MQTransProducer(instance_id, topic_name, group_id, self, self.debug) 76 | 77 | 78 | class MQCycleConsumer(MQConsumer, LoopTask): 79 | 80 | def __init__(self, message_handler, instance_id, topic_name, consumer, message_tag, mq_client, debug=False): 81 | 82 | MQConsumer.__init__(self, instance_id, topic_name, consumer, message_tag, mq_client, debug) 83 | LoopTask.__init__(self, self._do_task, 1) 84 | 85 | self._message_handler = message_handler 86 | 87 | async def _do_task(self): 88 | 89 | msg_list = None 90 | 91 | try: 92 | msg_list = await self.consume_message(16, 30) 93 | except MQServerException as err: 94 | if err.type == r'MessageNotExist': 95 | Utils.log.debug(err) 96 | else: 97 | Utils.log.error(err) 98 | 99 | if msg_list: 100 | await Utils.awaitable_wrapper( 101 | self._message_handler(self, msg_list) 102 | ) 103 | 104 | 105 | class MQMultiCycleConsumers(TaskInterface): 106 | 107 | def __init__(self, client_num, host, access_id, access_key, security_token=r'', debug=False, logger=None): 108 | 109 | self._clients = [ 110 | MQClient(host, access_id, access_key, security_token, debug, logger) 111 | for _ in range(client_num) 112 | ] 113 | 114 | self._consumers = None 115 | 116 | self._running = False 117 | 118 | def init_consumers(self, 119 | message_handler, instance_id, topic_name, consumer, message_tag, debug=False, 120 | *, consumer_cls=MQCycleConsumer 121 | ): 122 | 123 | if self._consumers: 124 | self.stop() 125 | 126 | self._consumers = [ 127 | consumer_cls(message_handler, instance_id, topic_name, consumer, message_tag, client, debug) 128 | for client in self._clients 129 | ] 130 | 131 | def start(self): 132 | 133 | if self._running or self._consumers is None: 134 | return False 135 | 136 | self._running = True 137 | 138 | for consumer in self._consumers: 139 | consumer.start() 140 | 141 | return True 142 | 143 | def stop(self): 144 | 145 | if not self._running or self._consumers is None: 146 | return False 147 | 148 | self._running = False 149 | 150 | for consumer in self._consumers: 151 | consumer.stop() 152 | 153 | return True 154 | 155 | def is_running(self): 156 | 157 | return self._running 158 | 159 | 160 | class MQCycleTransProducer(MQTransProducer, LoopTask): 161 | 162 | def __init__(self, message_handler, instance_id, topic_name, group_id, mq_client, debug=False): 163 | 164 | MQTransProducer.__init__(self, instance_id, topic_name, group_id, mq_client, debug) 165 | LoopTask.__init__(self, self._do_task, 1) 166 | 167 | self._message_handler = message_handler 168 | 169 | async def _do_task(self): 170 | 171 | msg_list = None 172 | 173 | try: 174 | msg_list = await self.consume_half_message(16, 30) 175 | except MQServerException as err: 176 | if err.type == r'MessageNotExist': 177 | Utils.log.debug(err) 178 | else: 179 | Utils.log.error(err) 180 | 181 | if msg_list: 182 | await Utils.awaitable_wrapper( 183 | self._message_handler(self, msg_list) 184 | ) 185 | 186 | 187 | class MQMultiCycleTransProducers(TaskInterface): 188 | 189 | def __init__(self, client_num, host, access_id, access_key, security_token=r'', debug=False, logger=None): 190 | 191 | self._clients = [ 192 | MQClient(host, access_id, access_key, security_token, debug, logger) 193 | for _ in range(client_num) 194 | ] 195 | 196 | self._producers = None 197 | 198 | self._running = False 199 | 200 | def init_producers(self, 201 | message_handler, instance_id, topic_name, group_id, debug=False, 202 | *, producer_cls=MQCycleTransProducer 203 | ): 204 | 205 | if self._producers: 206 | self.stop() 207 | 208 | self._producers = [ 209 | producer_cls(message_handler, instance_id, topic_name, group_id, client, debug) 210 | for client in self._clients 211 | ] 212 | 213 | def start(self): 214 | 215 | if self._running or self._producers is None: 216 | return False 217 | 218 | self._running = True 219 | 220 | for consumer in self._producers: 221 | consumer.start() 222 | 223 | return True 224 | 225 | def stop(self): 226 | 227 | if not self._running or self._producers is None: 228 | return False 229 | 230 | self._running = False 231 | 232 | for producer in self._producers: 233 | producer.stop() 234 | 235 | return True 236 | 237 | def is_running(self): 238 | 239 | return self._running 240 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | 2 | pip3 uninstall -y hagworm 3 | python3 setup.py install 4 | 5 | rm -rf ./dist 6 | rm -rf ./build 7 | rm -rf ./hagworm.egg-info 8 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [easy_install] 2 | index_url = https://mirrors.aliyun.com/pypi/simple -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import setuptools 4 | 5 | from hagworm import __version__ as package_version 6 | 7 | 8 | with open(r'README.md', r'r', encoding=r'utf8') as stream: 9 | long_description = stream.read() 10 | 11 | setuptools.setup( 12 | name=r'hagworm', 13 | version=package_version, 14 | license=r'Apache License Version 2.0', 15 | platforms=[r'all'], 16 | author=r'Shaobo.Wang', 17 | author_email=r'wsb310@gmail.com', 18 | description=r'Network Development Suite', 19 | long_description=long_description, 20 | long_description_content_type=r'text/markdown', 21 | url=r'https://github.com/wsb310/hagworm', 22 | packages=setuptools.find_packages(), 23 | package_data={r'hagworm': [r'static/*.*']}, 24 | python_requires=r'>= 3.7', 25 | install_requires=[ 26 | r'aiohttp==3.6.2', 27 | r'aiomysql==0.0.20', 28 | r'aioredis==1.3.1', 29 | r'async-timeout==3.0.1', 30 | r'cachetools==4.1.1', 31 | r'crontab==0.22.8', 32 | r'cryptography==2.9.2', 33 | r'hiredis==1.0.1', 34 | r'Jinja2==2.11.2', 35 | r'tornado-jinja2==0.2.4', 36 | r'loguru==0.5.1', 37 | r'motor==2.1.0', 38 | r'numpy==1.19.0', 39 | r'ntplib==0.3.4', 40 | r'objgraph==3.4.1', 41 | r'Pillow==7.2.0', 42 | r'psutil==5.7.0', 43 | r'PyJWT==1.7.1', 44 | r'pytest==5.4.3', 45 | r'pytest-asyncio==0.14.0', 46 | r'python-stdnum==1.13', 47 | r'pyzmq==19.0.1', 48 | r'qrcode==6.1', 49 | r'mq-http-sdk==1.0.1', 50 | r'Sphinx==3.1.1', 51 | r'SQLAlchemy==1.3.18', 52 | r'tornado==6.0.4', 53 | r'terminal-table==2.0.1', 54 | r'ujson==3.0.0', 55 | r'WTForms==2.3.1', 56 | r'wtforms-tornado==0.0.2', 57 | r'xlwt==1.3.0', 58 | r'xmltodict==0.12.0', 59 | r'uvloop==0.14.0 ; sys_platform!="win32"', 60 | ], 61 | classifiers=[ 62 | r'Programming Language :: Python :: 3.7', 63 | r'License :: OSI Approved :: Apache Software License', 64 | r'Operating System :: POSIX :: Linux', 65 | ], 66 | ) 67 | -------------------------------------------------------------------------------- /testing/main.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | import sys 5 | import pytest 6 | 7 | os.chdir(os.path.dirname(__file__)) 8 | sys.path.insert(0, os.path.abspath(r'../')) 9 | 10 | 11 | if __name__ == r'__main__': 12 | 13 | pytest.main() 14 | -------------------------------------------------------------------------------- /testing/test_extend_asyncio_base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | 5 | from hagworm.extend.base import Ignore 6 | from hagworm.extend.asyncio.base import Utils, MultiTasks, SliceTasks, QueueTasks, ShareFuture, async_adapter 7 | from hagworm.extend.asyncio.base import FutureWithTimeout, AsyncConstructor, AsyncCirculator, AsyncCirculatorForSecond 8 | from hagworm.extend.asyncio.base import AsyncContextManager, AsyncFuncWrapper, FuncCache, TimeDiff 9 | from hagworm.extend.asyncio.transaction import Transaction 10 | 11 | 12 | pytestmark = pytest.mark.asyncio 13 | # pytest.skip(allow_module_level=True) 14 | 15 | 16 | class TestUtils: 17 | 18 | async def test_context_manager_1(self): 19 | 20 | class _ContextManager(AsyncContextManager): 21 | async def _context_release(self): 22 | pass 23 | 24 | result = False 25 | 26 | try: 27 | 28 | async with _ContextManager(): 29 | raise Ignore() 30 | 31 | result = True 32 | 33 | except Ignore: 34 | 35 | result = False 36 | 37 | return result 38 | 39 | async def test_context_manager_2(self): 40 | 41 | class _ContextManager(AsyncContextManager): 42 | async def _context_release(self): 43 | pass 44 | 45 | result = False 46 | 47 | try: 48 | 49 | async with _ContextManager(): 50 | 51 | async with _ContextManager(): 52 | raise Ignore(layers=2) 53 | 54 | result = False 55 | 56 | result = True 57 | 58 | except Ignore: 59 | 60 | result = False 61 | 62 | return result 63 | 64 | async def test_multi_tasks_and_share_future(self): 65 | 66 | @ShareFuture() 67 | async def _do_acton(): 68 | await Utils.sleep(1) 69 | return Utils.randint(0, 0xffff) 70 | 71 | assert (await _do_acton()) != (await _do_acton()) 72 | 73 | tasks = MultiTasks() 74 | 75 | for _ in range(0xff): 76 | tasks.append(_do_acton()) 77 | 78 | await tasks 79 | 80 | assert len(set(tasks)) == 1 81 | 82 | async def test_async_adapter_and_wait_frame(self): 83 | 84 | result = False 85 | 86 | async def _temp(): 87 | nonlocal result 88 | result = True 89 | await Utils.wait_frame(0xf0) 90 | 91 | async_adapter(_temp)() 92 | 93 | await Utils.wait_frame(0xff) 94 | 95 | assert result 96 | 97 | async def test_future_with_timeout(self): 98 | 99 | time_diff = TimeDiff() 100 | 101 | await FutureWithTimeout(1) 102 | 103 | assert Utils.math.floor(time_diff.check()[0]) == 1 104 | 105 | async def test_slice_tasks(self): 106 | 107 | async def _do_acton(): 108 | await Utils.sleep(1) 109 | 110 | time_diff = TimeDiff() 111 | 112 | tasks = SliceTasks(3) 113 | 114 | for _ in range(10): 115 | tasks.append(_do_acton()) 116 | 117 | await tasks 118 | 119 | assert Utils.math.floor(time_diff.check()[0]) == 4 120 | 121 | async def test_queue_tasks(self): 122 | 123 | async def _do_acton(val): 124 | await Utils.sleep(val) 125 | 126 | time_diff = TimeDiff() 127 | 128 | tasks = QueueTasks(5) 129 | 130 | tasks.append(_do_acton(8)) 131 | 132 | for _ in range(2): 133 | 134 | tasks.append(_do_acton(4)) 135 | 136 | for _ in range(2): 137 | tasks.append(_do_acton(1)) 138 | tasks.append(_do_acton(1)) 139 | tasks.append(_do_acton(2)) 140 | 141 | await tasks 142 | 143 | assert Utils.math.floor(time_diff.check()[0]) == 8 144 | 145 | async def test_async_constructor(self): 146 | 147 | result = False 148 | 149 | class _Temp(AsyncConstructor): 150 | 151 | async def __ainit__(self): 152 | nonlocal result 153 | result = True 154 | 155 | temp = await _Temp() 156 | 157 | assert isinstance(temp, _Temp) 158 | assert result 159 | 160 | async def test_async_circulator_1(self): 161 | 162 | time_diff = TimeDiff() 163 | 164 | async for _ in AsyncCirculator(1, 0xff): 165 | pass 166 | 167 | assert Utils.math.floor(time_diff.check()[0]) == 1 168 | 169 | async def test_async_circulator_2(self): 170 | 171 | async for index in AsyncCirculator(0, 0xff, 0xff): 172 | pass 173 | else: 174 | assert index == 0xff 175 | 176 | async def test_async_circulator_for_second_1(self): 177 | 178 | time_diff = TimeDiff() 179 | 180 | async for index in AsyncCirculatorForSecond(1, 0.1): 181 | pass 182 | 183 | assert (index >= 9) and (index <= 11) 184 | assert Utils.math.floor(time_diff.check()[0]) == 1 185 | 186 | async def test_async_circulator_for_second_2(self): 187 | 188 | time_diff = TimeDiff() 189 | 190 | async for index in AsyncCirculatorForSecond(0, 0.1, 10): 191 | pass 192 | else: 193 | assert index == 10 194 | 195 | check_time = time_diff.check()[0] 196 | 197 | assert (check_time >= 0.9) and (check_time <= 1.1) 198 | 199 | async def test_async_context_manager(self): 200 | 201 | result = False 202 | 203 | class _Temp(AsyncContextManager): 204 | 205 | async def _context_release(self): 206 | nonlocal result 207 | result = True 208 | await Utils.wait_frame(0xff) 209 | 210 | async with _Temp() as temp: 211 | pass 212 | 213 | assert result 214 | 215 | async def test_async_func_wrapper(self): 216 | 217 | result1 = False 218 | result2 = False 219 | 220 | async def _temp1(): 221 | nonlocal result1 222 | result1 = True 223 | await Utils.wait_frame(0xff) 224 | 225 | async def _temp2(): 226 | nonlocal result2 227 | result2 = True 228 | await Utils.wait_frame(0xff) 229 | 230 | wrapper = AsyncFuncWrapper() 231 | 232 | wrapper.add(_temp1) 233 | wrapper.add(_temp2) 234 | 235 | await wrapper() 236 | 237 | assert result1 and result2 238 | 239 | async def test_transaction(self): 240 | 241 | result = [] 242 | 243 | def _temp1(): 244 | nonlocal result 245 | result.append(True) 246 | 247 | async def _temp2(): 248 | nonlocal result 249 | result.append(True) 250 | await Utils.wait_frame(0xff) 251 | 252 | async with Transaction() as trx1: 253 | trx1.add_commit_callback(_temp1) 254 | trx1.add_commit_callback(_temp2) 255 | await trx1.commit() 256 | 257 | assert all(result) and len(result) == 2 258 | 259 | async with Transaction() as trx2: 260 | trx2.add_rollback_callback(_temp1) 261 | trx2.add_rollback_callback(_temp2) 262 | await trx1.rollback() 263 | 264 | assert all(result) and len(result) == 4 265 | 266 | async with Transaction() as trx3: 267 | trx3.add_rollback_callback(_temp1) 268 | trx3.add_rollback_callback(_temp2) 269 | 270 | assert all(result) and len(result) == 6 271 | 272 | async def test_func_cache(self): 273 | 274 | @FuncCache(ttl=1) 275 | async def _do_acton(): 276 | return Utils.randint(0, 0xffff) 277 | 278 | res1 = await _do_acton() 279 | res2 = await _do_acton() 280 | 281 | assert res1 == res2 282 | 283 | await Utils.sleep(1) 284 | res3 = await _do_acton() 285 | 286 | assert res1 != res3 287 | -------------------------------------------------------------------------------- /testing/test_extend_asyncio_future.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import time 4 | import pytest 5 | 6 | from hagworm.extend.asyncio.base import Utils, TimeDiff 7 | from hagworm.extend.asyncio.future import ThreadWorker 8 | 9 | 10 | pytestmark = pytest.mark.asyncio 11 | # pytest.skip(allow_module_level=True) 12 | 13 | 14 | class TestWorker: 15 | 16 | @ThreadWorker(1) 17 | def _temp_for_thread_worker(self, *args, **kwargs): 18 | time.sleep(1) 19 | return True 20 | 21 | async def test_thread_worker(self): 22 | 23 | time_diff = TimeDiff() 24 | 25 | assert await self._temp_for_thread_worker() 26 | assert await self._temp_for_thread_worker(1, 2) 27 | assert await self._temp_for_thread_worker(1, 2, t1=1, t2=2) 28 | 29 | assert Utils.math.floor(time_diff.check()[0]) == 3 30 | -------------------------------------------------------------------------------- /testing/test_extend_asyncio_net.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | 5 | from hagworm.extend.asyncio.base import MultiTasks 6 | from hagworm.extend.asyncio.net import HTTPClient, HTTPTextClient, HTTPJsonClient, HTTPTouchClient 7 | from hagworm.extend.asyncio.net import HTTPClientPool, HTTPTextClientPool, HTTPJsonClientPool, HTTPTouchClientPool 8 | 9 | 10 | pytestmark = pytest.mark.asyncio 11 | # pytest.skip(allow_module_level=True) 12 | 13 | TEST_URLS = [ 14 | r'https://lib.sinaapp.com/js/bootstrap/4.1.3/js/bootstrap.min.js.map', 15 | r'https://lib.sinaapp.com/js/angular.js/angular-1.2.19/angular.min.js.map', 16 | ] 17 | 18 | 19 | class TestHTTPClient: 20 | 21 | async def _http_client(self, client): 22 | 23 | for url in TEST_URLS: 24 | response = await client.get(url) 25 | assert response 26 | 27 | async def test_http_client(self): 28 | 29 | await self._http_client(HTTPClient()) 30 | 31 | async def test_http_text_client(self): 32 | 33 | await self._http_client(HTTPTextClient()) 34 | 35 | async def test_http_json_client(self): 36 | 37 | await self._http_client(HTTPJsonClient()) 38 | 39 | async def test_http_touch_client(self): 40 | 41 | await self._http_client(HTTPTouchClient()) 42 | 43 | async def _http_client_pool(self, client): 44 | 45 | tasks = MultiTasks() 46 | 47 | for url in TEST_URLS: 48 | tasks.append(client.get(url)) 49 | 50 | await tasks 51 | 52 | assert all(tasks) 53 | 54 | async def test_http_client_pool(self): 55 | 56 | pool = HTTPClientPool() 57 | 58 | await self._http_client(pool) 59 | await self._http_client_pool(pool) 60 | 61 | await pool.close() 62 | 63 | async def test_http_text_client_pool(self): 64 | 65 | pool = HTTPTextClientPool() 66 | 67 | await self._http_client(pool) 68 | await self._http_client_pool(pool) 69 | 70 | await pool.close() 71 | 72 | async def test_http_json_client_pool(self): 73 | 74 | pool = HTTPJsonClientPool() 75 | 76 | await self._http_client(pool) 77 | await self._http_client_pool(pool) 78 | 79 | await pool.close() 80 | 81 | async def test_http_touch_client_pool(self): 82 | 83 | pool = HTTPTouchClientPool() 84 | 85 | await self._http_client(pool) 86 | await self._http_client_pool(pool) 87 | 88 | await pool.close() 89 | -------------------------------------------------------------------------------- /testing/test_extend_asyncio_task.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | 5 | from hagworm.extend.asyncio.base import Utils, TimeDiff 6 | from hagworm.extend.asyncio.task import RateLimiter 7 | 8 | 9 | pytestmark = pytest.mark.asyncio 10 | # pytest.skip(allow_module_level=True) 11 | 12 | 13 | class TestHTTPClient: 14 | 15 | async def test_rate_limiter(self): 16 | 17 | async def _temp(): 18 | await Utils.sleep(1) 19 | return True 20 | 21 | limiter = RateLimiter(2, 5, 2) 22 | 23 | time_diff = TimeDiff() 24 | 25 | _f1 = limiter.append_with_name(r'f1', _temp) 26 | _f2 = limiter.append_with_name(r'f2', _temp) 27 | _f3 = limiter.append_with_name(r'f3', _temp) 28 | _f4 = limiter.append_with_name(r'f4', _temp) 29 | _f5 = limiter.append_with_name(r'f4', _temp) 30 | _f6 = limiter.append_with_name(r'f6', _temp) 31 | 32 | await _f1 33 | await _f2 34 | await _f3 35 | await _f4 36 | assert _f5 is None 37 | 38 | try: 39 | await _f6 40 | except: 41 | assert True 42 | else: 43 | assert False 44 | 45 | assert Utils.math.floor(time_diff.check()[0]) == 2 46 | -------------------------------------------------------------------------------- /testing/test_extend_base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | 5 | from hagworm.extend.base import catch_error, Ignore, ContextManager, FuncWrapper 6 | 7 | 8 | pytestmark = pytest.mark.asyncio 9 | # pytest.skip(allow_module_level=True) 10 | 11 | 12 | class TestUtils: 13 | 14 | async def test_catch_error_1(self): 15 | 16 | result = False 17 | 18 | try: 19 | 20 | with catch_error(): 21 | raise Ignore() 22 | 23 | result = True 24 | 25 | except Ignore: 26 | 27 | result = False 28 | 29 | return result 30 | 31 | async def test_catch_error_2(self): 32 | 33 | result = False 34 | 35 | try: 36 | 37 | with catch_error(): 38 | 39 | with catch_error(): 40 | raise Ignore(layers=2) 41 | 42 | result = False 43 | 44 | result = True 45 | 46 | except Ignore: 47 | 48 | result = False 49 | 50 | return result 51 | 52 | async def test_context_manager_1(self): 53 | 54 | class _ContextManager(ContextManager): 55 | def _context_release(self): 56 | pass 57 | 58 | result = False 59 | 60 | try: 61 | 62 | with _ContextManager(): 63 | raise Ignore() 64 | 65 | result = True 66 | 67 | except Ignore: 68 | 69 | result = False 70 | 71 | return result 72 | 73 | async def test_context_manager_2(self): 74 | 75 | class _ContextManager(ContextManager): 76 | def _context_release(self): 77 | pass 78 | 79 | result = False 80 | 81 | try: 82 | 83 | with _ContextManager(): 84 | 85 | with _ContextManager(): 86 | raise Ignore(layers=2) 87 | 88 | result = False 89 | 90 | result = True 91 | 92 | except Ignore: 93 | 94 | result = False 95 | 96 | return result 97 | 98 | async def test_func_wrapper(self): 99 | 100 | result1 = False 101 | result2 = False 102 | 103 | def _temp1(): 104 | nonlocal result1 105 | result1 = True 106 | 107 | def _temp2(): 108 | nonlocal result2 109 | result2 = True 110 | 111 | wrapper = FuncWrapper() 112 | 113 | wrapper.add(_temp1) 114 | wrapper.add(_temp2) 115 | 116 | wrapper() 117 | 118 | assert result1 and result2 119 | -------------------------------------------------------------------------------- /testing/test_extend_cache.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | 5 | from hagworm.extend.asyncio.base import Utils 6 | from hagworm.extend.cache import StackCache, PeriodCounter 7 | 8 | 9 | pytestmark = pytest.mark.asyncio 10 | # pytest.skip(allow_module_level=True) 11 | 12 | 13 | class TestCache: 14 | 15 | async def test_stack_cache(self): 16 | 17 | cache = StackCache(ttl=0.5) 18 | 19 | ckey = Utils.uuid1() 20 | cval = Utils.uuid1() 21 | 22 | res1 = cache.has(ckey) 23 | res2 = cache.size() 24 | 25 | assert not res1 and res2 == 0 26 | 27 | cache.set(ckey, cval) 28 | 29 | res3 = cache.has(ckey) 30 | res4 = cache.get(ckey) 31 | res5 = cache.size() 32 | 33 | assert res3 and res4 == cval and res5 == 1 34 | 35 | await Utils.sleep(1) 36 | 37 | res6 = cache.has(ckey) 38 | res7 = cache.get(ckey) 39 | res8 = cache.size() 40 | 41 | assert not res6 and res7 is None and res8 == 0 42 | 43 | async def test_period_counter(self): 44 | 45 | counter = PeriodCounter(2) 46 | 47 | res1 = counter.incr(2) 48 | 49 | assert res1 == 2 50 | 51 | res2, trx = counter.incr_with_trx(1) 52 | trx.rollback() 53 | res3 = counter.get() 54 | 55 | assert res2 == 3 and res3 == 2 56 | 57 | await Utils.sleep(2) 58 | 59 | res4 = counter.get() 60 | 61 | assert res4 == 0 62 | -------------------------------------------------------------------------------- /testing/test_extend_struct.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pytest 4 | 5 | from hagworm.extend.asyncio.base import Utils 6 | from hagworm.extend.struct import KeyLowerDict 7 | 8 | 9 | pytestmark = pytest.mark.asyncio 10 | # pytest.skip(allow_module_level=True) 11 | 12 | 13 | class TestStruct: 14 | 15 | async def test_key_lower_dict(self): 16 | 17 | temp1 = { 18 | r'MySQL1': { 19 | r'MySQL1': Utils.randint(0x10, 0xff), 20 | r'MYSQL2': Utils.randint(0x10, 0xff), 21 | r'Mysql3': Utils.randint(0x10, 0xff), 22 | }, 23 | r'MYSQL2': { 24 | r'MySQL1': { 25 | r'MySQL1': Utils.randint(0x10, 0xff), 26 | r'MYSQL2': Utils.randint(0x10, 0xff), 27 | r'Mysql3': Utils.randint(0x10, 0xff), 28 | }, 29 | r'MYSQL2': { 30 | r'MySQL1': Utils.randint(0x10, 0xff), 31 | r'MYSQL2': Utils.randint(0x10, 0xff), 32 | r'Mysql3': Utils.randint(0x10, 0xff), 33 | }, 34 | r'Mysql3': { 35 | r'MySQL1': Utils.randint(0x10, 0xff), 36 | r'MYSQL2': Utils.randint(0x10, 0xff), 37 | r'Mysql3': Utils.randint(0x10, 0xff), 38 | }, 39 | }, 40 | r'Mysql3': { 41 | r'MySQL1': { 42 | r'MySQL1': { 43 | r'MySQL1': Utils.randint(0x10, 0xff), 44 | r'MYSQL2': Utils.randint(0x10, 0xff), 45 | r'Mysql3': Utils.randint(0x10, 0xff), 46 | }, 47 | r'MYSQL2': { 48 | r'MySQL1': Utils.randint(0x10, 0xff), 49 | r'MYSQL2': Utils.randint(0x10, 0xff), 50 | r'Mysql3': Utils.randint(0x10, 0xff), 51 | }, 52 | r'Mysql3': { 53 | r'MySQL1': Utils.randint(0x10, 0xff), 54 | r'MYSQL2': Utils.randint(0x10, 0xff), 55 | r'Mysql3': Utils.randint(0x10, 0xff), 56 | }, 57 | }, 58 | r'MYSQL2': { 59 | r'MySQL1': { 60 | r'MySQL1': Utils.randint(0x10, 0xff), 61 | r'MYSQL2': Utils.randint(0x10, 0xff), 62 | r'Mysql3': Utils.randint(0x10, 0xff), 63 | }, 64 | r'MYSQL2': { 65 | r'MySQL1': Utils.randint(0x10, 0xff), 66 | r'MYSQL2': Utils.randint(0x10, 0xff), 67 | r'Mysql3': Utils.randint(0x10, 0xff), 68 | }, 69 | r'Mysql3': { 70 | r'MySQL1': Utils.randint(0x10, 0xff), 71 | r'MYSQL2': Utils.randint(0x10, 0xff), 72 | r'Mysql3': Utils.randint(0x10, 0xff), 73 | }, 74 | }, 75 | r'Mysql3': { 76 | r'MySQL1': { 77 | r'MySQL1': Utils.randint(0x10, 0xff), 78 | r'MYSQL2': Utils.randint(0x10, 0xff), 79 | r'Mysql3': Utils.randint(0x10, 0xff), 80 | }, 81 | r'MYSQL2': { 82 | r'MySQL1': Utils.randint(0x10, 0xff), 83 | r'MYSQL2': Utils.randint(0x10, 0xff), 84 | r'Mysql3': Utils.randint(0x10, 0xff), 85 | }, 86 | r'Mysql3': { 87 | r'MySQL1': Utils.randint(0x10, 0xff), 88 | r'MYSQL2': Utils.randint(0x10, 0xff), 89 | r'Mysql3': Utils.randint(0x10, 0xff), 90 | }, 91 | }, 92 | }, 93 | } 94 | 95 | temp2 = KeyLowerDict(temp1) 96 | 97 | assert r'my_sql1' in temp2 and r'mysql2' in temp2 and r'mysql3' in temp2 98 | -------------------------------------------------------------------------------- /upload.sh: -------------------------------------------------------------------------------- 1 | 2 | python3 setup.py sdist build 3 | twine upload ./dist/* 4 | 5 | rm -rf ./dist 6 | rm -rf ./build 7 | rm -rf ./hagworm.egg-info 8 | --------------------------------------------------------------------------------