├── .gitattributes ├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── LICENSE ├── Makefile ├── README.md ├── doc ├── http.md ├── mysql.md ├── others.md ├── pywf.md └── redis.md ├── pyproject.toml ├── pywf ├── __init__.py └── mysql_iterator.py ├── requirements └── requirements-lint.txt ├── scripts └── lint.sh ├── setup.py ├── src ├── common_types.cc ├── common_types.h ├── http_types.cc ├── http_types.h ├── mysql_types.cc ├── mysql_types.h ├── network_types.cc ├── network_types.h ├── other_types.cc ├── other_types.h ├── pyworkflow.cc ├── redis_types.cc └── redis_types.h └── tutorial ├── tutorial01-wget.py ├── tutorial02-redis_cli.py ├── tutorial03-wget_to_redis.py ├── tutorial04-http_echo_server.py ├── tutorial05-http_proxy.py ├── tutorial06-parallel_wget.py ├── tutorial09-http_file_server.py └── tutorial12-mysql_cli.py /.gitattributes: -------------------------------------------------------------------------------- 1 | *.h linguist-language=python 2 | *.cc linguist-language=python 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # build 2 | build 3 | dist 4 | /pywf.* 5 | cmake_* 6 | localtest 7 | 8 | # pytest 9 | .pytest_cache 10 | unittest/__pycache__ 11 | 12 | # pyenv 13 | .python-version 14 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "workflow"] 2 | path = workflow 3 | url = https://github.com/sogou/workflow.git 4 | ignore = dirty 5 | [submodule "pybind11"] 6 | path = pybind11 7 | url = https://github.com/pybind/pybind11.git 8 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.4) 2 | project(cpp_pyworkflow CXX) 3 | set(CMAKE_BUILD_TYPE "RelWithDebInfo") # Attention Release will strip debug info 4 | set(CMAKE_VERBOSE_MAKEFILE ON) 5 | add_subdirectory(workflow) 6 | 7 | find_package(OpenSSL REQUIRED) 8 | include_directories(${OPENSSL_INCLUDE_DIR}) 9 | 10 | add_compile_options(-Wall -Wextra -Wno-unused-parameter) 11 | 12 | if (APPLE) 13 | add_compile_options(-fvisibility=default) 14 | endif () 15 | 16 | set(CMAKE_CXX_FLAGS, "-std=c++11 -Wall -Wextra") 17 | set(CMAKE_CXX_FLAGS_DEBUG, "-O0 -g") 18 | set(CMAKE_CXX_FLAGS_RELEASE, "-O2 -g") 19 | add_subdirectory(pybind11) 20 | pybind11_add_module(cpp_pyworkflow 21 | src/common_types.cc 22 | src/network_types.cc 23 | src/http_types.cc 24 | src/redis_types.cc 25 | src/mysql_types.cc 26 | src/other_types.cc 27 | src/pyworkflow.cc) 28 | 29 | include_directories(./workflow/_include) 30 | target_link_libraries(cpp_pyworkflow PRIVATE 31 | workflow-static pthread ssl crypto) 32 | target_compile_definitions(cpp_pyworkflow PRIVATE VERSION_INFO=${PYWORKFLOW_VERSION_INFO}) 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: wheel 2 | 3 | .PHONY: build wheel install uninstall reinstall clean 4 | 5 | build: 6 | python3 setup.py build 7 | 8 | wheel: 9 | python3 setup.py bdist_wheel 10 | 11 | install: wheel 12 | pip3 install dist/pywf-0.0.10-*.whl --user 13 | 14 | uninstall: 15 | pip3 uninstall -y pywf 16 | 17 | reinstall: build wheel uninstall install 18 | 19 | clean: 20 | rm -rf ./build ./pywf.egg-info ./dist 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyWorkflow(PyWF) - A Python Binding of C++ Workflow 2 | [![License](https://img.shields.io/badge/License-Apache%202.0-green.svg)](https://github.com/sogou/pyworkflow/blob/master/LICENSE) 3 | [![Language](https://img.shields.io/badge/language-c++-red.svg)](https://en.cppreference.com/) 4 | [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/pywf.svg)](https://pypi.python.org/pypi/pywf) 5 | [![PyPI](https://img.shields.io/pypi/v/pywf.svg)](https://pypi.python.org/pypi/pywf) 6 | 7 | ## 概览 8 | [C++ Workflow](https://github.com/sogou/workflow)是一个高性能的异步引擎，本项目着力于实现一个Python版的Workflow，让Python用户也能享受Workflow带来的绝佳体验。 9 | 10 | ### 快速上手 11 | 在用户深入了解Workflow相关概念之前，先来看几个简单的示例，可以对使用方法有一个初步印象。pywf是本项目Python包的名称，在文档中有时会直接使用`wf`作为其简称。 12 | 13 | 14 | #### 发起一个Http请求 15 | ```py 16 | import pywf as wf 17 | 18 | def http_callback(http_task): 19 | resp = http_task.get_resp() 20 | print("Http status:{}\n{}".format( 21 | resp.get_status_code(), resp.get_body())) # body is bytes 22 | 23 | http_task = wf.create_http_task("http://www.sogou.com/", redirect_max=4, retry_max=2, callback=http_callback) 24 | http_task.start() 25 | wf.wait_finish() 26 | ``` 27 | 28 | #### 依次发起多个Http请求 29 | ```py 30 | import pywf as wf 31 | 32 | def series_callback(s): 33 | print("All task in this series is done") 34 | 35 | def http_callback(http_task): 36 | req = http_task.get_req() 37 | resp = http_task.get_resp() 38 | print("uri:{} status:{}".format( 39 | req.get_request_uri(), 40 | resp.get_status_code())) 41 | 42 | def create_http_task(url): 43 | return wf.create_http_task(url, 4, 2, http_callback) 44 | 45 | first_task = create_http_task("http://www.sogou.com") 46 | series = wf.create_series_work(first_task, series_callback) 47 | series.push_back(create_http_task("https://www.zhihu.com/people/kedixa")) 48 | series.push_back(create_http_task("https://fanyi.sogou.com/document")) 49 | series.start() 50 | wf.wait_finish() 51 | ``` 52 | 53 | #### 同时发起多个Http请求 54 | ```py 55 | import pywf as wf 56 | 57 | def parallel_callback(p): 58 | print("All series in this parallel is done") 59 | 60 | def http_callback(http_task): 61 | req = http_task.get_req() 62 | resp = http_task.get_resp() 63 | print("uri:{} status:{}".format( 64 | req.get_request_uri(), 65 | resp.get_status_code())) 66 | 67 | url = [ 68 | "http://www.sogou.com", 69 | "https://www.zhihu.com/people/kedixa", 70 | "https://fanyi.sogou.com/document" 71 | ] 72 | parallel = wf.create_parallel_work(parallel_callback) 73 | for u in url: 74 | task = wf.create_http_task(u, 4, 2, http_callback) 75 | series = wf.create_series_work(task, None) # without callback 76 | parallel.add_series(series) 77 | parallel.start() 78 | 79 | wf.wait_finish() 80 | ``` 81 | 82 | ### 基本概念 83 | #### 任务 84 | 通过`create_xxx_task`等工厂函数创建的对象称作任务(task)，例如`create_http_task`。一个任务被创建后，必须被启动或取消，通过执行`http_task.start()`，会自动以`http_task`为`first_task`创建一个串行并立即启动任务。如果用户指定了回调函数，当任务完成时回调函数会被调用，但在任务启动后且回调函数前，用户不能再操作该任务。当回调函数结束后，该任务被立即释放，用户也不能再操作该任务。 85 | 86 | #### 串行 87 | 通过`create_series_work`创建的对象称作串行(series)，用户在创建时需要指定一个`first_task`来作为启动该`series`启动时应当执行的任务，用户可选地指定一个回调函数，当所有任务执行完成后，回调函数会被调用。 88 | 89 | `series`的回调函数用于通知用户该串行中的任务均已完成，不能再继续添加新的任务，且回调函数结束后，该串行会立即被销毁。 90 | 91 | #### 并行 92 | 通过`create_parallel_work`创建的对象称作并行(parallel)，用户可以创建一个空的并行，然后通过`add_series`接口向并行中添加串行，也可以在创建时指定一组串行。并行本身也是一种任务，所以并行也可以放到串行中。`parallel.start()`就会自动创建一个串行，并将`parallel`作为`first_task`立即开始执行。 93 | 94 | `parallel`的回调函数用于通知用户该并行中的串行均已完成，不能再继续添加新的串行，且回调函数结束后，该并行会立即被销毁。 95 | 96 | 有了上述三个概念，就可以构建出各种复杂的任务结构，并在Workflow的管理下高效执行。 97 | 98 | ### 详细说明 99 | - [任务流](./doc/pywf.md) 100 | - [http任务](./doc/http.md) 101 | - [redis任务](./doc/redis.md) 102 | - [mysql任务](./doc/mysql.md) 103 | - [其他任务](./doc/others.md) 104 | 105 | ### 设计理念 106 | Workflow认为，一个典型的后端程序由三个部分组成，并且完全独立开发。即：程序=协议+算法+任务流。 107 | 108 | - 协议 109 | - 大多数情况下，用户使用的是内置的通用网络协议，例如http，redis或各种rpc。 110 | - PyWF未支持用户自定义协议。 111 | - 算法 112 | - 算法是与协议对称的概念。 113 | - 如果说协议的调用是rpc，算法的调用就是一次apc（Async Procedure Call）。 114 | - 任何一次边界清晰的复杂计算，都应该包装成算法。 115 | - 任务流 116 | - 任务流就是实际的业务逻辑，就是把开发好的协议与算法放在流程图里使用起来。 117 | - 典型的任务流是一个闭合的串并联图。复杂的业务逻辑，可能是一个非闭合的DAG。 118 | - 任务流图可以直接构建，也可以根据每一步的结果动态生成。所有任务都是异步执行的。 119 | 120 | Python Workflow将会逐步支持Workflow的六种基础任务：通讯，文件IO，CPU，GPU，定时器，计数器。 121 | 122 | ### 注意事项 123 | - 框架本身不抛出异常，也未处理任何异常，所以用户需要保证回调函数不会抛出异常，context的构造和析构不抛出异常。 124 | - 所有通过工厂函数创建出的task，必须start、dismiss或添加至一个series中。 125 | - 所有创建出的series必须start、dismiss或添加至一个parallel中。 126 | - 所有创建出的parallel必须start、dismiss或添加至一个series中。 127 | - 由PyWF工厂函数创建的对象的生命周期均由内部管理，在Python层面仅是一个引用，用户不能使用超出生命周期的对象。 128 | - 用户使用大部分`get`接口获取的对象可以自由使用，例如Http中的`get_body`。 129 | 130 | ## 构建和安装 131 | 132 | ### 通过pip安装 133 | 本项目支持Python 3.6以上，在pypi上发布了一组Python 3.6、3.7、3.8、3.9的`manylinux2014`版本，用户可以通过较高版本的pip直接安装。 134 | 135 | ```bash 136 | pip3 install pywf 137 | ``` 138 | 139 | ### 编译安装 140 | 用户可以下载本项目源码进行编译安装。 141 | 142 | ```bash 143 | # CentOS 7 144 | yum install cmake3 ninja-build python36 python36-devel python36-pip 145 | yum install gcc-c++ # if needed 146 | git clone https://github.com/sogou/pyworkflow --recursive 147 | cd pyworkflow 148 | pip3 install wheel 149 | python3 setup.py bdist_wheel 150 | pip3 install dist/*.whl --user 151 | ``` 152 | 153 | ```bash 154 | # CentOS 8 155 | yum install cmake ninja-build python36 python36-devel python3-pip 156 | git clone https://github.com/sogou/pyworkflow --recursive 157 | cd pyworkflow 158 | pip3 install wheel 159 | python3 setup.py bdist_wheel 160 | pip3 install dist/*.whl --user 161 | ``` 162 | 163 | ```bash 164 | # Mac 165 | # Build Requirement 166 | brew install ninja cmake openssl 167 | pip3 install wheel 168 | 169 | # OpenSSL Env, see 'brew info openssl' for more infomation 170 | echo 'export PATH="/usr/local/opt/openssl@1.1/bin:$PATH"' >> ~/.bash_profile 171 | echo 'export LDFLAGS="-L/usr/local/opt/openssl@1.1/lib"' >> ~/.bash_profile 172 | echo 'export CPPFLAGS="-I/usr/local/opt/openssl@1.1/include"' >> ~/.bash_profile 173 | echo 'export PKG_CONFIG_PATH="/usr/local/opt/openssl@1.1/lib/pkgconfig"' >> ~/.bash_profile 174 | echo 'export OPENSSL_ROOT_DIR="/usr/local/opt/openssl@1.1/"' >> ~/.bash_profile 175 | 176 | # zsh Env, if needed 177 | echo 'test -f ~/.bash_profile && source ~/.bash_profile' >> ~/.zshrc 178 | source ~/.zshrc 179 | 180 | # Build 181 | git clone https://github.com/sogou/pyworkflow --recursive 182 | cd pyworkflow 183 | python3 setup.py bdist_wheel 184 | pip3 install dist/*.whl --user 185 | ``` 186 | -------------------------------------------------------------------------------- /doc/http.md: -------------------------------------------------------------------------------- 1 | ## Http Client/Server 2 | 3 | ### Tutorials 4 | - [tutorial01-wget.py](../tutorial/tutorial01-wget.py) 5 | - [tutorial04-http_echo_server.py](../tutorial/tutorial04-http_echo_server.py) 6 | - [tutorial05-http_proxy.py](../tutorial/tutorial05-http_proxy.py) 7 | - [tutorial06-parallel_wget.py](../tutorial/tutorial06-parallel_wget.py) 8 | 9 | ### HttpRequest 10 | - is_chunked() -> bool 11 | - is_keep_alive() -> bool 12 | - get_method() -> str 13 | - get_request_uri() -> str 14 | - get_http_version() -> str 15 | - get_headers() -> list[tuple] 16 | - get_body() -> bytes 17 | - 获取body，注意返回类型为bytes 18 | - set_method(str) -> bool 19 | - set_request_uri(str) -> bool 20 | - set_http_version(str) -> bool 21 | - add_header_pair(str key, str value) -> bool 22 | - set_header_pair(str key, str value) -> bool 23 | - append_body(bytes) -> bool 24 | - 追加http body，此bytes会被内部引用一份，不会发生拷贝 25 | - append_body(str) -> bool 26 | - 追加http body，会发生拷贝 27 | - clear_body() -> None 28 | - 清空body 29 | - get_body_size() -> int 30 | - set_size_limit(int) -> None 31 | - get_size_limit() -> int 32 | - move_to(wf.HttpRequest) -> None 33 | - 将当前HttpRequest内容移动至另一个HttpRequest中，在实现Http Proxy时可以用来减少拷贝 34 | - 移动完成后当前对象已经不可使用，即使回调函数还未结束 35 | 36 | ### HttpResponse 37 | - is_chunked() -> bool 38 | - is_keep_alive() -> bool 39 | - get_status_code() -> str 40 | - get_reason_phrase() -> str 41 | - get_http_version() -> str 42 | - get_headers() -> list[tuple] 43 | - get_body() -> bytes 44 | - 获取body，注意返回类型为bytes 45 | - set_status_code(str) -> bool 46 | - set_reason_phrase(str) -> bool 47 | - set_http_version(str) -> bool 48 | - add_header_pair(str key, str value) -> bool 49 | - set_header_pair(str key, str value) -> bool 50 | - append_body(bytes) -> bool 51 | - 追加http body，此bytes会被内部引用一份，不会发生拷贝 52 | - append_body(str) -> bool 53 | - 追加http body，会发生拷贝 54 | - clear_body() -> None 55 | - 清空body 56 | - get_body_size() -> int 57 | - set_size_limit(int) -> None 58 | - get_size_limit() -> int 59 | - move_to(wf.HttpRequest) -> None 60 | - 将当前HttpRequest内容移动至另一个HttpRequest中，在实现Http Proxy时可以用来减少拷贝 61 | - 移动完成后当前对象已经不可使用，即使回调函数还未结束 62 | 63 | ### HttpTask 64 | - start() -> None 65 | - dismiss() -> None 66 | - noreply() -> None 67 | - get_req() -> wf.HttpRequest 68 | - get_resp() -> wf.HttpResponse 69 | - get_state() -> int 70 | - get_error() -> int 71 | - get_timeout_reason() -> int 72 | - wf.TOR_NOT_TIMEOUT 73 | - wf.TOR_WAIT_TIMEOUT 74 | - wf.TOR_CONNECT_TIMEOUT 75 | - wf.TOR_TRANSMIT_TIMEOUT 76 | - get_task_seq() -> int 77 | - set_send_timeout(int) -> None 78 | - set_receive_timeout(int) -> None 79 | - set_keep_alive(int) -> None 80 | - get_peer_addr() -> tuple(str, int) 81 | - 返回元组(ip, port)，若获取失败则ip为空字符串 82 | - set_callback(Callable[[wf.HttpTask], None]) -> None 83 | - set_user_data(object) -> None 84 | - get_user_data() -> object 85 | 86 | ### HttpServer 87 | - HttpServer(Callable[[wf.HttpTask], None]) 88 | - HttpServer(wf.ServerParams, Callable[[wf.HttpTask], None]) 89 | - start(int port, str cert_file = '', str key_file = '') -> int 90 | - 启动server，cert_file和key_file任意一个未指定时等同于`start(port)` 91 | - 函数返回0表示启动成功 92 | - start(int family, str host, int port, str cert_file = '', str key_file = '') -> int 93 | - stop() -> None 94 | - 停止server，该函数同步等待当前处理中的请求完成 95 | 96 | ### 任务工厂等 97 | - wf.create_http_task(str url, int redirect_max, int retry_max, Callable[[wf.HttpTask], None]) -> wf.HttpTask 98 | - wf.create_http_task(str url, str proxy_url, int redirect_max, int retry_max, Callable[[wf.HttpTask], None]) -> wf.HttpTask 99 | - wf.ServerParams同workflow的WFServerParams 100 | 101 | Workflow中关于Server Params的定义，wf.ServerParams的默认构造会返回SERVER_PARAMS_DEFAULT 102 | ```cpp 103 | struct WFServerParams 104 | { 105 | size_t max_connections; 106 | int peer_response_timeout; /* timeout of each read or write operation */ 107 | int receive_timeout; /* timeout of receiving the whole message */ 108 | int keep_alive_timeout; 109 | size_t request_size_limit; 110 | int ssl_accept_timeout; /* if not ssl, this will be ignored */ 111 | }; 112 | 113 | static constexpr struct WFServerParams SERVER_PARAMS_DEFAULT = 114 | { 115 | .max_connections = 2000, 116 | .peer_response_timeout = 10 * 1000, 117 | .receive_timeout = -1, 118 | .keep_alive_timeout = 60 * 1000, 119 | .request_size_limit = (size_t)-1, 120 | .ssl_accept_timeout = 10 * 1000, 121 | }; 122 | ``` 123 | ### 示例 124 | ```py 125 | # 获取Http body示例 126 | import pywf as wf 127 | def series_callback(x): 128 | pass 129 | 130 | def http_callback(x): 131 | # x的类型是wf.HttpTask，作用域仅限于此函数内 132 | # req 和 resp仅限于此函数中，函数结束时会立刻被回收 133 | req = x.get_req() 134 | resp = x.get_resp() 135 | # 此时get_body获取到的是http请求的结果 136 | # body的生命周期由Python管理，可以随意引用至函数外部 137 | body = resp.get_body() 138 | # print(body) 139 | # 一般情况下不需要向resp里添加内容，若此时向resp追加内容，则请求结果会被清空，仅保留新追加的内容 140 | resp.append_body("") # 发生拷贝 141 | resp.append_body(b"") # 不会拷贝，bytes会被resp引用一份 142 | print(resp.get_body()) # 输出 b'' 143 | 144 | def create_http_task(): 145 | return wf.create_http_task("http://www.sogou.com/", 1, 1, http_callback) 146 | 147 | series = wf.create_series_work(create_http_task(), series_callback) 148 | series.start() 149 | wf.wait() 150 | ``` 151 | -------------------------------------------------------------------------------- /doc/mysql.md: -------------------------------------------------------------------------------- 1 | ## MySQL Client/Server 2 | 3 | ### Tutorials 4 | 5 | - [tutorial12-mysql_cli.py](../tutorial/tutorial12-mysql_cli.py) 6 | 7 | 8 | # MySQL URL的格式 9 | 10 | mysql://username:password@host:port/dbname?character_set=charset&character_set_results=charset 11 | 12 | - 如果以SSL连接访问MySQL，则scheme设为**mysqls://**。MySQL server 5.7及以上支持； 13 | 14 | - username和password按需填写； 15 | 16 | - port默认为3306； 17 | 18 | - dbname为要用的数据库名，一般如果SQL语句只操作一个db的话建议填写； 19 | 20 | - character_set为client的字符集，等价于使用官方客户端启动时的参数``--default-character-set``的配置，默认utf8，具体可以参考MySQL官方文档[character-set.html](https://dev.mysql.com/doc/internals/en/character-set.html)。 21 | 22 | - character_set_results为client、connection和results的字符集，如果想要在SQL语句里使用``SET NAME``来指定这些字符集的话，请把它配置到url的这个位置。 23 | 24 | MySQL URL示例： 25 | 26 | mysql://root:password@127.0.0.1 27 | 28 | mysql://@test.mysql.com:3306/db1?character_set=utf8&character_set_results=utf8 29 | 30 | mysqls://localhost/db1?character\_set=big5 31 | 32 | ### MySQL结果集 33 | 与workflow其他任务类似，可以用`task.get_resp()`拿到`MySQLResponse`，我们可以通过`MySQLResultCursor`遍历结果集及其中的每个列的信息`MySQLField`、每行和每个`MySQLCell` 34 | 35 | **注意**：用户在遍历MySQL结果集的过程中获得的`MySQLResultCursor`、`MySQLField`、`MySQLCell`都是`MySQLResponse`内部对象的引用，即仅可以在回调函数中使用 36 | 37 | 一次请求所对应的回复中，其数据是一个三维结构 38 | - 一个回复中包含了一个或多个结果集（result set） 39 | - 一个结果集包含了一行或多行（row） 40 | - 一行包含了一到多个阈（Field/Cell） 41 | 42 | - 具体使用从外到内的步骤应该是： 43 | 44 | 1. 判断任务状态（代表通信层面状态）：用户通过判断 **task.get_state()** 是否为WFT_STATE_SUCCESS来查看任务执行是否成功； 45 | 46 | 2. 判断回复包类型（代表返回包解析状态）：调用 **resp.get_packet_type()** 查看最后一条MySQL语句的返回包类型，常见的几个类型为： 47 | - MYSQL_PACKET_OK：返回非结果集的请求: 解析成功； 48 | - MYSQL_PACKET_EOF：返回结果集的请求: 解析成功； 49 | - MYSQL_PACKET_ERROR：请求失败； 50 | 51 | 3. 判断结果集状态（代表结果集读取状态）：用户可以使用MySQLResultCursor读取结果集中的内容，因为MySQL server返回的数据是多结果集的，因此一开始cursor会**自动指向第一个结果集**的读取位置。通过 **cursor.get_cursor_status()** 可以拿到的几种状态： 52 | - MYSQL_STATUS_GET_RESULT：有数据可读； 53 | - MYSQL_STATUS_END：当前结果集已读完最后一行； 54 | - MYSQL_STATUS_OK：此回复包为非结果集包，无需通过结果集接口读数据； 55 | - MYSQL_STATUS_ERROR：解析错误； 56 | 57 | 4. 读取columns中每个field： 58 | - `get_field_count()` 59 | - `fetch_fields()` 60 | 61 | 5. 读取每一行：按行读取可以使用 **cursor.fetch_row()** 直到返回值为false。其中会移动cursor内部对当前结果集的指向每行的offset： 62 | - `get_rows_count()` 63 | - `fetch_row()` 64 | 65 | 6. 直接把当前结果集的所有行拿出：所有行的读取可以使用 **cursor.fetch_all()** ，内部用来记录行的cursor会直接移动到最后；cursor状态会变成MYSQL_STATUS_END： 66 | - `fetch_all()`: 返回 list[list[MySQLCell]] 67 | 68 | 7. 返回当前结果集的头部：如果有必要重读这个结果集，可以使用 **cursor.rewind()** 回到当前结果集头部，再通过第5步或第6步进行读取； 69 | 70 | 8. 拿到下一个结果集：因为MySQL server返回的数据包可能是包含多结果集的（比如每个select语句为一个结果集；或者call procedure返回的多结果集数据），因此用户可以通过 **cursor.next_result_set()** 跳到下一个结果集，返回值为false表示所有结果集已取完。 71 | 72 | 9. 返回第一个结果集：**cursor.first_result_set()** 可以让我们返回到所有结果集的头部，然后可以从第3步开始重新拿数据； 73 | 74 | 10. 每列具体数据MySQLCell：第5步中读取到的一行，由多列组成，每列结果为MySQLCell，基本使用接口有： 75 | - `get_data_type()` 返回MYSQL_TYPE_LONG、MYSQL_TYPE_STRING... 76 | - `is_TYPE()` TYPE为int、string、ulonglong，判断是否是某种类型 77 | - `as_TYPE()` 以某种类型读出MySQLCell的数据 78 | - `as_object()` 按数据类型返回Python对象 79 | 80 | ### MySQLCell 81 | - get_data_type() -> int 82 | - 返回Cell中的数据类型，即`wf.MYSQL_TYPE_*` 83 | - is_null() -> bool 84 | - 判断是否为`MYSQL_TYPE_NULL` 85 | - is_int() -> bool 86 | - 判断是否为`MYSQL_TYPE_TINY`、`MYSQL_TYPE_SHORT`、`MYSQL_TYPE_INT24`、`MYSQL_TYPE_LONG` 87 | - is_string() -> bool 88 | - 判断是否为`MYSQL_TYPE_DECIMAL`、`MYSQL_TYPE_STRING`、`MYSQL_TYPE_VARCHAR`、`MYSQL_TYPE_VAR_STRING`、`MYSQL_TYPE_JSON` 89 | - is_float() -> bool 90 | - 判断是否为`MYSQL_TYPE_FLOAT` 91 | - is_double() -> bool 92 | - 判断是否为`MYSQL_TYPE_DOUBLE` 93 | - is_ulonglong() -> bool 94 | - 判断是否为`MYSQL_TYPE_LONGLONG` 95 | - is_date() -> bool 96 | - 判断是否为`MYSQL_TYPE_DATE` 97 | - is_time() -> bool 98 | - 判断是否为`MYSQL_TYPE_TIME` 99 | - is_datetime() -> bool 100 | - 判断是否为`MYSQL_TYPE_DATETIME`、`MYSQL_TYPE_TIMESTAMP` 101 | - as_int() -> int 102 | - as_float() -> float 103 | - as_double() -> float 104 | - as_ulonglong() -> int 105 | - as_string() -> bytes 106 | - `is_string()`时以bytes类型返回底层数据 107 | - 非`is_string()`不应该调用该接口，若调用则返回长度为零的bytes 108 | - as_date() -> datetime.date 109 | - as_time() -> datetime.timedelta 110 | - MySQL的time类型可以表示一个时间段，`datetime.timedelta`更相符 111 | - as_datetime() -> datetime.datetime 112 | - as_bytes() -> bytes 113 | - 返回底层数据的bytes类型表示，不考虑真实类型 114 | - as_object() -> object 115 | - 返回该Cell的Python对象表示，如同正确调用了`as_*`函数 116 | - `__str__` -> str 117 | - 返回该Cell的字符串表示，如同正确调用了`str(cell.as_object())` 118 | - 例外: 若`as_object()`返回了`utf-8`编码的bytes，则会尝试将其解码为str类型，否则会调用`bytes.__str__` 119 | 120 | ### MySQLField 121 | - get_name() -> bytes 122 | - 字段名称，如果为字段提供了带有AS子句的别名，则值为别名。对于过程参数，为参数名称。 123 | - get_org_name() -> bytes 124 | - 字段名称，忽略别名 125 | - get_table() -> bytes 126 | - 包含此字段的表的名称。对于计算字段，该值为空字符串。如果为表或视图提供了带有AS子句的别名，则值为别名。对于过程参数，为过程名称。 127 | - get_org_table() -> bytes 128 | - 表的名称，忽略别名 129 | - get_db() -> bytes 130 | - 字段来源的数据库的名称。如果该字段是计算字段，则为空字符串。对于过程参数，是包含该过程的数据库的名称。 131 | - get_catalog() -> bytes 132 | - 此值始终为b"def" 133 | - get_def() -> bytes 134 | - 此字段的默认值。仅在使用`mysql_list_fields()`时设置此值 135 | - get_charsetnr() -> int 136 | - get_length() -> int 137 | - get_flags() -> int 138 | - get_decimals() -> int 139 | - get_data_type() -> int 140 | - 返回该列的数据类型，即`wf.MYSQL_TYPE_*` 141 | - `__str__()` -> str 142 | - 返回`get_name()`的str表示 143 | 144 | ### MySQLResultCursor 145 | - 构造函数 146 | - MySQLResultCursor(wf.MySQLResponse) 147 | - fetch_fields() -> list[wf.MySQLField] 148 | - 返回当前ResultSet的所有Fields 149 | - next_result_set() -> bool 150 | - first_result_set() -> None 151 | - fetch_row() -> list[wf.MySQLCell]/None 152 | - 若当前ResultSet还有未返回的行，则返回下一行并移动下标，否则返回None 153 | - fetch_all() -> list[list[wf.MySQLCell]] 154 | - 返回当前ResultSet中所有剩余的行 155 | - get_cursor_status() -> int 156 | - 返回cursor状态，即`wf.MYSQL_STATUS_*` 157 | - get_server_status() -> int 158 | - get_field_count() -> int 159 | - 返回当前ResultSet的field个数 160 | - get_rows_count() -> int 161 | - 返回当前ResultSet的行数 162 | - get_affected_rows() -> int 163 | - get_insert_id() -> int 164 | - get_warnings() -> int 165 | - get_info() -> bytes 166 | - rewind() 167 | - 将行标移动至当前ResultSet开始处 168 | 169 | ### MySQLRequest 170 | - move_to(MySQLRequest) -> None 171 | - 移动当前对象至新对象，模仿C++的std::move 172 | - set_query(bytes/str) -> None 173 | - 设置sql请求 174 | - get_query() -> bytes 175 | - 获取已设置的请求 176 | - query_is_unset() -> bool 177 | - get_seqid() -> int 178 | - set_seqid(int) -> None 179 | - get_command() -> int 180 | - set_size_limit(int) -> None 181 | - get_size_limit() -> int 182 | 183 | ### MySQLResponse 184 | - move_to(MySQLResponse) -> None 185 | - 移动当前对象至新对象，模仿C++的std::move 186 | - is_ok_packet() -> bool 187 | - is_error_packet() -> bool 188 | - get_packet_type() -> int 189 | - 返回Response状态，即`wf.MYSQL_PACKET_*` 190 | - get_affected_rows() -> int 191 | - get_last_insert_id() -> int 192 | - get_warnings() -> int 193 | - get_error_code() -> int 194 | - get_error_msg() -> bytes 195 | - get_sql_state() -> bytes 196 | - get_info() -> bytes 197 | - set_ok_packet() -> None 198 | - get_seqid() -> int 199 | - set_seqid(int) 200 | - get_command() -> int 201 | - set_size_limit(int) -> None 202 | - get_size_limit() -> int 203 | 204 | ### MySQLTask 205 | - start() -> None 206 | - dismiss() -> None 207 | - noreply() -> None 208 | - get_req() -> wf.MySQLRequest 209 | - get_resp() -> wf.MySQLResponse 210 | - get_state() -> int 211 | - get_error() -> int 212 | - get_timeout_reason() -> int 213 | - wf.TOR_NOT_TIMEOUT 214 | - wf.TOR_WAIT_TIMEOUT 215 | - wf.TOR_CONNECT_TIMEOUT 216 | - wf.TOR_TRANSMIT_TIMEOUT 217 | - get_task_seq() -> int 218 | - set_send_timeout(int) -> None 219 | - set_receive_timeout(int) -> None 220 | - set_keep_alive(int) -> None 221 | - get_peer_addr() -> tuple(str, int) 222 | - 返回元组(ip, port)，若获取失败则ip为空字符串 223 | - set_callback(Callable[[wf.MySQLTask], None]) -> None 224 | - set_user_data(object) -> None 225 | - get_user_data() -> object 226 | 227 | ### MySQLServer 228 | - MySQLServer(Callable[[wf.MySQLTask], None]) 229 | - MySQLServer(wf.ServerParams, Callable[[wf.MySQLTask], None]) 230 | - start(int port, str cert_file = '', str key_file = '') -> int 231 | - 启动server，cert_file和key_file任意一个未指定时等同于`start(port)` 232 | - 函数返回0表示启动成功 233 | - start(int family, str host, int port, str cert_file = '', str key_file = '') -> int 234 | - stop() -> None 235 | - 停止server，该函数同步等待当前处理中的请求完成 236 | 237 | ### 其他 238 | - wf.mysql_datatype2str(int) -> str 239 | - 返回`wf.MYSQL_TYPE_*`的str表示 240 | - wf.create_mysql_task(str url, int retry_max, Callable[[wf.MySQLTask], None]) -> wf.MySQLTask 241 | 242 | - wf.MySQLRowIterator(wf.MySQLResultCursor) 243 | - 获得一个遍历当前ResultSet所有行的可迭代对象，每一次迭代返回一个`list[wf.MySQLCell]` 244 | - wf.MySQLRowObjectIterator(wf.MySQLResultCursor) 245 | - 获得一个遍历当前ResultSet所有行的可迭代对象，每一次迭代返回一个`list[object]`，其中object为`wf.MySQLCell.as_object()`的结果 246 | - wf.MySQLResultSetIterator(wf.MySQLResultCursor) 247 | - 获得一个遍历当前MySQLResultCursor所有ResultSet的可迭代对象 248 | - 每一次迭代返回的结果都是wf.MySQLResultCursor的一个引用，该cursor在遍历过程中含有状态，用户不应该在遍历过程中施加额外的操作 249 | 250 | #### MySQL status 251 | - wf.MYSQL_STATUS_NOT_INIT 252 | - wf.MYSQL_STATUS_OK 253 | - wf.MYSQL_STATUS_GET_RESULT 254 | - wf.MYSQL_STATUS_ERROR 255 | - wf.MYSQL_STATUS_END 256 | 257 | #### MySQL type 258 | - wf.MYSQL_TYPE_DECIMAL 259 | - wf.MYSQL_TYPE_TINY 260 | - wf.MYSQL_TYPE_SHORT 261 | - wf.MYSQL_TYPE_LONG 262 | - wf.MYSQL_TYPE_FLOAT 263 | - wf.MYSQL_TYPE_DOUBLE 264 | - wf.MYSQL_TYPE_NULL 265 | - wf.MYSQL_TYPE_TIMESTAMP 266 | - wf.MYSQL_TYPE_LONGLONG 267 | - wf.MYSQL_TYPE_INT24 268 | - wf.MYSQL_TYPE_DATE 269 | - wf.MYSQL_TYPE_TIME 270 | - wf.MYSQL_TYPE_DATETIME 271 | - wf.MYSQL_TYPE_YEAR 272 | - wf.MYSQL_TYPE_NEWDATE 273 | - wf.MYSQL_TYPE_VARCHAR 274 | - wf.MYSQL_TYPE_BIT 275 | - wf.MYSQL_TYPE_TIMESTAMP2 276 | - wf.MYSQL_TYPE_DATETIME2 277 | - wf.MYSQL_TYPE_TIME2 278 | - wf.MYSQL_TYPE_TYPED_ARRAY 279 | - wf.MYSQL_TYPE_JSON 280 | - wf.MYSQL_TYPE_NEWDECIMAL 281 | - wf.MYSQL_TYPE_ENUM 282 | - wf.MYSQL_TYPE_SET 283 | - wf.MYSQL_TYPE_TINY_BLOB 284 | - wf.MYSQL_TYPE_MEDIUM_BLOB 285 | - wf.MYSQL_TYPE_LONG_BLOB 286 | - wf.MYSQL_TYPE_BLOB 287 | - wf.MYSQL_TYPE_VAR_STRING 288 | - wf.MYSQL_TYPE_STRING 289 | - wf.MYSQL_TYPE_GEOMETRY 290 | 291 | #### MySQL packet 292 | - wf.MYSQL_PACKET_OTHER 293 | - wf.MYSQL_PACKET_OK 294 | - wf.MYSQL_PACKET_NULL 295 | - wf.MYSQL_PACKET_EOF 296 | - wf.MYSQL_PACKET_ERROR 297 | - wf.MYSQL_PACKET_GET_RESULT 298 | - wf.MYSQL_PACKET_LOCAL_INLINE 299 | 300 | ### 示例 301 | ```python 302 | # 发起一个MySQL请求 303 | import pywf as wf 304 | 305 | def mysql_callback(task): 306 | print(task.get_state(), task.get_error()) 307 | 308 | url = "mysql://user:password@host:port/database" 309 | mysql_task = wf.create_mysql_task(url=url, retry_max=1, callback=mysql_callback) 310 | req = mysql_task.get_req() 311 | req.set_query("select * from your_table limit 10;") 312 | mysql_task.start() 313 | wf.wait_finish() 314 | ``` 315 | 316 | ```python 317 | # 遍历MySQL请求结果 318 | default_column_width = 20 319 | def header_line(sz): 320 | return '+' + '+'.join(['-' * default_column_width for i in range(sz)]) + '+' 321 | 322 | def format_row(row): 323 | return '|' + '|'.join([f"{str(x):^{default_column_width}}" for x in row]) + '|' 324 | 325 | def mysql_callback(task): 326 | state = task.get_state() 327 | error = task.get_error() 328 | 329 | if state != wf.WFT_STATE_SUCCESS: 330 | print(wf.get_error_string(state, error)) 331 | return 332 | 333 | resp = task.get_resp() 334 | cursor = wf.MySQLResultCursor(resp) 335 | 336 | # 使用迭代语法遍历所有结果集 337 | for result_set in wf.MySQLResultSetIterator(cursor): 338 | # 1. 遍历每个结果集 339 | if result_set.get_cursor_status() == wf.MYSQL_STATUS_GET_RESULT: 340 | fields = result_set.fetch_fields() 341 | 342 | print(header_line(len(fields))) 343 | print(format_row(fields)) 344 | print(header_line(len(fields))) 345 | # 2. 遍历每一行 346 | for row in wf.MySQLRowIterator(result_set): 347 | # 3. 遍历每个Cell 348 | print(format_row(row)) 349 | 350 | print(header_line(len(fields))) 351 | print("{} {} in set\n".format( 352 | result_set.get_rows_count(), 353 | "row" if result_set.get_rows_count() == 1 else "rows" 354 | )) 355 | elif result_set.get_cursor_status() == wf.MYSQL_STATUS_OK: 356 | print("OK. {} {} affected. {} warnings. insert_id={}. {}".format( 357 | result_set.get_affected_rows(), 358 | "row" if result_set.get_affected_rows() == 1 else "rows", 359 | result_set.get_warnings(), 360 | result_set.get_insert_id(), 361 | str(result_set.get_info()) 362 | )) 363 | 364 | # ... 365 | ``` 366 | 367 | ```python 368 | # 遍历MySQL请求结果 369 | import pywf as wf 370 | 371 | default_column_width = 20 372 | def header_line(sz): 373 | return '+' + '+'.join(['-' * default_column_width for i in range(sz)]) + '+' 374 | 375 | def format_row(row): 376 | return '|' + '|'.join([f"{str(x):^{default_column_width}}" for x in row]) + '|' 377 | 378 | def mysql_callback(task): 379 | state = task.get_state() 380 | error = task.get_error() 381 | 382 | if state != wf.WFT_STATE_SUCCESS: 383 | print(wf.get_error_string(state, error)) 384 | return 385 | 386 | resp = task.get_resp() 387 | cursor = wf.MySQLResultCursor(resp) 388 | 389 | # 遍历所有结果集 390 | while True: 391 | # 1. 遍历每个结果集 392 | if cursor.get_cursor_status() == wf.MYSQL_STATUS_GET_RESULT: 393 | fields = cursor.fetch_fields() 394 | 395 | print(header_line(len(fields))) 396 | print(format_row(fields)) 397 | print(header_line(len(fields))) 398 | 399 | # 2. 遍历每一行，两种方案任选其一 400 | # 2.1 使用fetch_all 401 | all_row = cursor.fetch_all() 402 | for row in all_row: 403 | print(format_row(row)) 404 | # 2.2 使用fetch_row 405 | cursor.rewind() # 回到当前结果集起始位置 406 | while True: 407 | row = cursor.fetch_row() 408 | if row == None: 409 | break 410 | # 3. 遍历每个Cell 411 | print(format_row(row)) 412 | print(header_line(len(fields))) 413 | print("{} {} in set\n".format( 414 | cursor.get_rows_count(), 415 | "row" if cursor.get_rows_count() == 1 else "rows" 416 | )) 417 | elif cursor.get_cursor_status() == wf.MYSQL_STATUS_OK: 418 | print("OK. {} {} affected. {} warnings. insert_id={}. {}".format( 419 | cursor.get_affected_rows(), 420 | "row" if cursor.get_affected_rows() == 1 else "rows", 421 | cursor.get_warnings(), 422 | cursor.get_insert_id(), 423 | str(cursor.get_info()) 424 | )) 425 | if cursor.next_result_set() == False: 426 | break 427 | 428 | # ... 429 | ``` 430 | -------------------------------------------------------------------------------- /doc/others.md: -------------------------------------------------------------------------------- 1 | ## Other Tasks 2 | 本文档介绍结构较为简单的任务 3 | 4 | ### FileTasks 5 | 文件相关任务用于对打开的fd进行异步读写等操作，包括`FileIOTask`, `FileVIOTask`, `FileSyncTask` 6 | 7 | #### FileIOTask 8 | - start() -> None 9 | - dismiss() -> None 10 | - get_state() -> int 11 | - get_error() -> int 12 | - get_retval() -> int 13 | - 文件读写操作的返回值 14 | - set_callback(Callable[[wf.FileIOTask], None]) -> None 15 | - set_user_data(object) -> None 16 | - get_user_data() -> object 17 | - get_fd() -> int 18 | - 获取和该任务相关的fd 19 | - get_offset() -> int 20 | - get_count() -> int 21 | - 文件操作的字节数 22 | - get_data() -> bytes 23 | - 文件读取操作的结果 24 | 25 | #### FileVIOTask 26 | - start() -> None 27 | - dismiss() -> None 28 | - get_state() -> int 29 | - get_error() -> int 30 | - get_retval() -> int 31 | - 文件写操作的返回值 32 | - set_callback(Callable[[wf.FileVIOTask], None]) -> None 33 | - set_user_data(object) -> None 34 | - get_user_data() -> object 35 | - get_fd() -> int 36 | - 获取和该任务相关的fd 37 | - get_offset() -> int 38 | - get_data() -> list[bytes] 39 | - 文件读取操作的结果 40 | 41 | #### FileSyncTask 42 | - start() -> None 43 | - dismiss() -> None 44 | - get_state() -> int 45 | - get_error() -> int 46 | - get_retval() -> int 47 | - 文件操作的返回值 48 | - set_callback(Callable[[wf.FileSyncTask], None]) -> None 49 | - set_user_data(object) -> None 50 | - get_user_data() -> object 51 | - get_fd() -> int 52 | - 获取和该任务相关的fd 53 | 54 | ### TimerTask 55 | - start() -> None 56 | - dismiss() -> None 57 | - get_state() -> int 58 | - get_error() -> int 59 | - set_callback(Callable[[wf.TimerTask], None]) -> None 60 | - set_user_data(object) -> None 61 | - get_user_data() -> object 62 | 63 | ### CounterTask 64 | - start() -> None 65 | - dismiss() -> None 66 | - get_state() -> int 67 | - get_error() -> int 68 | - set_callback(Callable[[wf.CounterTask], None]) -> None 69 | - set_user_data(object) -> None 70 | - get_user_data() -> object 71 | - count() -> None 72 | - 增加一次计数 73 | 74 | ### GoTask 75 | - start() -> None 76 | - dismiss() -> None 77 | - get_state() -> int 78 | - get_error() -> int 79 | - set_callback(Callable[[wf.GoTask], None]) -> None 80 | - set_user_data(object) -> None 81 | - get_user_data() -> object 82 | 83 | ### EmptyTask 84 | - start() -> None 85 | - dismiss() -> None 86 | - get_state() -> int 87 | - get_error() -> int 88 | 89 | ### DynamicTask 90 | - start() -> None 91 | - dismiss() -> None 92 | - get_state() -> int 93 | - get_error() -> int 94 | 95 | ### 任务工厂等 96 | - wf.create_pread_task(int fd, int count, int offset, callback) -> wf.FileIOTask 97 | - wf.create_pwrite_task(int fd, bytes data, int count, int offset, callback) -> wf.FileIOTask 98 | - 若data长度小于count，则以真实长度为准 99 | - wf.create_pwritev_task(int fd, list[bytes], int offset, callback) -> wf.FileVIOTask 100 | - wf.create_fsync_task(int fd, callback) -> wf.FileSyncTask 101 | - wf.create_fdsync_task(int fd, callback) -> wf.FileSyncTask 102 | - wf.create_timer_task(int microseconds, callback) -> wf.TimerTask 103 | - wf.create_counter_task(int target, callback) -> wf.CounterTask 104 | - wf.create_counter_task(str name, int target, callback) -> wf.TimerTask 105 | - wf.count_by_name(str name, int n) -> None 106 | - 通过任务名称进行n次计数 107 | - wf.create_go_task(Callable[[*args, **kwargs], None], args, kwargs) -> wf.GoTask 108 | - wf.create_empty_task() -> wf.EmptyTask 109 | - wf.create_dynamic_task(Callable[[wf.DynamicTask], wf.SubTask]) -> wf.DynamicTask 110 | 111 | ### 示例 112 | ```py 113 | # GoTask 114 | import pywf as wf 115 | 116 | def go_callback(task): 117 | print("state: {}, error: {}".format( 118 | task.get_state(), task.get_error() 119 | )) 120 | print("user_data: ", task.get_user_data()) 121 | pass 122 | 123 | def go(a, b, c): 124 | print(a, b, c) 125 | 126 | go_task = wf.create_go_task(go, 1, "pywf", 3.1415926) 127 | go_task.set_user_data({"key": "value"}) 128 | go_task.set_callback(go_callback) 129 | go_task.start() 130 | wf.wait_finish() 131 | ``` 132 | 133 | ```py 134 | # CounterTask 135 | import pywf as wf 136 | 137 | def counter_callback(task): 138 | print('counter callback') 139 | 140 | task = wf.create_counter_task("counter", 2, counter_callback) 141 | task.count() 142 | task.start() 143 | wf.count_by_name("counter", 1) 144 | wf.wait_finish() 145 | ``` 146 | 147 | ```py 148 | # TimerTask 149 | import pywf as wf 150 | import time 151 | 152 | start = 0 153 | 154 | def timer_callback(task): 155 | stop = time.time() 156 | print(stop - start) 157 | 158 | # sleep for 10 ms 159 | task = wf.create_timer_task(10 * 1000, timer_callback) 160 | start = time.time() 161 | task.start() 162 | wf.wait_finish() 163 | ``` 164 | -------------------------------------------------------------------------------- /doc/pywf.md: -------------------------------------------------------------------------------- 1 | # pywf 2 | 3 | ## 基础类型 4 | 在C++ Workflow中，所有由Workflow创建的`xxxTask`或`xxxMessage`等资源均由Workflow负责释放，用户只需要在资源生命周期结束之前通过指针或引用等方式访问公开接口。在Python Workflow中，为了实现这一特性，所有的原生指针被包装成一个含有该指针的Python对象，且均派生自`WFBase`类。Python用户同样需要注意，所有由被包装的指针指向的对象的生命周期均由Workflow管理，在对应的任务`callback`结束后就会立即被释放，虽然Python层面仍保留有一个指针，但此时已经成为野指针，用户切不可再次使用。可能有一些奇技淫巧可以在运行时报告这种错误，但我们暂时没有计划这样做。如果用户要判断哪种对象需要注意生命周期的问题，可以进行如下判断 5 | ```py 6 | t = wf.create_http_task("http://www.sogou.com/", 1, 1, None) 7 | isinstance(t, wf.WFBase) # True 8 | issubclass(wf.HttpTask, wf.WFBase) # True 9 | t.dismiss() 10 | # 在一个task被直接或间接 dismiss/start 之后，用户不再拥有其所有权 11 | # 此后用户只能在该task的回调函数内部进行操作 12 | ``` 13 | 14 | ### SubTask 15 | `SubTask`是所有Task类型的基类，大部分情况下用户不需要关心它。除非用户需要了解某个对象是否是一个Task 16 | ```py 17 | t = wf.create_http_task("http://www.sogou.com/", 1, 1, None) 18 | p = wf.create_parallel_work(None) 19 | isinstance(t, wf.SubTask) # 所有task都是SubTask 20 | isinstance(p, wf.SubTask) # parallel work是一种task 21 | ``` 22 | 23 | ### SeriesWork 24 | - is_null() -> bool 25 | - 判断该对象是否为空指针 26 | - start() -> None 27 | - 启动串行 28 | - dismiss() -> None 29 | - 放弃串行中的所有任务 30 | - push_back(wf.SubTask) -> None 31 | - 在串行尾部添加任务 32 | - push_front(wf.SubTask) -> None 33 | - 在串行头部添加任务 34 | - cancel() -> None 35 | - 取消串行 36 | - is_canceled() -> bool 37 | - 串行是否被取消 38 | - set_callback(Callable[[wf.ConstSeriesWork], None]) -> None 39 | - 为串行设置回调函数，将参数设置为`None`可以取消回调 40 | - set_context(object) -> None 41 | - 用户可以为串行设置一个Python object作为context，在串行生命周期存续的任意时刻将object取回任意次 42 | - 串行将在自身被释放时取消对该object的引用 43 | - 将参数设置为`None`可以提前取消串行对该object的引用 44 | - get_context() -> object 45 | - 取回context，若从未设置过context，此调用返回`None` 46 | - `__lshift__`(wf.SubTask) -> wf.SeriesWork 47 | - 将一个任务加入到串行中，作用同`push_back` 48 | - 但用户可以方便地执行 `series << task1 << task2 << task3` 49 | 50 | 另有一个`ConstSeriesWork`，仅可调用`is_null`、`is_canceled`、`get_context`几个函数 51 | 52 | ### ParallelWork 53 | - is_null() -> bool 54 | - 判断该对象是否为空指针 55 | - start() -> None 56 | - 并行是一种任务，启动并行会将其放入串行，并启动串行 57 | - dismiss() -> None 58 | - 取消并行中的所有任务 59 | - add_series(wf.SeriesWork) 60 | - 将串行加入到并行中，并行由一组串行构成 61 | - series_at(int) -> wf.ConstSeriesWork 62 | - 获取指定位置上的Series，用户需保证该位置合法 63 | - set_context(object) -> None 64 | - 用户可以为并行设置一个Python object作为context，在并行生命周期存续的任意时刻将object取回任意次 65 | - 并行将在自身被释放时取消对该object的引用 66 | - 将参数设置为`None`可以提前取消并行对该object的引用 67 | - get_context() -> object 68 | - 取回context，若从未设置过context，此调用返回`None` 69 | - size() -> int 70 | - 获取并行中串行的个数 71 | - set_callback(Callable[[wf.ConstParallelWork], None]) -> None 72 | - 为并行设置回调函数，将参数设置为`None`可以取消回调 73 | 74 | 另有一个`ConstParallelWork`，仅可调用`is_null`、`series_at`、`get_context`、`size`几个函数 75 | 76 | ### 几个全局函数 77 | - wf.WORKFLOW_library_init(wf.GlobalSettings) -> None 78 | - 初始化workflow全局参数 79 | - wf.create_series_work(wf.SubTask, Callable[[wf.ConstSeriesWork], None]) -> None 80 | - wf.create_series_work(wf.SubTask, wf.SubTask, Callable[[wf.ConstSeriesWork], None]) -> None 81 | - wf.start_series_work(wf.SubTask, Callable[[wf.ConstSeriesWork], None]) -> None 82 | - wf.start_series_work(wf.SubTask, wf.SubTask, Callable[[wf.ConstSeriesWork], None]) -> None 83 | - wf.create_parallel_work(Callable[[wf.ConstParallelWork], None]) -> None 84 | - wf.create_parallel_work(list[wf.SeriesWork], Callable[[wf.ConstParallelWork], None]) -> None 85 | - wf.start_parallel_work(list[wf.SeriesWork], Callable[[wf.ConstParallelWork], None]) -> None 86 | - wf.series_of(wf.SubTask) -> wf.SeriesWork 87 | - 获取task所在的series 88 | - 未被加入到sereis的task会返回空指针，用series.is_null()来检查 89 | - wf.wait_finish() -> None 90 | - **重要** 91 | - workflow设计了一套严密的管理体系来保证所有的任务都会在正确的时机被释放，但在PyWF中，一些Python对象会被引用在Series或者Task中，需要保证在Python线程退出前，这些对象均已被成功释放 92 | - 在PyWF中，用户只需在需要等待所有串行完成的地方调用`wf.wait_finish()`，该函数返回时所有被启动的串行均已被执行，被放弃的串行均已被析构 93 | - `wf.wait_finish()`可以作为一种序列点，若用户创建了串行但既不启动也不放弃，则`wf.wait_finish()`不会返回 94 | - wf.wait_finish_timeout(float seconds) -> None 95 | - 等待串行完成，可以传入一个以秒计时的参数 96 | - 函数返回`True`时表示所有串行执行完成 97 | - wf.get_error_string(int state, int error) -> None 98 | - 获取`state, error`状态码对应的可读的字符串表示 99 | 100 | ### 其他 101 | - 状态码，同workflow 102 | - wf.WFT_STATE_UNDEFINED 103 | - wf.WFT_STATE_SUCCESS 104 | - wf.WFT_STATE_TOREPLY 105 | - wf.WFT_STATE_NOREPLY 106 | - wf.WFT_STATE_SYS_ERROR 107 | - wf.WFT_STATE_SSL_ERROR 108 | - wf.WFT_STATE_DNS_ERROR 109 | - wf.WFT_STATE_TASK_ERROR 110 | - wf.WFT_STATE_ABORTED 111 | - wf.EndpointParams同workflow的EndpointParams 112 | - wf.GlobalSettings同workflow的WFGlobalSettings 113 | -------------------------------------------------------------------------------- /doc/redis.md: -------------------------------------------------------------------------------- 1 | ## Redis Client/Server 2 | 3 | ### Tutorials 4 | - [tutorial02-redis_cli.py](../tutorial/tutorial02-redis_cli.py) 5 | - [tutorial03-wget_to_redis.py](../tutorial/tutorial03-wget_to_redis.py) 6 | 7 | ### RedisValue 8 | - 构造函数 9 | - RedisValue() 10 | - RedisValue(RedisValue) 11 | - copy() -> RedisValue 12 | - 获得当前对象的一份深拷贝 13 | - move_to(RedisValue) -> None 14 | - 将当前对象移动至另一对象，模仿C++的std::move 15 | - move_from(RedisValue) -> None 16 | - 将另一个对象的内容移动至当前对象，模仿C++的std::move 17 | 18 | - set_nil() -> None 19 | - set_int(int) -> None 20 | - set_array(uint newsize) -> None 21 | - 设置RedisValue为数组，并指定大小 22 | - set_string(str/bytes) -> None 23 | - set_status(str/bytes) -> None 24 | - set_error(str/bytes) -> None 25 | 26 | - is_ok() -> bool 27 | - is_error() -> bool 28 | - is_nil() -> bool 29 | - is_int() -> bool 30 | - is_array() -> bool 31 | - is_string() -> bool 32 | 33 | - string_value() -> bytes 34 | - int_value() -> int 35 | - arr_size() -> int 36 | - arr_clear() -> None 37 | - arr_resize(uint newsize) -> None 38 | 39 | - clear() -> None 40 | - 清空对象，等价于`set_nil` 41 | - debug_string() -> bytes 42 | - 获取当前对象的字符串表示 43 | - arr_at(uint pos) -> wf.RedisValue 44 | - 若当前对象为数组，返回位于pos位置的RedisValue的拷贝，否则返回None 45 | - arr_at_ref(uint pos) -> wf.RedisValue 46 | - 若当前对象为数组，返回位于pos位置的RedisValue的引用，用户自行保证生命周期，否则返回None 47 | - arr_at_object(uint pos) -> object 48 | - 若当前对象为数组，返回位于pos位置的RedisValue的Python对象表示，否则返回None 49 | - as_object() -> object 50 | - 返回当前对象的Python对象表示 51 | - string、error、status均为Python bytes 52 | - nil为Python None 53 | - array为Python list 54 | 55 | ### RedisRequest 56 | - move_to(RedisRequest) -> None 57 | - 移动当前对象至新对象，模仿C++的std::move 58 | - set_request(str cmd, list[str/bytes]) -> None 59 | - get_command() -> str 60 | - get_params() -> list[bytes] 61 | - 获取参数列表，注意参数类型均为bytes 62 | - set_size_limit(uint) 63 | - get_size_limit() -> int 64 | 65 | ### RedisResponse 66 | - move_to(RedisResponse) -> None 67 | - 移动当前对象至新对象，模仿C++的std::move 68 | - get_result() -> wf.RedisValue 69 | - 返回RedisValue的拷贝 70 | - set_result(RedisValue) -> None 71 | - 将RedisValue设置到RedisResponse中 72 | - set_size_limit(uint) 73 | - get_size_limit() -> int 74 | 75 | ### RedisTask 76 | - start() -> None 77 | - dismiss() -> None 78 | - noreply() -> None 79 | - get_req() -> wf.RedisRequest 80 | - get_resp() -> wf.RedisResponse 81 | - get_state() -> int 82 | - get_error() -> int 83 | - get_timeout_reason() -> int 84 | - wf.TOR_NOT_TIMEOUT 85 | - wf.TOR_WAIT_TIMEOUT 86 | - wf.TOR_CONNECT_TIMEOUT 87 | - wf.TOR_TRANSMIT_TIMEOUT 88 | - get_task_seq() -> int 89 | - set_send_timeout(int) -> None 90 | - set_receive_timeout(int) -> None 91 | - set_keep_alive(int) -> None 92 | - get_peer_addr() -> tuple(str, int) 93 | - 返回元组(ip, port)，若获取失败则ip为空字符串 94 | - set_callback(Callable[[wf.RedisTask], None]) -> None 95 | - set_user_data(object) -> None 96 | - get_user_data() -> object 97 | 98 | ### RedisServer 99 | - RedisServer(Callable[[wf.RedisTask], None]) 100 | - RedisServer(wf.ServerParams, Callable[[wf.RedisTask], None]) 101 | - start(int port, str cert_file = '', str key_file = '') -> int 102 | - 启动server，cert_file和key_file任意一个未指定时等同于`start(port)` 103 | - 函数返回0表示启动成功 104 | - start(int family, str host, int port, str cert_file = '', str key_file = '') -> int 105 | - stop() -> None 106 | - 停止server，该函数同步等待当前处理中的请求完成 107 | 108 | ### 任务工厂等 109 | - wf.create_redis_task(str url, int retry_max, Callable[[wf.RedisTask], None]) -> wf.RedisTask 110 | 111 | ### 示例 112 | 113 | RedisTask使用较为简单，直接参考tutorials即可。 114 | 115 | ```py 116 | # RedisValue 117 | import pywf as wf 118 | 119 | def construct(): 120 | v1 = wf.RedisValue() 121 | 122 | print(v1.as_object()) # None 123 | 124 | def set_check(): 125 | v1 = wf.RedisValue() 126 | v2 = wf.RedisValue() 127 | v1.set_int(1024) 128 | v2.set_status(b'status') 129 | 130 | print(v1.is_int(), v1.int_value()) # True 1024 131 | print(v2.is_ok(), v2.is_string(), v2.string_value()) # True True b'status' 132 | 133 | def array(): 134 | v1 = wf.RedisValue() 135 | v1.set_array(3) 136 | 137 | v1[0].set_string('string') 138 | v11_ref = v1.arr_at_ref(1) 139 | v11_ref.set_array(2) 140 | v11_ref[0].set_int(1) 141 | v11_ref[1].set_string(b'value') 142 | v1[2] = v1[0] # v1[2] is a copy of v1[0] 143 | v1[0].set_nil() 144 | 145 | print(v1.as_object()) # [None, [1, b'value'], b'string'] 146 | 147 | v2 = wf.RedisValue() 148 | v3 = wf.RedisValue() 149 | v3.set_string('string') 150 | v2.set_array(2) 151 | v3.move_to(v2[0]) 152 | v21 = v2.arr_at(1) # arr_at returns a copy 153 | v21.set_error('error') # So v2[1] not changed 154 | print(v2.as_object()) # [b'string', None] 155 | 156 | if __name__ == "__main__": 157 | construct() 158 | set_check() 159 | array() 160 | ``` 161 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools>=42", 4 | "wheel", 5 | "ninja; sys_platform != 'win32'", 6 | "cmake>=3.12", 7 | ] 8 | build-backend = "setuptools.build_meta" 9 | 10 | -------------------------------------------------------------------------------- /pywf/__init__.py: -------------------------------------------------------------------------------- 1 | '''Root module of pywf''' 2 | from .cpp_pyworkflow import * 3 | from .mysql_iterator import MySQLResultSetIterator 4 | from .mysql_iterator import MySQLRowIterator 5 | from .mysql_iterator import MySQLRowObjectIterator 6 | 7 | 8 | inner_init() 9 | del inner_init 10 | -------------------------------------------------------------------------------- /pywf/mysql_iterator.py: -------------------------------------------------------------------------------- 1 | from .cpp_pyworkflow import MySQLResultCursor 2 | from .cpp_pyworkflow import MYSQL_STATUS_GET_RESULT 3 | from .cpp_pyworkflow import MYSQL_STATUS_OK 4 | 5 | def MySQLRowIterator(cursor): 6 | assert type(cursor) == MySQLResultCursor 7 | cursor.rewind() 8 | while True: 9 | row = cursor.fetch_row() 10 | if row is None: 11 | break 12 | yield row 13 | 14 | def MySQLRowObjectIterator(cursor): 15 | assert type(cursor) == MySQLResultCursor 16 | cursor.rewind() 17 | while True: 18 | row = cursor.fetch_row() 19 | if row is None: 20 | break 21 | yield [cell.as_object() for cell in row] 22 | 23 | def MySQLResultSetIterator(cursor): 24 | assert type(cursor) == MySQLResultCursor 25 | if cursor.get_cursor_status() != MYSQL_STATUS_GET_RESULT and \ 26 | cursor.get_cursor_status() != MYSQL_STATUS_OK: 27 | return 28 | cursor.first_result_set() 29 | yield cursor 30 | while cursor.next_result_set(): 31 | yield cursor 32 | -------------------------------------------------------------------------------- /requirements/requirements-lint.txt: -------------------------------------------------------------------------------- 1 | autoflake 2 | black 3 | isort>=5.0.0 4 | -------------------------------------------------------------------------------- /scripts/lint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | export PREFIX="" 4 | if [ -d 'venv' ] ; then 5 | export PREFIX="venv/bin/" 6 | fi 7 | 8 | set -x 9 | 10 | pip install -r requirements/requirements-lint.txt 11 | 12 | FILES="tutorial setup.py" 13 | 14 | ${PREFIX}autoflake --in-place --recursive --remove-all-unused-imports --remove-unused-variables ${FILES} 15 | ${PREFIX}black --exclude=".pyi$" ${FILES} 16 | ${PREFIX}isort --multi-line=3 --trailing-comma --force-grid-wrap=0 --combine-as --line-width 88 ${FILES} 17 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import shutil 4 | import subprocess 5 | import sys 6 | 7 | from setuptools import Extension, setup 8 | from setuptools.command.build_ext import build_ext 9 | 10 | # Convert distutils Windows platform specifiers to CMake -A arguments 11 | PLAT_TO_CMAKE = { 12 | "win32": "Win32", 13 | "win-amd64": "x64", 14 | "win-arm32": "ARM", 15 | "win-arm64": "ARM64", 16 | } 17 | 18 | 19 | # A CMakeExtension needs a sourcedir instead of a file list. 20 | # The name must be the _single_ output extension from the CMake build. 21 | # If you need multiple extensions, see scikit-build. 22 | class CMakeExtension(Extension): 23 | def __init__(self, name, sourcedir=""): 24 | Extension.__init__(self, name, sources=[]) 25 | self.sourcedir = os.path.abspath(sourcedir) 26 | 27 | 28 | class CMakeBuild(build_ext): 29 | def build_extension(self, ext): 30 | extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) 31 | 32 | # required for auto-detection of auxiliary "native" libs 33 | if not extdir.endswith(os.path.sep): 34 | extdir += os.path.sep 35 | 36 | cfg = "Debug" if self.debug else "Release" 37 | 38 | # CMake lets you override the generator - we need to check this. 39 | # Can be set with Conda-Build, for example. 40 | cmake_generator = os.environ.get("CMAKE_GENERATOR", "") 41 | 42 | # Set Python_EXECUTABLE instead if you use PYBIND11_FINDPYTHON 43 | # PYWORKFLOW_VERSION_INFO shows you how to pass a value into the C++ code 44 | # from Python. 45 | cmake_args = [ 46 | "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}".format(extdir), 47 | "-DPYTHON_EXECUTABLE={}".format(sys.executable), 48 | "-DPYWORKFLOW_VERSION_INFO={}".format(self.distribution.get_version()), 49 | "-DCMAKE_BUILD_TYPE={}".format(cfg), # not used on MSVC, but no harm 50 | ] 51 | build_args = [] 52 | 53 | if self.compiler.compiler_type != "msvc": 54 | # Using Ninja-build since it a) is available as a wheel and b) 55 | # multithreads automatically. MSVC would require all variables be 56 | # exported for Ninja to pick it up, which is a little tricky to do. 57 | # Users can override the generator with CMAKE_GENERATOR in CMake 58 | # 3.15+. 59 | 60 | if not cmake_generator: 61 | cmake_args += ["-GNinja"] 62 | 63 | else: 64 | 65 | # Single config generators are handled "normally" 66 | single_config = any(x in cmake_generator for x in {"NMake", "Ninja"}) 67 | 68 | # CMake allows an arch-in-generator style for backward compatibility 69 | contains_arch = any(x in cmake_generator for x in {"ARM", "Win64"}) 70 | 71 | # Specify the arch if using MSVC generator, but only if it doesn't 72 | # contain a backward-compatibility arch spec already in the 73 | # generator name. 74 | if not single_config and not contains_arch: 75 | cmake_args += ["-A", PLAT_TO_CMAKE[self.plat_name]] 76 | 77 | # Multi-config generators have a different way to specify configs 78 | if not single_config: 79 | cmake_args += [ 80 | "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}".format(cfg.upper(), extdir) 81 | ] 82 | build_args += ["--config", cfg] 83 | 84 | # Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level 85 | # across all generators. 86 | if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ: 87 | # self.parallel is a Python 3 only way to set parallel jobs by hand 88 | # using -j in the build_ext call, not supported by pip or PyPA-build. 89 | if hasattr(self, "parallel") and self.parallel: 90 | # CMake 3.12+ only. 91 | build_args += ["-j{}".format(self.parallel)] 92 | 93 | if not os.path.exists(self.build_temp): 94 | os.makedirs(self.build_temp) 95 | 96 | cmake = "cmake" 97 | if shutil.which("cmake3") != None: 98 | cmake = "cmake3" 99 | 100 | subprocess.check_call([cmake, ext.sourcedir] + cmake_args, cwd=self.build_temp) 101 | subprocess.check_call([cmake, "--build", "."] + build_args, cwd=self.build_temp) 102 | 103 | 104 | setup( 105 | name="pywf", 106 | version="0.0.10", 107 | author="sogou", 108 | author_email="liukaidx@sogou-inc.com", 109 | description="A C++ Workflow Wrapper for Python", 110 | long_description="A C++ Workflow Wrapper for Python", 111 | long_description_content_type="text/plain", 112 | url="https://github.com/sogou/pyworkflow", 113 | ext_modules=[CMakeExtension("pywf.cpp_pyworkflow")], 114 | cmdclass={"build_ext": CMakeBuild}, 115 | zip_safe=False, 116 | license="Apache License 2.0", 117 | packages=["pywf"], 118 | classifiers=[ 119 | "Development Status :: 3 - Alpha", 120 | "Intended Audience :: Developers", 121 | "Programming Language :: Python :: 3", 122 | "Programming Language :: Python :: 3.6", 123 | "Programming Language :: Python :: 3.7", 124 | "Programming Language :: Python :: 3.8", 125 | "Programming Language :: Python :: 3 :: Only", 126 | "Programming Language :: Python :: Implementation :: CPython", 127 | ], 128 | ) 129 | -------------------------------------------------------------------------------- /src/common_types.cc: -------------------------------------------------------------------------------- 1 | #include "common_types.h" 2 | #include "workflow/EndpointParams.h" 3 | #include "workflow/WFGlobal.h" 4 | #include "workflow/WFTask.h" 5 | #include 6 | 7 | static constexpr struct WFGlobalSettings PYWF_GLOBAL_SETTINGS_DEFAULT = 8 | { 9 | .endpoint_params = ENDPOINT_PARAMS_DEFAULT, 10 | .dns_server_params = ENDPOINT_PARAMS_DEFAULT, 11 | .dns_ttl_default = 12 * 3600, 12 | .dns_ttl_min = 180, 13 | .dns_threads = 4, 14 | .poller_threads = 4, 15 | .handler_threads = 4, 16 | .compute_threads = 4, 17 | .resolv_conf_path = "/etc/resolv.conf", 18 | .hosts_path = "/etc/hosts", 19 | }; 20 | 21 | std::mutex CountableSeriesWork::series_mtx; 22 | size_t CountableSeriesWork::series_counter = 0; 23 | std::condition_variable CountableSeriesWork::series_cv; 24 | 25 | PySeriesWork create_series_work(PySubTask &first, py_series_callback_t cb) { 26 | auto ptr = CountableSeriesWork::create_series_work( 27 | first.get(), [cb](const SeriesWork *p) { 28 | py_callback_wrapper(cb, PyConstSeriesWork(const_cast(p))); 29 | }); 30 | return PySeriesWork(ptr); 31 | } 32 | PySeriesWork create_series_work(PySubTask &first, PySubTask &last, py_series_callback_t cb) { 33 | auto ptr = CountableSeriesWork::create_series_work( 34 | first.get(), last.get(), [cb](const SeriesWork *p) { 35 | py_callback_wrapper(cb, PyConstSeriesWork(const_cast(p))); 36 | }); 37 | return PySeriesWork(ptr); 38 | } 39 | void start_series_work(PySubTask &first, py_series_callback_t cb) { 40 | CountableSeriesWork::start_series_work( 41 | first.get(), [cb](const SeriesWork *p) { 42 | py_callback_wrapper(cb, PyConstSeriesWork(const_cast(p))); 43 | }); 44 | } 45 | void start_series_work(PySubTask &first, PySubTask &last, py_series_callback_t cb) { 46 | CountableSeriesWork::start_series_work( 47 | first.get(), last.get(), [cb](const SeriesWork *p) { 48 | py_callback_wrapper(cb, PyConstSeriesWork(const_cast(p))); 49 | }); 50 | } 51 | 52 | PyParallelWork create_parallel_work(py_parallel_callback_t cb) { 53 | auto ptr = CountableParallelWork::create_parallel_work([cb](const ParallelWork *p) { 54 | py_callback_wrapper(cb, PyConstParallelWork(const_cast(p))); 55 | }); 56 | return PyParallelWork(ptr); 57 | } 58 | PyParallelWork create_parallel_work(std::vector &v, py_parallel_callback_t cb) { 59 | std::vector works(v.size()); 60 | for(size_t i = 0; i < v.size(); i++) works[i] = v[i].get(); 61 | auto ptr = CountableParallelWork::create_parallel_work(works.data(), works.size(), 62 | [cb](const ParallelWork *p) { 63 | py_callback_wrapper(cb, PyConstParallelWork(const_cast(p))); 64 | }); 65 | return PyParallelWork(ptr); 66 | } 67 | void start_parallel_work(std::vector &v, py_parallel_callback_t cb) { 68 | std::vector works(v.size()); 69 | for(size_t i = 0; i < v.size(); i++) works[i] = v[i].get(); 70 | CountableParallelWork::start_parallel_work(works.data(), works.size(), 71 | [cb](const ParallelWork *p) { 72 | py_callback_wrapper(cb, PyConstParallelWork(const_cast(p))); 73 | }); 74 | } 75 | 76 | void PyWorkflow_library_init(const struct WFGlobalSettings s) { 77 | WORKFLOW_library_init(&s); 78 | } 79 | 80 | WFGlobalSettings get_global_settings() { 81 | return *WFGlobal::get_global_settings(); 82 | } 83 | 84 | PySeriesWork py_series_of(const PySubTask &t) { 85 | SeriesWork *s = series_of(t.get()); 86 | return PySeriesWork(s); 87 | } 88 | 89 | std::string get_error_string(int state, int error) { 90 | std::string s; 91 | const char *p = WFGlobal::get_error_string(state, error); 92 | if(p) s.assign(p); 93 | return s; 94 | } 95 | 96 | /** 97 | * Do some initialize before start. 98 | * The user should not call this function. 99 | */ 100 | void inner_init() { 101 | #if OPENSSL_VERSION_NUMBER >= 0x10100000L 102 | OPENSSL_init_ssl(0, NULL); 103 | #endif 104 | WORKFLOW_library_init(&PYWF_GLOBAL_SETTINGS_DEFAULT); 105 | } 106 | 107 | void init_common_types(py::module_ &wf) { 108 | wf.attr("WFT_STATE_UNDEFINED") = (int)WFT_STATE_UNDEFINED; 109 | wf.attr("WFT_STATE_SUCCESS") = (int)WFT_STATE_SUCCESS; 110 | wf.attr("WFT_STATE_TOREPLY") = (int)WFT_STATE_TOREPLY; 111 | wf.attr("WFT_STATE_NOREPLY") = (int)WFT_STATE_NOREPLY; 112 | wf.attr("WFT_STATE_SYS_ERROR") = (int)WFT_STATE_SYS_ERROR; 113 | wf.attr("WFT_STATE_SSL_ERROR") = (int)WFT_STATE_SSL_ERROR; 114 | wf.attr("WFT_STATE_DNS_ERROR") = (int)WFT_STATE_DNS_ERROR; 115 | wf.attr("WFT_STATE_TASK_ERROR") = (int)WFT_STATE_TASK_ERROR; 116 | wf.attr("WFT_STATE_ABORTED") = (int)WFT_STATE_ABORTED; 117 | 118 | wf.attr("TOR_NOT_TIMEOUT") = (int)TOR_NOT_TIMEOUT; 119 | wf.attr("TOR_WAIT_TIMEOUT") = (int)TOR_WAIT_TIMEOUT; 120 | wf.attr("TOR_CONNECT_TIMEOUT") = (int)TOR_CONNECT_TIMEOUT; 121 | wf.attr("TOR_TRANSMIT_TIMEOUT") = (int)TOR_TRANSMIT_TIMEOUT; 122 | 123 | py::class_(wf, "EndpointParams") 124 | .def(py::init([]() { return ENDPOINT_PARAMS_DEFAULT; })) 125 | .def("__str__", [](const EndpointParams &self) -> std::string { 126 | std::ostringstream oss; 127 | oss << "EndpointParams {" 128 | << "max_connections: " << self.max_connections 129 | << ", connect_timeout: " << self.connect_timeout 130 | << ", response_timeout: " << self.response_timeout 131 | << ", ssl_connect_timeout: " << self.ssl_connect_timeout 132 | << ", use_tls_sni: " << std::boolalpha << self.use_tls_sni 133 | << "}"; 134 | return oss.str(); 135 | }) 136 | .def_readwrite("max_connections", &EndpointParams::max_connections) 137 | .def_readwrite("connect_timeout", &EndpointParams::connect_timeout) 138 | .def_readwrite("response_timeout", &EndpointParams::response_timeout) 139 | .def_readwrite("ssl_connect_timeout", &EndpointParams::ssl_connect_timeout) 140 | .def_readwrite("use_tls_sni", &EndpointParams::use_tls_sni) 141 | ; 142 | py::class_(wf, "GlobalSettings") 143 | .def(py::init([]() { return PYWF_GLOBAL_SETTINGS_DEFAULT; })) 144 | .def("__str__", [](const WFGlobalSettings &self) -> std::string { 145 | (void)self; 146 | std::ostringstream oss; 147 | oss << "GlobalSettings {EndpointParams {" 148 | << "max_connections: " << self.endpoint_params.max_connections 149 | << ", connect_timeout: " << self.endpoint_params.connect_timeout 150 | << ", response_timeout: " << self.endpoint_params.response_timeout 151 | << ", ssl_connect_timeout: " << self.endpoint_params.ssl_connect_timeout 152 | << ", use_tls_sni: " << std::boolalpha << self.endpoint_params.use_tls_sni 153 | << "}, dns_ttl_default: " << self.dns_ttl_default 154 | << ", dns_ttl_min: " << self.dns_ttl_min 155 | << ", dns_threads: " << self.dns_threads 156 | << ", poller_threads: " << self.poller_threads 157 | << ", handler_threads: " << self.handler_threads 158 | << ", compute_threads: " << self.compute_threads 159 | << "}"; 160 | return oss.str(); 161 | }) 162 | .def_readwrite("endpoint_params", &WFGlobalSettings::endpoint_params) 163 | .def_readwrite("dns_server_params", &WFGlobalSettings::dns_server_params) 164 | .def_readwrite("dns_ttl_default", &WFGlobalSettings::dns_ttl_default) 165 | .def_readwrite("dns_ttl_min", &WFGlobalSettings::dns_ttl_min) 166 | .def_readwrite("dns_threads", &WFGlobalSettings::dns_threads) 167 | .def_readwrite("poller_threads", &WFGlobalSettings::poller_threads) 168 | .def_readwrite("handler_threads", &WFGlobalSettings::handler_threads) 169 | .def_readwrite("compute_threads", &WFGlobalSettings::compute_threads) 170 | ; 171 | py::class_(wf, "WFBase"); 172 | py::class_(wf, "SubTask") 173 | .def("is_null", &PySubTask::is_null) 174 | ; 175 | 176 | py::class_(wf, "ConstSeriesWork") 177 | .def("is_null", &PyConstSeriesWork::is_null) 178 | .def("is_canceled", &PyConstSeriesWork::is_canceled) 179 | .def("get_context", &PyConstSeriesWork::get_context) 180 | ; 181 | py::class_(wf, "ConstParallelWork") 182 | .def("is_null", &PyConstParallelWork::is_null) 183 | .def("series_at", &PyConstParallelWork::series_at) 184 | .def("get_context", &PyConstParallelWork::get_context) 185 | .def("size", &PyConstParallelWork::size) 186 | ; 187 | 188 | py::class_(wf, "SeriesWork") 189 | .def("is_null", &PySeriesWork::is_null) 190 | .def("__lshift__", &PySeriesWork::operator<<) 191 | .def("start", &PySeriesWork::start) 192 | .def("dismiss", &PySeriesWork::dismiss) 193 | .def("push_back", &PySeriesWork::push_back) 194 | .def("push_front", &PySeriesWork::push_front) 195 | .def("cancel", &PySeriesWork::cancel) 196 | .def("is_canceled", &PySeriesWork::is_canceled) 197 | .def("set_callback", &PySeriesWork::set_callback) 198 | .def("set_context", &PySeriesWork::set_context) 199 | .def("get_context", &PySeriesWork::get_context) 200 | ; 201 | py::class_(wf, "ParallelWork") 202 | .def("__mul__", [](PyParallelWork &self, PySeriesWork &t) -> PyParallelWork& { 203 | self.add_series(t); 204 | return self; 205 | }) 206 | .def("is_null", &PyParallelWork::is_null) 207 | .def("start", &PyParallelWork::start) 208 | .def("dismiss", &PyParallelWork::dismiss) 209 | .def("add_series", &PyParallelWork::add_series) 210 | .def("series_at", &PyParallelWork::series_at) 211 | .def("size", &PyParallelWork::size) 212 | .def("set_callback", &PyParallelWork::set_callback) 213 | .def("set_context", &PyParallelWork::set_context) 214 | .def("get_context", &PyParallelWork::get_context) 215 | ; 216 | 217 | wf.def("WORKFLOW_library_init", &PyWorkflow_library_init); 218 | wf.def("get_global_settings", &get_global_settings); 219 | wf.def("series_of", &py_series_of); 220 | 221 | wf.def("create_series_work", 222 | static_cast(&create_series_work), 223 | py::arg("first"), py::arg("callback") 224 | ); 225 | wf.def("create_series_work", 226 | static_cast(&create_series_work), 227 | py::arg("first"), py::arg("last"), py::arg("callback") 228 | ); 229 | wf.def("start_series_work", 230 | static_cast(&start_series_work), 231 | py::arg("first"), py::arg("callback") 232 | ); 233 | wf.def("start_series_work", 234 | static_cast(&start_series_work), 235 | py::arg("first"), py::arg("last"), py::arg("callback") 236 | ); 237 | wf.def("create_parallel_work", 238 | static_cast(&create_parallel_work), 239 | py::arg("callback") 240 | ); 241 | wf.def("create_parallel_work", 242 | static_cast&, py_parallel_callback_t)>(&create_parallel_work), 243 | py::arg("all_series"), py::arg("callback") 244 | ); 245 | wf.def("start_parallel_work", &start_parallel_work, py::arg("all_series"), py::arg("callback")); 246 | wf.def("wait_finish", &CountableSeriesWork::wait_finish, py::call_guard()); 247 | wf.def("wait_finish_timeout", &CountableSeriesWork::wait_finish_timeout, py::call_guard()); 248 | wf.def("get_error_string", &get_error_string, py::arg("state"), py::arg("error")); 249 | wf.def("inner_init", &inner_init); 250 | } 251 | -------------------------------------------------------------------------------- /src/common_types.h: -------------------------------------------------------------------------------- 1 | #ifndef PYWF_COMMON_H 2 | #define PYWF_COMMON_H 3 | #include 4 | #include 5 | #include 6 | #include "workflow/Workflow.h" 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | namespace py = pybind11; 15 | 16 | inline bool has_gil() { 17 | return PyGILState_Check(); 18 | } 19 | 20 | /** 21 | * All call of python function from workflow threads need to acquire 22 | * gil first. It is not allowed to throw exceptions from python 23 | * callback functions, if it does, the program print the error info to 24 | * stderr and exit immediately. 25 | */ 26 | template 27 | void py_callback_wrapper(Callable &&C, Args&& ...args) { 28 | py::gil_scoped_acquire acquire; 29 | try { 30 | if(C) C(std::forward(args)...); 31 | } 32 | catch(py::error_already_set &e) { 33 | std::cerr << e.what() << std::endl; 34 | #ifdef __APPLE__ 35 | std::_Exit(1); 36 | #else 37 | std::quick_exit(1); 38 | #endif 39 | } 40 | } 41 | 42 | /** 43 | * This is used to destruct a std::function object, we need to acquire 44 | * gil because there may be python object captured by std::function. 45 | */ 46 | template 47 | void release_wrapped_function(std::function &f) { 48 | py::gil_scoped_acquire acquire; 49 | if(f) f = nullptr; 50 | } 51 | 52 | template 53 | struct pytype { 54 | using type = void; 55 | }; 56 | 57 | template 58 | using pytype_t = typename pytype::type; 59 | 60 | template 61 | class TaskDeleterWrapper { 62 | public: 63 | TaskDeleterWrapper(Func &&f, Task *t) 64 | : f(std::forward(f)), t(t) { } 65 | TaskDeleterWrapper(const TaskDeleterWrapper&) = delete; 66 | TaskDeleterWrapper& operator=(const TaskDeleterWrapper&) = delete; 67 | Func& get_func() { 68 | return f; 69 | } 70 | void *get_context() { 71 | if(t) return t->user_data; 72 | return nullptr; 73 | } 74 | ~TaskDeleterWrapper() { 75 | py::gil_scoped_acquire acquire; 76 | if(f) f = nullptr; 77 | void *context = this->get_context(); 78 | if(context != nullptr) { 79 | delete static_cast(context); 80 | } 81 | t->user_data = nullptr; 82 | } 83 | private: 84 | Func f; 85 | Task *t{nullptr}; 86 | }; 87 | 88 | /** 89 | * PyWFBase is the Base wrapper class of raw pointers in Workflow. 90 | * All wrapper class has prefix Py, but no need for exported python class names. 91 | * The wrapper does not own the pointer, which is managed by Workflow. 92 | * OriginType is the wrapped type. 93 | */ 94 | class PyWFBase { 95 | public: 96 | using OriginType = void; 97 | PyWFBase() : ptr(nullptr) {} 98 | PyWFBase(void *p) : ptr(p) {} 99 | PyWFBase(const PyWFBase &o) : ptr(o.ptr) {} 100 | PyWFBase& operator=(const PyWFBase &o) { 101 | ptr = o.ptr; 102 | return *this; 103 | } 104 | void* get() const { return ptr; } 105 | bool is_null() const { return ptr == nullptr; } 106 | virtual ~PyWFBase() = default; 107 | protected: 108 | void *ptr; 109 | }; 110 | 111 | class PySubTask : public PyWFBase { 112 | public: 113 | using OriginType = SubTask; 114 | PySubTask() : PyWFBase() {} 115 | PySubTask(OriginType *p) : PyWFBase(p) {} 116 | PySubTask(const PySubTask &o) : PyWFBase(o) {} 117 | OriginType* get() const { return static_cast(ptr); } 118 | }; 119 | 120 | // Derive SeriesWork, release something in destructor 121 | class CountableSeriesWork final : public SeriesWork { 122 | public: 123 | static std::mutex series_mtx; 124 | static size_t series_counter; 125 | static std::condition_variable series_cv; 126 | static void wait_finish() { 127 | std::unique_lock lk(series_mtx); 128 | series_cv.wait(lk, [&](){ return series_counter == 0; }); 129 | } 130 | static bool wait_finish_timeout(double seconds) { 131 | auto dur = std::chrono::duration(seconds); 132 | std::unique_lock lk(series_mtx); 133 | return series_cv.wait_for(lk, dur, [&](){ return series_counter == 0; }); 134 | } 135 | static SeriesWork* create_series_work(SubTask *first, series_callback_t cb) { 136 | return new CountableSeriesWork(first, std::move(cb)); 137 | } 138 | static void start_series_work(SubTask *first, series_callback_t cb) { 139 | new CountableSeriesWork(first, std::move(cb)); 140 | first->dispatch(); 141 | } 142 | static SeriesWork* create_series_work(SubTask *first, SubTask *last, series_callback_t cb) { 143 | SeriesWork *series = new CountableSeriesWork(first, std::move(cb)); 144 | series->set_last_task(last); 145 | return series; 146 | } 147 | static void start_series_work(SubTask *first, SubTask *last, series_callback_t cb) { 148 | SeriesWork *series = new CountableSeriesWork(first, std::move(cb)); 149 | series->set_last_task(last); 150 | first->dispatch(); 151 | } 152 | protected: 153 | CountableSeriesWork(SubTask *first, series_callback_t &&callback) 154 | : SeriesWork(first, std::move(callback)) { 155 | std::unique_lock lk(series_mtx); 156 | ++series_counter; 157 | } 158 | ~CountableSeriesWork() { 159 | { 160 | py::gil_scoped_acquire acquire; 161 | void *context = get_context(); 162 | if(context != nullptr) { 163 | delete static_cast(context); 164 | } 165 | SeriesWork::callback = nullptr; 166 | } 167 | { 168 | std::unique_lock lk(series_mtx); 169 | --series_counter; 170 | if(series_counter == 0) series_cv.notify_all(); 171 | } 172 | } 173 | }; 174 | 175 | // Derive ParallelWork, release something in destructor 176 | class CountableParallelWork final : public ParallelWork { 177 | public: 178 | static ParallelWork* create_parallel_work(parallel_callback_t cb) { 179 | return new CountableParallelWork(std::move(cb)); 180 | } 181 | static ParallelWork* create_parallel_work(SeriesWork *const all_series[], size_t n, 182 | parallel_callback_t cb) { 183 | return new CountableParallelWork(all_series, n, std::move(cb)); 184 | } 185 | static void start_parallel_work(SeriesWork *const all_series[], size_t n, 186 | parallel_callback_t cb) { 187 | CountableParallelWork *p = new CountableParallelWork(all_series, n, std::move(cb)); 188 | CountableSeriesWork::start_series_work(p, nullptr); 189 | } 190 | protected: 191 | CountableParallelWork(parallel_callback_t &&cb) : ParallelWork(std::move(cb)) {} 192 | CountableParallelWork(SeriesWork *const all_series[], size_t n, parallel_callback_t &&cb) 193 | : ParallelWork(all_series, n, std::move(cb)) {} 194 | virtual ~CountableParallelWork() { 195 | { 196 | py::gil_scoped_acquire acquire; 197 | void *context = get_context(); 198 | if(context != nullptr) { 199 | delete static_cast(context); 200 | } 201 | ParallelWork::callback = nullptr; 202 | } 203 | } 204 | }; 205 | 206 | class PyConstSeriesWork : public PyWFBase { 207 | public: 208 | using OriginType = SeriesWork; 209 | PyConstSeriesWork() : PyWFBase() {} 210 | PyConstSeriesWork(OriginType *p) : PyWFBase(p) {} 211 | PyConstSeriesWork(const PyConstSeriesWork &o) : PyWFBase(o) {} 212 | const OriginType* get() const { return static_cast(ptr); } 213 | 214 | bool is_canceled() const { return this->get()->is_canceled(); } 215 | py::object get_context() const { 216 | void *context = this->get()->get_context(); 217 | if(context == nullptr) return py::none(); 218 | return *static_cast(context); 219 | } 220 | }; 221 | 222 | class PyConstParallelWork : public PySubTask { 223 | public: 224 | using OriginType = ParallelWork; 225 | PyConstParallelWork() : PySubTask() {} 226 | PyConstParallelWork(OriginType *p) : PySubTask(p) {} 227 | PyConstParallelWork(const PyConstParallelWork &o) : PySubTask(o) {} 228 | const OriginType* get() const { return static_cast(ptr); } 229 | 230 | PyConstSeriesWork series_at(size_t index) const { 231 | auto p = this->get()->series_at(index); 232 | return PyConstSeriesWork(const_cast(p)); 233 | } 234 | py::object get_context() const { 235 | void *context = this->get()->get_context(); 236 | if(context == nullptr) return py::none(); 237 | return *static_cast(context); 238 | } 239 | size_t size() const { 240 | return this->get()->size(); 241 | } 242 | }; 243 | 244 | class PySeriesWork : public PyWFBase { 245 | public: 246 | using OriginType = SeriesWork; 247 | PySeriesWork() : PyWFBase() {} 248 | PySeriesWork(OriginType *p) : PyWFBase(p) {} 249 | PySeriesWork(const PySeriesWork &o) : PyWFBase(o) {} 250 | OriginType* get() const { return static_cast(ptr); } 251 | PySeriesWork& operator<<(PySubTask &t) { 252 | push_back(t); 253 | return *this; 254 | } 255 | 256 | void start() { this->get()->start(); } 257 | void dismiss() { this->get()->dismiss(); } 258 | void push_back(PySubTask &t) { this->get()->push_back(t.get()); } 259 | void push_front(PySubTask &t) { this->get()->push_front(t.get()); } 260 | void cancel() { this->get()->cancel(); } 261 | bool is_canceled() const { return this->get()->is_canceled(); } 262 | 263 | void set_callback(std::function cb) { 264 | this->get()->set_callback([cb](const SeriesWork *p) mutable { 265 | py_callback_wrapper(cb, PyConstSeriesWork(const_cast(p))); 266 | }); 267 | } 268 | void set_context(py::object obj) { 269 | void *old = this->get()->get_context(); 270 | if(old != nullptr) { 271 | delete static_cast(old); 272 | } 273 | py::object *p = nullptr; 274 | if(obj.is_none() == false) p = new py::object(obj); 275 | this->get()->set_context(static_cast(p)); 276 | } 277 | py::object get_context() const { 278 | void *context = this->get()->get_context(); 279 | if(context == nullptr) return py::none(); 280 | return *static_cast(context); 281 | } 282 | }; 283 | 284 | class PyParallelWork : public PySubTask { 285 | public: 286 | using OriginType = ParallelWork; 287 | PyParallelWork() : PySubTask() {} 288 | PyParallelWork(OriginType *p) : PySubTask(p) {} 289 | PyParallelWork(const PyParallelWork &o) : PySubTask(o) {} 290 | OriginType* get() const { return static_cast(ptr); } 291 | 292 | void start() { 293 | assert(!series_of(this->get())); 294 | CountableSeriesWork::start_series_work(this->get(), nullptr); 295 | } 296 | void dismiss() { this->get()->dismiss(); } 297 | void add_series(PySeriesWork &series) { this->get()->add_series(series.get()); } 298 | 299 | PyConstSeriesWork series_at(size_t index) const { 300 | auto p = this->get()->series_at(index); 301 | return PyConstSeriesWork(const_cast(p)); 302 | } 303 | 304 | void set_context(py::object obj) { 305 | void *old = this->get()->get_context(); 306 | if(old != nullptr) { 307 | delete static_cast(old); 308 | } 309 | py::object *p = nullptr; 310 | if(obj.is_none() == false) p = new py::object(obj); 311 | this->get()->set_context(static_cast(p)); 312 | } 313 | py::object get_context() const { 314 | void *context = this->get()->get_context(); 315 | if(context == nullptr) return py::none(); 316 | return *static_cast(context); 317 | } 318 | size_t size() const { 319 | return this->get()->size(); 320 | } 321 | void set_callback(std::function cb) { 322 | this->get()->set_callback([cb](const ParallelWork *p) { 323 | py_callback_wrapper(cb, PyConstParallelWork(const_cast(p))); 324 | }); 325 | } 326 | }; 327 | 328 | using py_series_callback_t = std::function; 329 | using py_parallel_callback_t = std::function; 330 | 331 | #endif // PYWF_COMMON_H 332 | -------------------------------------------------------------------------------- /src/http_types.cc: -------------------------------------------------------------------------------- 1 | #include "http_types.h" 2 | 3 | void __network_helper::client_prepare(WFHttpTask *p) { 4 | auto resp = p->get_resp(); 5 | const void *data = nullptr; 6 | size_t size = 0; 7 | if(resp->get_parsed_body(&data, &size) && size > 0) { 8 | resp->append_output_body_nocopy(data, size); 9 | } 10 | } 11 | void __network_helper::server_prepare(WFHttpTask *p) { 12 | auto req = p->get_req(); 13 | const void *data = nullptr; 14 | size_t size = 0; 15 | if(req->get_parsed_body(&data, &size) && size > 0) { 16 | req->append_output_body_nocopy(data, size); 17 | } 18 | auto pytask = PyWFHttpTask(p); 19 | pytask.set_callback(nullptr); 20 | } 21 | 22 | PyWFHttpTask create_http_task(const std::string &url, int redirect_max, 23 | int retry_max, py_http_callback_t cb) { 24 | WFHttpTask *ptr = WFTaskFactory::create_http_task(url, redirect_max, 25 | retry_max, nullptr); 26 | PyWFHttpTask t(ptr); 27 | t.set_callback(std::move(cb)); 28 | return t; 29 | } 30 | 31 | PyWFHttpTask create_http_proxy_task(const std::string &url, 32 | const std::string proxy_url, int redirect_max, int retry_max, 33 | py_http_callback_t cb) { 34 | WFHttpTask *ptr = WFTaskFactory::create_http_task(url, proxy_url, 35 | redirect_max, retry_max, nullptr); 36 | PyWFHttpTask t(ptr); 37 | t.set_callback(std::move(cb)); 38 | return t; 39 | } 40 | 41 | void init_http_types(py::module_ &wf) { 42 | py::class_(wf, "HttpTask") 43 | .def("start", &PyWFHttpTask::start) 44 | .def("dismiss", &PyWFHttpTask::dismiss) 45 | .def("noreply", &PyWFHttpTask::noreply) 46 | .def("get_req", &PyWFHttpTask::get_req) 47 | .def("get_resp", &PyWFHttpTask::get_resp) 48 | .def("get_state", &PyWFHttpTask::get_state) 49 | .def("get_error", &PyWFHttpTask::get_error) 50 | .def("get_timeout_reason", &PyWFHttpTask::get_timeout_reason) 51 | .def("get_task_seq", &PyWFHttpTask::get_task_seq) 52 | .def("set_send_timeout", &PyWFHttpTask::set_send_timeout) 53 | .def("set_receive_timeout", &PyWFHttpTask::set_receive_timeout) 54 | .def("set_keep_alive", &PyWFHttpTask::set_keep_alive) 55 | .def("get_peer_addr", &PyWFHttpTask::get_peer_addr) 56 | .def("set_callback", &PyWFHttpTask::set_callback) 57 | .def("set_user_data", &PyWFHttpTask::set_user_data) 58 | .def("get_user_data", &PyWFHttpTask::get_user_data) 59 | ; 60 | py::class_(wf, "HttpMessage"); // Just export a class name 61 | py::class_(wf, "HttpRequest") 62 | .def("move_to", &PyHttpRequest::move_to) 63 | .def("is_chunked", &PyHttpRequest::is_chunked) 64 | .def("is_keep_alive", &PyHttpRequest::is_keep_alive) 65 | .def("get_method", &PyHttpRequest::get_method) 66 | .def("get_request_uri", &PyHttpRequest::get_request_uri) 67 | .def("get_http_version", &PyHttpRequest::get_http_version) 68 | .def("get_headers", &PyHttpRequest::get_headers) 69 | .def("get_body", &PyHttpRequest::get_body) 70 | .def("end_parsing", &PyHttpRequest::end_parsing) 71 | .def("set_method", &PyHttpRequest::set_method) 72 | .def("set_request_uri", &PyHttpRequest::set_request_uri) 73 | .def("set_http_version", &PyHttpRequest::set_http_version) 74 | .def("add_header_pair", &PyHttpRequest::add_header_pair) 75 | .def("set_header_pair", &PyHttpRequest::set_header_pair) 76 | .def("append_body", &PyHttpRequest::append_bytes_body) 77 | .def("append_body", &PyHttpRequest::append_str_body) 78 | .def("clear_body", &PyHttpRequest::clear_output_body) 79 | .def("get_body_size", &PyHttpRequest::get_output_body_size) 80 | .def("set_size_limit", &PyHttpRequest::set_size_limit) 81 | .def("get_size_limit", &PyHttpRequest::get_size_limit) 82 | ; 83 | py::class_(wf, "HttpResponse") 84 | .def("move_to", &PyHttpResponse::move_to) 85 | .def("is_chunked", &PyHttpResponse::is_chunked) 86 | .def("is_keep_alive", &PyHttpResponse::is_keep_alive) 87 | .def("get_status_code", &PyHttpResponse::get_status_code) 88 | .def("get_reason_phrase", &PyHttpResponse::get_reason_phrase) 89 | .def("get_http_version", &PyHttpResponse::get_http_version) 90 | .def("get_headers", &PyHttpResponse::get_headers) 91 | .def("get_body", &PyHttpResponse::get_body) 92 | .def("end_parsing", &PyHttpResponse::end_parsing) 93 | .def("set_status_code", &PyHttpResponse::set_status_code) 94 | .def("set_reason_phrase", &PyHttpResponse::set_reason_phrase) 95 | .def("set_http_version", &PyHttpResponse::set_http_version) 96 | .def("add_header_pair", &PyHttpResponse::add_header_pair) 97 | .def("set_header_pair", &PyHttpResponse::set_header_pair) 98 | .def("append_body", &PyHttpResponse::append_bytes_body) 99 | .def("append_body", &PyHttpResponse::append_str_body) 100 | .def("clear_body", &PyHttpResponse::clear_output_body) 101 | .def("get_body_size", &PyHttpResponse::get_output_body_size) 102 | .def("set_size_limit", &PyHttpResponse::set_size_limit) 103 | .def("get_size_limit", &PyHttpResponse::get_size_limit) 104 | ; 105 | py::class_(wf, "HttpServer") 106 | .def(py::init()) 107 | .def(py::init()) 108 | .def("start", &PyWFHttpServer::start_1, py::arg("port"), py::arg("cert_file") = std::string(), 109 | py::arg("key_file") = std::string()) 110 | .def("start", &PyWFHttpServer::start_2, py::arg("family"), py::arg("host"), py::arg("port"), 111 | py::arg("cert_file") = std::string(), py::arg("key_file") = std::string()) 112 | .def("shutdown", &PyWFHttpServer::shutdown, py::call_guard()) 113 | .def("wait_finish", &PyWFHttpServer::wait_finish, py::call_guard()) 114 | .def("stop", &PyWFHttpServer::stop, py::call_guard()) 115 | ; 116 | wf.def("create_http_task", &create_http_task, py::arg("url"), py::arg("redirect_max"), 117 | py::arg("retry_max"), py::arg("callback")); 118 | wf.def("create_http_task", &create_http_proxy_task, py::arg("url"), py::arg("proxy_url"), 119 | py::arg("redirect_max"), py::arg("retry_max"), py::arg("callback")); 120 | } 121 | -------------------------------------------------------------------------------- /src/http_types.h: -------------------------------------------------------------------------------- 1 | #ifndef PYWF_HTTP_TYPES_H 2 | #define PYWF_HTTP_TYPES_H 3 | 4 | #include "network_types.h" 5 | 6 | static inline std::string __as_string(const char *p) { 7 | std::string s; 8 | if(p) s.assign(p); 9 | return s; 10 | } 11 | 12 | class HttpAttachment final : public protocol::ProtocolMessage::Attachment { 13 | public: 14 | HttpAttachment() : total_size(0) {} 15 | HttpAttachment(const HttpAttachment&) = delete; 16 | ~HttpAttachment() { 17 | { 18 | py::gil_scoped_acquire acquire; 19 | pybytes.clear(); 20 | } 21 | nocopy_body.clear(); 22 | } 23 | 24 | // I suppose the caller has GIL for append, get_body, clear 25 | void append(py::bytes b, const char *p, size_t sz) { 26 | if(sz > 0) { 27 | pybytes.emplace_back(b); 28 | nocopy_body.emplace_back(p, sz); 29 | total_size += sz; 30 | } 31 | } 32 | void append(const char *p, size_t sz) noexcept { 33 | nocopy_body.emplace_back(p, sz); 34 | total_size += sz; 35 | } 36 | 37 | std::string get_body() const { 38 | std::string body; 39 | body.reserve(total_size); 40 | for(const auto &b : nocopy_body) { 41 | body.append(b.first, b.second); 42 | } 43 | return body; 44 | } 45 | 46 | void clear() { 47 | pybytes.clear(); 48 | nocopy_body.clear(); 49 | total_size = 0; 50 | } 51 | private: 52 | std::vector pybytes; 53 | std::vector> nocopy_body; 54 | size_t total_size; 55 | }; 56 | 57 | /** 58 | * This is a common supper class for PyHttpRequest and PyHttpResponse. 59 | * There is no need to export this class to python. 60 | */ 61 | class PyHttpMessage : public PyWFBase { 62 | public: 63 | using OriginType = protocol::HttpMessage; 64 | PyHttpMessage() : PyWFBase() {} 65 | PyHttpMessage(OriginType *p) : PyWFBase(p) {} 66 | PyHttpMessage(const PyHttpMessage &o) : PyWFBase(o) {} 67 | OriginType* get() const { return static_cast(ptr); } 68 | 69 | bool is_chunked() const { return this->get()->is_chunked(); } 70 | bool is_keep_alive() const { return this->get()->is_keep_alive(); } 71 | 72 | bool add_header_pair(const std::string &k, const std::string &v) { 73 | return this->get()->add_header_pair(k.c_str(), v.c_str()); 74 | } 75 | 76 | bool set_header_pair(const std::string &k, const std::string &v) { 77 | return this->get()->set_header_pair(k.c_str(), v.c_str()); 78 | } 79 | 80 | std::string get_http_version() const { 81 | return __as_string(this->get()->get_http_version()); 82 | } 83 | 84 | std::vector> get_headers() const { 85 | std::vector> headers; 86 | protocol::HttpHeaderCursor resp_cursor(this->get()); 87 | std::string name, value; 88 | while(resp_cursor.next(name, value)) { 89 | headers.emplace_back(name, value); 90 | } 91 | return headers; 92 | } 93 | 94 | py::bytes get_body() const { 95 | auto attach = static_cast(this->get()->get_attachment()); 96 | if(attach) { 97 | return attach->get_body(); 98 | } 99 | return protocol::HttpUtil::decode_chunked_body(this->get()); 100 | } 101 | 102 | bool set_http_version(const std::string &s) { return this->get()->set_http_version(s); } 103 | bool append_bytes_body(py::bytes b) { 104 | auto attach = static_cast(this->get()->get_attachment()); 105 | if(attach == nullptr) { // Which means it is the first time to append body 106 | this->get()->clear_output_body(); 107 | attach = new HttpAttachment(); 108 | this->get()->set_attachment(attach); 109 | } 110 | char *buffer = nullptr; 111 | ssize_t length = 0; 112 | if(PYBIND11_BYTES_AS_STRING_AND_SIZE(b.ptr(), &buffer, &length)) { 113 | // there is an error 114 | return false; 115 | } 116 | if(length <= 0) return true; // Nothing todo 117 | if(this->get()->append_output_body((const void*)buffer, (size_t)length)) { 118 | attach->append(b, buffer, length); 119 | return true; 120 | } 121 | return false; 122 | } 123 | 124 | bool append_str_body(py::str s) { 125 | return append_bytes_body((py::bytes)s); 126 | } 127 | 128 | void clear_output_body() { 129 | auto attach = static_cast(this->get()->get_attachment()); 130 | if(attach) attach->clear(); 131 | this->get()->clear_output_body(); 132 | } 133 | 134 | void end_parsing() { this->get()->end_parsing(); } 135 | size_t get_output_body_size() const { return this->get()->get_output_body_size(); } 136 | void set_size_limit(size_t size) { this->get()->set_size_limit(size); } 137 | size_t get_size_limit() const { return this->get()->get_size_limit(); } 138 | 139 | protected: 140 | std::string _get_parsed_body() const { 141 | auto attach = static_cast(this->get()->get_attachment()); 142 | if(attach) { 143 | return attach->get_body(); 144 | } 145 | return protocol::HttpUtil::decode_chunked_body(this->get()); 146 | } 147 | }; 148 | 149 | class PyHttpRequest : public PyHttpMessage { 150 | public: 151 | using OriginType = protocol::HttpRequest; 152 | PyHttpRequest() : PyHttpMessage() {} 153 | PyHttpRequest(OriginType *p) : PyHttpMessage(p) {} 154 | PyHttpRequest(const PyHttpRequest &o) : PyHttpMessage(o) {} 155 | OriginType* get() const { return static_cast(ptr); } 156 | 157 | void move_to(PyHttpRequest &o) { 158 | *(o.get()) = std::move(*(this->get())); 159 | } 160 | 161 | std::string get_method() const { 162 | return __as_string(this->get()->get_method()); 163 | } 164 | 165 | std::string get_request_uri() const { 166 | return __as_string(this->get()->get_request_uri()); 167 | } 168 | 169 | bool set_method(const std::string &s) { return this->get()->set_method(s); } 170 | bool set_request_uri(const std::string &s) { return this->get()->set_request_uri(s); } 171 | private: 172 | }; 173 | 174 | class PyHttpResponse : public PyHttpMessage { 175 | public: 176 | using OriginType = protocol::HttpResponse; 177 | PyHttpResponse() : PyHttpMessage() {} 178 | PyHttpResponse(OriginType *p) : PyHttpMessage(p) {} 179 | PyHttpResponse(const PyHttpResponse &o) : PyHttpMessage(o) {} 180 | OriginType* get() const { return static_cast(ptr); } 181 | 182 | void move_to(PyHttpResponse &o) { 183 | *(o.get()) = std::move(*(this->get())); 184 | } 185 | 186 | std::string get_status_code() const { 187 | return __as_string(this->get()->get_status_code()); 188 | } 189 | 190 | std::string get_reason_phrase() const { 191 | return __as_string(this->get()->get_reason_phrase()); 192 | } 193 | 194 | bool set_status_code(const std::string &status_code) { 195 | return this->get()->set_status_code(status_code); 196 | } 197 | 198 | bool set_reason_phrase(const std::string &phrase) { 199 | return this->get()->set_reason_phrase(phrase); 200 | } 201 | }; 202 | 203 | using PyWFHttpTask = PyWFNetworkTask; 204 | using PyWFHttpServer = PyWFServer; 205 | using py_http_callback_t = std::function; 206 | using py_http_process_t = std::function; 207 | 208 | template<> 209 | struct pytype { 210 | using type = PyWFHttpTask; 211 | }; 212 | template<> 213 | struct pytype { 214 | using type = PyWFHttpServer; 215 | }; 216 | 217 | #endif // PYWF_HTTP_TYPES_H 218 | -------------------------------------------------------------------------------- /src/mysql_types.cc: -------------------------------------------------------------------------------- 1 | #include "mysql_types.h" 2 | using namespace std; 3 | 4 | PyWFMySQLTask create_mysql_task(const std::string &url, int retry_max, py_mysql_callback_t cb) { 5 | WFMySQLTask *ptr = WFTaskFactory::create_mysql_task(url, retry_max, nullptr); 6 | PyWFMySQLTask t(ptr); 7 | t.set_callback(std::move(cb)); 8 | return t; 9 | } 10 | 11 | std::string mysql_datatype2str(int data_type) { 12 | std::string str; 13 | const char *p = datatype2str(data_type); 14 | if(p) str.assign(p); 15 | return p; 16 | } 17 | 18 | void init_mysql_types(py::module_ &wf) { 19 | if(!PyDateTimeAPI) { PyDateTime_IMPORT; } 20 | 21 | // MySQL status constant 22 | wf.attr("MYSQL_STATUS_NOT_INIT") = (int)MYSQL_STATUS_NOT_INIT; 23 | wf.attr("MYSQL_STATUS_OK") = (int)MYSQL_STATUS_OK; 24 | wf.attr("MYSQL_STATUS_GET_RESULT") = (int)MYSQL_STATUS_GET_RESULT; 25 | wf.attr("MYSQL_STATUS_ERROR") = (int)MYSQL_STATUS_ERROR; 26 | wf.attr("MYSQL_STATUS_END") = (int)MYSQL_STATUS_END; 27 | 28 | // MySQL type constant 29 | wf.attr("MYSQL_TYPE_DECIMAL") = (int)MYSQL_TYPE_DECIMAL; 30 | wf.attr("MYSQL_TYPE_TINY") = (int)MYSQL_TYPE_TINY; 31 | wf.attr("MYSQL_TYPE_SHORT") = (int)MYSQL_TYPE_SHORT; 32 | wf.attr("MYSQL_TYPE_LONG") = (int)MYSQL_TYPE_LONG; 33 | wf.attr("MYSQL_TYPE_FLOAT") = (int)MYSQL_TYPE_FLOAT; 34 | wf.attr("MYSQL_TYPE_DOUBLE") = (int)MYSQL_TYPE_DOUBLE; 35 | wf.attr("MYSQL_TYPE_NULL") = (int)MYSQL_TYPE_NULL; 36 | wf.attr("MYSQL_TYPE_TIMESTAMP") = (int)MYSQL_TYPE_TIMESTAMP; 37 | wf.attr("MYSQL_TYPE_LONGLONG") = (int)MYSQL_TYPE_LONGLONG; 38 | wf.attr("MYSQL_TYPE_INT24") = (int)MYSQL_TYPE_INT24; 39 | wf.attr("MYSQL_TYPE_DATE") = (int)MYSQL_TYPE_DATE; 40 | wf.attr("MYSQL_TYPE_TIME") = (int)MYSQL_TYPE_TIME; 41 | wf.attr("MYSQL_TYPE_DATETIME") = (int)MYSQL_TYPE_DATETIME; 42 | wf.attr("MYSQL_TYPE_YEAR") = (int)MYSQL_TYPE_YEAR; 43 | wf.attr("MYSQL_TYPE_NEWDATE") = (int)MYSQL_TYPE_NEWDATE; 44 | wf.attr("MYSQL_TYPE_VARCHAR") = (int)MYSQL_TYPE_VARCHAR; 45 | wf.attr("MYSQL_TYPE_BIT") = (int)MYSQL_TYPE_BIT; 46 | wf.attr("MYSQL_TYPE_TIMESTAMP2") = (int)MYSQL_TYPE_TIMESTAMP2; 47 | wf.attr("MYSQL_TYPE_DATETIME2") = (int)MYSQL_TYPE_DATETIME2; 48 | wf.attr("MYSQL_TYPE_TIME2") = (int)MYSQL_TYPE_TIME2; 49 | wf.attr("MYSQL_TYPE_TYPED_ARRAY") = (int)MYSQL_TYPE_TYPED_ARRAY; 50 | wf.attr("MYSQL_TYPE_JSON") = (int)MYSQL_TYPE_JSON; 51 | wf.attr("MYSQL_TYPE_NEWDECIMAL") = (int)MYSQL_TYPE_NEWDECIMAL; 52 | wf.attr("MYSQL_TYPE_ENUM") = (int)MYSQL_TYPE_ENUM; 53 | wf.attr("MYSQL_TYPE_SET") = (int)MYSQL_TYPE_SET; 54 | wf.attr("MYSQL_TYPE_TINY_BLOB") = (int)MYSQL_TYPE_TINY_BLOB; 55 | wf.attr("MYSQL_TYPE_MEDIUM_BLOB") = (int)MYSQL_TYPE_MEDIUM_BLOB; 56 | wf.attr("MYSQL_TYPE_LONG_BLOB") = (int)MYSQL_TYPE_LONG_BLOB; 57 | wf.attr("MYSQL_TYPE_BLOB") = (int)MYSQL_TYPE_BLOB; 58 | wf.attr("MYSQL_TYPE_VAR_STRING") = (int)MYSQL_TYPE_VAR_STRING; 59 | wf.attr("MYSQL_TYPE_STRING") = (int)MYSQL_TYPE_STRING; 60 | wf.attr("MYSQL_TYPE_GEOMETRY") = (int)MYSQL_TYPE_GEOMETRY; 61 | 62 | // MySQL packet constant 63 | wf.attr("MYSQL_PACKET_OTHER") = (int)MYSQL_PACKET_OTHER; 64 | wf.attr("MYSQL_PACKET_OK") = (int)MYSQL_PACKET_OK; 65 | wf.attr("MYSQL_PACKET_NULL") = (int)MYSQL_PACKET_NULL; 66 | wf.attr("MYSQL_PACKET_EOF") = (int)MYSQL_PACKET_EOF; 67 | wf.attr("MYSQL_PACKET_ERROR") = (int)MYSQL_PACKET_ERROR; 68 | wf.attr("MYSQL_PACKET_GET_RESULT") = (int)MYSQL_PACKET_GET_RESULT; 69 | wf.attr("MYSQL_PACKET_LOCAL_INLINE") = (int)MYSQL_PACKET_LOCAL_INLINE; 70 | 71 | py::class_(wf, "MySQLRequest") 72 | .def("is_null", &PyMySQLRequest::is_null) 73 | .def("move_to", &PyMySQLRequest::move_to) 74 | .def("set_query", &PyMySQLRequest::set_query) 75 | .def("get_query", &PyMySQLRequest::get_query) 76 | .def("query_is_unset", &PyMySQLRequest::query_is_unset) 77 | .def("get_seqid", &PyMySQLRequest::get_seqid) 78 | .def("set_seqid", &PyMySQLRequest::set_seqid) 79 | .def("get_command", &PyMySQLRequest::get_command) 80 | .def("set_size_limit", &PyMySQLRequest::set_size_limit) 81 | .def("get_size_limit", &PyMySQLRequest::get_size_limit) 82 | ; 83 | 84 | py::class_(wf, "MySQLResponse") 85 | .def("is_null", &PyMySQLResponse::is_null) 86 | .def("move_to", &PyMySQLResponse::move_to) 87 | .def("is_ok_packet", &PyMySQLResponse::is_ok_packet) 88 | .def("is_error_packet", &PyMySQLResponse::is_error_packet) 89 | .def("get_packet_type", &PyMySQLResponse::get_packet_type) 90 | .def("get_affected_rows", &PyMySQLResponse::get_affected_rows) 91 | .def("get_last_insert_id", &PyMySQLResponse::get_last_insert_id) 92 | .def("get_warnings", &PyMySQLResponse::get_warnings) 93 | .def("get_error_code", &PyMySQLResponse::get_error_code) 94 | .def("get_error_msg", &PyMySQLResponse::get_error_msg) 95 | .def("get_sql_state", &PyMySQLResponse::get_sql_state) 96 | .def("get_info", &PyMySQLResponse::get_info) 97 | .def("set_ok_packet", &PyMySQLResponse::set_ok_packet) 98 | .def("get_seqid", &PyMySQLResponse::get_seqid) 99 | .def("set_seqid", &PyMySQLResponse::set_seqid) 100 | .def("get_command", &PyMySQLResponse::get_command) 101 | .def("set_size_limit", &PyMySQLResponse::set_size_limit) 102 | .def("get_size_limit", &PyMySQLResponse::get_size_limit) 103 | ; 104 | 105 | py::class_(wf, "MySQLCell") 106 | .def("__str__", &PyMySQLCell::__str__) 107 | .def("get_data_type", &PyMySQLCell::get_data_type) 108 | .def("is_null", &PyMySQLCell::is_null) 109 | .def("is_int", &PyMySQLCell::is_int) 110 | .def("is_string", &PyMySQLCell::is_string) 111 | .def("is_float", &PyMySQLCell::is_float) 112 | .def("is_double", &PyMySQLCell::is_double) 113 | .def("is_ulonglong", &PyMySQLCell::is_ulonglong) 114 | .def("is_date", &PyMySQLCell::is_date) 115 | .def("is_time", &PyMySQLCell::is_time) 116 | .def("is_datetime", &PyMySQLCell::is_datetime) 117 | .def("as_int", &PyMySQLCell::as_int) 118 | .def("as_float", &PyMySQLCell::as_float) 119 | .def("as_double", &PyMySQLCell::as_double) 120 | .def("as_ulonglong", &PyMySQLCell::as_ulonglong) 121 | .def("as_string", &PyMySQLCell::as_string) 122 | .def("as_date", &PyMySQLCell::as_date) 123 | .def("as_time", &PyMySQLCell::as_time) 124 | .def("as_datetime", &PyMySQLCell::as_datetime) 125 | .def("as_bytes", &PyMySQLCell::as_bytes) 126 | .def("as_object", &PyMySQLCell::as_object) 127 | ; 128 | 129 | py::class_(wf, "MySQLField") 130 | .def("__str__", &PyMySQLField::__str__) 131 | .def("__repr__", &PyMySQLField::__repr__) 132 | .def("get_name", &PyMySQLField::get_name) 133 | .def("get_org_name", &PyMySQLField::get_org_name) 134 | .def("get_table", &PyMySQLField::get_table) 135 | .def("get_org_table", &PyMySQLField::get_org_table) 136 | .def("get_db", &PyMySQLField::get_db) 137 | .def("get_catalog", &PyMySQLField::get_catalog) 138 | .def("get_def", &PyMySQLField::get_def) 139 | .def("get_charsetnr", &PyMySQLField::get_charsetnr) 140 | .def("get_length", &PyMySQLField::get_length) 141 | .def("get_flags", &PyMySQLField::get_flags) 142 | .def("get_decimals", &PyMySQLField::get_decimals) 143 | .def("get_data_type", &PyMySQLField::get_data_type) 144 | ; 145 | 146 | py::class_(wf, "MySQLResultCursor") 147 | .def(py::init()) 148 | .def("next_result_set", &PyMySQLResultCursor::next_result_set) 149 | .def("first_result_set", &PyMySQLResultCursor::first_result_set) 150 | .def("fetch_fields", &PyMySQLResultCursor::fetch_fields) 151 | .def("fetch_row", &PyMySQLResultCursor::fetch_row) 152 | .def("fetch_all", &PyMySQLResultCursor::fetch_all) 153 | .def("get_cursor_status", &PyMySQLResultCursor::get_cursor_status) 154 | .def("get_server_status", &PyMySQLResultCursor::get_server_status) 155 | .def("get_field_count", &PyMySQLResultCursor::get_field_count) 156 | .def("get_rows_count", &PyMySQLResultCursor::get_rows_count) 157 | .def("get_affected_rows", &PyMySQLResultCursor::get_affected_rows) 158 | .def("get_insert_id", &PyMySQLResultCursor::get_insert_id) 159 | .def("get_warnings", &PyMySQLResultCursor::get_warnings) 160 | .def("get_info", &PyMySQLResultCursor::get_info) 161 | .def("rewind", &PyMySQLResultCursor::rewind) 162 | ; 163 | 164 | py::class_(wf, "MySQLTask") 165 | .def("is_null", &PyWFMySQLTask::is_null) 166 | .def("start", &PyWFMySQLTask::start) 167 | .def("dismiss", &PyWFMySQLTask::dismiss) 168 | .def("noreply", &PyWFMySQLTask::noreply) 169 | .def("get_req", &PyWFMySQLTask::get_req) 170 | .def("get_resp", &PyWFMySQLTask::get_resp) 171 | .def("get_state", &PyWFMySQLTask::get_state) 172 | .def("get_error", &PyWFMySQLTask::get_error) 173 | .def("get_timeout_reason", &PyWFMySQLTask::get_timeout_reason) 174 | .def("get_task_seq", &PyWFMySQLTask::get_task_seq) 175 | .def("set_send_timeout", &PyWFMySQLTask::set_send_timeout) 176 | .def("set_receive_timeout", &PyWFMySQLTask::set_receive_timeout) 177 | .def("set_keep_alive", &PyWFMySQLTask::set_keep_alive) 178 | .def("get_peer_addr", &PyWFMySQLTask::get_peer_addr) 179 | .def("set_callback", &PyWFMySQLTask::set_callback) 180 | .def("set_user_data", &PyWFMySQLTask::set_user_data) 181 | .def("get_user_data", &PyWFMySQLTask::get_user_data) 182 | ; 183 | 184 | py::class_(wf, "MySQLConnection") 185 | .def(py::init()) 186 | .def("init", &PyWFMySQLConnection::init) 187 | .def("deinit", &PyWFMySQLConnection::deinit) 188 | .def("create_query_task", &PyWFMySQLConnection::create_query_task) 189 | .def("create_disconnect_task", &PyWFMySQLConnection::create_disconnect_task) 190 | ; 191 | 192 | py::class_(wf, "MySQLServer") 193 | .def(py::init()) 194 | .def(py::init()) 195 | .def("start", &PyWFMySQLServer::start_1, py::arg("port"), py::arg("cert_file") = std::string(), 196 | py::arg("key_file") = std::string()) 197 | .def("start", &PyWFMySQLServer::start_2, py::arg("family"), py::arg("host"), py::arg("port"), 198 | py::arg("cert_file") = std::string(), py::arg("key_file") = std::string()) 199 | .def("shutdown", &PyWFMySQLServer::shutdown, py::call_guard()) 200 | .def("wait_finish", &PyWFMySQLServer::wait_finish, py::call_guard()) 201 | .def("stop", &PyWFMySQLServer::stop, py::call_guard()) 202 | ; 203 | 204 | wf.def("mysql_datatype2str", &mysql_datatype2str, py::arg("datatype")); 205 | 206 | wf.def("create_mysql_task", &create_mysql_task, py::arg("url"), py::arg("retry_max"), 207 | py::arg("callback")); 208 | } 209 | -------------------------------------------------------------------------------- /src/mysql_types.h: -------------------------------------------------------------------------------- 1 | #ifndef PYWF_MYSQL_TYPES_H 2 | #define PYWF_MYSQL_TYPES_H 3 | 4 | #include "network_types.h" 5 | #include "workflow/MySQLMessage.h" 6 | #include "workflow/MySQLResult.h" 7 | #include "workflow/WFMySQLConnection.h" 8 | #include 9 | 10 | class PyMySQLCell { 11 | public: 12 | using OriginType = protocol::MySQLCell; 13 | using ulonglong = unsigned long long; 14 | PyMySQLCell() {} 15 | PyMySQLCell(OriginType &&c) : cell(std::move(c)) {} 16 | 17 | int get_data_type() { return cell.get_data_type(); } 18 | bool is_null() { return cell.is_null(); } 19 | bool is_int() { return cell.is_int(); } 20 | bool is_string() { return cell.is_string(); } 21 | bool is_float() { return cell.is_float(); } 22 | bool is_double() { return cell.is_double(); } 23 | bool is_ulonglong() { return cell.is_ulonglong(); } 24 | bool is_date() { return cell.is_date(); } 25 | bool is_time() { return cell.is_time(); } 26 | bool is_datetime() { return cell.is_datetime(); } 27 | 28 | int as_int() { return cell.as_int(); } 29 | float as_float() { return cell.as_float(); } 30 | double as_double() { return cell.as_double(); } 31 | ulonglong as_ulonglong() { return cell.as_ulonglong(); } 32 | 33 | py::bytes as_string() { 34 | if(is_string() || is_time() || is_date() || is_datetime()) { 35 | return as_bytes(); 36 | } 37 | return py::bytes(); 38 | } 39 | 40 | py::object to_datetime() { 41 | std::string str = cell.as_string(); 42 | int year, month, day, hour, min, sec, usec, ret; 43 | if(is_date()) { 44 | if(str.size() != 10) return py::none(); 45 | ret = sscanf(str.c_str(), "%4d-%2d-%2d", &year, &month, &day); 46 | if(ret != 3) return py::none(); 47 | return py::reinterpret_steal(PyDate_FromDate(year, month, day)); 48 | } 49 | else if(is_time()) { 50 | if(str.size() < 8) return py::none(); 51 | usec = 0; 52 | ret = sscanf(str.c_str(), "%d:%2d:%2d.%d", &hour, &min, &sec, &usec); 53 | if(ret != 3 && ret != 4) return py::none(); 54 | return py::reinterpret_steal( 55 | PyDelta_FromDSU(0, hour * 3600 + min * 60 + sec, usec) 56 | ); 57 | } 58 | else if(is_datetime()) { 59 | if(str.size() < 19) return py::none(); 60 | usec = 0; 61 | ret = sscanf(str.c_str(), "%4d-%2d-%2d %2d:%2d:%2d.%d", 62 | &year, &month, &day, &hour, &min, &sec, &usec); 63 | if(ret != 6 && ret != 7) return py::none(); 64 | return py::reinterpret_steal( 65 | PyDateTime_FromDateAndTime(year, month, day, hour, min, sec, usec) 66 | ); 67 | } 68 | return py::none(); 69 | } 70 | py::object as_date() { return to_datetime(); } 71 | py::object as_time() { return to_datetime(); } 72 | py::object as_datetime() { return to_datetime(); } 73 | 74 | py::bytes as_bytes() { 75 | const void *data; 76 | size_t len; 77 | int data_type; 78 | cell.get_cell_nocopy(&data, &len, &data_type); 79 | return py::bytes(static_cast(data), len); 80 | } 81 | 82 | py::object as_object() { 83 | switch (get_data_type()) { 84 | case MYSQL_TYPE_NULL: 85 | return py::none(); 86 | case MYSQL_TYPE_TINY: 87 | case MYSQL_TYPE_SHORT: 88 | case MYSQL_TYPE_INT24: 89 | case MYSQL_TYPE_LONG: 90 | return py::int_(as_int()); 91 | case MYSQL_TYPE_LONGLONG: 92 | return py::int_(as_ulonglong()); 93 | case MYSQL_TYPE_FLOAT: 94 | return py::float_(as_float()); 95 | case MYSQL_TYPE_DOUBLE: 96 | return py::float_(as_double()); 97 | case MYSQL_TYPE_DATE: 98 | return as_date(); 99 | case MYSQL_TYPE_TIME: 100 | return as_time(); 101 | case MYSQL_TYPE_DATETIME: 102 | case MYSQL_TYPE_TIMESTAMP: 103 | return as_datetime(); 104 | case MYSQL_TYPE_DECIMAL: 105 | case MYSQL_TYPE_STRING: 106 | case MYSQL_TYPE_VARCHAR: 107 | case MYSQL_TYPE_VAR_STRING: 108 | case MYSQL_TYPE_JSON: 109 | return as_string(); 110 | default: 111 | return as_bytes(); 112 | } 113 | } 114 | 115 | py::str __str__() { 116 | py::object obj = as_object(); 117 | py::str result; 118 | bool str_is_set = false; 119 | if(py::bytes::check_(obj)) { 120 | try { 121 | py::str s(static_cast(obj)); 122 | result = s; 123 | str_is_set = true; 124 | } 125 | // this py::bytes cannot convert to py::str 126 | // but we can also try bytes' __str__ 127 | catch(const std::runtime_error&) { 128 | PyErr_Clear(); 129 | } 130 | } 131 | if(!str_is_set) { 132 | try { 133 | result = py::str(obj.ptr()); 134 | str_is_set = true; 135 | } 136 | catch(const std::runtime_error&) { 137 | PyErr_Clear(); 138 | } 139 | } 140 | if(!str_is_set) result = py::str("Exception"); 141 | return result; 142 | } 143 | private: 144 | OriginType cell; 145 | }; 146 | 147 | class PyMySQLField : public PyWFBase { 148 | public: 149 | using OriginType = protocol::MySQLField; 150 | PyMySQLField() : PyWFBase() {} 151 | PyMySQLField(OriginType *p) : PyWFBase(p) {} 152 | PyMySQLField(const PyMySQLField &o) : PyWFBase(o) {} 153 | OriginType* get() const { return static_cast(ptr); } 154 | 155 | py::str as_str(const std::string &s) { 156 | try { 157 | return py::str(s); 158 | } 159 | catch(const std::runtime_error&) { 160 | PyErr_Clear(); 161 | } 162 | try { 163 | py::bytes b(s); 164 | return py::str(b.ptr()); 165 | } 166 | catch(const std::runtime_error&) { 167 | PyErr_Clear(); 168 | } 169 | return py::str(); 170 | } 171 | py::str __str__() { 172 | return as_str(this->get()->get_name()); 173 | } 174 | py::str __repr__() { 175 | return py::str("pywf.MySQLField of ") + __str__(); 176 | } 177 | 178 | py::bytes get_name() { return this->get()->get_name(); } 179 | py::bytes get_org_name() { return this->get()->get_org_name(); } 180 | py::bytes get_table() { return this->get()->get_table(); } 181 | py::bytes get_org_table() { return this->get()->get_org_table(); } 182 | py::bytes get_db() { return this->get()->get_db(); } 183 | py::bytes get_catalog() { return this->get()->get_catalog(); } 184 | py::bytes get_def() { return this->get()->get_def(); } 185 | 186 | int get_charsetnr() { return this->get()->get_charsetnr(); } 187 | int get_length() { return this->get()->get_length(); } 188 | int get_flags() { return this->get()->get_flags(); } 189 | int get_decimals() { return this->get()->get_decimals(); } 190 | int get_data_type() { return this->get()->get_data_type(); } 191 | }; 192 | 193 | class PyMySQLRequest : public PyWFBase { 194 | public: 195 | using OriginType = protocol::MySQLRequest; 196 | PyMySQLRequest() : PyWFBase() {} 197 | PyMySQLRequest(OriginType *p) : PyWFBase(p) {} 198 | PyMySQLRequest(const PyMySQLRequest &o) : PyWFBase(o) {} 199 | OriginType* get() const { return static_cast(ptr); } 200 | 201 | void move_to(PyMySQLRequest &o) { 202 | *(o.get()) = std::move(*(this->get())); 203 | } 204 | 205 | void set_query(const std::string &query) { this->get()->set_query(query); } 206 | py::bytes get_query() const { return this->get()->get_query(); } 207 | bool query_is_unset() const { return this->get()->query_is_unset(); } 208 | 209 | int get_seqid() const { return this->get()->get_seqid(); } 210 | void set_seqid(int id) { this->get()->set_seqid(id); } 211 | int get_command() const { return this->get()->get_command(); } 212 | 213 | void set_size_limit(size_t limit) { this->get()->set_size_limit(limit); } 214 | size_t get_size_limit() const { return this->get()->get_size_limit(); } 215 | }; 216 | 217 | class PyMySQLResponse : public PyWFBase { 218 | public: 219 | using OriginType = protocol::MySQLResponse; 220 | PyMySQLResponse() : PyWFBase() {} 221 | PyMySQLResponse(OriginType *p) : PyWFBase(p) {} 222 | PyMySQLResponse(const PyMySQLResponse &o) : PyWFBase(o) {} 223 | OriginType* get() const { return static_cast(ptr); } 224 | 225 | void move_to(PyMySQLResponse &o) { 226 | *(o.get()) = std::move(*(this->get())); 227 | } 228 | 229 | bool is_ok_packet() const { return this->get()->is_ok_packet(); } 230 | bool is_error_packet() const { return this->get()->is_error_packet(); } 231 | int get_packet_type() const { return this->get()->get_packet_type(); } 232 | unsigned long long get_affected_rows() const { 233 | return this->get()->get_affected_rows(); 234 | } 235 | unsigned long long get_last_insert_id() const { 236 | return this->get()->get_last_insert_id(); 237 | } 238 | 239 | int get_warnings() const { return this->get()->get_warnings(); } 240 | int get_error_code() const { return this->get()->get_error_code(); } 241 | py::bytes get_error_msg() const { return this->get()-> get_error_msg(); } 242 | py::bytes get_sql_state() const { return this->get()->get_sql_state(); } 243 | py::bytes get_info() const { return this->get()->get_info(); } 244 | void set_ok_packet() { return this->get()->set_ok_packet(); } 245 | 246 | int get_seqid() const { return this->get()->get_seqid(); } 247 | void set_seqid(int id) { this->get()->set_seqid(id); } 248 | int get_command() const { return this->get()->get_command(); } 249 | 250 | void set_size_limit(size_t limit) { this->get()->set_size_limit(limit); } 251 | size_t get_size_limit() const { return this->get()->get_size_limit(); } 252 | }; 253 | 254 | class PyMySQLResultCursor { 255 | public: 256 | using OriginType = protocol::MySQLResultCursor; 257 | PyMySQLResultCursor() {} 258 | PyMySQLResultCursor(OriginType &&o) : csr(std::move(o)) {} 259 | PyMySQLResultCursor(PyMySQLResponse &resp) : csr(resp.get()) {} 260 | 261 | bool next_result_set() { return csr.next_result_set(); } 262 | void first_result_set() { return csr.first_result_set(); } 263 | 264 | std::vector fetch_fields() { 265 | const protocol::MySQLField * const * fields = csr.fetch_fields(); 266 | std::vector fields_vec; 267 | fields_vec.reserve(csr.get_field_count()); 268 | for(int i = 0; i < csr.get_field_count(); i++) 269 | fields_vec.push_back(PyMySQLField(const_cast(fields[i]))); 270 | return fields_vec; 271 | } 272 | 273 | py::object fetch_row() { 274 | py::list lst; 275 | std::vector cells; 276 | if(csr.fetch_row(cells)) { 277 | for(auto &cell : cells) 278 | lst.append(PyMySQLCell(std::move(cell))); 279 | return static_cast(lst); 280 | } 281 | else { 282 | return py::none(); 283 | } 284 | } 285 | py::list fetch_all() { 286 | py::list all; 287 | while(true) { 288 | py::object row; 289 | row = this->fetch_row(); 290 | if(row.is_none()) break; 291 | all.append(row); 292 | } 293 | return all; 294 | } 295 | 296 | int get_cursor_status() const { return csr.get_cursor_status(); } 297 | int get_server_status() const { return csr.get_server_status(); } 298 | 299 | using ULL = unsigned long long; 300 | int get_field_count() const { return csr.get_field_count(); } 301 | int get_rows_count() const { return csr.get_rows_count(); } 302 | ULL get_affected_rows() const { return csr.get_affected_rows(); } 303 | ULL get_insert_id() const { return csr.get_insert_id(); } 304 | int get_warnings() const { return csr.get_warnings(); } 305 | py::bytes get_info() const { return csr.get_info(); } 306 | 307 | void rewind() { csr.rewind(); } 308 | 309 | private: 310 | OriginType csr; 311 | }; 312 | 313 | using PyWFMySQLTask = PyWFNetworkTask; 314 | using PyWFMySQLServer = PyWFServer; 315 | using py_mysql_callback_t = std::function; 316 | using py_mysql_process_t = std::function; 317 | 318 | class PyWFMySQLConnection { 319 | public: 320 | using OriginType = WFMySQLConnection; 321 | PyWFMySQLConnection(int id) : conn(id) {} 322 | 323 | int init(const std::string &url) { return conn.init(url); } 324 | void deinit() { conn.deinit(); } 325 | 326 | PyWFMySQLTask create_query_task(const std::string &query, py_mysql_callback_t cb) { 327 | WFMySQLTask *ptr = conn.create_query_task(query, nullptr); 328 | PyWFMySQLTask t(ptr); 329 | t.set_callback(std::move(cb)); 330 | return t; 331 | } 332 | PyWFMySQLTask create_disconnect_task(py_mysql_callback_t cb) { 333 | WFMySQLTask *ptr = conn.create_disconnect_task(nullptr); 334 | PyWFMySQLTask t(ptr); 335 | t.set_callback(std::move(cb)); 336 | return t; 337 | } 338 | private: 339 | OriginType conn; 340 | }; 341 | 342 | #endif // PYWF_MYSQL_TYPES_H 343 | -------------------------------------------------------------------------------- /src/network_types.cc: -------------------------------------------------------------------------------- 1 | #include "network_types.h" 2 | 3 | void init_http_types(py::module_&); 4 | void init_redis_types(py::module_&); 5 | void init_mysql_types(py::module_&); 6 | 7 | void init_network_types(py::module_ &wf) { 8 | py::class_(wf, "ServerParams") 9 | .def(py::init([](){ return SERVER_PARAMS_DEFAULT; })) 10 | .def_readwrite("max_connections", &WFServerParams::max_connections) 11 | .def_readwrite("peer_response_timeout", &WFServerParams::peer_response_timeout) 12 | .def_readwrite("receive_timeout", &WFServerParams::receive_timeout) 13 | .def_readwrite("keep_alive_timeout", &WFServerParams::keep_alive_timeout) 14 | .def_readwrite("request_size_limit", &WFServerParams::request_size_limit) 15 | .def_readwrite("ssl_accept_timeout", &WFServerParams::ssl_accept_timeout) 16 | ; 17 | 18 | init_http_types(wf); 19 | init_redis_types(wf); 20 | init_mysql_types(wf); 21 | } 22 | -------------------------------------------------------------------------------- /src/network_types.h: -------------------------------------------------------------------------------- 1 | #ifndef PYWF_NETWORK_TYPES_H 2 | #define PYWF_NETWORK_TYPES_H 3 | #include 4 | 5 | #include "common_types.h" 6 | #include "workflow/HttpMessage.h" 7 | #include "workflow/HttpUtil.h" 8 | #include "workflow/WFHttpServer.h" 9 | 10 | class __network_helper { 11 | public: 12 | template 13 | static void client_prepare(Task*) {} 14 | static void client_prepare(WFHttpTask*); 15 | 16 | template 17 | static void server_prepare(Task*) {} 18 | static void server_prepare(WFHttpTask*); 19 | }; 20 | 21 | template 22 | class PyWFNetworkTask : public PySubTask { 23 | public: 24 | using ReqType = Req; 25 | using RespType = Resp; 26 | using OriginType = WFNetworkTask; 27 | using _py_callback_t = std::function)>; 28 | using _deleter_t = TaskDeleterWrapper<_py_callback_t, OriginType>; 29 | 30 | PyWFNetworkTask() : PySubTask() {} 31 | PyWFNetworkTask(OriginType *p) : PySubTask(p) {} 32 | PyWFNetworkTask(const PyWFNetworkTask &o) : PySubTask(o) {} 33 | OriginType* get() const { return static_cast(ptr); } 34 | 35 | void start() { 36 | assert(!series_of(this->get())); 37 | CountableSeriesWork::start_series_work(this->get(), nullptr); 38 | } 39 | 40 | void dismiss() { this->get()->dismiss(); } 41 | void noreply() { this->get()->noreply(); } 42 | ReqType get_req() { return ReqType(this->get()->get_req()); } 43 | RespType get_resp() { return RespType(this->get()->get_resp()); } 44 | int get_state() const { return this->get()->get_state(); } 45 | int get_error() const { return this->get()->get_error(); } 46 | int get_timeout_reason() const { return this->get()->get_timeout_reason(); } 47 | long long get_task_seq() const { return this->get()->get_task_seq(); } 48 | void set_send_timeout(int t) { this->get()->set_send_timeout(t); } 49 | void set_receive_timeout(int t) { this->get()->set_receive_timeout(t); } 50 | void set_keep_alive(int t) { this->get()->set_keep_alive(t); } 51 | 52 | py::object get_peer_addr() const { 53 | char ip_str[INET6_ADDRSTRLEN + 1] = { 0 }; 54 | struct sockaddr_storage addr; 55 | socklen_t addrlen = sizeof (addr); 56 | uint16_t port = 0; 57 | 58 | if (this->get()->get_peer_addr((struct sockaddr *)&addr, &addrlen) == 0) { 59 | if (addr.ss_family == AF_INET) { 60 | struct sockaddr_in *sin = (struct sockaddr_in *)(&addr); 61 | inet_ntop(AF_INET, &sin->sin_addr, ip_str, addrlen); 62 | port = ntohs(sin->sin_port); 63 | } 64 | else if (addr.ss_family == AF_INET6) { 65 | struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)(&addr); 66 | inet_ntop(AF_INET6, &sin6->sin6_addr, ip_str, addrlen); 67 | port = ntohs(sin6->sin6_port); 68 | } 69 | } 70 | return py::make_tuple(py::str(ip_str), py::int_(port)); 71 | } 72 | 73 | void set_callback(_py_callback_t cb) { 74 | // The deleter will destruct both cb and user_data, 75 | // but now we just want to reset cb, so we must clear user_data first, 76 | // and set user_data at the end 77 | void *user_data = this->get()->user_data; 78 | this->get()->user_data = nullptr; 79 | auto deleter = std::make_shared<_deleter_t>(std::move(cb), this->get()); 80 | this->get()->set_callback([deleter](OriginType *p) { 81 | __network_helper::client_prepare(p); 82 | py_callback_wrapper(deleter->get_func(), PyWFNetworkTask(p)); 83 | }); 84 | this->get()->user_data = user_data; 85 | } 86 | 87 | void set_user_data(py::object obj) { 88 | void *old = this->get()->user_data; 89 | if(old != nullptr) { 90 | delete static_cast(old); 91 | } 92 | py::object *p = nullptr; 93 | if(obj.is_none() == false) p = new py::object(obj); 94 | this->get()->user_data = static_cast(p); 95 | } 96 | 97 | py::object get_user_data() const { 98 | void *context = this->get()->user_data; 99 | if(context == nullptr) return py::none(); 100 | return *static_cast(context); 101 | } 102 | }; 103 | 104 | template 105 | class PyWFServer { 106 | public: 107 | using ReqType = Req; 108 | using RespType = Resp; 109 | using OriginType = WFServer; 110 | using _py_process_t = std::function)>; 111 | using _task_t = WFNetworkTask; 112 | using _pytask_t = PyWFNetworkTask; 113 | 114 | PyWFServer(_py_process_t proc) 115 | : process(std::move(proc)), server([this](_task_t *p) { 116 | __network_helper::server_prepare(p); 117 | py_callback_wrapper(this->process, _pytask_t(p)); 118 | }) {} 119 | PyWFServer(WFServerParams params, _py_process_t proc) 120 | : process(std::move(proc)), server(¶ms, [this](_task_t *p) { 121 | __network_helper::server_prepare(p); 122 | py_callback_wrapper(this->process, _pytask_t(p)); 123 | }) {} 124 | 125 | int start_0(unsigned short port) { 126 | return server.start(port); 127 | } 128 | 129 | int start_1(unsigned short port, const std::string &cert_file, const std::string &key_file) { 130 | if(cert_file.empty() || key_file.empty()) { 131 | return server.start(port); 132 | } 133 | return server.start(port, cert_file.c_str(), key_file.c_str()); 134 | } 135 | 136 | int start_2(int family, const std::string &host, unsigned short port, 137 | const std::string &cert_file, const std::string &key_file) { 138 | if(cert_file.empty() || key_file.empty()) { 139 | return server.start(family, host.c_str(), port, nullptr, nullptr); 140 | } 141 | return server.start(family, host.c_str(), port, cert_file.c_str(), key_file.c_str()); 142 | } 143 | 144 | void shutdown() { server.shutdown(); } 145 | void wait_finish() { server.wait_finish(); } 146 | void stop() { server.stop(); } 147 | ~PyWFServer() { release_wrapped_function(this->process); } 148 | 149 | _py_process_t process; 150 | private: 151 | OriginType server; 152 | }; 153 | 154 | #endif // PYWF_NETWORK_TYPES_H 155 | -------------------------------------------------------------------------------- /src/other_types.cc: -------------------------------------------------------------------------------- 1 | #include "other_types.h" 2 | 3 | class GoTaskWrapper { 4 | struct Params { 5 | Params(const py::function &f, const py::args &a, const py::kwargs &kw) 6 | : f(f), a(a), kw(kw) {} 7 | ~Params() = default; 8 | py::function f; 9 | py::args a; 10 | py::kwargs kw; 11 | }; 12 | public: 13 | GoTaskWrapper(const py::function &f, const py::args &a, const py::kwargs &kw) 14 | : params(new Params(f, a, kw)) {} 15 | GoTaskWrapper(const GoTaskWrapper&) = delete; 16 | GoTaskWrapper& operator=(const GoTaskWrapper&) = delete; 17 | void go() { 18 | py::gil_scoped_acquire acquire; 19 | (params->f)(*(params->a), **(params->kw)); 20 | } 21 | ~GoTaskWrapper() { 22 | py::gil_scoped_acquire acquire; 23 | delete params; 24 | } 25 | private: 26 | Params *params; 27 | }; 28 | 29 | PyWFFileIOTask create_pread_task(int fd, size_t count, off_t offset, py_fio_callback_t cb) { 30 | void *buf = malloc(count); 31 | FileIOTaskData *data = new FileIOTaskData(buf, nullptr); 32 | auto ptr = WFTaskFactory::create_pread_task(fd, buf, count, offset, nullptr); 33 | ptr->user_data = data; 34 | PyWFFileIOTask t(ptr); 35 | t.set_callback(std::move(cb)); 36 | return t; 37 | } 38 | 39 | PyWFFileIOTask create_pwrite_task(int fd, const py::bytes &b, size_t count, off_t offset, 40 | py_fio_callback_t cb) { 41 | char *buffer; 42 | ssize_t length; 43 | if(PYBIND11_BYTES_AS_STRING_AND_SIZE(b.ptr(), &buffer, &length)) { 44 | // there is an error 45 | return nullptr; 46 | } 47 | py::bytes *bytes = new py::bytes(b); 48 | size_t write_size = std::min(count, (size_t)length); 49 | FileIOTaskData *data = new FileIOTaskData(nullptr, bytes); 50 | auto ptr = WFTaskFactory::create_pwrite_task(fd, buffer, write_size, offset, nullptr); 51 | ptr->user_data = data; 52 | PyWFFileIOTask t(ptr); 53 | t.set_callback(std::move(cb)); 54 | return t; 55 | } 56 | 57 | PyWFFileVIOTask create_pwritev_task(int fd, const std::vector &b, off_t offset, 58 | py_fvio_callback_t cb) { 59 | size_t size = b.size(); 60 | struct iovec *iov = new iovec[size]; 61 | py::object *bytes = new py::object[size]; 62 | bool failed = false; 63 | for(size_t i = 0; i < size; i++) { 64 | char *buffer; 65 | ssize_t length; 66 | if(PYBIND11_BYTES_AS_STRING_AND_SIZE(b[i].ptr(), &buffer, &length)) { 67 | failed = true; 68 | break; 69 | } 70 | iov[i].iov_base = buffer; 71 | iov[i].iov_len = (size_t)length; 72 | bytes[i] = b[i]; 73 | } 74 | if(failed) { 75 | delete[] iov; 76 | delete[] bytes; 77 | return nullptr; 78 | } 79 | FileVIOTaskData *data = new FileVIOTaskData(iov, false, bytes, size); 80 | auto ptr = WFTaskFactory::create_pwritev_task(fd, iov, (int)size, offset, nullptr); 81 | ptr->user_data = data; 82 | PyWFFileVIOTask t(ptr); 83 | t.set_callback(std::move(cb)); 84 | return t; 85 | } 86 | 87 | PyWFFileSyncTask create_fsync_task(int fd, py_fsync_callback_t cb) { 88 | FileTaskData *data = new FileTaskData(); 89 | auto ptr = WFTaskFactory::create_fsync_task(fd, nullptr); 90 | ptr->user_data = data; 91 | PyWFFileSyncTask t(ptr); 92 | t.set_callback(std::move(cb)); 93 | return t; 94 | } 95 | 96 | PyWFFileSyncTask create_fdsync_task(int fd, py_fsync_callback_t cb) { 97 | FileTaskData *data = new FileTaskData(); 98 | auto ptr = WFTaskFactory::create_fdsync_task(fd, nullptr); 99 | ptr->user_data = data; 100 | PyWFFileSyncTask t(ptr); 101 | t.set_callback(std::move(cb)); 102 | return t; 103 | } 104 | 105 | PyWFTimerTask create_timer_task(unsigned int microseconds, py_timer_callback_t cb) { 106 | auto ptr = WFTaskFactory::create_timer_task(microseconds, nullptr); 107 | PyWFTimerTask task(ptr); 108 | task.set_callback(std::move(cb)); 109 | return task; 110 | } 111 | 112 | PyWFCounterTask create_counter_task(const std::string &name, unsigned int target, py_counter_callback_t cb) { 113 | auto ptr = WFTaskFactory::create_counter_task(name, target, nullptr); 114 | PyWFCounterTask task(ptr); 115 | task.set_callback(std::move(cb)); 116 | return task; 117 | } 118 | 119 | PyWFCounterTask create_counter_task_no_name(unsigned int target, py_counter_callback_t cb) { 120 | auto ptr = WFTaskFactory::create_counter_task(target, nullptr); 121 | PyWFCounterTask task(ptr); 122 | task.set_callback(std::move(cb)); 123 | return task; 124 | } 125 | 126 | void count_by_name(const std::string &name, unsigned int n) { 127 | WFTaskFactory::count_by_name(name, n); 128 | } 129 | 130 | PyWFGoTask create_go_task_with_name(const std::string &name, py::function f, py::args a, py::kwargs kw) { 131 | auto deleter = std::make_shared(f, a, kw); 132 | auto ptr = WFTaskFactory::create_go_task(name, [deleter]() { 133 | deleter->go(); 134 | }); 135 | PyWFGoTask task(ptr); 136 | task.set_callback(nullptr); // We need callback to delete user_data 137 | return task; 138 | } 139 | 140 | PyWFGoTask create_go_task(py::function f, py::args a, py::kwargs kw) { 141 | return create_go_task_with_name("pywf-default", f, a, kw); 142 | } 143 | 144 | PyWFEmptyTask create_empty_task() { 145 | auto ptr = WFTaskFactory::create_empty_task(); 146 | PyWFEmptyTask task(ptr); 147 | return task; 148 | } 149 | 150 | PyWFDynamicTask create_dynamic_task(py_dynamic_create_t& create) { 151 | auto ptr = WFTaskFactory::create_dynamic_task([create](WFDynamicTask *t) { 152 | PySubTask&& new_task = create(PyWFDynamicTask(t)); 153 | return new_task.get(); 154 | }); 155 | return PyWFDynamicTask(ptr); 156 | } 157 | 158 | void init_other_types(py::module_ &wf) { 159 | py::class_(wf, "FileIOTask") 160 | .def("is_null", &PyWFFileIOTask::is_null) 161 | .def("start", &PyWFFileIOTask::start) 162 | .def("dismiss", &PyWFFileIOTask::dismiss) 163 | .def("get_state", &PyWFFileIOTask::get_state) 164 | .def("get_error", &PyWFFileIOTask::get_error) 165 | .def("get_retval", &PyWFFileIOTask::get_retval) 166 | .def("set_callback", &PyWFFileIOTask::set_callback) 167 | .def("set_user_data", &PyWFFileIOTask::set_user_data) 168 | .def("get_user_data", &PyWFFileIOTask::get_user_data) 169 | .def("get_fd", &PyWFFileIOTask::get_fd) 170 | .def("get_offset", &PyWFFileIOTask::get_offset) 171 | .def("get_count", &PyWFFileIOTask::get_count) 172 | .def("get_data", &PyWFFileIOTask::get_data) 173 | ; 174 | py::class_(wf, "FileVIOTask") 175 | .def("is_null", &PyWFFileVIOTask::is_null) 176 | .def("start", &PyWFFileVIOTask::start) 177 | .def("dismiss", &PyWFFileVIOTask::dismiss) 178 | .def("get_state", &PyWFFileVIOTask::get_state) 179 | .def("get_error", &PyWFFileVIOTask::get_error) 180 | .def("get_retval", &PyWFFileVIOTask::get_retval) 181 | .def("set_callback", &PyWFFileVIOTask::set_callback) 182 | .def("set_user_data", &PyWFFileVIOTask::set_user_data) 183 | .def("get_user_data", &PyWFFileVIOTask::get_user_data) 184 | .def("get_fd", &PyWFFileVIOTask::get_fd) 185 | .def("get_offset", &PyWFFileVIOTask::get_offset) 186 | .def("get_data", &PyWFFileVIOTask::get_data) 187 | ; 188 | py::class_(wf, "FileSyncTask") 189 | .def("is_null", &PyWFFileSyncTask::is_null) 190 | .def("start", &PyWFFileSyncTask::start) 191 | .def("dismiss", &PyWFFileSyncTask::dismiss) 192 | .def("get_state", &PyWFFileSyncTask::get_state) 193 | .def("get_error", &PyWFFileSyncTask::get_error) 194 | .def("get_retval", &PyWFFileSyncTask::get_retval) 195 | .def("set_callback", &PyWFFileSyncTask::set_callback) 196 | .def("set_user_data", &PyWFFileSyncTask::set_user_data) 197 | .def("get_user_data", &PyWFFileSyncTask::get_user_data) 198 | .def("get_fd", &PyWFFileSyncTask::get_fd) 199 | ; 200 | 201 | py::class_(wf, "TimerTask") 202 | .def("is_null", &PyWFTimerTask::is_null) 203 | .def("start", &PyWFTimerTask::start) 204 | .def("dismiss", &PyWFTimerTask::dismiss) 205 | .def("get_state", &PyWFTimerTask::get_state) 206 | .def("get_error", &PyWFTimerTask::get_error) 207 | .def("set_user_data", &PyWFTimerTask::set_user_data) 208 | .def("get_user_data", &PyWFTimerTask::get_user_data) 209 | .def("set_callback", &PyWFTimerTask::set_callback) 210 | ; 211 | 212 | py::class_(wf, "CounterTask") 213 | .def("is_null", &PyWFCounterTask::is_null) 214 | .def("start", &PyWFCounterTask::start) 215 | .def("dismiss", &PyWFCounterTask::dismiss) 216 | .def("get_state", &PyWFCounterTask::get_state) 217 | .def("get_error", &PyWFCounterTask::get_error) 218 | .def("set_user_data", &PyWFCounterTask::set_user_data) 219 | .def("get_user_data", &PyWFCounterTask::get_user_data) 220 | .def("set_callback", &PyWFCounterTask::set_callback) 221 | .def("count", &PyWFCounterTask::count) 222 | ; 223 | 224 | py::class_(wf, "GoTask") 225 | .def("is_null", &PyWFGoTask::is_null) 226 | .def("start", &PyWFGoTask::start) 227 | .def("dismiss", &PyWFGoTask::dismiss) 228 | .def("get_state", &PyWFGoTask::get_state) 229 | .def("get_error", &PyWFGoTask::get_error) 230 | .def("set_user_data", &PyWFGoTask::set_user_data) 231 | .def("get_user_data", &PyWFGoTask::get_user_data) 232 | .def("set_callback", &PyWFGoTask::set_callback) 233 | ; 234 | 235 | py::class_(wf, "EmptyTask") 236 | .def("is_null", &PyWFEmptyTask::is_null) 237 | .def("start", &PyWFEmptyTask::start) 238 | .def("dismiss", &PyWFEmptyTask::dismiss) 239 | .def("get_state", &PyWFEmptyTask::get_state) 240 | .def("get_error", &PyWFEmptyTask::get_error) 241 | ; 242 | 243 | py::class_(wf, "DynamicTask") 244 | .def("is_null", &PyWFDynamicTask::is_null) 245 | .def("start", &PyWFDynamicTask::start) 246 | .def("dismiss", &PyWFDynamicTask::dismiss) 247 | .def("get_state", &PyWFDynamicTask::get_state) 248 | .def("get_error", &PyWFDynamicTask::get_error) 249 | ; 250 | 251 | py::class_(wf, "WaitGroup") 252 | .def(py::init()) 253 | .def("done", &PyWaitGroup::done, py::call_guard()) 254 | .def("wait", &PyWaitGroup::wait, py::call_guard()) 255 | ; 256 | 257 | wf.def("create_pread_task", &create_pread_task, py::arg("fd"), py::arg("count"), 258 | py::arg("offset"), py::arg("callback")); 259 | wf.def("create_pwrite_task", &create_pwrite_task, py::arg("fd"), py::arg("data"), 260 | py::arg("count"), py::arg("offset"), py::arg("callback")); 261 | wf.def("create_pwritev_task", &create_pwritev_task, py::arg("fd"), py::arg("data_list"), 262 | py::arg("offset"), py::arg("callback")); 263 | wf.def("create_fsync_task", &create_fsync_task, py::arg("fd"), py::arg("callback")); 264 | wf.def("create_fdsync_task", &create_fdsync_task, py::arg("fd"), py::arg("callback")); 265 | 266 | wf.def("create_timer_task", &create_timer_task, py::arg("microseconds"), py::arg("callback")); 267 | wf.def("create_counter_task", &create_counter_task_no_name, py::arg("target"), py::arg("callback")); 268 | wf.def("create_counter_task", &create_counter_task, py::arg("name"), py::arg("target"), 269 | py::arg("callback")); 270 | wf.def("count_by_name", &count_by_name, py::arg("name"), py::arg("n")); 271 | wf.def("create_go_task", &create_go_task, py::arg("function")); 272 | wf.def("create_go_task", &create_go_task_with_name, py::arg("name"), py::arg("function")); 273 | wf.def("create_empty_task", &create_empty_task); 274 | wf.def("create_dynamic_task", &create_dynamic_task, py::arg("creater")); 275 | } 276 | -------------------------------------------------------------------------------- /src/other_types.h: -------------------------------------------------------------------------------- 1 | #ifndef PYWF_OTHER_TYPES_H 2 | #define PYWF_OTHER_TYPES_H 3 | #include "common_types.h" 4 | #include "workflow/WFTask.h" 5 | #include "workflow/WFTaskFactory.h" 6 | #include "workflow/WFFacilities.h" 7 | 8 | class FileTaskData { 9 | public: 10 | FileTaskData() : obj(nullptr) {} 11 | FileTaskData(const FileTaskData&) = delete; 12 | FileTaskData& operator=(const FileTaskData&) = delete; 13 | virtual ~FileTaskData() { 14 | if(obj) { 15 | py::gil_scoped_acquire acquire; 16 | delete obj; 17 | obj = nullptr; 18 | } 19 | } 20 | void set_obj(const py::object &o) { 21 | void *old = obj; 22 | if(old != nullptr) { 23 | delete static_cast(old); 24 | } 25 | obj = nullptr; 26 | if(o.is_none() == false) obj = new py::object(o); 27 | } 28 | py::object get_obj() const { 29 | if(obj == nullptr) return py::none(); 30 | else return *obj; 31 | } 32 | private: 33 | py::object *obj; 34 | }; 35 | 36 | /** 37 | * FileIOTaskData OWNS buf and bytes, you need this class 38 | * to destruct buf and bytes on the end of task's destructor. 39 | **/ 40 | class FileIOTaskData : public FileTaskData { 41 | public: 42 | FileIOTaskData(void *buf, py::object *bytes) 43 | : buf(buf), bytes(bytes) { } 44 | ~FileIOTaskData() { 45 | if(buf) free(buf); 46 | if(bytes) { 47 | py::gil_scoped_acquire acquire; 48 | delete bytes; 49 | } 50 | } 51 | private: 52 | void *buf; 53 | py::object *bytes; 54 | }; 55 | 56 | /** 57 | * FileVIOTaskData OWNS buf[] and bytes[], you need this class 58 | * to destruct buf[] and bytes[] on the end of task's destructor. 59 | **/ 60 | class FileVIOTaskData : public FileTaskData { 61 | public: 62 | FileVIOTaskData(struct iovec *iov, bool with_buf, py::object *bytes, size_t count) 63 | : iov(iov), with_buf(with_buf), bytes(bytes), count(count) {} 64 | ~FileVIOTaskData() { 65 | if(iov && with_buf) { 66 | for(size_t i = 0; i < count; i++) { 67 | free(iov[i].iov_base); 68 | } 69 | } 70 | if(iov) delete[] iov; 71 | if(bytes) { 72 | py::gil_scoped_acquire acquire; 73 | delete[] bytes; 74 | } 75 | } 76 | private: 77 | struct iovec *iov; 78 | bool with_buf; 79 | py::object *bytes; 80 | size_t count; 81 | }; 82 | 83 | template 84 | class TaskDeleterWrapper> { 85 | using Task = WFFileTask; 86 | public: 87 | TaskDeleterWrapper(Func &&f, Task *t) 88 | : f(std::forward(f)), t(t) { } 89 | TaskDeleterWrapper(const TaskDeleterWrapper&) = delete; 90 | TaskDeleterWrapper& operator=(const TaskDeleterWrapper&) = delete; 91 | Func& get_func() { 92 | return f; 93 | } 94 | ~TaskDeleterWrapper() { 95 | if(t->user_data) { 96 | delete static_cast(t->user_data); 97 | t->user_data = nullptr; 98 | } 99 | } 100 | private: 101 | Func f; 102 | Task *t{nullptr}; 103 | }; 104 | 105 | class __file_helper { 106 | public: 107 | template 108 | static int get_fd(Task*) { return -1; } 109 | static int get_fd(WFFileIOTask *t) { return t->get_args()->fd; } 110 | static int get_fd(WFFileVIOTask *t) { return t->get_args()->fd; } 111 | static int get_fd(WFFileSyncTask *t) { return t->get_args()->fd; } 112 | 113 | template 114 | static off_t get_offset(Task*) { return -1; } 115 | static off_t get_offset(WFFileIOTask *t) { return t->get_args()->offset; } 116 | static off_t get_offset(WFFileVIOTask *t) { return t->get_args()->offset; } 117 | 118 | template 119 | static size_t get_count(Task*) { return 0; } 120 | static size_t get_count(WFFileIOTask *t) { return t->get_args()->count; } 121 | 122 | template 123 | static py::object get_data(Task*) { return py::none(); } 124 | static py::object get_data(WFFileIOTask *t) { 125 | auto *arg = t->get_args(); 126 | const char *buf = static_cast(arg->buf); 127 | return py::bytes(buf, arg->count); 128 | } 129 | static py::object get_data(WFFileVIOTask *t) { 130 | py::list contents; 131 | auto *arg = t->get_args(); 132 | for(int i = 0; i < arg->iovcnt; i++) { 133 | const iovec &iov = arg->iov[i]; 134 | contents.append(py::bytes((const char*)iov.iov_base, iov.iov_len)); 135 | } 136 | return static_cast(contents); 137 | } 138 | }; 139 | 140 | template 141 | class PyWFFileTask : public PySubTask { 142 | using _py_callback_t = std::function)>; 143 | public: 144 | using ArgType = Arg; 145 | using OriginType = WFFileTask; 146 | PyWFFileTask() : PySubTask() {} 147 | PyWFFileTask(OriginType *p) : PySubTask(p) {} 148 | PyWFFileTask(const PyWFFileTask &o) : PySubTask(o) {} 149 | OriginType* get() const { return static_cast(ptr); } 150 | void start() { 151 | assert(!series_of(this->get())); 152 | CountableSeriesWork::start_series_work(this->get(), nullptr); 153 | } 154 | void dismiss() { 155 | this->get()->dismiss(); 156 | } 157 | int get_fd() const { return __file_helper::get_fd(this->get()); } 158 | off_t get_offset() const { return __file_helper::get_offset(this->get()); } 159 | size_t get_count() const { return __file_helper::get_count(this->get()); } 160 | py::object get_data() const { return __file_helper::get_data(this->get()); } 161 | long get_retval() const { return this->get()->get_retval(); } 162 | int get_state() const { return this->get()->get_state(); } 163 | int get_error() const { return this->get()->get_error(); } 164 | void set_callback(_py_callback_t cb) { 165 | auto *task = this->get(); 166 | void *user_data = task->user_data; 167 | task->user_data = nullptr; 168 | auto deleter = std::make_shared>( 169 | std::move(cb), task); 170 | task->set_callback([deleter](OriginType *p) { 171 | py_callback_wrapper(deleter->get_func(), PyWFFileTask(p)); 172 | }); 173 | task->user_data = user_data; 174 | } 175 | void set_user_data(const py::object &obj) { 176 | auto *data = static_cast(this->get()->user_data); 177 | data->set_obj(obj); 178 | } 179 | py::object get_user_data() const { 180 | auto *data = static_cast(this->get()->user_data); 181 | return data->get_obj(); 182 | } 183 | }; 184 | 185 | class PyFileIOArgs : public PyWFBase { 186 | public: 187 | using OriginType = FileIOArgs; 188 | PyFileIOArgs() : PyWFBase() {} 189 | PyFileIOArgs(OriginType *p) : PyWFBase(p) {} 190 | PyFileIOArgs(const PyFileIOArgs &o) : PyWFBase(o) {} 191 | OriginType* get() const { return static_cast(ptr); } 192 | }; 193 | 194 | class PyFileVIOArgs : public PyWFBase { 195 | public: 196 | using OriginType = FileVIOArgs; 197 | PyFileVIOArgs() : PyWFBase() {} 198 | PyFileVIOArgs(OriginType *p) : PyWFBase(p) {} 199 | PyFileVIOArgs(const PyFileVIOArgs &o) : PyWFBase(o) {} 200 | OriginType* get() const { return static_cast(ptr); } 201 | }; 202 | 203 | class PyFileSyncArgs : public PyWFBase { 204 | public: 205 | using OriginType = FileSyncArgs; 206 | PyFileSyncArgs() : PyWFBase() {} 207 | PyFileSyncArgs(OriginType *p) : PyWFBase(p) {} 208 | PyFileSyncArgs(const PyFileSyncArgs &o) : PyWFBase(o) {} 209 | OriginType* get() const { return static_cast(ptr); } 210 | }; 211 | 212 | class PyWFTimerTask : public PySubTask { 213 | public: 214 | using OriginType = WFTimerTask; 215 | using _py_callback_t = std::function; 216 | PyWFTimerTask() : PySubTask() {} 217 | PyWFTimerTask(OriginType *p) : PySubTask(p) {} 218 | PyWFTimerTask(const PyWFTimerTask &o) : PySubTask(o) {} 219 | OriginType* get() const { return static_cast(ptr); } 220 | void start() { 221 | assert(!series_of(this->get())); 222 | CountableSeriesWork::start_series_work(this->get(), nullptr); 223 | } 224 | void dismiss() { return this->get()->dismiss(); } 225 | int get_state() const { return this->get()->get_state(); } 226 | int get_error() const { return this->get()->get_error(); } 227 | void set_user_data(py::object obj) { 228 | void *old = this->get()->user_data; 229 | if(old != nullptr) { 230 | delete static_cast(old); 231 | } 232 | py::object *p = nullptr; 233 | if(obj.is_none() == false) p = new py::object(obj); 234 | this->get()->user_data = static_cast(p); 235 | } 236 | py::object get_user_data() const { 237 | void *context = this->get()->user_data; 238 | if(context == nullptr) return py::none(); 239 | return *static_cast(context); 240 | } 241 | void set_callback(_py_callback_t cb) { 242 | auto *task = this->get(); 243 | void *user_data = task->user_data; 244 | task->user_data = nullptr; 245 | auto deleter = std::make_shared>( 246 | std::move(cb), this->get()); 247 | this->get()->set_callback([deleter](OriginType *p) { 248 | py_callback_wrapper(deleter->get_func(), PyWFTimerTask(p)); 249 | }); 250 | task->user_data = user_data; 251 | } 252 | }; 253 | 254 | class PyWFCounterTask : public PySubTask { 255 | public: 256 | using OriginType = WFCounterTask; 257 | using _py_callback_t = std::function; 258 | PyWFCounterTask() : PySubTask() {} 259 | PyWFCounterTask(OriginType *p) : PySubTask(p) {} 260 | PyWFCounterTask(const PyWFCounterTask &o) : PySubTask(o) {} 261 | OriginType* get() const { return static_cast(ptr); } 262 | void start() { 263 | assert(!series_of(this->get())); 264 | CountableSeriesWork::start_series_work(this->get(), nullptr); 265 | } 266 | void dismiss() { return this->get()->dismiss(); } 267 | int get_state() const { return this->get()->get_state(); } 268 | int get_error() const { return this->get()->get_error(); } 269 | void count() { this->get()->count(); } 270 | void set_user_data(py::object obj) { 271 | void *old = this->get()->user_data; 272 | if(old != nullptr) { 273 | delete static_cast(old); 274 | } 275 | py::object *p = nullptr; 276 | if(obj.is_none() == false) p = new py::object(obj); 277 | this->get()->user_data = static_cast(p); 278 | } 279 | py::object get_user_data() const { 280 | void *context = this->get()->user_data; 281 | if(context == nullptr) return py::none(); 282 | return *static_cast(context); 283 | } 284 | void set_callback(_py_callback_t cb) { 285 | auto *task = this->get(); 286 | void *user_data = task->user_data; 287 | task->user_data = nullptr; 288 | auto deleter = std::make_shared>( 289 | std::move(cb), this->get()); 290 | this->get()->set_callback([deleter](OriginType *p) { 291 | py_callback_wrapper(deleter->get_func(), PyWFCounterTask(p)); 292 | }); 293 | task->user_data = user_data; 294 | } 295 | }; 296 | 297 | class PyWFGoTask : public PySubTask { 298 | public: 299 | using OriginType = WFGoTask; 300 | using _py_callback_t = std::function; 301 | PyWFGoTask() : PySubTask() {} 302 | PyWFGoTask(OriginType *p) : PySubTask(p) {} 303 | PyWFGoTask(const PyWFGoTask &o) : PySubTask(o) {} 304 | OriginType* get() const { return static_cast(ptr); } 305 | void start() { 306 | assert(!series_of(this->get())); 307 | CountableSeriesWork::start_series_work(this->get(), nullptr); 308 | } 309 | void dismiss() { this->get()->dismiss(); } 310 | int get_state() const { return this->get()->get_state(); } 311 | int get_error() const { return this->get()->get_error(); } 312 | void set_user_data(py::object obj) { 313 | void *old = this->get()->user_data; 314 | if(old != nullptr) { 315 | delete static_cast(old); 316 | } 317 | py::object *p = nullptr; 318 | if(obj.is_none() == false) p = new py::object(obj); 319 | this->get()->user_data = static_cast(p); 320 | } 321 | py::object get_user_data() const { 322 | void *context = this->get()->user_data; 323 | if(context == nullptr) return py::none(); 324 | return *static_cast(context); 325 | } 326 | void set_callback(_py_callback_t cb) { 327 | auto *task = this->get(); 328 | void *user_data = task->user_data; 329 | task->user_data = nullptr; 330 | auto deleter = std::make_shared>( 331 | std::move(cb), this->get()); 332 | this->get()->set_callback([deleter](OriginType *p) { 333 | py_callback_wrapper(deleter->get_func(), PyWFGoTask(p)); 334 | }); 335 | task->user_data = user_data; 336 | } 337 | }; 338 | 339 | class PyWFEmptyTask : public PySubTask { 340 | public: 341 | using OriginType = WFEmptyTask; 342 | using _py_callback_t = std::function; 343 | PyWFEmptyTask() : PySubTask() {} 344 | PyWFEmptyTask(OriginType *p) : PySubTask(p) {} 345 | PyWFEmptyTask(const PyWFEmptyTask &o) : PySubTask(o) {} 346 | OriginType* get() const { return static_cast(ptr); } 347 | void start() { 348 | assert(!series_of(this->get())); 349 | CountableSeriesWork::start_series_work(this->get(), nullptr); 350 | } 351 | void dismiss() { this->get()->dismiss(); } 352 | int get_state() const { return this->get()->get_state(); } 353 | int get_error() const { return this->get()->get_error(); } 354 | }; 355 | 356 | class PyWFDynamicTask : public PySubTask { 357 | public: 358 | using OriginType = WFDynamicTask; 359 | using _py_create_t = std::function; 360 | PyWFDynamicTask() : PySubTask() {} 361 | PyWFDynamicTask(OriginType *p) : PySubTask(p) {} 362 | PyWFDynamicTask(const PyWFDynamicTask &o) : PySubTask(o) {} 363 | OriginType* get() const { return static_cast(ptr); } 364 | 365 | void start() { 366 | assert(!series_of(this->get())); 367 | CountableSeriesWork::start_series_work(this->get(), nullptr); 368 | } 369 | void dismiss() { this->get()->dismiss(); } 370 | int get_state() const { return this->get()->get_state(); } 371 | int get_error() const { return this->get()->get_error(); } 372 | }; 373 | 374 | // TODO Cannot call done and wait in same (main) thread 375 | class PyWaitGroup { 376 | public: 377 | using OriginType = WFFacilities::WaitGroup; 378 | PyWaitGroup(int n) : wg(n) {} 379 | void done() { wg.done(); } 380 | void wait() const { wg.wait(); } 381 | private: 382 | OriginType wg; 383 | }; 384 | 385 | using PyWFFileIOTask = PyWFFileTask; 386 | using PyWFFileVIOTask = PyWFFileTask; 387 | using PyWFFileSyncTask = PyWFFileTask; 388 | using py_fio_callback_t = std::function; 389 | using py_fvio_callback_t = std::function; 390 | using py_fsync_callback_t = std::function; 391 | using py_timer_callback_t = std::function; 392 | using py_counter_callback_t = std::function; 393 | using py_dynamic_create_t = std::function; 394 | 395 | #endif // PYWF_OTHER_TYPES_H 396 | -------------------------------------------------------------------------------- /src/pyworkflow.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | namespace py = pybind11; 4 | void init_common_types(py::module_&); 5 | void init_network_types(py::module_&); 6 | void init_other_types(py::module_&); 7 | 8 | PYBIND11_MODULE(cpp_pyworkflow, wf) { 9 | wf.doc() = "python3 binding for workflow"; 10 | 11 | init_common_types(wf); 12 | init_network_types(wf); 13 | init_other_types(wf); 14 | } 15 | -------------------------------------------------------------------------------- /src/redis_types.cc: -------------------------------------------------------------------------------- 1 | #include "redis_types.h" 2 | using namespace std; 3 | 4 | using protocol::RedisValue; 5 | py::object redis_as_object(RedisValue &value) { 6 | switch(value.get_type()) { 7 | case REDIS_REPLY_TYPE_STRING: 8 | case REDIS_REPLY_TYPE_STATUS: 9 | case REDIS_REPLY_TYPE_ERROR: 10 | return py::bytes(value.string_value()); 11 | case REDIS_REPLY_TYPE_ARRAY: 12 | { 13 | py::list lst; 14 | for(size_t i = 0; i < value.arr_size(); i++) { 15 | lst.append(redis_as_object(value[i])); 16 | } 17 | return static_cast(lst); 18 | } 19 | case REDIS_REPLY_TYPE_INTEGER: 20 | return py::int_(value.int_value()); 21 | case REDIS_REPLY_TYPE_NIL: 22 | default: 23 | return py::none(); 24 | } 25 | } 26 | void redis_set_string(RedisValue &value, const string &s) { 27 | value.set_string(s); 28 | } 29 | void redis_set_status(RedisValue &value, const string &s) { 30 | value.set_status(s); 31 | } 32 | void redis_set_error(RedisValue &value, const string &s) { 33 | value.set_error(s); 34 | } 35 | 36 | py::bytes redis_bytes_value(RedisValue &value) { 37 | const std::string *sv = value.string_view(); 38 | if(sv == nullptr) return py::bytes(); 39 | else return py::bytes(*sv); 40 | } 41 | py::bytes redis_debug_bytes(RedisValue &value) { 42 | std::string str = value.debug_string(); 43 | return py::bytes(str); 44 | } 45 | 46 | py::object redis_arr_at_ref(RedisValue &value, size_t pos) { 47 | if(pos >= value.arr_size()) return py::none(); 48 | auto &v = value.arr_at(pos); 49 | return py::cast(v, py::return_value_policy::reference); 50 | } 51 | py::object redis_arr_at_object(RedisValue &value, size_t pos) { 52 | if(pos >= value.arr_size()) return py::none(); 53 | return redis_as_object(value.arr_at(pos)); 54 | } 55 | py::object redis_arr_at(RedisValue &value, size_t pos) { 56 | if(pos >= value.arr_size()) return py::none(); 57 | return py::cast(value.arr_at(pos), py::return_value_policy::copy); 58 | } 59 | void redis_arr_set(RedisValue &value, size_t pos, RedisValue &v) { 60 | // if pos is invalid, do nothing 61 | if(pos < value.arr_size()) { 62 | value.arr_at(pos) = v; 63 | } 64 | } 65 | 66 | RedisValue redis_copy(RedisValue &value) { 67 | return value; 68 | } 69 | void redis_move_to(RedisValue &value, RedisValue &o) { 70 | o = std::move(value); 71 | } 72 | void redis_move_from(RedisValue &value, RedisValue &o) { 73 | value = std::move(o); 74 | } 75 | 76 | PyWFRedisTask create_redis_task(const std::string &url, int retry_max, py_redis_callback_t cb) { 77 | WFRedisTask *ptr = WFTaskFactory::create_redis_task(url, retry_max, nullptr); 78 | PyWFRedisTask t(ptr); 79 | t.set_callback(std::move(cb)); 80 | return t; 81 | } 82 | 83 | void init_redis_types(py::module_ &wf) { 84 | 85 | py::class_(wf, "RedisValue") 86 | .def(py::init()) 87 | .def(py::init()) 88 | 89 | .def("__len__", &RedisValue::arr_size) 90 | .def("__getitem__", &redis_arr_at_ref) 91 | .def("__setitem__", &redis_arr_set) 92 | .def("__copy__", &redis_copy) 93 | .def("__deepcopy__", &redis_copy) 94 | 95 | .def("copy", &redis_copy) 96 | .def("move_to", &redis_move_to) 97 | .def("move_from", &redis_move_from) 98 | 99 | .def("set_nil", &RedisValue::set_nil) 100 | .def("set_int", &RedisValue::set_int) 101 | .def("set_array", &RedisValue::set_array) 102 | .def("set_string", &redis_set_string) 103 | .def("set_status", &redis_set_status) 104 | .def("set_error", &redis_set_error) 105 | 106 | .def("is_ok", &RedisValue::is_ok) 107 | .def("is_error", &RedisValue::is_error) 108 | .def("is_nil", &RedisValue::is_nil) 109 | .def("is_int", &RedisValue::is_int) 110 | .def("is_array", &RedisValue::is_array) 111 | .def("is_string", &RedisValue::is_string) 112 | 113 | .def("string_value", &redis_bytes_value) 114 | .def("int_value", &RedisValue::int_value) 115 | .def("arr_size", &RedisValue::arr_size) 116 | .def("arr_clear", &RedisValue::arr_clear) 117 | .def("arr_resize", &RedisValue::arr_resize) 118 | 119 | .def("clear", &RedisValue::clear) 120 | .def("debug_string", &redis_debug_bytes) 121 | .def("arr_at", &redis_arr_at) 122 | .def("arr_at_ref", &redis_arr_at_ref) 123 | .def("arr_at_object", &redis_arr_at_object) 124 | .def("as_object", &redis_as_object) 125 | ; 126 | 127 | py::class_(wf, "RedisTask") 128 | .def("is_null", &PyWFRedisTask::is_null) 129 | .def("start", &PyWFRedisTask::start) 130 | .def("dismiss", &PyWFRedisTask::dismiss) 131 | .def("noreply", &PyWFRedisTask::noreply) 132 | .def("get_req", &PyWFRedisTask::get_req) 133 | .def("get_resp", &PyWFRedisTask::get_resp) 134 | .def("get_state", &PyWFRedisTask::get_state) 135 | .def("get_error", &PyWFRedisTask::get_error) 136 | .def("get_timeout_reason", &PyWFRedisTask::get_timeout_reason) 137 | .def("get_task_seq", &PyWFRedisTask::get_task_seq) 138 | .def("set_send_timeout", &PyWFRedisTask::set_send_timeout) 139 | .def("set_receive_timeout", &PyWFRedisTask::set_receive_timeout) 140 | .def("set_keep_alive", &PyWFRedisTask::set_keep_alive) 141 | .def("get_peer_addr", &PyWFRedisTask::get_peer_addr) 142 | .def("set_callback", &PyWFRedisTask::set_callback) 143 | .def("set_user_data", &PyWFRedisTask::set_user_data) 144 | .def("get_user_data", &PyWFRedisTask::get_user_data) 145 | ; 146 | 147 | py::class_(wf, "RedisRequest") 148 | .def("is_null", &PyRedisRequest::is_null) 149 | .def("move_to", &PyRedisRequest::move_to) 150 | .def("set_request", &PyRedisRequest::set_request) 151 | .def("get_command", &PyRedisRequest::get_command) 152 | .def("get_params", &PyRedisRequest::get_params) 153 | 154 | .def("set_size_limit", &PyRedisRequest::set_size_limit) 155 | .def("get_size_limit", &PyRedisRequest::get_size_limit) 156 | ; 157 | 158 | py::class_(wf, "RedisResponse") 159 | .def("is_null", &PyRedisResponse::is_null) 160 | .def("move_to", &PyRedisResponse::move_to) 161 | .def("get_result", &PyRedisResponse::get_result) 162 | .def("set_result", &PyRedisResponse::set_result) 163 | 164 | .def("set_size_limit", &PyRedisResponse::set_size_limit) 165 | .def("get_size_limit", &PyRedisResponse::get_size_limit) 166 | ; 167 | 168 | py::class_(wf, "RedisServer") 169 | .def(py::init()) 170 | .def(py::init()) 171 | .def("start", &PyWFRedisServer::start_1, py::arg("port"), py::arg("cert_file") = std::string(), 172 | py::arg("key_file") = std::string()) 173 | .def("start", &PyWFRedisServer::start_2, py::arg("family"), py::arg("host"), py::arg("port"), 174 | py::arg("cert_file") = std::string(), py::arg("key_file") = std::string()) 175 | .def("shutdown", &PyWFRedisServer::shutdown, py::call_guard()) 176 | .def("wait_finish", &PyWFRedisServer::wait_finish, py::call_guard()) 177 | .def("stop", &PyWFRedisServer::stop, py::call_guard()) 178 | ; 179 | 180 | wf.def("create_redis_task", &create_redis_task, py::arg("url"), py::arg("retry_max"), 181 | py::arg("callback")); 182 | } 183 | -------------------------------------------------------------------------------- /src/redis_types.h: -------------------------------------------------------------------------------- 1 | #ifndef PYWF_REDIS_TYPES_H 2 | #define PYWF_REDIS_TYPES_H 3 | 4 | #include "network_types.h" 5 | #include "workflow/RedisMessage.h" 6 | 7 | class PyRedisRequest : public PyWFBase { 8 | public: 9 | using OriginType = protocol::RedisRequest; 10 | PyRedisRequest() : PyWFBase() {} 11 | PyRedisRequest(OriginType *p) : PyWFBase(p) {} 12 | PyRedisRequest(const PyRedisRequest &o) : PyWFBase(o) {} 13 | OriginType* get() const { return static_cast(ptr); } 14 | 15 | void move_to(PyRedisRequest &o) { 16 | *(o.get()) = std::move(*(this->get())); 17 | } 18 | 19 | void set_request(const std::string &cmd, const std::vector ¶ms) { 20 | this->get()->set_request(cmd, params); 21 | } 22 | 23 | py::str get_command() const { 24 | std::string cmd; 25 | this->get()->get_command(cmd); 26 | return cmd; 27 | } 28 | 29 | py::list get_params() const { 30 | py::list params; 31 | std::vector params_vec; 32 | this->get()->get_params(params_vec); 33 | for(const auto ¶m : params_vec) { 34 | params.append(py::bytes(param)); 35 | } 36 | return params; 37 | } 38 | 39 | void set_size_limit(size_t limit) { this->get()->set_size_limit(limit); } 40 | size_t get_size_limit() const { return this->get()->get_size_limit(); } 41 | }; 42 | 43 | class PyRedisResponse : public PyWFBase { 44 | public: 45 | using OriginType = protocol::RedisResponse; 46 | PyRedisResponse() : PyWFBase() {} 47 | PyRedisResponse(OriginType *p) : PyWFBase(p) {} 48 | PyRedisResponse(const PyRedisResponse &o) : PyWFBase(o) {} 49 | OriginType* get() const { return static_cast(ptr); } 50 | 51 | void move_to(PyRedisResponse &o) { 52 | *(o.get()) = std::move(*(this->get())); 53 | } 54 | 55 | protocol::RedisValue get_result() const { 56 | protocol::RedisValue v; 57 | this->get()->get_result(v); 58 | return v; 59 | } 60 | 61 | bool set_result(const protocol::RedisValue &v) { 62 | return this->get()->set_result(v); 63 | } 64 | 65 | void set_size_limit(size_t limit) { this->get()->set_size_limit(limit); } 66 | size_t get_size_limit() const { return this->get()->get_size_limit(); } 67 | }; 68 | 69 | using PyWFRedisTask = PyWFNetworkTask; 70 | using PyWFRedisServer = PyWFServer; 71 | using py_redis_callback_t = std::function; 72 | using py_redis_process_t = std::function; 73 | 74 | #endif // PYWF_REDIS_TYPES_H 75 | -------------------------------------------------------------------------------- /tutorial/tutorial01-wget.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import pywf as wf 4 | 5 | 6 | def wget_callback(t): 7 | req = t.get_req() 8 | resp = t.get_resp() 9 | state = t.get_state() 10 | error = t.get_error() 11 | 12 | if state != wf.WFT_STATE_SUCCESS: 13 | print( 14 | "state:{} error:{} errstr:{}".format( 15 | state, error, wf.get_error_string(state, error) 16 | ) 17 | ) 18 | return 19 | print( 20 | "{} {} {}".format( 21 | req.get_method(), req.get_http_version(), req.get_request_uri() 22 | ) 23 | ) 24 | req_headers = req.get_headers() 25 | for h in req_headers: 26 | print("{}: {}".format(h[0], h[1])) 27 | print("") 28 | print( 29 | "{} {} {}".format( 30 | resp.get_http_version(), resp.get_status_code(), resp.get_reason_phrase() 31 | ) 32 | ) 33 | headers = resp.get_headers() 34 | for h in headers: 35 | print("{}: {}".format(h[0], h[1])) 36 | print( 37 | "\nbody size:{} (ignore body for a clear output)".format(len(resp.get_body())) 38 | ) 39 | # print(resp.get_body()) # body of resp is bytes type 40 | 41 | 42 | def main(): 43 | if len(sys.argv) != 2: 44 | print("USAGE: {} ".format(sys.argv[0])) 45 | sys.exit(1) 46 | url = sys.argv[1] 47 | if url[:7].lower() != "http://" and url[:8].lower() != "https://": 48 | url = "http://" + url 49 | task = wf.create_http_task(url, redirect_max=4, retry_max=2, callback=wget_callback) 50 | req = task.get_req() 51 | req.add_header_pair("Accept", "*/*") 52 | req.add_header_pair("User-Agent", "Wget/1.14 (linux-gnu)") 53 | req.add_header_pair("Connection", "close") 54 | task.start() 55 | wf.wait_finish() 56 | 57 | 58 | # Usage: python3 tutorial01-wget.py http://www.sogou.com/ 59 | if __name__ == "__main__": 60 | main() 61 | -------------------------------------------------------------------------------- /tutorial/tutorial02-redis_cli.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import pywf as wf 4 | 5 | class UserData: 6 | def __init__(self): 7 | pass 8 | 9 | 10 | def redis_callback(t): 11 | req = t.get_req() 12 | resp = t.get_resp() 13 | state = t.get_state() 14 | error = t.get_error() 15 | 16 | if state != wf.WFT_STATE_SUCCESS: 17 | print( 18 | "state:{} error:{} errstr:{}".format( 19 | state, error, wf.get_error_string(state, error) 20 | ) 21 | ) 22 | return 23 | else: 24 | val = resp.get_result() 25 | if val.is_error(): 26 | print( 27 | "Error reply. Need a password?" 28 | ) 29 | return 30 | 31 | cmd = req.get_command() 32 | if cmd == "SET": 33 | user_data = t.get_user_data() 34 | n = wf.create_redis_task(user_data.url, retry_max=2, callback=redis_callback) 35 | n.get_req().set_request("GET", [user_data.key]) 36 | wf.series_of(t).push_back(n) 37 | print("Redis SET request success. Trying to GET...") 38 | else: 39 | if val.is_string(): 40 | print("Redis GET success. Value = {}".format(val.string_value())) 41 | else: 42 | print("Error. Not a string value") 43 | 44 | print("Finish") 45 | 46 | def main(): 47 | if len(sys.argv) != 4: 48 | print("USAGE: {} ".format(sys.argv[0])) 49 | sys.exit(1) 50 | user_data = UserData() 51 | url = sys.argv[1] 52 | if url[:8].lower() != "redis://" and url[:9].lower() != "rediss://": 53 | url = "redis://" + url 54 | user_data.url = url 55 | task = wf.create_redis_task(url, retry_max=2, callback=redis_callback) 56 | req = task.get_req() 57 | req.set_request("SET", [sys.argv[2], sys.argv[3]]) 58 | user_data.key = sys.argv[2] 59 | task.set_user_data(user_data) 60 | task.start() 61 | wf.wait_finish() 62 | 63 | 64 | # Usage: python3 tutorial02-redis_cli redis://:password@hostname/dbnum key value 65 | if __name__ == "__main__": 66 | main() 67 | -------------------------------------------------------------------------------- /tutorial/tutorial03-wget_to_redis.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import pywf as wf 4 | 5 | class Context: 6 | def __init__(self): 7 | pass 8 | 9 | def redis_callback(t): 10 | req = t.get_req() 11 | resp = t.get_resp() 12 | state = t.get_state() 13 | error = t.get_error() 14 | 15 | if state != wf.WFT_STATE_SUCCESS: 16 | print( 17 | "redis: state:{} error:{} errstr:{}".format( 18 | state, error, wf.get_error_string(state, error) 19 | ) 20 | ) 21 | return 22 | else: 23 | val = resp.get_result() 24 | if val.is_error(): 25 | print( 26 | "Error reply. Need a password?" 27 | ) 28 | return 29 | 30 | context = wf.series_of(t).get_context() 31 | print("redis SET success: key: {}, value size: {}".format(context.http_url, context.body_len)) 32 | context.success = True 33 | 34 | def http_callback(t): 35 | req = t.get_req() 36 | resp = t.get_resp() 37 | state = t.get_state() 38 | error = t.get_error() 39 | 40 | if state != wf.WFT_STATE_SUCCESS: 41 | print( 42 | "http: state:{} error:{} errstr:{}".format( 43 | state, error, wf.get_error_string(state, error) 44 | ) 45 | ) 46 | return 47 | 48 | body_len = resp.get_body_size() 49 | if body_len == 0: 50 | print("Error: empty Http body!") 51 | return 52 | 53 | series = wf.series_of(t) 54 | context = series.get_context() 55 | context.body_len = body_len 56 | redis_url = context.redis_url 57 | redis_task = wf.create_redis_task(redis_url, retry_max=2, callback=redis_callback) 58 | redis_task.get_req().set_request("SET", [context.http_url, resp.get_body()]) 59 | series.push_back(redis_task) 60 | 61 | def series_callback(s): 62 | context = s.get_context() 63 | print("Series finished.", end=" ") 64 | if context.success: 65 | print("all success") 66 | else: 67 | print("failed") 68 | 69 | def main(): 70 | if len(sys.argv) != 3: 71 | print("USAGE: {} ".format(sys.argv[0])) 72 | sys.exit(1) 73 | context = Context() 74 | context.success = False 75 | 76 | url = sys.argv[1] 77 | if url[:7].lower() != "http://" and url[:8].lower() != "https://": 78 | url = "https://" + url 79 | context.http_url = url 80 | 81 | url = sys.argv[2] 82 | if url[:8].lower() != "redis://" and url[:9].lower() != "rediss://": 83 | url = "redis://" + url 84 | context.redis_url = url 85 | 86 | task = wf.create_http_task(context.http_url, 3, 2, http_callback) 87 | req = task.get_req() 88 | req.add_header_pair("Accept", "*/*") 89 | req.add_header_pair("User-Agent", "Wget/1.14 (linux-gnu)") 90 | req.add_header_pair("Connection", "close") 91 | 92 | # Limit the http response to size 20M 93 | task.get_resp().set_size_limit(20 * 1024 * 1024) 94 | 95 | # no more than 30 seconds receiving http response 96 | task.set_receive_timeout(30 * 1000) 97 | 98 | series = wf.create_series_work(task, series_callback) 99 | series.set_context(context) 100 | series.start() 101 | 102 | wf.wait_finish() 103 | 104 | 105 | # Usage: python3 tutorial03-wget_to_redis 106 | if __name__ == "__main__": 107 | main() 108 | -------------------------------------------------------------------------------- /tutorial/tutorial04-http_echo_server.py: -------------------------------------------------------------------------------- 1 | import signal 2 | import sys 3 | import threading 4 | 5 | import pywf as wf 6 | 7 | 8 | def process(task): 9 | req = task.get_req() 10 | resp = task.get_resp() 11 | 12 | resp.append_body(b"\n") # as bytes 13 | resp.append_body( 14 | f"

{req.get_method()} " 15 | f"{req.get_request_uri()} " 16 | f"{req.get_http_version()}

\n" 17 | ) 18 | headers = req.get_headers() 19 | for header in headers: 20 | resp.append_body(f"

{header[0]}: {header[1]}

\n") 21 | resp.append_body("\n") # as str 22 | 23 | resp.set_http_version("HTTP/1.1") 24 | resp.set_status_code("200") 25 | resp.set_reason_phrase("OK") 26 | resp.add_header_pair("Content-Type", "text/html") 27 | resp.add_header_pair("Server", "Sogou Python3 WFHttpServer") 28 | seq = task.get_task_seq() 29 | if seq == 9: # close after 10 reqs, seq start from 0 30 | resp.add_header_pair("Connection", "close") 31 | 32 | 33 | def main(): 34 | if len(sys.argv) != 2: 35 | print("Usage {} ".format(sys.argv[0])) 36 | sys.exit(1) 37 | 38 | port = int(sys.argv[1]) 39 | 40 | server = wf.HttpServer(process) 41 | stop_event = threading.Event() 42 | 43 | def stop(*args): 44 | if not stop_event.is_set(): 45 | stop_event.set() 46 | 47 | for sig in (signal.SIGTERM, signal.SIGINT): 48 | signal.signal(sig, stop) 49 | 50 | # You can use server.start(socket.AddressFamily.AF_INET, "localhost", port) too 51 | if server.start(port) == 0: 52 | stop_event.wait() 53 | server.stop() 54 | """ server.stop() equal to: 55 | server.shutdown() 56 | server.wait_finish() 57 | """ 58 | else: 59 | print("Cannot start server") 60 | sys.exit(1) 61 | 62 | 63 | if __name__ == "__main__": 64 | main() 65 | -------------------------------------------------------------------------------- /tutorial/tutorial05-http_proxy.py: -------------------------------------------------------------------------------- 1 | import errno 2 | import os 3 | import signal 4 | import sys 5 | import threading 6 | 7 | import pywf as wf 8 | 9 | 10 | class Context: 11 | def __init__(self): 12 | pass 13 | 14 | 15 | cv = threading.Condition() 16 | 17 | 18 | def Stop(signum, frame): 19 | print("Stop server:", signum) 20 | cv.acquire() 21 | cv.notify() 22 | cv.release() 23 | 24 | 25 | def reply_callback(proxy_task): 26 | series = wf.series_of(proxy_task) 27 | ctx = series.get_context() 28 | proxy_resp = proxy_task.get_resp() 29 | sz = proxy_resp.get_body_size() 30 | if proxy_task.get_state() == wf.WFT_STATE_SUCCESS: 31 | print( 32 | "{} Success, Http Status:{} Body Length:{}".format( 33 | ctx.url, proxy_resp.get_status_code(), sz 34 | ) 35 | ) 36 | else: 37 | print( 38 | "{} Reply failed:{} Body Length:{}".format( 39 | ctx.url, os.strerror(proxy_task.get_error()), sz 40 | ) 41 | ) 42 | 43 | 44 | def http_callback(t): 45 | state = t.get_state() 46 | error = t.get_error() 47 | resp = t.get_resp() 48 | series = wf.series_of(t) 49 | ctx = series.get_context() 50 | proxy_resp = ctx.proxy_task.get_resp() 51 | 52 | if state == wf.WFT_STATE_SUCCESS: 53 | ctx.proxy_task.set_callback(reply_callback) 54 | # move content from resp to proxy_resp, then you cannot use resp 55 | resp.move_to(proxy_resp) 56 | if not ctx.is_keep_alive: 57 | proxy_resp.set_header_pair("Connection", "close") 58 | else: 59 | errstr = "" 60 | if state == wf.WFT_STATE_SYS_ERROR: 61 | errstr = "system error: {}".format(os.strerror(error)) 62 | elif state == wf.WFT_STATE_DNS_ERROR: 63 | errstr = "DNS error: {}".format(error) 64 | elif state == wf.WFT_STATE_SSL_ERROR: 65 | errstr = "SSL error: {}".format(error) 66 | else: 67 | errstr = "URL error (Cannot be a HTTPS proxy)" 68 | print( 69 | "{} Fetch failed, state:{} error:{} {}".format( 70 | ctx.url, state, error, errstr 71 | ) 72 | ) 73 | proxy_resp.set_status_code("404") 74 | proxy_resp.append_body(b"404 Not Found.") 75 | 76 | 77 | def process(t): 78 | req = t.get_req() 79 | series = wf.series_of(t) 80 | ctx = Context() 81 | ctx.url = req.get_request_uri() 82 | ctx.proxy_task = t 83 | series.set_context(ctx) 84 | ctx.is_keep_alive = req.is_keep_alive() 85 | http_task = wf.create_http_task(req.get_request_uri(), 0, 0, http_callback) 86 | req.set_request_uri(http_task.get_req().get_request_uri()) 87 | req.move_to(http_task.get_req()) 88 | http_task.get_resp().set_size_limit(200 * 1024 * 1024) 89 | series << http_task 90 | 91 | 92 | def main(): 93 | if len(sys.argv) != 2: 94 | print("Usage {} ".format(sys.argv[0])) 95 | sys.exit(1) 96 | port = int(sys.argv[1]) 97 | signal.signal(signal.SIGINT, Stop) 98 | server_params = wf.ServerParams() 99 | server_params.request_size_limit = 8 * 1024 * 1024 100 | server = wf.HttpServer(server_params, process) 101 | if server.start(port) == 0: 102 | cv.acquire() 103 | cv.wait() 104 | cv.release() 105 | server.stop() 106 | else: 107 | print("Cannot start server") 108 | sys.exit(1) 109 | 110 | 111 | # Test it: curl -x http://localhost:10086/ http://sogou.com 112 | if __name__ == "__main__": 113 | main() 114 | -------------------------------------------------------------------------------- /tutorial/tutorial06-parallel_wget.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import pywf as wf 4 | 5 | 6 | class Context: 7 | def __init__(self): 8 | pass 9 | 10 | 11 | def parallel_callback(p): 12 | sz = p.size() 13 | for i in range(sz): 14 | ctx = p.series_at(i).get_context() 15 | if ctx.state == wf.WFT_STATE_SUCCESS: 16 | print( 17 | "url:{} chunked:{} keep_alive:{} status_code:{} http_version:{}".format( 18 | ctx.url, 19 | ctx.chunked, 20 | ctx.keep_alive, 21 | ctx.status_code, 22 | ctx.http_version, 23 | ) 24 | ) 25 | else: 26 | print("url:{} state:{} error:{}".format(ctx.url, ctx.state, ctx.error)) 27 | 28 | 29 | def http_callback(t): 30 | ctx = wf.series_of(t).get_context() 31 | ctx.state = t.get_state() 32 | ctx.error = t.get_error() 33 | # Attention: YOU ARE NOT ALLOW TO OWN RESP AFTER THIS CALLBACK, SO MAKE A COPY 34 | resp = t.get_resp() 35 | ctx.chunked = resp.is_chunked() 36 | ctx.keep_alive = resp.is_keep_alive() 37 | ctx.status_code = resp.get_status_code() 38 | ctx.http_version = resp.get_http_version() 39 | ctx.body = resp.get_body() 40 | 41 | 42 | def main(): 43 | parallel_work = wf.create_parallel_work(parallel_callback) 44 | for url in sys.argv[1:]: 45 | if url[:7].lower() != "http://" and url[:8].lower() != "https://": 46 | url = "http://" + url 47 | task = wf.create_http_task( 48 | url, redirect_max=4, retry_max=2, callback=http_callback 49 | ) 50 | req = task.get_req() 51 | req.add_header_pair("Accept", "*/*") 52 | req.add_header_pair("User-Agent", "Wget/1.14 (linux-gnu)") 53 | req.add_header_pair("Connection", "close") 54 | ctx = Context() 55 | ctx.url = url 56 | series = wf.create_series_work(task, None) 57 | series.set_context(ctx) 58 | parallel_work.add_series(series) 59 | wf.start_series_work(parallel_work, None) 60 | wf.wait_finish() 61 | 62 | 63 | # Usage: python3 tutorial06-parallel_wget.py https://www.sogou.com/ https://zhihu.sogou.com/ 64 | if __name__ == "__main__": 65 | main() 66 | -------------------------------------------------------------------------------- /tutorial/tutorial09-http_file_server.py: -------------------------------------------------------------------------------- 1 | import errno 2 | import os 3 | import signal 4 | import sys 5 | import threading 6 | 7 | import pywf as wf 8 | 9 | 10 | class Context: 11 | def __init__(self): 12 | pass 13 | 14 | 15 | cv = threading.Condition() 16 | 17 | 18 | def Stop(signum, frame): 19 | print("Stop server:", signum) 20 | cv.acquire() 21 | cv.notify() 22 | cv.release() 23 | 24 | 25 | def pread_callback(t): 26 | state = t.get_state() 27 | error = t.get_error() 28 | resp = t.get_user_data() 29 | 30 | os.close(t.get_fd()) 31 | if state != wf.WFT_STATE_SUCCESS or t.get_retval() < 0: 32 | resp.set_status_code("503") 33 | resp.append_body("503 Internal Server Error") 34 | else: 35 | resp.append_body(t.get_data()) 36 | 37 | def process(root, t): 38 | req = t.get_req() 39 | resp = t.get_resp() 40 | 41 | request_uri = req.get_request_uri() 42 | index = request_uri.find("?") 43 | path = root + (request_uri if index == -1 else request_uri[:index]) 44 | 45 | if path[-1] == "/": 46 | path += "index.html" 47 | 48 | try: 49 | fd = os.open(path, os.O_RDONLY) 50 | except: 51 | resp.set_status_code("404") 52 | resp.append_body("404 Not Found") 53 | return 54 | 55 | series = wf.series_of(t) 56 | size = os.lseek(fd, 0, os.SEEK_END) 57 | pread_task = wf.create_pread_task(fd, size, 0, pread_callback) 58 | pread_task.set_user_data(resp) 59 | series.push_back(pread_task) 60 | 61 | def get_process(root): 62 | def wrapper(task): 63 | process(root, task) 64 | return wrapper 65 | 66 | def main(): 67 | argc = len(sys.argv) 68 | if argc != 2 and argc != 3 and argc != 5: 69 | print("Usage {} [root path] [cert file] [key file]".format(sys.argv[0])) 70 | sys.exit(1) 71 | port = int(sys.argv[1]) 72 | root = sys.argv[2] if argc > 2 else "." 73 | signal.signal(signal.SIGINT, Stop) 74 | server = wf.HttpServer(get_process(root)) 75 | 76 | if argc >= 4: 77 | ret = server.start(port, sys.argv[3], sys.argv[4]) 78 | else: 79 | ret = server.start(port) 80 | 81 | if ret == 0: 82 | cv.acquire() 83 | cv.wait() 84 | cv.release() 85 | server.stop() 86 | else: 87 | print("Cannot start server") 88 | sys.exit(1) 89 | 90 | 91 | # Run it: python3 tutorial09-http_file_server.py 10086 . 92 | # Test it: curl "http://localhost:10086/" 93 | if __name__ == "__main__": 94 | main() 95 | -------------------------------------------------------------------------------- /tutorial/tutorial12-mysql_cli.py: -------------------------------------------------------------------------------- 1 | import readline 2 | import sys 3 | 4 | import pywf as wf 5 | 6 | default_column_width = 20 7 | retry_max = 2 8 | 9 | def header_line(sz): 10 | return '+' + '+'.join(['-' * default_column_width for i in range(sz)]) + '+' 11 | 12 | def as_str(obj): 13 | try: 14 | return obj.decode() if type(obj) is bytes else str(obj) 15 | except: 16 | return 'Exception' 17 | 18 | def format_row(row): 19 | return '|' + '|'.join([f"{str(x):^{default_column_width}}" for x in row]) + '|' 20 | 21 | def next_line(): 22 | sql = '' 23 | while True: 24 | line = '' 25 | prompt = 'mysql> ' if len(sql) == 0 else '-----> ' 26 | try: 27 | line = input(prompt).strip() 28 | except: 29 | return None 30 | sql = line if len(sql) == 0 else sql + ' ' + line 31 | 32 | if sql[:4].lower() == 'quit' or sql[:4].lower() == 'exit': 33 | return None 34 | if len(sql) > 0 and sql[-1] == ';': 35 | return sql 36 | 37 | def create_next_task(url, callback): 38 | sql = next_line() 39 | if sql is None: 40 | print('\nBye') 41 | return None 42 | readline.add_history(sql) 43 | 44 | task = wf.create_mysql_task(url, retry_max, callback) 45 | task.get_req().set_query(sql) 46 | return task 47 | 48 | def mysql_callback(task): 49 | state = task.get_state() 50 | error = task.get_error() 51 | 52 | if state != wf.WFT_STATE_SUCCESS: 53 | print(wf.get_error_string(state, error)) 54 | return 55 | 56 | resp = task.get_resp() 57 | cursor = wf.MySQLResultCursor(resp) 58 | 59 | for result_set in wf.MySQLResultSetIterator(cursor): 60 | if result_set.get_cursor_status() == wf.MYSQL_STATUS_GET_RESULT: 61 | fields = result_set.fetch_fields() 62 | 63 | print(header_line(len(fields))) 64 | print(format_row(fields)) 65 | print(header_line(len(fields))) 66 | 67 | for row in wf.MySQLRowIterator(result_set): 68 | print(format_row(row)) 69 | 70 | print(header_line(len(fields))) 71 | print("{} {} in set\n".format( 72 | result_set.get_rows_count(), 73 | "row" if result_set.get_rows_count() == 1 else "rows" 74 | )) 75 | elif result_set.get_cursor_status() == wf.MYSQL_STATUS_OK: 76 | print("OK. {} {} affected. {} warnings. insert_id={}. {}".format( 77 | result_set.get_affected_rows(), 78 | "row" if result_set.get_affected_rows() == 1 else "rows", 79 | result_set.get_warnings(), 80 | result_set.get_insert_id(), 81 | as_str(result_set.get_info()) 82 | )) 83 | 84 | if resp.get_packet_type() == wf.MYSQL_PACKET_OK: 85 | print("OK. {} {} affected. {} warnings. insert_id={}. {}".format( 86 | resp.get_affected_rows(), 87 | "row" if resp.get_affected_rows() == 1 else "rows", 88 | resp.get_warnings(), 89 | resp.get_last_insert_id(), 90 | as_str(resp.get_info()) 91 | )) 92 | 93 | elif resp.get_packet_type() == wf.MYSQL_PACKET_ERROR: 94 | print("ERROR. error_code={} {}".format( 95 | resp.get_error_code(), 96 | as_str(resp.get_error_msg()) 97 | )) 98 | 99 | url = wf.series_of(task).get_context() 100 | next_task = create_next_task(url, mysql_callback) 101 | if next_task is not None: 102 | wf.series_of(task).push_back(next_task) 103 | pass 104 | 105 | 106 | def main(): 107 | if len(sys.argv) != 2: 108 | print("USAGE: {} ".format(sys.argv[0])) 109 | sys.exit(1) 110 | 111 | url = sys.argv[1] 112 | if url[:8].lower() != "mysql://" and url[:9].lower() != "mysqls://": 113 | url = "mysql://" + url 114 | 115 | readline.set_auto_history(False) 116 | 117 | mysql_task = wf.create_mysql_task(url, retry_max, mysql_callback) 118 | 119 | first_sql = "show databases;" 120 | req = mysql_task.get_req() 121 | req.set_query(first_sql) 122 | readline.add_history(first_sql) 123 | 124 | series = wf.create_series_work(mysql_task, None) 125 | series.set_context(url) 126 | series.start() 127 | wf.wait_finish() 128 | 129 | # Usage: python3 tutorial12-mysql_cli.py mysql://user:password@host:port/dbname 130 | if __name__ == '__main__': 131 | main() 132 | --------------------------------------------------------------------------------