├── .gitignore ├── DETAIL.md ├── Dockerfile ├── LICENSE ├── README.md ├── examples ├── dns_service.py ├── dns_web.py ├── gossip │ ├── node1.py │ ├── node2.py │ └── node3.py ├── http_rpc_client.py ├── http_rpc_server.py ├── tcp_rpc_client.py ├── tcp_rpc_server.py ├── udp_rpc_client.py └── udp_rpc_server.py ├── exec_test.sh ├── setup.py ├── sunyata ├── __init__.py ├── algorithm │ ├── __init__.py │ ├── avlsearchtree.py │ ├── avltree.py │ ├── binsearchtree.py │ ├── bintree.py │ ├── bloomfilter.py │ ├── bplustree.py │ ├── btree.py │ ├── hashtable.py │ ├── lru.py │ ├── rbtree.py │ ├── sort │ │ ├── __init__.py │ │ ├── base.py │ │ └── quicksort.py │ └── trie.py ├── cache_wrap.py ├── cli │ ├── __init__.py │ ├── cli.py │ └── entry.py ├── compress.py ├── concurrent.py ├── consul.py ├── db.py ├── etcd.py ├── eventloop.py ├── gossip │ ├── __init__.py │ └── core.py ├── http │ ├── __init__.py │ ├── factory.py │ ├── middleware.py │ ├── rawserver.py │ ├── request.py │ ├── request_stream.py │ ├── response.py │ ├── router.py │ ├── server.py │ ├── status.py │ └── transport.py ├── log.py ├── orm.py ├── redislock.py ├── rpc │ ├── __init__.py │ ├── client.py │ ├── compress.py │ ├── const.py │ ├── discovery.py │ ├── encrypt.py │ ├── exception.py │ ├── method.py │ ├── protocal.py │ ├── serialize.py │ ├── server.py │ └── transport.py ├── table_writer.py ├── util.py └── wrap.py ├── test.py └── upload.py /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | dist 3 | pub.sh 4 | sunyata.egg-info 5 | __pycache__ 6 | .idea 7 | MANIFEST.in 8 | setup.cfg 9 | *.pyc 10 | upload.sh 11 | *.log 12 | restart.sh 13 | *.ini 14 | test_server.py 15 | test_client.py 16 | test_perf.py 17 | todo.txt 18 | test*.py 19 | .vscode 20 | myservice.py 21 | TODO -------------------------------------------------------------------------------- /DETAIL.md: -------------------------------------------------------------------------------- 1 | #### 目录 2 | - [简介](#简介) 3 | - [特性](#特性) 4 | - [安装](#安装) 5 | - [快速开始](#快速开始) 6 | - [详细介绍](#详细介绍) 7 | - [TCP RPC 服务端](#tcp-rpc-服务端) 8 | - [TCP RPC 客户端](#tcp-rpc-客户端) 9 | - [指定多个服务端地址](#指定多个服务端地址) 10 | - [HTTP RPC 服务端](#http-rpc-服务端) 11 | - [HTTP RPC Client](#http-rpc-client) 12 | - [UDP RPC 服务端](#udp-rpc-服务端) 13 | - [UDP RPC 客户端](#udp-rpc-客户端) 14 | - [服务发现](#服务发现) 15 | - [健康检查](#健康检查) 16 | - [快速开始](#快速开始-1) 17 | - [完整的服务端示例 (UDP/HTTP调用方式相同)](#完整的服务端示例-udphttp调用方式相同) 18 | - [完整的客户端示例（UDP/HTTP调用方式相同）](#完整的客户端示例udphttp调用方式相同) 19 | - [数据压缩](#数据压缩) 20 | - [内置Web框架](#内置web框架) 21 | 22 | ## 简介 23 | sunyata是一个Python3 RPC框架，client和server既可以直连，也可以通过Consul做服务注册发现。 24 | 25 | ## 特性 26 | - 像本地函数一样调用 27 | - 使用简单，用户只需要关注业务即可 28 | - HTTP/UDP/TCP 全协议支持 29 | - 支持异步 async/await 30 | - 支持服务注册、发现 31 | 32 | ## 安装 33 | Python 版本 >= 3.6 34 | ``` 35 | pip install sunyata 36 | ``` 37 | 38 | ## 快速开始 39 | 创建文件myservice.py 40 | ```python 41 | from sunyata.rpc import rpc 42 | 43 | @rpc 44 | def hello(name): 45 | return 'hello ' + name 46 | ``` 47 | 启动： 48 | ```shell 49 | sunyata --run myservice 50 | ``` 51 | ![pic.png](./docs/pic.png) 52 | 53 | ## 详细介绍 54 | 55 | ### TCP RPC 服务端 56 | 下面是一个TCP协议的服务端例子。 57 | - 创建一个TcpRpcServer对象, 指定服务端监听地址和端口 58 | - 通过@rpc装饰器注册需要被客户端请求的方法 59 | - 调用serve()方法，开始处理客户端请求 60 | ```python 61 | from sunyata.rpc.server import TcpRpcServer, rpc 62 | import asyncio 63 | 64 | @rpc 65 | class TestService: 66 | 67 | def hello(self, name): 68 | return "Hello, {}!".format(name) 69 | 70 | async def add(self, a, b, c): 71 | asyncio.sleep(1) 72 | return a + b + c 73 | 74 | @rpc 75 | def hello(name): 76 | return "Hello, {}!".format(name) 77 | 78 | server = TcpRpcServer('0.0.0.0', 9988) 79 | server.serve() 80 | ``` 81 | 82 | ### TCP RPC 客户端 83 | - 创建TcpRpcClient对象，指定RPC服务端地址 84 | - 通过call()方法，指定服务端方法名称和参数（注意：如果方法名不存在，或者服务端未调用@rpc装饰器注册，那么call()方法将抛出异常） 85 | - call() 方法的返回值和在本地调用一样，原来是什么返回类型，就还是什么（例如返回字典、列表、对象甚至内置类型，经过序列化后，不会发生改变） 86 | ```python 87 | from sunyata.rpc.client import TcpRpcClient 88 | 89 | cli = TcpRpcClient('127.0.0.1', 9988, timeout = 2) 90 | 91 | resp = cli.TestService.hello('xiaoming') 92 | print(resp) 93 | 94 | #或者使用call方法 95 | resp = cli.call('TestService.add', a=1, b=2, c=3) 96 | print(resp) 97 | 98 | resp = cli.call('hello', name = 'xiaoming') 99 | print(resp) 100 | ``` 101 | 102 | ### 指定多个服务端地址 103 | - 通过servers参数，你可以创建一个指定多个服务端地址的client对象，默认采用轮询的负载均衡策略，将请求转发到多个server上，如果请求其中一个server出现了失败，那么会自动重试。 104 | ```python 105 | from sunyata.rpc.client import TcpRpcClient 106 | 107 | c = TcpRpcClient(servers = ['127.0.0.1:9988', '127.0.0.1:9989']) 108 | resp = c.call('hello', 'zhangsan') 109 | print(resp) 110 | ``` 111 | 112 | ### HTTP RPC 服务端 113 | 底层是基于内置web框架实现的，使用起来非常简单，和TcpRpcServer的用法类似: 114 | ```python 115 | from sunyata.rpc.server import HttpRpcServer, rpc 116 | 117 | @rpc 118 | def sayHello(name): 119 | return 'hello ' + name 120 | 121 | s = HttpRpcServer('0.0.0.0', 9988, workers=1) 122 | s.serve() 123 | ``` 124 | 125 | ### HTTP RPC Client 126 | 客户端使用对应的HttpRpcClient对象: 127 | ```python 128 | from sunyata.rpc.client import HttpRpcClient 129 | 130 | c = HttpRpcClient('127.0.0.1', 9988) 131 | resp = c.call('sayHello', 'zhangsan') 132 | print(resp) 133 | ``` 134 | 135 | ### UDP RPC 服务端 136 | 将TcpRpcServer替换为UdpRpcServer即可。 137 | - 创建UdpRpcServer对象，指定监听的地址和端口 138 | - 调用regist()方法，将需要被客户端请求的方法注册进去 139 | - 调用serve()方法开始处理客户端请求 140 | - 返回的内容和调用本地方法没有差别，框架内部通过序列化和反序列化，将数据转化为程序内的对象（字典、列表、内置类型、各种类对象等等） 141 | ```python 142 | from sunyata.rpc.server import UdpRpcServer, rpc 143 | 144 | @rpc 145 | def sayHello(name): 146 | return 'hello ' + name 147 | 148 | server = UdpRpcServer('0.0.0.0', 9988) 149 | server.serve() 150 | ``` 151 | ### UDP RPC 客户端 152 | - 创建UdpRpcClient对象，指定服务端地址和端口 153 | - 调用call()方法，并指定服务端的方法名称和参数 154 | - 返回的内容和调用本地方法没有差别，框架内部通过序列化和反序列化，将数据转化为程序内的对象（字典、列表、内置类型、各种类对象等等） 155 | ```python 156 | from sunyata.rpc.client import UdpRpcClient 157 | cli = UdpRpcClient('127.0.0.1', 9988) 158 | resp = cli.call('sayHello', name = 'xiaoming' ) 159 | print(resp) 160 | ``` 161 | 162 | ## 服务发现 163 | 除了客户端与服务端直连，也支持服务注册发现（客户端与服务端直连的例子，请参考上面的TcpRpcServer部分）。 164 | 目前仅支持基于Consul的服务发现，未来计划支持etcd。下面的例子以TCP为例。 165 | 166 | 167 | ### 健康检查 168 | 基于Consul的Check机制，服务注册后，自动添加一个定期的检查任务。默认为TCP端口检查，支持TCP/HTTP RPC服务端，UDP服务端暂不支持。一旦服务进程挂掉，那么客户端会请求到其他健康的服务端节点上。 169 | 170 | ### 快速开始 171 | - 第一步，你需要定义一个DiscoverConfig对象。 172 | 指定用于服务注册发现的Consul的地址和端口。同时通过serviceName参数指定一个全局唯一的服务名称（用于标记服务端服务）。同时指定服务端监听的地址和端口。 173 | 174 | ```python 175 | from sunyata.rpc.discovery import DiscoveryConfig 176 | 177 | disconf = DiscoveryConfig( 178 | consulHost = '192.168.19.103', 179 | consulPort = 8500, 180 | serviceName = 'test-rpc-server', 181 | serviceHost = local_ip(), 182 | servicePort = 9988 183 | ) 184 | ``` 185 | > 说明: 186 | > 1.consulHost 和 consulPort 参数指定Consul的地址和端口 187 | > 2.ServiceName 参数用于标记服务端名称，并通过服务名称进行服务发现，需要保证全局唯一 188 | > 3.serviceHost和servicePort参数指定服务端监听的端口和地址 189 | 190 | - 第二步、调用setDiscoverConfig()方法将DiscoveryConfig对象传入 191 | - 第三步，调用serve()方法，开始处理请求 192 | ```python 193 | s = TcpRpcServer('0.0.0.0', 9988) 194 | s.regist(sayHello) 195 | disconf = DiscoveryConfig( 196 | consulHost = '192.168.19.103', 197 | consulPort = 8500, 198 | serviceName = 'test-rpc-server', 199 | serviceHost = local_ip(), 200 | servicePort = 9988 201 | ) 202 | s.setDiscoverConfig(disconf) 203 | s.serve() 204 | ``` 205 | ### 完整的服务端示例 (UDP/HTTP调用方式相同) 206 | ```python 207 | from sunyata.rpc.server import TcpRpcServer, rpc 208 | from sunyata.rpc.discovery import DiscoveryConfig 209 | from sunyata.util import local_ip 210 | 211 | @rpc 212 | def sayHello(name): 213 | return 'hello ' + name 214 | 215 | disconf = DiscoveryConfig( 216 | consulHost = '192.168.19.103', 217 | consulPort = 8500, 218 | consulToken = 'd8ba9c48-c01a-a78e-ce8d-b65593a56419', 219 | serviceName = 'UserService', 220 | serviceHost = local_ip(), 221 | servicePort = 9988, 222 | ) 223 | 224 | server = TcpRpcServer('0.0.0.0', 9988) 225 | server.setDiscoverConfig(disconf) 226 | server.serve() 227 | ``` 228 | 229 | ### 完整的客户端示例（UDP/HTTP调用方式相同） 230 | - 创建DiscoveryConfig对象，指定Consul的地址端口（serviceName参数和服务端的保持一致，且全局唯一） 231 | - 调用setDiscoveryConfig()方法传入服务发现配置 232 | ```python 233 | from sunyata.rpc.client import TcpRpcClient 234 | from sunyata.rpc.discovery import DiscoveryConfig 235 | cli = TcpRpcClient() 236 | disconf = DiscoveryConfig( 237 | consulHost= '192.168.19.103', 238 | consulPort= 8500, 239 | serviceName='test-rpc-server' 240 | ) 241 | cli.setDiscoveryConfig(disconf) 242 | resp = cli.call('sayHello', name = 'mary') 243 | print(resp) 244 | ``` 245 | 246 | ## 数据压缩 247 | 默认采用lz4进行压缩、解压缩（经过测试，它的压缩效果和gzip, zlib比较接近，压缩、解压缩性能是zlib的10倍左右）。 248 | 在数据传输大于4KB时，自动开启进行压缩。对端根据一个标记位进行判断，自动进行解压缩处理（或不处理，未经过压缩的情况）。开发者无需关心 249 | 数据的压缩、解压缩过程，经过测试对性能的影响极低（由于采用了level1级别的压缩），最高可减少75%的网络IO。 250 | 251 | ## 内置Web框架 252 | sunyata也可以作为一个web框架来使用, HttpRpcServer在此基础上构建。 253 | ```python 254 | from sunyata.http.server import HttpServer, route 255 | 256 | @route('/hello', methods=['GET']) 257 | def hello(request): 258 | name = request.data.get('name', '') 259 | return 'Hello ' + name 260 | 261 | hs = HttpServer(bind='0.0.0.0', port=9989) 262 | hs.serve() 263 | ``` 264 | 265 | 266 | [![Stargazers repo roster for @lycclsltt/sunyata](https://reporoster.com/stars/lycclsltt/sunyata)](https://github.com/lycclsltt/sunyata/stargazers) 267 | 268 | [![Forkers repo roster for @lycclsltt/sunyata](https://reporoster.com/forks/lycclsltt/sunyata)](https://github.com/lycclsltt/sunyata/network/members) 269 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9.17 2 | RUN mkdir -p /sunyata 3 | COPY . /sunyata 4 | WORKDIR /sunyata 5 | RUN python setup.py install -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 lycclsltt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | #### 目录 2 | - [简介](#简介) 3 | - [特性](#特性) 4 | - [安装](#安装) 5 | - [快速开始](#快速开始) 6 | - [详细介绍](#详细介绍) 7 | - [TCP RPC 服务端](#tcp-rpc-服务端) 8 | - [TCP RPC 客户端](#tcp-rpc-客户端) 9 | - [指定多个服务端地址](#指定多个服务端地址) 10 | - [HTTP RPC 服务端](#http-rpc-服务端) 11 | - [HTTP RPC 客户端](#http-rpc-客户端) 12 | - [UDP RPC 服务端](#udp-rpc-服务端) 13 | - [UDP RPC 客户端](#udp-rpc-客户端) 14 | - [服务发现](#服务发现) 15 | - [健康检查](#健康检查) 16 | - [快速开始](#快速开始-1) 17 | - [完整的服务端示例 (UDP/HTTP调用方式相同)](#完整的服务端示例-udphttp调用方式相同) 18 | - [完整的客户端示例（UDP/HTTP调用方式相同）](#完整的客户端示例udphttp调用方式相同) 19 | - [数据压缩](#数据压缩) 20 | - [内置Web框架](#内置web框架) 21 | - [微服务案例](#微服务案例) 22 | 23 | ## 简介 24 | sunyata是一个Python3 RPC框架，client和server既可以直连，也可以通过Consul或ETCD做服务注册发现。 25 | 26 | ## 特性 27 | - 像本地函数一样调用 28 | - 使用简单，用户只需要关注业务即可 29 | - 支持HTTP/UDP/TCP协议 30 | - 支持异步async/await 31 | - 支持通过consul或etcd的服务注册发现 32 | - 支持所有数据类型，包括自定义的类对象等都可作为参数 33 | 34 | ## 安装 35 | Python 版本 >= 3.6 36 | ``` 37 | pip install sunyata 38 | ``` 39 | 40 | ## 快速开始 41 | 创建文件myservice.py 42 | ```python 43 | from sunyata.rpc import rpc 44 | 45 | @rpc 46 | def hello(name): 47 | return 'hello ' + name 48 | ``` 49 | 启动： 50 | ```shell 51 | sunyata --run myservice 52 | ``` 53 | 54 | ## 详细介绍 55 | 56 | ### TCP RPC 服务端 57 | 下面是一个TCP协议的服务端例子。 58 | - 创建一个TcpRpcServer对象, 指定服务端监听地址和端口 59 | - 通过@rpc装饰器注册需要被客户端请求的方法 60 | - 调用serve()方法，开始处理客户端请求 61 | ```python 62 | from sunyata.rpc.server import TcpRpcServer, rpc 63 | import asyncio 64 | 65 | @rpc 66 | class TestService: 67 | 68 | def hello(self, name): 69 | return "Hello, {}!".format(name) 70 | 71 | async def add(self, a, b, c): 72 | asyncio.sleep(1) 73 | return a + b + c 74 | 75 | @rpc 76 | def hello(name): 77 | return "Hello, {}!".format(name) 78 | 79 | server = TcpRpcServer('0.0.0.0', 9988) 80 | server.serve() 81 | ``` 82 | 83 | ### TCP RPC 客户端 84 | - 创建TcpRpcClient对象，指定RPC服务端地址 85 | - 通过call()方法，指定服务端方法名称和参数（注意：如果方法名不存在，或者服务端未调用@rpc装饰器注册，那么call()方法将抛出异常） 86 | - call() 方法的返回值和在本地调用一样，原来是什么返回类型，就还是什么（例如返回字典、列表、对象甚至内置类型，经过序列化后，不会发生改变） 87 | ```python 88 | from sunyata.rpc.client import TcpRpcClient 89 | 90 | cli = TcpRpcClient('127.0.0.1', 9988, timeout = 2) 91 | 92 | resp = cli.TestService.hello('xiaoming') 93 | print(resp) 94 | 95 | #或者使用call方法 96 | resp = cli.call('TestService.add', a=1, b=2, c=3) 97 | print(resp) 98 | 99 | resp = cli.call('hello', name = 'xiaoming') 100 | print(resp) 101 | ``` 102 | 103 | ### 指定多个服务端地址 104 | - 通过servers参数，你可以创建一个指定多个服务端地址的client对象，默认采用轮询的负载均衡策略，将请求转发到多个server上，如果请求其中一个server出现了失败，那么会自动重试。 105 | ```python 106 | from sunyata.rpc.client import TcpRpcClient 107 | 108 | c = TcpRpcClient(servers = ['127.0.0.1:9988', '127.0.0.1:9989']) 109 | resp = c.call('hello', 'zhangsan') 110 | print(resp) 111 | ``` 112 | 113 | ### HTTP RPC 服务端 114 | 底层是基于内置web框架实现的，使用起来非常简单，和TcpRpcServer的用法类似: 115 | ```python 116 | from sunyata.rpc.server import HttpRpcServer, rpc 117 | 118 | @rpc 119 | def sayHello(name): 120 | return 'hello ' + name 121 | 122 | s = HttpRpcServer('0.0.0.0', 9988, workers=1) 123 | s.serve() 124 | ``` 125 | 126 | ### HTTP RPC 客户端 127 | 客户端使用对应的HttpRpcClient对象: 128 | ```python 129 | from sunyata.rpc.client import HttpRpcClient 130 | 131 | c = HttpRpcClient('127.0.0.1', 9988) 132 | resp = c.call('sayHello', 'zhangsan') 133 | print(resp) 134 | ``` 135 | 136 | ### UDP RPC 服务端 137 | 将TcpRpcServer替换为UdpRpcServer即可。 138 | - 创建UdpRpcServer对象，指定监听的地址和端口 139 | - 调用regist()方法，将需要被客户端请求的方法注册进去 140 | - 调用serve()方法开始处理客户端请求 141 | - 返回的内容和调用本地方法没有差别，框架内部通过序列化和反序列化，将数据转化为程序内的对象（字典、列表、内置类型、各种类对象等等） 142 | ```python 143 | from sunyata.rpc.server import UdpRpcServer, rpc 144 | 145 | @rpc 146 | def sayHello(name): 147 | return 'hello ' + name 148 | 149 | server = UdpRpcServer('0.0.0.0', 9988) 150 | server.serve() 151 | ``` 152 | ### UDP RPC 客户端 153 | - 创建UdpRpcClient对象，指定服务端地址和端口 154 | - 调用call()方法，并指定服务端的方法名称和参数 155 | - 返回的内容和调用本地方法没有差别，框架内部通过序列化和反序列化，将数据转化为程序内的对象（字典、列表、内置类型、各种类对象等等） 156 | ```python 157 | from sunyata.rpc.client import UdpRpcClient 158 | cli = UdpRpcClient('127.0.0.1', 9988) 159 | resp = cli.call('sayHello', name = 'xiaoming' ) 160 | print(resp) 161 | ``` 162 | 163 | ## 服务发现 164 | 除了客户端与服务端直连，也支持服务注册发现（客户端与服务端直连的例子，请参考上面的TcpRpcServer部分）。 165 | 目前支持基于Consul 或 etcd 进行服务注册发现， 下面的例子先以Consul为例。 166 | 167 | 168 | ### 基于Consul的服务注册发现 169 | 基于Consul的Check机制，服务注册后，自动添加一个定期的检查任务。默认为TCP端口检查，支持TCP/HTTP RPC服务端，UDP服务端暂不支持。一旦服务进程挂掉，那么客户端会请求到其他健康的服务端节点上。 170 | 171 | ### 快速开始 172 | - 第一步，你需要定义一个DiscoverConfig对象。 173 | 指定用于服务注册发现的Consul的地址和端口。同时通过serviceName参数指定一个全局唯一的服务名称（用于标记服务端服务）。同时指定服务端监听的地址和端口。 174 | 175 | ```python 176 | from sunyata.rpc.discovery import DiscoveryConfig 177 | 178 | disconf = DiscoveryConfig( 179 | consulHost = '192.168.19.103', 180 | consulPort = 8500, 181 | serviceName = 'test-rpc-server', 182 | serviceHost = local_ip(), 183 | servicePort = 9988 184 | ) 185 | ``` 186 | > 说明: 187 | > 1.consulHost 和 consulPort 参数指定Consul的地址和端口 188 | > 2.ServiceName 参数用于标记服务端名称，并通过服务名称进行服务发现，需要保证全局唯一 189 | > 3.serviceHost和servicePort参数指定服务端监听的端口和地址 190 | 191 | - 第二步、调用setDiscoverConfig()方法将DiscoveryConfig对象传入 192 | - 第三步，调用serve()方法，开始处理请求 193 | ```python 194 | s = TcpRpcServer('0.0.0.0', 9988) 195 | s.regist(sayHello) 196 | disconf = DiscoveryConfig( 197 | consulHost = '192.168.19.103', 198 | consulPort = 8500, 199 | serviceName = 'test-rpc-server', 200 | serviceHost = local_ip(), 201 | servicePort = 9988 202 | ) 203 | s.setDiscoverConfig(disconf) 204 | s.serve() 205 | ``` 206 | ### 完整的服务端示例 (UDP/HTTP调用方式相同) 207 | ```python 208 | from sunyata.rpc.server import TcpRpcServer, rpc 209 | from sunyata.rpc.discovery import DiscoveryConfig 210 | from sunyata.util import local_ip 211 | 212 | @rpc 213 | def sayHello(name): 214 | return 'hello ' + name 215 | 216 | disconf = DiscoveryConfig( 217 | consulHost = '192.168.19.103', 218 | consulPort = 8500, 219 | consulToken = 'd8ba9c48-c01a-a78e-ce8d-b65593a56419', 220 | serviceName = 'UserService', 221 | serviceHost = local_ip(), 222 | servicePort = 9988, 223 | ) 224 | 225 | server = TcpRpcServer('0.0.0.0', 9988) 226 | server.setDiscoverConfig(disconf) 227 | server.serve() 228 | ``` 229 | 230 | ### 完整的客户端示例（UDP/HTTP调用方式相同） 231 | - 创建DiscoveryConfig对象，指定Consul的地址端口（serviceName参数和服务端的保持一致，且全局唯一） 232 | - 调用setDiscoveryConfig()方法传入服务发现配置 233 | ```python 234 | from sunyata.rpc.client import TcpRpcClient 235 | from sunyata.rpc.discovery import DiscoveryConfig 236 | cli = TcpRpcClient() 237 | disconf = DiscoveryConfig( 238 | consulHost= '192.168.19.103', 239 | consulPort= 8500, 240 | serviceName='test-rpc-server' 241 | ) 242 | cli.setDiscoveryConfig(disconf) 243 | resp = cli.call('sayHello', name = 'mary') 244 | print(resp) 245 | ``` 246 | 247 | ### 基于Etcd的服务注册发现 248 | > 说明: 249 | > 1.etcdHost 和 etcdPort 参数指定etcd的地址和端口 250 | > 2.ServiceName 参数用于标记服务端名称，并通过服务名称进行服务发现，需要保证全局唯一 251 | > 3.serviceHost和servicePort参数指定服务端监听的端口和地址 252 | 253 | - 第二步、调用setDiscoverConfig()方法将DiscoveryConfig对象传入 254 | - 第三步，调用serve()方法，开始处理请求 255 | 256 | ### 完整的服务端示例 257 | ```python 258 | from sunyata.rpc.server import HttpRpcServer 259 | from sunyata.rpc.discovery import DiscoveryConfig 260 | from sunyata.util import local_ip 261 | 262 | def sayHello(name): 263 | return 'hello ' + name 264 | 265 | server = HttpRpcServer('0.0.0.0', 10031) 266 | disconf = DiscoveryConfig( 267 | etcdHost='192.168.19.103', 268 | etcdPort=2379, 269 | serviceName = 'test-http-rpc-server-etcd', 270 | serviceHost = local_ip(), 271 | servicePort = 10031 272 | ) 273 | server.setDiscoverConfig(disconf) 274 | server.regist(sayHello) 275 | server.serve() 276 | ``` 277 | ### 完整的客户端示例 278 | ```python 279 | from sunyata.rpc.client import HttpRpcClient 280 | from sunyata.rpc.discovery import DiscoveryConfig 281 | 282 | client = HttpRpcClient() 283 | disconf = DiscoveryConfig( 284 | etcdHost='192.168.19.103', 285 | etcdPort=2379, 286 | serviceName = 'myservice' 287 | ) 288 | client.setDiscoveryConfig(disconf) 289 | resp = client.sayHello('xiaoming') 290 | assert (resp == 'hello xiaoming') 291 | ``` 292 | 293 | 294 | 295 | 296 | 297 | ## 数据压缩 298 | 默认采用lz4进行压缩、解压缩（经过测试，它的压缩效果和gzip, zlib比较接近，压缩、解压缩性能是zlib的10倍左右）。 299 | 在数据传输大于4KB时，自动开启进行压缩。对端根据一个标记位进行判断，自动进行解压缩处理（或不处理，未经过压缩的情况）。开发者无需关心 300 | 数据的压缩、解压缩过程，经过测试对性能的影响极低（由于采用了level1级别的压缩），最高可减少75%的网络IO。 301 | 302 | ## 内置Web框架 303 | sunyata也可以作为一个web框架来使用, HttpRpcServer在此基础上构建。 304 | ```python 305 | from sunyata.http.server import HttpServer, route 306 | 307 | @route('/hello', methods=['GET']) 308 | def hello(request): 309 | name = request.data.get('name', '') 310 | return 'Hello ' + name 311 | 312 | hs = HttpServer(bind='0.0.0.0', port=9989) 313 | hs.serve() 314 | ``` 315 | 316 | 317 | ## 中间件 318 | 可以通过继承Middleware类，重写handle方法，添加中间件。所有请求的都会先经过多个中间件，可用于身份验证的场景，下面是一个例子。 319 | 320 | ```python 321 | from sunyata.http.server import HttpServer, route 322 | from sunyata.http.middleware import Middleware 323 | import logging 324 | 325 | 326 | class LogMiddleware(Middleware): 327 | 328 | def handle(self, request): 329 | logging.info('path:' + request.uri + ' method:' + request.method) 330 | 331 | 332 | class AuthMiddleware(Middleware): 333 | 334 | def handle(self, request): 335 | if request.headers.get('token') != 'abc': 336 | self.abort(403, 'invalid token') 337 | 338 | 339 | @route('/api/v1/getUserName', methods=['POST', 'GET']) 340 | def getUserName(request): 341 | return 'tom' 342 | 343 | app = HttpServer(port=9990, accessLog=True) 344 | app.middlewares = [ 345 | LogMiddleware(), 346 | AuthMiddleware(), 347 | ] 348 | app.serve() 349 | ``` 350 | 351 | ## 微服务案例 352 | 下面是一个dns查询场景的微服务案例,dns_service.py负责提供底层dns查询服务,dns_web.py启动http api负责提供对外接口。dns_web与dns_service之间通过rpc进行通信。 353 | 354 | dns_service.py 355 | ```python 356 | from sunyata.rpc.server import HttpRpcServer, rpc 357 | 358 | @rpc 359 | class DnsService(object): 360 | 361 | def query(self, domain): 362 | print('domain', domain) 363 | return domain + ' A IN 192.168.1.1' 364 | 365 | rpcServer = HttpRpcServer(host='0.0.0.0', port=9988) 366 | rpcServer.serve() 367 | ``` 368 | 369 | dns_web.py 370 | ```python 371 | from sunyata.http.server import HttpServer, route 372 | from sunyata.rpc.client import HttpRpcClient 373 | 374 | @route('/query', methods=['GET']) 375 | def query(request): 376 | domain = request.data.get('domain') 377 | cli = HttpRpcClient('127.0.0.1', 9988) 378 | ip = cli.DnsService.query(domain) 379 | return ip 380 | 381 | app = HttpServer() 382 | app.serve() 383 | ``` 384 | 385 | 请求dns_web curl 'http://127.0.0.1:9989/query?domain=www.a.com' 返回 www.a.com A IN 192.168.1.1 386 | 387 | ### 局限性 388 | 由于底层通过python特有的pickle方式进行序列化，目前只支持python语言编写的服务之间的rpc通信，暂不支持其他语言。好处是不需要 389 | 编写protobuf文件。 390 | 391 | [![Stargazers repo roster for @lycclsltt/sunyata](https://reporoster.com/stars/lycclsltt/sunyata)](https://github.com/lycclsltt/sunyata/stargazers) 392 | 393 | [![Forkers repo roster for @lycclsltt/sunyata](https://reporoster.com/forks/lycclsltt/sunyata)](https://github.com/lycclsltt/sunyata/network/members) 394 | -------------------------------------------------------------------------------- /examples/dns_service.py: -------------------------------------------------------------------------------- 1 | from sunyata.rpc.server import HttpRpcServer, rpc 2 | 3 | @rpc 4 | class DnsService(object): 5 | 6 | def query(self, domain): 7 | print('domain', domain) 8 | return domain + ' A IN 192.168.1.1' 9 | 10 | rpcServer = HttpRpcServer(host='0.0.0.0', port=9988) 11 | rpcServer.serve() -------------------------------------------------------------------------------- /examples/dns_web.py: -------------------------------------------------------------------------------- 1 | from sunyata.http.server import HttpServer, route 2 | from sunyata.rpc.client import HttpRpcClient 3 | 4 | @route('/query', methods=['GET']) 5 | def query(request): 6 | domain = request.data.get('domain') 7 | cli = HttpRpcClient('127.0.0.1', 9988) 8 | ip = cli.DnsService.query(domain) 9 | return ip 10 | 11 | app = HttpServer() 12 | app.serve() -------------------------------------------------------------------------------- /examples/gossip/node1.py: -------------------------------------------------------------------------------- 1 | from sunyata.gossip.core import GossipCore, GossipSyncedCallback, GossipMsg 2 | import time 3 | 4 | 5 | class MyCallback(GossipSyncedCallback): 6 | 7 | def __init__(self): 8 | self.m = {} 9 | 10 | def synced(self, msg): 11 | self.m[msg.k] = msg 12 | 13 | callbackClass = MyCallback() 14 | gc = GossipCore(bindHost='127.0.0.1', bindPort=8082, memberAddrs=[{'host':'127.0.0.1', 'port':8083}, {'host':'127.0.0.1', 'port':8082}, {'host':'127.0.0.1', 'port':8081}], syncedCallback=callbackClass) 15 | gc.start() 16 | 17 | time.sleep(3) 18 | 19 | for i in range(3): 20 | time.sleep(1) 21 | gc.broadcast(GossipMsg('node1_' + str(i), str(i), 0)) 22 | 23 | time.sleep(5) 24 | s = "" 25 | for k, msg in callbackClass.m.items(): 26 | s = s + msg.k + '-' + msg.v 27 | 28 | print(s) 29 | -------------------------------------------------------------------------------- /examples/gossip/node2.py: -------------------------------------------------------------------------------- 1 | from sunyata.gossip.core import GossipCore, GossipSyncedCallback, GossipMsg 2 | import time 3 | 4 | class MyCallback(GossipSyncedCallback): 5 | 6 | def __init__(self): 7 | self.m = {} 8 | 9 | def synced(self, msg): 10 | self.m[msg.k] = msg 11 | 12 | callbackClass = MyCallback() 13 | gc = GossipCore(bindHost='127.0.0.1', bindPort=8083, memberAddrs=[{'host':'127.0.0.1', 'port':8083}, {'host':'127.0.0.1', 'port':8082}, {'host':'127.0.0.1', 'port':8081}], syncedCallback=callbackClass) 14 | gc.start() 15 | 16 | time.sleep(3) 17 | 18 | for i in range(3): 19 | time.sleep(1) 20 | gc.broadcast(GossipMsg('node2_' + str(i), str(i), 0)) 21 | 22 | time.sleep(5) 23 | s = "" 24 | for k, msg in callbackClass.m.items(): 25 | s = s + msg.k + '-' + msg.v 26 | 27 | print(s) 28 | 29 | -------------------------------------------------------------------------------- /examples/gossip/node3.py: -------------------------------------------------------------------------------- 1 | from sunyata.gossip.core import GossipCore, GossipSyncedCallback, GossipMsg 2 | import time 3 | 4 | class MyCallback(GossipSyncedCallback): 5 | 6 | def __init__(self): 7 | self.m = {} 8 | 9 | def synced(self, msg): 10 | self.m[msg.k] = msg 11 | 12 | callbackClass = MyCallback() 13 | gc = GossipCore(bindHost='127.0.0.1', bindPort=8081, memberAddrs=[{'host':'127.0.0.1', 'port':8083}, {'host':'127.0.0.1', 'port':8082}, {'host':'127.0.0.1', 'port':8081}], syncedCallback=callbackClass) 14 | gc.start() 15 | 16 | time.sleep(3) 17 | 18 | for i in range(3): 19 | time.sleep(1) 20 | gc.broadcast(GossipMsg('node3_' + str(i), str(i), 0)) 21 | 22 | time.sleep(5) 23 | s = "" 24 | for k, msg in callbackClass.m.items(): 25 | s = s + msg.k + '-' + msg.v 26 | 27 | print(s) 28 | 29 | -------------------------------------------------------------------------------- /examples/http_rpc_client.py: -------------------------------------------------------------------------------- 1 | from sunyata.rpc.client import HttpRpcClient 2 | 3 | cli = HttpRpcClient('127.0.0.1', 9998) 4 | res = cli.hello('world') 5 | print(res) -------------------------------------------------------------------------------- /examples/http_rpc_server.py: -------------------------------------------------------------------------------- 1 | from sunyata.rpc.server import HttpRpcServer, rpc 2 | 3 | @rpc 4 | def hello(name): 5 | return 'hello ' + name 6 | 7 | app = HttpRpcServer('0.0.0.0', 9998) 8 | app.serve() -------------------------------------------------------------------------------- /examples/tcp_rpc_client.py: -------------------------------------------------------------------------------- 1 | from sunyata.rpc.client import TcpRpcClient 2 | 3 | cli = TcpRpcClient('127.0.0.1', 9998) 4 | res = cli.hello('world') 5 | print(res) -------------------------------------------------------------------------------- /examples/tcp_rpc_server.py: -------------------------------------------------------------------------------- 1 | from sunyata.rpc.server import TcpRpcServer, rpc 2 | 3 | @rpc 4 | def hello(name): 5 | return 'hello ' + name 6 | 7 | app = TcpRpcServer('0.0.0.0', 9998) 8 | app.serve() -------------------------------------------------------------------------------- /examples/udp_rpc_client.py: -------------------------------------------------------------------------------- 1 | from sunyata.rpc.client import UdpRpcClient 2 | 3 | cli = UdpRpcClient('127.0.0.1', 9998) 4 | res = cli.hello('world') 5 | print(res) -------------------------------------------------------------------------------- /examples/udp_rpc_server.py: -------------------------------------------------------------------------------- 1 | from sunyata.rpc.server import UdpRpcServer, rpc 2 | 3 | @rpc 4 | def hello(name): 5 | return 'hello ' + name 6 | 7 | app = UdpRpcServer('0.0.0.0', 9998) 8 | app.serve() -------------------------------------------------------------------------------- /exec_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | python3 -W ignore test.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | DEFINE_VERSION = '0.0.46' 2 | from setuptools import setup 3 | 4 | requireList = [ 5 | 'lz4==3.1.3', 6 | 'ujson==1.35', 7 | 'uvloop==0.17.0', 8 | 'uvicorn==0.18.0', 9 | 'aiohttp==3.8.4', 10 | 'requests==2.30.0', 11 | #'PyMySQL==0.10.1', 12 | #'DBUtils==1.3', 13 | #'dataclasses-json==0.5.13', 14 | ] 15 | 16 | setup( 17 | name='sunyata', 18 | version=DEFINE_VERSION, 19 | description='Light, simple, asynchronous RPC framework for Python', 20 | author='tank', 21 | license='MIT', 22 | platforms="any", 23 | install_requires=requireList, 24 | classifiers=[ 25 | 'Programming Language :: Python :: 3.7', 26 | 'Programming Language :: Python :: 3.8', 27 | 'Programming Language :: Python :: 3.9', 28 | 'Programming Language :: Python :: 3.10', 29 | 'Programming Language :: Python :: 3.11', 30 | 'Programming Language :: Python :: 3.12', 31 | ], 32 | keywords='sunyata', 33 | packages=[ 34 | 'sunyata', 35 | 'sunyata/rpc', 36 | 'sunyata/algorithm', 37 | 'sunyata/cli', 38 | 'sunyata/http' 39 | ], 40 | include_package_data=True, 41 | entry_points = { 42 | 'console_scripts' : [ 43 | 'sunyata=sunyata.cli.entry:main' 44 | ] 45 | } 46 | ) -------------------------------------------------------------------------------- /sunyata/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lycclsltt/sunyata/a1e09860c65c4c65a50ea4d7b0034c567c97088c/sunyata/__init__.py -------------------------------------------------------------------------------- /sunyata/algorithm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lycclsltt/sunyata/a1e09860c65c4c65a50ea4d7b0034c567c97088c/sunyata/algorithm/__init__.py -------------------------------------------------------------------------------- /sunyata/algorithm/avlsearchtree.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | 3 | from sunyata.algorith.avltree import AVLTree 4 | 5 | class AVLSearchTree(AVLTree): 6 | pass -------------------------------------------------------------------------------- /sunyata/algorithm/avltree.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | 3 | from sunyata.algorithm.bintree import BinTree 4 | 5 | class AVLTree(BinTree): 6 | pass -------------------------------------------------------------------------------- /sunyata/algorithm/binsearchtree.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | 3 | from sunyata.algorithm.bintree import BinTree 4 | 5 | class BinSearchTree(BinTree): 6 | pass -------------------------------------------------------------------------------- /sunyata/algorithm/bintree.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | 3 | class BinTree(object): 4 | pass -------------------------------------------------------------------------------- /sunyata/algorithm/bloomfilter.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | ''' 3 | Usage: 4 | bloom_filter = BloomFilter() 5 | urls = ['www.baidu.com','mathpretty.com','sina.com'] 6 | urls_check = ['mathpretty.com','zhihu.com', 'sina.com', 'ali.com'] 7 | for url in urls: bloom_filter.add(url) 8 | for url_check in urls_check: 9 | result = bloom_filter.contains(url_check) 10 | print('url : ',url_check,' contain : ',result) 11 | ''' 12 | 13 | import crypt 14 | 15 | 16 | class BloomFilter(object): 17 | def __init__(self, bitsize=30000): 18 | self.bitarr = [0 for _ in range(bitsize)] 19 | self.bitsize = bitsize 20 | 21 | def getPosList(self, string): 22 | posList = [] 23 | for salt in ['41', '42', '43', '44', '45']: 24 | cryStr = crypt.crypt(string, salt) 25 | pos = hash(cryStr) % self.bitsize 26 | posList.append(pos) 27 | return posList 28 | 29 | def add(self, string): 30 | posList = self.getPosList(string) 31 | for pos in posList: 32 | self.bitarr[pos] = 1 33 | 34 | def contains(self, string): 35 | posList = self.getPosList(string) 36 | for pos in posList: 37 | if self.bitarr[pos] != 1: 38 | return False 39 | return True 40 | -------------------------------------------------------------------------------- /sunyata/algorithm/bplustree.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | 3 | from sunyata.algorithm.btree import BTree 4 | 5 | class BPlusTree(BTree): 6 | pass -------------------------------------------------------------------------------- /sunyata/algorithm/btree.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | 3 | from sunyata.algorithm.avlsearchtree import AVLSearchTree 4 | 5 | class BTree(AVLSearchTree): 6 | pass -------------------------------------------------------------------------------- /sunyata/algorithm/hashtable.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | ''' 3 | Usage: 4 | 5 | h = HashTable() 6 | h.set("a", "1") 7 | h.set("b", "2") 8 | print(len(h)) 9 | print(h.get("a")) 10 | h["abc"] = "abc" 11 | print(h["abc"]) 12 | print(h.hasKey("a")) 13 | for k, v in h: 14 | print(k, v) 15 | for k, v in h.items(): 16 | print(k, v) 17 | ''' 18 | 19 | 20 | class HashTable(object): 21 | def __init__(self, size=99999): 22 | self.size = size 23 | self.hashList = [list() for _ in range(size)] 24 | 25 | def set(self, key, value): 26 | hashKey = hash(key) % self.size 27 | for item in self.hashList[hashKey]: 28 | if item[0] == key: 29 | item[1] = value 30 | return 31 | self.hashList[hashKey].append([key, value]) 32 | 33 | def get(self, key): 34 | hashKey = hash(key) % self.size 35 | for item in self.hashList[hashKey]: 36 | if item[0] == key: 37 | return item[1] 38 | raise KeyError 39 | 40 | def hasKey(self, key): 41 | try: 42 | self.get(key) 43 | except KeyError: 44 | return False 45 | return True 46 | 47 | def __len__(self): 48 | length = 0 49 | for item in self.hashList: 50 | length = length + len(item) 51 | return length 52 | 53 | def __getitem__(self, key): 54 | return self.get(key) 55 | 56 | def __setitem__(self, key, value): 57 | return self.set(key, value) 58 | 59 | def __iter__(self): 60 | return self.items() 61 | 62 | def items(self): 63 | kvList = [] 64 | for item in self.hashList: 65 | if len(item) > 0: 66 | kvList = kvList + item 67 | return iter(kvList) 68 | -------------------------------------------------------------------------------- /sunyata/algorithm/lru.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | ''' 3 | Usage: 4 | 5 | 6 | lru = LRUCache(size = 10) 7 | for i in range(10): 8 | key = str(i) 9 | lru.put(key, key) 10 | lru.dump() 11 | for i in range(5): 12 | lru.get(str(i)) 13 | lru.dump() 14 | for k, v in lru.items(): 15 | print(k, v) 16 | ''' 17 | 18 | import collections 19 | 20 | 21 | class Node(object): 22 | def __init__(self, key, value): 23 | self.key = key 24 | self.value = value 25 | self.next = None 26 | 27 | 28 | class LinkNode(object): 29 | def __init__(self): 30 | self.head = None 31 | self.tail = None 32 | self.length = 0 33 | 34 | def append(self, node): 35 | if self.head == None and self.tail == None: 36 | node.next = None 37 | self.head = node 38 | self.tail = node 39 | else: 40 | self.tail.next = node 41 | node.next = None 42 | self.tail = node 43 | self.length = self.length + 1 44 | 45 | def dump(self): 46 | node = self.head 47 | while node != None: 48 | print(node.key, node.value) 49 | node = node.next 50 | print('---------------------') 51 | 52 | def items(self): 53 | kvList = [] 54 | node = self.head 55 | while node != None: 56 | kvList.append([node.key, node.value]) 57 | node = node.next 58 | return iter(kvList) 59 | 60 | def find(self, key): 61 | node = self.head 62 | while node != None: 63 | if node.key == key: 64 | return node 65 | node = node.next 66 | return None 67 | 68 | def appendHead(self, node): 69 | node.next = self.head 70 | if self.head == None and self.tail == None: 71 | self.tail = node 72 | self.head = node 73 | self.length = self.length + 1 74 | 75 | def removeTail(self): 76 | if self.tail == None: return 77 | preNode = self.head 78 | curNode = self.head 79 | while 1: 80 | if curNode != self.tail: 81 | preNode = curNode 82 | curNode = curNode.next 83 | else: 84 | break 85 | preNode.next = None 86 | self.tail = preNode 87 | self.length = self.length - 1 88 | 89 | def removeNode(self, node): 90 | if self.head == None and self.tail == None: 91 | return 92 | if self.head == node: 93 | self.head = node.next 94 | self.length = self.length - 1 95 | return 96 | if self.tail == node: 97 | self.removeTail() 98 | return 99 | preNode = curNode = self.head 100 | while curNode != self.tail: 101 | if curNode.key == node.key: 102 | preNode.next = node.next 103 | self.length = self.length - 1 104 | return 105 | preNode = curNode 106 | curNode = curNode.next 107 | 108 | 109 | class LRUCache(object): 110 | def __init__(self, size=9999): 111 | self.size = size 112 | self.link = LinkNode() 113 | 114 | def get(self, key): 115 | node = self.link.find(key) 116 | if node != None: 117 | self.link.removeNode(node) 118 | self.link.appendHead(node) 119 | else: 120 | raise KeyError 121 | 122 | def put(self, key, value): 123 | node = self.link.find(key) 124 | if node != None: 125 | self.link.removeNode(node) 126 | self.link.appendHead(node) 127 | else: 128 | node = Node(key, value) 129 | if (self.link.length + 1) <= self.size: 130 | self.link.appendHead(node) 131 | else: 132 | self.link.appendHead(node) 133 | self.link.removeTail() 134 | return self 135 | 136 | def length(self): 137 | return self.link.length 138 | 139 | def dump(self): 140 | return self.link.dump() 141 | 142 | def items(self): 143 | return self.link.items() 144 | -------------------------------------------------------------------------------- /sunyata/algorithm/rbtree.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | 3 | from sunyata.algorithm.bintree import BinTree 4 | 5 | class RBTree(BinTree): 6 | pass -------------------------------------------------------------------------------- /sunyata/algorithm/sort/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lycclsltt/sunyata/a1e09860c65c4c65a50ea4d7b0034c567c97088c/sunyata/algorithm/sort/__init__.py -------------------------------------------------------------------------------- /sunyata/algorithm/sort/base.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | 3 | class Sort(object): 4 | 5 | @classmethod 6 | def sort(cls, arr): 7 | pass -------------------------------------------------------------------------------- /sunyata/algorithm/sort/quicksort.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | 3 | from sunyata.algorithm.sort.base import Sort 4 | 5 | class QuickSort(Sort): 6 | 7 | @classmethod 8 | def sort(cls, arr): 9 | ''' 10 | 递归版本 11 | ''' 12 | length = len(arr) 13 | if length <= 1: 14 | return arr 15 | stard = int(length / 2) 16 | larr = [] 17 | rarr = [] 18 | for i, v in enumerate(arr): 19 | if i == stard: 20 | continue 21 | else: 22 | if v <= arr[stard]: 23 | larr.append(v) 24 | else: 25 | rarr.append(v) 26 | return cls.sort(larr) + [ arr[stard] ] + cls.sort(rarr) -------------------------------------------------------------------------------- /sunyata/algorithm/trie.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | 3 | ''' 4 | Usage: 5 | 6 | from sunyata.algorithm.trie import TrieNode, Trie 7 | import random 8 | arr = [u'运维', u'运维开发', u'运营'] 9 | print('arr', arr) 10 | t = Trie() 11 | for elem in arr: 12 | t.append(elem) 13 | words = t.getWords() 14 | print(words) 15 | isExists = t.isContains('abcv') 16 | print(isExists) 17 | t.update(u'运维开发', u'运气') 18 | words = t.getWordsByKw('运') 19 | print(words) 20 | 21 | ''' 22 | 23 | class TrieNode(object): 24 | 25 | def __init__(self): 26 | self.char = '' 27 | self.pre = None 28 | self.isRoot = False 29 | self.childMap = {} 30 | self.data = None 31 | self.isWordEnd = False 32 | 33 | def isLeaf(self): 34 | if len(self.childMap) <= 0: 35 | return True 36 | return False 37 | 38 | def isRoot(self): 39 | return self.isRoot 40 | 41 | def childNodes(self): 42 | nodes = [] 43 | for _, node in (self.childMap).items(): 44 | nodes.append(node) 45 | return nodes 46 | 47 | def hasChild(self): 48 | childNodes = self.childNodes() 49 | if len(childNodes) <= 0: 50 | return False 51 | return True 52 | 53 | 54 | class Trie(object): 55 | 56 | def __init__(self): 57 | self.root = TrieNode() 58 | self.root.isRoot = True 59 | self.curString = '' 60 | self.words = [] 61 | 62 | def getRootNode(self): 63 | return self.root 64 | 65 | def isHasChildNodeByChar(self, node, char): 66 | if char in node.childMap: 67 | return True 68 | return False 69 | 70 | def getChildNodeByChar(self, node, char): 71 | return node.childMap.get(char) 72 | 73 | def append(self, string): 74 | curnode = self.root 75 | linkstr = '' 76 | for char in string: 77 | linkstr = linkstr + char 78 | if self.isHasChildNodeByChar(curnode, char): 79 | curnode = self.getChildNodeByChar(curnode, char) 80 | else: 81 | childNode = TrieNode() 82 | childNode.char = char 83 | childNode.pre = curnode 84 | childNode.isRoot = False 85 | childNode.data = linkstr 86 | curnode.childMap[char] = childNode 87 | curnode = childNode 88 | curnode.isWordEnd = True 89 | 90 | def getNodeWord(self, node): 91 | words = [] 92 | if node.isWordEnd: 93 | words.append(node.data) 94 | if node.hasChild(): 95 | for childNode in node.childNodes(): 96 | words = words + self.getNodeWord(childNode) 97 | return words 98 | 99 | def getWords(self): 100 | words = self.getNodeWord(self.root) 101 | return words 102 | 103 | def isContains(self, kw): 104 | curnode = self.root 105 | for char in kw: 106 | if self.isHasChildNodeByChar(curnode, char): 107 | curnode = self.getChildNodeByChar(curnode, char) 108 | else: 109 | return False 110 | return True 111 | 112 | def getWordsByKw(self, kw): 113 | words = [] 114 | curnode = self.root 115 | for char in kw: 116 | if self.isHasChildNodeByChar(curnode, char): 117 | curnode = self.getChildNodeByChar(curnode, char) 118 | else: 119 | return words 120 | words = self.getNodeWord(curnode) 121 | return words 122 | 123 | def remove(self, string): 124 | curnode = self.root 125 | for char in string: 126 | if self.isHasChildNodeByChar(curnode, char): 127 | curnode = self.getChildNodeByChar(curnode, char) 128 | else: 129 | return 130 | if curnode.hasChild(): 131 | curnode.isWordEnd = False 132 | return 133 | while curnode != self.root: 134 | if not curnode.hasChild(): 135 | if not curnode.pre.isWordEnd: 136 | del curnode.pre.childMap[curnode.char] 137 | curnode = curnode.pre 138 | else: 139 | break 140 | else: 141 | break 142 | 143 | def update(self, string, newstring): 144 | self.remove(string) 145 | self.append(newstring) -------------------------------------------------------------------------------- /sunyata/cache_wrap.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import redis 3 | import pickle 4 | 5 | FUNC_CACHE_MAP = {} 6 | 7 | def _get_key(funcname, *args, **kwargs): 8 | key = 'sunyata_cache_func:'+funcname + ':' 9 | if len(args) > 0: 10 | key = key + str(list(args)) + ':' 11 | if len(kwargs) > 0: 12 | key = key + str(dict(kwargs)) 13 | return key 14 | 15 | def cache_func(*args, **kwargs): 16 | def function(func): 17 | @functools.wraps(func) 18 | def wrapper(*args, **kwargs): 19 | global FUNC_CACHE_MAP 20 | key = _get_key(func.__name__, *args, **kwargs) 21 | if key in FUNC_CACHE_MAP: 22 | return FUNC_CACHE_MAP[key] 23 | ret = func(*args, **kwargs) 24 | FUNC_CACHE_MAP[key] = ret 25 | return ret 26 | return wrapper 27 | return function 28 | 29 | def get_redis_connection(connstr): 30 | host, port, db = connstr.split(':') 31 | conn = redis.Redis(host=host,port=port,db=db) 32 | return conn 33 | 34 | def redis_cache_func(connstr='127.0.0.1:6379:1', expireSeconds=3600*24): 35 | conn = get_redis_connection(connstr) 36 | def function(func): 37 | @functools.wraps(func) 38 | def wrapper(*args, **kwargs): 39 | key = _get_key(func.__name__, *args, **kwargs) 40 | val = conn.get(key) 41 | if val != None: 42 | return pickle.loads(val) 43 | ret = func(*args, **kwargs) 44 | val = pickle.dumps(ret) 45 | conn.set(key, val) 46 | conn.expire(key, expireSeconds) 47 | return ret 48 | return wrapper 49 | return function -------------------------------------------------------------------------------- /sunyata/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lycclsltt/sunyata/a1e09860c65c4c65a50ea4d7b0034c567c97088c/sunyata/cli/__init__.py -------------------------------------------------------------------------------- /sunyata/cli/cli.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import sys 3 | import os 4 | from sunyata.rpc.server import TcpRpcServer 5 | 6 | class Cli(object): 7 | 8 | def entryPoint(self): 9 | args = self.parseArgs() 10 | self.runServer(args) 11 | 12 | def parseArgs(self): 13 | parser = argparse.ArgumentParser(description="sunyata console cli tool") 14 | parser.add_argument('-r', '--run', type=str, help = 'module', required=True, metavar='module', dest='module') 15 | parser.add_argument('-b', '--bind', type=str, help='bind interface, default 0.0.0.0', default='0.0.0.0', required=False, metavar='0.0.0.0', dest='bind') 16 | parser.add_argument('-p', '--port', type=int, help='bind port, default 9988', default=9988, required=False, metavar='9988', dest='port') 17 | args = parser.parse_args() 18 | try: 19 | module = args.module 20 | if module == '': 21 | raise Exception('') 22 | except Exception as ex: 23 | print(ex) 24 | parser.print_usage() 25 | sys.exit(1) 26 | return args 27 | 28 | def runServer(self, args): 29 | sys.path.append( os.getcwd() ) 30 | __import__(args.module) 31 | s = TcpRpcServer(args.bind, args.port) 32 | s.serve() -------------------------------------------------------------------------------- /sunyata/cli/entry.py: -------------------------------------------------------------------------------- 1 | from sunyata.cli.cli import Cli 2 | 3 | def main(): 4 | cli = Cli() 5 | cli.entryPoint() 6 | print('finish') 7 | 8 | if __name__ == '__main__': 9 | main() -------------------------------------------------------------------------------- /sunyata/compress.py: -------------------------------------------------------------------------------- 1 | import lz4.block as lb 2 | 3 | class Compress(object): 4 | 5 | level = 3 6 | 7 | DEBUG = False 8 | 9 | @classmethod 10 | def compress(cls, bytearr: bytes): 11 | compressed = lb.compress(bytearr) 12 | if cls.DEBUG: 13 | print('debug, do compress', 'orig len', len(bytearr), 'compress len', len(compressed)) 14 | return compressed 15 | 16 | @classmethod 17 | def decompress(cls, bytearr): 18 | if cls.DEBUG: 19 | print('debug, do decompress') 20 | origin = lb.decompress(bytearr) 21 | return origin -------------------------------------------------------------------------------- /sunyata/concurrent.py: -------------------------------------------------------------------------------- 1 | from threading import Thread 2 | 3 | class Concurrent(object): 4 | 5 | thlist = [] 6 | 7 | @classmethod 8 | def go(cls, target, *args, **kwargs): 9 | print(target) 10 | print(args) 11 | print(kwargs) 12 | th = Thread(target=target, args=args, kwargs=kwargs) 13 | cls.thlist.append(th) 14 | th.start() -------------------------------------------------------------------------------- /sunyata/consul.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import ujson 3 | from sunyata.util import local_ip 4 | from dataclasses import dataclass 5 | import aiohttp 6 | 7 | 8 | @dataclass 9 | class Instance(object): 10 | service: str 11 | address: str 12 | port: int 13 | 14 | 15 | class ConsulApi(object): 16 | 17 | __slots__ = ('host', 'port', 'baseUrl', 'token', 'timeout', 'headers') 18 | 19 | def __init__(self, host: str, port: int, token = ''): 20 | self.host = host 21 | self.port = port 22 | self.baseUrl = 'http://%s:%s' % (self.host, self.port) 23 | self.token = token 24 | self.timeout = 5 25 | self.headers = {} 26 | if self.token: 27 | self.headers = {'X-Consul-Token':self.token} 28 | 29 | async def getInstanceMap(self): 30 | async with aiohttp.ClientSession() as sess: 31 | url = self.baseUrl + '/v1/agent/services' 32 | async with sess.get(url, headers=self.headers, timeout = self.timeout) as r: 33 | text = await r.text() 34 | if r.status != 200: 35 | raise Exception(text) 36 | return ujson.loads(text) 37 | 38 | def genInstanceID(self, service: str, host: str, port: int): 39 | key = "%s:%s:%s" % (service, host, port) 40 | m = hashlib.md5() 41 | m.update(key.encode()) 42 | return m.hexdigest() 43 | 44 | async def registService(self, serviceName: str, port = 80, address=None, ttl=30, degisterAfter = '1m'): 45 | params = { 46 | 'Name' : serviceName, 47 | 'Port' : port 48 | } 49 | if address: 50 | params['Address'] = address 51 | else: 52 | params['Address'] = local_ip() 53 | params['ID'] = self.genInstanceID(serviceName, params['Address'], port) 54 | ''' 55 | params['Check'] = { 56 | 'Interval' : "%ss" % checkInterval, 57 | 'DeregisterCriticalServiceAfter' : '1m', 58 | 'Timeout' : '5s', 59 | 'TCP' : "%s:%s" % (params['Address'], port) 60 | } 61 | ''' 62 | params['Check'] = { 63 | 'DeregisterCriticalServiceAfter' : degisterAfter, 64 | 'TTL' : '%ss' % ttl, 65 | } 66 | async with aiohttp.ClientSession() as sess: 67 | url = self.baseUrl + '/v1/agent/service/register' 68 | async with sess.put(url, headers = self.headers, data=ujson.dumps(params), timeout = self.timeout) as r: 69 | text = await r.text() 70 | print('[consul] regist ', text, r.status) 71 | 72 | async def get(self, k: str): 73 | async with aiohttp.ClientSession() as sess: 74 | url = self.baseUrl + '/v1/kv/%s?raw=true' % k 75 | async with sess.get(url, headers=self.headers, timeout = self.timeout) as r: 76 | if r.status_code != 200: 77 | raise Exception(r.text) 78 | return r.text 79 | 80 | async def getServiceInstanceList(self, serviceName: str) -> list: 81 | instanceMap = await self.getInstanceMap() 82 | instanceList = [] 83 | for instanceID, info in instanceMap.items(): 84 | if info.get('Service') == serviceName: 85 | instance = Instance( 86 | service = serviceName, 87 | address = info.get('Address'), 88 | port = int(info.get('Port')) 89 | ) 90 | instanceList.append(instance) 91 | return instanceList 92 | 93 | async def ttlHeartbeat(self, service, address, port): 94 | async with aiohttp.ClientSession() as sess: 95 | url = self.baseUrl + '/v1/agent/check/pass/%s' % ('service:' + self.genInstanceID(service, address, port)) 96 | async with sess.put(url, headers=self.headers, timeout = self.timeout) as r: 97 | if r.status_code != 200: 98 | raise Exception(r.text) -------------------------------------------------------------------------------- /sunyata/db.py: -------------------------------------------------------------------------------- 1 | ''' 2 | mysql conneciton pool 3 | ''' 4 | 5 | import pymysql 6 | from DBUtils.PooledDB import PooledDB 7 | 8 | import sys 9 | if sys.version[0:1] == '3': unicode = str 10 | 11 | 12 | class PoolDB(object): 13 | 14 | def __init__(self, 15 | host, 16 | port, 17 | user, 18 | passwd, 19 | dbName, 20 | log=None, 21 | connect_timeout=10, 22 | read_timeout=None, 23 | min_conn_num=10): 24 | self.host = host 25 | self.port = port 26 | self.user = user 27 | self.passwd = passwd 28 | self.dbName = dbName 29 | self.log = log 30 | self.connectTimeout = connect_timeout 31 | self.readTimeout = read_timeout 32 | self.minConnNum = min_conn_num 33 | self.charset = 'utf8' 34 | self.pool = None 35 | 36 | def connect(self): 37 | if self.pool == None: 38 | self.pool = PooledDB(pymysql, 39 | self.minConnNum, 40 | host=self.host, 41 | port=self.port, 42 | user=self.user, 43 | passwd=self.passwd, 44 | db=self.dbName, 45 | charset=self.charset, 46 | connect_timeout=self.connectTimeout, 47 | read_timeout=self.readTimeout) 48 | print('PoolDB init finish') 49 | 50 | def close(self): 51 | if self.pool != None: 52 | self.pool.close() 53 | self.pool = None 54 | 55 | def query(self, sql): 56 | conn = self.pool.connection() 57 | cur = conn.cursor(pymysql.cursors.DictCursor) 58 | cur.execute(sql) 59 | res = cur.fetchall() 60 | cur.close() 61 | conn.close() 62 | return res 63 | 64 | def update(self, sql): 65 | conn = self.pool.connection() 66 | cur = conn.cursor() 67 | effect_rows = cur.execute(sql) 68 | conn.commit() 69 | lastrowId = cur.lastrowid 70 | cur.close() 71 | conn.close() 72 | return effect_rows, lastrowId 73 | 74 | 75 | class Orm: 76 | 77 | ORDER_ASC = 'asc' 78 | ORDER_DESC = 'desc' 79 | 80 | def __init__(self, sunyataDb): 81 | self.db = sunyataDb 82 | self.tableName = '' 83 | self.whereCon = '' 84 | self.updateData = None 85 | self.fields = [] 86 | self.raw = '' 87 | 88 | def table(self, tableName): 89 | self.tableName = tableName 90 | return self 91 | 92 | def field(self, field_name): 93 | tmpFields = list(set( [ item.strip() for item in field_name.split(',') ] )) 94 | for tf in tmpFields: 95 | self.fields.append(tf) 96 | self.fields = list(set(self.fields)) 97 | return self 98 | 99 | def sql(self, raw): 100 | self.raw = raw 101 | return self 102 | 103 | def where(self, *args): 104 | colName = '' 105 | data = '' 106 | compareTag = '' 107 | 108 | if len(args) == 0: 109 | return self 110 | elif len(args) == 1: 111 | if self.whereCon != '': 112 | self.whereCon = self.whereCon + ' and %s ' % (args[0]) 113 | else: 114 | self.whereCon = ' where %s ' % (args[0]) 115 | return self 116 | elif len(args) == 2: 117 | compareTag = '=' 118 | colName = args[0] 119 | data = args[1] 120 | if '__in' in colName: 121 | compareTag = 'in' 122 | colName = colName.replace('__in', '') 123 | if type(data) == list: 124 | itemList = [ "'" + str(item) + "'" for item in data ] 125 | data = '(' + ','.join(itemList) + ')' 126 | elif type(data) == str: 127 | splitList = data.split(',') 128 | itemList = [ "'" + str(item).strip() + "'" for item in splitList ] 129 | data = '(' + ','.join(itemList) + ')' 130 | 131 | elif len(args) == 3: 132 | colName = args[0] 133 | compareTag = args[1] 134 | data = args[2] 135 | else: 136 | raise Exception('where param length not support') 137 | 138 | if self.whereCon != '': 139 | self.whereCon = self.whereCon + ' and `%s` %s' % (colName, 140 | compareTag) 141 | else: 142 | self.whereCon = ' where `%s` %s' % (colName, compareTag) 143 | 144 | if (type(data) == int) or (type(data) == float): 145 | self.whereCon = self.whereCon + " %s " % data 146 | else: 147 | if compareTag == 'in': 148 | self.whereCon = self.whereCon + " %s " % data 149 | else: 150 | data = pymysql.escape_string(data) 151 | self.whereCon = self.whereCon + " '%s' " % data 152 | 153 | return self 154 | 155 | def orWhere(self, *args): 156 | colName = '' 157 | data = '' 158 | compareTag = '' 159 | 160 | if len(args) == 0: 161 | return self 162 | elif len(args) == 1: 163 | if self.whereCon != '': 164 | self.whereCon = self.whereCon + ' or %s ' % (args[0]) 165 | else: 166 | self.whereCon = ' where %s ' % (args[0]) 167 | return self 168 | elif len(args) == 2: 169 | compareTag = '=' 170 | colName = args[0] 171 | data = args[1] 172 | if '__in' in colName: 173 | compareTag = 'in' 174 | colName = colName.replace('__in', '') 175 | if type(data) == list: 176 | itemList = [ "'" + str(item) + "'" for item in data ] 177 | data = '(' + ','.join(itemList) + ')' 178 | elif type(data) == str: 179 | splitList = data.split(',') 180 | itemList = [ "'" + str(item).strip() + "'" for item in splitList ] 181 | data = '(' + ','.join(itemList) + ')' 182 | 183 | elif len(args) == 3: 184 | colName = args[0] 185 | compareTag = args[1] 186 | data = args[2] 187 | else: 188 | raise Exception('where param length not support') 189 | 190 | if self.whereCon != '': 191 | self.whereCon = self.whereCon + ' or `%s` %s' % (colName, 192 | compareTag) 193 | else: 194 | self.whereCon = ' where `%s` %s' % (colName, compareTag) 195 | 196 | if (type(data) == int) or (type(data) == float): 197 | self.whereCon = self.whereCon + " %s " % data 198 | else: 199 | if compareTag == 'in': 200 | self.whereCon = self.whereCon + " %s " % data 201 | else: 202 | data = pymysql.escape_string(data) 203 | self.whereCon = self.whereCon + " '%s' " % data 204 | 205 | return self 206 | 207 | 208 | def andWhere(self, *args): 209 | return self.where(*args) 210 | 211 | def getSql(self, cmd): 212 | #cmd 'insert, delete, update, select' 213 | if self.raw != '': 214 | return self.raw 215 | 216 | sql = '' 217 | if cmd == 'select': 218 | field_str = '*' 219 | if len(self.fields) == 0: 220 | field_str = '*' 221 | else: 222 | if len(self.fields) == 1: 223 | if self.fields[0] == '' or self.fields[0] == '*': 224 | field_str = '*' 225 | else: 226 | field_str = ','.join([ '`' + colname + '`' for colname in self.fields ]) 227 | sql = 'select %s from %s %s' % (field_str, self.tableName, self.whereCon) 228 | if cmd == 'insert': 229 | sql_list = [] 230 | if type(self.updateData) == list: 231 | sql = 'insert into `%s`' % self.tableName 232 | for row in self.updateData: 233 | cols = [] 234 | for k, v in row.items(): 235 | cols.append("`" + k + "`") 236 | col = ','.join(cols) 237 | sql = sql + '(' + col + ') values' 238 | break 239 | 240 | values = [] 241 | for row in self.updateData: 242 | tup = [] 243 | for k, v in row.items(): 244 | if type(v) == int or type(v) == float: 245 | tup.append(str(v)) 246 | elif type(v) == str or type(v) == unicode: 247 | tup.append("'" + v + "'") 248 | tupStr = '(' + ','.join(tup) + ')' 249 | values.append(tupStr) 250 | valueStr = ', '.join(values) 251 | sql = sql + valueStr 252 | self.whereCon = '' 253 | return sql 254 | if type(self.updateData) == dict: 255 | cols = [] 256 | for k, v in self.updateData.items(): 257 | cols.append("`" + k + "`") 258 | sql = 'insert into %s' % (self.tableName) + '(' + ','.join( 259 | cols) + ') values(' 260 | for k, v in self.updateData.items(): 261 | if type(v) == int or type(v) == float: 262 | sql = sql + str(v) + ',' 263 | if type(v) == str or type(v) == unicode: 264 | sql = sql + "'" + v + "'," 265 | sql = sql[0:-1] 266 | sql = sql + ')' 267 | self.whereCon = '' 268 | return sql 269 | if cmd == 'delete': 270 | sql = """delete from %s %s""" % (self.tableName, self.whereCon) 271 | if cmd == 'update': 272 | sql = "update %s set" % (self.tableName) 273 | for k, v in self.updateData.items(): 274 | if type(v) == int: 275 | sql = sql + """ %s=%s,""" % ("`" + k + "`", v) 276 | if type(v) == str: 277 | sql = sql + """ %s='%s',""" % ("`" + k + "`", v) 278 | sql = sql[0:-1] 279 | sql = sql + ' ' + self.whereCon 280 | if cmd == 'count': 281 | sql = """select count(*) as cnt from %s %s""" % (self.tableName, 282 | self.whereCon) 283 | 284 | self.whereCon = '' 285 | return sql 286 | 287 | def data(self, updateData): 288 | if type(updateData) == list: 289 | for i in range(len(updateData)): 290 | for k, v in updateData[i].items(): 291 | if type(v) == str: 292 | updateData[i][k] = pymysql.escape_string(v) 293 | elif type(updateData) == dict: 294 | for k, v in updateData.items(): 295 | if type(v) == str: 296 | updateData[k] = pymysql.escape_string(v) 297 | else: 298 | pass 299 | self.updateData = updateData 300 | return self 301 | 302 | def get(self): 303 | sql = self.getSql('select') 304 | return self.db.query(sql) 305 | 306 | def values(self, field_name): 307 | rows = self.get() 308 | ret = [] 309 | if rows != None: 310 | for row in rows: 311 | ret.append(row.get(field_name)) 312 | return ret 313 | 314 | def first(self): 315 | rows = self.get() 316 | if len(rows) > 0: 317 | row = rows[0] 318 | return row 319 | return None 320 | 321 | def update(self, retSql=False): 322 | sql = self.getSql('update') 323 | return self.db.update(sql) 324 | 325 | def insert(self): 326 | sql = self.getSql('insert') 327 | self.db.update(sql) 328 | 329 | def delete(self): 330 | sql = self.getSql('delete') 331 | return self.db.update(sql) 332 | 333 | def count(self): 334 | sql = self.getSql('count') 335 | rows = self.db.query(sql) 336 | try: 337 | row = rows[0] 338 | num = int(row['cnt']) 339 | except: 340 | num = len(rows) 341 | return num 342 | 343 | def limit(self, limitNum, endpos = None): 344 | if not str(limitNum).isdigit(): 345 | raise Exception('param limit is not a number') 346 | if '.' in str(limitNum): 347 | raise Exception('param limit is not a integer, but a float') 348 | limitNum = int(limitNum) 349 | if limitNum < 0: raise Exception('param limit < 0') 350 | self.whereCon = self.whereCon + " limit %s" % limitNum 351 | 352 | if type(endpos) == int or type(endpos) == str: 353 | if not str(endpos).isdigit(): 354 | raise Exception('param limit is not a number') 355 | if int(endpos) < 0 or '.' in str(endpos): 356 | raise Exception('param limit is invalid') 357 | self.whereCon = self.whereCon + ", " + str(endpos) 358 | 359 | return self 360 | 361 | def orderBy(self, colName, order=ORDER_ASC): 362 | if order not in ['asc', 'desc', 'ASC', 'DESC']: 363 | raise Exception('order by order is invalid') 364 | self.whereCon = self.whereCon + " order by `%s` %s" % (colName, order) 365 | return self 366 | 367 | 368 | class DBCluster(object): 369 | def __init__(self, masterDb, slaveDbList=[]): 370 | self._masterDb = masterDb 371 | self._slaveDbList = slaveDbList 372 | self._slaveDbCount = len(slaveDbList) 373 | self._curSlave = 0 374 | 375 | def query(self, sql): 376 | if len(self._slaveDbList) == 0: 377 | return self._masterDb.query(sql) 378 | self._curSlave = self._curSlave + 1 379 | slaveIndex = self._curSlave % self._slaveDbCount 380 | slaveDb = self._slaveDbList[slaveIndex] 381 | return slaveDb.query(sql) 382 | 383 | def queryInstance(self): 384 | self._curSlave = self._curSlave + 1 385 | slaveIndex = self._curSlave % self._slaveDbCount 386 | return slaveIndex 387 | 388 | def update(self, sql): 389 | return self._masterDb.update(sql) 390 | 391 | 392 | class DBProxy(object): 393 | def __init__(self, masterDb, slaveDbList=[]): 394 | self._masterDb = masterDb 395 | self._slaveDbList = slaveDbList 396 | self._slaveDbCount = len(slaveDbList) 397 | self._curSlave = 0 398 | 399 | def query(self, sql): 400 | if len(self._slaveDbList) == 0: 401 | return self._masterDb.query(sql) 402 | self._curSlave = self._curSlave + 1 403 | slaveIndex = self._curSlave % self._slaveDbCount 404 | slaveDb = self._slaveDbList[slaveIndex] 405 | return slaveDb.query(sql) 406 | 407 | def queryInstance(self): 408 | self._curSlave = self._curSlave + 1 409 | slaveIndex = self._curSlave % self._slaveDbCount 410 | return slaveIndex 411 | 412 | def update(self, sql): 413 | return self._masterDb.update(sql) -------------------------------------------------------------------------------- /sunyata/etcd.py: -------------------------------------------------------------------------------- 1 | import ujson 2 | from sunyata.util import local_ip 3 | from sunyata.consul import Instance 4 | import aiohttp 5 | 6 | class EtcdApi(object): 7 | 8 | def __init__(self, host: str, port: int): 9 | self.host = host 10 | self.port = port 11 | self.baseUrl = 'http://%s:%s' % (self.host, self.port) 12 | self.timeout = 5 13 | self.headers = {} 14 | 15 | async def registService(self, serviceName: str, port = 80, address=None, ttl=30): 16 | async with aiohttp.ClientSession() as sess: 17 | url = self.baseUrl + '/v2/keys/sunyata-services/%s?dir=true&ttl=%s&prevExist=true' % (serviceName, ttl) 18 | async with sess.put(url, headers=self.headers, timeout=self.timeout) as r: 19 | text = await r.text() 20 | print('[etcd] regist dir', text, r.status) 21 | if address: 22 | key = address + ':' + str(port) 23 | else: 24 | key = local_ip() + ':' + str(port) 25 | val = key 26 | url = self.baseUrl + '/v2/keys/sunyata-services/%s/%s?value=%s&ttl=%s' % (serviceName, key, val, ttl) 27 | async with sess.put(url, headers=self.headers, timeout=self.timeout) as r: 28 | text = await r.text() 29 | print('[etcd] regist key', text, r.status) 30 | 31 | async def getServiceInstanceList(self, serviceName: str) -> list: 32 | async with aiohttp.ClientSession() as sess: 33 | url = self.baseUrl + '/v2/keys/sunyata-services/%s' % serviceName 34 | async with sess.get(url, timeout=self.timeout, headers=self.headers) as r: 35 | text = await r.text() 36 | dic = ujson.loads(text) 37 | nodes = dic.get('node').get('nodes', []) 38 | instanceList = [] 39 | for node in nodes: 40 | value = node.get('value') 41 | ip, port = value.split(':') 42 | instance = Instance( 43 | service=serviceName, 44 | address=ip, 45 | port=int(port) 46 | ) 47 | instanceList.append(instance) 48 | return instanceList 49 | 50 | async def ttlHeartbeat(self, service, address, port): 51 | await self.registService(serviceName=service, address=address, port=port) -------------------------------------------------------------------------------- /sunyata/eventloop.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | class EventLoop(object): 4 | @classmethod 5 | def runUntilComplete(self, asyncFunc): 6 | tasks = [ asyncFunc ] 7 | loop = asyncio.new_event_loop() 8 | asyncio.set_event_loop(loop) 9 | loop = asyncio.get_event_loop() 10 | loop.run_until_complete( asyncio.gather(*tasks) ) 11 | loop.run_forever() 12 | loop.close() -------------------------------------------------------------------------------- /sunyata/gossip/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lycclsltt/sunyata/a1e09860c65c4c65a50ea4d7b0034c567c97088c/sunyata/gossip/__init__.py -------------------------------------------------------------------------------- /sunyata/gossip/core.py: -------------------------------------------------------------------------------- 1 | from sunyata.rpc.server import HttpRpcServer 2 | from sunyata.rpc.client import HttpRpcClient 3 | from sunyata.rpc import rpc 4 | import queue 5 | import threading 6 | import time 7 | 8 | 9 | @rpc 10 | class GossipSyncedCallback(object): 11 | 12 | syncedCallbackClassInstance = None 13 | 14 | def synced(self, msg): 15 | if self.syncedCallbackClassInstance: 16 | self.syncedCallbackClassInstance.synced(msg) 17 | else: 18 | print('synced recv', msg.k, msg.v, msg.version) 19 | 20 | 21 | class GossipMsg(object): 22 | 23 | def __init__(self, k, v, version): 24 | self.k = k 25 | self.v = v 26 | self.version = version 27 | 28 | 29 | class GossipCore(object): 30 | 31 | def __init__(self, bindHost, bindPort, memberAddrs, syncedCallback=None): 32 | self.memberAddrs = memberAddrs 33 | self.bindHost = bindHost 34 | self.bindPort = bindPort 35 | self.rpcServer = HttpRpcServer(self.bindHost, self.bindPort) 36 | self.rpcServer.accessLog = False 37 | self.rpcServer.app.accessLog = False 38 | self.rpcClientList = [ HttpRpcClient(memberAddr['host'], memberAddr['port']) for memberAddr in self.memberAddrs ] 39 | if syncedCallback: 40 | GossipSyncedCallback.syncedCallbackClassInstance = syncedCallback 41 | self.queueMaxSize = 100000 42 | self.Q = queue.Queue(self.queueMaxSize) 43 | self.thlist = [] 44 | 45 | def startRpcServer(self): 46 | self.rpcServer.serve() 47 | 48 | def startBroadcast(self): 49 | while 1: 50 | msg = self.Q.get() 51 | for cli in self.rpcClientList: 52 | #if cli.host == self.bindHost and cli.port == self.bindPort: 53 | # continue 54 | try: 55 | cli.GossipSyncedCallback.synced(msg) 56 | except Exception as ex: 57 | print(ex) 58 | 59 | def start(self): 60 | thRpcServer = threading.Thread(target=self.startRpcServer) 61 | thRpcServer.start() 62 | self.thlist.append(thRpcServer) 63 | 64 | thBroadcast = threading.Thread(target=self.startBroadcast) 65 | thBroadcast.start() 66 | self.thlist.append(thBroadcast) 67 | 68 | def broadcast(self, msg): 69 | self.Q.put(msg) 70 | 71 | def join(self): 72 | for th in self.thlist: 73 | th.join() -------------------------------------------------------------------------------- /sunyata/http/__init__.py: -------------------------------------------------------------------------------- 1 | from sunyata.http.server import HttpServer 2 | 3 | route = HttpServer.route -------------------------------------------------------------------------------- /sunyata/http/factory.py: -------------------------------------------------------------------------------- 1 | from sunyata.http.request import HttpRequest 2 | from sunyata.http.response import HttpResponse 3 | from sunyata.http.router import HttpRouter 4 | from sunyata.util import bytes2str 5 | import ujson 6 | 7 | class HttpFactory(object): 8 | 9 | def __init__(self) -> None: 10 | super().__init__() 11 | 12 | @classmethod 13 | def genLineHeaderBody(self, lines): 14 | getHeader = 1 15 | reqHeaders = [] 16 | reqBody = b'' 17 | reqLine = lines[0] 18 | for l in lines[1:]: 19 | if not l: 20 | getHeader = 0 21 | if getHeader: 22 | reqHeaders.append(l) 23 | else: 24 | reqBody = reqBody + l 25 | return reqLine, reqHeaders, reqBody 26 | 27 | @classmethod 28 | def parseRequestData(cls, uri, body): 29 | requestData = {} 30 | if uri: 31 | try: 32 | kvPairs = uri.split(b'?')[1].split(b'&') 33 | for kvPair in kvPairs: 34 | k, v = kvPair.split(b'=') 35 | requestData[bytes2str(k)] = bytes2str(v) 36 | except: 37 | pass 38 | if body: 39 | try: 40 | bodydic = ujson.decode(body) 41 | requestData.update(bodydic) 42 | except: 43 | pass 44 | return requestData 45 | 46 | @classmethod 47 | async def genHttpRequest(cls, requestStream): 48 | linesStr = await requestStream.reader.read(requestStream.bufsize) 49 | req = HttpRequest() 50 | lines = linesStr.split(b"\r\n") 51 | reqLine, reqHeaders, reqBody = cls.genLineHeaderBody(lines) 52 | method, uri, httpVersion = reqLine.split(b' ') 53 | req.method = bytes2str(method) 54 | req.httpVersion = bytes2str(httpVersion) 55 | req.uri = bytes2str( uri.split(b'?')[0] ) 56 | req.body = reqBody 57 | for header in reqHeaders: 58 | if header == b'': 59 | continue 60 | k, v = header.split(b': ') 61 | try: 62 | req.headers[bytes2str(k)] = bytes2str(v) 63 | except: 64 | req.headers[str(k)] = str(v) 65 | req.data = cls.parseRequestData(uri, req.body) 66 | contentLength = int(req.headers.get('Content-Length', 0)) 67 | hasRead = len(req.body) 68 | toRead = contentLength - hasRead 69 | while toRead: 70 | rdata = requestStream.reader.read(toRead) 71 | if not rdata: 72 | raise Exception('peer closed') 73 | req.body = req.body + rdata 74 | hasRead = hasRead + len(rdata) 75 | toRead = contentLength - hasRead 76 | return req 77 | 78 | @classmethod 79 | def genHttpResponse(cls, status, body = ''): 80 | return HttpResponse(status, body) 81 | 82 | @classmethod 83 | def genHttpRouter(cls, path, func, methods): 84 | httpRouter = HttpRouter() 85 | httpRouter.path = path 86 | httpRouter.func = func 87 | httpRouter.methods = methods 88 | return httpRouter -------------------------------------------------------------------------------- /sunyata/http/middleware.py: -------------------------------------------------------------------------------- 1 | from sunyata.http.factory import HttpFactory 2 | from sunyata.http.status import HttpStatus 3 | import abc 4 | 5 | 6 | class Middleware(metaclass=abc.ABCMeta): 7 | 8 | def __init__(self): 9 | self.resetResponse() 10 | 11 | def resetResponse(self): 12 | self.resp = None 13 | 14 | def abort(self, statusCode, responseText): 15 | status = HttpStatus() 16 | status.code = statusCode 17 | httpResponse = HttpFactory.genHttpResponse(status, responseText) 18 | self.resp = httpResponse 19 | 20 | @abc.abstractmethod 21 | def handle(self, request): 22 | pass -------------------------------------------------------------------------------- /sunyata/http/rawserver.py: -------------------------------------------------------------------------------- 1 | from sunyata.http.factory import HttpFactory 2 | from sunyata.http.status import * 3 | from sunyata.http.request_stream import RequestStream 4 | from sunyata.http.response import HttpResponse 5 | from sunyata.table_writer import TableWriter 6 | from traceback import format_exc 7 | from types import MethodType,FunctionType 8 | import asyncio 9 | import uvloop 10 | import inspect 11 | asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) 12 | 13 | 14 | class RawHttpServer(object): 15 | 16 | routerMap = {} 17 | responseTypeConvertMap = { 18 | set: HttpResponse.responseConvertSetToJson, 19 | dict: HttpResponse.responseConvertToJson, 20 | list: HttpResponse.responseConvertToJson, 21 | tuple: HttpResponse.responseConvertToJson, 22 | bytes: HttpResponse.responseConvertRaw, 23 | } 24 | 25 | def __init__(self, bind = '0.0.0.0', port=9989): 26 | super().__init__() 27 | self.bind = bind 28 | self.port = port 29 | self.middlewares = [] 30 | 31 | @classmethod 32 | def addRoute(cls, path, func, methods = None): 33 | if isinstance(func, FunctionType) or isinstance(func, MethodType): 34 | cls.routerMap[path] = HttpFactory.genHttpRouter(path, func, methods) 35 | elif isinstance(func, object): 36 | controllerObj = func() 37 | if not path.endswith('/'): path += '/' 38 | for classFunc in dir(controllerObj): 39 | if classFunc.startswith('_'): 40 | continue 41 | cls.routerMap[path + classFunc] = HttpFactory.genHttpRouter(path + classFunc, getattr(controllerObj, classFunc), methods) 42 | 43 | async def handleRequest(self, httpRequest): 44 | router = self.routerMap.get(httpRequest.uri, None) 45 | if not router: 46 | return HttpFactory.genHttpResponse(HttpStatus404) 47 | if router.methods and httpRequest.method not in router.methods: 48 | return HttpFactory.genHttpResponse(HttpStatus405) 49 | try: 50 | for middleware in self.middlewares: 51 | middleware.resetResponse() 52 | middleware.handle(httpRequest) 53 | if middleware.resp: 54 | return middleware.resp 55 | if inspect.iscoroutinefunction(router.getFunc()): 56 | resp = await router.getFunc()(httpRequest) 57 | else: 58 | resp = router.getFunc()(httpRequest) 59 | respType = type(resp) 60 | convertMethod = self.responseTypeConvertMap.get(respType, HttpResponse.responseConvertToStr) 61 | resp = convertMethod(resp) 62 | return HttpFactory.genHttpResponse(HttpStatus200, resp) 63 | except Exception as ex: 64 | return HttpFactory.genHttpResponse(HttpStatus500, format_exc()) 65 | 66 | async def acceptStream(self, reader, writer): 67 | try: 68 | requestStream = RequestStream(reader, writer) 69 | httpRequest = await HttpFactory.genHttpRequest(requestStream) 70 | httpResponse = await self.handleRequest(httpRequest) 71 | except Exception as ex: 72 | httpResponse = HttpFactory.genHttpResponse(HttpStatus500, str(ex)) 73 | writer.write(httpResponse.toBytes()) 74 | await writer.drain() 75 | writer.close() 76 | 77 | async def listenAndServe(self): 78 | tasklist = [] 79 | server = await asyncio.start_server(self.acceptStream, self.bind, self.port) 80 | tasklist.append(server.serve_forever()) 81 | for task in tasklist: 82 | await task 83 | 84 | def serve(self): 85 | print('http running on http://%s:%s' % (self.bind, self.port) ) 86 | asyncio.run(self.listenAndServe()) 87 | 88 | @classmethod 89 | def route(cls, path, methods = None): 90 | def wrapper(func): 91 | cls.addRoute(path, func, methods) 92 | return func 93 | return wrapper 94 | 95 | def dumpRoutes(self): 96 | twHeaders = ['Path', 'Methods'] 97 | twRows = [] 98 | for path, httpRouter in self.routerMap.items(): 99 | twRows.append([path, httpRouter.methods]) 100 | tw = TableWriter(twHeaders, twRows) 101 | tw.dump() 102 | 103 | 104 | route = RawHttpServer.route -------------------------------------------------------------------------------- /sunyata/http/request.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass, field 2 | 3 | 4 | @dataclass 5 | class RequestClient(object): 6 | remoteAddr: str = field(default='') 7 | port: int = field(default=0) 8 | 9 | 10 | class HttpRequest(object): 11 | 12 | def __init__(self): 13 | super().__init__() 14 | self.method = '' 15 | self.uri = '' 16 | self.httpVersion = '' 17 | self.body = '' 18 | self.headers = {} 19 | self.data = {} 20 | self.scheme = 'http' 21 | self.queryString = '' 22 | self.client = RequestClient() 23 | 24 | 25 | def setBody(self, body): 26 | self.body = body -------------------------------------------------------------------------------- /sunyata/http/request_stream.py: -------------------------------------------------------------------------------- 1 | class RequestStream(object): 2 | 3 | def __init__(self, reader, writer, bufsize = 1024 * 1024): 4 | self.reader = reader 5 | self.writer = writer 6 | self.bufsize = bufsize 7 | -------------------------------------------------------------------------------- /sunyata/http/response.py: -------------------------------------------------------------------------------- 1 | from sunyata.util import str2bytes, bytes2str 2 | import ujson 3 | 4 | class HttpResponse(object): 5 | 6 | __slots__ = ('httpVersion', 'status', 'headers', 'body') 7 | 8 | def __init__(self, status, body) -> None: 9 | super().__init__() 10 | self.httpVersion = 'HTTP/1.1' 11 | self.status = status 12 | self.headers = {} 13 | self.body = body 14 | 15 | def genHeaders(self): 16 | self.headers['Content-Type'] = 'text/html' 17 | self.headers['Content-Length'] = len(self.body) 18 | 19 | def toBytes(self): 20 | resLine = f"{self.httpVersion} {self.status.code} {self.status.msg}" 21 | self.genHeaders() 22 | resHeaders = '' 23 | for k, v in self.headers.items(): 24 | resHeaders += f"{k}: {v}\r\n" 25 | resHeaders += "\r\n" 26 | resBody = self.body 27 | if type(resBody) == str: 28 | resBody = str2bytes(resBody) 29 | res = str2bytes(resLine + resHeaders) + resBody 30 | return res 31 | 32 | def responseBody(self): 33 | resBody = self.body 34 | if type(resBody) == str: 35 | resBody = str2bytes(resBody) 36 | return resBody 37 | 38 | @classmethod 39 | def responseConvertSetToJson(cls, resp): 40 | return ujson.encode(list(resp)) 41 | 42 | @classmethod 43 | def responseConverBytesToStr(cls, resp): 44 | return bytes2str(resp) 45 | 46 | @classmethod 47 | def responseConvertToJson(cls, resp): 48 | return ujson.encode(resp) 49 | 50 | @classmethod 51 | def responseConvertToBytes(cls, resp): 52 | return bytes(resp) 53 | 54 | @classmethod 55 | def responseConvertToStr(cls, resp): 56 | return str(resp) 57 | 58 | @classmethod 59 | def responseConvertRaw(self, resp): 60 | return resp -------------------------------------------------------------------------------- /sunyata/http/router.py: -------------------------------------------------------------------------------- 1 | class HttpRouter(object): 2 | 3 | __slots__ = ('path', 'methods', 'func') 4 | 5 | def __init__(self) -> None: 6 | super().__init__() 7 | self.path = '' 8 | self.methods = None 9 | self.func = None 10 | 11 | def getFunc(self): 12 | return self.func -------------------------------------------------------------------------------- /sunyata/http/server.py: -------------------------------------------------------------------------------- 1 | from sunyata.http.factory import HttpFactory 2 | from sunyata.http.status import * 3 | import uvicorn 4 | from sunyata.http.rawserver import RawHttpServer 5 | from sunyata.http.request import HttpRequest 6 | import sunyata.util as util 7 | import traceback 8 | import time 9 | import logging 10 | logging.basicConfig(format = '%(asctime)s|%(levelname)s|%(message)s', level=logging.DEBUG) 11 | 12 | 13 | class HttpServer(RawHttpServer): 14 | 15 | routerMap = {} 16 | 17 | def __init__(self, bind = '0.0.0.0', port=9989, accessLog=True): 18 | RawHttpServer.__init__(self, bind=bind, port=port) 19 | self.config = uvicorn.Config(self, host=self.bind, port=self.port, log_level='error') 20 | self.server = uvicorn.Server(self.config) 21 | self.accessLog = accessLog 22 | 23 | def serve(self): 24 | print('http running on http://%s:%s' % (self.bind, self.port) ) 25 | self.dumpRoutes() 26 | self.server.run() 27 | 28 | async def asyncServe(self): 29 | self.server.config.setup_event_loop() 30 | await self.server.serve() 31 | 32 | async def genHttpRequest(self, scheme, httpVersion, method, path, queryString, headers, body, clientTuple): 33 | httpRequest = HttpRequest() 34 | httpRequest.scheme = scheme 35 | httpRequest.httpVersion = httpVersion 36 | httpRequest.method = method 37 | httpRequest.uri = path 38 | httpRequest.queryString = queryString 39 | httpRequest.headers = headers 40 | httpRequest.body = body 41 | httpRequest.client.remoteAddr = clientTuple[0] 42 | httpRequest.client.port = clientTuple[1] 43 | httpRequest.data = HttpFactory.parseRequestData(util.str2bytes(path + "?") + queryString, body) 44 | return httpRequest 45 | 46 | async def readBody(self, receive): 47 | """ 48 | Read and return the entire body from an incoming ASGI message. 49 | """ 50 | body = b'' 51 | more_body = True 52 | while more_body: 53 | message = await receive() 54 | body += message.get('body', b'') 55 | more_body = message.get('more_body', False) 56 | return body 57 | 58 | async def rawHeadersToDict(self, rawHeaders): 59 | headers = {} 60 | for line in rawHeaders: 61 | headers[util.bytes2str(line[0])] = str(util.bytes2str(line[1])) 62 | return headers 63 | 64 | async def __call__(self, scope, receive, send): 65 | assert scope['type'] == 'http' 66 | headers = await self.rawHeadersToDict(scope['headers']) 67 | body = await self.readBody(receive) 68 | begin = time.time() 69 | try: 70 | httpRequest = await self.genHttpRequest( 71 | scope['scheme'], 72 | scope['http_version'], 73 | scope['method'], 74 | scope['path'], 75 | scope['query_string'], 76 | headers, 77 | body, 78 | scope['client'] 79 | ) 80 | httpResponse = await self.handleRequest(httpRequest) 81 | except Exception as ex: 82 | logging.error(str(ex) + ' ' + traceback.format_exc()) 83 | httpResponse = HttpFactory.genHttpResponse(HttpStatus500, str(ex)) 84 | if self.accessLog: 85 | end = time.time() 86 | cost = end - begin 87 | logging.debug('|'.join([scope['client'][0] + ':' + str(scope['client'][1]), scope['method'], scope['scheme'], scope['path'], str(httpResponse.status.code), str(round(cost*1000, 6))+'ms'])) 88 | await send( 89 | { 90 | "type": "http.response.start", 91 | "status": httpResponse.status.code, 92 | "headers": [[b"content-type", b"text/plain"]], 93 | } 94 | ) 95 | await send( 96 | { 97 | "type": "http.response.body", 98 | "body": httpResponse.responseBody(), 99 | } 100 | ) 101 | 102 | 103 | route = HttpServer.route -------------------------------------------------------------------------------- /sunyata/http/status.py: -------------------------------------------------------------------------------- 1 | class HttpStatus(object): 2 | code = 0 3 | msg = '' 4 | 5 | 6 | class HttpStatus200(HttpStatus): 7 | code = 200 8 | msg = 'OK' 9 | 10 | 11 | class HttpStatus400(HttpStatus): 12 | code = 400 13 | msg = 'Bad Request' 14 | 15 | 16 | class HttpStatus401(HttpStatus): 17 | code = 401 18 | msg = 'Unauthorized' 19 | 20 | 21 | class HttpStatus403(HttpStatus): 22 | code = 403 23 | msg = 'Forbidden' 24 | 25 | 26 | class HttpStatus404(HttpStatus): 27 | code = 404 28 | msg = 'Not Found' 29 | 30 | 31 | class HttpStatus405(HttpStatus): 32 | code = 405 33 | msg = 'Method Not Allowed' 34 | 35 | 36 | class HttpStatus500(HttpStatus): 37 | code = 500 38 | msg = 'Internal Server Error' 39 | 40 | 41 | class HttpStatus503(HttpStatus): 42 | code = 503 43 | msg = 'Server Unavailable' -------------------------------------------------------------------------------- /sunyata/http/transport.py: -------------------------------------------------------------------------------- 1 | from socket import * 2 | 3 | class TcpTransport(object): 4 | 5 | __slots__ = ('host', 'port', 'timeout', 'socket', 'keepaliveTimeout', 'backlog') 6 | 7 | def __init__(self, host, port, timeout = 30, keepaliveTimeout=30): 8 | self.host = host 9 | self.port = port 10 | self.timeout = timeout 11 | self.socket = None 12 | self.keepaliveTimeout = keepaliveTimeout 13 | self.backlog = 100 14 | 15 | def bind(self): 16 | self.socket = socket(AF_INET, SOCK_STREAM) 17 | self.socket.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) 18 | self.socket.bind( (self.host, self.port) ) 19 | self.socket.listen(self.backlog) 20 | 21 | def setKeepaliveTimeout(self, keepaliveTimeout: int): 22 | self.keepaliveTimeout = keepaliveTimeout 23 | 24 | def connect(self): 25 | if self.socket == None: 26 | self.socket = socket(AF_INET, SOCK_STREAM) 27 | self.socket.settimeout(self.timeout) 28 | self.socket.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) 29 | self.socket.connect( (self.host, self.port) ) 30 | 31 | def accept(self): 32 | cli, addr = self.socket.accept() 33 | cli.settimeout(self.keepaliveTimeout) 34 | return cli, addr 35 | 36 | def close(self): 37 | if self.socket: 38 | self.socket.close() 39 | self.socket = None 40 | 41 | def reconnect(self): 42 | self.close() 43 | self.connect() 44 | print('tcp client reconnect') 45 | 46 | def recvUntil(self, conn, until): 47 | retbytearr = b'' 48 | index = 0 - len(until) 49 | while True: 50 | rbytes = conn.recv(1) 51 | if rbytes == b'': 52 | break 53 | retbytearr += rbytes 54 | if retbytearr[index:] == until: 55 | break 56 | return retbytearr 57 | 58 | def recvFullHeader(self, conn): 59 | fullbytes = b'' 60 | while True: 61 | rbytes = self.recvUntil(conn, b'\r\n') 62 | fullbytes += rbytes 63 | if rbytes == b'\r\n' or rbytes == b'': 64 | break 65 | return fullbytes 66 | 67 | def recvn(self, conn, n): 68 | toRecv = n 69 | hasRecv = 0 70 | bytearr = b'' 71 | while True: 72 | if hasRecv >= toRecv: 73 | break 74 | bytearr += conn.recv(1) 75 | hasRecv += 1 76 | return bytearr 77 | 78 | def sendAll(self, conn, bytearr): 79 | toSend = len(bytearr) 80 | while toSend > 0: 81 | toSend -= conn.send(bytearr) 82 | bytearr = bytearr[0-toSend:] -------------------------------------------------------------------------------- /sunyata/log.py: -------------------------------------------------------------------------------- 1 | import sys 2 | try: 3 | from imp import reload 4 | except: 5 | pass 6 | reload(sys) 7 | try: 8 | sys.setdefaultencoding('utf-8') 9 | except: 10 | pass 11 | 12 | import logging 13 | from logging.handlers import WatchedFileHandler 14 | from logging import handlers 15 | 16 | 17 | class Log: 18 | 19 | __slots__ = [ 20 | '__path', '__switch', '__fileHandler', '__logger', '_errorCallBack', 21 | '_warningCallBack', '_isOutput' 22 | ] 23 | 24 | __fmt = '%(asctime)s [%(levelname)s] [p:%(process)d] [t:%(thread)d] [%(threadName)s] %(message)s' 25 | __shortFmt = '%(asctime)s-%(levelname)s-%(message)s' 26 | __level = logging.DEBUG 27 | 28 | def __init__(self, path, isRotate=True, logSaveDays=10, isShortLog=False): 29 | self.__path = path 30 | self.__switch = True 31 | fmter = None 32 | if isShortLog: 33 | fmter = logging.Formatter(Log.__shortFmt) 34 | else: 35 | fmter = logging.Formatter(Log.__fmt) 36 | if isRotate: 37 | self.__fileHandler = handlers.TimedRotatingFileHandler( 38 | self.__path, 39 | when='D', 40 | interval=1, 41 | backupCount=logSaveDays, 42 | encoding='utf-8') 43 | else: 44 | self.__fileHandler = WatchedFileHandler(self.__path) 45 | self.__fileHandler.setFormatter(fmter) 46 | self.__logger = logging.getLogger(path) 47 | self.__logger.addHandler(self.__fileHandler) 48 | self.__logger.setLevel(Log.__level) 49 | self._errorCallBack = None 50 | self._warningCallBack = None 51 | self._isOutput = False 52 | 53 | def setOutput(self, tag): 54 | self._isOutput = tag 55 | 56 | def getOutput(self): 57 | return self._isOutput 58 | 59 | def setSwitch(self, switch): 60 | self.__switch = switch 61 | 62 | def getSwitch(self): 63 | return self.__switch 64 | 65 | def getPath(self): 66 | return self.__path 67 | 68 | def debug(self, *args): 69 | logInfo = ' '.join(str(item) for item in args) 70 | if self.__switch: self.__logger.debug(loginfo) 71 | if self._isOutput: print(logInfo) 72 | 73 | def info(self, *args): 74 | logInfo = ' '.join(str(item) for item in args) 75 | if self.__switch: self.__logger.info(logInfo) 76 | if self._isOutput: print(logInfo) 77 | 78 | def warning(self, *args): 79 | logInfo = ' '.join(str(item) for item in args) 80 | if self.__switch: self.__logger.warning(logInfo) 81 | if self._isOutput: print(logInfo) 82 | try: 83 | if self._warningCallBack: self._warningCallBack(logInfo) 84 | except: 85 | pass 86 | 87 | def error(self, *args): 88 | logInfo = ' '.join(str(item) for item in args) 89 | if self.__switch: self.__logger.error('\033[31m' + logInfo + "\033[0m") 90 | if self._isOutput: print('\033[31m' + logInfo + '\033[0m') 91 | try: 92 | if self._errorCallBack: self._errorCallBack(logInfo) 93 | except: 94 | pass 95 | 96 | def fatal(self, *args): 97 | logInfo = ' '.join(str(item) for item in args) 98 | print(logInfo) 99 | import os 100 | os._exit(1) 101 | 102 | def setErrorCallBack(self, func): 103 | self._errorCallBack = func 104 | 105 | def setWarningCallBack(self, func): 106 | self._warningCallBack = func -------------------------------------------------------------------------------- /sunyata/orm.py: -------------------------------------------------------------------------------- 1 | ''' 2 | from sunyata.orm import Model, IntField, CharField 3 | 4 | #orm init 5 | Model.init('127.0.0.1', 3306, 'root', '', 'test2', min_conn_num=10) 6 | 7 | 8 | #define a model 9 | class Nation(Model): 10 | tableName = 'nation' #required 11 | primaryKey = 'id' #required 12 | id = IntField() #field type int 13 | name = CharField() #field type char 14 | 15 | 16 | #create 17 | nation = Nation(id = 30, name = 'dqdsaw') 18 | nation.create() 19 | 20 | #read 21 | obj = Nation.filter('id', '=', 30).get() 22 | print(obj, obj.id, obj.name) 23 | objs = Nation.all() 24 | 25 | #update 26 | obj.name = 'hahaha' 27 | obj.update() 28 | obj = Nation.filter('id', '=', 30).get() 29 | print(obj, obj.id, obj.name) 30 | 31 | #delete 32 | obj.delete() 33 | obj = Nation.filter('id', '=', 30).first() 34 | print(obj) 35 | ''' 36 | 37 | from sunyata.db import PoolDB 38 | from sunyata.db import Orm 39 | 40 | class OrmObject(object): 41 | 42 | def __init__(self): 43 | pass 44 | 45 | 46 | class OrmModel(OrmObject): 47 | _connection = None 48 | tableName = '' 49 | primaryKey = 'id' 50 | 51 | def __init__(self): 52 | self.fieldMap = {} 53 | 54 | @classmethod 55 | def init(cls, host, port, user, passwd, dbName, connect_timeout=10, read_timeout=10, min_conn_num=10,): 56 | cls._connection = PoolDB( 57 | host, port, user, passwd, dbName, 58 | connect_timeout=connect_timeout, read_timeout=read_timeout, 59 | min_conn_num=min_conn_num) 60 | cls._connection.connect() 61 | 62 | @classmethod 63 | def getOrm(cls): 64 | return Orm(cls._connection).table(cls.tableName) 65 | 66 | @classmethod 67 | def getConnection(cls): 68 | return cls.getOrm() 69 | 70 | @classmethod 71 | def getRawConnection(cls): 72 | return cls._connection 73 | 74 | 75 | class OrmTypeObject(OrmObject): 76 | 77 | def __init__(self, comment = ''): 78 | pass 79 | 80 | class IntField(OrmTypeObject): 81 | typeName = 'int' 82 | 83 | class CharField(OrmTypeObject): 84 | typeName = 'char' 85 | 86 | class FloatField(OrmTypeObject): 87 | typeName = 'float' 88 | 89 | 90 | class Model(OrmModel): 91 | 92 | def __init__(self, **args): 93 | super().__init__() 94 | for k, v in args.items(): 95 | if not hasattr(self, k): 96 | raise Exception('no attr' + str(k)) 97 | self.fieldMap[k] = v 98 | 99 | def create(self): 100 | return self.getOrm().data(self.fieldMap).insert() 101 | 102 | @classmethod 103 | def filter(cls, field, cmp, val, raw = False): 104 | rows = cls.getOrm().where(field, cmp, val).get() 105 | if raw: 106 | return rows 107 | querySet = OrmQuerySet() 108 | for row in rows: 109 | obj = cls() 110 | for k, v in row.items(): 111 | if hasattr(obj, k): 112 | setattr(obj, k, v) 113 | obj.fieldMap[k] = v 114 | querySet.append(obj) 115 | return querySet 116 | 117 | def update(self): 118 | data = {} 119 | for k in self.fieldMap.keys(): 120 | data[k] = getattr(self, k) 121 | return self.getOrm().data(data).where(self.primaryKey, '=', getattr(self, self.primaryKey)).update() 122 | 123 | def delete(self): 124 | return self.getOrm().where(self.primaryKey, '=', getattr(self, self.primaryKey)).delete() 125 | 126 | @classmethod 127 | def all(cls, raw = False): 128 | rows = cls.getOrm().get() 129 | if raw: 130 | return rows 131 | querySet = OrmQuerySet() 132 | for row in rows: 133 | obj = cls() 134 | for k, v in row.items(): 135 | if hasattr(obj, k): 136 | setattr(obj, k, v) 137 | obj.fieldMap[k] = v 138 | querySet.append(obj) 139 | return querySet 140 | 141 | 142 | class OrmQuerySet(Model): 143 | 144 | __slots__ = ['set', 'index'] 145 | 146 | def __init__(self): 147 | self.set = [] 148 | self.index = 0 149 | 150 | def append(self, elem): 151 | self.set.append(elem) 152 | 153 | def __iter__(self): 154 | return self 155 | 156 | def __next__(self): 157 | try: 158 | elem = self.set[ self.index ] 159 | self.index = self.index + 1 160 | return elem 161 | except: 162 | raise StopIteration('') 163 | 164 | def objects(self): 165 | return self.set 166 | 167 | def delete(self): 168 | for elem in self.set: 169 | OrmQuerySet.tableName = elem.tableName 170 | self.getOrm().where(elem.primaryKey, '=', getattr(elem, elem.primaryKey)).delete() 171 | 172 | def first(self): 173 | if len(self.set) <= 0: 174 | return None 175 | return self.set[0] 176 | 177 | def get(self): 178 | if len(self.set) <= 0: 179 | raise Exception('no result') 180 | return self.set[0] 181 | 182 | def count(self): 183 | return len(self.set) -------------------------------------------------------------------------------- /sunyata/redislock.py: -------------------------------------------------------------------------------- 1 | ''' 2 | example: 3 | 4 | from sunyata.redislock import get_redis_lock 5 | import time 6 | 7 | rlock = get_redis_lock('192.168.1.1', 6379, 'abcd1234') 8 | 9 | while 1: 10 | if rlock.acquireLock(): 11 | print('accquire lock succed') 12 | time.sleep(25) 13 | print('oper finish') 14 | rlock.releaseLock() 15 | print('release lock') 16 | time.sleep(2) 17 | else: 18 | time.sleep(2) 19 | print('wait lock') 20 | ''' 21 | 22 | import redis 23 | import uuid 24 | 25 | class RedisLock(object): 26 | 27 | def __init__(self, redisConn, expire=30): 28 | super().__init__() 29 | self.redisConn = redisConn 30 | self.key = 'redis_lock_key' 31 | self.expire = expire 32 | self.lastvalue = str(uuid.uuid1()) 33 | 34 | def acquireLock(self): 35 | ret = self.redisConn.setnx(self.key, self.lastvalue) 36 | if ret: 37 | self.redisConn.expire(self.key, self.expire) 38 | return True 39 | return False 40 | 41 | def releaseLock(self): 42 | value = self.redisConn.get(self.key) 43 | if value == None: 44 | return False 45 | if value != self.lastvalue: 46 | return False 47 | self.redisConn.delete(self.key) 48 | return True 49 | 50 | def get_redis_lock(host,port,password): 51 | pool = redis.ConnectionPool(host=host, port=port, decode_responses=True, password=password) 52 | rconn = redis.Redis(connection_pool=pool) 53 | rlock = RedisLock(rconn) 54 | return rlock -------------------------------------------------------------------------------- /sunyata/rpc/__init__.py: -------------------------------------------------------------------------------- 1 | from sunyata.rpc.server import RpcServer 2 | 3 | rpc = RpcServer.rpc -------------------------------------------------------------------------------- /sunyata/rpc/client.py: -------------------------------------------------------------------------------- 1 | from sunyata.rpc.protocal import TcpProtocal, UdpProtocal, HttpProtocal, RpcProtocal, ClientUdpProtocal 2 | from sunyata.rpc.exception import FuncNotFoundException 3 | from sunyata.rpc.discovery import DiscoveryConfig 4 | from sunyata.rpc.discovery import RpcDiscovery 5 | from sunyata.wrap import retryTimes 6 | import socket 7 | import asyncio 8 | 9 | 10 | class Address(object): 11 | 12 | __slots__ = ('host', 'port') 13 | 14 | def __init__(self, host, port): 15 | self.host = host 16 | self.port = port 17 | 18 | 19 | class Callable(object): 20 | 21 | def __init__(self, rpcClient, remoteMethodName): 22 | self.rpcClient = rpcClient 23 | self.remoteMethodName = remoteMethodName 24 | 25 | def __call__(self, *args, **kwargs): 26 | return self.rpcClient.call(self.remoteMethodName, *args, **kwargs) 27 | 28 | def __getattr__(self, key): 29 | if key in dir(self): 30 | return getattr(self, key) 31 | return Callable(self.rpcClient, self.remoteMethodName+'.'+f"{key}") 32 | 33 | 34 | class RpcClient(object): 35 | 36 | __slots__ = ('discoveryConfig', 'discovery', 'instanceIndex', 'protocalMap', 'isSync', 'syncInterval', 'protocal', 'servers', 'serversCount', 'serversProtocalMap', 'serverIndex', 'lastServer', 'timeout') 37 | 38 | def __init__(self): 39 | self.discoveryConfig = None 40 | self.discovery = None 41 | self.instanceIndex = 0 42 | self.protocalMap = {} 43 | self.isSync = False 44 | self.syncInterval = 30 45 | self.protocal = RpcProtocal() 46 | self.servers = [] 47 | self.serversCount = len(self.servers) 48 | self.serversProtocalMap = {} 49 | self.serverIndex = 0 50 | self.lastServer = '' 51 | self.timeout = 10 52 | 53 | def getInstance(self): 54 | instanceList = asyncio.run(self.discovery.getInstanceList(self.discoveryConfig.serviceName)) 55 | instanceLength = len(instanceList) 56 | if instanceLength == 0: 57 | raise Exception('no instance found') 58 | index = self.instanceIndex % instanceLength 59 | self.instanceIndex = self.instanceIndex + 1 60 | return instanceList[index] 61 | 62 | def refreshProtocal(self): 63 | if self.discovery: 64 | instance = self.getInstance() 65 | self.protocal = self.getProtocalInstance(instance) 66 | if self.servers: 67 | self.serversCount = len(self.servers) 68 | index = self.serverIndex % self.serversCount 69 | self.serverIndex = self.serverIndex + 1 70 | server = self.servers[ index ] 71 | self.lastServer = server 72 | self.protocal = self.serversProtocalMap[ server ] 73 | 74 | def getProtocalInstance(self, instance): 75 | #overide 76 | pass 77 | 78 | def setDiscoveryConfig(self, config: DiscoveryConfig): 79 | self.discoveryConfig = config 80 | self.discovery = RpcDiscovery(self.discoveryConfig.consulHost, self.discoveryConfig.consulPort, self.discoveryConfig.consulToken, self.discoveryConfig.etcdHost, self.discoveryConfig.etcdPort) 81 | 82 | def close(self): 83 | self.protocal.transport.close() 84 | 85 | def __del__(self): 86 | self.close() 87 | 88 | def beforeCall(self): 89 | self.refreshProtocal() 90 | 91 | def debugGetLastServer(self): 92 | return self.lastServer 93 | 94 | @retryTimes(retryTimes=3) 95 | def call(self, func, *args, **kwargs): 96 | pass 97 | 98 | def __getattr__(self, key): 99 | if key in dir(self): 100 | return getattr(self, key) 101 | return Callable(self, f"{key}") 102 | 103 | 104 | class TcpRpcClient(RpcClient): 105 | 106 | def __init__(self, host = '', port = 0, servers=[], timeout=10, keepconnect = True): 107 | RpcClient.__init__(self) 108 | self.host = host 109 | self.port = port 110 | self.keepconnect = keepconnect 111 | self.protocal = TcpProtocal(host, port, timeout=timeout) 112 | self.servers = servers 113 | self.timeout = timeout 114 | if len(self.servers) == 0 and self.host != '' and self.port != 0: 115 | self.servers = [ host+':'+str(port) ] 116 | for server in self.servers: 117 | host, port = server.split(':') 118 | self.serversProtocalMap[ server ] = TcpProtocal(host, int(port), timeout = timeout) 119 | 120 | @retryTimes(retryTimes=3) 121 | def call(self, func, *args, **kwargs): 122 | try: 123 | self.beforeCall() 124 | self.protocal.transport.connect() 125 | package = { 126 | 'func' : func, 127 | 'args' : args, 128 | 'kwargs' : kwargs, 129 | } 130 | msg = self.protocal.serialize(package) 131 | self.protocal.transport.send(msg) 132 | respmsg = self.protocal.transport.recv() 133 | resp = self.protocal.unserialize(respmsg) 134 | if isinstance(resp, Exception): 135 | if not self.keepconnect: self.protocal.transport.close() 136 | raise resp 137 | if not self.keepconnect: self.protocal.transport.close() 138 | return resp 139 | except socket.timeout as ex: 140 | self.protocal.transport.close() 141 | raise ex 142 | 143 | def getProtocalInstance(self, instance): 144 | key = "%s:%s:%s" % (instance.service, instance.address, instance.port) 145 | if key in self.protocalMap: 146 | return self.protocalMap.get(key) 147 | self.protocalMap[key] = TcpProtocal(instance.address, instance.port) 148 | return self.protocalMap[key] 149 | 150 | 151 | class UdpRpcClient(RpcClient): 152 | 153 | def __init__(self, host = '', port = 0, servers=[], timeout = 10): 154 | RpcClient.__init__(self) 155 | self.host = host 156 | self.port = port 157 | self.protocal = ClientUdpProtocal(host, port, timeout=timeout) 158 | self.servers = servers 159 | self.timeout = timeout 160 | if len(self.servers) == 0 and self.host != '' and self.port != 0: 161 | self.servers = [ host+':'+str(port) ] 162 | for server in self.servers: 163 | host, port = server.split(':') 164 | self.serversProtocalMap[ server ] = ClientUdpProtocal(host, int(port), timeout=timeout) 165 | 166 | @retryTimes(retryTimes=3) 167 | def call(self, func, *args, **kwargs): 168 | try: 169 | self.beforeCall() 170 | package = { 171 | 'func' : func, 172 | 'args' : args, 173 | 'kwargs' : kwargs 174 | } 175 | msg = self.protocal.serialize(package) 176 | self.protocal.transport.send(msg) 177 | respmsg, _ = self.protocal.transport.recv() 178 | resp = self.protocal.unserialize(respmsg) 179 | if isinstance(resp, Exception): 180 | raise resp 181 | return resp 182 | except socket.timeout as ex: 183 | self.protocal.newTransport() 184 | raise ex 185 | 186 | def getProtocalInstance(self, instance): 187 | key = "%s:%s:%s" % (instance.service, instance.address, instance.port) 188 | if key in self.protocalMap: 189 | return self.protocalMap.get(key) 190 | self.protocalMap[key] = UdpProtocal(instance.address, instance.port) 191 | return self.protocalMap[key] 192 | 193 | 194 | class HttpRpcClient(RpcClient): 195 | 196 | def __init__(self, host = '', port = 0, servers = [], timeout = 10): 197 | RpcClient.__init__(self) 198 | self.host = host 199 | self.port = port 200 | self.protocal = HttpProtocal(host, port, timeout=timeout) 201 | self.servers = servers 202 | self.timeout = timeout 203 | if len(self.servers) == 0 and self.host != '' and self.port != 0: 204 | self.servers = [ host+':'+str(port) ] 205 | for server in self.servers: 206 | host, port = server.split(':') 207 | self.serversProtocalMap[ server ] = HttpProtocal(host, int(port), timeout=timeout) 208 | 209 | @retryTimes(retryTimes=3) 210 | def call(self, func, *args, **kwargs): 211 | self.beforeCall() 212 | package = { 213 | 'func' : func, 214 | 'args' : args, 215 | 'kwargs' : kwargs, 216 | } 217 | msg = self.protocal.serialize(package) 218 | respmsg = asyncio.run(self.protocal.transport.send(msg)) 219 | resp = self.protocal.unserialize(respmsg) 220 | if isinstance(resp, FuncNotFoundException): 221 | raise resp 222 | return resp 223 | 224 | def getProtocalInstance(self, instance): 225 | key = "%s:%s:%s" % (instance.service, instance.address, instance.port) 226 | if key in self.protocalMap: 227 | return self.protocalMap.get(key) 228 | self.protocalMap[key] = HttpProtocal(instance.address, instance.port) 229 | return self.protocalMap[key] -------------------------------------------------------------------------------- /sunyata/rpc/compress.py: -------------------------------------------------------------------------------- 1 | from sunyata.compress import Compress 2 | 3 | class RpcCompress(Compress): 4 | 5 | enableCompressLen = 1024 * 4 #more than 4k enable compress 6 | -------------------------------------------------------------------------------- /sunyata/rpc/const.py: -------------------------------------------------------------------------------- 1 | SERVER_TIMEOUT = 10 -------------------------------------------------------------------------------- /sunyata/rpc/discovery.py: -------------------------------------------------------------------------------- 1 | from sunyata.consul import ConsulApi, Instance 2 | import time 3 | import threading 4 | import asyncio 5 | from sunyata.etcd import EtcdApi 6 | 7 | 8 | class DiscoveryConfig(object): 9 | 10 | def __init__(self, serviceName, consulHost = None, consulPort = None, etcdHost=None, etcdPort=None, serviceHost = '', servicePort = 0, consulToken=''): 11 | self.consulHost = consulHost 12 | self.consulPort = consulPort 13 | self.consulToken = consulToken 14 | self.serviceName = serviceName 15 | self.serviceHost = serviceHost 16 | self.servicePort = servicePort 17 | self.etcdHost = etcdHost 18 | self.etcdPort = etcdPort 19 | 20 | 21 | class RpcDiscovery(object): 22 | 23 | def __init__(self, consulHost=None, consulPort=None, consulToken='', etcdHost=None, etcdPort=None): 24 | self.consulHost = consulHost 25 | self.consulPort = consulPort 26 | self.consulToken = consulToken 27 | self.etcdHost = etcdHost 28 | self.etcdPort = etcdPort 29 | self.api = None 30 | if self.consulHost and self.consulPort: 31 | self.api = ConsulApi(consulHost, consulPort, consulToken) 32 | if self.etcdHost and self.etcdPort: 33 | self.api = EtcdApi(etcdHost, etcdPort) 34 | self.instanceList = [] 35 | self.heartbeatInterval = 15 36 | 37 | #def doHeartbeat(self, service, address, port, interval): 38 | # while 1: 39 | # try: 40 | # self.api.ttlHeartbeat(service, address, port) 41 | # except: 42 | # pass 43 | # time.sleep(interval) 44 | 45 | async def asyncDoHeartbeat(self, service, address, port, interval): 46 | while 1: 47 | try: 48 | await self.api.ttlHeartbeat(service, address, port) 49 | except: 50 | pass 51 | await asyncio.sleep(interval) 52 | 53 | #def regist(self, service, address, port, ttlHeartBeat = True): 54 | # self.api.registService(serviceName = service, address = address, port=port) 55 | # if ttlHeartBeat: 56 | # t = threading.Thread(target=self.doHeartbeat, args=(service, address, port, self.heartbeatInterval)) 57 | # t.setDaemon(True) 58 | # t.start() 59 | 60 | async def asyncRegist(self, service, address, port, ttlHeartBeat = True): 61 | await self.api.registService(serviceName = service, address = address, port=port) 62 | await self.asyncDoHeartbeat(service, address, port, self.heartbeatInterval) 63 | 64 | async def getInstanceList(self, service): 65 | instanceList = await self.api.getServiceInstanceList(service) 66 | return instanceList -------------------------------------------------------------------------------- /sunyata/rpc/encrypt.py: -------------------------------------------------------------------------------- 1 | class Encrypt(object): 2 | 3 | PRIVATE_KEY = b'sunyata' 4 | 5 | @classmethod 6 | def encrypt(cls, bytearr): 7 | return cls.PRIVATE_KEY + bytearr 8 | 9 | @classmethod 10 | def unencrypt(cls, bytearr): 11 | keyLength = len(cls.PRIVATE_KEY) 12 | if len(bytearr) < keyLength: 13 | return False, b'' 14 | header = bytearr[:keyLength] 15 | if header != cls.PRIVATE_KEY: 16 | return False, b'' 17 | return True, bytearr[keyLength:] -------------------------------------------------------------------------------- /sunyata/rpc/exception.py: -------------------------------------------------------------------------------- 1 | class FuncNotFoundException(Exception): 2 | def __init__(self, errmsg): 3 | self.errmsg = errmsg -------------------------------------------------------------------------------- /sunyata/rpc/method.py: -------------------------------------------------------------------------------- 1 | import operator 2 | from sunyata.eventloop import EventLoop 3 | 4 | class RpcMethod(object): 5 | 6 | __slots__ = ('_methodType', '_method', '_classDefine', '_isCoroutine', '_methodType') 7 | 8 | TYPE_WITHOUT_CLASS = 0 9 | TYPE_WITH_CLASS = 1 10 | 11 | def __init__(self, methodType, method, classDefine = None, isCoroutine = False): 12 | self._methodType = methodType 13 | self._method = method 14 | self._classDefine = classDefine 15 | self._isCoroutine = isCoroutine 16 | if self._methodType == self.TYPE_WITH_CLASS and self._classDefine == None: 17 | raise Exception('classDefine is None') 18 | 19 | def call(self, *args, **kwargs): 20 | if self._methodType == self.TYPE_WITH_CLASS: 21 | classInstance = self._classDefine() 22 | if self._isCoroutine: 23 | resp = EventLoop.runUntilComplete( operator.methodcaller(self._method.__name__, *args, **kwargs)(classInstance) ) 24 | else: 25 | resp = operator.methodcaller(self._method.__name__, *args, **kwargs)(classInstance) 26 | else: 27 | if self._isCoroutine: 28 | resp = EventLoop.runUntilComplete( self._method(*args, **kwargs) ) 29 | else: 30 | resp = self._method(*args, **kwargs) 31 | return resp 32 | 33 | async def asyncCall(self, *args, **kwargs): 34 | if self._methodType == self.TYPE_WITH_CLASS: 35 | classInstance = self._classDefine() 36 | if self._isCoroutine: 37 | resp = await ( operator.methodcaller(self._method.__name__, *args, **kwargs)(classInstance) ) 38 | else: 39 | resp = operator.methodcaller(self._method.__name__, *args, **kwargs)(classInstance) 40 | else: 41 | if self._isCoroutine: 42 | resp = await self._method(*args, **kwargs) 43 | else: 44 | resp = self._method(*args, **kwargs) 45 | return resp -------------------------------------------------------------------------------- /sunyata/rpc/protocal.py: -------------------------------------------------------------------------------- 1 | from sunyata.rpc.transport import TcpTransport, UdpTransport, HttpTransport, ClientUdpTransport, RpcTransport 2 | from sunyata.rpc.serialize import BinarySerialize, JsonSerialize 3 | from multiprocessing import cpu_count 4 | 5 | 6 | class RpcProtocal(object): 7 | 8 | __slots__ = ('transport') 9 | 10 | def __init__(self): 11 | self.transport = RpcTransport() 12 | 13 | def serialize(self, obj): 14 | serializer = None 15 | if self.serializeType == 'bin': 16 | serializer = BinarySerialize() 17 | elif self.serializeType == 'json': 18 | serializer = JsonSerialize() 19 | if serializer == None: 20 | raise Exception('unknown serializeType') 21 | return serializer.serialize(obj) 22 | 23 | def unserialize(self, msg): 24 | serializer = None 25 | if self.serializeType == 'bin': 26 | serializer = BinarySerialize() 27 | elif self.serializeType == 'json': 28 | serializer = JsonSerialize() 29 | if serializer == None: 30 | raise Exception('unknown serializeType') 31 | return serializer.unserialize(msg) 32 | 33 | def parseRequest(self, package): 34 | func = package['func'] 35 | args = package['args'] 36 | kwargs = package['kwargs'] 37 | return func, args, kwargs 38 | 39 | 40 | class UdpProtocal(RpcProtocal): 41 | 42 | def __init__(self): 43 | pass 44 | 45 | 46 | class HttpProtocal(RpcProtocal): 47 | 48 | def __init__(self, host, port, worker = cpu_count(), serializeType = 'bin', timeout = 10, poolConnection=5, poolMaxSize = 20, maxRetries = 3): 49 | RpcProtocal.__init__(self) 50 | self.host = host 51 | self.port = port 52 | self.worker = worker 53 | self.serializeType = serializeType 54 | self.timeout = timeout 55 | self.transport = HttpTransport(host, port, timeout, poolConnection, poolMaxSize, maxRetries) 56 | 57 | 58 | class TcpProtocal(RpcProtocal): 59 | 60 | def __init__(self, host, port, serializeType = 'bin', timeout = 10): 61 | RpcProtocal.__init__(self) 62 | self.serializeType = serializeType 63 | self.timeout = timeout 64 | self.transport = TcpTransport(host, port, timeout) 65 | 66 | 67 | class UdpProtocal(RpcProtocal): 68 | 69 | def __init__(self, host, port, serializeType = 'bin', timeout = 10): 70 | RpcProtocal.__init__(self) 71 | self.serializeType = serializeType 72 | self.timeout = timeout 73 | self.transport = UdpTransport(host, port) 74 | 75 | 76 | class ClientUdpProtocal(UdpProtocal): 77 | 78 | def __init__(self, host, port, serializeType = 'bin', timeout = 10): 79 | UdpProtocal.__init__(self, host, port, serializeType=serializeType, timeout=timeout) 80 | self.serializeType = serializeType 81 | self.host = host 82 | self.port = port 83 | self.timeout = timeout 84 | self.transport = ClientUdpTransport(host, port, timeout) 85 | 86 | def newTransport(self): 87 | self.transport.close() 88 | self.transport = ClientUdpTransport(self.host, self.port, self.timeout) -------------------------------------------------------------------------------- /sunyata/rpc/serialize.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import ujson 3 | from abc import ABCMeta, abstractmethod 4 | 5 | 6 | class RpcSerialize(object): 7 | 8 | __metaclass__ = ABCMeta 9 | 10 | @abstractmethod 11 | def serialize(cls, obj) -> bytes: 12 | pass 13 | 14 | @abstractmethod 15 | def unserialize(cls, bytearr): 16 | pass 17 | 18 | 19 | class BinarySerialize(RpcSerialize): 20 | 21 | @classmethod 22 | def serialize(cls, obj) -> bytes: 23 | return pickle.dumps(obj) 24 | 25 | @classmethod 26 | def unserialize(cls, bytearr): 27 | return pickle.loads(bytearr) 28 | 29 | 30 | class JsonSerialize(RpcSerialize): 31 | 32 | @classmethod 33 | def serialize(cls, obj) -> bytes: 34 | return ujson.dumps(obj) 35 | 36 | @classmethod 37 | def unserialize(cls, bytearr): 38 | return ujson.loads(bytearr) -------------------------------------------------------------------------------- /sunyata/rpc/server.py: -------------------------------------------------------------------------------- 1 | from sunyata.rpc.protocal import TcpProtocal, UdpProtocal, HttpProtocal, RpcProtocal 2 | import multiprocessing 3 | from sunyata.rpc.exception import FuncNotFoundException 4 | import queue 5 | import threading 6 | import socket 7 | from sunyata.rpc.discovery import DiscoveryConfig, RpcDiscovery 8 | import struct 9 | from sunyata.rpc.compress import RpcCompress 10 | from types import FunctionType 11 | from sunyata.rpc.method import RpcMethod 12 | import asyncio 13 | import uvloop 14 | import inspect 15 | from sunyata.http.server import HttpServer 16 | from sunyata.table_writer import TableWriter 17 | import time 18 | asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) 19 | 20 | 21 | class RpcServer(object): 22 | 23 | funcMap = {} 24 | funcList = [] 25 | 26 | def __init__(self): 27 | self.discoveryConfig = None 28 | self.discovery = None 29 | self.protocal = RpcProtocal() 30 | self.accessLog = True 31 | 32 | @classmethod 33 | def regist(cls, func): 34 | if isinstance(func, FunctionType): 35 | if inspect.iscoroutinefunction(func): 36 | cls.funcMap[ func.__name__ ] = RpcMethod(RpcMethod.TYPE_WITHOUT_CLASS, func, isCoroutine=True) 37 | cls.funcList = cls.funcMap.keys() 38 | else: 39 | cls.funcMap[ func.__name__ ] = RpcMethod(RpcMethod.TYPE_WITHOUT_CLASS, func) 40 | cls.funcList = cls.funcMap.keys() 41 | else: 42 | classDefine = func 43 | serMethods = list( filter(lambda m: not m.startswith('_'), dir(classDefine)) ) 44 | for methodName in serMethods: 45 | funcName = "{}.{}".format(classDefine.__name__, methodName) 46 | funcObj = getattr(classDefine, methodName) 47 | if inspect.iscoroutinefunction(funcObj): 48 | cls.funcMap [ funcName ] = RpcMethod(RpcMethod.TYPE_WITH_CLASS, funcObj, classDefine, isCoroutine=True) 49 | cls.funcList = cls.funcMap.keys() 50 | else: 51 | cls.funcMap [ funcName ] = RpcMethod(RpcMethod.TYPE_WITH_CLASS, funcObj, classDefine) 52 | cls.funcList = cls.funcMap.keys() 53 | 54 | @classmethod 55 | def getRegistedMethods(cls): 56 | return list(cls.funcMap.keys()) 57 | 58 | def run(self, func, args, kwargs): 59 | try: 60 | if func not in self.funcList: 61 | return FuncNotFoundException('func not found') 62 | methodObj = self.funcMap[func] 63 | args = tuple(args) 64 | begin = time.time() * 1000 65 | if len(args) == 0 and len(kwargs) == 0: 66 | resp = methodObj.call() 67 | else: 68 | resp = methodObj.call(*args, **kwargs) 69 | end = time.time() * 1000 70 | cost = end - begin 71 | if self.accessLog: 72 | print("time:%s, method:%s, args:%s, cost:%sms" % (time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()), func, str(args), round(cost, 6))) 73 | return resp 74 | except Exception as ex: 75 | return Exception('server exception, ' + str(ex)) 76 | 77 | def setDiscoverConfig(self, config: DiscoveryConfig): 78 | self.discoveryConfig = config 79 | self.discovery = RpcDiscovery(self.discoveryConfig.consulHost, self.discoveryConfig.consulPort, self.discoveryConfig.consulToken, self.discoveryConfig.etcdHost, self.discoveryConfig.etcdPort) 80 | 81 | def setKeepaliveTimeout(self, keepaliveTimeout: int): 82 | self.protocal.transport.setKeepaliveTimeout(keepaliveTimeout) 83 | 84 | @classmethod 85 | def rpc(cls, func): 86 | cls.regist(func) 87 | return func 88 | 89 | def printLogo(self): 90 | logo = """""" 91 | print(logo) 92 | 93 | def dumpRpcFuncs(self): 94 | twHeaders = ['RPC Methods'] 95 | twRows = [ [elem] for elem in self.funcList ] 96 | tw = TableWriter(twHeaders, list(twRows)) 97 | tw.dump() 98 | 99 | 100 | class SimpleTcpRpcServer(RpcServer): 101 | 102 | def __init__(self, host, port): 103 | RpcServer.__init__(self) 104 | self.host = host 105 | self.port = port 106 | self.protocal = TcpProtocal(host, port) 107 | 108 | def serve(self): 109 | self.protocal.transport.bind(self.keepaliveTimeout) 110 | self.printLogo() 111 | while 1: 112 | msg = self.protocal.transport.recv() 113 | request = self.protocal.unserialize(msg) 114 | func, args, kwargs = self.protocal.parseRequest(request) 115 | resp = self.run(func, args, kwargs) 116 | self.protocal.transport.send(self.protocal.serialize(resp)) 117 | 118 | 119 | class BlockTcpRpcServer(SimpleTcpRpcServer): 120 | 121 | def __init__(self, host, port, workers = multiprocessing.cpu_count()): 122 | SimpleTcpRpcServer.__init__(self, host, port) 123 | self.worker = workers 124 | 125 | def handle(self, conn): 126 | while 1: 127 | try: 128 | msg = self.protocal.transport.recvPeer(conn) 129 | request = self.protocal.unserialize(msg) 130 | func, args, kwargs = self.protocal.parseRequest(request) 131 | resp = self.run(func, args, kwargs) 132 | self.protocal.transport.sendPeer(self.protocal.serialize(resp), conn) 133 | except Exception as ex: 134 | conn.close() 135 | return 136 | 137 | def serve(self): 138 | self.protocal.transport.bind() 139 | self.printLogo() 140 | self.dumpRpcFuncs() 141 | if self.discovery and self.discoveryConfig: 142 | self.discovery.regist(self.discoveryConfig.serviceName, self.discoveryConfig.serviceHost, self.discoveryConfig.servicePort, ttlHeartBeat=True) 143 | while 1: 144 | conn, _ = self.protocal.transport.accept() 145 | t = threading.Thread(target=self.handle, args=(conn,) ) 146 | t.setDaemon(True) 147 | t.start() 148 | 149 | 150 | class HttpRpcServer(RpcServer): 151 | 152 | def __init__(self, host, port, workers = multiprocessing.cpu_count()): 153 | RpcServer.__init__(self) 154 | self.host = host 155 | self.port = port 156 | self.protocal = HttpProtocal(host, port, workers) 157 | self.app = HttpServer(self.host, self.port) 158 | self.app.addRoute('/', self.handle) 159 | 160 | def handle(self, request): 161 | body = request.body 162 | resp = self.callback(body) 163 | return resp 164 | 165 | def callback(self, package): 166 | isEnableCompress = package[:1] 167 | msg = package[1:] 168 | if isEnableCompress == b'1': 169 | msg = RpcCompress.decompress(msg) 170 | request = self.protocal.unserialize(msg) 171 | func, args, kwargs = self.protocal.parseRequest(request) 172 | resp = self.run(func, args, kwargs) 173 | resp = self.protocal.serialize(resp) 174 | return resp 175 | 176 | async def asyncServe(self): 177 | tasks = [] 178 | tasks.append(asyncio.create_task(self.app.asyncServe())) 179 | if self.discovery and self.discoveryConfig: 180 | registTask = asyncio.create_task(self.discovery.asyncRegist(self.discoveryConfig.serviceName, self.discoveryConfig.serviceHost, self.discoveryConfig.servicePort, ttlHeartBeat=True)) 181 | tasks.append(registTask) 182 | await asyncio.wait(tasks) 183 | 184 | def serve(self): 185 | print('http rpc running on http://%s:%s' % (self.host, self.port) ) 186 | self.dumpRpcFuncs() 187 | asyncio.run(self.asyncServe()) 188 | 189 | 190 | class TcpRpcServer(BlockTcpRpcServer): 191 | 192 | def __init__(self, host, port): 193 | BlockTcpRpcServer.__init__(self, host, port) 194 | self.host = host 195 | self.port = port 196 | 197 | async def handle(self, reader, writer): 198 | while 1: 199 | try: 200 | data = await reader.read(5) 201 | if data == b'': 202 | raise Exception('peer closed') 203 | lengthField = data[:4] 204 | compressField = data[4:5] 205 | isEnableCompress = 0 206 | if compressField == b'1': 207 | isEnableCompress = 1 208 | toread = struct.unpack("i", lengthField)[0] 209 | msg = b'' 210 | readn = toread 211 | while 1: 212 | rmsg = await reader.read(readn) 213 | if rmsg == b'': 214 | raise Exception('peer closed') 215 | msg = msg + rmsg 216 | if len(msg) == toread: 217 | readn = toread - len(rmsg) 218 | break 219 | if isEnableCompress: 220 | msg = RpcCompress.decompress(msg) 221 | request = self.protocal.unserialize(msg) 222 | func, args, kwargs = self.protocal.parseRequest(request) 223 | resp = await self.run(func, args, kwargs) 224 | respbytes = self.protocal.serialize(resp) 225 | 226 | isEnableCompress = b'0' 227 | if len(msg) >= RpcCompress.enableCompressLen: 228 | isEnableCompress = b'1' 229 | respbytes = RpcCompress.compress(respbytes) 230 | lenbytes = struct.pack("i", len(respbytes)) 231 | writer.write( lenbytes + isEnableCompress + respbytes) 232 | except Exception as ex: 233 | #print(ex, traceback.format_exc()) 234 | writer.close() 235 | return 236 | 237 | async def main(self): 238 | loop = asyncio.get_event_loop() 239 | coro = asyncio.start_server(self.handle, self.host, self.port, loop=loop) 240 | server = loop.run_until_complete(coro) 241 | print('tcp rpc running on tcp://%s:%s' % (self.host, self.port)) 242 | try: 243 | loop.run_forever() 244 | except KeyboardInterrupt: 245 | pass 246 | # Close the server 247 | server.close() 248 | loop.run_until_complete(server.wait_closed()) 249 | loop.close() 250 | 251 | async def run(self, func, args, kwargs): 252 | try: 253 | if func not in self.funcList: 254 | return FuncNotFoundException('func not found') 255 | methodObj = self.funcMap[func] 256 | args = tuple(args) 257 | if len(args) == 0 and len(kwargs) == 0: 258 | resp = await methodObj.asyncCall() 259 | else: 260 | resp = await methodObj.asyncCall(*args, **kwargs) 261 | return resp 262 | except Exception as ex: 263 | return Exception('server exception, ' + str(ex)) 264 | 265 | def serve(self): 266 | loop = asyncio.get_event_loop() 267 | tasks = [] 268 | tRegist = None 269 | if self.discovery and self.discoveryConfig: 270 | tRegist = self.discovery.asyncRegist(self.discoveryConfig.serviceName, self.discoveryConfig.serviceHost, self.discoveryConfig.servicePort, ttlHeartBeat=True) 271 | tasks.append(tRegist) 272 | coro = asyncio.start_server(self.handle, self.host, self.port, loop=loop) 273 | tasks.append(coro) 274 | print('tcp rpc running on tcp://%s:%s' % (self.host, self.port) ) 275 | self.dumpRpcFuncs() 276 | rs = loop.run_until_complete(asyncio.gather(*tasks)) 277 | try: 278 | loop.run_forever() 279 | except KeyboardInterrupt: 280 | pass 281 | server = rs[-1] 282 | # Close the server 283 | server.close() 284 | loop.run_until_complete(server.wait_closed()) 285 | loop.close() 286 | 287 | 288 | class UdpRpcServer(RpcServer): 289 | 290 | def __init__(self, host, port, workers = multiprocessing.cpu_count()): 291 | RpcServer.__init__(self) 292 | self.protocal = UdpProtocal(host, port) 293 | self.worker = workers 294 | self.queueMaxSize = 100000 295 | self.queue = queue.Queue(self.queueMaxSize) 296 | self.host = host 297 | self.port = port 298 | 299 | def startWorkers(self): 300 | for i in range(self.worker): 301 | t = threading.Thread(target=self.handle) 302 | t.setDaemon(True) 303 | t.start() 304 | 305 | def handle(self): 306 | while 1: 307 | try: 308 | body = self.queue.get() 309 | addr = body.get('addr') 310 | msg = body.get('msg') 311 | request = self.protocal.unserialize(msg) 312 | func, args, kwargs = self.protocal.parseRequest(request) 313 | resp = self.run(func, args, kwargs) 314 | self.protocal.transport.sendPeer(self.protocal.serialize(resp), addr = addr) 315 | except Exception as ex: 316 | print('UDP handler exception:', ex) 317 | 318 | async def handlePackage(self): 319 | while 1: 320 | try: 321 | msg, cliAddr = self.protocal.transport.recv() 322 | self.queue.put({'msg' : msg, 'addr' : cliAddr}) 323 | except socket.timeout: 324 | pass 325 | 326 | async def asyncServe(self): 327 | self.startWorkers() 328 | self.protocal.transport.bind() 329 | self.printLogo() 330 | print('udp rpc running on udp://%s:%s' % (self.host, self.port)) 331 | self.dumpRpcFuncs() 332 | tasks = [] 333 | if self.discovery and self.discoveryConfig: 334 | tasks.append(asyncio.create_task(self.discovery.asyncRegist(self.discoveryConfig.serviceName, self.discoveryConfig.serviceHost, self.discoveryConfig.servicePort, ttlHeartBeat=True))) 335 | tasks.append(asyncio.create_task(self.handlePackage())) 336 | loop = asyncio.get_event_loop() 337 | rs = loop.run_until_complete(asyncio.gather(*tasks)) 338 | try: 339 | loop.run_forever() 340 | except KeyboardInterrupt: 341 | pass 342 | server = rs[-1] 343 | # Close the server 344 | server.close() 345 | loop.run_until_complete(server.wait_closed()) 346 | loop.close() 347 | 348 | def serve(self): 349 | asyncio.run(self.asyncServe()) 350 | 351 | 352 | rpc = RpcServer.rpc -------------------------------------------------------------------------------- /sunyata/rpc/transport.py: -------------------------------------------------------------------------------- 1 | from socket import * 2 | import struct 3 | from sunyata.rpc.const import SERVER_TIMEOUT 4 | from sunyata.rpc.compress import RpcCompress 5 | import aiohttp 6 | 7 | 8 | class RpcTransport(object): 9 | 10 | def close(self): 11 | pass 12 | 13 | 14 | class TcpTransport(RpcTransport): 15 | 16 | __slots__ = ('host', 'port', 'timeout', 'socket', 'keepaliveTimeout') 17 | 18 | def __init__(self, host, port, timeout): 19 | RpcTransport.__init__(self) 20 | self.host = host 21 | self.port = port 22 | self.timeout = timeout 23 | self.socket = None 24 | self.keepaliveTimeout = SERVER_TIMEOUT 25 | 26 | def bind(self): 27 | self.socket = socket(AF_INET, SOCK_STREAM) 28 | self.socket.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) 29 | self.socket.bind( (self.host, self.port) ) 30 | self.socket.listen(100) 31 | 32 | def setKeepaliveTimeout(self, keepaliveTimeout: int): 33 | self.keepaliveTimeout = keepaliveTimeout 34 | 35 | def connect(self): 36 | if self.socket == None: 37 | self.socket = socket(AF_INET, SOCK_STREAM) 38 | self.socket.settimeout(self.timeout) 39 | self.socket.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) 40 | self.socket.connect( (self.host, self.port) ) 41 | 42 | def accept(self): 43 | cli, addr = self.socket.accept() 44 | cli.settimeout(self.keepaliveTimeout) 45 | return cli, addr 46 | 47 | def send(self, msg: bytes): 48 | isEnableCompress = b'0' 49 | oldMsgLen = len(msg) 50 | oldMsg = msg 51 | if len(msg) >= RpcCompress.enableCompressLen: 52 | isEnableCompress = b'1' 53 | msg = RpcCompress.compress(msg) 54 | newMsgLen = len(msg) 55 | if newMsgLen > oldMsgLen: 56 | isEnableCompress = b'0' 57 | msg = oldMsg 58 | newbyte = struct.pack("i", len(msg)) 59 | try: 60 | self.socket.sendall(newbyte + isEnableCompress + msg) 61 | except BrokenPipeError: 62 | self.reconnect() 63 | self.socket.sendall(newbyte + isEnableCompress + msg) 64 | 65 | def sendPeer(self, msg: bytes, conn): 66 | isEnableCompress = b'0' 67 | if len(msg) >= RpcCompress.enableCompressLen: 68 | isEnableCompress = b'1' 69 | msg = RpcCompress.compress(msg) 70 | newbyte = struct.pack("i", len(msg)) 71 | conn.sendall(newbyte + isEnableCompress + msg) 72 | 73 | def getSendByte(self, msg: bytes, conn = None): 74 | newbyte = struct.pack("i", len(msg)) 75 | ret = newbyte + msg 76 | return ret 77 | 78 | def recv(self): 79 | #每次读一个完整的字节，再接收前4个字节，再取body 80 | conn = self.socket 81 | toread = 5 82 | readn = 0 83 | lengthbyte = b'' 84 | while 1: 85 | bufsize = toread - readn 86 | if bufsize <= 0: 87 | break 88 | bytearr = conn.recv(bufsize) 89 | if len(bytearr) == 0: 90 | raise Exception('peer closed') 91 | lengthbyte = lengthbyte + bytearr 92 | readn = readn + len(bytearr) 93 | lengthField = lengthbyte[:4] 94 | compressField = lengthbyte[4:5] 95 | isEnableCompress = 0 96 | if compressField == b'1': 97 | isEnableCompress = 1 98 | toread = struct.unpack("i", lengthField)[0] 99 | readn = 0 100 | msg = b'' 101 | while 1: 102 | bufsize = toread - readn 103 | if bufsize <= 0: 104 | break 105 | bytearr = conn.recv(bufsize) 106 | if len(bytearr) == 0: 107 | raise Exception('peer closed') 108 | msg = msg + bytearr 109 | readn = readn + len(bytearr) 110 | if isEnableCompress: 111 | msg = RpcCompress.decompress(msg) 112 | return msg 113 | 114 | def recvPeer(self, conn): 115 | #每次读一个完整的字节，再接收前4个字节，再取body 116 | toread = 5 117 | readn = 0 118 | lengthbyte = b'' 119 | while 1: 120 | bufsize = toread - readn 121 | if bufsize <= 0: 122 | break 123 | bytearr = conn.recv(bufsize) 124 | if len(bytearr) == 0: 125 | raise Exception('peer closed') 126 | lengthbyte = lengthbyte + bytearr 127 | readn = readn + len(bytearr) 128 | lengthField = lengthbyte[:4] 129 | compressField = lengthbyte[4:5] 130 | isEnableCompress = 0 131 | if compressField == b'1': 132 | isEnableCompress = 1 133 | toread = struct.unpack("i", lengthField)[0] 134 | readn = 0 135 | msg = b'' 136 | while 1: 137 | bufsize = toread - readn 138 | if bufsize <= 0: 139 | break 140 | bytearr = conn.recv(bufsize) 141 | if len(bytearr) == 0: 142 | raise Exception('peer closed') 143 | msg = msg + bytearr 144 | readn = readn + len(bytearr) 145 | if isEnableCompress: 146 | msg = RpcCompress.decompress(msg) 147 | return msg 148 | 149 | def close(self): 150 | if self.socket: 151 | self.socket.close() 152 | self.socket = None 153 | 154 | def reconnect(self): 155 | self.close() 156 | self.connect() 157 | print('tcp client reconnect') 158 | 159 | 160 | class UdpTransport(RpcTransport): 161 | 162 | __slots__ = ('host', 'port', 'socket', 'timeout') 163 | 164 | def __init__(self, host, port, timeout = SERVER_TIMEOUT): 165 | RpcTransport.__init__(self) 166 | self.host = host 167 | self.port = port 168 | self.socket = None 169 | self.timeout = timeout 170 | self.socket = socket(AF_INET, SOCK_DGRAM) 171 | self.socket.settimeout(self.timeout) 172 | self.socket.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) 173 | self.maxUdpPackageSize = 65535 174 | 175 | def bind(self): 176 | self.socket.bind( (self.host, self.port) ) 177 | 178 | def sendPeer(self, msg: bytes, addr): 179 | isEnableCompress = b'0' 180 | if len(msg) >= RpcCompress.enableCompressLen: 181 | isEnableCompress = b'1' 182 | msg = RpcCompress.compress(msg) 183 | package = isEnableCompress + msg 184 | self.socket.sendto(package, addr) 185 | 186 | def send(self, msg: bytes): 187 | isEnableCompress = b'0' 188 | if len(msg) >= RpcCompress.enableCompressLen: 189 | isEnableCompress = b'1' 190 | msg = RpcCompress.compress(msg) 191 | if len(msg) >= self.maxUdpPackageSize: 192 | raise Exception('msg too long > udp allow size 65535') 193 | addr = (self.host, self.port) 194 | package = isEnableCompress + msg 195 | self.socket.sendto(package, addr) 196 | 197 | def recv(self, conn = None): 198 | if conn == None: 199 | conn = self.socket 200 | package, addr = conn.recvfrom(100000) 201 | isEnableCompress = package[:1] 202 | msg = package[1:] 203 | if isEnableCompress == b'1': 204 | msg = RpcCompress.decompress(msg) 205 | return msg, addr 206 | 207 | def close(self): 208 | if self.socket: 209 | self.socket.close() 210 | self.socket = None 211 | 212 | 213 | class ClientUdpTransport(UdpTransport): 214 | 215 | __slots__ = ('timeout') 216 | 217 | def __init__(self, host, port, timeout = 10): 218 | UdpTransport.__init__(self, host, port) 219 | self.timeout = timeout 220 | self.socket.settimeout(self.timeout) 221 | 222 | 223 | class HttpTransport(RpcTransport): 224 | 225 | def __init__(self, host, port, timeout = 10, poolConnection=20, poolMaxSize=20, maxRetries=2): 226 | RpcTransport.__init__(self) 227 | self.host = host 228 | self.port = port 229 | self.timeout = timeout 230 | self.url = self.makeUrl() 231 | self.headers = { 232 | 'Content-type' : 'application/octet-stream' 233 | } 234 | 235 | def makeUrl(self): 236 | return "http://%s:%s/" % (self.host, self.port) 237 | 238 | async def send(self, msg): 239 | isEnableCompress = b'0' 240 | if len(msg) >= RpcCompress.enableCompressLen: 241 | isEnableCompress = b'1' 242 | msg = RpcCompress.compress(msg) 243 | msg = isEnableCompress + msg 244 | async with aiohttp.ClientSession() as sess: 245 | async with sess.post(self.url, headers = {}, data=msg, timeout = self.timeout) as r: 246 | content = await r.read() 247 | return content -------------------------------------------------------------------------------- /sunyata/table_writer.py: -------------------------------------------------------------------------------- 1 | ''' 2 | usage: 3 | 4 | header = ['name', 'age'] 5 | rows = [ ['wangermazi', 25], ['zhangsan', 11], ['lisi', 30] ] 6 | tbWriter = TableWriter(header, rows) 7 | tbWriter.dump() 8 | string = tbWriter.getString() 9 | print(string) 10 | 11 | +--------------+-------+ 12 | | name | age | 13 | +--------------+-------+ 14 | | wangermazi | 25 | 15 | | zhangsan | 11 | 16 | | lisi | 30 | 17 | +--------------+-------+ 18 | ''' 19 | 20 | 21 | class TableWriter: 22 | 23 | __slots__ = ['_maxCol', '_header', '_rows', '_colMaxLengthList', '_lWidth', '_rWidth'] 24 | 25 | def __init__(self, header=[], rows=[], lWidth=2, rWidth=2): 26 | if type(header) != list or type(rows) != list: 27 | raise Exception('param headers, rows must be list type') 28 | 29 | self._maxCol = len(header) 30 | for row in rows: 31 | if len(row) > self._maxCol: 32 | self._maxCol = len(row) 33 | 34 | self._header = header 35 | self._rows = rows 36 | self._colMaxLengthList = self._getColsMaxLength() 37 | self._lWidth = lWidth 38 | self._rWidth = rWidth 39 | 40 | def _getColsMaxLength(self): 41 | allRows = [] 42 | allRows.append(self._header) 43 | for row in self._rows: 44 | allRows.append(row) 45 | 46 | colMaxLengthList = [] 47 | for i in range(self._maxCol): 48 | colMaxLength = 0 49 | 50 | for row in allRows: 51 | try: 52 | fieldLength = len(str(row[i])) 53 | except: 54 | continue 55 | if fieldLength > colMaxLength: 56 | colMaxLength = fieldLength 57 | 58 | colMaxLengthList.append(colMaxLength) 59 | 60 | return colMaxLengthList 61 | 62 | def _genFieldStr(self, string, maxLength): 63 | string = str(string) 64 | restSpaceLength = maxLength - len(string) 65 | ret = ''.join([' ' for i in range(self._lWidth)]) 66 | ret = ret + string + ''.join( 67 | [' ' for i in range(restSpaceLength + self._rWidth)]) 68 | return ret 69 | 70 | def getString(self): 71 | headerFieldList = [] 72 | for i in range(len(self._header)): 73 | fieldStr = self._genFieldStr(self._header[i], 74 | self._colMaxLengthList[i]) 75 | headerFieldList.append(fieldStr) 76 | headerStr = '|' + '|'.join(headerFieldList) + '|' + "\n" 77 | 78 | rowListStr = '' 79 | for row in self._rows: 80 | rowFieldList = [] 81 | for i in range(len(row)): 82 | fieldStr = self._genFieldStr(row[i], self._colMaxLengthList[i]) 83 | rowFieldList.append(fieldStr) 84 | rowStr = '|' + '|'.join(rowFieldList) + '|' 85 | rowListStr = rowListStr + rowStr + "\n" 86 | 87 | indentList = [] 88 | for maxLength in self._colMaxLengthList: 89 | fieldStr = ''.join( 90 | ['-' for i in range(maxLength + self._lWidth + self._rWidth)]) 91 | indentList.append(fieldStr) 92 | indentStr = '+' + '+'.join(indentList) + '+' + "\n" 93 | 94 | ret = '' 95 | if len(self._header) > 0: 96 | ret = ret + indentStr + headerStr + indentStr 97 | if len(self._rows) > 0: 98 | ret = ret + rowListStr + indentStr 99 | 100 | return ret 101 | 102 | def dump(self): 103 | print(self.getString()) -------------------------------------------------------------------------------- /sunyata/util.py: -------------------------------------------------------------------------------- 1 | import socket 2 | 3 | def hostname(): 4 | return socket.gethostname() 5 | 6 | def local_ip(): 7 | try: 8 | ip = socket.gethostbyname(socket.gethostname()) 9 | if ip == '127.0.0.1': 10 | s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 11 | s.connect(('8.8.8.8', 80)) 12 | ip = s.getsockname()[0] 13 | except: 14 | ip = '127.0.0.1' 15 | return ip 16 | 17 | def bytes2str(bytes): 18 | return str(bytes, encoding = 'utf-8') 19 | 20 | def str2bytes(string): 21 | return string.encode(encoding="utf-8") -------------------------------------------------------------------------------- /sunyata/wrap.py: -------------------------------------------------------------------------------- 1 | from multiprocessing import Process 2 | import functools 3 | import time 4 | import socket 5 | from dataclasses import dataclass 6 | 7 | 8 | def stdout_log(func): 9 | ''' 10 | print to stdout 11 | for example: [2020-06-23 10:19:09] call test() method, args:(1, 2), return:3, cost:2.113 seconds 12 | ''' 13 | @functools.wraps(func) 14 | def wrapper(*args, **kwargs): 15 | begin = time.time() 16 | ret = func(*args, **kwargs) 17 | cost = time.time() - begin 18 | return ret 19 | 20 | return wrapper 21 | 22 | 23 | EXCEPTION = 'sunyata_wrap_exception' 24 | 25 | 26 | def safe(func): 27 | ''' 28 | catch exception 29 | ''' 30 | @functools.wraps(func) 31 | def wrapper(*args, **kwargs): 32 | ret = EXCEPTION 33 | try: 34 | ret = func(*args, **kwargs) 35 | except Exception as ex: 36 | pass 37 | return ret 38 | 39 | return wrapper 40 | 41 | 42 | def retryTimes(retryTimes=2): 43 | if retryTimes <= 0: 44 | retryTimes = 1 45 | 46 | def function(func): 47 | @functools.wraps(func) 48 | def wrapper(*args, **kwargs): 49 | ret = None 50 | lastEx = None 51 | for i in range(retryTimes): 52 | try: 53 | ret = func(*args, **kwargs) 54 | return ret 55 | except socket.timeout as ex: 56 | raise ex 57 | except Exception as ex: 58 | lastEx = ex 59 | raise lastEx 60 | 61 | return wrapper 62 | 63 | return function 64 | 65 | 66 | def retry(func): 67 | @functools.wraps(func) 68 | def wrapper(*args, **kwargs): 69 | ret = None 70 | lastEx = None 71 | for i in range(2): 72 | try: 73 | ret = func(*args, **kwargs) 74 | return ret 75 | except socket.timeout as ex: 76 | raise ex 77 | except Exception as ex: 78 | lastEx = ex 79 | raise lastEx 80 | 81 | return wrapper 82 | 83 | 84 | def loop(func): 85 | ''' 86 | loop exec function when it return 87 | ''' 88 | @functools.wraps(func) 89 | def wrapper(*args, **kwargs): 90 | while 1: 91 | func(*args, **kwargs) 92 | 93 | return wrapper 94 | 95 | 96 | def subprocess(func): 97 | ''' 98 | run in sub process 99 | ''' 100 | @functools.wraps(func) 101 | def wrapper(*args, **kwargs): 102 | p = Process(target=func, args=args, kwargs=kwargs) 103 | p.start() 104 | 105 | return wrapper 106 | 107 | 108 | FUNC_EXEC_MAP = {} 109 | 110 | 111 | def exec_once(func): 112 | @functools.wraps(func) 113 | def wrapper(*args, **kwargs): 114 | global FUNC_EXEC_MAP 115 | if func.__name__ in FUNC_EXEC_MAP: 116 | return 117 | ret = func(*args, **kwargs) 118 | FUNC_EXEC_MAP[func.__name__] = 1 119 | return ret 120 | 121 | return wrapper 122 | 123 | 124 | def force_return_string(func): 125 | @functools.wraps(func) 126 | def wrapper(*args, **kwargs): 127 | ret = func(*args, **kwargs) 128 | return str(ret) 129 | 130 | return wrapper 131 | 132 | 133 | FUNC_REQUESTNUM_MAP = {} 134 | 135 | 136 | @dataclass 137 | class RateLimitItem: 138 | laststamp: int 139 | count: int 140 | 141 | 142 | def ratelimit(perSecondMaxRequest): 143 | if type(perSecondMaxRequest) != int or perSecondMaxRequest < 0: 144 | raise Exception('ratelimit param perSecondMaxRequest is invalid') 145 | def function(func): 146 | @functools.wraps(func) 147 | def wrapper(*args, **kwargs): 148 | now = int(time.time()) 149 | global FUNC_REQUESTNUM_MAP 150 | if func.__name__ in FUNC_REQUESTNUM_MAP: 151 | ritem = FUNC_REQUESTNUM_MAP[func.__name__] 152 | if now - ritem.laststamp >= 1: 153 | ritem.laststamp = now 154 | ritem.count = 1 155 | FUNC_REQUESTNUM_MAP[func.__name__] = ritem 156 | else: 157 | FUNC_REQUESTNUM_MAP[func.__name__].count += 1 158 | else: 159 | FUNC_REQUESTNUM_MAP[func.__name__] = RateLimitItem(laststamp=now, count=1) 160 | if FUNC_REQUESTNUM_MAP[func.__name__].count > perSecondMaxRequest: 161 | raise Exception('execced per second max request %s' % perSecondMaxRequest) 162 | ret = func(*args, **kwargs) 163 | return ret 164 | return wrapper 165 | 166 | return function 167 | 168 | 169 | def singleton(cls): 170 | _instance = {} 171 | 172 | def inner(): 173 | if cls not in _instance: 174 | _instance[cls] = cls() 175 | return _instance[cls] 176 | return inner -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | """ 2 | 单元测试脚本: 3 | 4 | python3 -W ignore test.py 5 | """ 6 | 7 | import threading 8 | import time 9 | import unittest 10 | from multiprocessing import Process 11 | from sunyata.util import local_ip 12 | import socket 13 | import random 14 | from sunyata.rpc.compress import RpcCompress 15 | from sunyata.rpc.server import TcpRpcServer 16 | from sunyata.rpc.client import TcpRpcClient 17 | from sunyata.rpc.server import UdpRpcServer 18 | from sunyata.rpc.client import UdpRpcClient 19 | from sunyata.rpc.discovery import DiscoveryConfig 20 | from sunyata.rpc.compress import RpcCompress 21 | from sunyata.rpc.server import HttpRpcServer 22 | from sunyata.rpc.client import HttpRpcClient 23 | from sunyata.rpc import rpc 24 | from sunyata.http.server import HttpServer 25 | from multiprocessing import Process 26 | import asyncio 27 | import aiohttp 28 | from sunyata.http.middleware import Middleware 29 | import requests 30 | 31 | 32 | CONSUL_HOST = '192.168.19.103' 33 | CONSUL_PORT = 8500 34 | CONSUL_TOKEN = 'c594c65b-0f00-bc9b-5ba1-e8242e287f12' 35 | ETCD_HOST = '192.168.19.103' 36 | ETCD_PORT = 2379 37 | SLEEP = 2 38 | IS_TEST_DISCOVERY = False 39 | 40 | 41 | @rpc 42 | class TestService(object): 43 | def add(self, n1, n2): 44 | return n1 + n2 45 | def big_array(self, array): 46 | array.sort() 47 | return array 48 | 49 | def sayHello(name): return 'hello ' + name 50 | 51 | def testTimeout(): 52 | time.sleep(3) 53 | return 'finish' 54 | 55 | class A(object): 56 | def __init__(self, val): 57 | self.val = val 58 | def dump(self): 59 | print('self.val', self.val) 60 | 61 | 62 | class TestRpcServerClient(unittest.TestCase): 63 | 64 | def test_tcp_server_client(self): 65 | #测试TCP 66 | def create_server(): 67 | server = TcpRpcServer('0.0.0.0', 9988) 68 | server.regist(sayHello) 69 | server.serve() 70 | 71 | def create_client(): 72 | client = TcpRpcClient('127.0.0.1', 9988) 73 | for i in range(3): 74 | resp = client.call('sayHello', 'zhangsan') 75 | self.assertEqual(resp, 'hello zhangsan') 76 | 77 | tServer = Process(target=create_server) 78 | tServer.start() 79 | time.sleep(SLEEP) 80 | tClient = Process(target=create_client) 81 | tClient.start() 82 | 83 | def test_udp_server_client(self): 84 | #测试UDP 85 | def create_server(): 86 | server = UdpRpcServer('0.0.0.0', 9999) 87 | server.regist(sayHello) 88 | server.serve() 89 | 90 | def create_client(): 91 | client = UdpRpcClient('127.0.0.1', 9999) 92 | for i in range(3): 93 | resp = client.call('sayHello', 'lisi' ) 94 | self.assertEqual(resp, 'hello lisi') 95 | 96 | tServer = threading.Thread(target=create_server) 97 | tServer.start() 98 | time.sleep(SLEEP) 99 | tClient = threading.Thread(target=create_client) 100 | tClient.start() 101 | 102 | def test_tcp_server_discovery(self): 103 | if not IS_TEST_DISCOVERY: return 104 | #测试TCP服务发现 105 | def create_server(): 106 | disconf = DiscoveryConfig( 107 | consulHost = CONSUL_HOST, 108 | consulPort = CONSUL_PORT, 109 | serviceName = 'test-rpc-server', 110 | serviceHost = local_ip(), 111 | servicePort = 10001, 112 | consulToken=CONSUL_TOKEN 113 | ) 114 | server = TcpRpcServer('0.0.0.0', 10001) 115 | server.setDiscoverConfig(disconf) 116 | server.regist(sayHello) 117 | server.serve() 118 | 119 | def create_client(): 120 | cli = TcpRpcClient() 121 | disconf = DiscoveryConfig( 122 | consulHost= CONSUL_HOST, 123 | consulPort= CONSUL_PORT, 124 | serviceName='test-rpc-server', 125 | consulToken=CONSUL_TOKEN 126 | ) 127 | cli.setDiscoveryConfig(disconf) 128 | for i in range(3): 129 | resp = cli.call('sayHello', 'mary') 130 | self.assertEqual(resp, 'hello mary') 131 | 132 | tServer = Process(target=create_server) 133 | tServer.start() 134 | time.sleep(SLEEP) 135 | tClient = Process(target=create_client) 136 | tClient.start() 137 | 138 | def test_udp_server_discovery(self): 139 | if not IS_TEST_DISCOVERY: return 140 | if socket.gethostname() == 'ubuntu': return 141 | #测试UDP服务发现 142 | def create_server(): 143 | disconf = DiscoveryConfig( 144 | consulHost = CONSUL_HOST, 145 | consulPort = CONSUL_PORT, 146 | serviceName = 'test-udp-rpc-server', 147 | serviceHost = local_ip(), 148 | servicePort = 10002, 149 | consulToken=CONSUL_TOKEN 150 | ) 151 | server = UdpRpcServer('0.0.0.0', 10002) 152 | server.setDiscoverConfig(disconf) 153 | server.regist(sayHello) 154 | server.serve() 155 | 156 | def create_client(): 157 | client = UdpRpcClient() 158 | disconf = DiscoveryConfig( 159 | consulHost= CONSUL_HOST, 160 | consulPort= CONSUL_PORT, 161 | serviceName='test-udp-rpc-server', 162 | consulToken=CONSUL_TOKEN 163 | ) 164 | client.setDiscoveryConfig(disconf) 165 | for i in range(3): 166 | resp = client.call('sayHello', 'mary') 167 | self.assertEqual(resp, 'hello mary') 168 | 169 | tServer = threading.Thread(target=create_server) 170 | tServer.start() 171 | time.sleep(SLEEP) 172 | tClient = threading.Thread(target=create_client) 173 | tClient.start() 174 | 175 | def test_serialize_obj(self): 176 | #测试TCP 177 | def create_server(): 178 | def retObj(): 179 | return A(10) 180 | server = TcpRpcServer('0.0.0.0', 10004) 181 | server.regist(retObj) 182 | server.serve() 183 | 184 | def create_client(): 185 | client = TcpRpcClient('127.0.0.1', 10004) 186 | for i in range(3): 187 | resp = client.call('retObj') 188 | self.assertEqual(resp.val, 10) 189 | 190 | tServer = Process(target=create_server) 191 | tServer.start() 192 | time.sleep(SLEEP) 193 | tClient = Process(target=create_client) 194 | tClient.start() 195 | 196 | def test_multi_server(self): 197 | def create_server_10005(): 198 | server = TcpRpcServer('0.0.0.0', 10005) 199 | server.regist(sayHello) 200 | server.serve() 201 | def create_server_10006(): 202 | server = TcpRpcServer('0.0.0.0', 10006) 203 | server.regist(sayHello) 204 | server.serve() 205 | def create_server_10007(): 206 | server = TcpRpcServer('0.0.0.0', 10007) 207 | server.regist(sayHello) 208 | server.serve() 209 | def create_client(): 210 | client = TcpRpcClient(servers=[ 211 | '127.0.0.1:10005', 212 | '127.0.0.1:10006', 213 | '127.0.0.1:10007', 214 | ]) 215 | for i in range(10): 216 | resp = client.call('sayHello', 'zhangsan') 217 | self.assertEqual(resp, 'hello zhangsan') 218 | tServer_10005 = Process(target=create_server_10005) 219 | tServer_10006 = Process(target=create_server_10006) 220 | tServer_10007 = Process(target=create_server_10007) 221 | tServer_10005.start() 222 | tServer_10006.start() 223 | tServer_10007.start() 224 | time.sleep(SLEEP) 225 | tClient = Process(target=create_client) 226 | tClient.start() 227 | 228 | def test_multi_udp_server_client(self): 229 | #测试UDP 230 | def create_server_10008(): 231 | server = UdpRpcServer('0.0.0.0', 10008) 232 | server.regist(sayHello) 233 | server.serve() 234 | def create_server_10009(): 235 | server = UdpRpcServer('0.0.0.0', 10009) 236 | server.regist(sayHello) 237 | server.serve() 238 | def create_server_10010(): 239 | server = UdpRpcServer('0.0.0.0', 10010) 240 | server.regist(sayHello) 241 | server.serve() 242 | def create_client(): 243 | client = UdpRpcClient(servers = [ 244 | '127.0.0.1:10008', 245 | '127.0.0.1:10009', 246 | '127.0.0.1:10010', 247 | ]) 248 | for i in range(10): 249 | resp = client.call('sayHello', 'lisi') 250 | self.assertEqual(resp, 'hello lisi') 251 | tServer_10008 = threading.Thread(target=create_server_10008) 252 | tServer_10009 = threading.Thread(target=create_server_10009) 253 | tServer_10010 = threading.Thread(target=create_server_10010) 254 | tServer_10008.start() 255 | tServer_10009.start() 256 | tServer_10010.start() 257 | time.sleep(SLEEP) 258 | tClient = Process(target=create_client) 259 | tClient.start() 260 | 261 | def test_tcp_server_client_timeout(self): 262 | #测试TCP 263 | def create_server(): 264 | server = TcpRpcServer('0.0.0.0', 10014) 265 | server.regist(testTimeout) 266 | server.regist(sayHello) 267 | server.serve() 268 | 269 | def create_client(): 270 | client = TcpRpcClient('127.0.0.1', 10014, timeout = 2) 271 | tag = '' 272 | try: 273 | resp = client.call('testTimeout') 274 | self.assertEqual(resp, 'finish') 275 | except socket.timeout as ex: 276 | tag = 'test timeout ok!' 277 | self.assertEqual(tag, 'test timeout ok!') 278 | resp = client.call('sayHello', 'xiaoming') 279 | self.assertEqual(resp, 'hello xiaoming') 280 | 281 | tServer = Process(target=create_server) 282 | tServer.start() 283 | time.sleep(3) 284 | tClient = Process(target=create_client) 285 | tClient.start() 286 | 287 | def test_udp_server_client_timeout(self): 288 | #测试UDP 289 | def create_server(): 290 | server = UdpRpcServer('0.0.0.0', 10015) 291 | server.regist(sayHello) 292 | server.regist(testTimeout) 293 | server.serve() 294 | 295 | def create_client(): 296 | client = UdpRpcClient('127.0.0.1', 10015, timeout=2) 297 | tag = '' 298 | try: 299 | resp = client.call('testTimeout') 300 | except socket.timeout as ex: 301 | tag = 'udp timeout' 302 | self.assertEqual(tag, 'udp timeout') 303 | resp = client.call('sayHello', 'xiaoming') 304 | time.sleep(5) 305 | self.assertEqual(resp, 'hello xiaoming') 306 | 307 | tServer = threading.Thread(target=create_server) 308 | tServer.start() 309 | time.sleep(SLEEP) 310 | tClient = threading.Thread(target=create_client) 311 | tClient.start() 312 | 313 | def test_tcp_server_server_timeout(self): 314 | #测试TCP 315 | def create_server(): 316 | server = TcpRpcServer('0.0.0.0', 10017) 317 | server.regist(sayHello) 318 | server.serve() 319 | 320 | def create_client(): 321 | client = TcpRpcClient('127.0.0.1', 10017, timeout = 2) 322 | resp = client.call('sayHello', 'xiaoming') 323 | self.assertEqual(resp, 'hello xiaoming') 324 | time.sleep(10) 325 | resp = client.call('sayHello', 'zhangsan') 326 | self.assertEqual(resp, 'hello zhangsan') 327 | 328 | tServer = Process(target=create_server) 329 | tServer.start() 330 | time.sleep(1) 331 | tClient = Process(target=create_client) 332 | tClient.start() 333 | 334 | def test_tcp_decorator(self): 335 | #测试TCP 336 | def create_server(): 337 | @rpc 338 | def add(n1, n2): 339 | return n1 +n2 340 | server = TcpRpcServer('0.0.0.0', 10018) 341 | server.serve() 342 | 343 | def create_client(): 344 | client = TcpRpcClient('127.0.0.1', 10018, timeout = 2) 345 | resp = client.call('add', n1=1, n2=1) 346 | self.assertEqual(resp, 2) 347 | 348 | tServer = Process(target=create_server) 349 | tServer.start() 350 | time.sleep(1) 351 | tClient = Process(target=create_client) 352 | tClient.start() 353 | 354 | def test_tcp_compress(self): 355 | #测试TCP 356 | def create_server(): 357 | RpcCompress.DEBUG = True 358 | RpcCompress.enableCompressLen = 200 359 | server = TcpRpcServer('0.0.0.0', 10019) 360 | server.regist(sayHello) 361 | server.serve() 362 | 363 | def create_client(): 364 | client = TcpRpcClient('127.0.0.1', 10019, timeout = 2) 365 | name = ''.join([ random.choice('abcdefghijklmnopqrstuvwxyz') for i in range(2000) ]) 366 | resp = client.call('sayHello', name) 367 | self.assertEqual(resp, 'hello ' + name) 368 | 369 | tServer = Process(target=create_server) 370 | tServer.start() 371 | time.sleep(1) 372 | tClient = Process(target=create_client) 373 | tClient.start() 374 | 375 | def test_udp_compress(self): 376 | #测试UDP 377 | def create_server(): 378 | RpcCompress.DEBUG = True 379 | RpcCompress.enableCompressLen = 200 380 | server = UdpRpcServer('0.0.0.0', 10020) 381 | server.regist(sayHello) 382 | server.serve() 383 | 384 | def create_client(): 385 | client = UdpRpcClient('127.0.0.1', 10020, timeout=2) 386 | name = ''.join([ random.choice('abcdefghijklmnopqrstuvwxyz') for i in range(2000) ]) 387 | resp = client.call('sayHello', name ) 388 | self.assertEqual(resp, 'hello ' + name) 389 | 390 | tServer = threading.Thread(target=create_server) 391 | tServer.start() 392 | time.sleep(SLEEP) 393 | tClient = threading.Thread(target=create_client) 394 | tClient.start() 395 | 396 | def test_regist_class(self): 397 | def create_server(): 398 | server = TcpRpcServer('0.0.0.0', 10022) 399 | server.regist(TestService) 400 | server.serve() 401 | 402 | def create_client(): 403 | client = TcpRpcClient('127.0.0.1', 10022, timeout = 2) 404 | resp = client.call('TestService.add', n1=1,n2=2) 405 | self.assertEqual(resp, 3) 406 | 407 | tServer = Process(target=create_server) 408 | tServer.start() 409 | time.sleep(1) 410 | tClient = Process(target=create_client) 411 | tClient.start() 412 | 413 | def test_inner_http_server(self): 414 | def inner_hello(req): 415 | name = req.data.get('name', '') 416 | print('get param name:', name) 417 | return 'hello ' + name 418 | def create_server(): 419 | hs = HttpServer(bind='0.0.0.0', port=10023) 420 | hs.addRoute('/hello', inner_hello) 421 | hs.serve() 422 | async def async_create_client(): 423 | async with aiohttp.ClientSession() as sess: 424 | async with sess.post('http://127.0.0.1:10023/hello', data={'name':'xiaoming'}) as r: 425 | text = await r.text() 426 | self.assertEqual(r.text, 'hello xiaoming') 427 | print('test inner http ok') 428 | def create_client(): 429 | asyncio.run(async_create_client()) 430 | tServer = Process(target=create_server) 431 | tServer.start() 432 | time.sleep(1) 433 | tClient = Process(target=create_client) 434 | tClient.start() 435 | 436 | def test_tcp_big_param(self): 437 | def create_server(): 438 | server = TcpRpcServer('0.0.0.0', 10024) 439 | server.regist(TestService) 440 | server.serve() 441 | def create_client(): 442 | client = TcpRpcClient('127.0.0.1', 10024, timeout = 30) 443 | resp = client.call('TestService.big_array', array=[ random.randint(0,10000) for _ in range(10000000) ]) 444 | self.assertEqual(len(resp), 10000000) 445 | print('test_tcp_big_param ok') 446 | tServer = Process(target=create_server) 447 | tServer.start() 448 | time.sleep(1) 449 | tClient = Process(target=create_client) 450 | tClient.start() 451 | 452 | def create_http_server(): 453 | server = HttpRpcServer('0.0.0.0', 10000) 454 | server.regist(sayHello) 455 | server.serve() 456 | 457 | def create_http_client(): 458 | client = HttpRpcClient('127.0.0.1', 10000) 459 | for i in range(3): 460 | resp = client.call('sayHello', 'xiaoming') 461 | assert (resp == 'hello xiaoming') 462 | 463 | def create_http_server_discovery(): 464 | if not IS_TEST_DISCOVERY: return 465 | server = HttpRpcServer('0.0.0.0', 10003) 466 | disconf = DiscoveryConfig( 467 | consulHost = CONSUL_HOST, 468 | consulPort = CONSUL_PORT, 469 | serviceName = 'test-http-rpc-server', 470 | serviceHost = local_ip(), 471 | servicePort = 10003, 472 | consulToken=CONSUL_TOKEN 473 | ) 474 | server.setDiscoverConfig(disconf) 475 | server.regist(sayHello) 476 | server.serve() 477 | 478 | def create_http_client_discovery(): 479 | if not IS_TEST_DISCOVERY: return 480 | client = HttpRpcClient() 481 | disconf = DiscoveryConfig( 482 | consulHost = CONSUL_HOST, 483 | consulPort = CONSUL_PORT, 484 | serviceName = 'test-http-rpc-server', 485 | consulToken=CONSUL_TOKEN 486 | ) 487 | client.setDiscoveryConfig(disconf) 488 | for i in range(3): 489 | resp = client.call('sayHello', 'xiaoming') 490 | assert (resp == 'hello xiaoming') 491 | 492 | def create_http_server_10011(): 493 | server = HttpRpcServer('0.0.0.0', 10011) 494 | server.regist(sayHello) 495 | server.serve() 496 | 497 | def create_http_server_10012(): 498 | server = HttpRpcServer('0.0.0.0', 10012) 499 | server.regist(sayHello) 500 | server.serve() 501 | 502 | def create_http_server_10013(): 503 | server = HttpRpcServer('0.0.0.0', 10013) 504 | server.regist(sayHello) 505 | server.serve() 506 | 507 | def create_http_client_multi(): 508 | client = HttpRpcClient(servers = [ 509 | '127.0.0.1:10011', 510 | '127.0.0.1:10012', 511 | '127.0.0.1:10013' 512 | ]) 513 | for i in range(10): 514 | resp = client.call('sayHello', 'xiaoming') 515 | assert (resp == 'hello xiaoming') 516 | 517 | def create_http_server_timeout(): 518 | server = HttpRpcServer('0.0.0.0', 10016) 519 | server.regist(testTimeout) 520 | server.regist(sayHello) 521 | server.serve() 522 | 523 | def create_http_client_timeout(): 524 | client = HttpRpcClient('127.0.0.1', 10016, timeout=2) 525 | tag = '' 526 | try: 527 | resp = client.call('testTimeout') 528 | except asyncio.exceptions.TimeoutError as ex: 529 | tag = 'http timeout' 530 | assert( tag == 'http timeout') 531 | resp = client.call('sayHello', 'xiaoming') 532 | assert( resp == 'hello xiaoming' ) 533 | 534 | def create_http_server_compress(): 535 | RpcCompress.DEBUG = True 536 | RpcCompress.enableCompressLen = 200 537 | server = HttpRpcServer('0.0.0.0', 10021) 538 | server.regist(sayHello) 539 | server.serve() 540 | 541 | def create_http_client_compress(): 542 | RpcCompress.DEBUG = True 543 | client = HttpRpcClient('127.0.0.1', 10021, timeout=2) 544 | name = ''.join([ random.choice('abcdefghijklmnopqrstuvwxyz') for i in range(2000) ]) 545 | resp = client.call('sayHello', name) 546 | assert( resp == 'hello ' + name ) 547 | 548 | def create_http_server_etcd_discovery(): 549 | if not IS_TEST_DISCOVERY: return 550 | server = HttpRpcServer('0.0.0.0', 10031) 551 | disconf = DiscoveryConfig( 552 | etcdHost='192.168.19.103', 553 | etcdPort=2379, 554 | serviceName = 'test-http-rpc-server-etcd-1', 555 | serviceHost = local_ip(), 556 | servicePort = 10031 557 | ) 558 | server.setDiscoverConfig(disconf) 559 | server.regist(sayHello) 560 | server.serve() 561 | 562 | def create_http_client_etcd_discovery(): 563 | if not IS_TEST_DISCOVERY: return 564 | client = HttpRpcClient() 565 | disconf = DiscoveryConfig( 566 | etcdHost = '192.168.19.103', 567 | etcdPort = 2379, 568 | serviceName = 'test-http-rpc-server-etcd-1', 569 | ) 570 | client.setDiscoveryConfig(disconf) 571 | for i in range(3): 572 | resp = client.call('sayHello', 'xiaoming') 573 | assert (resp == 'hello xiaoming') 574 | 575 | def create_http_server_with_middleware(): 576 | class TestMiddleware(Middleware): 577 | def handle(self, request): 578 | if request.headers.get('token') != 'test': 579 | self.abort(403, 'invalid token') 580 | 581 | def getUserName(request): 582 | return 'tom' 583 | 584 | httpServerMiddle = HttpServer(port=10032) 585 | httpServerMiddle.addRoute("/getUserName", getUserName) 586 | httpServerMiddle.middlewares = [ TestMiddleware() ] 587 | httpServerMiddle.serve() 588 | 589 | def create_http_client_totest_middle(): 590 | r = requests.get('http://127.0.0.1:10032/getUserName', headers={'token':'test'}) 591 | assert (r.status_code == 200) 592 | assert (r.text == 'tom') 593 | 594 | if __name__ == '__main__': 595 | #测试HTTP 596 | tServer = Process(target=create_http_server) 597 | tServer.start() 598 | time.sleep(SLEEP) 599 | tClient = threading.Thread(target=create_http_client) 600 | tClient.start() 601 | time.sleep(SLEEP) 602 | tServer.terminate() 603 | tServer.join() 604 | #测试HTTP服务发现 605 | httpServer = Process(target=create_http_server_discovery) 606 | httpServer.start() 607 | time.sleep(SLEEP) 608 | httpClient = threading.Thread(target=create_http_client_discovery) 609 | httpClient.start() 610 | #测试HTTP多个服务端地址 611 | httpServer_10011 = Process(target=create_http_server_10011) 612 | httpServer_10012 = Process(target=create_http_server_10012) 613 | httpServer_10013 = Process(target=create_http_server_10013) 614 | httpServer_10011.start() 615 | httpServer_10012.start() 616 | httpServer_10013.start() 617 | time.sleep(SLEEP) 618 | httpClient_multi = threading.Thread(target=create_http_client_multi) 619 | httpClient_multi.start() 620 | #测试http超时 621 | httpServer = Process(target=create_http_server_timeout) 622 | httpServer.start() 623 | time.sleep(SLEEP) 624 | httpClient = threading.Thread(target=create_http_client_timeout) 625 | httpClient.start() 626 | #测试http压缩 627 | httpServer = Process(target=create_http_server_compress) 628 | httpServer.start() 629 | time.sleep(SLEEP) 630 | httpClient = threading.Thread(target=create_http_client_compress) 631 | httpClient.start() 632 | print('\n\n\n') 633 | #测试etcd服务发现 634 | httpServer = Process(target=create_http_server_etcd_discovery) 635 | httpServer.start() 636 | time.sleep(SLEEP) 637 | httpClient = threading.Thread(target=create_http_client_etcd_discovery) 638 | httpClient.start() 639 | time.sleep(30) 640 | #测试middleware 641 | httpServer = Process(target=create_http_server_with_middleware) 642 | httpServer.start() 643 | time.sleep(SLEEP) 644 | httpClient = threading.Thread(target=create_http_client_totest_middle) 645 | httpClient.start() 646 | time.sleep(30) 647 | print('all ok') -------------------------------------------------------------------------------- /upload.py: -------------------------------------------------------------------------------- 1 | """ 2 | PYPI publish 3 | """ 4 | 5 | import sys 6 | import subprocess 7 | import platform 8 | 9 | binExec = 'python' 10 | system = platform.system() 11 | if platform.python_version()[0:1] == '3' and system == 'Linux': 12 | binExec = 'python3' 13 | 14 | if len(sys.argv) != 3: 15 | print('Usage:') 16 | print('%s upload.py [username] [password]' % binExec) 17 | sys.exit(1) 18 | 19 | cmdList = [ 20 | "rm -rf ./sunyata.egg-info", "rm -rf ./dist", 21 | "%s setup.py sdist" % binExec, 22 | "%s -m twine upload -u '%s' -p '%s' dist/*" % 23 | (binExec, sys.argv[1], sys.argv[2]), "rm -rf ./sunyata.egg-info", 24 | "rm -rf ./dist" 25 | ] 26 | 27 | for cmd in cmdList: 28 | print(cmd) 29 | _, output = subprocess.getstatusoutput(cmd) 30 | print(output) --------------------------------------------------------------------------------