├── cyberdb ├── data │ ├── __init__.py │ └── datas.py ├── __init__.py ├── extensions │ ├── nonce.py │ ├── signature.py │ └── __init__.py └── network │ ├── __init__.py │ ├── aioclient.py │ ├── server.py │ ├── route.py │ └── client.py ├── requirements.txt ├── .gitattributes ├── documentation_chn ├── mkdocs.yml └── docs │ ├── tutorial │ └── flask.md │ ├── index.md │ └── API.md ├── documentation ├── mkdocs.yml └── docs │ ├── tutorial │ └── flask.md │ ├── index.md │ └── API.md ├── setup.py ├── LICENSE ├── .gitignore ├── README_CHN.md ├── Performance Testing ├── CyberDB vs Redis_chn.md └── CyberDB vs Redis.md ├── README.md └── 文档.ipynb /cyberdb/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | obj-encrypt==0.7.0 2 | APScheduler>=3.9.1 -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.js linguist-language=Python 2 | *.css linguist-language=Python 3 | *.html linguist-language=Python -------------------------------------------------------------------------------- /documentation_chn/mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: CyberDB 2 | site_url: https://example.com/ 3 | 4 | nav: 5 | - 入门: index.md 6 | - 教程: 7 | - Flask 多进程访问: tutorial/flask.md 8 | - API: API.md -------------------------------------------------------------------------------- /cyberdb/__init__.py: -------------------------------------------------------------------------------- 1 | from .network.server import Server 2 | from .network.client import connect 3 | from .network.aioclient import connect as aioconnect 4 | from .network.client import CyberDict, CyberList -------------------------------------------------------------------------------- /documentation/mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: CyberDB 2 | site_url: https://example.com/ 3 | 4 | nav: 5 | - Getting Started: index.md 6 | - Tutorial: 7 | - Flask multi-process access: tutorial/flask.md 8 | - API: API.md -------------------------------------------------------------------------------- /cyberdb/extensions/nonce.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | 4 | seed = '1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' 5 | 6 | 7 | def generate(num: int): 8 | ''' 9 | Generate num random strings. 10 | ''' 11 | text = '' 12 | for i in range(num): 13 | text += random.choice(seed) 14 | return text 15 | -------------------------------------------------------------------------------- /cyberdb/extensions/signature.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | 3 | 4 | class Signature: 5 | ''' 6 | This class is based on the SHA-256 algorithm for signing and checking data integrity. 7 | ''' 8 | 9 | def __init__(self, salt: bytes, iterations: int=1): 10 | ''' 11 | salt -- SHA-256 salt\n 12 | iterations -- The number of iterations of the SHA-256 algorithm. 13 | ''' 14 | if type(salt) != type(b'1'): 15 | raise RuntimeError('Salt must be bytes.') 16 | self._salt = salt 17 | self._iterations = iterations 18 | 19 | def encrypt(self, content: bytes): 20 | ''' 21 | SHA-256 encryption 22 | ''' 23 | if type(content) != type(b'1'): 24 | raise RuntimeError('Content must be bytes.') 25 | dk = hashlib.pbkdf2_hmac('sha256', content, self._salt, self._iterations) 26 | return dk.hex() -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | 4 | with open("README.md", "r") as fh: 5 | long_description = fh.read() 6 | 7 | setuptools.setup( 8 | name="CyberDB", 9 | version="0.9.3", 10 | author="Cyberbolt", 11 | author_email="dtconlyone@gmail.com", 12 | description="CyberDB is a lightweight Python in-memory database. It is designed to use Python's built-in data structures Dictionaries, Lists for data storage, efficient communication through Socket TCP, and provide data persistence. This module can be used in hard disk database caching, Gunicorn inter-process communication, distributed computing and other fields.", 13 | long_description=long_description, 14 | long_description_content_type="text/markdown", 15 | packages=setuptools.find_packages(), 16 | classifiers=[ 17 | "Programming Language :: Python :: 3", 18 | "License :: OSI Approved :: MIT License", 19 | "Operating System :: OS Independent", 20 | ], 21 | install_requires=[ 22 | 'APScheduler>=3.9.1', 23 | 'obj-encrypt==0.7.0' 24 | ] 25 | ) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Cyberbolt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /cyberdb/extensions/__init__.py: -------------------------------------------------------------------------------- 1 | from threading import Thread 2 | 3 | 4 | class MyThread(Thread): 5 | def __init__(self, target, args): 6 | super(MyThread, self).__init__() 7 | self.target = target 8 | self.args = args 9 | 10 | def run(self): 11 | self.result = self.target(*self.args) 12 | 13 | def get_result(self): 14 | try: 15 | return self.result 16 | except Exception: 17 | return None 18 | 19 | 20 | class CyberDBError(RuntimeError): 21 | ''' 22 | CyberDB basic error 23 | ''' 24 | pass 25 | 26 | 27 | class DisconCyberDBError(CyberDBError): 28 | ''' 29 | TCP connection has been disconnected. 30 | ''' 31 | pass 32 | 33 | 34 | class WrongPasswordCyberDBError(CyberDBError): 35 | pass 36 | 37 | class WrongFilenameCyberDBError(CyberDBError): 38 | pass 39 | 40 | class BackupCyberDBError(CyberDBError): 41 | pass 42 | 43 | class TypeCyberDBError(CyberDBError): 44 | pass 45 | 46 | class WrongTableNameCyberDBError(CyberDBError): 47 | pass 48 | 49 | class WrongInputCyberDBError(CyberDBError): 50 | pass 51 | -------------------------------------------------------------------------------- /cyberdb/data/datas.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Data used for TCP transmission 3 | ''' 4 | 5 | 6 | import pickle 7 | 8 | from obj_encrypt import Secret 9 | 10 | from ..extensions.signature import Signature 11 | from ..extensions import CyberDBError 12 | 13 | 14 | def generate_client_obj(): 15 | return { 16 | 'route': None, 17 | 'message': None 18 | } 19 | 20 | 21 | def generate_server_obj(): 22 | return { 23 | 'code': None, 24 | 'message': None 25 | } 26 | 27 | 28 | errors_code = { 29 | 2: 'Incorrect password or data tampering.' 30 | } 31 | 32 | 33 | class DataParsing: 34 | ''' 35 | Convert TCP data and encrypted objects to each other. 36 | ''' 37 | 38 | def __init__(self, secret: Secret, encrypt: bool=False): 39 | self._secret = secret 40 | self._encrypt = encrypt 41 | 42 | def data_to_obj(self, data): 43 | ''' 44 | Restore TCP encrypted data as an object. 45 | ''' 46 | # Determine whether to decrypt and verify the signature. 47 | if self._encrypt: 48 | 49 | try: 50 | data = self._secret.decrypt(data) 51 | except (UnicodeDecodeError, KeyError): 52 | return { 53 | 'code': 2, 54 | 'errors-code': errors_code[2] 55 | } 56 | 57 | obj = pickle.loads(data['content']) 58 | return { 59 | 'code': 1, 60 | 'content': obj 61 | } 62 | 63 | else: 64 | 65 | try: 66 | obj = pickle.loads(data) 67 | return { 68 | 'code': 1, 69 | 'content': obj 70 | } 71 | except Exception as e: 72 | return { 73 | 'code': 2, 74 | 'errors-code': errors_code[2] 75 | } 76 | 77 | def obj_to_data(self, obj): 78 | ''' 79 | Convert object to TCP transmission data. 80 | ''' 81 | data = { 82 | 'content': None 83 | } 84 | 85 | # Determine whether to encrypt and sign. 86 | if self._encrypt: 87 | try: 88 | data['content'] = pickle.dumps(obj) 89 | except pickle.PickleError as e: 90 | raise CyberDBError('CyberDB does not support this data type.') 91 | 92 | data = self._secret.encrypt(data) 93 | 94 | else: 95 | data = pickle.dumps(obj) 96 | 97 | return data 98 | 99 | 100 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | #自定义忽略 2 | macvenv 3 | .idea 4 | dist 5 | pyvenv 6 | site 7 | .DS_Store 8 | data.cdb 9 | backup.py 10 | app.py 11 | test1.ipynb 12 | test1.py 13 | test2.ipynb 14 | test2.py 15 | test3.ipynb 16 | test3.py 17 | 18 | 19 | # Byte-compiled / optimized / DLL files 20 | __pycache__/ 21 | *.py[cod] 22 | *$py.class 23 | 24 | # C extensions 25 | *.so 26 | 27 | # Distribution / packaging 28 | .Python 29 | build/ 30 | develop-eggs/ 31 | downloads/ 32 | eggs/ 33 | .eggs/ 34 | lib/ 35 | lib64/ 36 | parts/ 37 | sdist/ 38 | var/ 39 | wheels/ 40 | share/python-wheels/ 41 | *.egg-info/ 42 | .installed.cfg 43 | *.egg 44 | MANIFEST 45 | 46 | # PyInstaller 47 | # Usually these files are written by a python script from a template 48 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 49 | *.manifest 50 | *.spec 51 | 52 | # Installer logs 53 | pip-log.txt 54 | pip-delete-this-directory.txt 55 | 56 | # Unit test / coverage reports 57 | htmlcov/ 58 | .tox/ 59 | .nox/ 60 | .coverage 61 | .coverage.* 62 | .cache 63 | nosetests.xml 64 | coverage.xml 65 | *.cover 66 | *.py,cover 67 | .hypothesis/ 68 | .pytest_cache/ 69 | cover/ 70 | 71 | # Translations 72 | *.mo 73 | *.pot 74 | 75 | # Django stuff: 76 | *.log 77 | local_settings.py 78 | db.sqlite3 79 | db.sqlite3-journal 80 | 81 | # Flask stuff: 82 | instance/ 83 | .webassets-cache 84 | 85 | # Scrapy stuff: 86 | .scrapy 87 | 88 | # Sphinx documentation 89 | docs/_build/ 90 | 91 | # PyBuilder 92 | .pybuilder/ 93 | target/ 94 | 95 | # Jupyter Notebook 96 | .ipynb_checkpoints 97 | 98 | # IPython 99 | profile_default/ 100 | ipython_config.py 101 | 102 | # pyenv 103 | # For a library or package, you might want to ignore these files since the code is 104 | # intended to run in multiple environments; otherwise, check them in: 105 | # .python-version 106 | 107 | # pipenv 108 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 109 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 110 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 111 | # install all needed dependencies. 112 | #Pipfile.lock 113 | 114 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 115 | __pypackages__/ 116 | 117 | # Celery stuff 118 | celerybeat-schedule 119 | celerybeat.pid 120 | 121 | # SageMath parsed files 122 | *.sage.py 123 | 124 | # Environments 125 | .env 126 | .venv 127 | env/ 128 | venv/ 129 | ENV/ 130 | env.bak/ 131 | venv.bak/ 132 | 133 | # Spyder project settings 134 | .spyderproject 135 | .spyproject 136 | 137 | # Rope project settings 138 | .ropeproject 139 | 140 | # mkdocs documentation 141 | /site 142 | 143 | # mypy 144 | .mypy_cache/ 145 | .dmypy.json 146 | dmypy.json 147 | 148 | # Pyre type checker 149 | .pyre/ 150 | 151 | # pytype static type analyzer 152 | .pytype/ 153 | 154 | # Cython debug symbols 155 | cython_debug/ 156 | 157 | -------------------------------------------------------------------------------- /documentation_chn/docs/tutorial/flask.md: -------------------------------------------------------------------------------- 1 | # 生产环境下使用 CyberDB 作为 Flask 的内存数据库 2 | 3 | 前面我们讲述了 CyberDB 的[快速上手](https://www.cyberlight.xyz/static/cyberdb-chn/)，现在我们需要把它带到能发挥其作用的地方，在生产环境中将 CyberDB 作为 Flask 的内存数据库，使用 Gunicorn 运行，并实现多进程间的通信。 4 | 5 | 这篇文章通过一个尽可能精简的 Flask 实例讲解，不会涉及复杂的 Web 知识。核心思路为 CyberDB + Gunicorn + Gevent + Flask (多进程 + 协程)，启动一个 CyberDB 服务器，使用 Gunicorn 多进程运行 Flask 实例，每个进程的实例通过 Gevent 运行，进程中使用 CyberDB 客户端连接至内存数据库，由此实现 CyberDB 数据库的高并发访问。 6 | 7 | 文章使用 PyPy 运行，同样适用 CPython。 8 | 9 | 运行环境: Python 3.8.12, PyPy 7.3.7 10 | 11 | 此项目的目录结构 12 | ```bash 13 | . 14 | ├── app.py 15 | ├── cyberdb_init.py 16 | ├── cyberdb_serve.py 17 | ├── requirements.txt 18 | └── venv 19 | ``` 20 | 我们通过列举每个文件的内容顺序讲解 CyberDB 的核心操作。 21 | 22 | 文件 requirements.txt 23 | ``` 24 | CyberDB>=0.7.1 25 | Flask==2.1.1 26 | gevent==21.12.0 27 | gunicorn==20.1.0 28 | ``` 29 | 此项目的依赖。这篇文章不是 Python 基础教程，请查询相关文档创建虚拟环境 venv 目录并安装 requirements.txt 中的依赖。 30 | 31 | 生成 venv 目录并安装好依赖后，下面所有操作都在激活的虚拟环境中运行。 32 | 33 | 文件 cyberdb_init.py 34 | ```python 35 | ''' 36 | 该模块用于初始化 CyberDB 的表结构， 37 | 只在第一次运行，后续不再使用。 38 | ''' 39 | 40 | 41 | import time 42 | 43 | import cyberdb 44 | 45 | db = cyberdb.Server() 46 | # 配置 CyberDB 服务端的 地址、端口、密码 47 | db.start(host='127.0.0.1', port=9980, password='123456') 48 | 49 | # 待服务端启动后，连接 CyberDB 服务端 50 | time.sleep(3) 51 | client = cyberdb.connect(host='127.0.0.1', port=9980, password='123456') 52 | # 生成 proxy 对象 53 | with client.get_proxy() as proxy: 54 | # 创建类型为 CyberDict 的表 centre，并初始化内容 55 | proxy.create_cyberdict('centre') 56 | centre = proxy.get_cyberdict('centre') 57 | centre['content'] = 'Hello CyberDB!' 58 | 59 | # 将 CyberDB 保存至 data.cdb 60 | db.save_db('data.cdb') 61 | ``` 62 | 63 | 在项目根目录执行 64 | ```bash 65 | python cyberdb_init.py 66 | ``` 67 | 此时完成了 CyberDB 数据库表的初始化，在 CyberDB 中创建了一个名为 centre、类型为 CyberDict 的表，初始化 'content' 键的值为 'Hello CyberDB!' ，最后将 CyberDB 数据库保存至硬盘(项目根目录生成了名为 data.cdb 的文件)。 68 | 69 | 文件 cyberdb_serve.py 70 | ```python 71 | import cyberdb 72 | 73 | 74 | def main(): 75 | # 后台运行 CyberDB 服务端，设置相关信息。 76 | db = cyberdb.Server() 77 | # 从硬盘读取 data.cdb 至 CyberDB 78 | db.load_db('data.cdb') 79 | # 每 300 秒备份一次数据库 80 | db.set_backup('data.cdb', cycle=300) 81 | db.run( 82 | host='127.0.0.1', # TCP 运行地址 83 | port=9980, # TCP 监听端口 84 | password='hWjYvVdqRC', # 数据库连接密码 85 | max_con=10000, # 最大并发数 86 | encrypt=True, # 加密通信 87 | print_log=False # 不打印日志 88 | ) 89 | 90 | 91 | if __name__ == '__main__': 92 | main() 93 | ``` 94 | 在项目根目录执行 95 | ```bash 96 | python cyberdb_serve.py 97 | ``` 98 | 以运行 CyberDB 服务端。 99 | 100 | 此处设置了 encrypt=True ，CyberDB 会将 TCP 通信内容使用 AES-256 算法加密。开启 encrypt=True 后，CyberDB 仅允许白名单中的 ip 通信，默认白名单为 ['127.0.0.1']，白名单[设置方法](https://www.cyberlight.xyz/static/cyberdb-chn/API/#cyberdbserver)。一般，若只需在本地进程间通信，无需开启 encrypt=True 和设置白名单，只有远程通信时需要此操作。 101 | 102 | 文件 app.py 103 | ```python 104 | import cyberdb 105 | from flask import Flask, g 106 | 107 | 108 | # 连接 CyberDB 并生成客户端实例。 109 | client = cyberdb.connect( 110 | host='127.0.0.1', 111 | port=9980, 112 | password='hWjYvVdqRC', 113 | # 服务端若加密，客户端必须加密，反之亦然 114 | encrypt=True, 115 | # 每个连接若超过900秒无操作，将舍弃该连接。 116 | # 连接由连接池智能管理，无需关系细节。 117 | time_out=900 118 | ) 119 | 120 | # 创建 Flask 实例，此部分请参考 121 | # Flask 文档 https://flask.palletsprojects.com/ 122 | app = Flask(__name__) 123 | 124 | 125 | @app.before_request 126 | def before_request(): 127 | # 每次请求执行前生成 proxy 对象。 128 | g.proxy = client.get_proxy() 129 | # 从连接池获取连接。 130 | g.proxy.connect() 131 | 132 | 133 | @app.get("/") 134 | def hello_world(): 135 | # 从数据库获取 centre 表 136 | centre = g.proxy.get_cyberdict('centre') 137 | 138 | return { 139 | 'code': 1, 140 | 'content': centre['content'] 141 | } 142 | 143 | 144 | @app.teardown_request 145 | def teardown_request(error): 146 | # 每次请求执行后归还连接至连接池 147 | g.proxy.close() 148 | 149 | 150 | if __name__ == '__main__': 151 | app.run(host='127.0.0.1', port=8000) 152 | ``` 153 | 该模块会在每次请求执行前( before_request )使用 client.get_proxy() 获取 proxy 对象，每个获取的 proxy 对象可以绑定一个 TCP 连接，此处使用 proxy.connect() 从连接池获取连接。视图函数 hello_world 中，由 proxy 获取的对象 centre，与 proxy 共用同一个连接，proxy 的连接释放后，centre 也会失去连接。在每次请求后( teardown_request )使用 proxy.close() 方法释放 proxy 绑定的连接，归还至连接池。 154 | 155 | cyberdb.connect 的 time_out 参数是连接池中每个连接的超时时间，此处每个连接超过 900 秒无操作将被舍弃。若不设置该参数，连接池的每个连接会维持到失效为止。 156 | 157 | 在项目根目录运行 158 | ```bash 159 | gunicorn -w 4 -b 127.0.0.1:8000 -k gevent app:app 160 | ``` 161 | 使用 4 进程、Gevent 启动 Flask 实例。 162 | 163 | 浏览器访问 127.0.0.1:8000 会得到如下响应 164 | ```JSON 165 | {"code":1,"content":"Hello CyberDB!"} 166 | ``` 167 | 通过此例，你可以把 CyberDB 部署到更复杂的 Web 环境中，充分享受内存的低延迟特性。 168 | -------------------------------------------------------------------------------- /cyberdb/network/__init__.py: -------------------------------------------------------------------------------- 1 | import math 2 | import socket 3 | import asyncio 4 | 5 | from ..data import datas 6 | from ..extensions import CyberDBError, DisconCyberDBError, \ 7 | WrongPasswordCyberDBError 8 | 9 | 10 | SLICE_SIZE = 4096 11 | 12 | 13 | class AioStream: 14 | ''' 15 | Encapsulates TCP read and write and object encryption. 16 | ''' 17 | 18 | def __init__( 19 | self, 20 | reader: asyncio.streams.StreamReader, 21 | writer: asyncio.streams.StreamWriter, 22 | dp: datas.DataParsing 23 | ): 24 | self._reader = reader 25 | self._writer = writer 26 | self._dp = dp 27 | 28 | async def read(self) -> dict: 29 | reader, writer = self._reader, self._writer 30 | 31 | # Gets the number of times TCP loops to receive data. 32 | number_of_times = await reader.read(SLICE_SIZE) 33 | 34 | # The client actively disconnects. 35 | if number_of_times == b'': 36 | raise DisconCyberDBError('The TCP connection was disconnected by the other end.') 37 | 38 | r = self._dp.data_to_obj(number_of_times) 39 | if r['code'] != 1: 40 | writer.close() 41 | raise WrongPasswordCyberDBError(r['errors-code']) 42 | number_of_times = r['content'] 43 | 44 | # Informs the client that it is ready to receive data. 45 | ready = self._dp.obj_to_data('ready') 46 | writer.write(ready) 47 | await writer.drain() 48 | 49 | buffer = [await reader.read(SLICE_SIZE) for i in range(number_of_times)] 50 | data = b''.join(buffer) # Splice into complete data. 51 | 52 | r = self._dp.data_to_obj(data) 53 | if r['code'] != 1: 54 | writer.close() 55 | raise WrongPasswordCyberDBError(r['errors-code']) 56 | 57 | return r['content'] 58 | 59 | async def write(self, obj: dict): 60 | reader, writer = self._reader, self._writer 61 | 62 | data = self._dp.obj_to_data(obj) 63 | 64 | # The number of times the other end TCP loops to 65 | # receive data. 66 | number_of_times = self._dp.obj_to_data( 67 | math.ceil(len(data) / SLICE_SIZE) 68 | ) 69 | writer.write(number_of_times) 70 | await writer.drain() 71 | 72 | # Get the client's readiness status. 73 | ready = await reader.read(SLICE_SIZE) 74 | r = self._dp.data_to_obj(ready) 75 | if r['code'] != 1: 76 | writer.close() 77 | raise WrongPasswordCyberDBError(r['errors-code']) 78 | ready = r['content'] 79 | 80 | if ready != 'ready': 81 | raise RuntimeError('error') 82 | 83 | writer.write(data) 84 | await writer.drain() 85 | 86 | def get_addr(self): 87 | return self._writer.get_extra_info('peername') 88 | 89 | 90 | class Stream: 91 | ''' 92 | Synchronous TCP read and write 93 | ''' 94 | 95 | def __init__(self, s: socket.socket, dp: datas.DataParsing): 96 | self._s = s 97 | self._dp = dp 98 | 99 | def read(self) -> dict: 100 | # Gets the number of times TCP loops to receive data. 101 | number_of_times = self._s.recv(SLICE_SIZE) 102 | 103 | # The client actively disconnects. 104 | if number_of_times == b'': 105 | raise DisconCyberDBError('The TCP connection was disconnected by the other end. It is also possible that the password entered by the client is incorrect.') 106 | 107 | r = self._dp.data_to_obj(number_of_times) 108 | if r['code'] != 1: 109 | self._s.close() 110 | raise WrongPasswordCyberDBError(r['errors-code']) 111 | number_of_times = r['content'] 112 | 113 | # Informs the client that it is ready to receive data. 114 | ready = self._dp.obj_to_data('ready') 115 | self._s.sendall(ready) 116 | 117 | buffer = [self._s.recv(SLICE_SIZE) for i in range(number_of_times)] 118 | data = b''.join(buffer) # Splice into complete data. 119 | 120 | r = self._dp.data_to_obj(data) 121 | if r['code'] != 1: 122 | self._s.close() 123 | raise WrongPasswordCyberDBError(r['errors-code']) 124 | 125 | return r['content'] 126 | 127 | def write(self, obj: dict): 128 | obj['password'] = self._dp._secret.key 129 | 130 | data = self._dp.obj_to_data(obj) 131 | 132 | # The number of times the other end TCP loops to 133 | # receive data. 134 | number_of_times = self._dp.obj_to_data( 135 | math.ceil(len(data) / SLICE_SIZE) 136 | ) 137 | try: 138 | self._s.sendall(number_of_times) 139 | except BrokenPipeError: 140 | raise DisconCyberDBError('The TCP connection has been lost, please run proxy.connect() to regain the connection.') 141 | except AttributeError: 142 | raise DisconCyberDBError('There is no connection to the CyberDB server.') 143 | 144 | # Get the client's readiness status. 145 | ready = self._s.recv(SLICE_SIZE) 146 | try: 147 | r = self._dp.data_to_obj(ready) 148 | except EOFError as e: 149 | raise DisconCyberDBError('The TCP connection has been lost, please run proxy.connect() to regain the connection.') 150 | 151 | if r['code'] != 1: 152 | self._s.close() 153 | raise WrongPasswordCyberDBError(r['errors-code']) 154 | ready = r['content'] 155 | 156 | if ready != 'ready': 157 | raise RuntimeError('error') 158 | 159 | self._s.sendall(data) 160 | -------------------------------------------------------------------------------- /README_CHN.md: -------------------------------------------------------------------------------- 1 | # CyberDB 2 | 3 | CyberDB 是一个轻量级的 Python 内存数据库。它旨在利用 Python 内置数据结构 Dictionaries、Lists 作数据存储，通过 Socket TCP 高效通信，并提供数据持久化。该模块可用于 硬盘数据库缓存、Gunicorn 进程间通信、分布式计算 等领域。 4 | 5 | CyberDB 服务端使用 Asyncio 进行 TCP 通信。客户端基于 Socket 开发，所以支持 Gevent 协程，暂未适配 Asyncio。服务端和客户端均支持 PyPy，推荐使用 PyPy 运行，以获得更好的性能。 6 | 7 | 高并发场景下，传统数据库的性能瓶颈主要在硬盘 I/O，即便 CyberDB 基于动态语言 Python 开发，速度仍然远快于硬盘数据库(如 MySQL)，CyberDB 可以作为它的缓存。此外，CyberDB 的核心在于使用 Pythonic 的方式编程，你可以像使用 Dictionaries 和 Lists 一样使用 CyberDB。 8 | 9 | ## 安装方法 10 | 11 | 1.进入命令窗口，创建虚拟环境，依次输入以下命令 12 | 13 | Linux 和 macOS: 14 | 15 | 16 | 17 | ```python 18 | python3 -m venv venv # 创建虚拟环境 19 | . venv/bin/activate # 激活虚拟环境 20 | ``` 21 | 22 | Windows: 23 | 24 | 25 | ```python 26 | python -m venv venv # 创建虚拟环境 27 | venv\Scripts\activate # 激活虚拟环境 28 | ``` 29 | 30 | 2.安装 CyberDB，依次输入 31 | 32 | 33 | ```python 34 | pip install --upgrade pip 35 | pip install cyberdb 36 | ``` 37 | 38 | 如果你的服务端和客户端在两个不同的项目目录运行，请分别在服务端、客户端的虚拟环境中安装 CyberDB。 39 | 40 | ## 链接 41 | 42 | - GitHub: [https://github.com/Cyberbolt/CyberDB](https://github.com/Cyberbolt/CyberDB) 43 | - PyPI: [https://pypi.org/project/CyberDB/](https://pypi.org/project/CyberDB/) 44 | - 文档: [https://www.cyberlight.xyz/static/cyberdb-chn](https://www.cyberlight.xyz/static/cyberdb-chn) 45 | - 电光笔记: [https://www.cyberlight.xyz/](https://www.cyberlight.xyz/) 46 | 47 | ## 快速使用 48 | 49 | 该模块中请使用 CyberDict 和 CyberList 替代 dict 和 list （一种基于 TCP 的类 Dictionaries、类 Lists 对象）。 50 | 51 | ### 服务端 52 | 53 | 54 | 服务端初始化，设置备份和 TCP 监听地址。 55 | 56 | 57 | ```python 58 | import time 59 | import cyberdb 60 | 61 | db = cyberdb.Server() 62 | # 数据持久化，备份文件为 data.cdb，备份周期 900 秒一次。 63 | db.set_backup('data.cdb', cycle=900) 64 | # 设置 TCP 地址、端口号、密码，生产环境中密码建议使用大小写字母和数字的组合。 65 | # start 方法不会阻塞运行，若希望该操作阻塞，请使用 run 方法代替 start，参数不变。 66 | db.start(host='127.0.0.1', port=9980, password='123456') 67 | 68 | while True: 69 | time.sleep(10000) 70 | ``` 71 | 72 | 上述服务端运行后，每 900 秒将在此项目根目录生成（或覆盖）data.cdb。下次启动数据库可使用 load 方法读取该文件。 73 | 74 | ### 客户端 75 | 76 | #### 连接数据库 77 | 78 | 79 | ```python 80 | import cyberdb 81 | 82 | # 生成客户端实例并连接。 83 | client = cyberdb.connect(host='127.0.0.1', port=9980, password='123456') 84 | ``` 85 | 86 | #### 生成 proxy 对象 87 | 88 | 89 | ```python 90 | # 生成本次请求的 proxy。 91 | proxy = client.get_proxy() 92 | # 从连接池自动获取数据库连接。 93 | proxy.connect() 94 | ``` 95 | 96 | 建议在每个线程(或协程)中单独生成 proxy 对象，并通过 connect 方法获取数据库连接。你只需在操作完成后使用 close 方法归还连接即可，归还后的连接由 client 对象智能管理。 97 | 98 | #### 操作 proxy 对象 99 | 100 | 创建 CyberDict 和 CyberList 101 | 102 | 103 | ```python 104 | # 在数据库中分别创建类型为 CyberDict 的 dict1、dict2 表， 105 | # 类型为 CyberList 的 list1 表。 106 | proxy.create_cyberdict('dict1') 107 | proxy.create_cyberdict('dict2') 108 | proxy.create_cyberlist('list1') 109 | ``` 110 | 111 | 112 | ```python 113 | dict1 = proxy.get_cyberdict('dict1') 114 | dict2 = proxy.get_cyberdict('dict2') 115 | list1 = proxy.get_cyberlist('list1') 116 | ``` 117 | 118 | 此处获取的 dict1、dict2、list1 均为网络对象，数据通过 TCP 传输。三个对象受 proxy 控制，当调用 proxy.close() 归还连接后，三个对象也会失效。同样，使用 proxy.connect() 可重新从连接池获取连接，dict1、dict2、list1 也变为可用。 119 | 120 | 了解此操作后，你便可以像操作 Dictionaries 一样操作 dict1、dict2，像操作 Lists 一样操作 list1 了！（CyberDict 和 CyberList 支持 Dictionaries、Lists 的大部分方法） 121 | 122 | 示例如下 123 | 124 | ##### CyberDict 常用操作 125 | 126 | 在 dict1 和 dict2 中新增键值对 127 | 128 | 129 | ```python 130 | dict1[0] = 100 131 | dict1['test'] = 'Hello CyberDB!' 132 | dict2[0] = 200 133 | ``` 134 | 135 | 获取对应的值 136 | 137 | 138 | ```python 139 | dict1.get(0) 140 | ``` 141 | 142 | 143 | 144 | 145 | 100 146 | 147 | 148 | 149 | 150 | ```python 151 | dict2[0] 152 | ``` 153 | 154 | 155 | 156 | 157 | 200 158 | 159 | 160 | 161 | 查看 dict1 和 dict2 (也可以使用 print 打印) 162 | 163 | 164 | ```python 165 | dict1 166 | ``` 167 | 168 | 169 | 170 | 171 | {0: 100, 'test': 'Hello CyberDB!'} 172 | 173 | 174 | 175 | 176 | ```python 177 | dict2 178 | ``` 179 | 180 | 181 | 182 | 183 | {0: 200} 184 | 185 | 186 | 187 | 获取长度 188 | 189 | 190 | ```python 191 | len(dict1) 192 | ``` 193 | 194 | 195 | 196 | 197 | 2 198 | 199 | 200 | 201 | 删除键值对 202 | 203 | 204 | ```python 205 | del dict1[0] 206 | dict1 207 | ``` 208 | 209 | 210 | 211 | 212 | {'test': 'Hello CyberDB!'} 213 | 214 | 215 | 216 | 清空 dict1 217 | 218 | 219 | ```python 220 | dict1.clear() 221 | dict1 222 | ``` 223 | 224 | 225 | 226 | 227 | {} 228 | 229 | 230 | 231 | ##### CyberList 常用操作 232 | 233 | 生成 list1 的内容 234 | 235 | 236 | ```python 237 | for i in range(5): 238 | list1.append(99) 239 | 240 | list1 241 | ``` 242 | 243 | 244 | 245 | 246 | [99, 99, 99, 99, 99] 247 | 248 | 249 | 250 | 更改坐标值 251 | 252 | 253 | ```python 254 | list1[3] = 100 255 | list1 256 | ``` 257 | 258 | 259 | 260 | 261 | [99, 99, 99, 100, 99] 262 | 263 | 264 | 265 | 进行切片 266 | 267 | 268 | ```python 269 | list1[3:] 270 | ``` 271 | 272 | 273 | 274 | 275 | [100, 99] 276 | 277 | 278 | 279 | 获取 list1 的长度 280 | 281 | 282 | ```python 283 | len(list1) 284 | ``` 285 | 286 | 287 | 288 | 289 | 5 290 | 291 | 292 | 293 | 通过迭代打印 list1 每个元素 294 | 295 | 296 | ```python 297 | for v in list1: 298 | print(v) 299 | ``` 300 | 301 | 99 302 | 99 303 | 99 304 | 100 305 | 99 306 | 307 | 308 | 强烈推荐使用 for 循环迭代 CyberList，每次迭代将从服务端获取 v，客户端的空间复杂度为 o(1)。迭代同样可用于 CyberDict，CyberDict 的迭代中，客户端空间复杂度为 o(n), n 为 CyberDict.keys() 的长度。 309 | 310 | #### 释放 proxy 对象 311 | 312 | 使用完成，将 proxy 的连接归还至连接池即可。 313 | 314 | 315 | ```python 316 | proxy.close() 317 | ``` 318 | 319 | proxy 对象同样支持上下文管理器，如 320 | 321 | 322 | ```python 323 | with client.get_proxy() as proxy: 324 | list1 = proxy.get_cyberlist('list1') 325 | print(list1) 326 | ``` 327 | 328 | [99, 99, 99, 100, 99] 329 | 330 | 331 | ## 概括 332 | 333 | 有了 CyberDB，便能充分利用内存性能，不同进程(甚至不同主机)能通过 Python 的数据结构通信。更多教程请参考文档，感谢你的支持！ 334 | 335 | ## 注意 336 | 337 | 由于编码限制，CyberDB 会将 0 识别为 None，但并不影响计算，请在所需位置将 None 转为 0。 338 | -------------------------------------------------------------------------------- /documentation_chn/docs/index.md: -------------------------------------------------------------------------------- 1 | # CyberDB 2 | 3 | CyberDB 是一个轻量级的 Python 内存数据库。它旨在利用 Python 内置数据结构 Dictionaries、Lists 作数据存储，通过 Socket TCP 高效通信，并提供数据持久化。该模块可用于 硬盘数据库缓存、Gunicorn 进程间通信、分布式计算 等领域。 4 | 5 | CyberDB 服务端使用 Asyncio 进行 TCP 通信。客户端基于 Socket 开发，所以支持 Gevent 协程，暂未适配 Asyncio。服务端和客户端均支持 PyPy，推荐使用 PyPy 运行，以获得更好的性能。 6 | 7 | 高并发场景下，传统数据库的性能瓶颈主要在硬盘 I/O，即便 CyberDB 基于动态语言 Python 开发，速度仍然远快于硬盘数据库(如 MySQL)，CyberDB 可以作为它的缓存。此外，CyberDB 的核心在于使用 Pythonic 的方式编程，你可以像使用 Dictionaries 和 Lists 一样使用 CyberDB。 8 | 9 | ## 安装方法 10 | 11 | 1.进入命令窗口，创建虚拟环境，依次输入以下命令 12 | 13 | Linux 和 macOS: 14 | 15 | 16 | 17 | ```python 18 | python3 -m venv venv # 创建虚拟环境 19 | . venv/bin/activate # 激活虚拟环境 20 | ``` 21 | 22 | Windows: 23 | 24 | 25 | ```python 26 | python -m venv venv # 创建虚拟环境 27 | venv\Scripts\activate # 激活虚拟环境 28 | ``` 29 | 30 | 2.安装 CyberDB，依次输入 31 | 32 | 33 | ```python 34 | pip install --upgrade pip 35 | pip install cyberdb 36 | ``` 37 | 38 | 如果你的服务端和客户端在两个不同的项目目录运行，请分别在服务端、客户端的虚拟环境中安装 CyberDB。 39 | 40 | ## 链接 41 | 42 | - GitHub: [https://github.com/Cyberbolt/CyberDB](https://github.com/Cyberbolt/CyberDB) 43 | - PyPI: [https://pypi.org/project/CyberDB/](https://pypi.org/project/CyberDB/) 44 | - 文档: [https://www.cyberlight.xyz/static/cyberdb-chn](https://www.cyberlight.xyz/static/cyberdb-chn) 45 | - 电光笔记: [https://www.cyberlight.xyz/](https://www.cyberlight.xyz/) 46 | 47 | ## 快速使用 48 | 49 | 该模块中请使用 CyberDict 和 CyberList 替代 dict 和 list （一种基于 TCP 的类 Dictionaries、类 Lists 对象）。 50 | 51 | ### 服务端 52 | 53 | 54 | 服务端初始化，设置备份和 TCP 监听地址。 55 | 56 | 57 | ```python 58 | import time 59 | import cyberdb 60 | 61 | db = cyberdb.Server() 62 | # 数据持久化，备份文件为 data.cdb，备份周期 900 秒一次。 63 | db.set_backup('data.cdb', cycle=900) 64 | # 设置 TCP 地址、端口号、密码，生产环境中密码建议使用大小写字母和数字的组合。 65 | # start 方法不会阻塞运行，若希望该操作阻塞，请使用 run 方法代替 start，参数不变。 66 | db.start(host='127.0.0.1', port=9980, password='123456') 67 | 68 | while True: 69 | time.sleep(10000) 70 | ``` 71 | 72 | 上述服务端运行后，每 900 秒将在此项目根目录生成（或覆盖）data.cdb。下次启动数据库可使用 load 方法读取该文件。 73 | 74 | ### 客户端 75 | 76 | #### 连接数据库 77 | 78 | 79 | ```python 80 | import cyberdb 81 | 82 | # 生成客户端实例并连接。 83 | client = cyberdb.connect(host='127.0.0.1', port=9980, password='123456') 84 | ``` 85 | 86 | #### 生成 proxy 对象 87 | 88 | 89 | ```python 90 | # 生成本次请求的 proxy。 91 | proxy = client.get_proxy() 92 | # 从连接池自动获取数据库连接。 93 | proxy.connect() 94 | ``` 95 | 96 | 建议在每个线程(或协程)中单独生成 proxy 对象，并通过 connect 方法获取数据库连接。你只需在操作完成后使用 close 方法归还连接即可，归还后的连接由 client 对象智能管理。 97 | 98 | #### 操作 proxy 对象 99 | 100 | 创建 CyberDict 和 CyberList 101 | 102 | 103 | ```python 104 | # 在数据库中分别创建类型为 CyberDict 的 dict1、dict2 表， 105 | # 类型为 CyberList 的 list1 表。 106 | proxy.create_cyberdict('dict1') 107 | proxy.create_cyberdict('dict2') 108 | proxy.create_cyberlist('list1') 109 | ``` 110 | 111 | 112 | ```python 113 | dict1 = proxy.get_cyberdict('dict1') 114 | dict2 = proxy.get_cyberdict('dict2') 115 | list1 = proxy.get_cyberlist('list1') 116 | ``` 117 | 118 | 此处获取的 dict1、dict2、list1 均为网络对象，数据通过 TCP 传输。三个对象受 proxy 控制，当调用 proxy.close() 归还连接后，三个对象也会失效。同样，使用 proxy.connect() 可重新从连接池获取连接，dict1、dict2、list1 也变为可用。 119 | 120 | 了解此操作后，你便可以像操作 Dictionaries 一样操作 dict1、dict2，像操作 Lists 一样操作 list1 了！（CyberDict 和 CyberList 支持 Dictionaries、Lists 的大部分方法） 121 | 122 | 示例如下 123 | 124 | ##### CyberDict 常用操作 125 | 126 | 在 dict1 和 dict2 中新增键值对 127 | 128 | 129 | ```python 130 | dict1[0] = 100 131 | dict1['test'] = 'Hello CyberDB!' 132 | dict2[0] = 200 133 | ``` 134 | 135 | 获取对应的值 136 | 137 | 138 | ```python 139 | dict1.get(0) 140 | ``` 141 | 142 | 143 | 144 | 145 | 100 146 | 147 | 148 | 149 | 150 | ```python 151 | dict2[0] 152 | ``` 153 | 154 | 155 | 156 | 157 | 200 158 | 159 | 160 | 161 | 查看 dict1 和 dict2 (也可以使用 print 打印) 162 | 163 | 164 | ```python 165 | dict1 166 | ``` 167 | 168 | 169 | 170 | 171 | {0: 100, 'test': 'Hello CyberDB!'} 172 | 173 | 174 | 175 | 176 | ```python 177 | dict2 178 | ``` 179 | 180 | 181 | 182 | 183 | {0: 200} 184 | 185 | 186 | 187 | 获取长度 188 | 189 | 190 | ```python 191 | len(dict1) 192 | ``` 193 | 194 | 195 | 196 | 197 | 2 198 | 199 | 200 | 201 | 删除键值对 202 | 203 | 204 | ```python 205 | del dict1[0] 206 | dict1 207 | ``` 208 | 209 | 210 | 211 | 212 | {'test': 'Hello CyberDB!'} 213 | 214 | 215 | 216 | 清空 dict1 217 | 218 | 219 | ```python 220 | dict1.clear() 221 | dict1 222 | ``` 223 | 224 | 225 | 226 | 227 | {} 228 | 229 | 230 | 231 | ##### CyberList 常用操作 232 | 233 | 生成 list1 的内容 234 | 235 | 236 | ```python 237 | for i in range(5): 238 | list1.append(99) 239 | 240 | list1 241 | ``` 242 | 243 | 244 | 245 | 246 | [99, 99, 99, 99, 99] 247 | 248 | 249 | 250 | 更改坐标值 251 | 252 | 253 | ```python 254 | list1[3] = 100 255 | list1 256 | ``` 257 | 258 | 259 | 260 | 261 | [99, 99, 99, 100, 99] 262 | 263 | 264 | 265 | 进行切片 266 | 267 | 268 | ```python 269 | list1[3:] 270 | ``` 271 | 272 | 273 | 274 | 275 | [100, 99] 276 | 277 | 278 | 279 | 获取 list1 的长度 280 | 281 | 282 | ```python 283 | len(list1) 284 | ``` 285 | 286 | 287 | 288 | 289 | 5 290 | 291 | 292 | 293 | 通过迭代打印 list1 每个元素 294 | 295 | 296 | ```python 297 | for v in list1: 298 | print(v) 299 | ``` 300 | 301 | 99 302 | 99 303 | 99 304 | 100 305 | 99 306 | 307 | 308 | 强烈推荐使用 for 循环迭代 CyberList，每次迭代将从服务端获取 v，客户端的空间复杂度为 o(1)。迭代同样可用于 CyberDict，CyberDict 的迭代中，客户端空间复杂度为 o(n), n 为 CyberDict.keys() 的长度。 309 | 310 | #### 释放 proxy 对象 311 | 312 | 使用完成，将 proxy 的连接归还至连接池即可。 313 | 314 | 315 | ```python 316 | proxy.close() 317 | ``` 318 | 319 | proxy 对象同样支持上下文管理器，如 320 | 321 | 322 | ```python 323 | with client.get_proxy() as proxy: 324 | list1 = proxy.get_cyberlist('list1') 325 | print(list1) 326 | ``` 327 | 328 | [99, 99, 99, 100, 99] 329 | 330 | 331 | ## 概括 332 | 333 | 有了 CyberDB，便能充分利用内存性能，不同进程(甚至不同主机)能通过 Python 的数据结构通信。更多教程请参考文档，感谢你的支持！ 334 | 335 | ## 注意 336 | 337 | 由于编码限制，CyberDB 会将 0 识别为 None，但并不影响计算，请在所需位置将 None 转为 0。 338 | -------------------------------------------------------------------------------- /Performance Testing/CyberDB vs Redis_chn.md: -------------------------------------------------------------------------------- 1 | # [性能测试] Python 内存数据库 CyberDB VS Redis 2 | 3 | 作为基于 Python 的内存数据库，[CyberDB](https://github.com/Cyberbolt/CyberDB) 的表现如何呢？本文主要测试 CyberDB 和 Redis 在 Python Web 中的性能表现。 4 | 5 | 由于 CyberDB 中 proxy 的 connect 方法会检测连接是否有效，为了确保测试的公平性，我们将使用 redis 的 ping 方法与之对应。 6 | 7 | 文章将采用 Gunicorn 3进程 + Gevent 协程的方法测试。环境: Python 3.8.12, PyPy 7.3.7 8 | 9 | 本项目的目录结构为 10 | ```bash 11 | . 12 | ├── app.py 13 | ├── app_redis.py 14 | ├── cyberdb_init.py 15 | ├── cyberdb_serve.py 16 | ├── redis_init.py 17 | └── requirements.txt 18 | ``` 19 | 20 | 每个文件的内容如下 21 | 22 | app.py 23 | ```python 24 | import cyberdb 25 | from flask import Flask, g 26 | 27 | 28 | client = cyberdb.connect( 29 | host='127.0.0.1', 30 | port=9980, 31 | password='hWjYvVdqRC' 32 | ) 33 | 34 | app = Flask(__name__) 35 | 36 | 37 | @app.before_request 38 | def before_request(): 39 | g.proxy = client.get_proxy() 40 | g.proxy.connect() 41 | 42 | 43 | @app.get("/") 44 | def hello_world(): 45 | centre = g.proxy.get_cyberdict('centre') 46 | 47 | return { 48 | 'code': 1, 49 | 'content': centre['content'] 50 | } 51 | 52 | 53 | @app.teardown_request 54 | def teardown_request(error): 55 | g.proxy.close() 56 | 57 | 58 | if __name__ == '__main__': 59 | app.run(host='127.0.0.1', port=8000) 60 | ``` 61 | 62 | app_redis.py 63 | ```python 64 | import redis 65 | from flask import Flask, g 66 | 67 | 68 | rdp = redis.ConnectionPool(host='127.0.0.1', port=6379, decode_responses=True) 69 | r = redis.StrictRedis(connection_pool=rdp) 70 | 71 | 72 | app = Flask(__name__) 73 | 74 | 75 | @app.before_request 76 | def before_request(): 77 | r.ping() 78 | g.r = r 79 | 80 | 81 | @app.get("/") 82 | def hello_world(): 83 | return { 84 | 'code': 1, 85 | 'content': g.r['content'] 86 | } 87 | 88 | 89 | if __name__ == '__main__': 90 | app.run(host='127.0.0.1', port=8000, debug=True) 91 | ``` 92 | 93 | cyberdb_init.py 94 | ```python 95 | import time 96 | 97 | import cyberdb 98 | 99 | db = cyberdb.Server() 100 | 101 | db.start(host='127.0.0.1', port=9980, password='123456') 102 | 103 | time.sleep(3) 104 | client = cyberdb.connect(host='127.0.0.1', port=9980, password='123456') 105 | with client.get_proxy() as proxy: 106 | proxy.create_cyberdict('centre') 107 | centre = proxy.get_cyberdict('centre') 108 | centre['content'] = 'Hello CyberDB!' 109 | 110 | db.save_db('data.cdb') 111 | ``` 112 | cyberdb_serve.py 113 | ```python 114 | import cyberdb 115 | 116 | 117 | def main(): 118 | db = cyberdb.Server() 119 | db.load_db('data.cdb') 120 | db.set_backup('data.cdb', cycle=300) 121 | db.run( 122 | host='127.0.0.1', 123 | port=9980, 124 | password='hWjYvVdqRC', 125 | max_con=10000, 126 | print_log=False 127 | ) 128 | 129 | 130 | if __name__ == '__main__': 131 | main() 132 | ``` 133 | 134 | redis_init.py 135 | ```python 136 | import redis 137 | 138 | 139 | r = redis.Redis(host='127.0.0.1', port=6379, decode_responses=True) 140 | r.set('content', 'Hello CyberDB!') 141 | ``` 142 | 143 | requirements.txt 144 | ``` 145 | APScheduler==3.9.1 146 | async-timeout==4.0.2 147 | backports.zoneinfo==0.2.1 148 | cffi==1.14.6 149 | click==8.1.2 150 | CyberDB==0.9.1 151 | Deprecated==1.2.13 152 | Flask==2.1.1 153 | gevent==21.12.0 154 | greenlet==0.4.13 155 | gunicorn==20.1.0 156 | hpy==0.0.3 157 | importlib-metadata==4.11.3 158 | itsdangerous==2.1.2 159 | Jinja2==3.1.1 160 | MarkupSafe==2.1.1 161 | obj-encrypt==0.7.0 162 | packaging==21.3 163 | pycryptodome==3.14.1 164 | pyparsing==3.0.8 165 | pytz==2022.1 166 | pytz-deprecation-shim==0.1.0.post0 167 | readline==6.2.4.1 168 | redis==4.2.2 169 | six==1.16.0 170 | tzdata==2022.1 171 | tzlocal==4.2 172 | Werkzeug==2.1.1 173 | wrapt==1.14.0 174 | zipp==3.8.0 175 | zope.event==4.5.0 176 | zope.interface==5.4.0 177 | 178 | ``` 179 | 180 | 测试条件: 自己的环境中拥有 Redis 和 wrk 。 181 | 182 | 测试步骤: 183 | 184 | (注：本文不是 Python 的基础教程，如果你不了解虚拟环境，请查询相关文档的操作) 185 | 186 | 1.创建并激活虚拟环境，安装 requirements.txt 中的依赖后，新建终端，运行 `python cyberdb_init.py` 初始化 CyberDB 的表结构，并运行 `python cyberdb_serve.py` 以开启 CyberDB 服务器。 187 | 188 | 2.并在另一个终端运行 `redis-server` 开启 Redis 服务器。在激活的虚拟环境中运行 `python redis_init.py` 初始化 Redis 的表结构。 189 | 190 | 3.运行 `gunicorn -w 3 -b 127.0.0.1:8000 -k gevent app:app` (Flask 运行 CyberDB)，之后使用 `wrk -t8 -c100 -d120s --latency http://127.0.0.1:8000` 测试。结果如下: 191 | 192 | ``` 193 | Running 2m test @ http://127.0.0.1:8000 194 | 8 threads and 100 connections 195 | Thread Stats Avg Stdev Max +/- Stdev 196 | Latency 24.49ms 12.17ms 310.04ms 86.68% 197 | Req/Sec 505.40 124.64 1.28k 77.15% 198 | Latency Distribution 199 | 50% 19.95ms 200 | 75% 29.53ms 201 | 90% 38.15ms 202 | 99% 66.11ms 203 | 482884 requests in 2.00m, 86.58MB read 204 | Requests/sec: 4020.84 205 | Transfer/sec: 738.20KB 206 | ``` 207 | 208 | 4.运行 `gunicorn -w 3 -b 127.0.0.1:8000 -k gevent app_redis:app` (Flask 运行 Redis)，之后使用 `wrk -t8 -c100 -d120s --latency http://127.0.0.1:8000` 测试。结果如下: 209 | 210 | ``` 211 | Running 2m test @ http://127.0.0.1:8000 212 | 8 threads and 100 connections 213 | Thread Stats Avg Stdev Max +/- Stdev 214 | Latency 22.59ms 14.58ms 697.21ms 95.14% 215 | Req/Sec 549.22 85.09 1.08k 78.60% 216 | Latency Distribution 217 | 50% 21.22ms 218 | 75% 25.28ms 219 | 90% 29.66ms 220 | 99% 54.46ms 221 | 524279 requests in 2.00m, 94.00MB read 222 | Requests/sec: 4365.66 223 | Transfer/sec: 801.51KB 224 | ``` 225 | 226 | ### 总结 227 | 228 | 本测试中，CyberDB 的结果为 4020.84 HPS，Redis 的结果为 4365.66 HPS，两者在同一水平。 229 | 230 | 该结果并不令人惊讶，CyberDB 和 redis-py 都是基于 socket 开发的接口，两者服务端也均为单线程。CyberDB 所依赖的 Python 字典和列表，与 Redis 相同，都是由 C 开发。 231 | 232 | 两者最大的区别是，[CyberDB](https://github.com/Cyberbolt/CyberDB) 原生基于 Python，你可以像使用字典和列表一样使用 CyberDB；Redis-py 只是 Python 中操作 Redis 的接口。 -------------------------------------------------------------------------------- /documentation_chn/docs/API.md: -------------------------------------------------------------------------------- 1 | # API 2 | 3 | 这部分文档涵盖了 CyberDB 的所有接口。对于 CyberDB 依赖于外部库的部分，我们在此处记录最重要的部分并提供指向规范文档的链接。 4 | 5 | 6 | ## cyberdb.Server 类 7 | 8 | **class cyberdb.Server** 9 | 10 | 该类用于创建 CyberDB 服务端对象。 11 | 12 | ```python 13 | def start(self, host: str = '127.0.0.1', port: int = 9980, 14 | password: str = None, max_con: int = 500, timeout: int = 0, 15 | print_log: bool = False, encrypt: bool = False): 16 | ''' 17 | 后台启动 CyberDB 服务器，该操作不会阻塞前台任务。 18 | 参数: 19 | host -- TCP 监听主机地址，如 127.0.0.1。 20 | port -- TCP 监听端口。 21 | password -- TCP 通信密码，建议采用 英文字母和数字的组合方式， 22 | 最长不超过 32 个字符。 23 | max_con -- 最大并发数。 24 | timeout -- 单个连接的超时时间，单位 秒。 25 | print_log -- 是否打印通信日志，Fasle 为不打印。 26 | encrypt -- 是否加密通信内容，Fasle 为不加密。此加密算法为 AES-256，密钥为 password。 27 | 返回类型: None 28 | ''' 29 | ``` 30 | 31 | ```python 32 | def run(self, host: str = '127.0.0.1', port: int = 9980, 33 | password: str = None, max_con: int = 500, timeout: int = 0, 34 | print_log: bool = False, encrypt: bool = False): 35 | ''' 36 | 前台运行 CyberDB 服务器，该操作会阻塞前台任务。 37 | 参数和 start 方法相同。 38 | 返回类型: None 39 | ''' 40 | ``` 41 | 42 | ```python 43 | def set_ip_whitelist(self, ips: list): 44 | ''' 45 | 设置 ip 白名单，CyberDB 加密通信时仅允许白名单的 ip 连接， 46 | 该方法仅在 cyberdb.Server.start(encrypt=True) 或 47 | cyberdb.Server.run(encrypt=True) 启用时有效。 48 | 参数: 49 | ips -- 类型为列表，格式如 ['192.168.1.1', '118.123.89.137'] 50 | 返回类型: None 51 | ''' 52 | ``` 53 | 54 | ```python 55 | def set_backup(self, file_name: str = 'data.cdb', cycle: int = 900): 56 | ''' 57 | 设置定时备份，该操作设置后，将在指定周期进行数据持久化备份，将 CyberDB 数据库保存至硬盘。 58 | 参数: 59 | file_name -- 备份生成的文件名，文件后缀必须是 .cdb。 60 | cycle -- 循环备份的周期，单位 秒。 61 | 返回类型: None 62 | ''' 63 | ``` 64 | 65 | ```python 66 | def save_db(self, file_name: str = 'data.cdb'): 67 | ''' 68 | 数据持久化，将 CyberDB 数据库保存至硬盘。 69 | 参数: 70 | file_name -- 备份生成的文件名，文件后缀必须是 .cdb。 71 | 返回类型: None 72 | ''' 73 | ``` 74 | 75 | ```python 76 | def load_db(self, file_name: str = 'data.cdb'): 77 | ''' 78 | 加载 .cdb 格式的文件，将硬盘中备份的 CyberDB 数据库加载回内存。 79 | 参数: 80 | file_name -- 经数据持久化生成的文件名，文件后缀必须是 .cdb。 81 | 返回类型: None 82 | ''' 83 | ``` 84 | 85 | ## cyberdb.connect 函数 86 | 87 | ```python 88 | def cyberdb.connect(host: str = '127.0.0.1', port: int = 9980, password: 89 | str = None, encrypt: bool = False, time_out: int = None) -> Client: 90 | ''' 91 | 将客户端连接至 CyberDB 服务端。 92 | 参数: 93 | host -- 连接地址，如 127.0.0.1 94 | port -- 连接端口 95 | password -- 连接密码 96 | encrypt -- 是否加密通信，如果服务端启用 encrypt 为 True，此处必须为 97 | True，反之亦然。 98 | time_out -- 连接池中每个连接的超时时间，单位 秒。连接池的连接经 time_out 99 | 秒无操作将被舍弃，下次连接将生成新连接。连接池将自动管理连接，开发者无需关注细节。 100 | 此参数若为 None，则不会超时，连接池将维持连接直至失效，之后将重新生成新连接。 101 | 返回类型: Client 102 | ''' 103 | ``` 104 | 105 | ## cyberdb.Client 类 106 | 107 | **class cyberdb.Client** 108 | 109 | cyberdb.connect 函数返回的 cyberdb.Client 对象，用于生成 Proxy 对象。 110 | 111 | ```python 112 | def get_proxy(self) -> Proxy: 113 | ''' 114 | 生成 Proxy 对象。 115 | 返回类型: None 116 | ''' 117 | ``` 118 | 119 | ## Proxy 类 120 | 121 | cyberdb.Client.get_proxy 方法生成的 Proxy 对象，可对 CyberDB 数据库进行操作，并管理由 Proxy 生成的 CyberDict、CyberList 子对象的 TCP 连接。Proxy 对象初始化后，执行 Proxy.connect 方法后才能使用。Proxy 对象及其子对象将执行远程操作，作用于服务端 CyberDB 数据库。 122 | 123 | **class Proxy** 124 | 125 | ```python 126 | def connect(self): 127 | ''' 128 | 从连接池获取 TCP 连接，绑定至该 Proxy 对象。 129 | 返回类型: None 130 | ''' 131 | ``` 132 | 133 | ```python 134 | def close(self): 135 | ''' 136 | 取消该 Proxy 对象绑定的 TCP 连接，归还至连接池，下次操作前需重新 137 | 执行 Proxy.connect 方法。 138 | 返回类型: None 139 | ''' 140 | ``` 141 | 142 | ```python 143 | def create_cyberdict(self, table_name: str, content: dict = {}): 144 | ''' 145 | 创建 CyberDict 表。 146 | 参数: 147 | table_name -- 表名。 148 | content -- 表内容，需要是字典类型，默认为空字典。 149 | 返回类型: None 150 | ''' 151 | ``` 152 | 153 | ```python 154 | def create_cyberlist(self, table_name: str, content: list = []): 155 | ''' 156 | 创建 CyberList 表。 157 | 参数: 158 | table_name -- 表名。 159 | content -- 表内容，需要是列表类型，默认为空列表。 160 | 返回类型: None 161 | ''' 162 | ``` 163 | 164 | ```python 165 | def get_cyberdict(self, table_name: str) -> CyberDict: 166 | ''' 167 | 获取 CyberDict 表。 168 | 参数: 169 | table_name -- 表名。 170 | 返回类型: CyberDict，该对象是 Proxy 生成的子对象，由 Proxy 控制 TCP 连接。 171 | ''' 172 | ``` 173 | 174 | ```python 175 | def get_cyberlist(self, table_name: str) -> CyberList: 176 | ''' 177 | 获取 CyberList 表。 178 | 参数: 179 | table_name -- 表名。 180 | 返回类型: CyberList，该对象是 Proxy 生成的子对象，由 Proxy 控制 TCP 连接。 181 | ''' 182 | ``` 183 | 184 | ```python 185 | def print_tables(self): 186 | ''' 187 | 打印 CyberDB 数据库中的所有表。 188 | 返回类型: None 189 | ''' 190 | ``` 191 | 192 | ```python 193 | def delete_table(self, table_name: str): 194 | ''' 195 | 删除 CyberDB 数据库中的 table_name 表。 196 | 参数: 197 | table_name -- 删除的表名。 198 | 返回类型: None 199 | ''' 200 | ``` 201 | 202 | ## cyberdb.CyberDict 类 203 | 204 | **class cyberdb.CyberDict** 205 | 206 | 由 Proxy 对象生成的子对象，用于执行 Dictionaries 操作。该对象将执行远程操作，作用于服务端 CyberDB 数据库。和 Proxy 对象共用同一个 TCP 连接。将跟随 Proxy 的连接而连接，Proxy 释放连接后，该对象也会失去连接。CyberDict 可以执行 Dictionaries 的 get、setdefault、update、keys、values、items、pop、popitem、clear 方法以及常用魔术方法，此部分请参考[ Python 字典官方文档](https://docs.python.org/3/library/stdtypes.html#mapping-types-dict)。 207 | 208 | 使用 for 循环迭代 CyberDict，客户端空间复杂度为 o(n), n 为 CyberDict.keys() 的长度。 209 | 210 | ```python 211 | def todict(self) -> Dict: 212 | ''' 213 | 将 CyberDict 转为 Dictionaries。 214 | 返回类型: Dict 215 | ''' 216 | ``` 217 | 218 | ## cyberdb.CyberList 类 219 | 220 | **class cyberdb.CyberList** 221 | 222 | 由 Proxy 对象生成的子对象，用于执行 Lists 操作。该对象将执行远程操作，作用于服务端 CyberDB 数据库。和 Proxy 对象共用同一个 TCP 连接。将跟随 Proxy 的连接而连接，Proxy 释放连接后，该对象也会失去连接。CyberList 可以执行 Lists 的 append、extend、insert、pop、remove、count、index、reverse、sort、clear 方法以及常用魔术方法，此部分请参考[ Python 列表官方文档](https://docs.python.org/3/tutorial/datastructures.html#more-on-lists)。 223 | 224 | 使用 for 循环迭代 CyberList，每次迭代将从服务端获取内容，客户端的空间复杂度为 o(1)。 225 | 226 | ```python 227 | def tolist(self) -> List: 228 | ''' 229 | 将 CyberList 转为 Lists。 230 | 返回类型: List 231 | ''' 232 | ``` 233 | 234 | -------------------------------------------------------------------------------- /Performance Testing/CyberDB vs Redis.md: -------------------------------------------------------------------------------- 1 | # [Performance Test] Python In-memory Database CyberDB VS Redis 2 | 3 | How does [CyberDB](https://github.com/Cyberbolt/CyberDB) perform as a Python-based in-memory database? This article mainly tests the performance of CyberDB and Redis in Python Web. 4 | 5 | Since the connect method of the proxy in CyberDB will detect whether the connection is valid, in order to ensure the fairness of the test, we will use the ping method of redis to correspond to it. 6 | 7 | The article will be tested using the Gunicorn 3 process + Gevent coroutine method. Environment: Python 3.8.12, PyPy 7.3.7 8 | 9 | The directory structure of this project is 10 | ```bash 11 | . 12 | ├── app.py 13 | ├── app_redis.py 14 | ├── cyberdb_init.py 15 | ├── cyberdb_serve.py 16 | ├── redis_init.py 17 | └── requirements.txt 18 | ``` 19 | 20 | The content of each file is as follows 21 | 22 | app.py 23 | ```python 24 | import cyberdb 25 | from flask import Flask, g 26 | 27 | 28 | client = cyberdb.connect( 29 | host='127.0.0.1', 30 | port=9980, 31 | password='hWjYvVdqRC' 32 | ) 33 | 34 | app = Flask(__name__) 35 | 36 | 37 | @app.before_request 38 | def before_request(): 39 | g.proxy = client.get_proxy() 40 | g.proxy.connect() 41 | 42 | 43 | @app.get("/") 44 | def hello_world(): 45 | centre = g.proxy.get_cyberdict('centre') 46 | 47 | return { 48 | 'code': 1, 49 | 'content': centre['content'] 50 | } 51 | 52 | 53 | @app.teardown_request 54 | def teardown_request(error): 55 | g.proxy.close() 56 | 57 | 58 | if __name__ == '__main__': 59 | app.run(host='127.0.0.1', port=8000) 60 | ``` 61 | 62 | app_redis.py 63 | ```python 64 | import redis 65 | from flask import Flask, g 66 | 67 | 68 | rdp = redis.ConnectionPool(host='127.0.0.1', port=6379, decode_responses=True) 69 | r = redis.StrictRedis(connection_pool=rdp) 70 | 71 | 72 | app = Flask(__name__) 73 | 74 | 75 | @app.before_request 76 | def before_request(): 77 | r.ping() 78 | g.r = r 79 | 80 | 81 | @app.get("/") 82 | def hello_world(): 83 | return { 84 | 'code': 1, 85 | 'content': g.r['content'] 86 | } 87 | 88 | 89 | if __name__ == '__main__': 90 | app.run(host='127.0.0.1', port=8000, debug=True) 91 | ``` 92 | 93 | cyberdb_init.py 94 | ```python 95 | import time 96 | 97 | import cyberdb 98 | 99 | db = cyberdb.Server() 100 | 101 | db.start(host='127.0.0.1', port=9980, password='123456') 102 | 103 | time.sleep(3) 104 | client = cyberdb.connect(host='127.0.0.1', port=9980, password='123456') 105 | with client.get_proxy() as proxy: 106 | proxy.create_cyberdict('centre') 107 | centre = proxy.get_cyberdict('centre') 108 | centre['content'] = 'Hello CyberDB!' 109 | 110 | db.save_db('data.cdb') 111 | ``` 112 | cyberdb_serve.py 113 | ```python 114 | import cyberdb 115 | 116 | 117 | def main(): 118 | db = cyberdb.Server() 119 | db.load_db('data.cdb') 120 | db.set_backup('data.cdb', cycle=300) 121 | db.run( 122 | host='127.0.0.1', 123 | port=9980, 124 | password='hWjYvVdqRC', 125 | max_con=10000, 126 | print_log=False 127 | ) 128 | 129 | 130 | if __name__ == '__main__': 131 | main() 132 | ``` 133 | 134 | redis_init.py 135 | ```python 136 | import redis 137 | 138 | 139 | r = redis.Redis(host='127.0.0.1', port=6379, decode_responses=True) 140 | r.set('content', 'Hello CyberDB!') 141 | ``` 142 | 143 | requirements.txt 144 | ``` 145 | APScheduler==3.9.1 146 | async-timeout==4.0.2 147 | backports.zoneinfo==0.2.1 148 | cffi==1.14.6 149 | click==8.1.2 150 | CyberDB==0.9.1 151 | Deprecated==1.2.13 152 | Flask==2.1.1 153 | gevent==21.12.0 154 | greenlet==0.4.13 155 | gunicorn==20.1.0 156 | hpy==0.0.3 157 | importlib-metadata==4.11.3 158 | itsdangerous==2.1.2 159 | Jinja2==3.1.1 160 | MarkupSafe==2.1.1 161 | obj-encrypt==0.7.0 162 | packaging==21.3 163 | pycryptodome==3.14.1 164 | pyparsing==3.0.8 165 | pytz==2022.1 166 | pytz-deprecation-shim==0.1.0.post0 167 | readline==6.2.4.1 168 | redis==4.2.2 169 | six==1.16.0 170 | tzdata==2022.1 171 | tzlocal==4.2 172 | Werkzeug==2.1.1 173 | wrapt==1.14.0 174 | zipp==3.8.0 175 | zope.event==4.5.0 176 | zope.interface==5.4.0 177 | 178 | ``` 179 | 180 | Test conditions: Redis and wrk in your own environment. 181 | 182 | Test steps: 183 | 184 | (Note: This article is not a basic tutorial of Python. If you do not understand virtual environment, please check the operation of related documents) 185 | 186 | 1. Create and activate a virtual environment, install the dependencies in requirements.txt, create a new terminal, run `python cyberdb_init.py` to initialize the table structure of CyberDB, and run `python cyberdb_serve.py` to start the CyberDB server. 187 | 188 | 2. And run `redis-server` in another terminal to start the Redis server. Run `python redis_init.py` in the activated virtual environment to initialize the Redis table structure. 189 | 190 | 3. Run `gunicorn -w 3 -b 127.0.0.1:8000 -k gevent app:app` (Flask runs CyberDB), then use `wrk -t8 -c100 -d120s --latency http://127.0.0.1:8000 ` test. The result is as follows: 191 | 192 | ``` 193 | Running 2m test @ http://127.0.0.1:8000 194 | 8 threads and 100 connections 195 | Thread Stats Avg Stdev Max +/- Stdev 196 | Latency 24.49ms 12.17ms 310.04ms 86.68% 197 | Req/Sec 505.40 124.64 1.28k 77.15% 198 | Latency Distribution 199 | 50% 19.95ms 200 | 75% 29.53ms 201 | 90% 38.15ms 202 | 99% 66.11ms 203 | 482884 requests in 2.00m, 86.58MB read 204 | Requests/sec: 4020.84 205 | Transfer/sec: 738.20KB 206 | ``` 207 | 208 | 1. Run `gunicorn -w 3 -b 127.0.0.1:8000 -k gevent app_redis:app` (Flask runs Redis), then use `wrk -t8 -c100 -d120s --latency http://127.0.0.1:8000 ` test. The result is as follows: 209 | 210 | ``` 211 | Running 2m test @ http://127.0.0.1:8000 212 | 8 threads and 100 connections 213 | Thread Stats Avg Stdev Max +/- Stdev 214 | Latency 22.59ms 14.58ms 697.21ms 95.14% 215 | Req/Sec 549.22 85.09 1.08k 78.60% 216 | Latency Distribution 217 | 50% 21.22ms 218 | 75% 25.28ms 219 | 90% 29.66ms 220 | 99% 54.46ms 221 | 524279 requests in 2.00m, 94.00MB read 222 | Requests/sec: 4365.66 223 | Transfer/sec: 801.51KB 224 | ``` 225 | 226 | ### Summarize 227 | 228 | In this test, the result of CyberDB is 4020.84 HPS, and the result of Redis is 4365.66 HPS, both are on the same level. 229 | 230 | This result is not surprising, both CyberDB and redis-py are socket-based interfaces, and both servers are also single-threaded. The Python dictionaries and lists that CyberDB relies on, the same as Redis, are developed in C. 231 | 232 | The biggest difference between the two is that [CyberDB](https://github.com/Cyberbolt/CyberDB) is natively based on Python, and you can use CyberDB like dictionaries and lists; Redis-py is just an interface for operating Redis in Python. -------------------------------------------------------------------------------- /documentation/docs/tutorial/flask.md: -------------------------------------------------------------------------------- 1 | # Using CyberDB as Flask's In-memory Database in Production 2 | 3 | Earlier we talked about CyberDB's [Quick Start](https://www.cyberlight.xyz/static/cyberdb/), now we need to bring it to a place where it can play its role, and use CyberDB as Flask's in the production environment An in-memory database that runs with Gunicorn and enables multi-process communication. 4 | 5 | This article is explained through a Flask example that is as concise as possible, and will not involve complex web knowledge. The core idea is CyberDB + Gunicorn + Gevent + Flask (multi-process + coroutine), start a CyberDB server, use Gunicorn multi-process to run the Flask instance, the instance of each process runs through Gevent, and the CyberDB client is used to connect to the in-memory database in the process , thereby achieving high concurrent access to the CyberDB database. 6 | 7 | Articles are run using PyPy, as well as CPython. 8 | 9 | Runtime: Python 3.8.12, PyPy 7.3.7 10 | 11 | The directory structure of this project 12 | ```bash 13 | . 14 | ├── app.py 15 | ├── cyberdb_init.py 16 | ├── cyberdb_serve.py 17 | ├── requirements.txt 18 | └── venv 19 | ``` 20 | We explain the core operations of CyberDB by listing the contents of each file in order. 21 | 22 | File requirements.txt 23 | ``` 24 | CyberDB>=0.7.1 25 | Flask==2.1.1 26 | gevent==21.12.0 27 | gunicorn==20.1.0 28 | ``` 29 | dependencies of this project. This article is not a basic Python tutorial, please refer to related documents to create a virtual environment venv directory and install the dependencies in requirements.txt. 30 | 31 | After generating the venv directory and installing the dependencies, all the following operations will run in the activated virtual environment. 32 | 33 | File cyberdb_init.py 34 | ```python 35 | ''' 36 | This module is used to initialize the table structure of CyberDB, and it is 37 | only used for the first run, and will not be used subsequently. 38 | ''' 39 | 40 | 41 | import time 42 | 43 | import cyberdb 44 | 45 | db = cyberdb.Server() 46 | # Configure the address, port and password of the CyberDB server. 47 | db.start(host='127.0.0.1', port=9980, password='123456') 48 | 49 | # After the server starts, connect to the CyberDB server. 50 | time.sleep(3) 51 | client = cyberdb.connect(host='127.0.0.1', port=9980, password='123456') 52 | # Generate proxy object. 53 | with client.get_proxy() as proxy: 54 | # Create a table centre of type CyberDict and initialize the contents. 55 | proxy.create_cyberdict('centre') 56 | centre = proxy.get_cyberdict('centre') 57 | centre['content'] = 'Hello CyberDB!' 58 | 59 | # Save CyberDB to data.cdb. 60 | db.save_db('data.cdb') 61 | ``` 62 | 63 | Execute in the project root directory 64 | ```bash 65 | python cyberdb_init.py 66 | ``` 67 | At this point, the initialization of the CyberDB database table is completed, a table named center and type CyberDict is created in CyberDB, the value of the initial 'content' key is 'Hello CyberDB!', and finally the CyberDB database is saved to the hard disk (project root directory generates a file named data.cdb). 68 | 69 | File cyberdb_serve.py 70 | ```python 71 | import cyberdb 72 | 73 | 74 | def main(): 75 | # Run the CyberDB server in the background and set relevant 76 | # information. 77 | db = cyberdb.Server() 78 | # Read data.cdb from hard disk to CyberDB. 79 | db.load_db('data.cdb') 80 | # Backup the database every 300 seconds. 81 | db.set_backup('data.cdb', cycle=300) 82 | db.run( 83 | host='127.0.0.1', # TCP run address 84 | port=9980, # TCP listening port 85 | password='hWjYvVdqRC', # Database connection password 86 | max_con=10000, # max concurrency 87 | encrypt=True, # encrypted communication 88 | print_log=False # do not print logs 89 | ) 90 | 91 | 92 | if __name__ == '__main__': 93 | main() 94 | ``` 95 | Execute it in the project root directory 96 | ```bash 97 | python cyberdb_serve.py 98 | ``` 99 | to run the CyberDB server. 100 | 101 | If encrypt=True is set here, CyberDB will encrypt the TCP communication content using the AES-256 algorithm. After enabling encrypt=True, CyberDB only allows ip communication in the whitelist, the default whitelist is ['127.0.0.1'], the whitelist [setting method](https://www.cyberlight.xyz/static/cyberdb/API/#cyberdbserver). Generally, if you only need to communicate between local processes, you do not need to enable encrypt=True and set a whitelist. This operation is only required for remote communication. 102 | 103 | File app.py 104 | ```python 105 | import cyberdb 106 | from flask import Flask, g 107 | 108 | 109 | # Connect to CyberDB and spawn a client instance. 110 | client = cyberdb.connect( 111 | host='127.0.0.1', 112 | port=9980, 113 | password='hWjYvVdqRC', 114 | # If the server is encrypted, the client must be encrypted, and vice 115 | # versa. 116 | encrypt=True, 117 | # Each connection will be discarded if there is no operation for more 118 | # than 900 seconds. 119 | # Connections are managed intelligently by the connection pool, 120 | # no relationship details are required. 121 | time_out=900 122 | ) 123 | 124 | # Create a Flask instance, please refer to the Flask documentation for this 125 | # section https://flask.palletsprojects.com/ 126 | app = Flask(__name__) 127 | 128 | 129 | @app.before_request 130 | def before_request(): 131 | # Generate a proxy object before each request is executed. 132 | g.proxy = client.get_proxy() 133 | # Get a connection from the connection pool. 134 | g.proxy.connect() 135 | 136 | 137 | @app.get("/") 138 | def hello_world(): 139 | # Get the centre table from the database. 140 | centre = g.proxy.get_cyberdict('centre') 141 | 142 | return { 143 | 'code': 1, 144 | 'content': centre['content'] 145 | } 146 | 147 | 148 | @app.teardown_request 149 | def teardown_request(error): 150 | # Return the connection to the connection pool after each request is executed. 151 | g.proxy.close() 152 | 153 | 154 | if __name__ == '__main__': 155 | app.run(host='127.0.0.1', port=8000) 156 | ``` 157 | This module will use client.get_proxy() to get the proxy object before each request is executed ( before_request ). Each obtained proxy object can be bound to a TCP connection. Here, proxy.connect() is used to get the connection from the connection pool. In the view function hello_world, the object center obtained by the proxy shares the same connection with the proxy. After the connection of the proxy is released, the center will also lose the connection. After each request ( teardown_request ), use the proxy.close() method to release the connection bound by the proxy and return it to the connection pool. 158 | 159 | The time_out parameter of cyberdb.connect is the timeout for each connection in the connection pool, where each connection will be discarded after 900 seconds of inactivity. If this parameter is not set, each connection in the connection pool will be maintained until it expires. 160 | 161 | Run in the project root directory 162 | ```bash 163 | gunicorn -w 4 -b 127.0.0.1:8000 -k gevent app:app 164 | ``` 165 | Start a Flask instance with 4 processes, Gevent. 166 | 167 | Browser access to 127.0.0.1:8000 will get the following response 168 | ```JSON 169 | {"code":1,"content":"Hello CyberDB!"} 170 | ``` 171 | With this example, you can deploy CyberDB into more complex web environments and take full advantage of the low-latency characteristics of memory. 172 | -------------------------------------------------------------------------------- /documentation/docs/index.md: -------------------------------------------------------------------------------- 1 | # CyberDB 2 | 3 | [Chinese Version](https://github.com/Cyberbolt/CyberDB/blob/main/README_CHN.md) 4 | 5 | CyberDB is a lightweight Python in-memory database. It is designed to use Python's built-in data structures Dictionaries, Lists for data storage, efficient communication through Socket TCP, and provide data persistence. This module can be used in hard disk database caching, Gunicorn inter-process communication, distributed computing and other fields. 6 | 7 | The CyberDB server uses Asyncio for TCP communication. The client is developed based on Socket, so it supports the Gevent coroutine, but it has not yet adapted to Asyncio. Both the server and the client support PyPy, and it is recommended to use PyPy to run for better performance. 8 | 9 | In high concurrency scenarios, the performance bottleneck of traditional databases is mainly hard disk I/O. Even if CyberDB is developed based on the dynamic language Python, the speed is still much faster than that of hard disk databases (such as MySQL), and CyberDB can be used as its cache. In addition, the core of CyberDB lies in programming in a Pythonic way, you can use CyberDB like Dictionaries and Lists. 10 | 11 | ## Installation 12 | 13 | 1. Enter the command window, create a virtual environment, and enter the following commands in turn 14 | 15 | Linux and macOS: 16 | 17 | 18 | 19 | ```python 20 | python3 -m venv venv # Create a virtual environment. 21 | . venv/bin/activate # Activate the virtual environment. 22 | ``` 23 | 24 | Windows: 25 | 26 | 27 | ```python 28 | python -m venv venv # Create a virtual environment. 29 | venv\Scripts\activate # Activate the virtual environment. 30 | ``` 31 | 32 | 2.Install CyberDB, enter 33 | 34 | 35 | ```python 36 | pip install --upgrade pip 37 | pip install cyberdb 38 | ``` 39 | 40 | If your server and client are running in two different project directories, please install CyberDB in the virtual environment of the server and client respectively. 41 | 42 | ## Links 43 | 44 | - GitHub: [https://github.com/Cyberbolt/CyberDB](https://github.com/Cyberbolt/CyberDB) 45 | - PyPI: [https://pypi.org/project/CyberDB/](https://pypi.org/project/CyberDB/) 46 | - Documentation: [https://www.cyberlight.xyz/static/cyberdb](https://www.cyberlight.xyz/static/cyberdb) 47 | - CyberLight: [https://www.cyberlight.xyz/](https://www.cyberlight.xyz/) 48 | 49 | ## Quick to Use 50 | 51 | In this module, please use CyberDict and CyberList instead of dict and list (a TCP-based Dictionaries-like, Lists-like object). 52 | 53 | ### Server 54 | 55 | Run the database server. 56 | 57 | ```python 58 | import time 59 | import cyberdb 60 | 61 | db = cyberdb.Server() 62 | # The data is persistent, the backup file is data.cdb, and the backup cycle is every 900 seconds. 63 | db.set_backup('data.cdb', cycle=900) 64 | # Set the TCP address, port number, and password. 65 | # The start method will not block the operation. If you want the operation to block, please use the run method instead of start, and the parameters remain unchanged. 66 | db.start(host='127.0.0.1', port=9980, password='123456') 67 | 68 | while True: 69 | time.sleep(10000) 70 | ``` 71 | 72 | After the above server runs, data.cdb and data_backup.cdb (backup files) will be generated (or overwritten) in the project root directory every 900 seconds. The file can be read the next time the database is started using the load method. 73 | 74 | ### Client 75 | 76 | #### Connect to the Database 77 | 78 | 79 | ```python 80 | import cyberdb 81 | 82 | # Generate a client instance and connect. 83 | client = cyberdb.connect(host='127.0.0.1', port=9980, password='123456') 84 | ``` 85 | 86 | #### Generate proxy Object 87 | 88 | 89 | ```python 90 | # Generate proxy for this request. 91 | proxy = client.get_proxy() 92 | # Automatically obtain database connections from the connection pool. 93 | proxy.connect() 94 | ``` 95 | 96 | The proxy object is not thread-safe, please generate the proxy object separately in each thread (or coroutine), and obtain the database connection through the connect method. You only need to use the close method to return the connection after the operation is completed, and the returned connection is managed intelligently by the client object. 97 | 98 | #### Manipulate proxy Objects 99 | 100 | Create CyberDict and CyberList 101 | 102 | 103 | ```python 104 | # Create dict1 and dict2 tables of type CyberDict and 105 | # list1 table of type CyberList in the database respectively. 106 | proxy.create_cyberdict('dict1') 107 | proxy.create_cyberdict('dict2') 108 | proxy.create_cyberlist('list1') 109 | ``` 110 | 111 | 112 | ```python 113 | dict1 = proxy.get_cyberdict('dict1') 114 | dict2 = proxy.get_cyberdict('dict2') 115 | list1 = proxy.get_cyberlist('list1') 116 | ``` 117 | 118 | The dict1, dict2, and list1 obtained here are all network objects, and the data is transmitted through TCP. The three objects are controlled by the proxy, and when proxy.close() is called to return the connection, the three objects will also be invalid. Similarly, use proxy.connect() to get connections from the connection pool again, and dict1, dict2, list1 also become available. 119 | 120 | Once you understand this operation, you can operate dict1, dict2 like Dictionaries, and list1 like Lists! (CyberDict and CyberList support most methods of Dictionaries, Lists) 121 | 122 | Examples are as follows 123 | 124 | ##### Common Operations of CyberDict 125 | 126 | Add key-value pairs in dict1 and dict2 127 | 128 | 129 | ```python 130 | dict1[0] = 100 131 | dict1['test'] = 'Hello CyberDB!' 132 | dict2[0] = 200 133 | ``` 134 | 135 | Get the corresponding value 136 | 137 | 138 | ```python 139 | dict1.get(0) 140 | ``` 141 | 142 | 143 | 144 | 145 | 100 146 | 147 | 148 | 149 | 150 | ```python 151 | dict2[0] 152 | ``` 153 | 154 | 155 | 156 | 157 | 200 158 | 159 | 160 | 161 | View dict1 and dict2 (can also be printed with print ) 162 | 163 | 164 | ```python 165 | dict1 166 | ``` 167 | 168 | 169 | 170 | 171 | {0: 100, 'test': 'Hello CyberDB!'} 172 | 173 | 174 | 175 | 176 | ```python 177 | dict2 178 | ``` 179 | 180 | 181 | 182 | 183 | {0: 200} 184 | 185 | 186 | 187 | Get length 188 | 189 | 190 | ```python 191 | len(dict1) 192 | ``` 193 | 194 | 195 | 196 | 197 | 2 198 | 199 | 200 | 201 | Delete key-value pair 202 | 203 | 204 | ```python 205 | del dict1[0] 206 | ``` 207 | 208 | 209 | ```python 210 | dict1 211 | ``` 212 | 213 | 214 | 215 | 216 | {'test': 'Hello CyberDB!'} 217 | 218 | 219 | 220 | Empty dict1 221 | 222 | 223 | ```python 224 | dict1.clear() 225 | dict1 226 | ``` 227 | 228 | 229 | 230 | 231 | {} 232 | 233 | 234 | 235 | ##### Common Operations of CyberList 236 | 237 | Generate the contents of list1 238 | 239 | 240 | ```python 241 | for i in range(5): 242 | list1.append(99) 243 | 244 | list1 245 | ``` 246 | 247 | 248 | 249 | 250 | [99, 99, 99, 99, 99] 251 | 252 | 253 | 254 | Change coordinate values 255 | 256 | 257 | ```python 258 | list1[3] = 100 259 | list1 260 | ``` 261 | 262 | 263 | 264 | 265 | [99, 99, 99, 100, 99] 266 | 267 | 268 | 269 | To slice 270 | 271 | 272 | ```python 273 | list1[3:] 274 | ``` 275 | 276 | 277 | 278 | 279 | [100, 99] 280 | 281 | 282 | 283 | Get the length of list1 284 | 285 | 286 | ```python 287 | len(list1) 288 | ``` 289 | 290 | 291 | 292 | 293 | 5 294 | 295 | 296 | 297 | Print each element of list1 by iterating 298 | 299 | 300 | ```python 301 | for v in list1: 302 | print(v) 303 | ``` 304 | 305 | 99 306 | 99 307 | 99 308 | 100 309 | 99 310 | 311 | 312 | It is strongly recommended to use a for loop to iterate CyberList, each iteration will get v from the server, and the space complexity of the client is o(1). Iteration can also be used for CyberDict. In the iteration of CyberDict, the client space complexity is o(n), and n is the size of CyberDict.keys(). 313 | 314 | #### Release the Proxy Object 315 | 316 | After the use is complete, return the connection of the proxy to the connection pool. 317 | 318 | 319 | ```python 320 | proxy.close() 321 | ``` 322 | 323 | The proxy object also supports context managers, such as 324 | 325 | 326 | ```python 327 | with client.get_proxy() as proxy: 328 | list1 = proxy.get_cyberlist('list1') 329 | print(list1) 330 | ``` 331 | 332 | [99, 99, 99, 100, 99] 333 | 334 | 335 | ## Generalize 336 | 337 | With CyberDB, memory performance can be fully utilized, and different processes (or even different hosts) can communicate through Python's data structures. For more tutorials, please refer to the documentation, Thank you! 338 | 339 | ## Notice 340 | 341 | Due to encoding limitations, CyberDB will recognize 0 as None, but it does not affect the calculation, please convert None to 0 in the desired position. 342 | -------------------------------------------------------------------------------- /cyberdb/network/aioclient.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from obj_encrypt import Secret 4 | 5 | from . import AioStream 6 | from ..data import datas 7 | from ..extensions import MyThread, CyberDBError 8 | from ..extensions.signature import Signature 9 | 10 | 11 | result = None # Check whether the connection is successful 12 | 13 | 14 | class Connection: 15 | ''' 16 | Asyncio TCP connection data type 17 | ''' 18 | 19 | def __init__( 20 | self, 21 | reader: asyncio.streams.StreamReader=None, 22 | writer: asyncio.streams.StreamWriter=None 23 | ): 24 | self.reader = reader 25 | self.writer = writer 26 | 27 | 28 | class ConPool: 29 | ''' 30 | Maintain connection pool. 31 | ''' 32 | 33 | def __init__(self, host: str, port: str, dp: datas.DataParsing): 34 | self._host = host 35 | self._port = port 36 | self._dp = dp 37 | self._connections = [] 38 | 39 | async def get(self): 40 | ''' 41 | Get the connection from the connection pool. 42 | ''' 43 | if self._connections: 44 | # If the connection is full, the loop waits until a connection 45 | # is free. 46 | while self._connections: 47 | reader, writer = self._connections.pop(0) 48 | if await self.exam(reader, writer): 49 | return reader, writer 50 | 51 | reader, writer = await asyncio.open_connection( 52 | self._host, self._port) 53 | return reader, writer 54 | 55 | def put(self, reader: asyncio.streams.StreamReader, 56 | writer: asyncio.streams.StreamWriter): 57 | ''' 58 | Return the connection to the connection pool. 59 | ''' 60 | self._connections.append((reader, writer)) 61 | 62 | async def exam(self, reader: asyncio.streams.StreamReader, 63 | writer: asyncio.streams.StreamWriter): 64 | ''' 65 | Check if the connection is valid. 66 | ''' 67 | # try: 68 | stream = AioStream(reader, writer, self._dp) 69 | client_obj = { 70 | 'route': '/connect' 71 | } 72 | await stream.write(client_obj) 73 | 74 | server_obj = await stream.read() 75 | return True 76 | # except: 77 | # return False 78 | 79 | 80 | class CyberDict: 81 | 82 | def __init__( 83 | self, 84 | table_name: str, 85 | dp: datas.DataParsing, 86 | con: Connection 87 | ): 88 | self._table_name = table_name 89 | self._dp = dp 90 | self._con = con 91 | self._route = '/cyberdict' 92 | 93 | async def __getitem__(self, key): 94 | stream = AioStream(self._con.reader, self._con.writer, self._dp) 95 | 96 | client_obj = { 97 | 'route': self._route + '/getitem', 98 | 'table_name': self._table_name, 99 | 'key': key 100 | } 101 | 102 | await stream.write(client_obj) 103 | 104 | server_obj = await stream.read() 105 | if server_obj['code'] == 0: 106 | self._con.writer.close() 107 | raise server_obj['Exception'] 108 | 109 | return server_obj['content'] 110 | 111 | 112 | class CyberList: 113 | 114 | def __init__( 115 | self, 116 | table_name: str, 117 | dp: datas.DataParsing, 118 | con: Connection 119 | ): 120 | self._table_name = table_name 121 | self._dp = dp 122 | self._con = con 123 | self._route = '/cyberlist' 124 | 125 | 126 | class Proxy: 127 | ''' 128 | Database instance per session 129 | 130 | This object is not thread safe, please regenerate in each thread or 131 | coroutine. 132 | ''' 133 | 134 | def __init__(self, con_pool: ConPool, dp: datas.DataParsing): 135 | self._con_pool = con_pool 136 | self._dp = dp 137 | # The connection used by the proxy, the first is the reader and the 138 | # second is the writer. 139 | self._con = Connection() 140 | 141 | async def connect(self): 142 | ''' 143 | Get the latest connection from the connection pool. 144 | 145 | This method does not necessarily create a new connection, if the 146 | connection of the connection pool is available, it will be 147 | obtained directly. 148 | ''' 149 | # If the proxy already has a connection, the connection will be 150 | # returned to the connection pool first. 151 | if self._con.reader != None and self._con.writer != None: 152 | self._con_pool.put(self._con.reader, self._con.writer) 153 | 154 | reader, writer = await self._con_pool.get() 155 | self._con.reader = reader 156 | self._con.writer = writer 157 | 158 | async def close(self): 159 | ''' 160 | Return the connection to the connection pool. 161 | ''' 162 | if self._con.reader == None or self._con == None: 163 | raise CyberDBError('The connection could not be closed, the proxy has not acquired a connection.') 164 | 165 | self._con_pool.put(self._con.reader, self._con.writer) 166 | self._con.reader = None 167 | self._con.writer = None 168 | 169 | async def create_cyberdict(self, table_name: str, content: dict={}): 170 | if type(table_name) != type(''): 171 | raise CyberDBError('Please use str for the table name.') 172 | if type(content) != type(dict()): 173 | raise CyberDBError('The input database table type is not a Python dictionary.') 174 | 175 | stream = AioStream(self._con.reader, self._con.writer, self._dp) 176 | client_obj = { 177 | 'route': '/create_cyberdict', 178 | 'table_name': table_name, 179 | 'content': content 180 | } 181 | await stream.write(client_obj) 182 | 183 | server_obj = await stream.read() 184 | 185 | if server_obj['code'] == 0: 186 | raise CyberDBError('Duplicate table names already exist!') 187 | 188 | return server_obj 189 | 190 | async def get_cyberdict(self, table_name: str) -> CyberDict: 191 | ''' 192 | Get the network object of the table from the database. 193 | ''' 194 | if type(table_name) != type(''): 195 | raise CyberDBError('Please use str for the table name.') 196 | 197 | stream = AioStream(self._con.reader, self._con.writer, self._dp) 198 | 199 | client_obj = { 200 | 'route': '/exam_cyberdict', 201 | 'table_name': table_name 202 | } 203 | await stream.write(client_obj) 204 | 205 | server_obj = await stream.read() 206 | if server_obj['code'] == 0: 207 | raise CyberDBError('{} table does not exist.'.format(table_name)) 208 | else: 209 | table = CyberDict(table_name, self._dp, self._con) 210 | return table 211 | 212 | 213 | class Client: 214 | ''' 215 | This object blocks execution and is recommended to be used as a 216 | global variable instead of running in an async function. 217 | ''' 218 | 219 | def __init__(self, con_pool: ConPool, dp: datas.DataParsing): 220 | self._con_pool = con_pool 221 | self._dp = dp 222 | 223 | def get_proxy(self) -> Proxy: 224 | ''' 225 | Get proxy data. 226 | ''' 227 | proxy = Proxy(self._con_pool, self._dp) 228 | return proxy 229 | 230 | def check_connection_pool(self): 231 | pass 232 | 233 | 234 | def connect(host: str='127.0.0.1', port: int=9980, password: 235 | str=None) -> Client: 236 | ''' 237 | Connect to the CyberDB server via TCP, this method will run 238 | synchronously. 239 | ''' 240 | if not password: 241 | raise RuntimeError('The password cannot be empty.') 242 | 243 | # Responsible for encrypting and decrypting objects. 244 | secret = Secret(key=password) 245 | # for digital signature 246 | signature = Signature(salt=password.encode()) 247 | dp = datas.DataParsing(secret, signature) 248 | con_pool = ConPool(host, port, dp) 249 | 250 | # Synchronously test whether the connection is successful. 251 | t = MyThread(target=asyncio.run, 252 | args=(confirm_the_connection(con_pool, dp), ) ) 253 | t.daemon = True 254 | t.start() 255 | t.join() 256 | result = t.get_result() 257 | if result['code'] == 0: 258 | raise result['Exception'] 259 | 260 | client = Client(con_pool, dp) 261 | return client 262 | 263 | 264 | async def confirm_the_connection(con_pool: ConPool, dp: datas.DataParsing) -> \ 265 | dict: 266 | ''' 267 | The connection is detected when the database connects for the first 268 | time. 269 | ''' 270 | try: 271 | reader, writer = await con_pool.get() 272 | stream = AioStream(reader, writer, dp) 273 | 274 | client_obj = { 275 | 'route': '/connect' 276 | } 277 | await stream.write(client_obj) 278 | 279 | server_obj = await stream.read() 280 | 281 | writer.close() 282 | 283 | return { 284 | 'code': 1, 285 | 'content': server_obj 286 | } 287 | except (ConnectionRefusedError, CyberDBError) as e: 288 | return { 289 | 'code': 0, 290 | 'Exception': e 291 | } 292 | # except Exception as e: 293 | # return { 294 | # 'code': 0, 295 | # 'Exception': CyberDBError('Incorrect address, port or password for database.') 296 | # } 297 | 298 | 299 | -------------------------------------------------------------------------------- /documentation/docs/API.md: -------------------------------------------------------------------------------- 1 | # API 2 | 3 | This part of the documentation covers all the interfaces of CyberDB. For the parts of CyberDB that depend on external libraries, we document the most important parts here and provide a link to the specification document. 4 | 5 | 6 | ## cyberdb.Server Class 7 | 8 | **class cyberdb.Server** 9 | 10 | This class is used to create CyberDB server objects. 11 | 12 | ```python 13 | def start(self, host: str = '127.0.0.1', port: int = 9980, 14 | password: str = None, max_con: int = 500, timeout: int = 0, 15 | print_log: bool = False, encrypt: bool = False): 16 | ''' 17 | Starts the CyberDB server in the background, which does not block 18 | foreground tasks. 19 | 20 | Parameters: 21 | 22 | host -- TCP listening host address, such as 127.0.0.1. 23 | 24 | port – TCP listening port. 25 | 26 | password -- TCP communication password, it is recommended to use 27 | a combination of English letters and numbers, 28 | Up to 32 characters long. 29 | 30 | max_con -- the maximum number of concurrency. 31 | 32 | timeout -- The timeout for a single connection, in seconds. 33 | 34 | print_log -- whether to print the communication log, Fasle does not 35 | print it. 36 | 37 | encrypt -- Whether to encrypt the communication content, Fasle is 38 | not encrypted. The encryption algorithm is AES-256 and the key is 39 | password. 40 | 41 | Return Type: None 42 | ''' 43 | ``` 44 | 45 | ```python 46 | def run(self, host: str = '127.0.0.1', port: int = 9980, 47 | password: str = None, max_con: int = 500, timeout: int = 0, 48 | print_log: bool = False, encrypt: bool = False): 49 | ''' 50 | Running the CyberDB server in the foreground blocks foreground tasks. 51 | The parameters are the same as the start method. 52 | 53 | Return Type: None 54 | ''' 55 | ``` 56 | 57 | ```python 58 | def set_ip_whitelist(self, ips: list): 59 | ''' 60 | Set the ip whitelist. When CyberDB encrypts communication, only 61 | whitelisted ip connections are allowed. 62 | 63 | This method only works if cyberdb.Server.start(encrypt=True) or 64 | cyberdb.Server.run(encrypt=True) are enabled. 65 | 66 | Parameters: 67 | 68 | ips -- the type is a list, the format is ['192.168.1.1', 69 | '118.123.89.137'] 70 | 71 | Return Type: None 72 | ''' 73 | ``` 74 | 75 | ```python 76 | def set_backup(self, file_name: str = 'data.cdb', cycle: int = 900): 77 | ''' 78 | Set timed backup. After this operation is set, data persistent backup 79 | will be performed at the specified period, and the CyberDB database 80 | will be saved to the hard disk. 81 | 82 | parameter: 83 | 84 | file_name -- the name of the file generated by the backup, the file 85 | suffix must be .cdb. 86 | 87 | cycle -- The cycle of the cyclic backup, in seconds. 88 | 89 | Return Type: None 90 | ''' 91 | ``` 92 | 93 | ```python 94 | def save_db(self, file_name: str = 'data.cdb'): 95 | ''' 96 | Data persistence, save the CyberDB database to the hard disk. 97 | 98 | parameter: 99 | 100 | file_name -- the name of the file generated by the backup, the file 101 | suffix must be .cdb. 102 | 103 | Return Type: None 104 | ''' 105 | ``` 106 | 107 | ```python 108 | def load_db(self, file_name: str = 'data.cdb'): 109 | ''' 110 | Load a file in .cdb format to load the CyberDB database backed up from 111 | the hard disk back into memory. 112 | 113 | parameter: 114 | file_name -- the file name generated by data persistence, the file 115 | suffix must be .cdb. 116 | 117 | Return Type: None 118 | ''' 119 | ``` 120 | 121 | ## cyberdb.connect Function 122 | 123 | ```python 124 | def cyberdb.connect(host: str = '127.0.0.1', port: int = 9980, password: 125 | str = None, encrypt: bool = False, time_out: int = None) -> Client: 126 | ''' 127 | Connect the client to the CyberDB server. 128 | 129 | Parameters: 130 | 131 | host -- the connection address, such as 127.0.0.1 132 | 133 | port -- connection port 134 | 135 | password -- connection password 136 | 137 | encrypt -- Whether to encrypt communication, if the server enables 138 | encrypt to be True, it must be True here, and vice versa. 139 | 140 | time_out -- The timeout for each connection in the connection pool, 141 | in seconds. Connections in the connection pool will be discarded 142 | after time_out seconds of inactivity, and a new connection will be 143 | generated next time. The connection pool will manage the 144 | connections automatically, and the developer does not need to pay 145 | attention to the details. If this parameter is None, there will be 146 | no timeout, and the connection pool will maintain the connection 147 | until it expires, after which a new connection will be regenerated. 148 | 149 | Return Type: Client 150 | ''' 151 | ``` 152 | 153 | ## cyberdb.Client Class 154 | 155 | **class cyberdb.Client** 156 | 157 | The cyberdb.Client object returned by the cyberdb.connect function is used to generate the Proxy object. 158 | 159 | ```python 160 | def get_proxy(self) -> Proxy: 161 | ''' 162 | Generate a Proxy object. 163 | 164 | Return Type: None 165 | ''' 166 | ``` 167 | 168 | ## Proxy Class 169 | 170 | The Proxy object generated by the cyberdb.Client.get_proxy method can operate on the CyberDB database and manage the TCP connections of the CyberDict and CyberList sub-objects generated by the Proxy. After the Proxy object is initialized, it can be used after executing the Proxy.connect method. The Proxy object and its sub-objects will perform remote operations on the server-side CyberDB database. 171 | 172 | **class Proxy** 173 | 174 | ```python 175 | def connect(self): 176 | ''' 177 | Get a TCP connection from the connection pool and bind it to the Proxy 178 | object. 179 | 180 | Return Type: None 181 | ''' 182 | ``` 183 | 184 | ```python 185 | def close(self): 186 | ''' 187 | Cancel the TCP connection bound to the Proxy object and return it to 188 | the connection pool. It needs to be reset before the next operation. 189 | Execute the Proxy.connect method. 190 | 191 | Return Type: None 192 | ''' 193 | ``` 194 | 195 | ```python 196 | def create_cyberdict(self, table_name: str, content: dict = {}): 197 | ''' 198 | Create a CyberDict table. 199 | Parameters: 200 | table_name – table name. 201 | content -- table content, needs to be a dictionary type, the 202 | default is an empty dictionary. 203 | Return Type: None 204 | ''' 205 | ``` 206 | 207 | ```python 208 | def create_cyberlist(self, table_name: str, content: list = []): 209 | ''' 210 | Create the CyberList table. 211 | 212 | Parameters: 213 | 214 | table_name – table name. 215 | 216 | content -- table content, it needs to be a list type, the default 217 | is an empty list. 218 | 219 | Return Type: None 220 | ''' 221 | ``` 222 | 223 | ```python 224 | def get_cyberdict(self, table_name: str) -> CyberDict: 225 | ''' 226 | Get the CyberDict table. 227 | 228 | Parameters: 229 | 230 | table_name – table name. 231 | 232 | Return Type: CyberDict, which is a sub-object generated by Proxy, 233 | which controls the TCP connection. 234 | ''' 235 | ``` 236 | 237 | ```python 238 | def get_cyberlist(self, table_name: str) -> CyberList: 239 | ''' 240 | Get the CyberList table. 241 | 242 | Parameters: 243 | 244 | table_name – table name. 245 | 246 | Return type: CyberList, which is a sub-object generated by Proxy, which 247 | controls the TCP connection. 248 | ''' 249 | ``` 250 | 251 | ```python 252 | def print_tables(self): 253 | ''' 254 | Print all tables in the CyberDB database. 255 | 256 | Return Type: None 257 | ''' 258 | ``` 259 | 260 | ```python 261 | def delete_table(self, table_name: str): 262 | ''' 263 | Drop the table_name table in the CyberDB database. 264 | 265 | Parameters: 266 | 267 | table_name – the name of the table to drop. 268 | 269 | Return Type: None 270 | ''' 271 | ``` 272 | 273 | ## cyberdb.CyberDict Class 274 | 275 | **class cyberdb.CyberDict** 276 | 277 | A child object generated by a Proxy object for performing Dictionaries operations. This object will perform remote operations on the server-side CyberDB database. Shares the same TCP connection with the Proxy object. The connection will follow the connection of the Proxy. After the Proxy releases the connection, the object will also lose the connection. CyberDict can execute the get, setdefault, update, keys, values, items, pop, popitem, clear methods and common magic methods of Dictionaries, please refer to [Python dictionary official documentation](https://docs.python.org/3/library/stdtypes.html#mapping-types-dict). 278 | 279 | Iterate over CyberDict using a for loop with client space complexity o(n), where n is the length of CyberDict.keys() . 280 | 281 | ```python 282 | def todict(self) -> Dict: 283 | ''' 284 | Convert CyberDict to Dictionaries. 285 | 286 | Return Type: Dict 287 | ''' 288 | ``` 289 | 290 | ## cyberdb.CyberList 类 291 | 292 | **class cyberdb.CyberList** 293 | 294 | A child object generated by a Proxy object for performing Lists operations. This object will perform remote operations on the server-side CyberDB database. Shares the same TCP connection with the Proxy object. The connection will follow the connection of the Proxy. After the Proxy releases the connection, the object will also lose the connection. CyberList can execute the append, extend, insert, pop, remove, count, index, reverse, sort, clear methods and common magic methods of Lists, please refer to [Python List Official Documentation](https://docs.python.org/3/tutorial/datastructures.html#more-on-lists). 295 | 296 | The CyberList is iterated using a for loop, each iteration will fetch the content from the server, and the space complexity of the client is o(1). 297 | 298 | ```python 299 | def tolist(self) -> List: 300 | ''' 301 | Convert CyberList to Lists. 302 | 303 | Return Type: List 304 | ''' 305 | ``` 306 | 307 | -------------------------------------------------------------------------------- /cyberdb/network/server.py: -------------------------------------------------------------------------------- 1 | import re 2 | import shutil 3 | import pickle 4 | import datetime 5 | import asyncio 6 | import threading 7 | 8 | from obj_encrypt import Secret 9 | from apscheduler.schedulers.background import BackgroundScheduler 10 | 11 | from . import AioStream 12 | from .route import Route 13 | from ..data import datas 14 | from ..extensions import DisconCyberDBError, WrongFilenameCyberDBError, \ 15 | WrongPasswordCyberDBError, BackupCyberDBError, TypeCyberDBError 16 | from ..extensions.signature import Signature 17 | 18 | 19 | class Server: 20 | ''' 21 | This class is used to create CyberDB server objects. 22 | ''' 23 | 24 | def __init__(self): 25 | self._data = { 26 | 'config': { 27 | 'host': None, 28 | 'port': None, 29 | 'password': None 30 | }, 31 | 'db': {} 32 | } 33 | self.ips = {'127.0.0.1'} # ip whitelist 34 | self.sched = None 35 | 36 | def start(self, host: str = '127.0.0.1', port: int = 9980, 37 | password: str = None, max_con: int = 500, timeout: int = 0, 38 | print_log: bool = False, encrypt: bool = False): 39 | ''' 40 | Starts the CyberDB server in the background, which does not block 41 | foreground tasks. 42 | 43 | Parameters: 44 | 45 | host -- TCP listening host address, such as 127.0.0.1. 46 | 47 | port – TCP listening port. 48 | 49 | password -- TCP communication password, it is recommended to use 50 | a combination of English letters and numbers, 51 | Up to 32 characters long. 52 | 53 | max_con -- the maximum number of concurrency. 54 | 55 | timeout -- The timeout for a single connection, in seconds. 56 | 57 | print_log -- whether to print the communication log, Fasle does not 58 | print it. 59 | 60 | encrypt -- Whether to encrypt the communication content, Fasle is 61 | not encrypted. The encryption algorithm is AES-256 and the key is 62 | password. 63 | 64 | Return Type: None 65 | ''' 66 | t = threading.Thread(target=self.run, 67 | args=(host, port, password, max_con, timeout, print_log, encrypt)) 68 | t.daemon = True 69 | t.start() 70 | 71 | def run(self, host: str = '127.0.0.1', port: int = 9980, 72 | password: str = None, max_con: int = 500, timeout: int = 0, 73 | print_log: bool = False, encrypt: bool = False): 74 | ''' 75 | Running the CyberDB server in the foreground blocks foreground tasks. 76 | The parameters are the same as the start method. 77 | 78 | Return Type: None 79 | ''' 80 | if not password: 81 | raise RuntimeError('The password cannot be empty.') 82 | 83 | self._data['config']['host'] = host 84 | self._data['config']['port'] = port 85 | self._data['config']['password'] = password 86 | self._data['config']['max_con'] = max_con 87 | self._data['config']['timeout'] = timeout 88 | self._data['config']['print_log'] = print_log 89 | self._data['config']['encrypt'] = encrypt 90 | 91 | # Responsible for encrypting and decrypting objects. 92 | secret = Secret(key=password) 93 | # Convert TCP data and encrypted objects to each other. 94 | self._dp = datas.DataParsing(secret, encrypt=encrypt) 95 | 96 | asyncio.run(self._main()) 97 | 98 | async def _main(self): 99 | server = await asyncio.start_server( 100 | self._listener, self._data['config']['host'], 101 | self._data['config']['port'], 102 | limit=2 ** 16, # 64 KiB 103 | # the maximum number of queued connections 104 | backlog=self._data['config']['max_con'] 105 | ) 106 | 107 | print('CyberDB server is starting, please wait a few seconds before operation!\n* Started at {}:{}'.format( 108 | self._data['config']['host'], 109 | self._data['config']['port'] 110 | )) 111 | 112 | async with server: 113 | await server.serve_forever() 114 | 115 | async def _listener(self, reader: asyncio.streams.StreamReader, 116 | writer: asyncio.streams.StreamWriter): 117 | ''' 118 | This method is entered as long as a TCP connection is established, 119 | even if no data is sent. 120 | ''' 121 | 122 | try: 123 | addr = writer.get_extra_info('peername') 124 | if self._data['config']['print_log']: 125 | print('{} {}:{} establishes a connection.'.format( 126 | datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 127 | addr[0], addr[1])) 128 | 129 | if self._data['config']['encrypt']: 130 | # Check if the ip is in the whitelist. 131 | if self.ips: 132 | if addr[0] not in self.ips: 133 | if self._data['config']['print_log']: 134 | print('{} The request for {}, the ip is not in the whitelist.'.format( 135 | datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 136 | addr[0])) 137 | writer.close() 138 | return 139 | 140 | # TCP route of this connection 141 | stream = AioStream(reader, writer, self._dp) 142 | route = Route(self._data['db'], self._dp, stream, 143 | print_log=self._data['config']['print_log']) 144 | 145 | # If the timeout is set, it will automatically disconnect. 146 | if self._data['config']['timeout'] == 0: 147 | await route.find() 148 | else: 149 | try: 150 | await asyncio.wait_for(route.find(), 151 | timeout=self._data['config']['timeout']) 152 | except asyncio.TimeoutError: 153 | if self._data['config']['print_log']: 154 | print('{} {}:{} connection timed out.'.format( 155 | datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 156 | addr[0], addr[1])) 157 | writer.close() 158 | 159 | except DisconCyberDBError: 160 | if self._data['config']['print_log']: 161 | print('{} {}:{} Client disconnected.'.format( 162 | datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 163 | addr[0], addr[1]) 164 | ) 165 | except WrongPasswordCyberDBError: 166 | if self._data['config']['print_log']: 167 | print('{} {}:{} The password entered by the client is incorrect.'.format( 168 | datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 169 | addr[0], addr[1])) 170 | 171 | def set_ip_whitelist(self, ips: list): 172 | ''' 173 | Set the ip whitelist. When CyberDB encrypts communication, only 174 | whitelisted ip connections are allowed. 175 | 176 | This method only works if cyberdb.Server.start(encrypt=True) or 177 | cyberdb.Server.run(encrypt=True) are enabled. 178 | 179 | Parameters: 180 | 181 | ips -- the type is a list, the format is ['192.168.1.1', 182 | '118.123.89.137'] 183 | 184 | Return Type: None 185 | ''' 186 | for ip in ips: 187 | if not re.match(r'^((2((5[0-5])|([0-4]\d)))|([0-1]?\d{1,2}))(\.((2((5[0-5])|([0-4]\d)))|([0-1]?\d{1,2}))){3}$', ip): 188 | raise RuntimeError('Please enter a valid ipv4 address.') 189 | ips = set(ips) 190 | ips.add('127.0.0.1') 191 | self.ips = ips 192 | 193 | def set_backup(self, file_name: str = 'data.cdb', cycle: int = 900): 194 | ''' 195 | Set timed backup. After this operation is set, data persistent backup 196 | will be performed at the specified period, and the CyberDB database 197 | will be saved to the hard disk. 198 | 199 | parameter: 200 | 201 | file_name -- the name of the file generated by the backup, the file 202 | suffix must be .cdb. 203 | 204 | cycle -- The cycle of the cyclic backup, in seconds. 205 | 206 | Return Type: None 207 | ''' 208 | prefix_name, suffix_name = file_name.rsplit('.', 1) 209 | 210 | if suffix_name != 'cdb': 211 | raise WrongFilenameCyberDBError( 212 | 'Please enter a filename with a .cdb suffix.') 213 | 214 | if type(cycle) != int: 215 | raise TypeCyberDBError('The type of cycle is not an integer.') 216 | 217 | # If the scheduled task exists, reset it. 218 | if self.sched: 219 | self.sched.shutdown(wait=False) 220 | 221 | # The time here is for looping only and does not affect usage anywhere in the world. 222 | self.sched = BackgroundScheduler(timezone='Asia/Shanghai') 223 | self.sched.add_job(self.save_db, 'interval', 224 | seconds=cycle, args=[file_name]) 225 | self.sched.start() 226 | 227 | def save_db(self, file_name: str = 'data.cdb'): 228 | ''' 229 | Data persistence, save the CyberDB database to the hard disk. 230 | 231 | parameter: 232 | 233 | file_name -- the name of the file generated by the backup, the file 234 | suffix must be .cdb. 235 | 236 | Return Type: None 237 | ''' 238 | prefix_name, suffix_name = file_name.rsplit('.', 1) 239 | 240 | if suffix_name != 'cdb': 241 | raise WrongFilenameCyberDBError( 242 | 'Please enter a filename with a .cdb suffix.') 243 | 244 | # save to hard drive 245 | file_name_temp = prefix_name + '_temp.cdb' 246 | with open(file_name_temp, 'wb') as f: 247 | pickle.dump(self._data, f) 248 | shutil.move(file_name_temp, file_name) 249 | 250 | def load_db(self, file_name: str = 'data.cdb'): 251 | ''' 252 | Load a file in .cdb format to load the CyberDB database backed up from 253 | the hard disk back into memory. 254 | 255 | parameter: 256 | file_name -- the file name generated by data persistence, the file 257 | suffix must be .cdb. 258 | 259 | Return Type: None 260 | ''' 261 | prefix_name, suffix_name = file_name.rsplit('.', 1) 262 | 263 | if suffix_name != 'cdb': 264 | raise WrongFilenameCyberDBError('The file suffix must be cdb.') 265 | 266 | with open(file_name, 'rb') as f: 267 | self._data = pickle.load(f) 268 | 269 | # print('File {} loaded successfully.'.format(file_name)) 270 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CyberDB 2 | 3 | [中文版](https://github.com/Cyberbolt/CyberDB#中文版) 4 | 5 | CyberDB is a lightweight Python in-memory database. It is designed to use Python's built-in data structures Dictionaries, Lists for data storage, efficient communication through Socket TCP, and provide data persistence. This module can be used in hard disk database caching, Gunicorn inter-process communication, distributed computing and other fields. 6 | 7 | The CyberDB server uses Asyncio for TCP communication. The client is developed based on Socket, so it supports the Gevent coroutine, but it has not yet adapted to Asyncio. Both the server and the client support PyPy, and it is recommended to use PyPy to run for better performance. 8 | 9 | In high concurrency scenarios, the performance bottleneck of traditional databases is mainly hard disk I/O. Even if CyberDB is developed based on the dynamic language Python, the speed is still much faster than that of hard disk databases (such as MySQL), and CyberDB can be used as its cache. In addition, the core of CyberDB lies in programming in a Pythonic way, you can use CyberDB like Dictionaries and Lists. 10 | 11 | ## Installation 12 | 13 | 1. Enter the command window, create a virtual environment, and enter the following commands in turn 14 | 15 | Linux and macOS: 16 | 17 | 18 | 19 | ```python 20 | python3 -m venv venv # Create a virtual environment. 21 | . venv/bin/activate # Activate the virtual environment. 22 | ``` 23 | 24 | Windows: 25 | 26 | 27 | ```python 28 | python -m venv venv # Create a virtual environment. 29 | venv\Scripts\activate # Activate the virtual environment. 30 | ``` 31 | 32 | 2.Install CyberDB, enter 33 | 34 | 35 | ```python 36 | pip install --upgrade pip 37 | pip install cyberdb 38 | ``` 39 | 40 | If your server and client are running in two different project directories, please install CyberDB in the virtual environment of the server and client respectively. 41 | 42 | ## Links 43 | 44 | - GitHub: [https://github.com/Cyberbolt/CyberDB](https://github.com/Cyberbolt/CyberDB) 45 | - PyPI: [https://pypi.org/project/CyberDB/](https://pypi.org/project/CyberDB/) 46 | - Documentation: [https://www.cyberlight.xyz/static/cyberdb](https://www.cyberlight.xyz/static/cyberdb) 47 | - CyberLight: [https://www.cyberlight.xyz/](https://www.cyberlight.xyz/) 48 | 49 | ## Quick to Use 50 | 51 | In this module, please use CyberDict and CyberList instead of dict and list (a TCP-based Dictionaries-like, Lists-like object). 52 | 53 | ### Server 54 | 55 | Run the database server. 56 | 57 | ```python 58 | import time 59 | import cyberdb 60 | 61 | db = cyberdb.Server() 62 | # The data is persistent, the backup file is data.cdb, and the backup cycle is every 900 seconds. 63 | db.set_backup('data.cdb', cycle=900) 64 | # Set the TCP address, port number, and password. 65 | # The start method will not block the operation. If you want the operation to block, please use the run method instead of start, and the parameters remain unchanged. 66 | db.start(host='127.0.0.1', port=9980, password='123456') 67 | 68 | while True: 69 | time.sleep(10000) 70 | ``` 71 | 72 | After the above server runs, data.cdb and data_backup.cdb (backup files) will be generated (or overwritten) in the project root directory every 900 seconds. The file can be read the next time the database is started using the load method. 73 | 74 | ### Client 75 | 76 | #### Connect to the Database 77 | 78 | 79 | ```python 80 | import cyberdb 81 | 82 | # Generate a client instance and connect. 83 | client = cyberdb.connect(host='127.0.0.1', port=9980, password='123456') 84 | ``` 85 | 86 | #### Generate proxy Object 87 | 88 | 89 | ```python 90 | # Generate proxy for this request. 91 | proxy = client.get_proxy() 92 | # Automatically obtain database connections from the connection pool. 93 | proxy.connect() 94 | ``` 95 | 96 | The proxy object is not thread-safe, please generate the proxy object separately in each thread (or coroutine), and obtain the database connection through the connect method. You only need to use the close method to return the connection after the operation is completed, and the returned connection is managed intelligently by the client object. 97 | 98 | #### Manipulate proxy Objects 99 | 100 | Create CyberDict and CyberList 101 | 102 | 103 | ```python 104 | # Create dict1 and dict2 tables of type CyberDict and 105 | # list1 table of type CyberList in the database respectively. 106 | proxy.create_cyberdict('dict1') 107 | proxy.create_cyberdict('dict2') 108 | proxy.create_cyberlist('list1') 109 | ``` 110 | 111 | 112 | ```python 113 | dict1 = proxy.get_cyberdict('dict1') 114 | dict2 = proxy.get_cyberdict('dict2') 115 | list1 = proxy.get_cyberlist('list1') 116 | ``` 117 | 118 | The dict1, dict2, and list1 obtained here are all network objects, and the data is transmitted through TCP. The three objects are controlled by the proxy, and when proxy.close() is called to return the connection, the three objects will also be invalid. Similarly, use proxy.connect() to get connections from the connection pool again, and dict1, dict2, list1 also become available. 119 | 120 | Once you understand this operation, you can operate dict1, dict2 like Dictionaries, and list1 like Lists! (CyberDict and CyberList support most methods of Dictionaries, Lists) 121 | 122 | Examples are as follows 123 | 124 | ##### Common Operations of CyberDict 125 | 126 | Add key-value pairs in dict1 and dict2 127 | 128 | 129 | ```python 130 | dict1[0] = 100 131 | dict1['test'] = 'Hello CyberDB!' 132 | dict2[0] = 200 133 | ``` 134 | 135 | Get the corresponding value 136 | 137 | 138 | ```python 139 | dict1.get(0) 140 | ``` 141 | 142 | 143 | 144 | 145 | 100 146 | 147 | 148 | 149 | 150 | ```python 151 | dict2[0] 152 | ``` 153 | 154 | 155 | 156 | 157 | 200 158 | 159 | 160 | 161 | View dict1 and dict2 (can also be printed with print ) 162 | 163 | 164 | ```python 165 | dict1 166 | ``` 167 | 168 | 169 | 170 | 171 | {0: 100, 'test': 'Hello CyberDB!'} 172 | 173 | 174 | 175 | 176 | ```python 177 | dict2 178 | ``` 179 | 180 | 181 | 182 | 183 | {0: 200} 184 | 185 | 186 | 187 | Get length 188 | 189 | 190 | ```python 191 | len(dict1) 192 | ``` 193 | 194 | 195 | 196 | 197 | 2 198 | 199 | 200 | 201 | Delete key-value pair 202 | 203 | 204 | ```python 205 | del dict1[0] 206 | ``` 207 | 208 | 209 | ```python 210 | dict1 211 | ``` 212 | 213 | 214 | 215 | 216 | {'test': 'Hello CyberDB!'} 217 | 218 | 219 | 220 | Empty dict1 221 | 222 | 223 | ```python 224 | dict1.clear() 225 | dict1 226 | ``` 227 | 228 | 229 | 230 | 231 | {} 232 | 233 | 234 | 235 | ##### Common Operations of CyberList 236 | 237 | Generate the contents of list1 238 | 239 | 240 | ```python 241 | for i in range(5): 242 | list1.append(99) 243 | 244 | list1 245 | ``` 246 | 247 | 248 | 249 | 250 | [99, 99, 99, 99, 99] 251 | 252 | 253 | 254 | Change coordinate values 255 | 256 | 257 | ```python 258 | list1[3] = 100 259 | list1 260 | ``` 261 | 262 | 263 | 264 | 265 | [99, 99, 99, 100, 99] 266 | 267 | 268 | 269 | To slice 270 | 271 | 272 | ```python 273 | list1[3:] 274 | ``` 275 | 276 | 277 | 278 | 279 | [100, 99] 280 | 281 | 282 | 283 | Get the length of list1 284 | 285 | 286 | ```python 287 | len(list1) 288 | ``` 289 | 290 | 291 | 292 | 293 | 5 294 | 295 | 296 | 297 | Print each element of list1 by iterating 298 | 299 | 300 | ```python 301 | for v in list1: 302 | print(v) 303 | ``` 304 | 305 | 99 306 | 99 307 | 99 308 | 100 309 | 99 310 | 311 | 312 | It is strongly recommended to use a for loop to iterate CyberList, each iteration will get v from the server, and the space complexity of the client is o(1). Iteration can also be used for CyberDict. In the iteration of CyberDict, the client space complexity is o(n), and n is the size of CyberDict.keys(). 313 | 314 | #### Release the Proxy Object 315 | 316 | After the use is complete, return the connection of the proxy to the connection pool. 317 | 318 | 319 | ```python 320 | proxy.close() 321 | ``` 322 | 323 | The proxy object also supports context managers, such as 324 | 325 | 326 | ```python 327 | with client.get_proxy() as proxy: 328 | list1 = proxy.get_cyberlist('list1') 329 | print(list1) 330 | ``` 331 | 332 | [99, 99, 99, 100, 99] 333 | 334 | 335 | ## Generalize 336 | 337 | With CyberDB, memory performance can be fully utilized, and different processes (or even different hosts) can communicate through Python's data structures. For more tutorials, please refer to the documentation, Thank you! 338 | 339 | ## Notice 340 | 341 | Due to encoding limitations, CyberDB will recognize 0 as None, but it does not affect the calculation, please convert None to 0 in the desired position. 342 | 343 | 344 | # 中文版 345 | 346 | CyberDB 是一个轻量级的 Python 内存数据库。它旨在利用 Python 内置数据结构 Dictionaries、Lists 作数据存储，通过 Socket TCP 高效通信，并提供数据持久化。该模块可用于 硬盘数据库缓存、Gunicorn 进程间通信、分布式计算 等领域。 347 | 348 | CyberDB 服务端使用 Asyncio 进行 TCP 通信。客户端基于 Socket 开发，所以支持 Gevent 协程，暂未适配 Asyncio。服务端和客户端均支持 PyPy，推荐使用 PyPy 运行，以获得更好的性能。 349 | 350 | 高并发场景下，传统数据库的性能瓶颈主要在硬盘 I/O，即便 CyberDB 基于动态语言 Python 开发，速度仍然远快于硬盘数据库(如 MySQL)，CyberDB 可以作为它的缓存。此外，CyberDB 的核心在于使用 Pythonic 的方式编程，你可以像使用 Dictionaries 和 Lists 一样使用 CyberDB。 351 | 352 | ## 安装方法 353 | 354 | 1.进入命令窗口，创建虚拟环境，依次输入以下命令 355 | 356 | Linux 和 macOS: 357 | 358 | 359 | 360 | ```python 361 | python3 -m venv venv # 创建虚拟环境 362 | . venv/bin/activate # 激活虚拟环境 363 | ``` 364 | 365 | Windows: 366 | 367 | 368 | ```python 369 | python -m venv venv # 创建虚拟环境 370 | venv\Scripts\activate # 激活虚拟环境 371 | ``` 372 | 373 | 2.安装 CyberDB，依次输入 374 | 375 | 376 | ```python 377 | pip install --upgrade pip 378 | pip install cyberdb 379 | ``` 380 | 381 | 如果你的服务端和客户端在两个不同的项目目录运行，请分别在服务端、客户端的虚拟环境中安装 CyberDB。 382 | 383 | ## 链接 384 | 385 | - GitHub: [https://github.com/Cyberbolt/CyberDB](https://github.com/Cyberbolt/CyberDB) 386 | - PyPI: [https://pypi.org/project/CyberDB/](https://pypi.org/project/CyberDB/) 387 | - 文档: [https://www.cyberlight.xyz/static/cyberdb-chn](https://www.cyberlight.xyz/static/cyberdb-chn) 388 | - 电光笔记: [https://www.cyberlight.xyz/](https://www.cyberlight.xyz/) 389 | 390 | ## 快速使用 391 | 392 | 该模块中请使用 CyberDict 和 CyberList 替代 dict 和 list （一种基于 TCP 的类 Dictionaries、类 Lists 对象）。 393 | 394 | ### 服务端 395 | 396 | 397 | 服务端初始化，设置备份和 TCP 监听地址。 398 | 399 | 400 | ```python 401 | import time 402 | import cyberdb 403 | 404 | db = cyberdb.Server() 405 | # 数据持久化，备份文件为 data.cdb，备份周期 900 秒一次。 406 | db.set_backup('data.cdb', cycle=900) 407 | # 设置 TCP 地址、端口号、密码，生产环境中密码建议使用大小写字母和数字的组合。 408 | # start 方法不会阻塞运行，若希望该操作阻塞，请使用 run 方法代替 start，参数不变。 409 | db.start(host='127.0.0.1', port=9980, password='123456') 410 | 411 | while True: 412 | time.sleep(10000) 413 | ``` 414 | 415 | 上述服务端运行后，每 900 秒将在此项目根目录生成（或覆盖）data.cdb。下次启动数据库可使用 load 方法读取该文件。 416 | 417 | ### 客户端 418 | 419 | #### 连接数据库 420 | 421 | 422 | ```python 423 | import cyberdb 424 | 425 | # 生成客户端实例并连接。 426 | client = cyberdb.connect(host='127.0.0.1', port=9980, password='123456') 427 | ``` 428 | 429 | #### 生成 proxy 对象 430 | 431 | 432 | ```python 433 | # 生成本次请求的 proxy。 434 | proxy = client.get_proxy() 435 | # 从连接池自动获取数据库连接。 436 | proxy.connect() 437 | ``` 438 | 439 | 建议在每个线程(或协程)中单独生成 proxy 对象，并通过 connect 方法获取数据库连接。你只需在操作完成后使用 close 方法归还连接即可，归还后的连接由 client 对象智能管理。 440 | 441 | #### 操作 proxy 对象 442 | 443 | 创建 CyberDict 和 CyberList 444 | 445 | 446 | ```python 447 | # 在数据库中分别创建类型为 CyberDict 的 dict1、dict2 表， 448 | # 类型为 CyberList 的 list1 表。 449 | proxy.create_cyberdict('dict1') 450 | proxy.create_cyberdict('dict2') 451 | proxy.create_cyberlist('list1') 452 | ``` 453 | 454 | 455 | ```python 456 | dict1 = proxy.get_cyberdict('dict1') 457 | dict2 = proxy.get_cyberdict('dict2') 458 | list1 = proxy.get_cyberlist('list1') 459 | ``` 460 | 461 | 此处获取的 dict1、dict2、list1 均为网络对象，数据通过 TCP 传输。三个对象受 proxy 控制，当调用 proxy.close() 归还连接后，三个对象也会失效。同样，使用 proxy.connect() 可重新从连接池获取连接，dict1、dict2、list1 也变为可用。 462 | 463 | 了解此操作后，你便可以像操作 Dictionaries 一样操作 dict1、dict2，像操作 Lists 一样操作 list1 了！（CyberDict 和 CyberList 支持 Dictionaries、Lists 的大部分方法） 464 | 465 | 示例如下 466 | 467 | ##### CyberDict 常用操作 468 | 469 | 在 dict1 和 dict2 中新增键值对 470 | 471 | 472 | ```python 473 | dict1[0] = 100 474 | dict1['test'] = 'Hello CyberDB!' 475 | dict2[0] = 200 476 | ``` 477 | 478 | 获取对应的值 479 | 480 | 481 | ```python 482 | dict1.get(0) 483 | ``` 484 | 485 | 486 | 487 | 488 | 100 489 | 490 | 491 | 492 | 493 | ```python 494 | dict2[0] 495 | ``` 496 | 497 | 498 | 499 | 500 | 200 501 | 502 | 503 | 504 | 查看 dict1 和 dict2 (也可以使用 print 打印) 505 | 506 | 507 | ```python 508 | dict1 509 | ``` 510 | 511 | 512 | 513 | 514 | {0: 100, 'test': 'Hello CyberDB!'} 515 | 516 | 517 | 518 | 519 | ```python 520 | dict2 521 | ``` 522 | 523 | 524 | 525 | 526 | {0: 200} 527 | 528 | 529 | 530 | 获取长度 531 | 532 | 533 | ```python 534 | len(dict1) 535 | ``` 536 | 537 | 538 | 539 | 540 | 2 541 | 542 | 543 | 544 | 删除键值对 545 | 546 | 547 | ```python 548 | del dict1[0] 549 | dict1 550 | ``` 551 | 552 | 553 | 554 | 555 | {'test': 'Hello CyberDB!'} 556 | 557 | 558 | 559 | 清空 dict1 560 | 561 | 562 | ```python 563 | dict1.clear() 564 | dict1 565 | ``` 566 | 567 | 568 | 569 | 570 | {} 571 | 572 | 573 | 574 | ##### CyberList 常用操作 575 | 576 | 生成 list1 的内容 577 | 578 | 579 | ```python 580 | for i in range(5): 581 | list1.append(99) 582 | 583 | list1 584 | ``` 585 | 586 | 587 | 588 | 589 | [99, 99, 99, 99, 99] 590 | 591 | 592 | 593 | 更改坐标值 594 | 595 | 596 | ```python 597 | list1[3] = 100 598 | list1 599 | ``` 600 | 601 | 602 | 603 | 604 | [99, 99, 99, 100, 99] 605 | 606 | 607 | 608 | 进行切片 609 | 610 | 611 | ```python 612 | list1[3:] 613 | ``` 614 | 615 | 616 | 617 | 618 | [100, 99] 619 | 620 | 621 | 622 | 获取 list1 的长度 623 | 624 | 625 | ```python 626 | len(list1) 627 | ``` 628 | 629 | 630 | 631 | 632 | 5 633 | 634 | 635 | 636 | 通过迭代打印 list1 每个元素 637 | 638 | 639 | ```python 640 | for v in list1: 641 | print(v) 642 | ``` 643 | 644 | 99 645 | 99 646 | 99 647 | 100 648 | 99 649 | 650 | 651 | 强烈推荐使用 for 循环迭代 CyberList，每次迭代将从服务端获取 v，客户端的空间复杂度为 o(1)。迭代同样可用于 CyberDict，CyberDict 的迭代中，客户端空间复杂度为 o(n), n 为 CyberDict.keys() 的长度。 652 | 653 | #### 释放 proxy 对象 654 | 655 | 使用完成，将 proxy 的连接归还至连接池即可。 656 | 657 | 658 | ```python 659 | proxy.close() 660 | ``` 661 | 662 | proxy 对象同样支持上下文管理器，如 663 | 664 | 665 | ```python 666 | with client.get_proxy() as proxy: 667 | list1 = proxy.get_cyberlist('list1') 668 | print(list1) 669 | ``` 670 | 671 | [99, 99, 99, 100, 99] 672 | 673 | 674 | ## 概括 675 | 676 | 有了 CyberDB，便能充分利用内存性能，不同进程(甚至不同主机)能通过 Python 的数据结构通信。更多教程请参考文档，感谢你的支持！ 677 | 678 | ## 注意 679 | 680 | 由于编码限制，CyberDB 会将 0 识别为 None，但并不影响计算，请在所需位置将 None 转为 0。 681 | -------------------------------------------------------------------------------- /文档.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "lightweight-session", 6 | "metadata": {}, 7 | "source": [ 8 | "# CyberDB\n", 9 | "\n", 10 | "CyberDB 是一个轻量级的 Python 内存数据库。它旨在利用 Python 内置数据结构 Dictionaries、Lists 作数据存储，通过 Socket TCP 高效通信，并提供数据持久化。该模块常用于 Gunicorn 进程间通信、分布式计算 等领域。\n", 11 | "\n", 12 | "## 安装方法\n", 13 | "\n", 14 | "1.进入命令窗口，创建虚拟环境，依次输入以下命令\n", 15 | "\n", 16 | "Linux 和 macOS:\n" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": null, 22 | "id": "understood-rider", 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "python3 -m venv venv # 创建虚拟环境\n", 27 | ". venv/bin/activate # 激活虚拟环境" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "id": "endless-coating", 33 | "metadata": {}, 34 | "source": [ 35 | "Windows:" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": null, 41 | "id": "greenhouse-encoding", 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "python -m venv venv # 创建虚拟环境\n", 46 | "venv\\Scripts\\activate # 激活虚拟环境" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "id": "wireless-stephen", 52 | "metadata": {}, 53 | "source": [ 54 | "2.安装 CyberDB，依次输入" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": null, 60 | "id": "acquired-burden", 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "pip install --upgrade pip\n", 65 | "pip install cyberdb" 66 | ] 67 | }, 68 | { 69 | "cell_type": "markdown", 70 | "id": "indirect-exhibit", 71 | "metadata": {}, 72 | "source": [ 73 | "如果你的服务端和客户端在两个不同的项目目录运行，请分别在服务端、客户端的虚拟环境中安装 CyberDB。\n", 74 | "\n", 75 | "## 链接\n", 76 | "\n", 77 | "- GitHub: [https://github.com/Cyberbolt/CyberDB](https://github.com/Cyberbolt/CyberDB) \n", 78 | "- PyPI: [https://pypi.org/project/CyberDB/](https://pypi.org/project/CyberDB/)\n", 79 | "- 文档: 正在撰写中\n", 80 | "- 电光笔记 [https://www.cyberlight.xyz/](https://www.cyberlight.xyz/)\n", 81 | "\n", 82 | "## 快速使用\n", 83 | "\n", 84 | "该模块中请使用 CyberDict 和 CyberList 替代 dict 和 list （一种基于 TCP 的类 Dictionaries、类 Lists 对象）。\n", 85 | "\n", 86 | "### 服务端\n" 87 | ] 88 | }, 89 | { 90 | "cell_type": "markdown", 91 | "id": "bibliographic-profit", 92 | "metadata": {}, 93 | "source": [ 94 | "首先单独开一个 py 文件，用于服务端初始化时创建数据库表，运行服务端后保存所建表到本地。（该文件只在建表时使用，后续将不会运行）" 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": 1, 100 | "id": "verified-therapist", 101 | "metadata": {}, 102 | "outputs": [], 103 | "source": [ 104 | "import cyberdb\n", 105 | "\n", 106 | "db = cyberdb.Server()\n", 107 | "# 数据持久化，备份文件为 data.cdb，备份周期 900 秒一次。\n", 108 | "db.set_backup('data.cdb', cycle=900)\n", 109 | "# 设置 TCP 地址、端口号、密码，生产环境中密码建议使用大小写字母和数字的组合。\n", 110 | "# start 方法不会阻塞运行，若希望该操作阻塞，请使用 run 方法代替 start，参数不变。\n", 111 | "db.start(host='127.0.0.1', port=9980, password='123456')" 112 | ] 113 | }, 114 | { 115 | "cell_type": "markdown", 116 | "id": "comparative-rebecca", 117 | "metadata": {}, 118 | "source": [ 119 | "上述服务端运行后，每 900 秒将在此项目根目录生成（或覆盖） data.cdb 和 data_backup.cdb (备用文件)。下次启动数据库可使用 load 方法读取该文件。" 120 | ] 121 | }, 122 | { 123 | "cell_type": "markdown", 124 | "id": "considered-diana", 125 | "metadata": {}, 126 | "source": [ 127 | "### 客户端\n", 128 | "\n", 129 | "#### 连接数据库" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": 2, 135 | "id": "anonymous-blanket", 136 | "metadata": {}, 137 | "outputs": [], 138 | "source": [ 139 | "import cyberdb\n", 140 | "\n", 141 | "# 生成客户端实例并连接。\n", 142 | "client = cyberdb.connect(host='127.0.0.1', port=9980, password='123456')" 143 | ] 144 | }, 145 | { 146 | "cell_type": "markdown", 147 | "id": "fewer-stake", 148 | "metadata": {}, 149 | "source": [ 150 | "#### 生成 proxy 对象" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": 3, 156 | "id": "annoying-rehabilitation", 157 | "metadata": {}, 158 | "outputs": [], 159 | "source": [ 160 | "# 生成本次请求的 proxy。\n", 161 | "proxy = client.get_proxy()\n", 162 | "# 从连接池自动获取数据库连接。\n", 163 | "proxy.connect()" 164 | ] 165 | }, 166 | { 167 | "cell_type": "markdown", 168 | "id": "behavioral-profession", 169 | "metadata": {}, 170 | "source": [ 171 | "proxy 对象是线程不安全的，请在每个线程(或协程)中单独生成 proxy 对象，并通过 connect 方法获取数据库连接。你只需在操作完成后使用 close 方法归还连接即可，归还后的连接由 client 对象智能管理。\n", 172 | "\n", 173 | "#### 操作 proxy 对象" 174 | ] 175 | }, 176 | { 177 | "cell_type": "markdown", 178 | "id": "weekly-contributor", 179 | "metadata": {}, 180 | "source": [ 181 | "创建 CyberDict 和 CyberList" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": 4, 187 | "id": "pressing-kingston", 188 | "metadata": {}, 189 | "outputs": [], 190 | "source": [ 191 | "# 在数据库中分别创建类型为 CyberDict 的 dict1、dict2 表，\n", 192 | "# 类型为 CyberList 的 list1 表。\n", 193 | "proxy.create_cyberdict('dict1')\n", 194 | "proxy.create_cyberdict('dict2')\n", 195 | "proxy.create_cyberlist('list1')" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": 5, 201 | "id": "quality-klein", 202 | "metadata": {}, 203 | "outputs": [], 204 | "source": [ 205 | "dict1 = proxy.get_cyberdict('dict1')\n", 206 | "dict2 = proxy.get_cyberdict('dict2')\n", 207 | "list1 = proxy.get_cyberlist('list1')" 208 | ] 209 | }, 210 | { 211 | "cell_type": "markdown", 212 | "id": "professional-minimum", 213 | "metadata": {}, 214 | "source": [ 215 | "此处获取的 dict1、dict2、list1 均为网络对象，数据通过 TCP 传输。三个对象受 proxy 控制，当调用 proxy.close() 归还连接后，三个对象也会失效。同样，使用 proxy.connect() 可重新从连接池获取连接，dict1、dict2、list1 也变为可用。\n", 216 | "\n", 217 | "了解此操作后，你便可以像操作 Dictionaries 一样操作 dict1、dict2，像操作 Lists 一样操作 list1 了！（CyberDict 和 CyberList 支持 Dictionaries、Lists 的大部分方法）\n", 218 | "\n", 219 | "示例如下" 220 | ] 221 | }, 222 | { 223 | "cell_type": "markdown", 224 | "id": "behavioral-tribe", 225 | "metadata": {}, 226 | "source": [ 227 | "##### CyberDict 常用操作" 228 | ] 229 | }, 230 | { 231 | "cell_type": "markdown", 232 | "id": "starting-waste", 233 | "metadata": {}, 234 | "source": [ 235 | "在 dict1 和 dict2 中新增键值对" 236 | ] 237 | }, 238 | { 239 | "cell_type": "code", 240 | "execution_count": 6, 241 | "id": "alpine-california", 242 | "metadata": {}, 243 | "outputs": [], 244 | "source": [ 245 | "dict1[0] = 100\n", 246 | "dict1['test'] = 'Hello CyberDB!'\n", 247 | "dict2[0] = 200" 248 | ] 249 | }, 250 | { 251 | "cell_type": "markdown", 252 | "id": "hawaiian-visibility", 253 | "metadata": {}, 254 | "source": [ 255 | "获取对应的值" 256 | ] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "execution_count": 7, 261 | "id": "partial-procurement", 262 | "metadata": {}, 263 | "outputs": [ 264 | { 265 | "data": { 266 | "text/plain": [ 267 | "100" 268 | ] 269 | }, 270 | "execution_count": 7, 271 | "metadata": {}, 272 | "output_type": "execute_result" 273 | } 274 | ], 275 | "source": [ 276 | "dict1.get(0)" 277 | ] 278 | }, 279 | { 280 | "cell_type": "code", 281 | "execution_count": 8, 282 | "id": "synthetic-warning", 283 | "metadata": {}, 284 | "outputs": [ 285 | { 286 | "data": { 287 | "text/plain": [ 288 | "200" 289 | ] 290 | }, 291 | "execution_count": 8, 292 | "metadata": {}, 293 | "output_type": "execute_result" 294 | } 295 | ], 296 | "source": [ 297 | "dict2[0]" 298 | ] 299 | }, 300 | { 301 | "cell_type": "markdown", 302 | "id": "entertaining-sport", 303 | "metadata": {}, 304 | "source": [ 305 | "查看 dict1 和 dict2 (也可以使用 print 打印)" 306 | ] 307 | }, 308 | { 309 | "cell_type": "code", 310 | "execution_count": 9, 311 | "id": "stuffed-coach", 312 | "metadata": {}, 313 | "outputs": [ 314 | { 315 | "data": { 316 | "text/plain": [ 317 | "{0: 100, 'test': 'Hello CyberDB!'}" 318 | ] 319 | }, 320 | "execution_count": 9, 321 | "metadata": {}, 322 | "output_type": "execute_result" 323 | } 324 | ], 325 | "source": [ 326 | "dict1" 327 | ] 328 | }, 329 | { 330 | "cell_type": "code", 331 | "execution_count": 10, 332 | "id": "extensive-cabinet", 333 | "metadata": {}, 334 | "outputs": [ 335 | { 336 | "data": { 337 | "text/plain": [ 338 | "{0: 200}" 339 | ] 340 | }, 341 | "execution_count": 10, 342 | "metadata": {}, 343 | "output_type": "execute_result" 344 | } 345 | ], 346 | "source": [ 347 | "dict2" 348 | ] 349 | }, 350 | { 351 | "cell_type": "markdown", 352 | "id": "cheap-handbook", 353 | "metadata": {}, 354 | "source": [ 355 | "获取长度" 356 | ] 357 | }, 358 | { 359 | "cell_type": "code", 360 | "execution_count": 11, 361 | "id": "celtic-harmony", 362 | "metadata": {}, 363 | "outputs": [ 364 | { 365 | "data": { 366 | "text/plain": [ 367 | "2" 368 | ] 369 | }, 370 | "execution_count": 11, 371 | "metadata": {}, 372 | "output_type": "execute_result" 373 | } 374 | ], 375 | "source": [ 376 | "len(dict1)" 377 | ] 378 | }, 379 | { 380 | "cell_type": "markdown", 381 | "id": "sixth-offense", 382 | "metadata": {}, 383 | "source": [ 384 | "删除键值对" 385 | ] 386 | }, 387 | { 388 | "cell_type": "code", 389 | "execution_count": 12, 390 | "id": "unavailable-mounting", 391 | "metadata": {}, 392 | "outputs": [], 393 | "source": [ 394 | "del dict1[0]" 395 | ] 396 | }, 397 | { 398 | "cell_type": "code", 399 | "execution_count": 13, 400 | "id": "pretty-peace", 401 | "metadata": {}, 402 | "outputs": [ 403 | { 404 | "data": { 405 | "text/plain": [ 406 | "{'test': 'Hello CyberDB!'}" 407 | ] 408 | }, 409 | "execution_count": 13, 410 | "metadata": {}, 411 | "output_type": "execute_result" 412 | } 413 | ], 414 | "source": [ 415 | "dict1" 416 | ] 417 | }, 418 | { 419 | "cell_type": "markdown", 420 | "id": "downtown-assurance", 421 | "metadata": {}, 422 | "source": [ 423 | "清空 dict1" 424 | ] 425 | }, 426 | { 427 | "cell_type": "code", 428 | "execution_count": 14, 429 | "id": "indoor-stanford", 430 | "metadata": {}, 431 | "outputs": [ 432 | { 433 | "data": { 434 | "text/plain": [ 435 | "{}" 436 | ] 437 | }, 438 | "execution_count": 14, 439 | "metadata": {}, 440 | "output_type": "execute_result" 441 | } 442 | ], 443 | "source": [ 444 | "dict1.clear()\n", 445 | "dict1" 446 | ] 447 | }, 448 | { 449 | "cell_type": "markdown", 450 | "id": "arbitrary-tulsa", 451 | "metadata": {}, 452 | "source": [ 453 | "##### CyberList 常用操作" 454 | ] 455 | }, 456 | { 457 | "cell_type": "markdown", 458 | "id": "ranking-monster", 459 | "metadata": {}, 460 | "source": [ 461 | "生成 list1 的内容" 462 | ] 463 | }, 464 | { 465 | "cell_type": "code", 466 | "execution_count": 15, 467 | "id": "phantom-publicity", 468 | "metadata": {}, 469 | "outputs": [ 470 | { 471 | "data": { 472 | "text/plain": [ 473 | "[99, 99, 99, 99, 99]" 474 | ] 475 | }, 476 | "execution_count": 15, 477 | "metadata": {}, 478 | "output_type": "execute_result" 479 | } 480 | ], 481 | "source": [ 482 | "for i in range(5):\n", 483 | " list1.append(99)\n", 484 | " \n", 485 | "list1" 486 | ] 487 | }, 488 | { 489 | "cell_type": "markdown", 490 | "id": "ongoing-inspector", 491 | "metadata": {}, 492 | "source": [ 493 | "更改坐标值" 494 | ] 495 | }, 496 | { 497 | "cell_type": "code", 498 | "execution_count": 16, 499 | "id": "russian-clearance", 500 | "metadata": {}, 501 | "outputs": [ 502 | { 503 | "data": { 504 | "text/plain": [ 505 | "[99, 99, 99, 100, 99]" 506 | ] 507 | }, 508 | "execution_count": 16, 509 | "metadata": {}, 510 | "output_type": "execute_result" 511 | } 512 | ], 513 | "source": [ 514 | "list1[3] = 100\n", 515 | "list1" 516 | ] 517 | }, 518 | { 519 | "cell_type": "markdown", 520 | "id": "continuous-denial", 521 | "metadata": {}, 522 | "source": [ 523 | "进行切片" 524 | ] 525 | }, 526 | { 527 | "cell_type": "code", 528 | "execution_count": 17, 529 | "id": "interstate-tribe", 530 | "metadata": {}, 531 | "outputs": [ 532 | { 533 | "data": { 534 | "text/plain": [ 535 | "[100, 99]" 536 | ] 537 | }, 538 | "execution_count": 17, 539 | "metadata": {}, 540 | "output_type": "execute_result" 541 | } 542 | ], 543 | "source": [ 544 | "list1[3:]" 545 | ] 546 | }, 547 | { 548 | "cell_type": "markdown", 549 | "id": "cosmetic-surfing", 550 | "metadata": {}, 551 | "source": [ 552 | "获取 list1 的长度" 553 | ] 554 | }, 555 | { 556 | "cell_type": "code", 557 | "execution_count": 18, 558 | "id": "opening-reporter", 559 | "metadata": {}, 560 | "outputs": [ 561 | { 562 | "data": { 563 | "text/plain": [ 564 | "5" 565 | ] 566 | }, 567 | "execution_count": 18, 568 | "metadata": {}, 569 | "output_type": "execute_result" 570 | } 571 | ], 572 | "source": [ 573 | "len(list1)" 574 | ] 575 | }, 576 | { 577 | "cell_type": "markdown", 578 | "id": "spoken-cardiff", 579 | "metadata": {}, 580 | "source": [ 581 | "通过迭代打印 list1 每个元素" 582 | ] 583 | }, 584 | { 585 | "cell_type": "code", 586 | "execution_count": 19, 587 | "id": "right-relative", 588 | "metadata": {}, 589 | "outputs": [ 590 | { 591 | "name": "stdout", 592 | "output_type": "stream", 593 | "text": [ 594 | "99\n", 595 | "99\n", 596 | "99\n", 597 | "100\n", 598 | "99\n" 599 | ] 600 | } 601 | ], 602 | "source": [ 603 | "for v in list1:\n", 604 | " print(v)" 605 | ] 606 | }, 607 | { 608 | "cell_type": "markdown", 609 | "id": "fifth-substitute", 610 | "metadata": {}, 611 | "source": [ 612 | "强烈推荐使用 for 循环迭代 CyberDict，每次迭代将从服务端获取 v，客户端的空间复杂度为 o(1)。迭代同样可用于 CyberDict，CyberDict 的迭代中，客户端空间复杂度为 o(n), n 为 CyberDict.keys() 的大小。\n", 613 | "\n", 614 | "#### 释放 proxy 对象\n", 615 | "\n", 616 | "使用完成，将 proxy 的连接归还至连接池即可。" 617 | ] 618 | }, 619 | { 620 | "cell_type": "code", 621 | "execution_count": 20, 622 | "id": "funded-consciousness", 623 | "metadata": {}, 624 | "outputs": [], 625 | "source": [ 626 | "proxy.close()" 627 | ] 628 | }, 629 | { 630 | "cell_type": "markdown", 631 | "id": "automotive-mirror", 632 | "metadata": {}, 633 | "source": [ 634 | "proxy 对象同样支持上下文管理器，如" 635 | ] 636 | }, 637 | { 638 | "cell_type": "code", 639 | "execution_count": 21, 640 | "id": "brutal-pleasure", 641 | "metadata": {}, 642 | "outputs": [ 643 | { 644 | "name": "stdout", 645 | "output_type": "stream", 646 | "text": [ 647 | "[99, 99, 99, 100, 99]\n" 648 | ] 649 | } 650 | ], 651 | "source": [ 652 | "with client.get_proxy() as proxy:\n", 653 | " list1 = proxy.get_cyberlist('list1')\n", 654 | " print(list1)" 655 | ] 656 | }, 657 | { 658 | "cell_type": "markdown", 659 | "id": "fixed-columbus", 660 | "metadata": {}, 661 | "source": [ 662 | "## 概括\n", 663 | "\n", 664 | "有了 CyberDB，便能充分利用内存性能，不同进程(甚至不同主机)能通过 Python 的数据结构通信。生产环境使用方法请参考官方文档，感谢你的支持！" 665 | ] 666 | }, 667 | { 668 | "cell_type": "markdown", 669 | "id": "noble-observation", 670 | "metadata": {}, 671 | "source": [ 672 | "## 注意\n", 673 | "\n", 674 | "由于编码限制，CyberDB 会将 0 识别为 None，但并不影响计算，请在所需位置将 None 转为 0。" 675 | ] 676 | } 677 | ], 678 | "metadata": { 679 | "kernelspec": { 680 | "display_name": "Python 3", 681 | "language": "python", 682 | "name": "python3" 683 | }, 684 | "language_info": { 685 | "codemirror_mode": { 686 | "name": "ipython", 687 | "version": 3 688 | }, 689 | "file_extension": ".py", 690 | "mimetype": "text/x-python", 691 | "name": "python", 692 | "nbconvert_exporter": "python", 693 | "pygments_lexer": "ipython3", 694 | "version": "3.8.6" 695 | }, 696 | "toc": { 697 | "base_numbering": 1, 698 | "nav_menu": {}, 699 | "number_sections": true, 700 | "sideBar": true, 701 | "skip_h1_title": false, 702 | "title_cell": "Table of Contents", 703 | "title_sidebar": "Contents", 704 | "toc_cell": false, 705 | "toc_position": {}, 706 | "toc_section_display": true, 707 | "toc_window_display": false 708 | } 709 | }, 710 | "nbformat": 4, 711 | "nbformat_minor": 5 712 | } 713 | -------------------------------------------------------------------------------- /cyberdb/network/route.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import datetime 3 | 4 | from . import AioStream 5 | from ..data import datas 6 | from ..extensions import nonce, WrongPasswordCyberDBError 7 | 8 | 9 | MAP = {} 10 | 11 | 12 | def bind(path): 13 | def decorator(func): 14 | async def wrapper(self, *args, **kw): 15 | server_obj = await func(self, *args, **kw) 16 | server_obj['password'] = self._dp._secret.key 17 | 18 | await self._stream.write(server_obj) 19 | 20 | MAP[path] = wrapper 21 | return wrapper 22 | return decorator 23 | 24 | 25 | class Route: 26 | ''' 27 | TCP event mapping. 28 | ''' 29 | 30 | def __init__(self, db: dict, dp: datas.DataParsing, stream: AioStream, 31 | print_log: bool = False): 32 | self._db = db 33 | self._dp = dp 34 | self._stream = stream 35 | self._print_log = print_log 36 | 37 | async def find(self): 38 | ''' 39 | Loop accepting client requests. 40 | ''' 41 | addr = self._stream.get_addr() 42 | addr = '{}:{}'.format(addr[0], addr[1]) 43 | 44 | while True: 45 | client_obj = await self._stream.read() 46 | self._client_obj = client_obj 47 | 48 | # Check if the password is correct. 49 | if self._stream._dp._secret.key != client_obj['password']: 50 | self._stream._writer.close() 51 | raise WrongPasswordCyberDBError( 52 | 'The password entered by the client is incorrect.') 53 | 54 | if self._print_log: 55 | print('{} {} {}'.format( 56 | datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 57 | addr, client_obj['route'])) 58 | 59 | # Jump to the specified function by routing. 60 | func = MAP[client_obj['route']] 61 | # Go to the corresponding routing function. 62 | await func(self) 63 | 64 | @bind('/connect') 65 | async def connect(self): 66 | server_obj = { 67 | 'code': 1, 68 | 'message': 'connection succeeded.' 69 | } 70 | return server_obj 71 | 72 | @bind('/create_cyberdict') 73 | async def create_cyberdict(self): 74 | table_name = self._client_obj['table_name'] 75 | content = self._client_obj['content'] 76 | if self._db.get(table_name) == None: 77 | # New CyberDict 78 | self._db[table_name] = content 79 | server_obj = { 80 | 'code': 1 81 | } 82 | else: 83 | server_obj = { 84 | 'code': 0 85 | } 86 | 87 | return server_obj 88 | 89 | @bind('/create_cyberlist') 90 | async def create_cyberlist(self): 91 | table_name = self._client_obj['table_name'] 92 | content = self._client_obj['content'] 93 | if self._db.get(table_name) == None: 94 | # New CyberList 95 | self._db[table_name] = content 96 | server_obj = { 97 | 'code': 1 98 | } 99 | else: 100 | server_obj = { 101 | 'code': 0 102 | } 103 | 104 | return server_obj 105 | 106 | @bind('/exam_cyberdict') 107 | async def exam_cyberdict(self): 108 | ''' 109 | Check if the table in the database exists. 110 | ''' 111 | 112 | table_name = self._client_obj['table_name'] 113 | if self._db.get(table_name) != None: 114 | server_obj = { 115 | 'code': 1 116 | } 117 | else: 118 | server_obj = { 119 | 'code': 0 120 | } 121 | 122 | return server_obj 123 | 124 | @bind('/exam_cyberlist') 125 | async def exam_cyberlist(self): 126 | ''' 127 | Check if the table in the database exists. 128 | ''' 129 | 130 | table_name = self._client_obj['table_name'] 131 | if self._db.get(table_name) != None: 132 | server_obj = { 133 | 'code': 1 134 | } 135 | else: 136 | server_obj = { 137 | 'code': 0 138 | } 139 | 140 | return server_obj 141 | 142 | @bind('/print_tables') 143 | async def print_tables(self): 144 | inform = [] 145 | for table_name in self._db: 146 | if type(self._db[table_name]) == dict: 147 | type_name = 'CyberDict' 148 | elif type(self._db[table_name]) == list: 149 | type_name = 'CyberList' 150 | inform.append((table_name, type_name)) 151 | 152 | server_obj = { 153 | 'code': 1, 154 | 'content': inform 155 | } 156 | 157 | return server_obj 158 | 159 | @bind('/delete_table') 160 | async def delete_table(self): 161 | table_name = self._client_obj['table_name'] 162 | try: 163 | del self._db[table_name] 164 | server_obj = { 165 | 'code': 1 166 | } 167 | except Exception as e: 168 | server_obj = { 169 | 'code': 0, 170 | 'Exception': e 171 | } 172 | 173 | return server_obj 174 | 175 | @bind('/cyberdict/repr') 176 | async def dict_repr(self): 177 | table_name = self._client_obj['table_name'] 178 | try: 179 | r = repr(self._db[table_name]) 180 | server_obj = { 181 | 'code': 1, 182 | 'content': r 183 | } 184 | except Exception as e: 185 | server_obj = { 186 | 'code': 0, 187 | 'Exception': e 188 | } 189 | 190 | return server_obj 191 | 192 | @bind('/cyberdict/str') 193 | async def dict_str(self): 194 | table_name = self._client_obj['table_name'] 195 | try: 196 | r = str(self._db[table_name]) 197 | server_obj = { 198 | 'code': 1, 199 | 'content': r 200 | } 201 | except Exception as e: 202 | server_obj = { 203 | 'code': 0, 204 | 'Exception': e 205 | } 206 | 207 | return server_obj 208 | 209 | @bind('/cyberdict/len') 210 | async def dict_len(self): 211 | table_name = self._client_obj['table_name'] 212 | try: 213 | r = len(self._db[table_name]) 214 | server_obj = { 215 | 'code': 1, 216 | 'content': r 217 | } 218 | except Exception as e: 219 | server_obj = { 220 | 'code': 0, 221 | 'Exception': e 222 | } 223 | 224 | return server_obj 225 | 226 | @bind('/cyberdict/getitem') 227 | async def dict_getitem(self): 228 | table_name = self._client_obj['table_name'] 229 | key = self._client_obj['key'] 230 | try: 231 | r = self._db[table_name][key] 232 | server_obj = { 233 | 'code': 1, 234 | 'content': r 235 | } 236 | except Exception as e: 237 | server_obj = { 238 | 'code': 0, 239 | 'Exception': e 240 | } 241 | 242 | return server_obj 243 | 244 | @bind('/cyberdict/setitem') 245 | async def dict_setitem(self): 246 | table_name = self._client_obj['table_name'] 247 | key = self._client_obj['key'] 248 | value = self._client_obj['value'] 249 | try: 250 | self._db[table_name][key] = value 251 | server_obj = { 252 | 'code': 1, 253 | } 254 | except Exception as e: 255 | server_obj = { 256 | 'code': 0, 257 | 'Exception': e 258 | } 259 | 260 | return server_obj 261 | 262 | @bind('/cyberdict/delitem') 263 | async def dict_delitem(self): 264 | table_name = self._client_obj['table_name'] 265 | key = self._client_obj['key'] 266 | try: 267 | del self._db[table_name][key] 268 | server_obj = { 269 | 'code': 1, 270 | } 271 | except Exception as e: 272 | server_obj = { 273 | 'code': 0, 274 | 'Exception': e 275 | } 276 | 277 | return server_obj 278 | 279 | @bind('/cyberdict/todict') 280 | async def todict(self): 281 | table_name = self._client_obj['table_name'] 282 | try: 283 | r = self._db[table_name] 284 | server_obj = { 285 | 'code': 1, 286 | 'content': r 287 | } 288 | except Exception as e: 289 | server_obj = { 290 | 'code': 0, 291 | 'Exception': e 292 | } 293 | 294 | return server_obj 295 | 296 | @bind('/cyberdict/get') 297 | async def dict_get(self): 298 | table_name = self._client_obj['table_name'] 299 | key = self._client_obj['key'] 300 | default = self._client_obj['default'] 301 | try: 302 | r = self._db[table_name].get(key, default) 303 | server_obj = { 304 | 'code': 1, 305 | 'content': r 306 | } 307 | except Exception as e: 308 | server_obj = { 309 | 'code': 0, 310 | 'Exception': e 311 | } 312 | 313 | return server_obj 314 | 315 | @bind('/cyberdict/setdefault') 316 | async def dict_setdefault(self): 317 | table_name = self._client_obj['table_name'] 318 | key = self._client_obj['key'] 319 | default = self._client_obj['default'] 320 | try: 321 | r = self._db[table_name].setdefault(key, default) 322 | server_obj = { 323 | 'code': 1, 324 | 'content': r 325 | } 326 | except Exception as e: 327 | server_obj = { 328 | 'code': 0, 329 | 'Exception': e 330 | } 331 | 332 | return server_obj 333 | 334 | @bind('/cyberdict/update') 335 | async def dict_update(self): 336 | table_name = self._client_obj['table_name'] 337 | dict2 = self._client_obj['dict2'] 338 | try: 339 | self._db[table_name].update(dict2) 340 | server_obj = { 341 | 'code': 1 342 | } 343 | except Exception as e: 344 | server_obj = { 345 | 'code': 0, 346 | 'Exception': e 347 | } 348 | 349 | return server_obj 350 | 351 | @bind('/cyberdict/keys') 352 | async def dict_keys(self): 353 | table_name = self._client_obj['table_name'] 354 | try: 355 | r = list(self._db[table_name].keys()) 356 | server_obj = { 357 | 'code': 1, 358 | 'content': r 359 | } 360 | except Exception as e: 361 | server_obj = { 362 | 'code': 0, 363 | 'Exception': e 364 | } 365 | 366 | return server_obj 367 | 368 | @bind('/cyberdict/values') 369 | async def dict_values(self): 370 | table_name = self._client_obj['table_name'] 371 | try: 372 | r = list(self._db[table_name].values()) 373 | server_obj = { 374 | 'code': 1, 375 | 'content': r 376 | } 377 | except Exception as e: 378 | server_obj = { 379 | 'code': 0, 380 | 'Exception': e 381 | } 382 | 383 | return server_obj 384 | 385 | @bind('/cyberdict/items') 386 | async def dict_items(self): 387 | table_name = self._client_obj['table_name'] 388 | try: 389 | r = list(self._db[table_name].items()) 390 | server_obj = { 391 | 'code': 1, 392 | 'content': r 393 | } 394 | except Exception as e: 395 | server_obj = { 396 | 'code': 0, 397 | 'Exception': e 398 | } 399 | 400 | return server_obj 401 | 402 | @bind('/cyberdict/pop') 403 | async def dict_pop(self): 404 | table_name = self._client_obj['table_name'] 405 | key = self._client_obj['key'] 406 | default = self._client_obj['default'] 407 | 408 | try: 409 | r = self._db[table_name].pop(key, default) 410 | server_obj = { 411 | 'code': 1, 412 | 'content': r 413 | } 414 | except Exception as e: 415 | server_obj = { 416 | 'code': 0, 417 | 'Exception': e 418 | } 419 | 420 | return server_obj 421 | 422 | @bind('/cyberdict/popitem') 423 | async def dict_popitem(self): 424 | table_name = self._client_obj['table_name'] 425 | try: 426 | r = self._db[table_name].popitem() 427 | server_obj = { 428 | 'code': 1, 429 | 'content': r 430 | } 431 | except Exception as e: 432 | server_obj = { 433 | 'code': 0, 434 | 'Exception': e 435 | } 436 | 437 | return server_obj 438 | 439 | @bind('/cyberdict/clear') 440 | async def dict_clear(self): 441 | table_name = self._client_obj['table_name'] 442 | try: 443 | self._db[table_name].clear() 444 | server_obj = { 445 | 'code': 1 446 | } 447 | except Exception as e: 448 | server_obj = { 449 | 'code': 0, 450 | 'Exception': e 451 | } 452 | 453 | return server_obj 454 | 455 | @bind('/cyberlist/repr') 456 | async def list_repr(self): 457 | table_name = self._client_obj['table_name'] 458 | try: 459 | r = repr(self._db[table_name]) 460 | server_obj = { 461 | 'code': 1, 462 | 'content': r 463 | } 464 | except Exception as e: 465 | server_obj = { 466 | 'code': 0, 467 | 'Exception': e 468 | } 469 | 470 | return server_obj 471 | 472 | @bind('/cyberlist/str') 473 | async def list_str(self): 474 | table_name = self._client_obj['table_name'] 475 | try: 476 | r = str(self._db[table_name]) 477 | server_obj = { 478 | 'code': 1, 479 | 'content': r 480 | } 481 | except Exception as e: 482 | server_obj = { 483 | 'code': 0, 484 | 'Exception': e 485 | } 486 | 487 | return server_obj 488 | 489 | @bind('/cyberlist/len') 490 | async def list_len(self): 491 | table_name = self._client_obj['table_name'] 492 | try: 493 | r = len(self._db[table_name]) 494 | server_obj = { 495 | 'code': 1, 496 | 'content': r 497 | } 498 | except Exception as e: 499 | server_obj = { 500 | 'code': 0, 501 | 'Exception': e 502 | } 503 | 504 | return server_obj 505 | 506 | @bind('/cyberlist/getitem') 507 | async def list_getitem(self): 508 | table_name = self._client_obj['table_name'] 509 | index = self._client_obj['index'] 510 | try: 511 | r = self._db[table_name][index] 512 | server_obj = { 513 | 'code': 1, 514 | 'content': r 515 | } 516 | except Exception as e: 517 | server_obj = { 518 | 'code': 0, 519 | 'Exception': e 520 | } 521 | 522 | return server_obj 523 | 524 | @bind('/cyberlist/setitem') 525 | async def list_setitem(self): 526 | table_name = self._client_obj['table_name'] 527 | index = self._client_obj['index'] 528 | value = self._client_obj['value'] 529 | try: 530 | self._db[table_name][index] = value 531 | server_obj = { 532 | 'code': 1, 533 | } 534 | except Exception as e: 535 | server_obj = { 536 | 'code': 0, 537 | 'Exception': e 538 | } 539 | 540 | return server_obj 541 | 542 | @bind('/cyberlist/delitem') 543 | async def list_delitem(self): 544 | table_name = self._client_obj['table_name'] 545 | index = self._client_obj['index'] 546 | try: 547 | del self._db[table_name][index] 548 | server_obj = { 549 | 'code': 1, 550 | } 551 | except Exception as e: 552 | server_obj = { 553 | 'code': 0, 554 | 'Exception': e 555 | } 556 | 557 | return server_obj 558 | 559 | @bind('/cyberlist/tolist') 560 | async def tolist(self): 561 | table_name = self._client_obj['table_name'] 562 | try: 563 | r = self._db[table_name] 564 | server_obj = { 565 | 'code': 1, 566 | 'content': r 567 | } 568 | except Exception as e: 569 | server_obj = { 570 | 'code': 0, 571 | 'Exception': e 572 | } 573 | 574 | return server_obj 575 | 576 | @bind('/cyberlist/append') 577 | async def list_append(self): 578 | table_name = self._client_obj['table_name'] 579 | value = self._client_obj['value'] 580 | try: 581 | self._db[table_name].append(value) 582 | server_obj = { 583 | 'code': 1 584 | } 585 | except Exception as e: 586 | server_obj = { 587 | 'code': 0, 588 | 'Exception': e 589 | } 590 | 591 | return server_obj 592 | 593 | @bind('/cyberlist/extend') 594 | async def list_extend(self): 595 | table_name = self._client_obj['table_name'] 596 | obj = self._client_obj['obj'] 597 | try: 598 | self._db[table_name].extend(obj) 599 | server_obj = { 600 | 'code': 1 601 | } 602 | except Exception as e: 603 | server_obj = { 604 | 'code': 0, 605 | 'Exception': e 606 | } 607 | 608 | return server_obj 609 | 610 | @bind('/cyberlist/insert') 611 | async def list_insert(self): 612 | table_name = self._client_obj['table_name'] 613 | index = self._client_obj['index'] 614 | value = self._client_obj['value'] 615 | try: 616 | self._db[table_name].insert(index, value) 617 | server_obj = { 618 | 'code': 1 619 | } 620 | except Exception as e: 621 | server_obj = { 622 | 'code': 0, 623 | 'Exception': e 624 | } 625 | 626 | return server_obj 627 | 628 | @bind('/cyberlist/pop') 629 | async def list_pop(self): 630 | table_name = self._client_obj['table_name'] 631 | index = self._client_obj['index'] 632 | try: 633 | self._db[table_name].pop(index) 634 | server_obj = { 635 | 'code': 1 636 | } 637 | except Exception as e: 638 | server_obj = { 639 | 'code': 0, 640 | 'Exception': e 641 | } 642 | 643 | return server_obj 644 | 645 | @bind('/cyberlist/remove') 646 | async def list_remove(self): 647 | table_name = self._client_obj['table_name'] 648 | value = self._client_obj['value'] 649 | try: 650 | self._db[table_name].remove(value) 651 | server_obj = { 652 | 'code': 1 653 | } 654 | except Exception as e: 655 | server_obj = { 656 | 'code': 0, 657 | 'Exception': e 658 | } 659 | 660 | return server_obj 661 | 662 | @bind('/cyberlist/count') 663 | async def list_count(self): 664 | table_name = self._client_obj['table_name'] 665 | value = self._client_obj['value'] 666 | try: 667 | r = self._db[table_name].count(value) 668 | server_obj = { 669 | 'code': 1, 670 | 'content': r 671 | } 672 | except Exception as e: 673 | server_obj = { 674 | 'code': 0, 675 | 'Exception': e 676 | } 677 | 678 | return server_obj 679 | 680 | @bind('/cyberlist/index') 681 | async def list_index(self): 682 | table_name = self._client_obj['table_name'] 683 | value = self._client_obj['value'] 684 | try: 685 | r = self._db[table_name].index(value) 686 | server_obj = { 687 | 'code': 1, 688 | 'content': r 689 | } 690 | except Exception as e: 691 | server_obj = { 692 | 'code': 0, 693 | 'Exception': e 694 | } 695 | 696 | return server_obj 697 | 698 | @bind('/cyberlist/reverse') 699 | async def list_reverse(self): 700 | table_name = self._client_obj['table_name'] 701 | try: 702 | self._db[table_name].reverse() 703 | server_obj = { 704 | 'code': 1 705 | } 706 | except Exception as e: 707 | server_obj = { 708 | 'code': 0, 709 | 'Exception': e 710 | } 711 | 712 | return server_obj 713 | 714 | @bind('/cyberlist/sort') 715 | async def list_sort(self): 716 | table_name = self._client_obj['table_name'] 717 | key = self._client_obj['key'] 718 | 719 | if key: 720 | # Dynamic get function. 721 | loc = locals() 722 | key = exec('{}'.format(key)) 723 | key = loc['func'] 724 | 725 | reverse = self._client_obj['reverse'] 726 | try: 727 | self._db[table_name].sort(key=key, reverse=reverse) 728 | server_obj = { 729 | 'code': 1 730 | } 731 | except Exception as e: 732 | server_obj = { 733 | 'code': 0, 734 | 'Exception': e 735 | } 736 | 737 | return server_obj 738 | 739 | @bind('/cyberlist/clear') 740 | async def list_clear(self): 741 | table_name = self._client_obj['table_name'] 742 | try: 743 | self._db[table_name].clear() 744 | server_obj = { 745 | 'code': 1 746 | } 747 | except Exception as e: 748 | server_obj = { 749 | 'code': 0, 750 | 'Exception': e 751 | } 752 | 753 | return server_obj 754 | -------------------------------------------------------------------------------- /cyberdb/network/client.py: -------------------------------------------------------------------------------- 1 | import time 2 | import inspect 3 | import socket 4 | from typing import List, Tuple, Dict 5 | 6 | from obj_encrypt import Secret 7 | 8 | from ..data import datas 9 | from ..extensions import CyberDBError, WrongInputCyberDBError, WrongPasswordCyberDBError, WrongTableNameCyberDBError 10 | from ..extensions.signature import Signature 11 | from . import Stream 12 | 13 | 14 | class Connection: 15 | ''' 16 | TCP connection data type 17 | ''' 18 | 19 | def __init__( 20 | self, 21 | s: socket.socket = None, 22 | ): 23 | self.s = s 24 | 25 | 26 | class ConPool: 27 | ''' 28 | Maintain connection pool. 29 | ''' 30 | 31 | def __init__(self, host: str, port: str, dp: datas.DataParsing, 32 | time_out: int = None): 33 | self._host = host 34 | self._port = port 35 | self._dp = dp 36 | self._time_out = time_out 37 | self._connections = [] 38 | 39 | def get(self): 40 | ''' 41 | Get the connection from the connection pool. 42 | ''' 43 | while self._connections: 44 | # If the connection is full, the loop waits until a connection 45 | # is free. 46 | connection = self._connections.pop() 47 | time_now = int(time.time()) 48 | if self._time_out: 49 | if time_now > connection['timestamp'] + self._time_out: 50 | continue 51 | # Check if the server is down. 52 | r = confirm_the_connection(connection['s'], self._dp) 53 | if r['code'] == 1: 54 | return connection['s'] 55 | else: 56 | pass 57 | 58 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 59 | s.connect((self._host, self._port)) 60 | s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 61 | return s 62 | 63 | def put(self, s: socket.socket): 64 | ''' 65 | Return the connection to the connection pool. 66 | ''' 67 | # If a connection timeout is set, get the current timestamp. 68 | timestamp = None 69 | if self._time_out: 70 | timestamp = int(time.time()) 71 | 72 | self._connections.insert(0, { 73 | 's': s, 74 | 'timestamp': timestamp 75 | }) 76 | 77 | 78 | def network(func): 79 | ''' 80 | Network operations before and after encapsulating CyberDB data 81 | structure methods. 82 | ''' 83 | 84 | def wrapper(self, *args, **kw): 85 | stream = Stream(self._con.s, self._dp) 86 | client_obj = func(self, *args, **kw) 87 | stream.write(client_obj) 88 | 89 | server_obj = stream.read() 90 | if server_obj['code'] == 0: 91 | self._con.s.close() 92 | raise server_obj['Exception'] 93 | 94 | if server_obj.get('content'): 95 | return server_obj['content'] 96 | 97 | return wrapper 98 | 99 | 100 | class CyberDict: 101 | ''' 102 | A child object generated by a Proxy object for performing 103 | Dictionaries operations. This object will perform remote 104 | operations on the server-side CyberDB database. Shares the 105 | same TCP connection with the Proxy object. The connection 106 | will follow the connection of the Proxy. After the Proxy 107 | releases the connection, the object will also lose the 108 | connection. CyberDict can execute the get, setdefault, update, 109 | keys, values, items, pop, popitem, clear methods and common 110 | magic methods of Dictionaries, please refer to [Python 111 | dictionary official documentation] 112 | (https://docs.python.org/3/library/stdtypes.html#mapping-types-dict). 113 | 114 | Iterate over CyberDict using a for loop with client space 115 | complexity o(n), where n is the length of CyberDict.keys() . 116 | ''' 117 | 118 | def __init__( 119 | self, 120 | table_name: str, 121 | dp: datas.DataParsing, 122 | con: Connection 123 | ): 124 | self._table_name = table_name 125 | self._dp = dp 126 | self._con = con 127 | self._route = '/cyberdict' 128 | 129 | @network 130 | def __repr__(self): 131 | return { 132 | 'route': self._route + '/repr', 133 | 'table_name': self._table_name 134 | } 135 | 136 | @network 137 | def __str__(self): 138 | return { 139 | 'route': self._route + '/str', 140 | 'table_name': self._table_name 141 | } 142 | 143 | @network 144 | def __len__(self): 145 | return { 146 | 'route': self._route + '/len', 147 | 'table_name': self._table_name 148 | } 149 | 150 | def __iter__(self): 151 | return self.generate() 152 | 153 | @network 154 | def __getitem__(self, key): 155 | return { 156 | 'route': self._route + '/getitem', 157 | 'table_name': self._table_name, 158 | 'key': key 159 | } 160 | 161 | @network 162 | def __setitem__(self, key, value): 163 | return { 164 | 'route': self._route + '/setitem', 165 | 'table_name': self._table_name, 166 | 'key': key, 167 | 'value': value 168 | } 169 | 170 | @network 171 | def __delitem__(self, key): 172 | return { 173 | 'route': self._route + '/delitem', 174 | 'table_name': self._table_name, 175 | 'key': key 176 | } 177 | 178 | @network 179 | def todict(self) -> Dict: 180 | ''' 181 | Convert CyberDict to Dictionaries. 182 | 183 | Return Type: Dict 184 | ''' 185 | return { 186 | 'route': self._route + '/todict', 187 | 'table_name': self._table_name 188 | } 189 | 190 | @network 191 | def get(self, key, default=None) -> any: 192 | return { 193 | 'route': self._route + '/get', 194 | 'table_name': self._table_name, 195 | 'key': key, 196 | 'default': default 197 | } 198 | 199 | @network 200 | def setdefault(self, key, default=None) -> None: 201 | return { 202 | 'route': self._route + '/setdefault', 203 | 'table_name': self._table_name, 204 | 'key': key, 205 | 'default': default 206 | } 207 | 208 | @network 209 | def update(self, dict2) -> None: 210 | return { 211 | 'route': self._route + '/update', 212 | 'table_name': self._table_name, 213 | 'dict2': dict2, 214 | } 215 | 216 | @network 217 | def keys(self) -> List: 218 | return { 219 | 'route': self._route + '/keys', 220 | 'table_name': self._table_name 221 | } 222 | 223 | @network 224 | def values(self) -> List: 225 | return { 226 | 'route': self._route + '/values', 227 | 'table_name': self._table_name 228 | } 229 | 230 | @network 231 | def items(self) -> List[Tuple]: 232 | return { 233 | 'route': self._route + '/items', 234 | 'table_name': self._table_name 235 | } 236 | 237 | @network 238 | def pop(self, key, default=None) -> any: 239 | return { 240 | 'route': self._route + '/pop', 241 | 'table_name': self._table_name, 242 | 'key': key, 243 | 'default': default 244 | } 245 | 246 | @network 247 | def popitem(self) -> Tuple[any, any]: 248 | return { 249 | 'route': self._route + '/popitem', 250 | 'table_name': self._table_name 251 | } 252 | 253 | @network 254 | def clear(self) -> None: 255 | return { 256 | 'route': self._route + '/clear', 257 | 'table_name': self._table_name 258 | } 259 | 260 | def generate(self): 261 | for key in self.keys(): 262 | yield key 263 | 264 | 265 | class CyberList: 266 | ''' 267 | A child object generated by a Proxy object for performing Lists 268 | operations. This object will perform remote operations on the 269 | server-side CyberDB database. Shares the same TCP connection 270 | with the Proxy object. The connection will follow the connection 271 | of the Proxy. After the Proxy releases the connection, the object 272 | will also lose the connection. CyberList can execute the append, 273 | extend, insert, pop, remove, count, index, reverse, sort, clear 274 | methods and common magic methods of Lists, please refer to 275 | [Python List Official Documentation] 276 | (https://docs.python.org/3/tutorial/datastructures.html#more-on-lists). 277 | 278 | The CyberList is iterated using a for loop, each iteration 279 | will fetch the content from the server, and the space complexity 280 | of the client is o(1). 281 | ''' 282 | 283 | def __init__( 284 | self, 285 | table_name: str, 286 | dp: datas.DataParsing, 287 | con: Connection 288 | ): 289 | self._table_name = table_name 290 | self._dp = dp 291 | self._con = con 292 | self._route = '/cyberlist' 293 | 294 | @network 295 | def __repr__(self): 296 | return { 297 | 'route': self._route + '/repr', 298 | 'table_name': self._table_name 299 | } 300 | 301 | @network 302 | def __str__(self): 303 | return { 304 | 'route': self._route + '/str', 305 | 'table_name': self._table_name 306 | } 307 | 308 | @network 309 | def __len__(self): 310 | return { 311 | 'route': self._route + '/len', 312 | 'table_name': self._table_name 313 | } 314 | 315 | def __iter__(self): 316 | return self.generate() 317 | 318 | @network 319 | def __getitem__(self, index): 320 | return { 321 | 'route': self._route + '/getitem', 322 | 'table_name': self._table_name, 323 | 'index': index 324 | } 325 | 326 | @network 327 | def __setitem__(self, index, value): 328 | return { 329 | 'route': self._route + '/setitem', 330 | 'table_name': self._table_name, 331 | 'index': index, 332 | 'value': value 333 | } 334 | 335 | @network 336 | def __delitem__(self, index): 337 | return { 338 | 'route': self._route + '/delitem', 339 | 'table_name': self._table_name, 340 | 'index': index 341 | } 342 | 343 | @network 344 | def tolist(self) -> List: 345 | return { 346 | 'route': self._route + '/tolist', 347 | 'table_name': self._table_name 348 | } 349 | 350 | @network 351 | def append(self, value) -> None: 352 | return { 353 | 'route': self._route + '/append', 354 | 'table_name': self._table_name, 355 | 'value': value 356 | } 357 | 358 | @network 359 | def extend(self, obj) -> None: 360 | if type(obj) == CyberList: 361 | obj = obj.tolist() 362 | 363 | return { 364 | 'route': self._route + '/extend', 365 | 'table_name': self._table_name, 366 | 'obj': obj 367 | } 368 | 369 | @network 370 | def insert(self, index, value) -> None: 371 | return { 372 | 'route': self._route + '/insert', 373 | 'table_name': self._table_name, 374 | 'index': index, 375 | 'value': value 376 | } 377 | 378 | @network 379 | def pop(self, index: int = -1) -> any: 380 | return { 381 | 'route': self._route + '/pop', 382 | 'table_name': self._table_name, 383 | 'index': index 384 | } 385 | 386 | @network 387 | def remove(self, value) -> None: 388 | return { 389 | 'route': self._route + '/remove', 390 | 'table_name': self._table_name, 391 | 'value': value 392 | } 393 | 394 | @network 395 | def count(self, value) -> int: 396 | return { 397 | 'route': self._route + '/count', 398 | 'table_name': self._table_name, 399 | 'value': value 400 | } 401 | 402 | @network 403 | def index(self, value) -> int: 404 | return { 405 | 'route': self._route + '/index', 406 | 'table_name': self._table_name, 407 | 'value': value 408 | } 409 | 410 | @network 411 | def reverse(self) -> None: 412 | return { 413 | 'route': self._route + '/reverse', 414 | 'table_name': self._table_name 415 | } 416 | 417 | @network 418 | def sort(self, key=None, reverse=False) -> None: 419 | # Reference the lambda part to func. 420 | if key: 421 | if key.__code__.co_name == '': 422 | key = 'func = ' + \ 423 | inspect.getsource(key).split('=')[1].rsplit(')')[0] 424 | # De-indent the code and reference the function to the func. 425 | elif key.__code__.co_name: 426 | code = inspect.getsource(key) 427 | num = 0 428 | for i in range(len(code)): 429 | if code[i] != ' ': 430 | num = i 431 | break 432 | if i != 0: 433 | code_new = '' 434 | for line in code.splitlines(): 435 | code_new += line[num:] + '\n' 436 | else: 437 | code_new = code 438 | key = code_new + '\nfunc = {}'.format(key.__code__.co_name) 439 | 440 | return { 441 | 'route': self._route + '/sort', 442 | 'table_name': self._table_name, 443 | 'key': key, 444 | 'reverse': reverse 445 | } 446 | 447 | @network 448 | def clear(self) -> None: 449 | return { 450 | 'route': self._route + '/clear', 451 | 'table_name': self._table_name, 452 | } 453 | 454 | def generate(self): 455 | for i in range(self.__len__()): 456 | yield self.__getitem__(i) 457 | 458 | 459 | class Proxy: 460 | ''' 461 | The Proxy object generated by the cyberdb.Client.get_proxy method 462 | can operate on the CyberDB database and manage the TCP connections 463 | of the CyberDict and CyberList sub-objects generated by the Proxy. 464 | After the Proxy object is initialized, it can be used after executing 465 | the Proxy.connect method. The Proxy object and its sub-objects will 466 | perform remote operations on the server-side CyberDB database. 467 | ''' 468 | 469 | def __init__(self, con_pool: ConPool, dp: datas.DataParsing): 470 | self._con_pool = con_pool 471 | self._dp = dp 472 | # The connection used by the proxy, the first is the reader and the 473 | # second is the writer. 474 | self._con = Connection() 475 | 476 | def __enter__(self): 477 | self.connect() 478 | return self 479 | 480 | def __exit__(self, exc_type, exc_val, exc_tb): 481 | self.close() 482 | return True 483 | 484 | def connect(self): 485 | ''' 486 | Get a TCP connection from the connection pool and bind it to the Proxy 487 | object. 488 | 489 | Return Type: None 490 | ''' 491 | # If the proxy already has a connection, the connection will be 492 | # returned to the connection pool first. 493 | if self._con.s != None: 494 | self._con_pool.put(self._con.s) 495 | 496 | s = self._con_pool.get() 497 | self._con.s = s 498 | 499 | def close(self): 500 | ''' 501 | Cancel the TCP connection bound to the Proxy object and return it to 502 | the connection pool. It needs to be reset before the next operation. 503 | Execute the Proxy.connect method. 504 | 505 | Return Type: None 506 | ''' 507 | if self._con == None: 508 | raise CyberDBError( 509 | 'The connection could not be closed, the proxy has not acquired a connection.') 510 | 511 | self._con_pool.put(self._con.s) 512 | self._con.s = None 513 | 514 | def create_cyberdict(self, table_name: str, content: dict = {}): 515 | ''' 516 | Create a CyberDict table. 517 | 518 | Parameters: 519 | 520 | table_name – table name. 521 | 522 | content -- table content, needs to be a dictionary type, the 523 | default is an empty dictionary. 524 | 525 | Return Type: None 526 | ''' 527 | if type(table_name) != str: 528 | raise WrongInputCyberDBError('Please use str for the table name.') 529 | if type(content) != dict: 530 | raise WrongInputCyberDBError( 531 | 'The input database table type is not a Python dictionary.') 532 | 533 | stream = Stream(self._con.s, self._dp) 534 | client_obj = { 535 | 'route': '/create_cyberdict', 536 | 'table_name': table_name, 537 | 'content': content 538 | } 539 | stream.write(client_obj) 540 | 541 | server_obj = stream.read() 542 | 543 | if server_obj['code'] == 0: 544 | raise WrongTableNameCyberDBError( 545 | 'Duplicate table names already exist!') 546 | 547 | def get_cyberdict(self, table_name: str) -> CyberDict: 548 | ''' 549 | Get the CyberDict table. 550 | 551 | Parameters: 552 | 553 | table_name – table name. 554 | 555 | Return Type: CyberDict, which is a sub-object generated by Proxy, 556 | which controls the TCP connection. 557 | ''' 558 | if type(table_name) != str: 559 | raise WrongInputCyberDBError('Please use str for the table name.') 560 | 561 | stream = Stream(self._con.s, self._dp) 562 | 563 | client_obj = { 564 | 'route': '/exam_cyberdict', 565 | 'table_name': table_name 566 | } 567 | stream.write(client_obj) 568 | 569 | server_obj = stream.read() 570 | if server_obj['code'] == 0: 571 | raise WrongTableNameCyberDBError( 572 | '{} table does not exist.'.format(table_name)) 573 | else: 574 | table = CyberDict(table_name, self._dp, self._con) 575 | return table 576 | 577 | def create_cyberlist(self, table_name: str, content: list = []): 578 | ''' 579 | Create the CyberList table. 580 | 581 | Parameters: 582 | 583 | table_name – table name. 584 | 585 | content -- table content, it needs to be a list type, the default 586 | is an empty list. 587 | 588 | Return Type: None 589 | ''' 590 | if type(table_name) != str: 591 | raise WrongInputCyberDBError('Please use str for the table name.') 592 | if type(content) != type(list()): 593 | raise WrongInputCyberDBError( 594 | 'The input database table type is not a Python dictionary.') 595 | 596 | stream = Stream(self._con.s, self._dp) 597 | client_obj = { 598 | 'route': '/create_cyberlist', 599 | 'table_name': table_name, 600 | 'content': content 601 | } 602 | stream.write(client_obj) 603 | 604 | server_obj = stream.read() 605 | 606 | if server_obj['code'] == 0: 607 | raise WrongTableNameCyberDBError( 608 | 'Duplicate table names already exist!') 609 | 610 | def get_cyberlist(self, table_name: str) -> CyberList: 611 | ''' 612 | Get the CyberList table. 613 | 614 | Parameters: 615 | 616 | table_name – table name. 617 | 618 | Return type: CyberList, which is a sub-object generated by Proxy, which 619 | controls the TCP connection.abase. 620 | ''' 621 | if type(table_name) != str: 622 | raise WrongInputCyberDBError('Please use str for the table name.') 623 | 624 | stream = Stream(self._con.s, self._dp) 625 | 626 | client_obj = { 627 | 'route': '/exam_cyberlist', 628 | 'table_name': table_name 629 | } 630 | stream.write(client_obj) 631 | 632 | server_obj = stream.read() 633 | if server_obj['code'] == 0: 634 | raise WrongTableNameCyberDBError( 635 | '{} table does not exist.'.format(table_name)) 636 | else: 637 | table = CyberList(table_name, self._dp, self._con) 638 | return table 639 | 640 | def print_tables(self): 641 | ''' 642 | Print all tables in the CyberDB database. 643 | 644 | Return Type: None 645 | ''' 646 | @network 647 | def get_tables(self): 648 | return { 649 | 'route': '/print_tables' 650 | } 651 | 652 | r = get_tables(self) 653 | for line in r: 654 | print('table name: {} type name: {}'.format(line[0], line[1])) 655 | 656 | def delete_table(self, table_name: str): 657 | ''' 658 | Drop the table_name table in the CyberDB database. 659 | 660 | Parameters: 661 | 662 | table_name – the name of the table to drop. 663 | 664 | Return Type: None 665 | ''' 666 | if type(table_name) != str: 667 | raise WrongInputCyberDBError('Please use str for the table name.') 668 | 669 | stream = Stream(self._con.s, self._dp) 670 | 671 | client_obj = { 672 | 'route': '/exam_cyberlist', 673 | 'table_name': table_name 674 | } 675 | stream.write(client_obj) 676 | 677 | server_obj = stream.read() 678 | if server_obj['code'] == 0: 679 | raise WrongTableNameCyberDBError( 680 | '{} table does not exist.'.format(table_name)) 681 | 682 | client_obj = { 683 | 'route': '/delete_table', 684 | 'table_name': table_name 685 | } 686 | stream.write(client_obj) 687 | stream.read() 688 | 689 | 690 | class Client: 691 | ''' 692 | The cyberdb.Client object returned by the cyberdb.connect function 693 | is used to generate the Proxy object. 694 | ''' 695 | 696 | def __init__(self, con_pool: ConPool, dp: datas.DataParsing): 697 | self._con_pool = con_pool 698 | self._dp = dp 699 | 700 | def get_proxy(self) -> Proxy: 701 | ''' 702 | Generate a Proxy object. 703 | 704 | Return Type: None 705 | ''' 706 | proxy = Proxy(self._con_pool, self._dp) 707 | return proxy 708 | 709 | 710 | def connect(host: str = '127.0.0.1', port: int = 9980, password: 711 | str = None, encrypt: bool = False, time_out: int = None) -> Client: 712 | ''' 713 | Connect the client to the CyberDB server. 714 | 715 | Parameters: 716 | 717 | host -- the connection address, such as 127.0.0.1 718 | 719 | port -- connection port 720 | 721 | password -- connection password 722 | 723 | encrypt -- Whether to encrypt communication, if the server enables 724 | encrypt to be True, it must be True here, and vice versa. 725 | 726 | time_out -- The timeout for each connection in the connection pool, 727 | in seconds. Connections in the connection pool will be discarded 728 | after time_out seconds of inactivity, and a new connection will be 729 | generated next time. The connection pool will manage the 730 | connections automatically, and the developer does not need to pay 731 | attention to the details. If this parameter is None, there will be 732 | no timeout, and the connection pool will maintain the connection 733 | until it expires, after which a new connection will be regenerated. 734 | 735 | Return Type: Client 736 | ''' 737 | if not password: 738 | raise WrongPasswordCyberDBError('The password cannot be empty.') 739 | if time_out and type(time_out) != int: 740 | raise CyberDBError('time_out must be an integer.') 741 | 742 | # Responsible for encrypting and decrypting objects. 743 | secret = Secret(key=password) 744 | dp = datas.DataParsing(secret, encrypt=encrypt) 745 | con_pool = ConPool(host, port, dp, time_out=time_out) 746 | 747 | # Synchronously test whether the connection is successful. 748 | s = con_pool.get() 749 | confirm_the_connection(s, dp) 750 | con_pool.put(s) 751 | 752 | client = Client(con_pool, dp) 753 | return client 754 | 755 | 756 | def confirm_the_connection(s: socket.socket, dp: datas.DataParsing) -> dict: 757 | ''' 758 | The connection is detected when the database connects for the first 759 | time. 760 | ''' 761 | try: 762 | stream = Stream(s, dp) 763 | 764 | client_obj = { 765 | 'route': '/connect' 766 | } 767 | stream.write(client_obj) 768 | 769 | server_obj = stream.read() 770 | 771 | return { 772 | 'code': 1, 773 | 'content': server_obj 774 | } 775 | except Exception as e: 776 | return { 777 | 'code': 0, 778 | 'Exception': e 779 | } 780 | --------------------------------------------------------------------------------