├── .github └── workflows │ └── pytest.yml ├── .gitignore ├── LICENSE ├── README.md ├── redis_pydict ├── __init__.py ├── const.py ├── install.py ├── pyredisdict.py └── ui │ ├── __init__.py │ ├── debugPrint.py │ └── loadingAnim.py ├── setup.py └── test_redis_pydict.py /.github/workflows/pytest.yml: -------------------------------------------------------------------------------- 1 | name: Pytest Unit Tests 2 | 3 | on: [push] 4 | 5 | jobs: 6 | test: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v3 10 | - name: Set up Python ${{ matrix.python-version }} 11 | uses: actions/setup-python@v3 12 | with: 13 | python-version: ${{ matrix.python-version }} 14 | - name: Install dependencies 15 | run: | 16 | python -m pip install --upgrade pip 17 | pip install pytest 18 | pip install . 19 | - name: Analysing the code with pytest 20 | run: | 21 | pytest -v 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | .idea/ 161 | 162 | # VSCode settings directory 163 | .vscode/ 164 | 165 | # Redis Server specifics 166 | redis_server/ 167 | *.rdb 168 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Narasimha Prasanna HN 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # redis-pydict 2 | A python dictionary that uses Redis as in-memory storage backend to facilitate distributed computing applications development. This library is a small wrapper around Redis providing a dictionary like class that acts as a drop-in replacement for python dict, but stores data in Redis instead of local program memory, thus key-value pairs written into the python dictionary can be used by other programs that are connected to the same Redis instance / cluster anywhere over the network. The library also provides wrapper around Redis Pub-Sub using which processes can publish and wait for events when new keys are added to the dictionary. 3 | 4 | ### Installation 5 | 1. From source: 6 | ```bash 7 | git clone git@github.com:Narasimha1997/redis-pydict.git 8 | cd redis-pydict 9 | pip3 install -e . 10 | ``` 11 | or 12 | 13 | ```bash 14 | pip3 install git+https://github.com/Narasimha1997/redis-pydict#egg=redis-pydict 15 | ``` 16 | 17 | 2. From PIP 18 | ```bash 19 | pip3 install redis-pydict 20 | ``` 21 | 22 | ### Usage 23 | 1. Creating the instance of RedisPyDict 24 | ```python 25 | from redis_pydict import PyRedisDict 26 | 27 | redis_dict = PyRedisDict( 28 | host='localhost', # redis host 29 | port=6379, # redis port 30 | db=0, # redis db (0-15) 31 | password=None, # password if using password auth 32 | namespace="", # namespace: this is the key prefix, every key inserted into the dict will be prefixed with this namespace string when inserting into Redis, this provides some degree of isolation between namespaces 33 | custom=None # use a custom connection, pass a Redis/Redis cluster connection object manually, parameters like `host`, `port`, `db` and `password` will be ignored if this is not None 34 | ) 35 | ``` 36 | 37 | 2. Supported functions: 38 | `PyRedisDict` is a replacement for dict, and it supports most of the dictionary methods: 39 | ``` 40 | >>> dir(redis_dict) 41 | ['__class__', '__contains__', '__delattr__', '__delitem__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__getitem__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__iter__', '__le__', '__len__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__setitem__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_iter_internal', '_iter_items', '_scan_iter', 'clear', 'enable_publish', 'get', 'items', 'items_matching', 'iter_matching', 'key_namespace', 'keys', 'notification_id', 'redis', 'set_notification_id', 'set_notification_mode', 'subscribe_to_key_events', 'to_dict', 'update', 'values'] 42 | ``` 43 | 44 | 3. Publish and subscribe to key set events 45 | `PyRedisDict` can also send key set events when enabled, by default it is not enabled. You can enable it by calling `set_notification_mode(true)` and `set_notification_id(unique_id)`. Every instance of `PyRedisDict` should be set to a unqiue ID to help diffreniciating with each of them. Here is an example 46 | 47 | On the sender side: 48 | 49 | ```python 50 | # enable notifications 51 | redis_dict.set_notification_mode(True) 52 | 53 | # set a unique id 54 | redis_dict.set_notification_id('myprocess1') 55 | 56 | # now this operation will also publish an event which can be captured by subscribers 57 | redis_dict['key'] = "Hello" 58 | ``` 59 | 60 | On receiver side: 61 | ```python 62 | # this will listen for key set events for key 'key' in the given namespace 63 | for (key, notification_id, value) in redis_dict.subscribe_to_key_events(pattern="key"): 64 | print(key, notification_id, value) 65 | 66 | # optionally you can also use a regex pattern, this example listen for all key events matching pattern key* 67 | for (key, notification_id, value) in redis_dict.subscribe_to_key_events(pattern="key*"): 68 | print(key, notification_id, value) 69 | ``` 70 | 71 | ### Examples: 72 | 1. Basic set and get 73 | ```python 74 | def test_basic_get_set(): 75 | d = PyRedisDict(namespace="mykeyspace") 76 | 77 | d.clear() 78 | d['none'] = None 79 | d['int'] = 10 80 | d['float'] = 20.131 81 | d['array'] = [10, 20, 30] 82 | d['hashmap'] = {'name': 'prasanna', 'meta': {'age': 24, 'hobbies': ['gaming']}} 83 | 84 | # assert values 85 | assert d['int'] == 10 86 | assert d['float'] == 20.131 87 | assert d['array'] == [10, 20, 30] 88 | assert (d['hashmap']['name'] == 'prasanna') and (d['hashmap']['meta']['age'] == 24) 89 | assert (d['none'] == None) 90 | 91 | d.clear() 92 | ``` 93 | 94 | 2. Custom objects 95 | ```python 96 | def test_custom_class_get_set(): 97 | d = PyRedisDict(namespace="mykeyspace") 98 | 99 | d.clear() 100 | d['obj'] = CustomClass() 101 | 102 | assert d['obj'].add() == (10 + 20 + 30) 103 | d.clear() 104 | ``` 105 | **Note**: `PyRedisDict` uses `cPickle` for custom objects. 106 | 107 | 3. Iterations 108 | ```python 109 | def test_iterations(): 110 | 111 | d = PyRedisDict(namespace="mykeyspace2") 112 | d.clear() 113 | d['mykey_none'] = None 114 | d['mykey_int'] = 10 115 | d['mykey_float'] = 20.131 116 | d['array'] = [10, 20, 30] 117 | d['hashmap'] = {'name': 'prasanna', 'meta': {'age': 24, 'hobbies': ['gaming']}} 118 | 119 | # assert values 120 | assert len(d) == 5 121 | assert len(d.items()) == 5 122 | 123 | # find by prefix 124 | assert len(d.items_matching('mykey*')) == 3 125 | d.clear() 126 | ``` 127 | 128 | 4. Key deletion 129 | ```python 130 | def test_deletion(): 131 | 132 | d = PyRedisDict(namespace="mykeyspace3") 133 | 134 | d.clear() 135 | d['x1'] = CustomClass() 136 | d['x2'] = CustomClass() 137 | d['x3'] = CustomClass() 138 | d['x4'] = CustomClass() 139 | d['x4'] = CustomClass() 140 | 141 | assert 'x1' in d 142 | del d['x1'] 143 | 144 | assert 'x1' not in d 145 | 146 | d.clear() 147 | assert len(d) == 0 148 | ``` 149 | 150 | 5. Converting to/from dict 151 | ```python 152 | def test_conversions(): 153 | d = PyRedisDict(namespace="mykeyspace") 154 | 155 | d.clear() 156 | 157 | d['mykey_none'] = None 158 | d['mykey_int'] = 10 159 | d['mykey_float'] = 20.131 160 | d['array'] = [10, 20, 30] 161 | d['hashmap'] = {'name': 'prasanna', 'meta': {'age': 24, 'hobbies': ['gaming']}} 162 | 163 | assert len(d.to_dict()) == 5 164 | 165 | normal_dict = {"mykey_1": 10, "mykey_2": 20, "mykey_3": "hello", "array": [50, 60, 70]} 166 | d.update(normal_dict) 167 | 168 | assert len(d) == (5 + 3) # another key is duplicate (array) 169 | assert d['array'] == [50, 60, 70] 170 | d.clear() 171 | ``` 172 | 173 | ### Running tests 174 | This project uses `pytest` for testing. Instal pytest and run `pytest` to validate tests, make sure you set-up Redis locally to test the process. 175 | 176 | ### Contributions 177 | Feel free to try this library, suggest changes, raise issues, send PRs. 178 | -------------------------------------------------------------------------------- /redis_pydict/__init__.py: -------------------------------------------------------------------------------- 1 | from .pyredisdict import PyRedisDict 2 | from .install import RedisServer 3 | -------------------------------------------------------------------------------- /redis_pydict/const.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import json 3 | 4 | NONE_TYPE_MAGIC = b'PyDictNoneType' 5 | 6 | TYPE_MAGIC = { 7 | ord('n'): "NoneType", 8 | ord('i'): "int", 9 | ord('f'): "float", 10 | ord('s'): "str", 11 | ord('d'): "dict", 12 | ord('l'): "list", 13 | ord('c'): "custom", 14 | } 15 | 16 | TYPE_ENCODE_FUNCTIONS = { 17 | None: lambda _: b'n', 18 | int: lambda i: b'i' + str(i).encode('utf-8'), 19 | float: lambda f: b'f' + str(f).encode('utf-8'), 20 | str: lambda s: b's' + str(s).encode("utf-8"), 21 | dict: lambda d: b'd' + json.dumps(d).encode('utf-8'), 22 | list: lambda l: b'l' + json.dumps(l).encode('utf-8'), 23 | "custom": lambda c: b'c' + pickle.dumps(c) 24 | } 25 | 26 | TYPE_DECODE_FUNCTIONS = { 27 | ord('n'): lambda _: None, 28 | ord('i'): lambda i: int(i.decode('utf-8')), 29 | ord('f'): lambda f: float(f.decode('utf-8')), 30 | ord('s'): lambda s: s.decode('utf-8'), 31 | ord('d'): lambda d: json.loads(d), 32 | ord('l'): lambda l: json.loads(l), 33 | ord('c'): lambda c: pickle.loads(c) 34 | } 35 | -------------------------------------------------------------------------------- /redis_pydict/install.py: -------------------------------------------------------------------------------- 1 | """ 2 | Written by Phani Pavan k . 3 | use this code as per gplv3 guidelines. 4 | """ 5 | import pgrep 6 | import re 7 | import shutil 8 | import os 9 | import tarfile 10 | import subprocess as sp 11 | from tqdm.auto import tqdm 12 | import requests as req 13 | from .ui import * 14 | import sys 15 | from threading import Thread 16 | from multiprocessing import Process 17 | import asyncio 18 | import time 19 | REDIS_STABLE_URL = "http://download.redis.io/redis-stable.tar.gz" 20 | 21 | 22 | class RedisServer: 23 | def __init__(self) -> None: 24 | self.proc = None 25 | self.procID = None 26 | 27 | @staticmethod 28 | def _makeLogFolder(): 29 | os.makedirs('redis_server/installLogs', exist_ok=True) 30 | # ? Implement saving logs to this folder. 31 | 32 | @staticmethod 33 | def _downloadRedis(): 34 | os.makedirs('redis_server', exist_ok=True) 35 | lst = os.listdir('./redis_server') 36 | if 'redis-stable.tar.gz' in lst: 37 | debug('Redis Source already found, skipping download', Log.WRN) 38 | else: 39 | debug('Downloading Redis Stable Source') 40 | with req.get(REDIS_STABLE_URL, stream=True) as r: 41 | size = int(r.headers.get('Content-Length')) 42 | with tqdm.wrapattr(r.raw, 'read', total=size, desc='') as data: 43 | with open('redis_server/redis-stable.tar.gz', 'wb') as fil: 44 | shutil.copyfileobj(data, fil) 45 | debug('Download Done', Log.SUC) 46 | 47 | @staticmethod 48 | def _findSource(): 49 | lst = os.listdir('./redis_server') 50 | if 'redis-stable' in lst: 51 | debug('Redis Already Extracted', Log.WRN) 52 | else: 53 | debug('Extracting Source') 54 | tarfile.open( 55 | 'redis_server/redis-stable.tar.gz').extractall('./redis_server/') 56 | debug('Done Extracting Source', Log.SUC) 57 | 58 | @staticmethod 59 | def _build(): 60 | coresToBuild = 1 if os.cpu_count() is not None else os.cpu_count()//3 61 | if 'src' in os.listdir('redis_server/redis-stable') and 'redis-server' in os.listdir('./redis_server/redis-stable/src'): 62 | # os.chdir('redis_server/redis-stable') 63 | debug('Redis Already Built', Log.WRN) 64 | else: 65 | debug('Running Redis Build On ' + 66 | str(coresToBuild+1 if type(os.cpu_count()) is int else 1)+' Cores') 67 | anim = Loader(desc='Building ').start() 68 | os.chdir('redis_server/redis-stable') 69 | x = sp.run(['make -j'+str(coresToBuild+1 if type(os.cpu_count()) is int else 1)], 70 | capture_output=True, text=True, shell=True) 71 | anim.stop() 72 | os.chdir('../../') 73 | debug('Done Building', Log.SUC) 74 | 75 | @staticmethod 76 | def _setupLink(): 77 | if 'redis-server' in os.listdir('./redis_server'): 78 | debug('Redis Already setup', Log.WRN) 79 | else: 80 | debug('Setting up Redis Server') 81 | y = sp.run(['chmod +x ./redis_server/redis-stable/src/redis-server'], 82 | shell=True, text=True, capture_output=True) 83 | # sp.run(['ls']) 84 | y = sp.run(['ln -s ./redis-stable/src/redis-server ./redis_server/redis-server'], 85 | shell=True, text=True, capture_output=True) 86 | y = sp.run(['chmod +x ./redis_server/redis-server'], 87 | shell=True, text=True, capture_output=True) 88 | debug('Redis Set', Log.SUC) 89 | 90 | @staticmethod 91 | def _verify(): 92 | debug('Veryfying Redis Install') 93 | # sp.run(['ls', '-l']) 94 | # sp.run(['pwd']) 95 | z = sp.run(['./redis_server/redis-server --version'], 96 | shell=True, capture_output=True, text=True) 97 | regexp = re.search('Redis server v=(.*) sha', z.stdout) 98 | if regexp.group(0): 99 | debug('Redis Version '+regexp.group(1)+' Found') 100 | debug('Redis working fine', Log.SUC) 101 | return 1 102 | else: 103 | return 0 104 | 105 | def _setProcID(self): 106 | if self.procID == None and self.proc != None: 107 | try: 108 | self.procID = pgrep.pgrep( 109 | r"-f 'redis_server\/redis-server \*\:6379'")[0] 110 | except IndexError as e: 111 | debug("Unable to set PID", Log.ERR) 112 | 113 | def install(self, saveLogs: bool = False, ): 114 | if sys.platform.lower() != 'linux': 115 | debug('OS not Linux, install redis manually', Log.ERR) 116 | debug( 117 | 'Support for intalling on other OSs will be implemented in the future.', Log.WRN) 118 | return 119 | if saveLogs: 120 | RedisServer._makeLogFolder() 121 | RedisServer._downloadRedis() 122 | RedisServer._findSource() 123 | RedisServer._build() 124 | RedisServer._setupLink() 125 | return RedisServer._verify() 126 | 127 | async def _startServer(self, threading: bool = False): 128 | def runServer(): 129 | sp.run('./redis_server/redis-server', 130 | capture_output=True, text=True, shell=False) 131 | 132 | if threading: 133 | self.proc = Thread(target=runServer) 134 | else: 135 | self.proc = Process(target=runServer) 136 | self.proc.start() 137 | while not self.proc.is_alive(): 138 | pass 139 | # self.procID = self.proc.pid 140 | # print('new: {} old: {}'.format(self.procID, pgrep.pgrep( 141 | # r"-f 'redis_server\/redis-server \*\:6379'")[0])) 142 | self._setProcID() 143 | 144 | def startServer(self): 145 | if self.proc is None: 146 | asyncio.run(self._startServer()) 147 | time.sleep(0.5) 148 | else: 149 | debug('Redis Server already running manually', Log.WRN) 150 | 151 | def stopServer(self): 152 | if self.proc is None: 153 | debug('Redis Server not running', Log.WRN) 154 | else: 155 | self._setProcID() 156 | self.proc.terminate() 157 | if self.procID != None: 158 | os.kill(self.procID, 15) 159 | anim = Loader(desc='Stopping Redis Server ').start() 160 | while self.proc.is_alive(): 161 | pass 162 | time.sleep(0.5) 163 | anim.stop() 164 | # if self.procID 165 | debug('Redis Server stopped', Log.SUC) 166 | print('\n') 167 | self.proc = None 168 | self.procID = None 169 | 170 | 171 | if __name__ == "__main__": 172 | a = RedisServer() 173 | a.install() 174 | a.startServer() 175 | print(a.procID) 176 | time.sleep(5) 177 | a.stopServer() 178 | -------------------------------------------------------------------------------- /redis_pydict/pyredisdict.py: -------------------------------------------------------------------------------- 1 | from typing import * 2 | import redis 3 | from .const import NONE_TYPE_MAGIC,\ 4 | TYPE_MAGIC,\ 5 | TYPE_ENCODE_FUNCTIONS,\ 6 | TYPE_DECODE_FUNCTIONS 7 | 8 | 9 | class DataFunctions: 10 | 11 | @staticmethod 12 | def encode(data: Any): 13 | type_ = type(data) 14 | if type_ not in TYPE_ENCODE_FUNCTIONS: 15 | return TYPE_ENCODE_FUNCTIONS["custom"](data) 16 | return TYPE_ENCODE_FUNCTIONS[type_](data) 17 | 18 | @staticmethod 19 | def decode(data: str): 20 | type_magic = data[0] 21 | 22 | rest_of_data = data[1:] 23 | return TYPE_DECODE_FUNCTIONS[type_magic](rest_of_data) 24 | 25 | @staticmethod 26 | def define_key(namespace: str, key: Any): 27 | return namespace + "&&" + str(key) 28 | 29 | @staticmethod 30 | def unpack_key(key: bytes): 31 | key_string = key.decode('utf-8') 32 | return '&&'.join(key_string.split("&&")[1:]) 33 | 34 | 35 | class PyRedisDict: 36 | 37 | def __init__(self, 38 | host: str = 'localhost', 39 | port: int = 6379, 40 | db: int = 0, 41 | password: str = None, 42 | namespace: str = "", 43 | custom: Any = None): 44 | self.redis = custom if custom else redis.Redis( 45 | host=host, port=port, db=db, password=password) 46 | 47 | self.key_namespace = namespace 48 | self.enable_publish = False 49 | self.notification_id = "" 50 | 51 | def set_notification_mode(self, enable: bool): 52 | self.enable_publish = enable 53 | 54 | def set_notification_id(self, note_id: str): 55 | self.notification_id = note_id 56 | 57 | def _scan_iter(self, pattern: Any): 58 | formatted_pattern = DataFunctions.define_key(self.key_namespace, 59 | pattern) 60 | return self.redis.scan_iter(formatted_pattern) 61 | 62 | # basic GET and SET functions 63 | 64 | def get(self, key: Any, default: Any): 65 | key = DataFunctions.define_key(self.key_namespace, key) 66 | data = self.redis.get(key) 67 | if not data: 68 | return default 69 | return DataFunctions.decode(data) 70 | 71 | def __setitem__(self, key: Any, value: Any): 72 | key = DataFunctions.define_key(self.key_namespace, key) 73 | data = DataFunctions.encode(value) 74 | self.redis.set(key, data) 75 | 76 | # push notification if enabled 77 | if self.enable_publish: 78 | self.redis.publish("event_" + key, self.notification_id) 79 | 80 | def __getitem__(self, key: Any): 81 | key_formatted = DataFunctions.define_key(self.key_namespace, key) 82 | data = self.redis.get(key_formatted) 83 | # print(data) 84 | if not data: 85 | raise KeyError(key) 86 | return DataFunctions.decode(data) 87 | 88 | def __delitem__(self, key: Any): 89 | key_formatted = DataFunctions.define_key(self.key_namespace, key) 90 | return_value = self.redis.delete(key_formatted) 91 | if return_value == 0: 92 | raise KeyError(key) 93 | 94 | def __contains__(self, key: Any): 95 | key_formatted = DataFunctions.define_key(self.key_namespace, key) 96 | return self.redis.exists(key_formatted) 97 | 98 | # iteration functions 99 | 100 | def _iter_internal(self, pattern: Any): 101 | scan_iter = self._scan_iter(pattern) 102 | keys = (DataFunctions.unpack_key(key) for key in scan_iter) 103 | return keys 104 | 105 | def _iter_items(self, pattern: Any): 106 | return [(key, self[key]) for key in self._iter_internal(pattern)] 107 | 108 | def __iter__(self): 109 | return self._iter_internal('*') 110 | 111 | def items(self): 112 | return self._iter_items('*') 113 | 114 | def iter_matching(self, pattern: Any = '*'): 115 | return self._iter_internal(pattern) 116 | 117 | def items_matching(self, pattern: Any = '*'): 118 | return self._iter_items(pattern) 119 | 120 | # mulit-data operations 121 | def keys(self): 122 | return self._iter_items('*') 123 | 124 | def clear(self): 125 | keys = list(self._scan_iter('*')) 126 | if len(keys) != 0: 127 | self.redis.delete(*keys) 128 | 129 | def values(self): 130 | keys = list(self._scan_iter('*')) 131 | return [ 132 | DataFunctions.decode(value) for value in self.redis.mget(*keys) 133 | ] 134 | 135 | def update(self, from_dic: Any): 136 | pipeline = self.redis.pipeline() 137 | for key, value in from_dic.items(): 138 | formatted_key, encoded_value = DataFunctions.define_key( 139 | self.key_namespace, key), DataFunctions.encode(value) 140 | pipeline.set(formatted_key, encoded_value) 141 | pipeline.execute() 142 | 143 | def to_dict(self): 144 | return {key: value for key, value in self.items()} 145 | 146 | def __len__(self): 147 | return len(list(self._scan_iter('*'))) 148 | 149 | def __str__(self) -> str: 150 | return str(self.to_dict()) 151 | 152 | def subscribe_to_key_events(self, pattern: Any = "*"): 153 | channel_name = DataFunctions.define_key("event_" + self.key_namespace, 154 | pattern) 155 | pusub = self.redis.pubsub() 156 | pusub.subscribe(channel_name) 157 | 158 | for message in pusub.listen(): 159 | if message and type( 160 | message) == dict and message['type'] == 'message': 161 | notification_id = message['data'].decode('utf-8') 162 | channel_complete_name = message['channel'].decode('utf-8') 163 | if not channel_complete_name.startswith('event_'): 164 | pass 165 | 166 | channel = channel_complete_name[6:] 167 | channel = '&&'.join(channel.split('&&')[1:]) 168 | value = self[channel] 169 | 170 | yield (channel, notification_id, value) 171 | -------------------------------------------------------------------------------- /redis_pydict/ui/__init__.py: -------------------------------------------------------------------------------- 1 | from .debugPrint import Log, debug 2 | from .loadingAnim import Loader 3 | -------------------------------------------------------------------------------- /redis_pydict/ui/debugPrint.py: -------------------------------------------------------------------------------- 1 | """ 2 | This program is free software; you can redistribute it and/or modify 3 | it under the terms of the GNU General Public License as published by 4 | the Free Software Foundation; either version 3 of the License, or 5 | (at your option) any later version. 6 | 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | 12 | Written by Phani Pavan K 13 | Use this code as per GPLv3 guidelines 14 | """ 15 | 16 | 17 | import enum 18 | 19 | 20 | class bcolors: 21 | # bcolors class taken from https://svn.blender.org/svnroot/bf-blender/trunk/blender/build_files/scons/tools/bcolors.py 22 | HEADER = '\033[95m' 23 | OKBLUE = '\033[94m' 24 | OKCYAN = '\033[96m' 25 | OKGREEN = '\033[92m' 26 | WARNING = '\033[93m' 27 | FAIL = '\033[91m' 28 | ENDC = '\033[0m' 29 | 30 | 31 | class Log(enum.Enum): 32 | SUC = 0 33 | INF = 1 34 | WRN = 2 35 | ERR = 3 36 | 37 | 38 | def debug(comment: str, level: Log = Log.INF): 39 | if level == Log.INF: 40 | print(f'{bcolors.OKBLUE}[i]', comment, bcolors.ENDC) 41 | elif level == Log.WRN: 42 | print(f'{bcolors.WARNING}[!]', comment, bcolors.ENDC) 43 | elif level == Log.ERR: 44 | print(f'{bcolors.FAIL}[x]', comment, bcolors.ENDC) 45 | elif level == Log.SUC: 46 | print(f'{bcolors.OKGREEN}[✓]', comment, bcolors.ENDC) 47 | 48 | 49 | if __name__ == "__main__": 50 | 51 | debug('info') 52 | debug('warn', Log.WRN) 53 | debug('error', Log.ERR) 54 | debug('success', Log.SUC) 55 | -------------------------------------------------------------------------------- /redis_pydict/ui/loadingAnim.py: -------------------------------------------------------------------------------- 1 | """ 2 | This class is taken from the following answer from StackOverflow 3 | https://stackoverflow.com/a/66558182/12580609 4 | 5 | Written by phani pavan k . 6 | use this code as per gplv3 guidelines. 7 | """ 8 | 9 | 10 | from itertools import cycle 11 | from shutil import get_terminal_size 12 | from threading import Thread 13 | from time import sleep 14 | from types import DynamicClassAttribute 15 | 16 | 17 | class Loader: 18 | def __init__(self, desc: str = "Loading...", end: str = "Done!", timeout: int = 0.1): 19 | """ 20 | A loader-like context manager 21 | 22 | Args: 23 | desc (str, optional): The loader's description. Defaults to "Loading...". 24 | end (str, optional): Final print. Defaults to "Done!". 25 | timeout (float, optional): Sleep time between prints. Defaults to 0.1. 26 | """ 27 | self.desc = desc 28 | self.end = end 29 | self.timeout = timeout 30 | 31 | self._thread = Thread(target=self._animate, daemon=True) 32 | # self.steps = ["⢿", "⡿", "⣟", "⣻", "⣽", "⣯", "⣷", "⣾", "⣽", "⣻"] 33 | self.steps = ["⢿⣿", "⡿⣿", "⣿⢿", "⣿⡿", "⣿⣟", "⣿⣻", "⣟⣿", "⣻⣿", 34 | "⣽⣿", "⣯⣿", "⣿⣽", "⣿⣯", "⣿⣷", "⣿⣾", "⣷⣿", "⣾⣿", "⣽⣿", "⣻⣿"] 35 | self.done = False 36 | 37 | def start(self): 38 | self._thread.start() 39 | return self 40 | 41 | def _animate(self): 42 | for c in cycle(self.steps): 43 | if self.done: 44 | break 45 | print(f"\r{self.desc} {c}", flush=True, end="") 46 | sleep(self.timeout) 47 | 48 | def __enter__(self): 49 | self.start() 50 | 51 | def stop(self): 52 | self.done = True 53 | cols = get_terminal_size((80, 20)).columns 54 | print("\r" + "" * cols, end="", flush=True) 55 | # print(f"\r{self.end}", flush=True) 56 | 57 | def __exit__(self, **kwargs: DynamicClassAttribute): 58 | # handle exceptions with those variables ^ 59 | self.stop() 60 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | long_description = open('README.md').read() 4 | 5 | setup( 6 | name='redis-pydict', 7 | version='0.2', 8 | author='Narasimha Prasanna HN', 9 | author_email='narasimhaprasannahn@gmail.com', 10 | url='https://github.com/Narasimha1997/redis-pydict', 11 | description="A python dictionary that uses Redis as in-memory storage backend to facilitate distributed computing applications development.", 12 | long_description=long_description, 13 | long_description_content_type="text/markdown", 14 | license='MIT', 15 | packages=find_packages(), 16 | classifiers=[ 17 | "Programming Language :: Python :: 3", 18 | "License :: OSI Approved :: MIT License", 19 | "Operating System :: OS Independent", 20 | ], 21 | keywords='python redis distributed-computing dict software-dev', 22 | zip_safe=False, 23 | install_requires=["redis", "pgrep", "tqdm", 'requests']) 24 | -------------------------------------------------------------------------------- /test_redis_pydict.py: -------------------------------------------------------------------------------- 1 | from redis_pydict import PyRedisDict, RedisServer 2 | 3 | 4 | class CustomClass: 5 | 6 | def __init__(self) -> None: 7 | self.x = 10 8 | self.y = 20 9 | self.z = 30 10 | 11 | def add(self): 12 | return self.x + self.y + self.z 13 | 14 | 15 | def test_install_in_local_dir(): 16 | serv = RedisServer() 17 | if not serv.install(): 18 | raise Exception('Server Install Failed') 19 | 20 | 21 | def test_basic_get_set(): 22 | serv = RedisServer() 23 | serv.startServer() 24 | d = PyRedisDict(namespace="mykeyspace") 25 | 26 | d.clear() 27 | d['none'] = None 28 | d['int'] = 10 29 | d['float'] = 20.131 30 | d['array'] = [10, 20, 30] 31 | d['hashmap'] = {'name': 'prasanna', 'meta': { 32 | 'age': 24, 'hobbies': ['gaming']}} 33 | 34 | # assert values 35 | assert d['int'] == 10 36 | assert d['float'] == 20.131 37 | assert d['array'] == [10, 20, 30] 38 | assert (d['hashmap']['name'] == 'prasanna') and ( 39 | d['hashmap']['meta']['age'] == 24) 40 | assert (d['none'] is None) 41 | 42 | d.clear() 43 | serv.stopServer() 44 | 45 | 46 | def test_custom_class_get_set(): 47 | serv = RedisServer() 48 | serv.startServer() 49 | d = PyRedisDict(namespace="mykeyspace") 50 | 51 | d.clear() 52 | d['obj'] = CustomClass() 53 | 54 | assert d['obj'].add() == (10 + 20 + 30) 55 | d.clear() 56 | serv.stopServer() 57 | 58 | 59 | def test_iterations(): 60 | serv = RedisServer() 61 | serv.startServer() 62 | d = PyRedisDict(namespace="mykeyspace2") 63 | d.clear() 64 | d['mykey_none'] = None 65 | d['mykey_int'] = 10 66 | d['mykey_float'] = 20.131 67 | d['array'] = [10, 20, 30] 68 | d['hashmap'] = {'name': 'prasanna', 'meta': { 69 | 'age': 24, 'hobbies': ['gaming']}} 70 | 71 | # assert values 72 | assert len(d) == 5 73 | assert len(d.items()) == 5 74 | 75 | # find by prefix 76 | assert len(d.items_matching('mykey*')) == 3 77 | d.clear() 78 | serv.stopServer() 79 | 80 | 81 | def test_deletion(): 82 | serv = RedisServer() 83 | serv.startServer() 84 | d = PyRedisDict(namespace="mykeyspace3") 85 | 86 | d.clear() 87 | d['x1'] = CustomClass() 88 | d['x2'] = CustomClass() 89 | d['x3'] = CustomClass() 90 | d['x4'] = CustomClass() 91 | d['x4'] = CustomClass() 92 | 93 | assert 'x1' in d 94 | del d['x1'] 95 | 96 | assert 'x1' not in d 97 | 98 | d.clear() 99 | assert len(d) == 0 100 | serv.stopServer() 101 | 102 | 103 | def test_conversions(): 104 | serv = RedisServer() 105 | serv.startServer() 106 | d = PyRedisDict(namespace="mykeyspace") 107 | 108 | d.clear() 109 | 110 | d['mykey_none'] = None 111 | d['mykey_int'] = 10 112 | d['mykey_float'] = 20.131 113 | d['array'] = [10, 20, 30] 114 | d['hashmap'] = {'name': 'prasanna', 'meta': { 115 | 'age': 24, 'hobbies': ['gaming']}} 116 | 117 | assert len(d.to_dict()) == 5 118 | 119 | normal_dict = {"mykey_1": 10, "mykey_2": 20, 120 | "mykey_3": "hello", "array": [50, 60, 70]} 121 | d.update(normal_dict) 122 | 123 | assert len(d) == (5 + 3) # another key is duplicate (array) 124 | assert d['array'] == [50, 60, 70] 125 | d.clear() 126 | serv.stopServer() 127 | 128 | 129 | def test_ints_as_keys(): 130 | serv = RedisServer() 131 | serv.startServer() 132 | d = PyRedisDict(namespace="mykeyspace4") 133 | d.clear() 134 | 135 | # testing string because testing 136 | d['stringKey'] = 'data' 137 | 138 | # using integer as a key 139 | for integer in range(0, 10): 140 | d[integer] = 'data'+str(integer) 141 | d.clear() 142 | serv.stopServer() 143 | # TODO: implement type encoding for key as well, but save it as string instead of bytes. 144 | --------------------------------------------------------------------------------