├── .bumpversion.cfg ├── .coveragerc ├── .flake8 ├── .gitignore ├── .gitmodules ├── Bakefile ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── dbdaora ├── __init__.py ├── _tests │ ├── test_unit_cache.py │ └── test_unit_repository_subclass_validation.py ├── boolean │ ├── __init__.py │ ├── _tests │ │ ├── datastore │ │ │ ├── conftest.py │ │ │ ├── test_integration_service_boolean_aioredis_datastore_add.py │ │ │ ├── test_integration_service_boolean_aioredis_datastore_delete.py │ │ │ ├── test_integration_service_boolean_aioredis_datastore_get_many.py │ │ │ └── test_integration_service_boolean_aioredis_datastore_get_one.py │ │ ├── test_integration_service_boolean_aioredis_add.py │ │ ├── test_integration_service_boolean_aioredis_delete.py │ │ ├── test_integration_service_boolean_aioredis_get_many.py │ │ ├── test_integration_service_boolean_aioredis_get_one.py │ │ └── test_unit_boolean_service.py │ ├── conftest.py │ ├── factory.py │ ├── repositories │ │ ├── __init__.py │ │ ├── _tests │ │ │ ├── test_integration_repository_aioredis_boolean_get.py │ │ │ ├── test_integration_repository_aioredis_boolean_get_many.py │ │ │ └── test_integration_repository_boolean_datastore.py │ │ └── datastore.py │ └── service.py ├── cache.py ├── circuitbreaker.py ├── conftest.py ├── data_sources │ ├── __init__.py │ ├── fallback │ │ ├── __init__.py │ │ ├── datastore.py │ │ ├── dict.py │ │ └── mongodb.py │ └── memory │ │ ├── __init__.py │ │ ├── aioredis.py │ │ └── dict.py ├── entity.py ├── exceptions.py ├── geospatial │ ├── __init__.py │ ├── _tests │ │ ├── datastore │ │ │ ├── conftest.py │ │ │ ├── test_integration_service_geospatial_aioredis_datastore_add.py │ │ │ └── test_integration_service_geospatial_aioredis_datastore_get_one.py │ │ ├── mongodb │ │ │ ├── conftest.py │ │ │ ├── test_integration_service_geospatial_aioredis_mongodb_add.py │ │ │ └── test_integration_service_geospatial_aioredis_mongodb_get_one.py │ │ ├── test_integration_service_geospatial_aioredis_add.py │ │ └── test_integration_service_geospatial_aioredis_get_one.py │ ├── conftest.py │ ├── entity.py │ ├── factory.py │ ├── query.py │ ├── repositories │ │ ├── __init__.py │ │ ├── _tests │ │ │ ├── test_integration_repository_geospatial_aioredis_datastore.py │ │ │ └── test_integration_repository_geospatial_aioredis_get.py │ │ ├── datastore.py │ │ └── mongodb.py │ └── service │ │ ├── __init__.py │ │ ├── datastore.py │ │ └── mongodb.py ├── hash │ ├── __init__.py │ ├── _tests │ │ ├── conftest.py │ │ ├── datastore │ │ │ ├── conftest.py │ │ │ ├── test_integration_service_hash_aioredis_datastore_add.py │ │ │ ├── test_integration_service_hash_aioredis_datastore_delete.py │ │ │ ├── test_integration_service_hash_aioredis_datastore_get_many.py │ │ │ └── test_integration_service_hash_aioredis_datastore_get_one.py │ │ ├── mongodb │ │ │ ├── conftest.py │ │ │ ├── test_integration_service_hash_aioredis_mongodb_add.py │ │ │ ├── test_integration_service_hash_aioredis_mongodb_delete.py │ │ │ ├── test_integration_service_hash_aioredis_mongodb_get_many.py │ │ │ ├── test_integration_service_hash_aioredis_mongodb_get_many_composed_key.py │ │ │ └── test_integration_service_hash_aioredis_mongodb_get_one.py │ │ ├── test_integration_service_hash_aioredis_add.py │ │ ├── test_integration_service_hash_aioredis_delete.py │ │ ├── test_integration_service_hash_aioredis_exists.py │ │ ├── test_integration_service_hash_aioredis_get_many.py │ │ ├── test_integration_service_hash_aioredis_get_one.py │ │ └── test_unit_hash_service.py │ ├── conftest.py │ ├── factory.py │ ├── query.py │ ├── repositories │ │ ├── __init__.py │ │ ├── _tests │ │ │ ├── test_integration_repository_aioredis_hash_get.py │ │ │ ├── test_integration_repository_aioredis_hash_get_fields.py │ │ │ ├── test_integration_repository_aioredis_hash_get_many.py │ │ │ ├── test_integration_repository_aioredis_hash_get_many_fields.py │ │ │ ├── test_integration_repository_hash_datastore.py │ │ │ ├── test_unit_repository_dict_hash_get.py │ │ │ └── test_unit_repository_dict_hash_get_fields.py │ │ ├── datastore.py │ │ └── mongodb.py │ └── service │ │ ├── __init__.py │ │ ├── datastore.py │ │ └── mongodb.py ├── hashring.py ├── keys.py ├── py.typed ├── query.py ├── repository │ ├── __init__.py │ └── datastore.py ├── service │ ├── __init__.py │ └── builder.py └── sorted_set │ ├── __init__.py │ ├── conftest.py │ ├── entity.py │ ├── factory.py │ ├── query.py │ ├── repositories │ ├── __init__.py │ ├── _tests │ │ ├── conftest.py │ │ ├── test_integration_repository_sorted_set_datastore.py │ │ ├── test_integration_repository_sorted_set_mongodb.py │ │ ├── test_unit_repository_sorted_set_add.py │ │ ├── test_unit_repository_sorted_set_add_typed_dict.py │ │ └── test_unit_repository_sorted_set_get.py │ ├── datastore.py │ └── mongodb.py │ └── service │ ├── __init__.py │ ├── _tests │ ├── conftest.py │ ├── datastore │ │ ├── conftest.py │ │ ├── test_integration_service_sorted_set_aioredis_datastore_add.py │ │ └── test_integration_service_sorted_set_aioredis_datastore_get_one.py │ ├── mongodb │ │ ├── conftest.py │ │ ├── test_integration_service_sorted_set_aioredis_mongodb_add.py │ │ └── test_integration_service_sorted_set_aioredis_mongodb_get_one.py │ ├── test_integration_service_sorted_set_aioredis_add.py │ ├── test_integration_service_sorted_set_aioredis_get_one.py │ ├── test_integration_service_sorted_set_aioredis_get_one_max_min_score.py │ ├── test_integration_service_sorted_set_aioredis_get_one_max_min_score_cache.py │ ├── test_integration_service_sorted_set_aioredis_get_one_max_min_score_fallback.py │ ├── test_integration_service_sorted_set_aioredis_get_one_pagination.py │ └── test_integration_service_sorted_set_aioredis_get_one_pagination_fallback.py │ ├── datastore.py │ └── mongodb.py ├── docs ├── changelog.md ├── contributing.md ├── dbdaora-white.svg ├── dbdaora.svg ├── domain-model │ └── simple.md ├── favicon.png ├── index.md ├── src │ ├── __init__.py │ ├── domain_model │ │ ├── __init__.py │ │ ├── simple.output │ │ ├── simple.py │ │ └── tests.bats │ ├── index │ │ ├── __init__.py │ │ ├── simple_hash.output │ │ ├── simple_hash.py │ │ ├── simple_service.output │ │ ├── simple_service.py │ │ ├── simple_sorted_set.output │ │ ├── simple_sorted_set.py │ │ └── tests.bats │ ├── test.sh │ ├── tests.bats │ ├── typed_dict.output │ └── typed_dict.py ├── stylesheets │ ├── codehilite.css │ └── dark_theme.css └── typed-dict.md ├── mkdocs.yml ├── mypy.ini ├── pyproject.toml ├── stubs ├── aioredis │ ├── __init__.pyi │ └── commands │ │ ├── __init__.pyi │ │ └── transaction.pyi ├── bson │ ├── __init__.pyi │ └── objectid.pyi ├── cachetools │ └── __init__.pyi ├── circuitbreaker │ └── __init__.pyi ├── google │ ├── __init__.pyi │ └── cloud │ │ ├── __init__.pyi │ │ └── datastore │ │ └── __init__.pyi ├── motor │ ├── __init__.pyi │ └── motor_asyncio.pyi ├── newrelic │ ├── __init__.pyi │ └── agent.pyi └── pymongo │ ├── __init__.pyi │ └── errors.pyi └── theme ├── main.html ├── manifest.json ├── open-book-2x.png ├── open-book-3x.png └── open-book-4x.png /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | commit = True 3 | current_version = 0.27.0 4 | tag = True 5 | tag_name = {new_version} 6 | 7 | [bumpversion:file:dbdaora/__init__.py] 8 | 9 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = 3 | dbdaora 4 | omit = 5 | */test_*.py 6 | *conftest.py 7 | *venv* 8 | dbdaora/__init__.py 9 | 10 | [report] 11 | show_missing = True 12 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 100 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # sublime 107 | *.sublime-project 108 | *.sublime-workspace 109 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "devtools"] 2 | path = devtools 3 | url = git@github.com:dutradda/devtools.git 4 | -------------------------------------------------------------------------------- /Bakefile: -------------------------------------------------------------------------------- 1 | include devtools/python/Bakefile devtools/common/Bakefile 2 | 3 | export MYPYPATH=./stubs 4 | export PYTHONPATH=. 5 | export PROJECT_NAME=dbdaora 6 | 7 | deploy: //check-virtualenv @confirm:secure deploy-docs release-pypi push-release 8 | 9 | setup-dbdaora: //check-virtualenv 10 | pip install --force-reinstall git+https://github.com/pycqa/pyflakes 11 | 12 | integration: check-code tests coverage 13 | 14 | tests-code: //check-virtualenv 15 | export DATASTORE_EMULATOR_HOST=localhost:8085 16 | export DATASTORE_PROJECT_ID=dbdaora 17 | export GOOGLE_CLOUD_PROJECT=dbdaora 18 | coverage run -p -m pytest -xvv --disable-warnings ${PROJECT_NAME} docs/src 19 | 20 | tests-query: 21 | export DATASTORE_EMULATOR_HOST=localhost:8085 22 | export DATASTORE_PROJECT_ID=dbdaora 23 | export GOOGLE_CLOUD_PROJECT=dbdaora 24 | coverage run -p -m pytest -xvv -k "${q}" --disable-warnings ${PROJECT_NAME} docs/src 25 | 26 | isort: //check-virtualenv 27 | isort --recursive --apply ${PROJECT_NAME} docs/src stubs 28 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Starting Development 2 | 3 | ```bash 4 | git clone git@github.com:dutradda/dbdaora.git --recursive 5 | cd dbdaora 6 | make setup-python-virtualenv 7 | source venv/bin/activate 8 | make setup-python-project 9 | bake setup-dbdaora 10 | bake dependencies 11 | ``` 12 | 13 | ## Running the integration suite: 14 | 15 | ```bash 16 | bake integration 17 | ``` 18 | 19 | ## Other bake tasks: 20 | 21 | ```bash 22 | bake check-code 23 | 24 | bake tests-docs 25 | 26 | bake serve-docs 27 | 28 | bake add-changelog m="Add my cool feature" 29 | ``` 30 | 31 | You can run `bake` to see all tasks available. 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Diogo Dutra 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md LICENSE NOTICE HISTORY.md pytest.ini requirements.txt Pipfile Pipfile.lock 2 | recursive-include tests *.py 3 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include devtools/common/Makefile devtools/python/Makefile 2 | 3 | export PYTHONPATH = . 4 | export PROJECT_ROOT = . 5 | export PROJECT_NAME = dbdaora 6 | export PYTHON_VERSION = 3.8 7 | -------------------------------------------------------------------------------- /dbdaora/_tests/test_unit_cache.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import pytest 4 | 5 | from dbdaora import TTLDaoraCache 6 | 7 | 8 | @pytest.fixture 9 | def cache(): 10 | return TTLDaoraCache(maxsize=2, ttl=2, ttl_failure_threshold=1) 11 | 12 | 13 | def test_should_set_and_get_data(cache): 14 | cache['fake'] = 'faked' 15 | assert cache.get('fake') == 'faked' 16 | 17 | 18 | def test_should_not_get_data(cache): 19 | assert cache.get('fake') is None 20 | 21 | 22 | def test_should_not_get_data_after_expired(cache): 23 | cache['fake'] = 'faked' 24 | 25 | assert cache.get('fake') == 'faked' 26 | 27 | time.sleep(2) 28 | 29 | assert cache.get('fake') == 'faked' 30 | assert cache.get('fake') is None 31 | 32 | 33 | def test_should_not_set_data_when_reach_maxsize(cache): 34 | cache['fake'] = 'faked' 35 | cache['fake2'] = 'faked2' 36 | cache['fake3'] = 'faked3' 37 | 38 | assert cache.get('fake') == 'faked' 39 | assert cache.get('fake2') == 'faked2' 40 | assert cache.get('fake3') is None 41 | 42 | 43 | def test_should_set_data_when_reach_maxsize_and_expires_some_key(cache): 44 | cache['fake'] = 'faked' 45 | cache['fake2'] = 'faked2' 46 | 47 | time.sleep(2) 48 | 49 | cache['fake3'] = 'faked3' 50 | 51 | assert cache.get('fake') is None 52 | assert cache.get('fake2') == 'faked2' 53 | assert cache.get('fake3') == 'faked3' 54 | -------------------------------------------------------------------------------- /dbdaora/_tests/test_unit_repository_subclass_validation.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from dbdaora.exceptions import RequiredClassAttributeError 4 | from dbdaora.repository import MemoryRepository 5 | 6 | 7 | def test_should_raise_validation_error_without_attributes(): 8 | with pytest.raises(RequiredClassAttributeError) as exc_info: 9 | 10 | class FakeRepository(MemoryRepository): 11 | ... 12 | 13 | assert exc_info.value.args == ( 14 | 'FakeRepository', 15 | 'entity_cls or get_entity_type', 16 | ) 17 | 18 | 19 | def test_should_create_class_with_entity_cls(): 20 | class FakeRepository(MemoryRepository): 21 | entity_cls = dict 22 | 23 | assert FakeRepository 24 | 25 | 26 | def test_should_create_class_with_get_entity_type(): 27 | class FakeRepository(MemoryRepository): 28 | def get_entity_type(self): 29 | ... 30 | 31 | assert FakeRepository 32 | -------------------------------------------------------------------------------- /dbdaora/boolean/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dutradda/dbdaora/5c87a3818e9d736bbf5e1438edc5929a2f5acd3f/dbdaora/boolean/__init__.py -------------------------------------------------------------------------------- /dbdaora/boolean/_tests/datastore/conftest.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | 3 | import pytest 4 | 5 | from dbdaora import DatastoreBooleanRepository, DatastoreDataSource 6 | 7 | 8 | @pytest.fixture 9 | def fallback_data_source(): 10 | return DatastoreDataSource() 11 | 12 | 13 | @dataclasses.dataclass 14 | class FakeEntity: 15 | id: str 16 | other_id: str 17 | 18 | 19 | @pytest.fixture 20 | def fake_entity_cls(): 21 | return FakeEntity 22 | 23 | 24 | @pytest.fixture 25 | def fake_entity(): 26 | return FakeEntity(id='fake', other_id='other_fake') 27 | 28 | 29 | @pytest.fixture 30 | def fake_entity2(): 31 | return FakeEntity(id='fake2', other_id='other_fake') 32 | 33 | 34 | @pytest.fixture 35 | def fake_entity3(): 36 | return FakeEntity(id='fake3', other_id='other_fake') 37 | 38 | 39 | @pytest.fixture 40 | def fake_boolean_repository_cls(): 41 | class FakeRepository(DatastoreBooleanRepository, entity_cls=FakeEntity): 42 | key_attrs = ('other_id', 'id') 43 | 44 | return FakeRepository 45 | -------------------------------------------------------------------------------- /dbdaora/boolean/_tests/datastore/test_integration_service_boolean_aioredis_datastore_add.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from aioredis import RedisError 4 | 5 | 6 | @pytest.fixture 7 | def has_add_cb(): 8 | return True 9 | 10 | 11 | @pytest.mark.asyncio 12 | async def test_should_add(fake_service, fake_entity): 13 | await fake_service.repository.add(fake_entity) 14 | 15 | entity = await fake_service.get_one('fake', other_id='other_fake') 16 | 17 | assert entity == fake_entity.id 18 | 19 | 20 | @pytest.mark.asyncio 21 | async def test_should_add_to_fallback_after_open_circuit_breaker( 22 | fake_service, fake_entity 23 | ): 24 | fake_get = asynctest.CoroutineMock(side_effect=RedisError) 25 | fake_service.repository.memory_data_source.get = fake_get 26 | await fake_service.add(fake_entity) 27 | 28 | entity = await fake_service.get_one('fake', other_id='other_fake') 29 | 30 | assert entity == fake_entity.id 31 | assert fake_service.logger.warning.call_count == 1 32 | -------------------------------------------------------------------------------- /dbdaora/boolean/_tests/datastore/test_integration_service_boolean_aioredis_datastore_delete.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from aioredis import RedisError 4 | 5 | from dbdaora.exceptions import EntityNotFoundError 6 | 7 | 8 | @pytest.fixture 9 | def has_delete_cb(): 10 | return True 11 | 12 | 13 | @pytest.mark.asyncio 14 | async def test_should_delete( 15 | fake_service, serialized_fake_entity, fake_entity 16 | ): 17 | await fake_service.add(fake_entity) 18 | 19 | assert await fake_service.get_one('fake', other_id='other_fake') 20 | 21 | await fake_service.delete(fake_entity.id, other_id='other_fake') 22 | fake_service.cache.clear() 23 | 24 | with pytest.raises(EntityNotFoundError): 25 | await fake_service.get_one('fake', other_id='other_fake') 26 | 27 | 28 | @pytest.mark.asyncio 29 | async def test_should_delete_from_fallback_after_open_circuit_breaker( 30 | fake_service, serialized_fake_entity, fake_entity, mocker 31 | ): 32 | await fake_service.repository.memory_data_source.delete( 33 | 'fake:other_fake:fake' 34 | ) 35 | await fake_service.repository.memory_data_source.delete( 36 | 'fake:not-found:other_fake:fake' 37 | ) 38 | key = fake_service.repository.fallback_data_source.make_key( 39 | 'fake', 'other_fake', 'fake' 40 | ) 41 | await fake_service.repository.fallback_data_source.put( 42 | key, {'value': True} 43 | ) 44 | 45 | assert await fake_service.get_one('fake', other_id='other_fake') 46 | 47 | fake_service.repository.memory_data_source.set = asynctest.CoroutineMock( 48 | side_effect=RedisError 49 | ) 50 | 51 | await fake_service.delete(fake_entity.id, other_id='other_fake') 52 | fake_service.cache.clear() 53 | 54 | with pytest.raises(EntityNotFoundError): 55 | await fake_service.get_one('fake', other_id='other_fake') 56 | 57 | assert fake_service.logger.warning.call_count == 2 58 | -------------------------------------------------------------------------------- /dbdaora/boolean/_tests/datastore/test_integration_service_boolean_aioredis_datastore_get_one.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from aioredis import RedisError 4 | 5 | 6 | @pytest.mark.asyncio 7 | async def test_should_get_one( 8 | fake_service, serialized_fake_entity, fake_entity 9 | ): 10 | await fake_service.repository.memory_data_source.set( 11 | 'fake:other_fake:fake', serialized_fake_entity, 12 | ) 13 | 14 | entity = await fake_service.get_one('fake', other_id='other_fake') 15 | 16 | assert entity == fake_entity.id 17 | 18 | 19 | @pytest.mark.asyncio 20 | async def test_should_get_one_with_fields( 21 | fake_service, serialized_fake_entity, fake_entity 22 | ): 23 | await fake_service.repository.memory_data_source.set( 24 | 'fake:other_fake:fake', serialized_fake_entity, 25 | ) 26 | fake_entity.number = None 27 | fake_entity.boolean = False 28 | 29 | entity = await fake_service.get_one( 30 | 'fake', 31 | fields=['id', 'other_id', 'integer', 'inner_entities'], 32 | other_id='other_fake', 33 | ) 34 | 35 | assert entity == fake_entity.id 36 | 37 | 38 | @pytest.mark.asyncio 39 | async def test_should_get_one_from_cache( 40 | fake_service, serialized_fake_entity, fake_entity 41 | ): 42 | fake_service.repository.memory_data_source.get = asynctest.CoroutineMock() 43 | fake_service.cache['fakeother_idother_fake'] = fake_entity.id 44 | 45 | entity = await fake_service.get_one('fake', other_id='other_fake') 46 | 47 | assert entity == fake_entity.id 48 | assert not fake_service.repository.memory_data_source.get.called 49 | 50 | 51 | @pytest.mark.asyncio 52 | async def test_should_get_one_from_fallback_when_not_found_on_memory( 53 | fake_service, serialized_fake_entity, fake_entity 54 | ): 55 | await fake_service.repository.memory_data_source.delete( 56 | 'fake:other_fake:fake' 57 | ) 58 | await fake_service.repository.memory_data_source.delete( 59 | 'fake:not-found:other_fake:fake' 60 | ) 61 | await fake_service.repository.fallback_data_source.put( 62 | fake_service.repository.fallback_data_source.make_key( 63 | 'fake', 'other_fake:fake' 64 | ), 65 | {'value': True}, 66 | ) 67 | 68 | entity = await fake_service.get_one('fake', other_id='other_fake') 69 | 70 | assert entity == fake_entity.id 71 | assert ( 72 | await fake_service.repository.memory_data_source.get( 73 | 'fake:other_fake:fake' 74 | ) 75 | == b'1' 76 | ) 77 | 78 | 79 | @pytest.mark.asyncio 80 | async def test_should_get_one_from_fallback_when_not_found_on_memory_with_fields( 81 | fake_service, serialized_fake_entity, fake_entity 82 | ): 83 | await fake_service.repository.memory_data_source.delete( 84 | 'fake:other_fake:fake' 85 | ) 86 | await fake_service.repository.fallback_data_source.put( 87 | fake_service.repository.fallback_data_source.make_key( 88 | 'fake', 'other_fake:fake' 89 | ), 90 | {'value': True}, 91 | ) 92 | fake_entity.number = None 93 | fake_entity.boolean = False 94 | 95 | entity = await fake_service.get_one( 96 | 'fake', 97 | other_id='other_fake', 98 | fields=['id', 'other_id', 'integer', 'inner_entities'], 99 | ) 100 | 101 | assert entity == fake_entity.id 102 | assert ( 103 | await fake_service.repository.memory_data_source.get( 104 | 'fake:other_fake:fake' 105 | ) 106 | == b'1' 107 | ) 108 | 109 | 110 | @pytest.mark.asyncio 111 | async def test_should_get_one_from_fallback_after_open_circuit_breaker( 112 | fake_service, fake_entity, mocker 113 | ): 114 | fake_service.repository.memory_data_source.get = asynctest.CoroutineMock( 115 | side_effect=RedisError 116 | ) 117 | key = fake_service.repository.fallback_data_source.make_key( 118 | 'fake', 'other_fake', 'fake' 119 | ) 120 | await fake_service.repository.fallback_data_source.put( 121 | key, {'value': True} 122 | ) 123 | 124 | entity = await fake_service.get_one('fake', other_id='other_fake') 125 | 126 | assert entity == fake_entity.id 127 | assert fake_service.logger.warning.call_count == 1 128 | -------------------------------------------------------------------------------- /dbdaora/boolean/_tests/test_integration_service_boolean_aioredis_add.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from aioredis import RedisError 4 | 5 | 6 | @pytest.fixture 7 | def has_add_cb(): 8 | return True 9 | 10 | 11 | @pytest.mark.asyncio 12 | async def test_should_add(fake_service, fake_entity): 13 | await fake_service.repository.add(fake_entity) 14 | 15 | entity = await fake_service.get_one('fake') 16 | 17 | assert entity == fake_entity.id 18 | 19 | 20 | @pytest.mark.asyncio 21 | async def test_should_add_to_fallback_after_open_circuit_breaker( 22 | fake_service, fake_entity 23 | ): 24 | fake_get = asynctest.CoroutineMock(side_effect=RedisError) 25 | fake_service.repository.memory_data_source.get = fake_get 26 | await fake_service.add(fake_entity) 27 | 28 | entity = await fake_service.get_one('fake') 29 | 30 | assert entity == fake_entity.id 31 | assert fake_service.logger.warning.call_count == 1 32 | -------------------------------------------------------------------------------- /dbdaora/boolean/_tests/test_integration_service_boolean_aioredis_delete.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from aioredis import RedisError 4 | from jsondaora import dataclasses 5 | 6 | from dbdaora.exceptions import EntityNotFoundError 7 | 8 | 9 | @pytest.fixture 10 | def has_delete_cb(): 11 | return True 12 | 13 | 14 | @pytest.mark.asyncio 15 | async def test_should_delete( 16 | fake_service, serialized_fake_entity, fake_entity 17 | ): 18 | await fake_service.add(fake_entity) 19 | 20 | assert await fake_service.get_one('fake') 21 | 22 | await fake_service.delete(fake_entity.id) 23 | fake_service.cache.clear() 24 | 25 | with pytest.raises(EntityNotFoundError): 26 | await fake_service.get_one('fake') 27 | 28 | 29 | @pytest.mark.asyncio 30 | async def test_should_delete_from_fallback_after_open_circuit_breaker( 31 | fake_service, fake_entity, mocker 32 | ): 33 | await fake_service.repository.memory_data_source.delete('fake:fake') 34 | fake_service.repository.fallback_data_source.db[ 35 | 'fake:fake' 36 | ] = dataclasses.asdict(fake_entity, dumps_value=True) 37 | 38 | assert await fake_service.get_one('fake') 39 | 40 | fake_service.repository.memory_data_source.set = asynctest.CoroutineMock( 41 | side_effect=RedisError 42 | ) 43 | 44 | await fake_service.delete(fake_entity.id) 45 | fake_service.cache.clear() 46 | 47 | with pytest.raises(EntityNotFoundError): 48 | await fake_service.get_one('fake') 49 | 50 | assert fake_service.logger.warning.call_count == 2 51 | -------------------------------------------------------------------------------- /dbdaora/boolean/_tests/test_integration_service_boolean_aioredis_get_many.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from aioredis import RedisError 4 | 5 | 6 | @pytest.mark.asyncio 7 | async def test_should_get_many( 8 | fake_service, 9 | serialized_fake_entity, 10 | fake_entity, 11 | serialized_fake_entity2, 12 | fake_entity2, 13 | ): 14 | await fake_service.repository.memory_data_source.set( 15 | 'fake:fake', serialized_fake_entity 16 | ) 17 | await fake_service.repository.memory_data_source.set( 18 | 'fake:fake2', serialized_fake_entity2 19 | ) 20 | entities = [e async for e in fake_service.get_many('fake', 'fake2')] 21 | 22 | assert entities == [fake_entity.id, fake_entity2.id] 23 | 24 | 25 | @pytest.mark.asyncio 26 | async def test_should_get_many_without_cache( 27 | fake_service, 28 | serialized_fake_entity, 29 | fake_entity, 30 | serialized_fake_entity2, 31 | fake_entity2, 32 | ): 33 | fake_service.cache = None 34 | await fake_service.repository.memory_data_source.set( 35 | 'fake:fake', serialized_fake_entity 36 | ) 37 | await fake_service.repository.memory_data_source.set( 38 | 'fake:fake2', serialized_fake_entity2 39 | ) 40 | entities = [e async for e in fake_service.get_many('fake', 'fake2')] 41 | 42 | assert entities == [fake_entity.id, fake_entity2.id] 43 | 44 | 45 | @pytest.mark.asyncio 46 | async def test_should_get_many_from_cache( 47 | fake_service, serialized_fake_entity, fake_entity, fake_entity2 48 | ): 49 | fake_service.repository.memory_data_source.get = asynctest.CoroutineMock() 50 | fake_service.cache['fake'] = fake_entity.id 51 | fake_service.cache['fake2'] = fake_entity2.id 52 | entities = [e async for e in fake_service.get_many('fake', 'fake2')] 53 | 54 | assert entities == [fake_entity.id, fake_entity2.id] 55 | assert not fake_service.repository.memory_data_source.get.called 56 | 57 | 58 | @pytest.mark.asyncio 59 | async def test_should_get_many_from_fallback_after_open_circuit_breaker( 60 | fake_service, fake_entity, fake_entity2, mocker 61 | ): 62 | fake_service.repository.memory_data_source.get = asynctest.CoroutineMock( 63 | side_effect=RedisError 64 | ) 65 | fake_service.repository.fallback_data_source.db['fake:fake'] = { 66 | 'value': True 67 | } 68 | fake_service.repository.fallback_data_source.db['fake:fake2'] = { 69 | 'value': True 70 | } 71 | 72 | entities = [e async for e in fake_service.get_many('fake', 'fake2')] 73 | 74 | assert entities == [fake_entity.id, fake_entity2.id] 75 | assert fake_service.logger.warning.call_count == 2 76 | 77 | 78 | @pytest.mark.asyncio 79 | async def test_should_get_many_from_fallback_after_open_circuit_breaker_without_cache( 80 | fake_service, fake_entity, fake_entity2, mocker 81 | ): 82 | fake_service.cache = None 83 | fake_service.repository.memory_data_source.get = asynctest.CoroutineMock( 84 | side_effect=RedisError 85 | ) 86 | fake_service.repository.fallback_data_source.db['fake:fake'] = { 87 | 'value': True 88 | } 89 | fake_service.repository.fallback_data_source.db['fake:fake2'] = { 90 | 'value': True 91 | } 92 | 93 | entities = [e async for e in fake_service.get_many('fake', 'fake2')] 94 | 95 | assert entities == [fake_entity.id, fake_entity2.id] 96 | assert fake_service.logger.warning.call_count == 1 97 | -------------------------------------------------------------------------------- /dbdaora/boolean/_tests/test_integration_service_boolean_aioredis_get_one.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from aioredis import RedisError 4 | 5 | 6 | @pytest.mark.asyncio 7 | async def test_should_get_one( 8 | fake_service, serialized_fake_entity, fake_entity 9 | ): 10 | await fake_service.repository.memory_data_source.set( 11 | 'fake:fake', serialized_fake_entity 12 | ) 13 | entity = await fake_service.get_one('fake') 14 | 15 | assert entity == fake_entity.id 16 | 17 | 18 | @pytest.mark.asyncio 19 | async def test_should_get_one_without_cache( 20 | fake_service, serialized_fake_entity, fake_entity 21 | ): 22 | fake_service.cache = None 23 | await fake_service.repository.memory_data_source.set( 24 | 'fake:fake', serialized_fake_entity 25 | ) 26 | entity = await fake_service.get_one('fake') 27 | 28 | assert entity == fake_entity.id 29 | 30 | 31 | @pytest.mark.asyncio 32 | async def test_should_get_one_from_cache( 33 | fake_service, serialized_fake_entity, fake_entity 34 | ): 35 | fake_service.repository.memory_data_source.get = asynctest.CoroutineMock() 36 | fake_service.cache['fake'] = fake_entity.id 37 | entity = await fake_service.get_one('fake') 38 | 39 | assert entity == fake_entity.id 40 | assert not fake_service.repository.memory_data_source.get.called 41 | 42 | 43 | @pytest.mark.asyncio 44 | async def test_should_get_one_from_fallback_after_open_circuit_breaker( 45 | fake_service, fake_entity, mocker 46 | ): 47 | fake_service.repository.memory_data_source.get = asynctest.CoroutineMock( 48 | side_effect=RedisError 49 | ) 50 | fake_service.repository.fallback_data_source.db['fake:fake'] = { 51 | 'value': True 52 | } 53 | 54 | entity = await fake_service.get_one('fake') 55 | 56 | assert entity == fake_entity.id 57 | assert fake_service.logger.warning.call_count == 1 58 | 59 | 60 | @pytest.mark.asyncio 61 | async def test_should_get_one_from_fallback_after_open_circuit_breaker_without_cache( 62 | fake_service, fake_entity, mocker 63 | ): 64 | fake_service.cache = None 65 | fake_service.repository.memory_data_source.get = asynctest.CoroutineMock( 66 | side_effect=RedisError 67 | ) 68 | fake_service.repository.fallback_data_source.db['fake:fake'] = { 69 | 'value': True 70 | } 71 | 72 | entity = await fake_service.get_one('fake') 73 | 74 | assert entity == fake_entity.id 75 | assert fake_service.logger.warning.call_count == 1 76 | -------------------------------------------------------------------------------- /dbdaora/boolean/_tests/test_unit_boolean_service.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from dbdaora import EntityNotFoundError, HashService 4 | from dbdaora.service import CACHE_ALREADY_NOT_FOUND 5 | 6 | 7 | @pytest.fixture 8 | def service(mocker): 9 | s = HashService( 10 | repository=mocker.MagicMock(id_name='id'), 11 | circuit_breaker=mocker.MagicMock(), 12 | fallback_circuit_breaker=mocker.MagicMock(), 13 | cache={}, 14 | ) 15 | s.entity_circuit = mocker.AsyncMock() 16 | return s 17 | 18 | 19 | @pytest.mark.asyncio 20 | async def test_should_set_cache_entity_not_found_when_getting_one(service): 21 | error = EntityNotFoundError() 22 | service.entity_circuit.side_effect = error 23 | 24 | with pytest.raises(EntityNotFoundError) as exc_info: 25 | await service.get_one('fake') 26 | 27 | assert exc_info.value == error 28 | assert service.cache['fake'] == CACHE_ALREADY_NOT_FOUND 29 | 30 | 31 | @pytest.mark.asyncio 32 | async def test_should_get_already_not_found_when_getting_one(service): 33 | service.cache['fake'] = CACHE_ALREADY_NOT_FOUND 34 | 35 | with pytest.raises(EntityNotFoundError) as exc_info: 36 | await service.get_one('fake') 37 | 38 | assert not service.entity_circuit.called 39 | assert exc_info.value.args == ('fake',) 40 | 41 | 42 | @pytest.mark.asyncio 43 | async def test_should_set_cache_entities_not_found_when_getting_many(service): 44 | fake_entity = {'id': 'fake'} 45 | service.entity_circuit.side_effect = [ 46 | fake_entity, 47 | EntityNotFoundError, 48 | EntityNotFoundError, 49 | ] 50 | service.entity_fallback_circuit.side_effect = [ 51 | EntityNotFoundError, 52 | EntityNotFoundError, 53 | ] 54 | service.repository.id_name = 'id' 55 | 56 | entities = [e async for e in service.get_many('fake', 'fake2', 'fake3')] 57 | 58 | assert entities == [fake_entity] 59 | assert service.cache['fake2'] == CACHE_ALREADY_NOT_FOUND 60 | assert service.cache['fake3'] == CACHE_ALREADY_NOT_FOUND 61 | 62 | 63 | @pytest.mark.asyncio 64 | async def test_should_get_already_not_found_when_getting_many(service): 65 | fake_entity = {'id': 'fake'} 66 | service.cache['fake'] = fake_entity 67 | service.cache['fake2'] = CACHE_ALREADY_NOT_FOUND 68 | service.cache['fake3'] = CACHE_ALREADY_NOT_FOUND 69 | 70 | entities = [e async for e in service.get_many('fake', 'fake2', 'fake3')] 71 | 72 | assert not service.repository.query.called 73 | assert entities == [fake_entity] 74 | assert service.cache['fake2'] == CACHE_ALREADY_NOT_FOUND 75 | assert service.cache['fake3'] == CACHE_ALREADY_NOT_FOUND 76 | -------------------------------------------------------------------------------- /dbdaora/boolean/conftest.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from functools import partial 3 | 4 | import pytest 5 | from aioredis import RedisError 6 | 7 | from dbdaora import ( 8 | BooleanRepository, 9 | CacheType, 10 | DictFallbackDataSource, 11 | make_aioredis_data_source, 12 | make_boolean_service, 13 | ) 14 | 15 | 16 | @pytest.fixture 17 | def memory_data_source_factory(): 18 | return partial( 19 | make_aioredis_data_source, 20 | 'redis://', 21 | 'redis://localhost/1', 22 | 'redis://localhost/2', 23 | ) 24 | 25 | 26 | @pytest.fixture 27 | def has_add_cb(): 28 | return False 29 | 30 | 31 | @pytest.fixture 32 | def has_delete_cb(): 33 | return False 34 | 35 | 36 | @pytest.mark.asyncio 37 | @pytest.fixture 38 | async def fake_service( 39 | memory_data_source_factory, 40 | mocker, 41 | fallback_data_source, 42 | fake_boolean_repository_cls, 43 | has_add_cb, 44 | has_delete_cb, 45 | ): 46 | async def fallback_data_source_factory(): 47 | return fallback_data_source 48 | 49 | service = await make_boolean_service( 50 | fake_boolean_repository_cls, 51 | memory_data_source_factory, 52 | fallback_data_source_factory, 53 | repository_expire_time=1, 54 | cache_type=CacheType.TTL, 55 | cache_ttl=1, 56 | cache_max_size=2, 57 | cb_failure_threshold=0, 58 | cb_recovery_timeout=10, 59 | cb_expected_exception=RedisError, 60 | cb_expected_fallback_exception=KeyError, 61 | logger=mocker.MagicMock(), 62 | has_add_circuit_breaker=has_add_cb, 63 | has_delete_circuit_breaker=has_delete_cb, 64 | ) 65 | 66 | yield service 67 | 68 | service.repository.memory_data_source.close() 69 | await service.repository.memory_data_source.wait_closed() 70 | 71 | 72 | @pytest.fixture 73 | def fallback_data_source(): 74 | return DictFallbackDataSource() 75 | 76 | 77 | @dataclasses.dataclass 78 | class FakeEntity: 79 | id: str 80 | 81 | 82 | class FakeBooleanRepository(BooleanRepository[FakeEntity, str]): 83 | name = 'fake' 84 | id_name = 'id' 85 | 86 | 87 | @pytest.fixture 88 | def fake_entity_cls(): 89 | return FakeEntity 90 | 91 | 92 | @pytest.fixture 93 | def fake_boolean_repository_cls(): 94 | return FakeBooleanRepository 95 | 96 | 97 | @pytest.fixture 98 | def dict_repository_cls(): 99 | return FakeBooleanRepository 100 | 101 | 102 | @pytest.fixture 103 | def fake_entity(): 104 | return FakeEntity(id='fake',) 105 | 106 | 107 | @pytest.fixture 108 | def fake_entity2(): 109 | return FakeEntity(id='fake2',) 110 | 111 | 112 | @pytest.fixture 113 | def serialized_fake_entity(): 114 | return '1' 115 | 116 | 117 | @pytest.fixture 118 | def serialized_fake_entity2(): 119 | return '1' 120 | 121 | 122 | @pytest.mark.asyncio 123 | @pytest.fixture 124 | async def repository( 125 | mocker, 126 | memory_data_source_factory, 127 | fallback_data_source, 128 | fake_boolean_repository_cls, 129 | ): 130 | memory_data_source = await memory_data_source_factory() 131 | yield fake_boolean_repository_cls( 132 | memory_data_source=memory_data_source, 133 | fallback_data_source=fallback_data_source, 134 | expire_time=1, 135 | ) 136 | memory_data_source.close() 137 | await memory_data_source.wait_closed() 138 | -------------------------------------------------------------------------------- /dbdaora/boolean/factory.py: -------------------------------------------------------------------------------- 1 | from logging import Logger, getLogger 2 | from typing import Any, Optional, Type 3 | 4 | from dbdaora.entity import Entity, EntityData 5 | from dbdaora.keys import FallbackKey 6 | from dbdaora.service.builder import build as build_base_service 7 | 8 | from ..cache import CacheType 9 | from ..repository import MemoryRepository 10 | from ..service import Service 11 | from .service import BooleanService 12 | 13 | 14 | async def make_service( 15 | repository_cls: Type[MemoryRepository[Entity, EntityData, FallbackKey]], 16 | memory_data_source_factory: Any, 17 | fallback_data_source_factory: Any, 18 | repository_expire_time: int, 19 | cache_type: Optional[CacheType] = None, 20 | cache_ttl: Optional[int] = None, 21 | cache_max_size: Optional[int] = None, 22 | cb_failure_threshold: Optional[int] = None, 23 | cb_recovery_timeout: Optional[int] = None, 24 | cb_expected_exception: Optional[Type[Exception]] = None, 25 | cb_expected_fallback_exception: Optional[Type[Exception]] = None, 26 | logger: Logger = getLogger(__name__), 27 | has_add_circuit_breaker: bool = False, 28 | has_delete_circuit_breaker: bool = False, 29 | ) -> Service[Entity, EntityData, FallbackKey]: 30 | return await build_base_service( 31 | BooleanService, # type: ignore 32 | repository_cls, 33 | memory_data_source_factory, 34 | fallback_data_source_factory, 35 | repository_expire_time, 36 | cache_type=cache_type, 37 | cache_ttl=cache_ttl, 38 | cache_max_size=cache_max_size, 39 | cb_failure_threshold=cb_failure_threshold, 40 | cb_recovery_timeout=cb_recovery_timeout, 41 | cb_expected_exception=cb_expected_exception, 42 | cb_expected_fallback_exception=cb_expected_fallback_exception, 43 | logger=logger, 44 | has_add_circuit_breaker=has_add_circuit_breaker, 45 | has_delete_circuit_breaker=has_delete_circuit_breaker, 46 | ) 47 | -------------------------------------------------------------------------------- /dbdaora/boolean/repositories/_tests/test_integration_repository_aioredis_boolean_get.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | 4 | from dbdaora import Query 5 | from dbdaora.exceptions import EntityNotFoundError 6 | 7 | 8 | @pytest.mark.asyncio 9 | async def test_should_get_from_memory( 10 | repository, serialized_fake_entity, fake_entity 11 | ): 12 | await repository.memory_data_source.set( 13 | 'fake:fake', serialized_fake_entity 14 | ) 15 | entity = await repository.query('fake').entity 16 | 17 | assert entity == fake_entity.id 18 | 19 | 20 | @pytest.mark.asyncio 21 | async def test_should_raise_not_found_error(repository, fake_entity, mocker): 22 | await repository.memory_data_source.delete('fake:fake') 23 | fake_query = Query(repository, memory=True, id=fake_entity.id) 24 | 25 | with pytest.raises(EntityNotFoundError) as exc_info: 26 | await repository.query(fake_entity.id).entity 27 | 28 | assert exc_info.value.args == (fake_query,) 29 | 30 | 31 | @pytest.mark.asyncio 32 | async def test_should_raise_not_found_error_when_already_raised_before( 33 | repository, mocker 34 | ): 35 | fake_entity = 'fake' 36 | expected_query = Query(repository, memory=True, id=fake_entity) 37 | repository.memory_data_source.get = asynctest.CoroutineMock( 38 | side_effect=[False, True] 39 | ) 40 | repository.memory_data_source.set = asynctest.CoroutineMock() 41 | 42 | with pytest.raises(EntityNotFoundError) as exc_info: 43 | await repository.query(fake_entity).entity 44 | 45 | assert exc_info.value.args == (expected_query,) 46 | assert repository.memory_data_source.get.call_args_list == [ 47 | mocker.call('fake:fake'), 48 | ] 49 | assert not repository.memory_data_source.set.called 50 | 51 | 52 | @pytest.mark.asyncio 53 | async def test_should_set_already_not_found_error(repository, mocker): 54 | fake_entity = 'fake' 55 | expected_query = Query(repository, memory=True, id=fake_entity) 56 | repository.fallback_data_source.get = asynctest.CoroutineMock( 57 | return_value=None 58 | ) 59 | repository.memory_data_source.get = asynctest.CoroutineMock( 60 | return_value=None 61 | ) 62 | repository.memory_data_source.set = asynctest.CoroutineMock() 63 | 64 | with pytest.raises(EntityNotFoundError) as exc_info: 65 | await repository.query(fake_entity).entity 66 | 67 | assert exc_info.value.args == (expected_query,) 68 | assert repository.fallback_data_source.get.call_args_list == [ 69 | mocker.call('fake:fake') 70 | ] 71 | assert repository.memory_data_source.set.call_args_list == [ 72 | mocker.call('fake:fake', '0') 73 | ] 74 | 75 | 76 | @pytest.mark.asyncio 77 | async def test_should_get_from_fallback(repository, fake_entity): 78 | await repository.memory_data_source.delete('fake:fake') 79 | repository.fallback_data_source.db['fake:fake'] = {'value': True} 80 | entity = await repository.query(fake_entity.id).entity 81 | 82 | assert entity == fake_entity.id 83 | assert await repository.memory_data_source.get('fake:fake') == b'1' 84 | 85 | 86 | @pytest.mark.asyncio 87 | async def test_should_set_memory_after_got_fallback( 88 | repository, fake_entity, mocker 89 | ): 90 | repository.memory_data_source.get = asynctest.CoroutineMock( 91 | side_effect=[None] 92 | ) 93 | repository.memory_data_source.set = asynctest.CoroutineMock() 94 | repository.fallback_data_source.db['fake:fake'] = {'value': True} 95 | entity = await repository.query(fake_entity.id).entity 96 | 97 | assert repository.memory_data_source.get.called 98 | assert repository.memory_data_source.set.call_args_list == [ 99 | mocker.call('fake:fake', '1') 100 | ] 101 | assert entity == fake_entity.id 102 | -------------------------------------------------------------------------------- /dbdaora/boolean/repositories/_tests/test_integration_repository_aioredis_boolean_get_many.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.mark.asyncio 5 | async def test_should_set_already_not_found_error_when_get_many( 6 | repository, mocker 7 | ): 8 | fake_entity = 'fake' 9 | repository.memory_data_source.get = mocker.AsyncMock(side_effect=[None]) 10 | repository.fallback_data_source.get = mocker.AsyncMock(side_effect=[None]) 11 | repository.memory_data_source.set = mocker.AsyncMock() 12 | 13 | assert [ 14 | e async for e in repository.query(many=[fake_entity]).entities 15 | ] == [] 16 | 17 | assert repository.memory_data_source.get.call_args_list == [ 18 | mocker.call('fake:fake'), 19 | ] 20 | assert repository.fallback_data_source.get.call_args_list == [ 21 | mocker.call('fake:fake') 22 | ] 23 | assert repository.memory_data_source.set.call_args_list == [ 24 | mocker.call('fake:fake', '0') 25 | ] 26 | 27 | 28 | @pytest.mark.asyncio 29 | async def test_should_get_many_from_fallback(repository, fake_entity): 30 | await repository.memory_data_source.delete('fake:fake') 31 | repository.fallback_data_source.db['fake:fake'] = {'value': True} 32 | 33 | entities = [ 34 | e async for e in repository.query(many=[fake_entity.id]).entities 35 | ] 36 | 37 | assert entities == [fake_entity.id] 38 | assert await repository.memory_data_source.get('fake:fake') == b'1' 39 | 40 | 41 | @pytest.mark.asyncio 42 | async def test_should_get_many_with_one_item_already_not_found_from_fallback( 43 | repository, fake_entity 44 | ): 45 | await repository.memory_data_source.delete('fake:fake') 46 | await repository.memory_data_source.delete('fake:fake2') 47 | repository.fallback_data_source.db['fake:fake'] = {'value': True} 48 | 49 | entities = [ 50 | e 51 | async for e in repository.query( 52 | many=[fake_entity.id, 'fake2'] 53 | ).entities 54 | ] 55 | 56 | assert entities == [fake_entity.id] 57 | assert await repository.memory_data_source.get('fake:fake') == b'1' 58 | assert await repository.memory_data_source.get('fake:fake2') == b'0' 59 | 60 | 61 | @pytest.mark.asyncio 62 | async def test_should_get_many_with_one_item_already_not_found_and_another_not_found_from_fallback( 63 | repository, fake_entity 64 | ): 65 | await repository.memory_data_source.delete('fake:fake') 66 | await repository.memory_data_source.set('fake:fake2', '0') 67 | await repository.memory_data_source.delete('fake:fake3') 68 | repository.fallback_data_source.db['fake:fake'] = {'value': True} 69 | 70 | entities = [ 71 | e 72 | async for e in repository.query( 73 | many=[fake_entity.id, 'fake2', 'fake3'] 74 | ).entities 75 | ] 76 | 77 | assert entities == [fake_entity.id] 78 | assert await repository.memory_data_source.get('fake:fake') == b'1' 79 | assert await repository.memory_data_source.get('fake:fake2') == b'0' 80 | assert await repository.memory_data_source.get('fake:fake3') == b'0' 81 | -------------------------------------------------------------------------------- /dbdaora/boolean/repositories/_tests/test_integration_repository_boolean_datastore.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from google.cloud import datastore 3 | 4 | from dbdaora import DatastoreBooleanRepository 5 | from dbdaora.data_sources.fallback.datastore import DatastoreDataSource 6 | 7 | 8 | @pytest.fixture 9 | def fallback_data_source(): 10 | return DatastoreDataSource() 11 | 12 | 13 | @pytest.fixture 14 | def fake_boolean_repository_cls(fake_entity_cls): 15 | class FakeRepository( 16 | DatastoreBooleanRepository, entity_cls=fake_entity_cls 17 | ): 18 | id_name = 'id' 19 | 20 | return FakeRepository 21 | 22 | 23 | @pytest.mark.asyncio 24 | async def test_should_exclude_attributes_from_indexes( 25 | repository, fake_entity_cls 26 | ): 27 | client = repository.fallback_data_source.client 28 | key = client.key('fake', 'fake') 29 | entity = datastore.Entity(key=key) 30 | entity.update({'value': True}) 31 | client.put(entity) 32 | 33 | assert not client.get(key).exclude_from_indexes 34 | 35 | await repository.add(fake_entity_cls(id='fake')) 36 | 37 | assert client.get(key).exclude_from_indexes == set(['value']) 38 | -------------------------------------------------------------------------------- /dbdaora/boolean/repositories/datastore.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | from google.cloud.datastore import Entity, Key 4 | from jsondaora.serializers import OrjsonDefaultTypes 5 | 6 | from . import BooleanRepository 7 | 8 | 9 | OrjsonDefaultTypes.types_default_map[Entity] = lambda e: dict(**e) 10 | 11 | 12 | class DatastoreBooleanRepository(BooleanRepository[Entity, Key]): 13 | __skip_cls_validation__ = ('DatastoreBooleanRepository',) 14 | fallback_data_source_key_cls = Key 15 | 16 | async def add_fallback( 17 | self, entity: Any, *entities: Any, **kwargs: Any 18 | ) -> None: 19 | await super().add_fallback( 20 | entity, *entities, exclude_from_indexes=('value',) 21 | ) 22 | -------------------------------------------------------------------------------- /dbdaora/boolean/service.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | from dbdaora.service import Service 4 | 5 | from ..keys import FallbackKey 6 | 7 | 8 | class BooleanService(Service[Any, bool, FallbackKey]): 9 | ... 10 | -------------------------------------------------------------------------------- /dbdaora/cache.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import random 3 | import time 4 | from enum import Enum 5 | from typing import Any, Dict, Optional, Tuple, Type, Union 6 | 7 | from cachetools import Cache, LFUCache, LRUCache, TTLCache 8 | 9 | 10 | @dataclasses.dataclass(init=False) 11 | class TTLDaoraCache: 12 | maxsize: int 13 | ttl: int 14 | ttl_failure_threshold: int 15 | cache: Dict[str, Tuple[Any, float]] 16 | first_set_time: float 17 | clean_keys_size: int 18 | 19 | def __init__( 20 | self, 21 | maxsize: int, 22 | ttl: int, 23 | ttl_failure_threshold: int = 0, 24 | cache: Optional[Dict[str, Tuple[Any, float]]] = None, 25 | first_set_time: float = 0, 26 | ): 27 | if cache is None: 28 | cache = {} 29 | 30 | self.maxsize = maxsize 31 | self.ttl = ttl 32 | self.ttl_failure_threshold = ttl_failure_threshold 33 | self.cache = cache 34 | self.first_set_time = first_set_time 35 | self.clean_keys_size = int(self.maxsize * 0.1) 36 | 37 | def __setitem__(self, key: str, data: Any) -> None: 38 | if len(self.cache) < self.maxsize: 39 | set_time = time.time() - self.ttl_threshold 40 | self.cache[key] = (data, set_time) 41 | 42 | if self.first_set_time <= 0: 43 | self.first_set_time = set_time 44 | 45 | elif self.first_set_time + self.ttl < time.time(): 46 | self.first_set_time = 0 47 | 48 | for i, key_to_pop in enumerate(self.cache.keys()): 49 | self.cache.pop(key_to_pop, None) 50 | 51 | if i >= self.clean_keys_size: 52 | break 53 | 54 | set_time = time.time() - self.ttl_threshold 55 | self.cache[key] = (data, set_time) 56 | 57 | def get(self, key: str, default: Any = None) -> Any: 58 | data = self.cache.get(key) 59 | 60 | if data is not None: 61 | if data[1] + self.ttl >= time.time(): 62 | return data[0] 63 | 64 | else: 65 | self.cache.pop(key, None) 66 | return data[0] 67 | 68 | return default 69 | 70 | @property 71 | def ttl_threshold(self) -> int: 72 | if self.ttl_failure_threshold: 73 | return random.randint(0, self.ttl_failure_threshold) 74 | 75 | return 0 76 | 77 | 78 | class CacheType(Enum): 79 | value: Union[Type[Cache]] 80 | 81 | TTL = TTLCache 82 | LFU = LFUCache 83 | LRU = LRUCache 84 | TTLDAORA = TTLDaoraCache 85 | -------------------------------------------------------------------------------- /dbdaora/circuitbreaker.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from typing import ( 3 | Any, 4 | Awaitable, 5 | Callable, 6 | Optional, 7 | Tuple, 8 | Type, 9 | TypeVar, 10 | Union, 11 | ) 12 | 13 | from circuitbreaker import ( 14 | STATE_CLOSED, 15 | STATE_OPEN, 16 | CircuitBreaker, 17 | CircuitBreakerError, 18 | ) 19 | 20 | 21 | FuncReturn = TypeVar('FuncReturn') 22 | 23 | 24 | class AsyncCircuitBreaker(CircuitBreaker): 25 | _last_failure: Optional[Exception] 26 | 27 | def __init__( 28 | self, 29 | failure_threshold: Optional[int] = None, 30 | recovery_timeout: Optional[int] = None, 31 | expected_exception: Optional[ 32 | Union[Type[Exception], Tuple[Type[Exception], ...]] 33 | ] = None, 34 | name: Optional[str] = None, 35 | fallback_function: Optional[ 36 | Callable[..., Awaitable[FuncReturn]] 37 | ] = None, 38 | ): 39 | self._last_failure = None 40 | self._failure_count = 0 41 | self._failure_threshold = ( 42 | failure_threshold 43 | if failure_threshold is not None 44 | else self.FAILURE_THRESHOLD 45 | ) 46 | self._recovery_timeout = recovery_timeout or self.RECOVERY_TIMEOUT 47 | self._expected_exception = ( 48 | expected_exception or self.EXPECTED_EXCEPTION 49 | ) 50 | self._fallback_function = fallback_function or self.FALLBACK_FUNCTION 51 | self._name = name 52 | self._state = STATE_CLOSED 53 | self._opened = datetime.utcnow() 54 | 55 | async def call( 56 | self, 57 | func: Callable[..., Awaitable[FuncReturn]], 58 | *args: Any, 59 | **kwargs: Any, 60 | ) -> FuncReturn: 61 | if self.opened: 62 | if self.fallback_function: 63 | return await self.fallback_function(*args, **kwargs) 64 | else: 65 | raise DBDaoraCircuitBreakerError(self, func.__name__) 66 | 67 | try: 68 | result = await func(*args, **kwargs) 69 | except self._expected_exception as e: 70 | self.set_failure(func.__name__, e) 71 | raise 72 | 73 | self.__call_succeeded() 74 | return result 75 | 76 | def __call_succeeded(self) -> None: 77 | self._state = STATE_CLOSED 78 | self._last_failure = None 79 | self._failure_count = 0 80 | 81 | def __call_failed(self) -> None: 82 | self._failure_count += 1 83 | if self._failure_count >= self._failure_threshold: 84 | self._state = STATE_OPEN 85 | self._opened = datetime.utcnow() 86 | 87 | @property 88 | def expected_exception( 89 | self, 90 | ) -> Union[Type[Exception], Tuple[Type[Exception], ...]]: 91 | return self._expected_exception 92 | 93 | def set_failure(self, func_name: str, error: Exception) -> None: 94 | self._last_failure = error 95 | self.__call_failed() 96 | 97 | if self._failure_threshold == 0: 98 | raise DBDaoraCircuitBreakerError(self, func_name) 99 | 100 | def set_success(self) -> None: 101 | self.__call_succeeded() 102 | 103 | 104 | class DBDaoraCircuitBreakerError(CircuitBreakerError): 105 | _circuit_breaker: AsyncCircuitBreaker 106 | last_failure: Optional[Exception] 107 | 108 | def __init__( 109 | self, 110 | circuit_breaker: CircuitBreaker, 111 | name_sufix: Optional[str] = None, 112 | *args: Any, 113 | **kwargs: Any, 114 | ): 115 | super().__init__(circuit_breaker, *args, **kwargs) 116 | self._name_sufix = name_sufix 117 | self.last_failure = self._circuit_breaker._last_failure 118 | 119 | def __str__(self, *args: Any, **kwargs: Any) -> str: 120 | return ( 121 | 'Circuit "%s" OPEN until %s (%d failures, %d sec remaining) (last_failure: %r)' 122 | % ( 123 | self._circuit_breaker.name 124 | if self._name_sufix is None 125 | else f'{self._circuit_breaker.name}_{self._name_sufix}', 126 | self._circuit_breaker.open_until, 127 | self._circuit_breaker.failure_count, 128 | round(self._circuit_breaker.open_remaining), 129 | self._circuit_breaker.last_failure, 130 | ) 131 | ) 132 | -------------------------------------------------------------------------------- /dbdaora/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from dbdaora.data_sources.fallback.dict import DictFallbackDataSource 4 | from dbdaora.data_sources.memory.dict import DictMemoryDataSource 5 | 6 | 7 | @pytest.fixture 8 | async def dict_repository(mocker, dict_repository_cls): 9 | return dict_repository_cls( 10 | memory_data_source=DictMemoryDataSource(), 11 | fallback_data_source=DictFallbackDataSource(), 12 | expire_time=1, 13 | ) 14 | 15 | 16 | @pytest.fixture 17 | def async_iterator(): 18 | return AsyncIterator 19 | 20 | 21 | class AsyncIterator: 22 | def __init__(self, seq): 23 | self.iter = iter(seq) 24 | 25 | def __aiter__(self): 26 | return self 27 | 28 | async def __anext__(self): 29 | try: 30 | return next(self.iter) 31 | except StopIteration: 32 | raise StopAsyncIteration 33 | -------------------------------------------------------------------------------- /dbdaora/data_sources/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Protocol 2 | 3 | 4 | class DataSource(Protocol): 5 | def make_key(self, *key_parts: Any) -> Any: 6 | raise NotImplementedError() # pragma: no cover 7 | -------------------------------------------------------------------------------- /dbdaora/data_sources/fallback/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Any, ClassVar, Dict, Generic, Iterable, Optional 2 | 3 | from dbdaora.keys import FallbackKey 4 | 5 | from .. import DataSource 6 | 7 | 8 | class FallbackDataSource(DataSource, Generic[FallbackKey]): 9 | key_separator: ClassVar[str] = ':' 10 | 11 | def make_key(self, *key_parts: Any) -> FallbackKey: 12 | raise NotImplementedError() # pragma: no cover 13 | 14 | async def get(self, key: FallbackKey) -> Optional[Dict[str, Any]]: 15 | raise NotImplementedError() # pragma: no cover 16 | 17 | async def put( 18 | self, key: FallbackKey, data: Dict[str, Any], **kwargs: Any 19 | ) -> None: 20 | raise NotImplementedError() # pragma: no cover 21 | 22 | async def delete(self, key: FallbackKey) -> None: 23 | raise NotImplementedError() # pragma: no cover 24 | 25 | async def query( 26 | self, key: FallbackKey, **kwargs: Any 27 | ) -> Iterable[Dict[str, Any]]: 28 | raise NotImplementedError() # pragma: no cover 29 | -------------------------------------------------------------------------------- /dbdaora/data_sources/fallback/datastore.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import dataclasses 3 | from concurrent.futures import ThreadPoolExecutor 4 | from functools import partial 5 | from typing import Any, Dict, Iterable, Optional 6 | 7 | from google.cloud.datastore import Client, Entity, Key 8 | 9 | from . import FallbackDataSource 10 | 11 | 12 | @dataclasses.dataclass 13 | class DatastoreDataSource(FallbackDataSource[Key]): 14 | client: Client = dataclasses.field(default_factory=Client) 15 | executor: ThreadPoolExecutor = ThreadPoolExecutor(max_workers=100) 16 | 17 | def make_key(self, *key_parts: Any) -> Key: 18 | return self.client.key( 19 | key_parts[0], 20 | self.key_separator.join([str(k) for k in key_parts[1:]]), 21 | ) 22 | 23 | async def get(self, key: Key) -> Optional[Dict[str, Any]]: 24 | loop = asyncio.get_running_loop() 25 | entity = await loop.run_in_executor( 26 | self.executor, partial(self.client.get, key) 27 | ) 28 | return None if entity is None else entity_asdict(entity) 29 | 30 | async def put( 31 | self, 32 | key: Key, 33 | data: Dict[str, Any], 34 | exclude_from_indexes: Iterable[str] = (), 35 | **kwargs: Any, 36 | ) -> None: 37 | entity = Entity(key, exclude_from_indexes=exclude_from_indexes) 38 | entity.update(data) 39 | loop = asyncio.get_running_loop() 40 | await loop.run_in_executor( 41 | self.executor, partial(self.client.put, entity) 42 | ) 43 | 44 | async def delete(self, key: Key) -> None: 45 | loop = asyncio.get_running_loop() 46 | await loop.run_in_executor( 47 | self.executor, partial(self.client.delete, key) 48 | ) 49 | 50 | async def query(self, key: Key, **kwargs: Any) -> Iterable[Dict[str, Any]]: 51 | return self.client.query(kind=key.kind, **kwargs).fetch() 52 | 53 | 54 | def entity_asdict(entity: Entity) -> Dict[str, Any]: 55 | return { 56 | k: entity_asdict(v) if isinstance(v, Entity) else v 57 | for k, v in entity.items() 58 | } 59 | 60 | 61 | class KindKeyDatastoreDataSource(DatastoreDataSource): 62 | def make_key(self, *key_parts: Any) -> Key: 63 | return self.client.key( 64 | self.key_separator.join([str(k) for k in key_parts[:-1]]), 65 | key_parts[-1], 66 | ) 67 | -------------------------------------------------------------------------------- /dbdaora/data_sources/fallback/dict.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import Any, ClassVar, Dict, Iterable, Optional 3 | 4 | from dbdaora.data_sources.fallback import FallbackDataSource 5 | 6 | 7 | @dataclasses.dataclass 8 | class DictFallbackDataSource(FallbackDataSource[str]): 9 | db: Dict[Optional[str], Dict[str, Any]] = dataclasses.field( 10 | default_factory=dict 11 | ) 12 | key_separator: ClassVar[str] = ':' 13 | 14 | def make_key(self, *key_parts: str) -> str: 15 | return self.key_separator.join([p for p in key_parts if p]) 16 | 17 | async def get(self, key: str) -> Optional[Dict[str, Any]]: 18 | return self.db.get(key) 19 | 20 | async def put(self, key: str, data: Dict[str, Any], **kwargs: Any) -> None: 21 | self.db[key] = data 22 | 23 | async def delete(self, key: str) -> None: 24 | self.db.pop(key, None) 25 | 26 | async def query(self, key: str, **kwargs: Any) -> Iterable[Dict[str, Any]]: 27 | return self.db.values() 28 | -------------------------------------------------------------------------------- /dbdaora/data_sources/fallback/mongodb.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import datetime 3 | from hashlib import sha256 4 | from typing import Any, ClassVar, Dict, Iterable, Optional, Set, Union 5 | 6 | import motor.motor_asyncio as motor 7 | from bson.objectid import ObjectId 8 | from pymongo.errors import OperationFailure 9 | 10 | from . import FallbackDataSource 11 | 12 | 13 | @dataclasses.dataclass 14 | class Key: 15 | collection_name: str 16 | document_id: Union[str, ObjectId] 17 | 18 | 19 | @dataclasses.dataclass 20 | class MongoDataSource(FallbackDataSource[Key]): 21 | database_name: str 22 | client: motor.AsyncIOMotorClient = dataclasses.field( 23 | default_factory=motor.AsyncIOMotorClient 24 | ) 25 | collections_has_ttl_index: ClassVar[Set[str]] = set() 26 | key_is_object_id: bool = True 27 | 28 | def make_key(self, *key_parts: Any) -> Key: 29 | str_key = self.key_separator.join([str(k) for k in key_parts[1:]]) 30 | return Key(key_parts[0], self.make_document_id(str_key)) 31 | 32 | def make_document_id(self, key: str) -> Union[str, ObjectId]: 33 | if self.key_is_object_id: 34 | return ObjectId(sha256(key.encode()).hexdigest()[:12].encode()) 35 | 36 | return key 37 | 38 | async def get(self, key: Key) -> Optional[Dict[str, Any]]: 39 | collection = self.collection(key) 40 | document = await collection.find_one({'_id': key.document_id}) 41 | 42 | if document: 43 | document.pop('_id') 44 | 45 | return document 46 | 47 | async def put( 48 | self, 49 | key: Key, 50 | data: Dict[str, Any], 51 | exclude_from_indexes: Iterable[str] = (), 52 | **kwargs: Any, 53 | ) -> None: 54 | document_ttl = kwargs.get('fallback_ttl') 55 | 56 | if document_ttl: 57 | data['last_modified'] = datetime.datetime.now() 58 | 59 | if key.collection_name not in type(self).collections_has_ttl_index: 60 | try: 61 | await self.create_ttl_index(key, document_ttl) 62 | except OperationFailure: 63 | if await self.drop_ttl_index(key, document_ttl): 64 | await self.create_ttl_index(key, document_ttl) 65 | 66 | type(self).collections_has_ttl_index.add(key.collection_name) 67 | 68 | collection = self.collection(key) 69 | await collection.replace_one( 70 | {'_id': key.document_id}, data, upsert=True, 71 | ) 72 | 73 | async def delete(self, key: Key) -> None: 74 | collection = self.collection(key) 75 | await collection.delete_one({'_id': key.document_id}) 76 | 77 | def collection(self, key: Key) -> motor.AsyncIOMotorCollection: 78 | return self.client[self.database_name][key.collection_name] 79 | 80 | async def query(self, key: Key, **kwargs: Any) -> Iterable[Dict[str, Any]]: 81 | return [i async for i in self.collection(key).find(**kwargs)] 82 | 83 | async def create_ttl_index(self, key: Key, document_ttl: int) -> None: 84 | await self.collection(key).create_index( 85 | 'last_modified', expireAfterSeconds=document_ttl, 86 | ) 87 | 88 | async def drop_ttl_index(self, key: Key, document_ttl: int) -> bool: 89 | collection = self.collection(key) 90 | 91 | async for index in collection.list_indexes(): 92 | if 'last_modified' in index['key']: 93 | await collection.drop_index(index['name']) 94 | return True 95 | 96 | return False 97 | 98 | 99 | class CollectionKeyMongoDataSource(MongoDataSource): 100 | def make_key(self, *key_parts: Any) -> Key: 101 | return Key( 102 | self.key_separator.join([str(k) for k in key_parts[:-1]]), 103 | self.make_document_id(key_parts[-1]), 104 | ) 105 | -------------------------------------------------------------------------------- /dbdaora/data_sources/memory/dict.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import Any, Dict, Optional, Sequence, Union 3 | 4 | from . import MemoryDataSource, RangeOutput 5 | 6 | 7 | @dataclasses.dataclass 8 | class DictMemoryDataSource(MemoryDataSource): 9 | db: Dict[str, Any] = dataclasses.field(default_factory=dict) 10 | 11 | async def set(self, key: str, data: str) -> None: 12 | self.db[key] = data.encode() 13 | 14 | async def delete(self, key: str) -> None: 15 | self.db.pop(key, None) 16 | 17 | async def expire(self, key: str, time: int) -> None: 18 | ... 19 | 20 | async def exists(self, key: str) -> bool: 21 | return key in self.db 22 | 23 | async def zrange( 24 | self, 25 | key: str, 26 | start: int = 0, 27 | stop: int = -1, 28 | withscores: bool = False, 29 | ) -> Optional[RangeOutput]: 30 | data: Optional[RangeOutput] = self.db.get(key) 31 | 32 | if data is None: 33 | return None 34 | 35 | return [i[0] for i in self.db[key]] 36 | 37 | async def zadd( 38 | self, key: str, score: float, member: str, *pairs: Union[float, str] 39 | ) -> None: 40 | data = [score, member] + list(pairs) 41 | self.db[key] = sorted( 42 | [ 43 | ( 44 | data[i].encode() if isinstance(data[i], str) else data[i], # type: ignore 45 | data[i - 1], 46 | ) 47 | for i in range(1, len(data), 2) 48 | ], 49 | key=lambda d: d[1], 50 | ) 51 | 52 | async def hmset( 53 | self, 54 | key: str, 55 | field: Union[str, bytes], 56 | value: Union[str, bytes], 57 | *pairs: Union[str, bytes], 58 | ) -> None: 59 | data = [field, value] + list(pairs) 60 | self.db[key] = { 61 | f.encode() 62 | if isinstance(f := data[i - 1], str) # noqa 63 | else (f if isinstance(f, bytes) else str(f).encode()): v.encode() 64 | if isinstance(v := data[i], str) # noqa 65 | else (v if isinstance(v, bytes) else str(v).encode()) 66 | for i in range(1, len(data), 2) 67 | } 68 | 69 | async def hmget( 70 | self, key: str, field: Union[str, bytes], *fields: Union[str, bytes] 71 | ) -> Sequence[Optional[bytes]]: 72 | data: Dict[bytes, Any] = self.db.get(key, {}) 73 | 74 | return [ 75 | None 76 | if (d := data.get(f.encode() if isinstance(f, str) else f)) # noqa 77 | is None 78 | else ( 79 | d 80 | if isinstance(d, bytes) 81 | else (d.encode() if isinstance(d, str) else str(d).encode()) 82 | ) 83 | for f in (field,) + fields 84 | ] 85 | 86 | async def hgetall(self, key: str) -> Dict[bytes, bytes]: 87 | return { 88 | f: d.encode() 89 | if isinstance(d, str) 90 | else ( 91 | d 92 | if isinstance(d, bytes) 93 | else (d.encode() if isinstance(d, str) else str(d).encode()) 94 | ) 95 | for f, d in self.db.get(key, {}).items() 96 | } 97 | -------------------------------------------------------------------------------- /dbdaora/entity.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import Tuple, Type, TypeVar 3 | 4 | 5 | Entity = TypeVar('Entity') 6 | 7 | EntityData = TypeVar('EntityData') 8 | 9 | 10 | def init_subclass(cls: Type[Entity], bases: Tuple[Type[Entity], ...]) -> None: 11 | if cls not in getattr(cls, '__skip_dataclass__', ()): 12 | if not hasattr(cls, '__annotations__'): 13 | cls.__annotations__ = {'id': str} 14 | 15 | for base_cls in bases: 16 | cls.__annotations__.update( 17 | { 18 | k: v 19 | for k, v in base_cls.__annotations__.items() 20 | if k not in cls.__annotations__ 21 | } 22 | ) 23 | 24 | dataclasses.dataclass(cls) 25 | -------------------------------------------------------------------------------- /dbdaora/exceptions.py: -------------------------------------------------------------------------------- 1 | class DBDaoraError(Exception): 2 | ... 3 | 4 | 5 | class EntityNotFoundError(DBDaoraError): 6 | ... 7 | 8 | 9 | class InvalidQueryError(DBDaoraError): 10 | ... 11 | 12 | 13 | class InvalidHashAttribute(DBDaoraError): 14 | ... 15 | 16 | 17 | class InvalidEntityAnnotationError(DBDaoraError): 18 | ... 19 | 20 | 21 | class RequiredKeyAttributeError(DBDaoraError): 22 | ... 23 | 24 | 25 | class InvalidKeyAttributeError(DBDaoraError): 26 | ... 27 | 28 | 29 | class InvalidEntityTypeError(DBDaoraError): 30 | ... 31 | 32 | 33 | class RequiredClassAttributeError(DBDaoraError): 34 | ... 35 | 36 | 37 | class InvalidGeoSpatialDataError(DBDaoraError): 38 | ... 39 | 40 | 41 | class CacheNotAvailableError(DBDaoraError): 42 | ... 43 | -------------------------------------------------------------------------------- /dbdaora/geospatial/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dutradda/dbdaora/5c87a3818e9d736bbf5e1438edc5929a2f5acd3f/dbdaora/geospatial/__init__.py -------------------------------------------------------------------------------- /dbdaora/geospatial/_tests/datastore/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from dbdaora import DatastoreGeoSpatialRepository, KindKeyDatastoreDataSource 4 | 5 | 6 | @pytest.fixture 7 | def fallback_data_source(): 8 | return KindKeyDatastoreDataSource() 9 | 10 | 11 | @pytest.fixture 12 | def fake_repository_cls(fake_entity_cls): 13 | class FakeGeoSpatialRepository(DatastoreGeoSpatialRepository): 14 | name = 'fake' 15 | key_attrs = ('fake2_id', 'fake_id') 16 | entity_cls = fake_entity_cls 17 | 18 | return FakeGeoSpatialRepository 19 | -------------------------------------------------------------------------------- /dbdaora/geospatial/_tests/datastore/test_integration_service_geospatial_aioredis_datastore_add.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from aioredis import RedisError 4 | 5 | from dbdaora import EntityNotFoundError 6 | 7 | 8 | @pytest.fixture 9 | def has_add_cb(): 10 | return True 11 | 12 | 13 | @pytest.mark.asyncio 14 | async def test_should_add( 15 | fake_service, fake_entity, fake_entity_add, fake_entity_add2 16 | ): 17 | await fake_service.add(fake_entity_add) 18 | await fake_service.add(fake_entity_add2) 19 | 20 | entity = await fake_service.get_one( 21 | fake_id=fake_entity.fake_id, 22 | fake2_id=fake_entity.fake2_id, 23 | latitude=5, 24 | longitude=6, 25 | max_distance=1, 26 | ) 27 | 28 | assert entity == fake_entity 29 | 30 | 31 | @pytest.mark.asyncio 32 | async def test_should_add_to_fallback_after_open_circuit_breaker( 33 | fake_service, fake_entity, fake_entity_add, fake_entity_add2 34 | ): 35 | fake_exists = asynctest.CoroutineMock(side_effect=[True, True]) 36 | fake_geoadd = asynctest.CoroutineMock(side_effect=RedisError) 37 | fake_service.repository.memory_data_source.exists = fake_exists 38 | fake_service.repository.memory_data_source.geoadd = fake_geoadd 39 | await fake_service.add(fake_entity_add) 40 | await fake_service.add(fake_entity_add2) 41 | 42 | with pytest.raises(EntityNotFoundError): 43 | await fake_service.get_one( 44 | fake_id=fake_entity.fake_id, 45 | fake2_id=fake_entity.fake2_id, 46 | latitude=5, 47 | longitude=6, 48 | max_distance=1, 49 | ) 50 | 51 | assert fake_service.logger.warning.call_count == 3 52 | -------------------------------------------------------------------------------- /dbdaora/geospatial/_tests/datastore/test_integration_service_geospatial_aioredis_datastore_get_one.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.mark.asyncio 5 | async def test_should_get_one( 6 | fake_service, fake_entity, repository, fake_fallback_data_entity 7 | ): 8 | await repository.memory_data_source.delete('fake:fake2:fake') 9 | key = fake_service.repository.fallback_data_source.make_key( 10 | 'fake', 'fake2', 'fake' 11 | ) 12 | await fake_service.repository.fallback_data_source.put( 13 | key, fake_fallback_data_entity 14 | ) 15 | entity = await repository.query( 16 | fake_id=fake_entity.fake_id, 17 | fake2_id=fake_entity.fake2_id, 18 | latitude=5, 19 | longitude=6, 20 | max_distance=1, 21 | ).entity 22 | 23 | assert entity == fake_entity 24 | -------------------------------------------------------------------------------- /dbdaora/geospatial/_tests/mongodb/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | from motor.motor_asyncio import AsyncIOMotorClient 5 | 6 | from dbdaora import CollectionKeyMongoDataSource, MongodbGeoSpatialRepository 7 | 8 | 9 | @pytest.fixture 10 | def fallback_data_source(event_loop): 11 | auth = os.environ.get('MONGO_AUTH', 'mongo:mongo') 12 | client = AsyncIOMotorClient( 13 | f'mongodb://{auth}@localhost:27017', io_loop=event_loop 14 | ) 15 | return CollectionKeyMongoDataSource(database_name='dbdaora', client=client) 16 | 17 | 18 | @pytest.fixture 19 | def fake_repository_cls(fake_entity_cls): 20 | class FakeGeoSpatialRepository(MongodbGeoSpatialRepository): 21 | name = 'fake' 22 | key_attrs = ('fake2_id', 'fake_id') 23 | entity_cls = fake_entity_cls 24 | 25 | return FakeGeoSpatialRepository 26 | -------------------------------------------------------------------------------- /dbdaora/geospatial/_tests/mongodb/test_integration_service_geospatial_aioredis_mongodb_add.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from aioredis import RedisError 4 | 5 | from dbdaora import EntityNotFoundError 6 | 7 | 8 | @pytest.fixture 9 | def has_add_cb(): 10 | return True 11 | 12 | 13 | @pytest.mark.asyncio 14 | async def test_should_add( 15 | fake_service, fake_entity, fake_entity_add, fake_entity_add2 16 | ): 17 | await fake_service.add(fake_entity_add) 18 | await fake_service.add(fake_entity_add2) 19 | 20 | entity = await fake_service.get_one( 21 | fake_id=fake_entity.fake_id, 22 | fake2_id=fake_entity.fake2_id, 23 | latitude=5, 24 | longitude=6, 25 | max_distance=1, 26 | ) 27 | 28 | assert entity == fake_entity 29 | 30 | 31 | @pytest.mark.asyncio 32 | async def test_should_add_to_fallback_after_open_circuit_breaker( 33 | fake_service, fake_entity, fake_entity_add, fake_entity_add2 34 | ): 35 | fake_exists = asynctest.CoroutineMock(side_effect=[True, True]) 36 | fake_geoadd = asynctest.CoroutineMock(side_effect=RedisError) 37 | fake_service.repository.memory_data_source.exists = fake_exists 38 | fake_service.repository.memory_data_source.geoadd = fake_geoadd 39 | await fake_service.add(fake_entity_add) 40 | await fake_service.add(fake_entity_add2) 41 | 42 | with pytest.raises(EntityNotFoundError): 43 | await fake_service.get_one( 44 | fake_id=fake_entity.fake_id, 45 | fake2_id=fake_entity.fake2_id, 46 | latitude=5, 47 | longitude=6, 48 | max_distance=1, 49 | ) 50 | 51 | assert fake_service.logger.warning.call_count == 3 52 | -------------------------------------------------------------------------------- /dbdaora/geospatial/_tests/mongodb/test_integration_service_geospatial_aioredis_mongodb_get_one.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.mark.asyncio 5 | async def test_should_get_one( 6 | fake_service, fake_entity, repository, fake_fallback_data_entity 7 | ): 8 | await repository.memory_data_source.delete('fake:fake2:fake') 9 | key = fake_service.repository.fallback_data_source.make_key( 10 | 'fake', 'fake2', 'fake' 11 | ) 12 | await fake_service.repository.fallback_data_source.put( 13 | key, fake_fallback_data_entity 14 | ) 15 | entity = await repository.query( 16 | fake_id=fake_entity.fake_id, 17 | fake2_id=fake_entity.fake2_id, 18 | latitude=5, 19 | longitude=6, 20 | max_distance=1, 21 | ).entity 22 | 23 | assert entity == fake_entity 24 | -------------------------------------------------------------------------------- /dbdaora/geospatial/_tests/test_integration_service_geospatial_aioredis_add.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from aioredis import RedisError 4 | from circuitbreaker import CircuitBreakerError 5 | from pymongo.errors import PyMongoError 6 | 7 | from dbdaora import EntityNotFoundError 8 | 9 | 10 | @pytest.fixture 11 | def has_add_cb(): 12 | return True 13 | 14 | 15 | @pytest.mark.asyncio 16 | async def test_should_add( 17 | fake_service, fake_entity, fake_entity_add, fake_entity_add2 18 | ): 19 | await fake_service.add(fake_entity_add) 20 | await fake_service.add(fake_entity_add2) 21 | 22 | entity = await fake_service.get_one( 23 | fake_id=fake_entity.fake_id, 24 | fake2_id=fake_entity.fake2_id, 25 | latitude=5, 26 | longitude=6, 27 | max_distance=1, 28 | ) 29 | 30 | assert entity == fake_entity 31 | 32 | 33 | @pytest.mark.asyncio 34 | async def test_should_add_to_fallback_after_open_circuit_breaker( 35 | fake_service, fake_entity, fake_entity_add, fake_entity_add2 36 | ): 37 | fake_exists = asynctest.CoroutineMock(side_effect=[True, True]) 38 | fake_geoadd = asynctest.CoroutineMock(side_effect=RedisError) 39 | fake_service.repository.memory_data_source.exists = fake_exists 40 | fake_service.repository.memory_data_source.geoadd = fake_geoadd 41 | await fake_service.add(fake_entity_add) 42 | await fake_service.add(fake_entity_add2) 43 | 44 | with pytest.raises(EntityNotFoundError): 45 | await fake_service.get_one( 46 | fake_id=fake_entity.fake_id, 47 | fake2_id=fake_entity.fake2_id, 48 | latitude=5, 49 | longitude=6, 50 | max_distance=1, 51 | ) 52 | 53 | assert fake_service.logger.warning.call_count == 3 54 | 55 | 56 | @pytest.mark.asyncio 57 | async def test_should_not_add_to_fallback_after_open_fallback_circuit_breaker( 58 | fake_service, fake_entity, fake_entity_add, fake_entity_add2 59 | ): 60 | fake_put = asynctest.CoroutineMock(side_effect=PyMongoError) 61 | fake_service.repository.fallback_data_source.put = fake_put 62 | 63 | with pytest.raises(CircuitBreakerError): 64 | await fake_service.add(fake_entity_add, memory=False) 65 | 66 | assert fake_service.logger.warning.call_count == 1 67 | -------------------------------------------------------------------------------- /dbdaora/geospatial/_tests/test_integration_service_geospatial_aioredis_get_one.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | import pytest 4 | 5 | 6 | @pytest.mark.asyncio 7 | async def test_should_get_one( 8 | fake_service, serialized_fake_entity, fake_entity, repository 9 | ): 10 | await repository.memory_data_source.geoadd( 11 | 'fake:fake2:fake', *itertools.chain(*serialized_fake_entity) 12 | ) 13 | entity = await repository.query( 14 | fake_id=fake_entity.fake_id, 15 | fake2_id=fake_entity.fake2_id, 16 | latitude=5, 17 | longitude=6, 18 | max_distance=1, 19 | ).entity 20 | 21 | assert entity == fake_entity 22 | -------------------------------------------------------------------------------- /dbdaora/geospatial/entity.py: -------------------------------------------------------------------------------- 1 | from typing import ( # type: ignore 2 | Any, 3 | Dict, 4 | Protocol, 5 | Tuple, 6 | Type, 7 | TypeVar, 8 | TypedDict, 9 | Union, 10 | _TypedDictMeta, 11 | ) 12 | 13 | from dbdaora.data_sources.memory import GeoMember, GeoRadiusOutput 14 | from dbdaora.entity import init_subclass 15 | 16 | 17 | GeoSpatialData = Union[GeoMember, GeoRadiusOutput] 18 | 19 | 20 | class GeoSpatialEntityProtocol(Protocol): 21 | data: GeoSpatialData 22 | 23 | def __init__(self, *, data: GeoSpatialData, **kwargs: Any): 24 | ... 25 | 26 | 27 | class GeoSpatialEntity(GeoSpatialEntityProtocol): 28 | data: GeoSpatialData 29 | 30 | def __init_subclass__(cls) -> None: 31 | init_subclass(cls, (GeoSpatialEntity,)) 32 | 33 | 34 | class GeoSpatialDictEntityMeta(_TypedDictMeta): # type: ignore 35 | def __init__( 36 | cls, name: str, bases: Tuple[Type[Any], ...], attrs: Dict[str, Any] 37 | ): 38 | super().__init__(name, bases, attrs) 39 | init_subclass(cls, bases) 40 | 41 | 42 | class GeoSpatialDictEntity(TypedDict, metaclass=GeoSpatialDictEntityMeta): 43 | data: GeoSpatialData 44 | 45 | 46 | GeoSpatialEntityHint = TypeVar( 47 | 'GeoSpatialEntityHint', 48 | bound=Union[GeoSpatialEntityProtocol, GeoSpatialDictEntity], 49 | ) 50 | -------------------------------------------------------------------------------- /dbdaora/geospatial/factory.py: -------------------------------------------------------------------------------- 1 | from logging import Logger, getLogger 2 | from typing import Any, Optional, Type 3 | 4 | from dbdaora.entity import Entity, EntityData 5 | from dbdaora.keys import FallbackKey 6 | from dbdaora.service.builder import build as build_base_service 7 | 8 | from ..repository import MemoryRepository 9 | from ..service import Service 10 | from .service import GeoSpatialService 11 | 12 | 13 | async def make_service( 14 | repository_cls: Type[MemoryRepository[Entity, EntityData, FallbackKey]], 15 | memory_data_source_factory: Any, 16 | fallback_data_source_factory: Any, 17 | repository_expire_time: int, 18 | cb_failure_threshold: Optional[int] = None, 19 | cb_recovery_timeout: Optional[int] = None, 20 | cb_expected_exception: Optional[Type[Exception]] = None, 21 | cb_expected_fallback_exception: Optional[Type[Exception]] = None, 22 | logger: Logger = getLogger(__name__), 23 | has_add_circuit_breaker: bool = False, 24 | has_delete_circuit_breaker: bool = False, 25 | ) -> Service[Entity, EntityData, FallbackKey]: 26 | return await build_base_service( 27 | GeoSpatialService, # type: ignore 28 | repository_cls, 29 | memory_data_source_factory, 30 | fallback_data_source_factory, 31 | repository_expire_time, 32 | cb_failure_threshold=cb_failure_threshold, 33 | cb_recovery_timeout=cb_recovery_timeout, 34 | cb_expected_exception=cb_expected_exception, 35 | cb_expected_fallback_exception=cb_expected_fallback_exception, 36 | logger=logger, 37 | has_add_circuit_breaker=has_add_circuit_breaker, 38 | has_delete_circuit_breaker=has_delete_circuit_breaker, 39 | ) 40 | -------------------------------------------------------------------------------- /dbdaora/geospatial/query.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from enum import Enum 3 | from typing import Any, List, Optional 4 | 5 | from dbdaora.keys import FallbackKey 6 | from dbdaora.query import BaseQuery, Query 7 | 8 | 9 | class GeoSpatialQueryType(Enum): 10 | RADIUS = 'radius' 11 | 12 | 13 | @dataclasses.dataclass(init=False) 14 | class GeoSpatialQuery( 15 | Query['GeoSpatialEntityHint', 'GeoSpatialData', FallbackKey] 16 | ): 17 | repository: 'GeoSpatialRepository[GeoSpatialEntityHint, FallbackKey]' 18 | type: GeoSpatialQueryType = GeoSpatialQueryType.RADIUS 19 | latitude: Optional[float] = None 20 | longitude: Optional[float] = None 21 | max_distance: Optional[float] = None 22 | distance_unit: str = 'km' 23 | with_dist: bool = True 24 | with_coord: bool = True 25 | count: Optional[int] = None 26 | 27 | def __init__( 28 | self, 29 | repository: 'GeoSpatialRepository[GeoSpatialEntityHint, FallbackKey]', 30 | *args: Any, 31 | memory: bool = True, 32 | key_parts: Optional[List[Any]] = None, 33 | type: GeoSpatialQueryType = GeoSpatialQueryType.RADIUS, 34 | latitude: Optional[float] = None, 35 | longitude: Optional[float] = None, 36 | max_distance: Optional[float] = None, 37 | distance_unit: str = 'km', 38 | with_dist: bool = True, 39 | with_coord: bool = True, 40 | count: Optional[int] = None, 41 | **kwargs: Any, 42 | ): 43 | super().__init__( 44 | repository, memory=memory, key_parts=key_parts, *args, **kwargs, 45 | ) 46 | self.type = type 47 | self.latitude = latitude 48 | self.longitude = longitude 49 | self.max_distance = max_distance 50 | self.distance_unit = distance_unit 51 | self.with_dist = with_dist 52 | self.with_coord = with_coord 53 | self.count = count 54 | 55 | 56 | def make( 57 | *args: Any, **kwargs: Any 58 | ) -> BaseQuery['GeoSpatialEntityHint', 'GeoSpatialData', FallbackKey]: 59 | return GeoSpatialQuery(*args, **kwargs) 60 | 61 | 62 | from .repositories import GeoSpatialRepository # noqa isort:skip 63 | from .entity import GeoSpatialData, GeoSpatialEntityHint # noqa isort:skip 64 | -------------------------------------------------------------------------------- /dbdaora/geospatial/repositories/_tests/test_integration_repository_geospatial_aioredis_datastore.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from aioredis import GeoMember, GeoPoint 3 | from google.cloud import datastore 4 | 5 | from dbdaora import ( 6 | DatastoreGeoSpatialRepository, 7 | GeoSpatialEntity, 8 | KindKeyDatastoreDataSource, 9 | ) 10 | 11 | 12 | class FakeGeoSpatialEntity(GeoSpatialEntity): 13 | id: str 14 | 15 | 16 | class FakeGeoSpatialRepository( 17 | DatastoreGeoSpatialRepository, entity_cls=FakeGeoSpatialEntity 18 | ): 19 | name = 'fakegeo' 20 | 21 | 22 | @pytest.fixture 23 | def fake_repository_cls(): 24 | return FakeGeoSpatialRepository 25 | 26 | 27 | @pytest.fixture 28 | def fallback_data_source(): 29 | return KindKeyDatastoreDataSource() 30 | 31 | 32 | @pytest.mark.asyncio 33 | async def test_should_exclude_all_attributes_from_indexes(repository): 34 | await repository.memory_data_source.delete('fakegeo:fake') 35 | client = repository.fallback_data_source.client 36 | key = client.key('fakegeo:fake', 'fake') 37 | entity = datastore.Entity(key=key) 38 | entity.update({'latitude': 1, 'longitude': 1, 'member': 'fake'}) 39 | client.put(entity) 40 | 41 | assert not client.get(key).exclude_from_indexes 42 | entity = FakeGeoSpatialEntity( 43 | id='fake', 44 | data=GeoMember( 45 | member='fake', dist=None, hash=None, coord=GeoPoint(1, 1) 46 | ), 47 | ) 48 | 49 | await repository.add(entity) 50 | 51 | assert client.get(key).exclude_from_indexes == set( 52 | FakeGeoSpatialRepository.exclude_from_indexes 53 | ) 54 | -------------------------------------------------------------------------------- /dbdaora/geospatial/repositories/datastore.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | from google.cloud.datastore import Entity, Key 4 | from jsondaora.serializers import OrjsonDefaultTypes 5 | 6 | from dbdaora.repository.datastore import DatastoreRepository 7 | 8 | from ..entity import GeoSpatialData, GeoSpatialEntityHint 9 | from . import GeoSpatialRepository 10 | 11 | 12 | OrjsonDefaultTypes.types_default_map[Entity] = lambda e: dict(**e) 13 | 14 | 15 | class DatastoreGeoSpatialRepository( 16 | GeoSpatialRepository[GeoSpatialEntityHint, Key], 17 | DatastoreRepository[GeoSpatialEntityHint, GeoSpatialData], 18 | ): 19 | __skip_cls_validation__ = ('DatastoreGeoSpatialRepository',) 20 | exclude_from_indexes = ('latitude', 'longitude', 'member') 21 | 22 | async def add_fallback( 23 | self, 24 | entity: GeoSpatialEntityHint, 25 | *entities: GeoSpatialEntityHint, 26 | **kwargs: Any, 27 | ) -> None: 28 | await super().add_fallback( 29 | entity, *entities, exclude_from_indexes=self.exclude_from_indexes 30 | ) 31 | -------------------------------------------------------------------------------- /dbdaora/geospatial/repositories/mongodb.py: -------------------------------------------------------------------------------- 1 | from dbdaora.data_sources.fallback.mongodb import Key 2 | 3 | from ..entity import GeoSpatialEntityHint 4 | from . import GeoSpatialRepository 5 | 6 | 7 | class MongodbGeoSpatialRepository( 8 | GeoSpatialRepository[GeoSpatialEntityHint, Key] 9 | ): 10 | __skip_cls_validation__ = ('MongodbGeoSpatialRepository',) 11 | fallback_data_source_key_cls = Key 12 | -------------------------------------------------------------------------------- /dbdaora/geospatial/service/__init__.py: -------------------------------------------------------------------------------- 1 | from logging import Logger, getLogger 2 | from typing import Any, AsyncGenerator, Optional, Sequence, Tuple, Union 3 | 4 | from cachetools import Cache 5 | 6 | from ...circuitbreaker import AsyncCircuitBreaker 7 | from ...entity import Entity 8 | from ...keys import FallbackKey 9 | from ...repository import MemoryRepository 10 | from ...service import CacheAlreadyNotFound, Service 11 | from ..entity import GeoSpatialData 12 | 13 | 14 | class GeoSpatialService(Service[Entity, GeoSpatialData, FallbackKey]): 15 | def __init__( 16 | self, 17 | repository: MemoryRepository[Entity, GeoSpatialData, FallbackKey], 18 | circuit_breaker: AsyncCircuitBreaker, 19 | fallback_circuit_breaker: AsyncCircuitBreaker, 20 | logger: Logger = getLogger(__name__), 21 | has_add_circuit_breaker: bool = False, 22 | has_delete_circuit_breaker: bool = False, 23 | ): 24 | super().__init__( 25 | repository=repository, 26 | circuit_breaker=circuit_breaker, 27 | fallback_circuit_breaker=fallback_circuit_breaker, 28 | logger=logger, 29 | has_add_circuit_breaker=has_add_circuit_breaker, 30 | has_delete_circuit_breaker=has_delete_circuit_breaker, 31 | ) 32 | 33 | def get_many( 34 | self, *ids: str, **filters: Any, 35 | ) -> AsyncGenerator[Entity, None]: 36 | raise NotImplementedError() # pragma: no cover 37 | 38 | def get_many_cached( 39 | self, 40 | ids: Sequence[str], 41 | cache: Cache, 42 | memory: bool = True, 43 | **filters: Any, 44 | ) -> AsyncGenerator[Entity, None]: 45 | raise NotImplementedError() # pragma: no cover 46 | 47 | def get_cached_entity( 48 | self, id: Union[str, Tuple[str, ...]], key_suffix: str, **filters: Any, 49 | ) -> Any: 50 | raise NotImplementedError() # pragma: no cover 51 | 52 | def cache_key(self, id: Union[str, Tuple[str, ...]], suffix: str) -> str: 53 | raise NotImplementedError() # pragma: no cover 54 | 55 | def set_cached_entity( 56 | self, 57 | id: Union[str, Tuple[str, ...]], 58 | key_suffix: str, 59 | entity: Union[Entity, CacheAlreadyNotFound], 60 | ) -> None: 61 | raise NotImplementedError() # pragma: no cover 62 | 63 | def cache_key_suffix(self, **filters: Any) -> str: 64 | raise NotImplementedError() # pragma: no cover 65 | 66 | async def get_one_cached( 67 | self, cache: Cache, memory: bool = True, **filters: Any, 68 | ) -> Any: 69 | raise NotImplementedError() # pragma: no cover 70 | 71 | async def delete( 72 | self, entity_id: Optional[str] = None, **filters: Any 73 | ) -> None: 74 | raise NotImplementedError() # pragma: no cover 75 | 76 | async def exists(self, id: Optional[str] = None, **filters: Any) -> bool: 77 | raise NotImplementedError() # pragma: no cover 78 | 79 | async def exists_cached( 80 | self, cache: Cache, memory: bool = True, **filters: Any, 81 | ) -> bool: 82 | raise NotImplementedError() # pragma: no cover 83 | -------------------------------------------------------------------------------- /dbdaora/geospatial/service/datastore.py: -------------------------------------------------------------------------------- 1 | from google.cloud.datastore import Key 2 | 3 | from ...entity import Entity 4 | from . import GeoSpatialService 5 | 6 | 7 | class DatastoreGeoSpatialService(GeoSpatialService[Entity, Key]): 8 | ... 9 | -------------------------------------------------------------------------------- /dbdaora/geospatial/service/mongodb.py: -------------------------------------------------------------------------------- 1 | from ...data_sources.fallback.mongodb import Key as MongoKey 2 | from ...entity import Entity 3 | from . import GeoSpatialService 4 | 5 | 6 | class MongoGeoSpatialService(GeoSpatialService[Entity, MongoKey]): 7 | ... 8 | -------------------------------------------------------------------------------- /dbdaora/hash/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dutradda/dbdaora/5c87a3818e9d736bbf5e1438edc5929a2f5acd3f/dbdaora/hash/__init__.py -------------------------------------------------------------------------------- /dbdaora/hash/_tests/conftest.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | 3 | import pytest 4 | from aioredis import RedisError 5 | 6 | from dbdaora import ( 7 | CacheType, 8 | DictFallbackDataSource, 9 | HashService, 10 | build_service, 11 | make_aioredis_data_source, 12 | ) 13 | 14 | 15 | @pytest.mark.asyncio 16 | @pytest.fixture 17 | async def fake_service( 18 | mocker, 19 | fallback_data_source, 20 | fake_hash_repository_cls, 21 | has_add_cb, 22 | has_delete_cb, 23 | ): 24 | memory_data_source_factory = partial( 25 | make_aioredis_data_source, 26 | 'redis://', 27 | 'redis://localhost/1', 28 | 'redis://localhost/2', 29 | ) 30 | 31 | async def fallback_data_source_factory(): 32 | return fallback_data_source 33 | 34 | service = await build_service( 35 | HashService, 36 | fake_hash_repository_cls, 37 | memory_data_source_factory, 38 | fallback_data_source_factory, 39 | repository_expire_time=1, 40 | cache_type=CacheType.TTL, 41 | cache_ttl=1, 42 | cache_max_size=2, 43 | cb_failure_threshold=0, 44 | cb_recovery_timeout=10, 45 | cb_expected_exception=RedisError, 46 | cb_expected_fallback_exception=KeyError, 47 | logger=mocker.MagicMock(), 48 | has_add_circuit_breaker=has_add_cb, 49 | has_delete_circuit_breaker=has_delete_cb, 50 | ) 51 | 52 | yield service 53 | 54 | await service.shutdown() 55 | 56 | 57 | @pytest.fixture 58 | def fallback_data_source(): 59 | return DictFallbackDataSource() 60 | -------------------------------------------------------------------------------- /dbdaora/hash/_tests/datastore/conftest.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import List, Optional 3 | 4 | import pytest 5 | 6 | from dbdaora import DatastoreDataSource, DatastoreHashRepository 7 | 8 | 9 | @pytest.fixture 10 | def fallback_data_source(): 11 | return DatastoreDataSource() 12 | 13 | 14 | @dataclasses.dataclass 15 | class FakeInnerEntity: 16 | id: str 17 | 18 | 19 | @dataclasses.dataclass 20 | class FakeDatastoreEntity: 21 | id: str 22 | other_id: str 23 | integer: int 24 | inner_entities: List[FakeInnerEntity] 25 | number: Optional[float] = None 26 | boolean: Optional[bool] = None 27 | 28 | 29 | @pytest.fixture 30 | def fake_entity(): 31 | return FakeDatastoreEntity( 32 | id='fake', 33 | other_id='other_fake', 34 | inner_entities=[FakeInnerEntity('inner1'), FakeInnerEntity('inner2')], 35 | integer=1, 36 | number=0.1, 37 | boolean=True, 38 | ) 39 | 40 | 41 | @pytest.fixture 42 | def fake_entity2(): 43 | return FakeDatastoreEntity( 44 | id='fake2', 45 | other_id='other_fake', 46 | inner_entities=[FakeInnerEntity('inner3'), FakeInnerEntity('inner4')], 47 | integer=2, 48 | number=0.2, 49 | boolean=False, 50 | ) 51 | 52 | 53 | @pytest.fixture 54 | def fake_entity3(): 55 | return FakeDatastoreEntity( 56 | id='fake3', 57 | other_id='other_fake', 58 | inner_entities=[FakeInnerEntity('inner3'), FakeInnerEntity('inner4')], 59 | integer=2, 60 | number=0.2, 61 | boolean=False, 62 | ) 63 | 64 | 65 | @pytest.fixture 66 | def serialized_fake_entity(): 67 | return { 68 | b'id': b'fake', 69 | b'other_id': b'other_fake', 70 | b'integer': b'1', 71 | b'number': b'0.1', 72 | b'boolean': b'1', 73 | b'inner_entities': b'[{"id":"inner1"},{"id":"inner2"}]', 74 | } 75 | 76 | 77 | @pytest.fixture 78 | def serialized_fake_entity2(): 79 | return { 80 | b'id': b'fake2', 81 | b'other_id': b'other_fake', 82 | b'integer': b'2', 83 | b'number': b'0.2', 84 | b'boolean': b'0', 85 | b'inner_entities': b'[{"id":"inner3"},{"id":"inner4"}]', 86 | } 87 | 88 | 89 | @pytest.fixture 90 | def serialized_fake_entity3(): 91 | return { 92 | b'id': b'fake3', 93 | b'other_id': b'other_fake', 94 | b'integer': b'2', 95 | b'number': b'0.2', 96 | b'boolean': b'0', 97 | b'inner_entities': b'[{"id":"inner3"},{"id":"inner4"}]', 98 | } 99 | 100 | 101 | class FakeDatastoreHashRepository(DatastoreHashRepository): 102 | name = 'fake' 103 | key_attrs = ('other_id', 'id') 104 | many_key_attrs = ('id',) 105 | entity_cls = FakeDatastoreEntity 106 | 107 | 108 | @pytest.fixture 109 | def fake_hash_repository_cls(): 110 | return FakeDatastoreHashRepository 111 | -------------------------------------------------------------------------------- /dbdaora/hash/_tests/datastore/test_integration_service_hash_aioredis_datastore_add.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from aioredis import RedisError 4 | 5 | 6 | @pytest.fixture 7 | def has_add_cb(): 8 | return True 9 | 10 | 11 | @pytest.mark.asyncio 12 | async def test_should_add(fake_service, fake_entity): 13 | await fake_service.repository.add(fake_entity) 14 | 15 | entity = await fake_service.get_one('fake', other_id='other_fake') 16 | 17 | assert entity == fake_entity 18 | 19 | 20 | @pytest.mark.asyncio 21 | async def test_should_add_to_fallback_after_open_circuit_breaker( 22 | fake_service, fake_entity 23 | ): 24 | fake_exists = asynctest.CoroutineMock(side_effect=RedisError) 25 | fake_service.repository.memory_data_source.exists = fake_exists 26 | await fake_service.add(fake_entity) 27 | 28 | entity = await fake_service.get_one('fake', other_id='other_fake') 29 | 30 | assert entity == fake_entity 31 | assert fake_service.logger.warning.call_count == 2 32 | -------------------------------------------------------------------------------- /dbdaora/hash/_tests/datastore/test_integration_service_hash_aioredis_datastore_delete.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from aioredis import RedisError 4 | from jsondaora import dataclasses 5 | 6 | from dbdaora.exceptions import EntityNotFoundError 7 | 8 | 9 | @pytest.fixture 10 | def has_delete_cb(): 11 | return True 12 | 13 | 14 | @pytest.mark.asyncio 15 | async def test_should_delete( 16 | fake_service, serialized_fake_entity, fake_entity 17 | ): 18 | await fake_service.add(fake_entity) 19 | 20 | assert await fake_service.get_one('fake', other_id='other_fake') 21 | 22 | await fake_service.delete(fake_entity.id, other_id='other_fake') 23 | fake_service.cache.clear() 24 | 25 | with pytest.raises(EntityNotFoundError): 26 | await fake_service.get_one('fake', other_id='other_fake') 27 | 28 | 29 | @pytest.mark.asyncio 30 | async def test_should_delete_from_fallback_after_open_circuit_breaker( 31 | fake_service, serialized_fake_entity, fake_entity, mocker 32 | ): 33 | await fake_service.repository.memory_data_source.delete( 34 | 'fake:other_fake:fake' 35 | ) 36 | await fake_service.repository.memory_data_source.delete( 37 | 'fake:not-found:other_fake:fake' 38 | ) 39 | key = fake_service.repository.fallback_data_source.make_key( 40 | 'fake', 'other_fake', 'fake' 41 | ) 42 | await fake_service.repository.fallback_data_source.put( 43 | key, dataclasses.asdict(fake_entity, dumps_value=True) 44 | ) 45 | 46 | assert await fake_service.get_one('fake', other_id='other_fake') 47 | 48 | fake_service.repository.memory_data_source.delete = asynctest.CoroutineMock( 49 | side_effect=RedisError 50 | ) 51 | 52 | await fake_service.delete(fake_entity.id, other_id='other_fake') 53 | fake_service.cache.clear() 54 | 55 | with pytest.raises(EntityNotFoundError): 56 | await fake_service.get_one('fake', other_id='other_fake') 57 | 58 | assert fake_service.logger.warning.call_count == 2 59 | -------------------------------------------------------------------------------- /dbdaora/hash/_tests/datastore/test_integration_service_hash_aioredis_datastore_get_one.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | import asynctest 4 | import pytest 5 | from aioredis import RedisError 6 | from jsondaora import dataclasses 7 | 8 | 9 | @pytest.mark.asyncio 10 | async def test_should_get_one( 11 | fake_service, serialized_fake_entity, fake_entity 12 | ): 13 | await fake_service.repository.memory_data_source.hmset( 14 | 'fake:other_fake:fake', 15 | *itertools.chain(*serialized_fake_entity.items()), 16 | ) 17 | 18 | entity = await fake_service.get_one('fake', other_id='other_fake') 19 | 20 | assert entity == fake_entity 21 | 22 | 23 | @pytest.mark.asyncio 24 | async def test_should_get_one_with_fields( 25 | fake_service, serialized_fake_entity, fake_entity 26 | ): 27 | await fake_service.repository.memory_data_source.hmset( 28 | 'fake:other_fake:fake', 29 | *itertools.chain(*serialized_fake_entity.items()), 30 | ) 31 | fake_entity.number = None 32 | fake_entity.boolean = None 33 | 34 | entity = await fake_service.get_one( 35 | 'fake', 36 | fields=['id', 'other_id', 'integer', 'inner_entities'], 37 | other_id='other_fake', 38 | ) 39 | 40 | assert entity == fake_entity 41 | 42 | 43 | @pytest.mark.asyncio 44 | async def test_should_get_one_from_cache( 45 | fake_service, serialized_fake_entity, fake_entity 46 | ): 47 | fake_service.repository.memory_data_source.hgetall = ( 48 | asynctest.CoroutineMock() 49 | ) 50 | fake_service.cache['fakeother_idother_fake'] = fake_entity 51 | 52 | entity = await fake_service.get_one('fake', other_id='other_fake') 53 | 54 | assert entity == fake_entity 55 | assert not fake_service.repository.memory_data_source.hgetall.called 56 | 57 | 58 | @pytest.mark.asyncio 59 | async def test_should_get_one_from_fallback_when_not_found_on_memory( 60 | fake_service, serialized_fake_entity, fake_entity 61 | ): 62 | await fake_service.repository.memory_data_source.delete( 63 | 'fake:other_fake:fake' 64 | ) 65 | await fake_service.repository.memory_data_source.delete( 66 | 'fake:not-found:other_fake:fake' 67 | ) 68 | await fake_service.repository.fallback_data_source.put( 69 | fake_service.repository.fallback_data_source.make_key( 70 | 'fake', 'other_fake:fake' 71 | ), 72 | dataclasses.asdict(fake_entity), 73 | ) 74 | 75 | entity = await fake_service.get_one('fake', other_id='other_fake') 76 | 77 | assert entity == fake_entity 78 | assert fake_service.repository.memory_data_source.exists( 79 | 'fake:other_fake:fake' 80 | ) 81 | 82 | 83 | @pytest.mark.asyncio 84 | async def test_should_get_one_from_fallback_when_not_found_on_memory_with_fields( 85 | fake_service, serialized_fake_entity, fake_entity 86 | ): 87 | await fake_service.repository.memory_data_source.delete( 88 | 'fake:other_fake:fake' 89 | ) 90 | await fake_service.repository.fallback_data_source.put( 91 | fake_service.repository.fallback_data_source.make_key( 92 | 'fake', 'other_fake:fake' 93 | ), 94 | dataclasses.asdict(fake_entity), 95 | ) 96 | fake_entity.number = None 97 | fake_entity.boolean = None 98 | 99 | entity = await fake_service.get_one( 100 | 'fake', 101 | other_id='other_fake', 102 | fields=['id', 'other_id', 'integer', 'inner_entities'], 103 | ) 104 | 105 | assert entity == fake_entity 106 | assert fake_service.repository.memory_data_source.exists( 107 | 'fake:other_fake:fake' 108 | ) 109 | 110 | 111 | @pytest.mark.asyncio 112 | async def test_should_get_one_from_fallback_after_open_circuit_breaker( 113 | fake_service, fake_entity, mocker 114 | ): 115 | fake_service.repository.memory_data_source.hgetall = asynctest.CoroutineMock( 116 | side_effect=RedisError 117 | ) 118 | key = fake_service.repository.fallback_data_source.make_key( 119 | 'fake', 'other_fake', 'fake' 120 | ) 121 | await fake_service.repository.fallback_data_source.put( 122 | key, dataclasses.asdict(fake_entity) 123 | ) 124 | 125 | entity = await fake_service.get_one('fake', other_id='other_fake') 126 | 127 | assert entity == fake_entity 128 | assert fake_service.logger.warning.call_count == 1 129 | -------------------------------------------------------------------------------- /dbdaora/hash/_tests/mongodb/conftest.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import os 3 | from typing import List, Optional 4 | 5 | import pytest 6 | from motor.motor_asyncio import AsyncIOMotorClient 7 | 8 | from dbdaora import HashRepository, MongoDataSource 9 | 10 | 11 | @pytest.fixture 12 | def fallback_data_source(event_loop): 13 | auth = os.environ.get('MONGO_AUTH', 'mongo:mongo') 14 | client = AsyncIOMotorClient( 15 | f'mongodb://{auth}@localhost:27017', io_loop=event_loop 16 | ) 17 | return MongoDataSource(database_name='dbdaora', client=client) 18 | 19 | 20 | @dataclasses.dataclass 21 | class FakeInnerEntity: 22 | id: str 23 | 24 | 25 | @dataclasses.dataclass 26 | class FakeEntity: 27 | id: str 28 | other_id: str 29 | integer: int 30 | inner_entities: List[FakeInnerEntity] 31 | number: Optional[float] = None 32 | boolean: Optional[bool] = None 33 | 34 | 35 | @pytest.fixture 36 | def fake_entity_cls(): 37 | return FakeEntity 38 | 39 | 40 | @pytest.fixture 41 | def fake_entity(): 42 | return FakeEntity( 43 | id='fake', 44 | other_id='other_fake', 45 | inner_entities=[FakeInnerEntity('inner1'), FakeInnerEntity('inner2')], 46 | integer=1, 47 | number=0.1, 48 | boolean=True, 49 | ) 50 | 51 | 52 | @pytest.fixture 53 | def fake_entity2(): 54 | return FakeEntity( 55 | id='fake2', 56 | other_id='other_fake', 57 | inner_entities=[FakeInnerEntity('inner3'), FakeInnerEntity('inner4')], 58 | integer=2, 59 | number=0.2, 60 | boolean=False, 61 | ) 62 | 63 | 64 | @pytest.fixture 65 | def fake_entity3(): 66 | return FakeEntity( 67 | id='fake3', 68 | other_id='other_fake', 69 | inner_entities=[FakeInnerEntity('inner3'), FakeInnerEntity('inner4')], 70 | integer=2, 71 | number=0.2, 72 | boolean=False, 73 | ) 74 | 75 | 76 | @pytest.fixture 77 | def serialized_fake_entity(): 78 | return { 79 | b'id': b'fake', 80 | b'other_id': b'other_fake', 81 | b'integer': b'1', 82 | b'number': b'0.1', 83 | b'boolean': b'1', 84 | b'inner_entities': b'[{"id":"inner1"},{"id":"inner2"}]', 85 | } 86 | 87 | 88 | @pytest.fixture 89 | def serialized_fake_entity2(): 90 | return { 91 | b'id': b'fake2', 92 | b'other_id': b'other_fake', 93 | b'integer': b'2', 94 | b'number': b'0.2', 95 | b'boolean': b'0', 96 | b'inner_entities': b'[{"id":"inner3"},{"id":"inner4"}]', 97 | } 98 | 99 | 100 | @pytest.fixture 101 | def serialized_fake_entity3(): 102 | return { 103 | b'id': b'fake3', 104 | b'other_id': b'other_fake', 105 | b'integer': b'2', 106 | b'number': b'0.2', 107 | b'boolean': b'0', 108 | b'inner_entities': b'[{"id":"inner3"},{"id":"inner4"}]', 109 | } 110 | 111 | 112 | class FakeHashRepository(HashRepository): 113 | name = 'fake' 114 | key_attrs = ('other_id', 'id') 115 | many_key_attrs = ('id',) 116 | entity_cls = FakeEntity 117 | 118 | 119 | @pytest.fixture 120 | def fake_hash_repository_cls(): 121 | return FakeHashRepository 122 | -------------------------------------------------------------------------------- /dbdaora/hash/_tests/mongodb/test_integration_service_hash_aioredis_mongodb_add.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from aioredis import RedisError 4 | 5 | 6 | @pytest.fixture 7 | def has_add_cb(): 8 | return True 9 | 10 | 11 | @pytest.mark.asyncio 12 | async def test_should_add(fake_service, fake_entity): 13 | await fake_service.repository.add(fake_entity) 14 | 15 | entity = await fake_service.get_one('fake', other_id='other_fake') 16 | 17 | assert entity == fake_entity 18 | 19 | 20 | @pytest.mark.asyncio 21 | async def test_should_add_to_fallback_after_open_circuit_breaker( 22 | fake_service, fake_entity 23 | ): 24 | fake_exists = asynctest.CoroutineMock(side_effect=RedisError) 25 | fake_service.repository.memory_data_source.exists = fake_exists 26 | await fake_service.add(fake_entity) 27 | 28 | entity = await fake_service.get_one('fake', other_id='other_fake') 29 | 30 | assert entity == fake_entity 31 | assert fake_service.logger.warning.call_count == 2 32 | -------------------------------------------------------------------------------- /dbdaora/hash/_tests/mongodb/test_integration_service_hash_aioredis_mongodb_delete.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from aioredis import RedisError 4 | from jsondaora import dataclasses 5 | 6 | from dbdaora.exceptions import EntityNotFoundError 7 | 8 | 9 | @pytest.fixture 10 | def has_delete_cb(): 11 | return True 12 | 13 | 14 | @pytest.mark.asyncio 15 | async def test_should_delete( 16 | fake_service, serialized_fake_entity, fake_entity 17 | ): 18 | await fake_service.add(fake_entity) 19 | 20 | assert await fake_service.get_one('fake', other_id='other_fake') 21 | 22 | await fake_service.delete(fake_entity.id, other_id='other_fake') 23 | fake_service.cache.clear() 24 | 25 | with pytest.raises(EntityNotFoundError): 26 | await fake_service.get_one('fake', other_id='other_fake') 27 | 28 | 29 | @pytest.mark.asyncio 30 | async def test_should_delete_from_fallback_after_open_circuit_breaker( 31 | fake_service, serialized_fake_entity, fake_entity, mocker 32 | ): 33 | await fake_service.repository.memory_data_source.delete( 34 | 'fake:other_fake:fake' 35 | ) 36 | await fake_service.repository.memory_data_source.delete( 37 | 'fake:not-found:other_fake:fake' 38 | ) 39 | key = fake_service.repository.fallback_data_source.make_key( 40 | 'fake', 'other_fake', 'fake' 41 | ) 42 | await fake_service.repository.fallback_data_source.put( 43 | key, dataclasses.asdict(fake_entity, dumps_value=True) 44 | ) 45 | 46 | assert await fake_service.get_one('fake', other_id='other_fake') 47 | 48 | fake_service.repository.memory_data_source.delete = asynctest.CoroutineMock( 49 | side_effect=RedisError 50 | ) 51 | 52 | await fake_service.delete(fake_entity.id, other_id='other_fake') 53 | fake_service.cache.clear() 54 | 55 | with pytest.raises(EntityNotFoundError): 56 | await fake_service.get_one('fake', other_id='other_fake') 57 | 58 | assert fake_service.logger.warning.call_count == 2 59 | -------------------------------------------------------------------------------- /dbdaora/hash/_tests/mongodb/test_integration_service_hash_aioredis_mongodb_get_one.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | import asynctest 4 | import pytest 5 | from aioredis import RedisError 6 | from jsondaora import dataclasses 7 | 8 | 9 | @pytest.mark.asyncio 10 | async def test_should_get_one( 11 | fake_service, serialized_fake_entity, fake_entity 12 | ): 13 | await fake_service.repository.memory_data_source.hmset( 14 | 'fake:other_fake:fake', 15 | *itertools.chain(*serialized_fake_entity.items()), 16 | ) 17 | 18 | entity = await fake_service.get_one('fake', other_id='other_fake') 19 | 20 | assert entity == fake_entity 21 | 22 | 23 | @pytest.mark.asyncio 24 | async def test_should_get_one_with_fields( 25 | fake_service, serialized_fake_entity, fake_entity 26 | ): 27 | await fake_service.repository.memory_data_source.hmset( 28 | 'fake:other_fake:fake', 29 | *itertools.chain(*serialized_fake_entity.items()), 30 | ) 31 | fake_entity.number = None 32 | fake_entity.boolean = None 33 | 34 | entity = await fake_service.get_one( 35 | 'fake', 36 | fields=['id', 'other_id', 'integer', 'inner_entities'], 37 | other_id='other_fake', 38 | ) 39 | 40 | assert entity == fake_entity 41 | 42 | 43 | @pytest.mark.asyncio 44 | async def test_should_get_one_from_cache( 45 | fake_service, serialized_fake_entity, fake_entity 46 | ): 47 | fake_service.repository.memory_data_source.hgetall = ( 48 | asynctest.CoroutineMock() 49 | ) 50 | fake_service.cache['fakeother_idother_fake'] = fake_entity 51 | 52 | entity = await fake_service.get_one('fake', other_id='other_fake') 53 | 54 | assert entity == fake_entity 55 | assert not fake_service.repository.memory_data_source.hgetall.called 56 | 57 | 58 | @pytest.mark.asyncio 59 | async def test_should_get_one_from_fallback_when_not_found_on_memory( 60 | fake_service, serialized_fake_entity, fake_entity 61 | ): 62 | await fake_service.repository.memory_data_source.delete( 63 | 'fake:other_fake:fake' 64 | ) 65 | await fake_service.repository.memory_data_source.delete( 66 | 'fake:not-found:other_fake:fake' 67 | ) 68 | await fake_service.repository.fallback_data_source.put( 69 | fake_service.repository.fallback_data_source.make_key( 70 | 'fake', 'other_fake:fake' 71 | ), 72 | dataclasses.asdict(fake_entity), 73 | ) 74 | 75 | entity = await fake_service.get_one('fake', other_id='other_fake') 76 | 77 | assert entity == fake_entity 78 | assert fake_service.repository.memory_data_source.exists( 79 | 'fake:other_fake:fake' 80 | ) 81 | 82 | 83 | @pytest.mark.asyncio 84 | async def test_should_get_one_from_fallback_when_not_found_on_memory_with_fields( 85 | fake_service, serialized_fake_entity, fake_entity 86 | ): 87 | await fake_service.repository.memory_data_source.delete( 88 | 'fake:other_fake:fake' 89 | ) 90 | await fake_service.repository.fallback_data_source.put( 91 | fake_service.repository.fallback_data_source.make_key( 92 | 'fake', 'other_fake:fake' 93 | ), 94 | dataclasses.asdict(fake_entity), 95 | ) 96 | fake_entity.number = None 97 | fake_entity.boolean = None 98 | 99 | entity = await fake_service.get_one( 100 | 'fake', 101 | other_id='other_fake', 102 | fields=['id', 'other_id', 'integer', 'inner_entities'], 103 | ) 104 | 105 | assert entity == fake_entity 106 | assert fake_service.repository.memory_data_source.exists( 107 | 'fake:other_fake:fake' 108 | ) 109 | 110 | 111 | @pytest.mark.asyncio 112 | async def test_should_get_one_from_fallback_after_open_circuit_breaker( 113 | fake_service, fake_entity, mocker 114 | ): 115 | fake_service.repository.memory_data_source.hgetall = asynctest.CoroutineMock( 116 | side_effect=RedisError 117 | ) 118 | key = fake_service.repository.fallback_data_source.make_key( 119 | 'fake', 'other_fake', 'fake' 120 | ) 121 | await fake_service.repository.fallback_data_source.put( 122 | key, dataclasses.asdict(fake_entity) 123 | ) 124 | 125 | entity = await fake_service.get_one('fake', other_id='other_fake') 126 | 127 | assert entity == fake_entity 128 | assert fake_service.logger.warning.call_count == 1 129 | -------------------------------------------------------------------------------- /dbdaora/hash/_tests/test_integration_service_hash_aioredis_add.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from aioredis import RedisError 4 | 5 | 6 | @pytest.fixture 7 | def has_add_cb(): 8 | return True 9 | 10 | 11 | @pytest.mark.asyncio 12 | async def test_should_add(fake_service, fake_entity): 13 | await fake_service.repository.add(fake_entity) 14 | 15 | entity = await fake_service.get_one('fake') 16 | 17 | assert entity == fake_entity 18 | 19 | 20 | @pytest.mark.asyncio 21 | async def test_should_add_to_fallback_after_open_circuit_breaker( 22 | fake_service, fake_entity 23 | ): 24 | fake_exists = asynctest.CoroutineMock(side_effect=RedisError) 25 | fake_service.repository.memory_data_source.exists = fake_exists 26 | await fake_service.add(fake_entity) 27 | 28 | entity = await fake_service.get_one('fake') 29 | 30 | assert entity == fake_entity 31 | assert fake_service.logger.warning.call_count == 2 32 | -------------------------------------------------------------------------------- /dbdaora/hash/_tests/test_integration_service_hash_aioredis_delete.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from aioredis import RedisError 4 | from jsondaora import dataclasses 5 | 6 | from dbdaora.exceptions import EntityNotFoundError 7 | 8 | 9 | @pytest.fixture 10 | def has_delete_cb(): 11 | return True 12 | 13 | 14 | @pytest.mark.asyncio 15 | async def test_should_delete( 16 | fake_service, serialized_fake_entity, fake_entity 17 | ): 18 | await fake_service.add(fake_entity) 19 | 20 | assert await fake_service.get_one('fake') 21 | 22 | await fake_service.delete(fake_entity.id) 23 | fake_service.cache.clear() 24 | 25 | with pytest.raises(EntityNotFoundError): 26 | await fake_service.get_one('fake') 27 | 28 | 29 | @pytest.mark.asyncio 30 | async def test_should_delete_from_fallback_after_open_circuit_breaker( 31 | fake_service, fake_entity, mocker 32 | ): 33 | await fake_service.repository.memory_data_source.delete('fake:fake') 34 | await fake_service.repository.memory_data_source.delete( 35 | 'fake:not-found:fake' 36 | ) 37 | fake_service.repository.fallback_data_source.db[ 38 | 'fake:fake' 39 | ] = dataclasses.asdict(fake_entity, dumps_value=True) 40 | 41 | assert await fake_service.get_one('fake') 42 | 43 | fake_service.repository.memory_data_source.delete = asynctest.CoroutineMock( 44 | side_effect=RedisError 45 | ) 46 | 47 | await fake_service.delete(fake_entity.id) 48 | fake_service.cache.clear() 49 | 50 | with pytest.raises(EntityNotFoundError): 51 | await fake_service.get_one('fake') 52 | 53 | assert fake_service.logger.warning.call_count == 2 54 | -------------------------------------------------------------------------------- /dbdaora/hash/_tests/test_integration_service_hash_aioredis_exists.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | import asynctest 4 | import pytest 5 | from aioredis import RedisError 6 | from jsondaora import dataclasses 7 | 8 | 9 | @pytest.mark.asyncio 10 | async def test_should_exists(fake_service, serialized_fake_entity): 11 | await fake_service.repository.memory_data_source.hmset( 12 | 'fake:fake', *itertools.chain(*serialized_fake_entity.items()) 13 | ) 14 | assert fake_service.exists('fake') 15 | 16 | 17 | @pytest.mark.asyncio 18 | async def test_should_exists_without_cache( 19 | fake_service, serialized_fake_entity 20 | ): 21 | fake_service.exists_cache = None 22 | await fake_service.repository.memory_data_source.hmset( 23 | 'fake:fake', *itertools.chain(*serialized_fake_entity.items()) 24 | ) 25 | assert await fake_service.exists('fake') 26 | 27 | 28 | @pytest.mark.asyncio 29 | async def test_should_exists_from_cache(fake_service, serialized_fake_entity): 30 | fake_service.repository.memory_data_source.exists = ( 31 | asynctest.CoroutineMock() 32 | ) 33 | fake_service.exists_cache['fake'] = True 34 | 35 | assert await fake_service.exists('fake') 36 | assert not fake_service.repository.memory_data_source.exists.called 37 | 38 | 39 | @pytest.mark.asyncio 40 | async def test_should_exists_from_fallback_after_open_circuit_breaker( 41 | fake_service, fake_entity, mocker 42 | ): 43 | fake_service.repository.memory_data_source.exists = asynctest.CoroutineMock( 44 | side_effect=RedisError 45 | ) 46 | fake_service.repository.fallback_data_source.db[ 47 | 'fake:fake' 48 | ] = dataclasses.asdict(fake_entity) 49 | 50 | assert await fake_service.exists('fake') 51 | assert fake_service.logger.warning.call_count == 1 52 | 53 | 54 | @pytest.mark.asyncio 55 | async def test_should_exists_from_fallback_after_open_circuit_breaker_without_cache( 56 | fake_service, fake_entity, mocker 57 | ): 58 | fake_service.exists_cache = None 59 | fake_service.repository.memory_data_source.exists = asynctest.CoroutineMock( 60 | side_effect=RedisError 61 | ) 62 | fake_service.repository.fallback_data_source.db[ 63 | 'fake:fake' 64 | ] = dataclasses.asdict(fake_entity) 65 | 66 | assert await fake_service.exists('fake') 67 | assert fake_service.logger.warning.call_count == 1 68 | -------------------------------------------------------------------------------- /dbdaora/hash/_tests/test_integration_service_hash_aioredis_get_one.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | import asynctest 4 | import pytest 5 | from aioredis import RedisError 6 | from jsondaora import dataclasses 7 | 8 | 9 | @pytest.mark.asyncio 10 | async def test_should_get_one( 11 | fake_service, serialized_fake_entity, fake_entity 12 | ): 13 | await fake_service.repository.memory_data_source.hmset( 14 | 'fake:fake', *itertools.chain(*serialized_fake_entity.items()) 15 | ) 16 | entity = await fake_service.get_one('fake') 17 | 18 | assert entity == fake_entity 19 | 20 | 21 | @pytest.mark.asyncio 22 | async def test_should_get_one_without_cache( 23 | fake_service, serialized_fake_entity, fake_entity 24 | ): 25 | fake_service.cache = None 26 | await fake_service.repository.memory_data_source.hmset( 27 | 'fake:fake', *itertools.chain(*serialized_fake_entity.items()) 28 | ) 29 | entity = await fake_service.get_one('fake') 30 | 31 | assert entity == fake_entity 32 | 33 | 34 | @pytest.mark.asyncio 35 | async def test_should_get_one_from_cache( 36 | fake_service, serialized_fake_entity, fake_entity 37 | ): 38 | fake_service.repository.memory_data_source.hgetall = ( 39 | asynctest.CoroutineMock() 40 | ) 41 | fake_service.cache['fake'] = fake_entity 42 | entity = await fake_service.get_one('fake') 43 | 44 | assert entity == fake_entity 45 | assert not fake_service.repository.memory_data_source.hgetall.called 46 | 47 | 48 | @pytest.mark.asyncio 49 | async def test_should_get_one_from_fallback_after_open_circuit_breaker( 50 | fake_service, fake_entity, mocker 51 | ): 52 | fake_service.repository.memory_data_source.hgetall = asynctest.CoroutineMock( 53 | side_effect=RedisError 54 | ) 55 | fake_service.repository.fallback_data_source.db[ 56 | 'fake:fake' 57 | ] = dataclasses.asdict(fake_entity) 58 | 59 | entity = await fake_service.get_one('fake') 60 | 61 | assert entity == fake_entity 62 | assert fake_service.logger.warning.call_count == 1 63 | 64 | 65 | @pytest.mark.asyncio 66 | async def test_should_get_one_from_fallback_after_open_circuit_breaker_without_cache( 67 | fake_service, fake_entity, mocker 68 | ): 69 | fake_service.cache = None 70 | fake_service.repository.memory_data_source.hgetall = asynctest.CoroutineMock( 71 | side_effect=RedisError 72 | ) 73 | fake_service.repository.fallback_data_source.db[ 74 | 'fake:fake' 75 | ] = dataclasses.asdict(fake_entity) 76 | 77 | entity = await fake_service.get_one('fake') 78 | 79 | assert entity == fake_entity 80 | assert fake_service.logger.warning.call_count == 1 81 | -------------------------------------------------------------------------------- /dbdaora/hash/_tests/test_unit_hash_service.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | 4 | from dbdaora import EntityNotFoundError, HashService 5 | from dbdaora.service import CACHE_ALREADY_NOT_FOUND 6 | 7 | 8 | @pytest.fixture 9 | def service(): 10 | s = HashService( 11 | repository=asynctest.MagicMock(id_name='id'), 12 | circuit_breaker=asynctest.MagicMock(), 13 | fallback_circuit_breaker=asynctest.MagicMock(), 14 | cache={}, 15 | ) 16 | s.entity_circuit = asynctest.CoroutineMock() 17 | s.entities_circuit = asynctest.CoroutineMock() 18 | return s 19 | 20 | 21 | @pytest.mark.asyncio 22 | async def test_should_set_cache_entity_not_found_when_getting_one(service): 23 | error = EntityNotFoundError() 24 | service.entity_circuit.side_effect = error 25 | 26 | with pytest.raises(EntityNotFoundError) as exc_info: 27 | await service.get_one('fake') 28 | 29 | assert exc_info.value == error 30 | assert service.cache['fake'] == CACHE_ALREADY_NOT_FOUND 31 | 32 | 33 | @pytest.mark.asyncio 34 | async def test_should_get_already_not_found_when_getting_one(service): 35 | service.cache['fake'] = CACHE_ALREADY_NOT_FOUND 36 | 37 | with pytest.raises(EntityNotFoundError) as exc_info: 38 | await service.get_one('fake') 39 | 40 | assert not service.entity_circuit.called 41 | assert exc_info.value.args == ('fake',) 42 | 43 | 44 | @pytest.mark.asyncio 45 | async def test_should_set_cache_entities_not_found_when_getting_many( 46 | service, async_iterator 47 | ): 48 | fake_entity = {'id': 'fake'} 49 | service.entity_circuit.side_effect = [ 50 | fake_entity, 51 | EntityNotFoundError, 52 | EntityNotFoundError, 53 | ] 54 | service.repository.id_name = 'id' 55 | 56 | entities = [e async for e in service.get_many('fake', 'fake2', 'fake3')] 57 | 58 | assert entities == [fake_entity] 59 | assert service.cache['fake2'] == CACHE_ALREADY_NOT_FOUND 60 | assert service.cache['fake3'] == CACHE_ALREADY_NOT_FOUND 61 | 62 | 63 | @pytest.mark.asyncio 64 | async def test_should_get_already_not_found_when_getting_many(service): 65 | fake_entity = {'id': 'fake'} 66 | service.cache['fake'] = fake_entity 67 | service.cache['fake2'] = CACHE_ALREADY_NOT_FOUND 68 | service.cache['fake3'] = CACHE_ALREADY_NOT_FOUND 69 | 70 | entities = [e async for e in service.get_many('fake', 'fake2', 'fake3')] 71 | 72 | assert not service.repository.query.called 73 | assert entities == [fake_entity] 74 | assert service.cache['fake2'] == CACHE_ALREADY_NOT_FOUND 75 | assert service.cache['fake3'] == CACHE_ALREADY_NOT_FOUND 76 | -------------------------------------------------------------------------------- /dbdaora/hash/conftest.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from functools import partial 3 | from typing import List, Optional 4 | 5 | import pytest 6 | from aioredis import RedisError 7 | 8 | from dbdaora import ( 9 | CacheType, 10 | DictFallbackDataSource, 11 | HashRepository, 12 | make_aioredis_data_source, 13 | make_hash_service, 14 | ) 15 | 16 | 17 | @pytest.fixture 18 | def has_add_cb(): 19 | return False 20 | 21 | 22 | @pytest.fixture 23 | def has_delete_cb(): 24 | return False 25 | 26 | 27 | @pytest.mark.asyncio 28 | @pytest.fixture 29 | async def fake_service( 30 | mocker, 31 | fallback_data_source, 32 | fake_hash_repository_cls, 33 | has_add_cb, 34 | has_delete_cb, 35 | ): 36 | memory_data_source_factory = partial( 37 | make_aioredis_data_source, 38 | 'redis://', 39 | 'redis://localhost/1', 40 | 'redis://localhost/2', 41 | ) 42 | 43 | async def fallback_data_source_factory(): 44 | return fallback_data_source 45 | 46 | service = await make_hash_service( 47 | fake_hash_repository_cls, 48 | memory_data_source_factory, 49 | fallback_data_source_factory, 50 | repository_expire_time=1, 51 | cache_type=CacheType.TTL, 52 | cache_ttl=1, 53 | cache_max_size=1, 54 | cb_failure_threshold=0, 55 | cb_recovery_timeout=10, 56 | cb_expected_exception=RedisError, 57 | logger=mocker.MagicMock(), 58 | has_add_circuit_breaker=has_add_cb, 59 | has_delete_circuit_breaker=has_delete_cb, 60 | ) 61 | 62 | yield service 63 | 64 | service.repository.memory_data_source.close() 65 | await service.repository.memory_data_source.wait_closed() 66 | 67 | 68 | @pytest.fixture 69 | def fallback_data_source(): 70 | return DictFallbackDataSource() 71 | 72 | 73 | @dataclasses.dataclass 74 | class FakeInnerEntity: 75 | id: str 76 | 77 | 78 | @dataclasses.dataclass 79 | class FakeEntity: 80 | id: str 81 | integer: int 82 | inner_entities: List[FakeInnerEntity] 83 | number: Optional[float] = None 84 | boolean: Optional[bool] = None 85 | 86 | 87 | class FakeHashRepository(HashRepository[FakeEntity, str]): 88 | name = 'fake' 89 | 90 | 91 | @pytest.fixture 92 | def fake_hash_repository_cls(): 93 | return FakeHashRepository 94 | 95 | 96 | @pytest.fixture 97 | def dict_repository_cls(): 98 | return FakeHashRepository 99 | 100 | 101 | @pytest.fixture 102 | def fake_entity(): 103 | return FakeEntity( 104 | id='fake', 105 | inner_entities=[FakeInnerEntity('inner1'), FakeInnerEntity('inner2')], 106 | integer=1, 107 | number=0.1, 108 | boolean=True, 109 | ) 110 | 111 | 112 | @pytest.fixture 113 | def fake_entity2(): 114 | return FakeEntity( 115 | id='fake2', 116 | inner_entities=[FakeInnerEntity('inner3'), FakeInnerEntity('inner4')], 117 | integer=2, 118 | number=0.2, 119 | boolean=False, 120 | ) 121 | 122 | 123 | @pytest.fixture 124 | def serialized_fake_entity(): 125 | return { 126 | b'id': b'fake', 127 | b'integer': b'1', 128 | b'number': b'0.1', 129 | b'boolean': b'1', 130 | b'inner_entities': b'[{"id":"inner1"},{"id":"inner2"}]', 131 | } 132 | 133 | 134 | @pytest.fixture 135 | def serialized_fake_entity2(): 136 | return { 137 | b'id': b'fake2', 138 | b'integer': b'2', 139 | b'number': b'0.2', 140 | b'boolean': b'0', 141 | b'inner_entities': b'[{"id":"inner3"},{"id":"inner4"}]', 142 | } 143 | 144 | 145 | @pytest.mark.asyncio 146 | @pytest.fixture 147 | async def repository(mocker): 148 | memory_data_source = await make_aioredis_data_source( 149 | 'redis://', 'redis://localhost/1', 'redis://localhost/2' 150 | ) 151 | yield FakeHashRepository( 152 | memory_data_source=memory_data_source, 153 | fallback_data_source=DictFallbackDataSource(), 154 | expire_time=1, 155 | ) 156 | memory_data_source.close() 157 | await memory_data_source.wait_closed() 158 | -------------------------------------------------------------------------------- /dbdaora/hash/factory.py: -------------------------------------------------------------------------------- 1 | from logging import Logger, getLogger 2 | from typing import Any, Optional, Type 3 | 4 | from dbdaora.entity import Entity, EntityData 5 | from dbdaora.keys import FallbackKey 6 | from dbdaora.service.builder import build as build_base_service 7 | 8 | from ..cache import CacheType 9 | from ..repository import MemoryRepository 10 | from ..service import Service 11 | from .service import HashService 12 | 13 | 14 | async def make_service( 15 | repository_cls: Type[MemoryRepository[Entity, EntityData, FallbackKey]], 16 | memory_data_source_factory: Any, 17 | fallback_data_source_factory: Any, 18 | repository_expire_time: int, 19 | cache_type: Optional[CacheType] = None, 20 | cache_ttl: Optional[int] = None, 21 | cache_max_size: Optional[int] = None, 22 | cb_failure_threshold: Optional[int] = None, 23 | cb_recovery_timeout: Optional[int] = None, 24 | cb_expected_exception: Optional[Type[Exception]] = None, 25 | cb_expected_fallback_exception: Optional[Type[Exception]] = None, 26 | logger: Logger = getLogger(__name__), 27 | has_add_circuit_breaker: bool = False, 28 | has_delete_circuit_breaker: bool = False, 29 | ) -> Service[Entity, EntityData, FallbackKey]: 30 | return await build_base_service( 31 | HashService, # type: ignore 32 | repository_cls, 33 | memory_data_source_factory, 34 | fallback_data_source_factory, 35 | repository_expire_time, 36 | cache_type=cache_type, 37 | cache_ttl=cache_ttl, 38 | cache_max_size=cache_max_size, 39 | cb_failure_threshold=cb_failure_threshold, 40 | cb_recovery_timeout=cb_recovery_timeout, 41 | cb_expected_exception=cb_expected_exception, 42 | cb_expected_fallback_exception=cb_expected_fallback_exception, 43 | logger=logger, 44 | has_add_circuit_breaker=has_add_circuit_breaker, 45 | has_delete_circuit_breaker=has_delete_circuit_breaker, 46 | ) 47 | -------------------------------------------------------------------------------- /dbdaora/hash/query.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import Any, ClassVar, List, Optional, Sequence, Tuple, Type, Union 3 | 4 | from dbdaora.keys import FallbackKey 5 | from dbdaora.query import BaseQuery, Query, QueryMany 6 | 7 | from .repositories import HashData, HashEntity, HashRepository 8 | 9 | 10 | @dataclasses.dataclass(init=False) 11 | class HashQuery(Query[HashEntity, HashData, FallbackKey]): 12 | repository: HashRepository[HashEntity, FallbackKey] 13 | fields: Optional[Sequence[str]] = None 14 | 15 | def __init__( 16 | self, 17 | repository: HashRepository[HashEntity, FallbackKey], 18 | *args: Any, 19 | memory: bool = True, 20 | key_parts: Optional[List[Any]] = None, 21 | fields: Optional[Sequence[str]] = None, 22 | **kwargs: Any, 23 | ): 24 | super().__init__( 25 | repository, memory=memory, key_parts=key_parts, *args, **kwargs, 26 | ) 27 | self.fields = fields 28 | 29 | 30 | @dataclasses.dataclass(init=False) 31 | class HashQueryMany(QueryMany[HashEntity, HashData, FallbackKey]): 32 | query_cls: ClassVar[Type[HashQuery[HashEntity, FallbackKey]]] = HashQuery[ 33 | HashEntity, FallbackKey 34 | ] 35 | queries: Sequence[HashQuery[HashEntity, FallbackKey]] # type: ignore 36 | repository: HashRepository[HashEntity, FallbackKey] 37 | fields: Optional[Sequence[str]] = None 38 | 39 | def __init__( 40 | self, 41 | repository: HashRepository[HashEntity, FallbackKey], 42 | *args: Any, 43 | many: List[Union[Any, Tuple[Any, ...]]], 44 | memory: bool = True, 45 | many_key_parts: Optional[List[List[Any]]] = None, 46 | fields: Optional[Sequence[str]] = None, 47 | **kwargs: Any, 48 | ): 49 | super().__init__( 50 | repository, 51 | memory=memory, 52 | many=many, 53 | many_key_parts=many_key_parts, 54 | *args, 55 | **kwargs, 56 | ) 57 | self.fields = fields 58 | 59 | for query in self.queries: 60 | query.fields = fields 61 | 62 | 63 | def make( 64 | *args: Any, **kwargs: Any 65 | ) -> BaseQuery[HashEntity, HashData, FallbackKey]: 66 | if kwargs.get('many') or kwargs.get('many_key_parts'): 67 | return HashQueryMany(*args, **kwargs) 68 | 69 | return HashQuery(*args, **kwargs) 70 | -------------------------------------------------------------------------------- /dbdaora/hash/repositories/_tests/test_integration_repository_aioredis_hash_get_many.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from jsondaora import dataclasses 4 | 5 | 6 | @pytest.mark.asyncio 7 | async def test_should_set_already_not_found_error_when_get_many( 8 | repository, mocker 9 | ): 10 | fake_entity = 'fake' 11 | repository.memory_data_source.hgetall = asynctest.CoroutineMock( 12 | return_value=None 13 | ) 14 | repository.memory_data_source.exists = asynctest.CoroutineMock( 15 | side_effect=[False] 16 | ) 17 | repository.fallback_data_source.get = asynctest.CoroutineMock( 18 | return_value=None 19 | ) 20 | repository.memory_data_source.set = asynctest.CoroutineMock() 21 | repository.memory_data_source.hmset = asynctest.CoroutineMock() 22 | 23 | assert [ 24 | e async for e in repository.query(many=[fake_entity]).entities 25 | ] == [] 26 | 27 | assert repository.memory_data_source.hgetall.call_args_list == [ 28 | mocker.call('fake:fake'), 29 | ] 30 | assert repository.memory_data_source.exists.call_args_list == [ 31 | mocker.call('fake:not-found:fake') 32 | ] 33 | assert repository.fallback_data_source.get.call_args_list == [ 34 | mocker.call('fake:fake') 35 | ] 36 | assert repository.memory_data_source.set.call_args_list == [ 37 | mocker.call('fake:not-found:fake', '1') 38 | ] 39 | assert not repository.memory_data_source.hmset.called 40 | 41 | 42 | @pytest.mark.asyncio 43 | async def test_should_get_many_from_fallback(repository, fake_entity): 44 | await repository.memory_data_source.delete('fake:fake') 45 | await repository.memory_data_source.delete('fake:not-found:fake') 46 | repository.fallback_data_source.db['fake:fake'] = dataclasses.asdict( 47 | fake_entity 48 | ) 49 | 50 | entities = [ 51 | e async for e in repository.query(many=[fake_entity.id]).entities 52 | ] 53 | 54 | assert entities == [fake_entity] 55 | assert repository.memory_data_source.exists('fake:fake') 56 | 57 | 58 | @pytest.mark.asyncio 59 | async def test_should_get_many_with_one_item_already_not_found_from_fallback( 60 | repository, fake_entity 61 | ): 62 | await repository.memory_data_source.delete('fake:fake') 63 | await repository.memory_data_source.delete('fake:fake2') 64 | await repository.memory_data_source.delete('fake:not-found:fake') 65 | await repository.memory_data_source.set('fake:not-found:fake2', '1') 66 | repository.fallback_data_source.db['fake:fake'] = dataclasses.asdict( 67 | fake_entity 68 | ) 69 | 70 | entities = [ 71 | e 72 | async for e in repository.query( 73 | many=[fake_entity.id, 'fake2'] 74 | ).entities 75 | ] 76 | 77 | assert entities == [fake_entity] 78 | assert repository.memory_data_source.exists('fake:fake') 79 | 80 | 81 | @pytest.mark.asyncio 82 | async def test_should_get_many_with_one_item_already_not_found_and_another_not_found_from_fallback( 83 | repository, fake_entity 84 | ): 85 | await repository.memory_data_source.delete('fake:fake') 86 | await repository.memory_data_source.delete('fake:fake2') 87 | await repository.memory_data_source.delete('fake:fake3') 88 | await repository.memory_data_source.delete('fake:not-found:fake') 89 | await repository.memory_data_source.delete('fake:not-found:fake3') 90 | await repository.memory_data_source.set('fake:not-found:fake2', '1') 91 | repository.fallback_data_source.db['fake:fake'] = dataclasses.asdict( 92 | fake_entity 93 | ) 94 | 95 | entities = [ 96 | e 97 | async for e in repository.query( 98 | many=[fake_entity.id, 'fake2', 'fake3'] 99 | ).entities 100 | ] 101 | 102 | assert entities == [fake_entity] 103 | assert repository.memory_data_source.exists('fake:fake') 104 | assert repository.memory_data_source.exists('fake:not-found:fake3') 105 | -------------------------------------------------------------------------------- /dbdaora/hash/repositories/_tests/test_integration_repository_aioredis_hash_get_many_fields.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from jsondaora import dataclasses 4 | 5 | 6 | @pytest.mark.asyncio 7 | async def test_should_set_already_not_found_error_when_get_many_with_fields( 8 | repository, mocker 9 | ): 10 | fake_entity = 'fake' 11 | repository.memory_data_source.hmget = asynctest.CoroutineMock( 12 | return_value=[None, None, None] 13 | ) 14 | repository.memory_data_source.exists = asynctest.CoroutineMock( 15 | side_effect=[False] 16 | ) 17 | repository.fallback_data_source.get = asynctest.CoroutineMock( 18 | return_value=None 19 | ) 20 | repository.memory_data_source.set = asynctest.CoroutineMock() 21 | repository.memory_data_source.hmset = asynctest.CoroutineMock() 22 | 23 | assert [ 24 | e 25 | async for e in repository.query( 26 | many=[fake_entity], fields=['id', 'integer', 'inner_entities'] 27 | ).entities 28 | ] == [] 29 | 30 | assert repository.memory_data_source.hmget.call_args_list == [ 31 | mocker.call('fake:fake', 'id', 'integer', 'inner_entities'), 32 | ] 33 | assert repository.memory_data_source.exists.call_args_list == [ 34 | mocker.call('fake:not-found:fake') 35 | ] 36 | assert repository.fallback_data_source.get.call_args_list == [ 37 | mocker.call('fake:fake') 38 | ] 39 | assert repository.memory_data_source.set.call_args_list == [ 40 | mocker.call('fake:not-found:fake', '1') 41 | ] 42 | assert not repository.memory_data_source.hmset.called 43 | 44 | 45 | @pytest.mark.asyncio 46 | async def test_should_get_many_from_fallback_with_fields( 47 | repository, fake_entity 48 | ): 49 | await repository.memory_data_source.delete('fake:fake') 50 | await repository.memory_data_source.delete('fake:not-found:fake') 51 | repository.fallback_data_source.db['fake:fake'] = dataclasses.asdict( 52 | fake_entity 53 | ) 54 | fake_entity.number = None 55 | fake_entity.boolean = None 56 | 57 | entities = [ 58 | e 59 | async for e in repository.query( 60 | many=[fake_entity.id], fields=['id', 'integer', 'inner_entities'] 61 | ).entities 62 | ] 63 | 64 | assert entities == [fake_entity] 65 | assert repository.memory_data_source.exists('fake:fake') 66 | -------------------------------------------------------------------------------- /dbdaora/hash/repositories/_tests/test_integration_repository_hash_datastore.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | import pytest 4 | from google.cloud import datastore 5 | 6 | from dbdaora import DatastoreHashRepository 7 | from dbdaora.data_sources.fallback.datastore import DatastoreDataSource 8 | from dbdaora.data_sources.memory.dict import DictMemoryDataSource 9 | 10 | 11 | @dataclass 12 | class FakeEntity: 13 | id: str 14 | fake_int: int 15 | 16 | 17 | class FakeRepository(DatastoreHashRepository, entity_cls=FakeEntity): 18 | key_attrs = ('id',) 19 | exclude_all_from_indexes = True 20 | 21 | 22 | @pytest.fixture 23 | async def repository(): 24 | return FakeRepository( 25 | memory_data_source=DictMemoryDataSource(), 26 | fallback_data_source=DatastoreDataSource(), 27 | expire_time=1, 28 | ) 29 | 30 | 31 | @pytest.mark.asyncio 32 | async def test_should_exclude_all_attributes_from_indexes(repository): 33 | client = repository.fallback_data_source.client 34 | key = client.key('fake', 'fake') 35 | entity = datastore.Entity(key=key) 36 | entity.update({'id': 'fake', 'fake_int': 1}) 37 | client.put(entity) 38 | 39 | assert not client.get(key).exclude_from_indexes 40 | 41 | await repository.add(FakeEntity(id='fake', fake_int=1)) 42 | 43 | assert client.get(key).exclude_from_indexes == set(['id', 'fake_int']) 44 | -------------------------------------------------------------------------------- /dbdaora/hash/repositories/_tests/test_unit_repository_dict_hash_get.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | import asynctest 4 | import pytest 5 | from jsondaora import dataclasses 6 | 7 | from dbdaora import HashQuery 8 | from dbdaora.exceptions import EntityNotFoundError 9 | 10 | 11 | @pytest.fixture 12 | def repository(dict_repository): 13 | return dict_repository 14 | 15 | 16 | @pytest.mark.asyncio 17 | async def test_should_get_from_memory( 18 | repository, serialized_fake_entity, fake_entity 19 | ): 20 | await repository.memory_data_source.hmset( 21 | 'fake:fake', *itertools.chain(*serialized_fake_entity.items()) 22 | ) 23 | entity = await repository.query('fake').entity 24 | 25 | assert entity == fake_entity 26 | 27 | 28 | @pytest.mark.asyncio 29 | async def test_should_raise_not_found_error(repository, fake_entity, mocker): 30 | fake_query = HashQuery(repository, memory=True, id=fake_entity.id) 31 | 32 | with pytest.raises(EntityNotFoundError) as exc_info: 33 | await repository.query(fake_entity.id).entity 34 | 35 | assert exc_info.value.args == (fake_query,) 36 | 37 | 38 | @pytest.mark.asyncio 39 | async def test_should_raise_not_found_error_when_already_raised_before( 40 | repository, mocker 41 | ): 42 | fake_entity = 'fake' 43 | expected_query = HashQuery(repository, memory=True, id=fake_entity) 44 | repository.memory_data_source.hgetall = asynctest.CoroutineMock( 45 | side_effect=[None] 46 | ) 47 | repository.memory_data_source.exists = asynctest.CoroutineMock( 48 | side_effect=[True] 49 | ) 50 | repository.memory_data_source.hmset = asynctest.CoroutineMock() 51 | 52 | with pytest.raises(EntityNotFoundError) as exc_info: 53 | await repository.query(fake_entity).entity 54 | 55 | assert exc_info.value.args == (expected_query,) 56 | assert repository.memory_data_source.hgetall.call_args_list == [ 57 | mocker.call('fake:fake'), 58 | ] 59 | assert repository.memory_data_source.exists.call_args_list == [ 60 | mocker.call('fake:not-found:fake') 61 | ] 62 | assert not repository.memory_data_source.hmset.called 63 | 64 | 65 | @pytest.mark.asyncio 66 | async def test_should_get_from_fallback(repository, fake_entity): 67 | repository.memory_data_source.hgetall = asynctest.CoroutineMock( 68 | side_effect=[None] 69 | ) 70 | repository.fallback_data_source.db['fake:fake'] = dataclasses.asdict( 71 | fake_entity 72 | ) 73 | entity = await repository.query(fake_entity.id).entity 74 | 75 | assert repository.memory_data_source.hgetall.called 76 | assert entity == fake_entity 77 | 78 | 79 | @pytest.mark.asyncio 80 | async def test_should_set_memory_after_got_fallback( 81 | repository, fake_entity, mocker 82 | ): 83 | repository.memory_data_source.hgetall = asynctest.CoroutineMock( 84 | side_effect=[None] 85 | ) 86 | repository.memory_data_source.hmset = asynctest.CoroutineMock() 87 | repository.fallback_data_source.db['fake:fake'] = dataclasses.asdict( 88 | fake_entity 89 | ) 90 | entity = await repository.query(fake_entity.id).entity 91 | 92 | assert repository.memory_data_source.hgetall.called 93 | assert repository.memory_data_source.hmset.call_args_list == [ 94 | mocker.call( 95 | 'fake:fake', 96 | b'id', 97 | 'fake', 98 | b'integer', 99 | 1, 100 | b'inner_entities', 101 | b'[{"id":"inner1"},{"id":"inner2"}]', 102 | b'number', 103 | 0.1, 104 | b'boolean', 105 | 1, 106 | ) 107 | ] 108 | assert entity == fake_entity 109 | -------------------------------------------------------------------------------- /dbdaora/hash/repositories/_tests/test_unit_repository_dict_hash_get_fields.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | import asynctest 4 | import pytest 5 | from jsondaora import dataclasses 6 | 7 | from dbdaora import HashQuery 8 | from dbdaora.exceptions import EntityNotFoundError 9 | 10 | 11 | @pytest.fixture 12 | def repository(dict_repository): 13 | return dict_repository 14 | 15 | 16 | @pytest.mark.asyncio 17 | async def test_should_get_from_memory( 18 | repository, serialized_fake_entity, fake_entity 19 | ): 20 | await repository.memory_data_source.hmset( 21 | 'fake:fake', *itertools.chain(*serialized_fake_entity.items()) 22 | ) 23 | fake_entity.number = None 24 | fake_entity.boolean = None 25 | 26 | entity = await repository.query( 27 | 'fake', fields=['id', 'integer', 'inner_entities'] 28 | ).entity 29 | 30 | assert entity == fake_entity 31 | 32 | 33 | @pytest.mark.asyncio 34 | async def test_should_raise_not_found_error(repository, fake_entity, mocker): 35 | fake_query = HashQuery(repository, memory=True, id=fake_entity.id) 36 | 37 | with pytest.raises(EntityNotFoundError) as exc_info: 38 | await repository.query(fake_entity.id).entity 39 | 40 | assert exc_info.value.args == (fake_query,) 41 | 42 | 43 | @pytest.mark.asyncio 44 | async def test_should_raise_not_found_error_when_already_raised_before( 45 | repository, mocker 46 | ): 47 | fake_entity = 'fake' 48 | fields = ['id', 'integer', 'inner_entities'] 49 | expected_query = HashQuery( 50 | repository, memory=True, id=fake_entity, fields=fields 51 | ) 52 | repository.memory_data_source.hmget = asynctest.CoroutineMock( 53 | side_effect=[[None]] 54 | ) 55 | repository.memory_data_source.exists = asynctest.CoroutineMock( 56 | side_effect=[True] 57 | ) 58 | repository.memory_data_source.hmset = asynctest.CoroutineMock() 59 | 60 | with pytest.raises(EntityNotFoundError) as exc_info: 61 | await repository.query('fake', fields=fields).entity 62 | 63 | assert exc_info.value.args == (expected_query,) 64 | assert repository.memory_data_source.hmget.call_args_list == [ 65 | mocker.call('fake:fake', *fields), 66 | ] 67 | assert repository.memory_data_source.exists.call_args_list == [ 68 | mocker.call('fake:not-found:fake') 69 | ] 70 | assert not repository.memory_data_source.hmset.called 71 | 72 | 73 | @pytest.mark.asyncio 74 | async def test_should_get_from_fallback(repository, fake_entity): 75 | repository.memory_data_source.hmget = asynctest.CoroutineMock( 76 | side_effect=[[None]] 77 | ) 78 | fields = ['id', 'integer', 'inner_entities'] 79 | repository.fallback_data_source.db['fake:fake'] = dataclasses.asdict( 80 | fake_entity 81 | ) 82 | fake_entity.number = None 83 | fake_entity.boolean = None 84 | 85 | entity = await repository.query('fake', fields=fields).entity 86 | 87 | assert repository.memory_data_source.hmget.called 88 | assert entity == fake_entity 89 | 90 | 91 | @pytest.mark.asyncio 92 | async def test_should_set_memory_after_got_fallback( 93 | repository, fake_entity, mocker 94 | ): 95 | repository.memory_data_source.hmget = asynctest.CoroutineMock( 96 | side_effect=[[None]] 97 | ) 98 | repository.memory_data_source.hmset = asynctest.CoroutineMock() 99 | repository.fallback_data_source.db['fake:fake'] = dataclasses.asdict( 100 | fake_entity 101 | ) 102 | fake_entity.number = None 103 | fake_entity.boolean = None 104 | 105 | entity = await repository.query( 106 | 'fake', fields=['id', 'integer', 'inner_entities'] 107 | ).entity 108 | 109 | assert repository.memory_data_source.hmget.called 110 | assert repository.memory_data_source.hmset.call_args_list == [ 111 | mocker.call( 112 | 'fake:fake', 113 | b'id', 114 | 'fake', 115 | b'integer', 116 | 1, 117 | b'inner_entities', 118 | b'[{"id":"inner1"},{"id":"inner2"}]', 119 | b'number', 120 | 0.1, 121 | b'boolean', 122 | 1, 123 | ) 124 | ] 125 | assert entity == fake_entity 126 | -------------------------------------------------------------------------------- /dbdaora/hash/repositories/datastore.py: -------------------------------------------------------------------------------- 1 | from google.cloud.datastore import Key 2 | 3 | from dbdaora.repository.datastore import DatastoreRepository 4 | 5 | from . import HashData, HashEntity, HashRepository 6 | 7 | 8 | class DatastoreHashRepository( 9 | DatastoreRepository[HashEntity, HashData], HashRepository[HashEntity, Key], 10 | ): 11 | __skip_cls_validation__ = ('DatastoreHashRepository',) 12 | -------------------------------------------------------------------------------- /dbdaora/hash/repositories/mongodb.py: -------------------------------------------------------------------------------- 1 | from dbdaora.data_sources.fallback.mongodb import Key 2 | 3 | from . import HashEntity, HashRepository 4 | 5 | 6 | class MongodbHashRepository(HashRepository[HashEntity, Key]): 7 | __skip_cls_validation__ = ('MongodbHashRepository',) 8 | fallback_data_source_key_cls = Key 9 | -------------------------------------------------------------------------------- /dbdaora/hash/service/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Tuple, Union 2 | 3 | from ...entity import Entity 4 | from ...keys import FallbackKey 5 | from ...service import Service 6 | from ..repositories import HashData 7 | 8 | 9 | class HashService(Service[Entity, HashData, FallbackKey]): 10 | def cache_key_suffix(self, **filters: Any) -> str: 11 | return ( 12 | ''.join(f'{f}{v}' for f, v in filters.items() if f != 'fields') 13 | if filters 14 | else '' 15 | ) 16 | 17 | def get_cached_entity( 18 | self, id: Union[str, Tuple[str, ...]], key_suffix: str, **filters: Any, 19 | ) -> Any: 20 | entity = super().get_cached_entity(id, key_suffix, **filters) 21 | fields = filters.get('fields') 22 | 23 | if fields is None: 24 | return entity 25 | 26 | if isinstance(entity, dict): 27 | for field in fields: 28 | if field not in entity: 29 | return None 30 | 31 | else: 32 | for field in fields: 33 | if hasattr(entity, field): 34 | return None 35 | 36 | return entity 37 | -------------------------------------------------------------------------------- /dbdaora/hash/service/datastore.py: -------------------------------------------------------------------------------- 1 | from google.cloud.datastore import Key as DatastoreKey 2 | 3 | from ...entity import Entity 4 | from . import HashService 5 | 6 | 7 | class DatastoreHashService(HashService[Entity, DatastoreKey]): 8 | ... 9 | -------------------------------------------------------------------------------- /dbdaora/hash/service/mongodb.py: -------------------------------------------------------------------------------- 1 | from ...data_sources.fallback.mongodb import Key as MongoKey 2 | from ...entity import Entity 3 | from . import HashService 4 | 5 | 6 | class MongoHashService(HashService[Entity, MongoKey]): 7 | ... 8 | -------------------------------------------------------------------------------- /dbdaora/hashring.py: -------------------------------------------------------------------------------- 1 | from hashlib import md5 2 | from typing import Generic, Optional, Sequence, TypeVar 3 | 4 | 5 | DataSource = TypeVar('DataSource') 6 | 7 | 8 | class HashRing(Generic[DataSource]): 9 | def __init__( 10 | self, nodes: Sequence[DataSource], nodes_size: Optional[int] = None 11 | ): 12 | self.nodes = nodes 13 | 14 | if nodes_size is None: 15 | nodes_size = len(nodes) 16 | 17 | self.nodes_size = nodes_size 18 | 19 | def get_index(self, key: str) -> int: 20 | return ( 21 | int(md5(str(key).encode('utf-8')).hexdigest(), 16) 22 | % self.nodes_size 23 | ) 24 | 25 | def get_node(self, key: str) -> DataSource: 26 | return self.nodes[self.get_index(key)] 27 | -------------------------------------------------------------------------------- /dbdaora/keys.py: -------------------------------------------------------------------------------- 1 | from typing import TypeVar 2 | 3 | 4 | FallbackKey = TypeVar('FallbackKey') 5 | -------------------------------------------------------------------------------- /dbdaora/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dutradda/dbdaora/5c87a3818e9d736bbf5e1438edc5929a2f5acd3f/dbdaora/py.typed -------------------------------------------------------------------------------- /dbdaora/repository/datastore.py: -------------------------------------------------------------------------------- 1 | from typing import Any, ClassVar, Optional, Sequence, Type, get_type_hints 2 | 3 | from google.cloud.datastore import Entity as DatastoreEntity 4 | from google.cloud.datastore import Key 5 | from jsondaora.serializers import OrjsonDefaultTypes 6 | 7 | from dbdaora.entity import Entity, EntityData 8 | 9 | from . import MemoryRepository 10 | 11 | 12 | OrjsonDefaultTypes.types_default_map[DatastoreEntity] = lambda e: dict(**e) 13 | 14 | 15 | class DatastoreRepository(MemoryRepository[Entity, EntityData, Key]): 16 | __skip_cls_validation__ = ('DatastoreRepository',) 17 | exclude_from_indexes: ClassVar[Sequence[str]] = () 18 | exclude_all_from_indexes: ClassVar[bool] = True 19 | fallback_data_source_key_cls = Key 20 | 21 | def __init_subclass__( 22 | cls, 23 | entity_cls: Optional[Type[Entity]] = None, 24 | name: Optional[str] = None, 25 | id_name: Optional[str] = None, 26 | key_attrs: Optional[Sequence[str]] = None, 27 | many_key_attrs: Optional[Sequence[str]] = None, 28 | ): 29 | super().__init_subclass__( 30 | entity_cls, name, id_name, key_attrs, many_key_attrs, 31 | ) 32 | 33 | if cls.__name__ not in cls.__skip_cls_validation__: 34 | if ( 35 | not cls.exclude_from_indexes 36 | and cls.exclude_all_from_indexes 37 | and cls.entity_cls 38 | ): 39 | cls.exclude_from_indexes = tuple( 40 | get_type_hints(cls.entity_cls).keys() 41 | ) 42 | 43 | async def add_fallback( 44 | self, entity: Entity, *entities: Entity, **kwargs: Any 45 | ) -> None: 46 | await super().add_fallback( 47 | entity, *entities, exclude_from_indexes=self.exclude_from_indexes 48 | ) 49 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dutradda/dbdaora/5c87a3818e9d736bbf5e1438edc5929a2f5acd3f/dbdaora/sorted_set/__init__.py -------------------------------------------------------------------------------- /dbdaora/sorted_set/conftest.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Optional 3 | 4 | import pytest 5 | 6 | from dbdaora import DictFallbackDataSource, SortedSetData, SortedSetRepository 7 | 8 | 9 | @dataclass 10 | class FakeEntity: 11 | id: str 12 | data: SortedSetData 13 | max_size: Optional[int] = None 14 | 15 | 16 | @pytest.fixture 17 | def fake_entity_cls(): 18 | return FakeEntity 19 | 20 | 21 | @pytest.fixture 22 | def fake_entity(fake_entity_cls): 23 | return fake_entity_cls(id='fake', data=[b'1', b'2']) 24 | 25 | 26 | @pytest.fixture 27 | def fake_entity_withscores(fake_entity_cls): 28 | return fake_entity_cls(id='fake', data=[(b'1', 0), (b'2', 1)]) 29 | 30 | 31 | @pytest.fixture 32 | def fake_repository_cls(fake_entity_cls): 33 | class FakeRepository(SortedSetRepository[fake_entity_cls, str]): 34 | ... 35 | 36 | return FakeRepository 37 | 38 | 39 | @pytest.fixture 40 | def fallback_data_source(): 41 | return DictFallbackDataSource() 42 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/entity.py: -------------------------------------------------------------------------------- 1 | from typing import ( # type: ignore 2 | Any, 3 | Dict, 4 | Optional, 5 | Protocol, 6 | Sequence, 7 | Tuple, 8 | Type, 9 | TypeVar, 10 | TypedDict, 11 | Union, 12 | _TypedDictMeta, 13 | ) 14 | 15 | from dbdaora.data_sources.memory import RangeOutput 16 | from dbdaora.entity import init_subclass 17 | 18 | 19 | SortedSetInput = Sequence[Union[str, float]] 20 | 21 | SortedSetData = Union[RangeOutput, SortedSetInput] 22 | 23 | 24 | class SortedSetEntityProtocol(Protocol): 25 | data: SortedSetData 26 | max_size: Optional[int] = None 27 | 28 | def __init__( 29 | self, 30 | *, 31 | data: SortedSetData, 32 | max_size: Optional[int] = None, 33 | **kwargs: Any, 34 | ): 35 | ... 36 | 37 | 38 | class SortedSetEntity(SortedSetEntityProtocol): 39 | data: SortedSetData 40 | max_size: Optional[int] = None 41 | 42 | def __init_subclass__(cls) -> None: 43 | init_subclass(cls, (SortedSetEntity,)) 44 | 45 | 46 | class SortedSetDictEntityMeta(_TypedDictMeta): # type: ignore 47 | def __init__( 48 | cls, name: str, bases: Tuple[Type[Any], ...], attrs: Dict[str, Any] 49 | ): 50 | super().__init__(name, bases, attrs) 51 | init_subclass(cls, bases) 52 | 53 | 54 | class SortedSetDictEntity(TypedDict, metaclass=SortedSetDictEntityMeta): 55 | data: SortedSetData 56 | max_size: Optional[int] 57 | 58 | 59 | SortedSetEntityHint = TypeVar( 60 | 'SortedSetEntityHint', 61 | bound=Union[SortedSetEntityProtocol, SortedSetDictEntity], 62 | ) 63 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/factory.py: -------------------------------------------------------------------------------- 1 | from logging import Logger, getLogger 2 | from typing import Any, Optional, Type 3 | 4 | from dbdaora.entity import Entity, EntityData 5 | from dbdaora.keys import FallbackKey 6 | from dbdaora.service.builder import build as build_base_service 7 | 8 | from ..cache import CacheType 9 | from ..repository import MemoryRepository 10 | from ..service import Service 11 | from .service import SortedSetService 12 | 13 | 14 | async def make_service( 15 | repository_cls: Type[MemoryRepository[Entity, EntityData, FallbackKey]], 16 | memory_data_source_factory: Any, 17 | fallback_data_source_factory: Any, 18 | repository_expire_time: int, 19 | cache_type: Optional[CacheType] = None, 20 | cache_ttl: Optional[int] = None, 21 | cache_max_size: Optional[int] = None, 22 | cb_failure_threshold: Optional[int] = None, 23 | cb_recovery_timeout: Optional[int] = None, 24 | cb_expected_exception: Optional[Type[Exception]] = None, 25 | cb_expected_fallback_exception: Optional[Type[Exception]] = None, 26 | logger: Logger = getLogger(__name__), 27 | has_add_circuit_breaker: bool = False, 28 | has_delete_circuit_breaker: bool = False, 29 | ) -> Service[Entity, EntityData, FallbackKey]: 30 | return await build_base_service( 31 | SortedSetService, # type: ignore 32 | repository_cls, 33 | memory_data_source_factory, 34 | fallback_data_source_factory, 35 | repository_expire_time, 36 | cache_type=cache_type, 37 | cache_ttl=cache_ttl, 38 | cache_max_size=cache_max_size, 39 | cb_failure_threshold=cb_failure_threshold, 40 | cb_recovery_timeout=cb_recovery_timeout, 41 | cb_expected_exception=cb_expected_exception, 42 | cb_expected_fallback_exception=cb_expected_fallback_exception, 43 | logger=logger, 44 | has_add_circuit_breaker=has_add_circuit_breaker, 45 | has_delete_circuit_breaker=has_delete_circuit_breaker, 46 | ) 47 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/query.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import Any, List, Optional 3 | 4 | from dbdaora.keys import FallbackKey 5 | from dbdaora.query import Query 6 | 7 | from .entity import SortedSetData, SortedSetEntityHint 8 | 9 | 10 | @dataclasses.dataclass 11 | class SortedSetQuery(Query[SortedSetEntityHint, SortedSetData, FallbackKey]): 12 | repository: 'SortedSetRepository[SortedSetEntityHint, FallbackKey]' 13 | reverse: bool = False 14 | withscores: bool = False 15 | page: Optional[int] = None 16 | page_size: Optional[int] = None 17 | min_score: Optional[float] = None 18 | max_score: Optional[float] = None 19 | withmaxsize: bool = False 20 | 21 | def __init__( 22 | self, 23 | repository: 'SortedSetRepository[SortedSetEntityHint, FallbackKey]', 24 | *args: Any, 25 | memory: bool = True, 26 | key_parts: Optional[List[Any]] = None, 27 | reverse: bool = False, 28 | withscores: bool = False, 29 | page: Optional[int] = None, 30 | page_size: Optional[int] = None, 31 | min_score: Optional[float] = None, 32 | max_score: Optional[float] = None, 33 | withmaxsize: bool = False, 34 | **kwargs: Any, 35 | ): 36 | super().__init__( 37 | repository, memory=memory, key_parts=key_parts, *args, **kwargs, 38 | ) 39 | self.reverse = reverse 40 | self.withscores = withscores 41 | self.page = page 42 | self.page_size = page_size 43 | self.min_score = min_score 44 | self.max_score = max_score 45 | self.withmaxsize = withmaxsize 46 | 47 | 48 | from .repositories import SortedSetRepository # noqa isort:skip 49 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/repositories/_tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from dbdaora.data_sources.memory.dict import DictMemoryDataSource 4 | 5 | 6 | @pytest.fixture 7 | async def repository(fake_repository_cls, fallback_data_source): 8 | return fake_repository_cls( 9 | memory_data_source=DictMemoryDataSource(), 10 | fallback_data_source=fallback_data_source, 11 | expire_time=1, 12 | ) 13 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/repositories/_tests/test_integration_repository_sorted_set_datastore.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from google.cloud import datastore 3 | 4 | from dbdaora import DatastoreDataSource, DatastoreSortedSetRepository 5 | 6 | 7 | @pytest.fixture 8 | def fallback_data_source(): 9 | return DatastoreDataSource() 10 | 11 | 12 | @pytest.fixture 13 | def fake_repository_cls(fake_entity_cls): 14 | class FakeRepository(DatastoreSortedSetRepository): 15 | entity_cls = fake_entity_cls 16 | 17 | return FakeRepository 18 | 19 | 20 | @pytest.mark.asyncio 21 | async def test_should_exclude_values_attributes_from_indexes( 22 | repository, fake_entity_cls 23 | ): 24 | client = repository.fallback_data_source.client 25 | data = ['v1', 1, 'v2', 2] 26 | key = client.key('fake', 'fake') 27 | entity = datastore.Entity(key=key) 28 | entity.update({'data': data}) 29 | client.put(entity) 30 | 31 | assert not client.get(key).exclude_from_indexes 32 | 33 | data = [('v1', 1), ('v2', 2)] 34 | await repository.add(fake_entity_cls(id='fake', data=data)) 35 | 36 | assert client.get(key).exclude_from_indexes == set(['data']) 37 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/repositories/_tests/test_integration_repository_sorted_set_mongodb.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | from motor.motor_asyncio import AsyncIOMotorClient 5 | from pymongo.errors import OperationFailure 6 | 7 | from dbdaora import MongoDataSource, MongodbSortedSetRepository 8 | 9 | 10 | @pytest.fixture 11 | def fallback_data_source(event_loop): 12 | auth = os.environ.get('MONGO_AUTH', 'mongo:mongo') 13 | client = AsyncIOMotorClient( 14 | f'mongodb://{auth}@localhost:27017', io_loop=event_loop 15 | ) 16 | MongoDataSource.collections_has_ttl_index.clear() 17 | return MongoDataSource(database_name='dbdaora', client=client) 18 | 19 | 20 | @pytest.fixture 21 | def fake_repository_cls(fake_entity_cls): 22 | class FakeRepository(MongodbSortedSetRepository): 23 | entity_cls = fake_entity_cls 24 | 25 | return FakeRepository 26 | 27 | 28 | @pytest.fixture(autouse=True) 29 | async def drop_ttl_index(repository): 30 | collection = repository.fallback_data_source.client.dbdaora.fake 31 | 32 | try: 33 | await collection.drop_index('last_modified_1') 34 | except OperationFailure: 35 | ... 36 | 37 | 38 | @pytest.fixture 39 | async def create_ttl_index(repository): 40 | collection = repository.fallback_data_source.client.dbdaora.fake 41 | 42 | try: 43 | await collection.create_index( 44 | 'last_modified', expireAfterSeconds=60, 45 | ) 46 | except OperationFailure: 47 | ... 48 | 49 | yield 50 | 51 | try: 52 | await collection.drop_index('last_modified_1') 53 | except OperationFailure: 54 | ... 55 | 56 | 57 | @pytest.mark.asyncio 58 | async def test_should_create_collection_ttl_index(repository, fake_entity_cls): 59 | collection = repository.fallback_data_source.client.dbdaora.fake 60 | 61 | assert not [ 62 | i 63 | async for i in collection.list_indexes() 64 | if 'last_modified' in i['key'] 65 | ] 66 | 67 | await repository.add( 68 | fake_entity_cls(id='fake', data=[('v1', 1), ('v2', 2)]), 69 | fallback_ttl=60, 70 | ) 71 | 72 | assert [ 73 | i 74 | async for i in collection.list_indexes() 75 | if 'last_modified' in i['key'] and i['expireAfterSeconds'] == 60 76 | ] 77 | 78 | 79 | @pytest.mark.asyncio 80 | async def test_should_drop_and_create_collection_ttl_index( 81 | create_ttl_index, repository, fake_entity_cls 82 | ): 83 | collection = repository.fallback_data_source.client.dbdaora.fake 84 | 85 | assert [ 86 | i 87 | async for i in collection.list_indexes() 88 | if 'last_modified' in i['key'] and i['expireAfterSeconds'] == 60 89 | ] 90 | 91 | await repository.add( 92 | fake_entity_cls(id='fake', data=[('v1', 1), ('v2', 2)]), 93 | fallback_ttl=61, 94 | ) 95 | 96 | assert [ 97 | i 98 | async for i in collection.list_indexes() 99 | if 'last_modified' in i['key'] and i['expireAfterSeconds'] == 61 100 | ] 101 | 102 | 103 | @pytest.mark.asyncio 104 | async def test_should_not_drop_collection_ttl_index( 105 | create_ttl_index, repository, fake_entity_cls 106 | ): 107 | collection = repository.fallback_data_source.client.dbdaora.fake 108 | 109 | assert [ 110 | i 111 | async for i in collection.list_indexes() 112 | if 'last_modified' in i['key'] and i['expireAfterSeconds'] == 60 113 | ] 114 | 115 | await repository.add( 116 | fake_entity_cls(id='fake', data=[('v1', 1), ('v2', 2)]), 117 | fallback_ttl=60, 118 | ) 119 | 120 | assert [ 121 | i 122 | async for i in collection.list_indexes() 123 | if 'last_modified' in i['key'] and i['expireAfterSeconds'] == 60 124 | ] 125 | 126 | 127 | @pytest.mark.asyncio 128 | async def test_should_get_entity_with_ttl_index( 129 | create_ttl_index, repository, fake_entity_cls 130 | ): 131 | await repository.add( 132 | fake_entity_cls(id='fake', data=[('v1', 1), ('v2', 2)]), 133 | fallback_ttl=60, 134 | ) 135 | 136 | fake_entity = await repository.query(id='fake').entity 137 | 138 | assert fake_entity.id == 'fake' 139 | assert fake_entity.data == [b'v1', b'v2'] 140 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/repositories/_tests/test_unit_repository_sorted_set_add.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | 4 | 5 | @pytest.mark.asyncio 6 | async def test_should_add_memory( 7 | repository, fake_entity, fake_entity_withscores, mocker 8 | ): 9 | repository.memory_data_source.exists = asynctest.CoroutineMock( 10 | return_value=True 11 | ) 12 | repository.memory_data_source.zadd = asynctest.CoroutineMock() 13 | await repository.add(fake_entity_withscores) 14 | 15 | assert repository.memory_data_source.zadd.call_args_list == [ 16 | mocker.call('fake:fake', 1, b'2', 0, b'1') 17 | ] 18 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/repositories/_tests/test_unit_repository_sorted_set_add_typed_dict.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | 4 | from dbdaora import SortedSetDictEntity 5 | 6 | 7 | @pytest.fixture 8 | def fake_entity_cls(): 9 | return SortedSetDictEntity 10 | 11 | 12 | @pytest.mark.asyncio 13 | async def test_should_add_memory( 14 | repository, fake_entity, fake_entity_withscores, mocker 15 | ): 16 | repository.memory_data_source.exists = asynctest.CoroutineMock( 17 | return_value=True 18 | ) 19 | repository.memory_data_source.zadd = asynctest.CoroutineMock() 20 | await repository.add(fake_entity_withscores) 21 | 22 | assert repository.memory_data_source.zadd.call_args_list == [ 23 | mocker.call('fake:fake', 1, b'2', 0, b'1') 24 | ] 25 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/repositories/_tests/test_unit_repository_sorted_set_get.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | import asynctest 4 | import pytest 5 | 6 | from dbdaora import SortedSetQuery 7 | from dbdaora.exceptions import EntityNotFoundError 8 | 9 | 10 | @pytest.mark.asyncio 11 | async def test_should_get_from_memory( 12 | repository, fake_entity, fake_entity_withscores 13 | ): 14 | await repository.memory_data_source.zadd('fake:fake', 0, '1', 2, '2') 15 | entity = await repository.query(fake_entity.id).entity 16 | 17 | assert entity == fake_entity 18 | 19 | 20 | @pytest.mark.asyncio 21 | async def test_should_raise_not_found_error(repository, fake_entity, mocker): 22 | fake_query = SortedSetQuery(repository, memory=True, id=fake_entity.id) 23 | 24 | with pytest.raises(EntityNotFoundError) as exc_info: 25 | await repository.query(fake_entity.id).entity 26 | 27 | assert exc_info.value.args == (fake_query,) 28 | 29 | 30 | @pytest.mark.asyncio 31 | async def test_should_raise_not_found_error_when_already_raised_before( 32 | repository, mocker 33 | ): 34 | fake_entity = 'fake' 35 | expected_query = SortedSetQuery(repository, memory=True, id=fake_entity) 36 | repository.memory_data_source.zrange = asynctest.CoroutineMock( 37 | side_effect=[None] 38 | ) 39 | repository.memory_data_source.exists = asynctest.CoroutineMock( 40 | side_effect=[True] 41 | ) 42 | repository.memory_data_source.zadd = asynctest.CoroutineMock() 43 | 44 | with pytest.raises(EntityNotFoundError) as exc_info: 45 | await repository.query(fake_entity).entity 46 | 47 | assert exc_info.value.args == (expected_query,) 48 | assert repository.memory_data_source.zrange.call_args_list == [ 49 | mocker.call('fake:fake', start=0, stop=-1, withscores=False), 50 | ] 51 | assert repository.memory_data_source.exists.call_args_list == [ 52 | mocker.call('fake:not-found:fake') 53 | ] 54 | assert not repository.memory_data_source.zadd.called 55 | 56 | 57 | @pytest.mark.asyncio 58 | async def test_should_get_from_fallback( 59 | repository, fake_entity_withscores, fake_entity 60 | ): 61 | repository.memory_data_source.zrange = asynctest.CoroutineMock( 62 | side_effect=[None] 63 | ) 64 | repository.fallback_data_source.db['fake:fake'] = { 65 | 'data': tuple(itertools.chain(*fake_entity_withscores.data)) 66 | } 67 | entity = await repository.query(fake_entity.id).entity 68 | 69 | assert repository.memory_data_source.zrange.called 70 | assert entity == fake_entity 71 | 72 | 73 | @pytest.mark.asyncio 74 | async def test_should_set_memory_after_got_fallback( 75 | repository, fake_entity_withscores, fake_entity, mocker 76 | ): 77 | repository.memory_data_source.zrange = asynctest.CoroutineMock( 78 | side_effect=[None, fake_entity.data] 79 | ) 80 | repository.memory_data_source.zadd = asynctest.CoroutineMock() 81 | repository.fallback_data_source.db['fake:fake'] = { 82 | 'data': tuple(itertools.chain(*fake_entity_withscores.data)) 83 | } 84 | entity = await repository.query(fake_entity.id).entity 85 | 86 | assert repository.memory_data_source.zrange.called 87 | assert repository.memory_data_source.zadd.call_args_list == [ 88 | mocker.call('fake:fake', 1, b'2', 0, b'1') 89 | ] 90 | assert entity == fake_entity 91 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/repositories/datastore.py: -------------------------------------------------------------------------------- 1 | from google.cloud.datastore import Key 2 | 3 | from dbdaora.repository.datastore import DatastoreRepository 4 | 5 | from ..entity import SortedSetData, SortedSetEntityHint 6 | from . import SortedSetRepository 7 | 8 | 9 | class DatastoreSortedSetRepository( 10 | DatastoreRepository[SortedSetEntityHint, SortedSetData], 11 | SortedSetRepository[SortedSetEntityHint, Key], 12 | ): 13 | __skip_cls_validation__ = ('DatastoreSortedSetRepository',) 14 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/repositories/mongodb.py: -------------------------------------------------------------------------------- 1 | from dbdaora.data_sources.fallback.mongodb import Key 2 | 3 | from ..entity import SortedSetEntityHint 4 | from . import SortedSetRepository 5 | 6 | 7 | class MongodbSortedSetRepository( 8 | SortedSetRepository[SortedSetEntityHint, Key] 9 | ): 10 | __skip_cls_validation__ = ('MongodbSortedSetRepository',) 11 | fallback_data_source_key_cls = Key 12 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/service/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Any, AsyncGenerator, Optional, Sequence 2 | 3 | from cachetools import Cache 4 | 5 | from ...keys import FallbackKey 6 | from ...service import Service 7 | from ..entity import SortedSetData, SortedSetEntity 8 | 9 | 10 | class SortedSetService(Service[SortedSetEntity, SortedSetData, FallbackKey]): 11 | def get_many( 12 | self, *ids: str, **filters: Any 13 | ) -> AsyncGenerator[SortedSetEntity, None]: 14 | raise NotImplementedError() # pragma: no cover 15 | 16 | def get_many_cached( 17 | self, 18 | ids: Sequence[str], 19 | cache: Cache, 20 | memory: bool = True, 21 | **filters: Any, 22 | ) -> AsyncGenerator[SortedSetEntity, None]: 23 | raise NotImplementedError() # pragma: no cover 24 | 25 | async def delete( 26 | self, entity_id: Optional[str] = None, **filters: Any 27 | ) -> None: 28 | raise NotImplementedError() # pragma: no cover 29 | 30 | async def exists(self, id: Optional[str] = None, **filters: Any) -> bool: 31 | raise NotImplementedError() # pragma: no cover 32 | 33 | async def exists_cached( 34 | self, cache: Cache, memory: bool = True, **filters: Any, 35 | ) -> bool: 36 | raise NotImplementedError() # pragma: no cover 37 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/service/_tests/conftest.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from functools import partial 3 | from typing import Optional 4 | 5 | import pytest 6 | from aioredis import RedisError 7 | 8 | from dbdaora import ( 9 | SortedSetData, 10 | SortedSetRepository, 11 | make_aioredis_data_source, 12 | make_sorted_set_service, 13 | ) 14 | 15 | 16 | @dataclass 17 | class FakeEntity: 18 | fake_id: str 19 | data: SortedSetData 20 | max_size: Optional[int] = None 21 | 22 | 23 | @pytest.fixture 24 | def fallback_cb_error(): 25 | return KeyError 26 | 27 | 28 | @pytest.fixture 29 | def fake_entity_cls(): 30 | return FakeEntity 31 | 32 | 33 | @pytest.fixture 34 | def fake_repository_cls(fake_entity_cls): 35 | class FakeRepository(SortedSetRepository): 36 | id_name = 'fake_id' 37 | entity_cls = fake_entity_cls 38 | 39 | return FakeRepository 40 | 41 | 42 | @pytest.fixture 43 | def fake_entity(fake_entity_cls): 44 | return fake_entity_cls(fake_id='fake', data=[b'1', b'2']) 45 | 46 | 47 | @pytest.fixture 48 | def fake_entity_withscores(fake_entity_cls): 49 | return fake_entity_cls(fake_id='fake', data=[(b'1', 0), (b'2', 1)]) 50 | 51 | 52 | @pytest.fixture 53 | def cache_config(): 54 | return {} 55 | 56 | 57 | @pytest.fixture 58 | def has_add_cb(): 59 | return False 60 | 61 | 62 | @pytest.fixture 63 | def has_delete_cb(): 64 | return False 65 | 66 | 67 | @pytest.mark.asyncio 68 | @pytest.fixture 69 | async def fake_service( 70 | mocker, 71 | fallback_data_source, 72 | fake_repository_cls, 73 | fallback_cb_error, 74 | cache_config, 75 | has_add_cb, 76 | has_delete_cb, 77 | ): 78 | memory_data_source_factory = partial( 79 | make_aioredis_data_source, 80 | 'redis://', 81 | 'redis://localhost/1', 82 | 'redis://localhost/2', 83 | ) 84 | 85 | async def fallback_data_source_factory(): 86 | return fallback_data_source 87 | 88 | service = await make_sorted_set_service( 89 | fake_repository_cls, 90 | memory_data_source_factory, 91 | fallback_data_source_factory, 92 | repository_expire_time=1, 93 | cb_failure_threshold=0, 94 | cb_recovery_timeout=10, 95 | cb_expected_exception=RedisError, 96 | cb_expected_fallback_exception=fallback_cb_error, 97 | logger=mocker.MagicMock(), 98 | has_add_circuit_breaker=has_add_cb, 99 | has_delete_circuit_breaker=has_delete_cb, 100 | **cache_config, 101 | ) 102 | 103 | yield service 104 | 105 | await service.shutdown() 106 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/service/_tests/datastore/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from google.api_core.exceptions import GoogleAPIError 3 | 4 | from dbdaora import DatastoreDataSource, DatastoreSortedSetRepository 5 | 6 | 7 | @pytest.fixture 8 | def fallback_data_source(): 9 | return DatastoreDataSource() 10 | 11 | 12 | @pytest.fixture 13 | def fake_repository_cls(fake_entity_cls): 14 | class FakeRepository(DatastoreSortedSetRepository): 15 | key_attrs = ('fake_id',) 16 | entity_cls = fake_entity_cls 17 | 18 | return FakeRepository 19 | 20 | 21 | @pytest.fixture 22 | def fallback_cb_error(): 23 | return GoogleAPIError 24 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/service/_tests/datastore/test_integration_service_sorted_set_aioredis_datastore_add.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from aioredis import RedisError 4 | from circuitbreaker import CircuitBreakerError 5 | 6 | 7 | @pytest.fixture 8 | def has_add_cb(): 9 | return True 10 | 11 | 12 | @pytest.mark.asyncio 13 | async def test_should_add(fake_service, fake_entity, fake_entity_withscores): 14 | await fake_service.add(fake_entity_withscores) 15 | 16 | entity = await fake_service.get_one(fake_id=fake_entity_withscores.fake_id) 17 | 18 | assert entity == fake_entity 19 | 20 | 21 | @pytest.mark.asyncio 22 | async def test_should_add_to_fallback_after_open_circuit_breaker( 23 | fake_service, fake_entity, fake_entity_withscores 24 | ): 25 | fake_exists = asynctest.CoroutineMock(side_effect=RedisError) 26 | fake_service.repository.memory_data_source.exists = fake_exists 27 | fake_zadd = asynctest.CoroutineMock(side_effect=RedisError) 28 | fake_service.repository.memory_data_source.zadd = fake_zadd 29 | await fake_service.add(fake_entity_withscores) 30 | 31 | assert fake_service.logger.warning.call_count == 1 32 | fake_service.logger.warning.reset_mock() 33 | 34 | entity = await fake_service.get_one( 35 | fake_id=fake_entity_withscores.fake_id, memory=False 36 | ) 37 | 38 | assert entity == fake_entity 39 | 40 | 41 | @pytest.mark.asyncio 42 | async def test_should_add_to_fallback_after_open_fallback_circuit_breaker( 43 | fake_service, fake_entity_withscores, fallback_cb_error 44 | ): 45 | fake_put = asynctest.CoroutineMock(side_effect=fallback_cb_error) 46 | fake_service.repository.fallback_data_source.put = fake_put 47 | 48 | with pytest.raises(CircuitBreakerError): 49 | await fake_service.add(fake_entity_withscores, memory=False) 50 | 51 | assert fake_service.logger.warning.call_count == 1 52 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/service/_tests/datastore/test_integration_service_sorted_set_aioredis_datastore_get_one.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | import pytest 4 | 5 | 6 | @pytest.mark.asyncio 7 | async def test_should_get_one( 8 | fake_service, fake_entity, fake_entity_withscores 9 | ): 10 | data = list(itertools.chain(*fake_entity_withscores.data)) 11 | data.reverse() 12 | await fake_service.repository.memory_data_source.zadd('fake:fake', *data) 13 | entity = await fake_service.get_one(fake_id=fake_entity_withscores.fake_id) 14 | 15 | assert entity == fake_entity 16 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/service/_tests/mongodb/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | from motor.motor_asyncio import AsyncIOMotorClient 5 | from pymongo.errors import PyMongoError 6 | 7 | from dbdaora import MongoDataSource, MongodbSortedSetRepository 8 | 9 | 10 | @pytest.fixture 11 | def fallback_data_source(event_loop): 12 | auth = os.environ.get('MONGO_AUTH', 'mongo:mongo') 13 | client = AsyncIOMotorClient( 14 | f'mongodb://{auth}@localhost:27017', io_loop=event_loop 15 | ) 16 | return MongoDataSource( 17 | database_name='dbdaora', client=client, key_is_object_id=True 18 | ) 19 | 20 | 21 | @pytest.fixture 22 | def fake_repository_cls(fake_entity_cls): 23 | class FakeRepository(MongodbSortedSetRepository): 24 | key_attrs = ('fake_id',) 25 | entity_cls = fake_entity_cls 26 | 27 | return FakeRepository 28 | 29 | 30 | @pytest.fixture 31 | def fallback_cb_error(): 32 | return PyMongoError 33 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/service/_tests/mongodb/test_integration_service_sorted_set_aioredis_mongodb_add.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from aioredis import RedisError 4 | from circuitbreaker import CircuitBreakerError 5 | 6 | 7 | @pytest.fixture 8 | def has_add_cb(): 9 | return True 10 | 11 | 12 | @pytest.mark.asyncio 13 | async def test_should_add(fake_service, fake_entity, fake_entity_withscores): 14 | await fake_service.add(fake_entity_withscores) 15 | 16 | entity = await fake_service.get_one(fake_id=fake_entity_withscores.fake_id) 17 | 18 | assert entity == fake_entity 19 | 20 | 21 | @pytest.mark.asyncio 22 | async def test_should_add_to_fallback_after_open_circuit_breaker( 23 | fake_service, fake_entity, fake_entity_withscores 24 | ): 25 | fake_exists = asynctest.CoroutineMock(side_effect=RedisError) 26 | fake_service.repository.memory_data_source.exists = fake_exists 27 | fake_zadd = asynctest.CoroutineMock(side_effect=RedisError) 28 | fake_service.repository.memory_data_source.zadd = fake_zadd 29 | await fake_service.add(fake_entity_withscores) 30 | 31 | assert fake_service.logger.warning.call_count == 1 32 | fake_service.logger.warning.reset_mock() 33 | 34 | entity = await fake_service.get_one( 35 | fake_id=fake_entity_withscores.fake_id, memory=False 36 | ) 37 | 38 | assert entity == fake_entity 39 | 40 | 41 | @pytest.mark.asyncio 42 | async def test_should_add_to_fallback_after_open_fallback_circuit_breaker( 43 | fake_service, fake_entity_withscores, fallback_cb_error 44 | ): 45 | fake_put = asynctest.CoroutineMock(side_effect=fallback_cb_error) 46 | fake_service.repository.fallback_data_source.put = fake_put 47 | 48 | with pytest.raises(CircuitBreakerError): 49 | await fake_service.add(fake_entity_withscores, memory=False) 50 | 51 | assert fake_service.logger.warning.call_count == 1 52 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/service/_tests/mongodb/test_integration_service_sorted_set_aioredis_mongodb_get_one.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | import pytest 4 | 5 | 6 | @pytest.mark.asyncio 7 | async def test_should_get_one( 8 | fake_service, fake_entity, fake_entity_withscores 9 | ): 10 | data = list(itertools.chain(*fake_entity_withscores.data)) 11 | data.reverse() 12 | await fake_service.repository.memory_data_source.zadd('fake:fake', *data) 13 | entity = await fake_service.get_one(fake_id=fake_entity_withscores.fake_id) 14 | 15 | assert entity == fake_entity 16 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/service/_tests/test_integration_service_sorted_set_aioredis_add.py: -------------------------------------------------------------------------------- 1 | import asynctest 2 | import pytest 3 | from aioredis import RedisError 4 | from circuitbreaker import CircuitBreakerError 5 | 6 | 7 | @pytest.fixture 8 | def has_add_cb(): 9 | return True 10 | 11 | 12 | @pytest.mark.asyncio 13 | async def test_should_add(fake_service, fake_entity, fake_entity_withscores): 14 | await fake_service.add(fake_entity_withscores) 15 | 16 | entity = await fake_service.get_one(fake_id=fake_entity_withscores.fake_id) 17 | 18 | assert entity == fake_entity 19 | 20 | 21 | @pytest.mark.asyncio 22 | async def test_should_add_to_fallback_after_open_circuit_breaker( 23 | fake_service, fake_entity, fake_entity_withscores 24 | ): 25 | fake_exists = asynctest.CoroutineMock(side_effect=RedisError) 26 | fake_service.repository.memory_data_source.exists = fake_exists 27 | fake_zadd = asynctest.CoroutineMock(side_effect=RedisError) 28 | fake_service.repository.memory_data_source.zadd = fake_zadd 29 | await fake_service.add(fake_entity_withscores) 30 | 31 | assert fake_service.logger.warning.call_count == 1 32 | fake_service.logger.warning.reset_mock() 33 | 34 | entity = await fake_service.get_one( 35 | fake_id=fake_entity_withscores.fake_id, memory=False 36 | ) 37 | 38 | assert entity == fake_entity 39 | 40 | 41 | @pytest.mark.asyncio 42 | async def test_should_add_to_fallback_after_open_fallback_circuit_breaker( 43 | fake_service, fake_entity_withscores 44 | ): 45 | fake_put = asynctest.CoroutineMock(side_effect=KeyError) 46 | fake_service.repository.fallback_data_source.put = fake_put 47 | 48 | with pytest.raises(CircuitBreakerError): 49 | await fake_service.add(fake_entity_withscores, memory=False) 50 | 51 | assert fake_service.logger.warning.call_count == 1 52 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/service/_tests/test_integration_service_sorted_set_aioredis_get_one.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | import pytest 4 | 5 | 6 | @pytest.fixture(autouse=True) 7 | async def set_data(fake_service, fake_entity_withscores): 8 | data = list(itertools.chain(*fake_entity_withscores.data)) 9 | data.reverse() 10 | await fake_service.repository.memory_data_source.zadd('fake:fake', *data) 11 | 12 | 13 | @pytest.mark.asyncio 14 | async def test_should_get_one( 15 | fake_service, fake_entity, fake_entity_withscores 16 | ): 17 | entity = await fake_service.get_one(fake_id=fake_entity_withscores.fake_id) 18 | 19 | assert entity == fake_entity 20 | 21 | 22 | @pytest.mark.asyncio 23 | async def test_should_get_one_reverse( 24 | fake_service, fake_entity, fake_entity_withscores 25 | ): 26 | entity = await fake_service.get_one( 27 | fake_id=fake_entity_withscores.fake_id, reverse=True, 28 | ) 29 | 30 | fake_entity.data.reverse() 31 | assert entity == fake_entity 32 | 33 | 34 | @pytest.mark.asyncio 35 | async def test_should_get_one_withscores(fake_service, fake_entity_withscores): 36 | entity = await fake_service.get_one( 37 | fake_id=fake_entity_withscores.fake_id, withscores=True 38 | ) 39 | 40 | assert entity == fake_entity_withscores 41 | 42 | 43 | @pytest.mark.asyncio 44 | async def test_should_get_one_withmaxsize( 45 | fake_service, fake_entity, fake_entity_withscores 46 | ): 47 | entity = await fake_service.get_one( 48 | fake_id=fake_entity_withscores.fake_id, withmaxsize=True 49 | ) 50 | 51 | fake_entity.max_size = 2 52 | assert entity == fake_entity 53 | 54 | 55 | @pytest.mark.asyncio 56 | async def test_should_get_one_withmaxsize_and_withscore( 57 | fake_service, fake_entity_withscores 58 | ): 59 | entity = await fake_service.get_one( 60 | fake_id=fake_entity_withscores.fake_id, 61 | withmaxsize=True, 62 | withscores=True, 63 | ) 64 | 65 | fake_entity_withscores.max_size = 2 66 | assert entity == fake_entity_withscores 67 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/service/datastore.py: -------------------------------------------------------------------------------- 1 | from google.cloud.datastore import Key 2 | 3 | from . import SortedSetService 4 | 5 | 6 | class DatastoreSortedSetService(SortedSetService[Key]): 7 | ... 8 | -------------------------------------------------------------------------------- /dbdaora/sorted_set/service/mongodb.py: -------------------------------------------------------------------------------- 1 | from ...data_sources.fallback.mongodb import Key as MongoKey 2 | from . import SortedSetService 3 | 4 | 5 | class MongoSortedSetService(SortedSetService[MongoKey]): 6 | ... 7 | -------------------------------------------------------------------------------- /docs/contributing.md: -------------------------------------------------------------------------------- 1 | ## Starting Development 2 | 3 | ```bash 4 | git clone git@github.com:dutradda/dbdaora.git --recursive 5 | cd dbdaora 6 | make setup-python-virtualenv 7 | source venv/bin/activate 8 | make setup-python-project 9 | bake setup-dbdaora 10 | bake dependencies 11 | ``` 12 | 13 | ## Running the integration suite: 14 | 15 | ```bash 16 | bake integration 17 | ``` 18 | 19 | ## Other bake tasks: 20 | 21 | ```bash 22 | bake check-code 23 | 24 | bake tests-docs 25 | 26 | bake serve-docs 27 | 28 | bake add-changelog m="Add my cool feature" 29 | ``` 30 | 31 | You can run `bake` to see all tasks available. 32 | -------------------------------------------------------------------------------- /docs/domain-model/simple.md: -------------------------------------------------------------------------------- 1 | # Simple Domain Model Example 2 | 3 | ```python 4 | {!./src/domain_model/simple.py!} 5 | ``` 6 | 7 | ```python 8 | {!./src/domain_model/simple.output!} 9 | ``` 10 | -------------------------------------------------------------------------------- /docs/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dutradda/dbdaora/5c87a3818e9d736bbf5e1438edc5929a2f5acd3f/docs/favicon.png -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # dbdaora 2 | 3 |

4 | 5 | dbdaora 6 | 7 |

8 | 9 |

10 | Communicates with NoSQL (and SQL for future) databases using repository and service patterns and python dataclasses 11 |

12 | 13 | --- 14 | 15 | **Documentation**: https://dutradda.github.io/dbdaora/ 16 | 17 | **Source Code**: https://github.com/dutradda/dbdaora 18 | 19 | --- 20 | 21 | 22 | ## Key Features 23 | 24 | - **Creates an optional service layer with cache and circuit breaker** 25 | 26 | - **Supports for redis data types:** 27 | + Hash 28 | + Sorted Set 29 | + *(Others data types are planned)* 30 | 31 | - **Backup redis data into other databases:** 32 | + Google Datastore 33 | + Mongodb *(soon)* 34 | + SQL databases with SQLAlchemy *(soon)* 35 | + *(Others data bases are planned)* 36 | 37 | - *Support for other databases are in development.* 38 | 39 | 40 | ## Requirements 41 | 42 | - Python 3.8+ 43 | - [jsondaora](https://github.com/dutradda/jsondaora) for data validation/parsing 44 | - circuitbreaker 45 | - cachetools 46 | 47 | - Optionals: 48 | + aioredis 49 | + google-cloud-datastore 50 | 51 | 52 | ## Instalation 53 | 54 | ``` 55 | $ pip install dbdaora 56 | ``` 57 | 58 | 59 | ## Simple redis hash example 60 | 61 | ```python 62 | {!./src/index/simple_hash.py!} 63 | ``` 64 | 65 | ```bash 66 | {!./src/index/simple_hash.output!} 67 | ``` 68 | 69 | 70 | ## Simple redis sorted set example 71 | 72 | ```python 73 | {!./src/index/simple_sorted_set.py!} 74 | ``` 75 | 76 | ```python 77 | {!./src/index/simple_sorted_set.output!} 78 | ``` 79 | 80 | 81 | ## Using the service layer 82 | 83 | The service layer uses the backup dataset when redis is offline, opening a circuit breaker. 84 | 85 | It has an optional cache system too. 86 | 87 | 88 | ```python 89 | {!./src/index/simple_service.py!} 90 | ``` 91 | 92 | ```python 93 | {!./src/index/simple_service.output!} 94 | ``` 95 | -------------------------------------------------------------------------------- /docs/src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dutradda/dbdaora/5c87a3818e9d736bbf5e1438edc5929a2f5acd3f/docs/src/__init__.py -------------------------------------------------------------------------------- /docs/src/domain_model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dutradda/dbdaora/5c87a3818e9d736bbf5e1438edc5929a2f5acd3f/docs/src/domain_model/__init__.py -------------------------------------------------------------------------------- /docs/src/domain_model/simple.output: -------------------------------------------------------------------------------- 1 | Person(id='john_doe', name='John Doe', age=33) 2 | Playlist(person_id='john_doe', data=[b'm1', b'm2', b'm3'], max_size=None) -------------------------------------------------------------------------------- /docs/src/domain_model/simple.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from dataclasses import dataclass 3 | from typing import Optional 4 | 5 | from dbdaora import ( 6 | DictFallbackDataSource, 7 | DictMemoryDataSource, 8 | HashRepository, 9 | SortedSetData, 10 | SortedSetRepository, 11 | make_hash_service, 12 | ) 13 | 14 | 15 | # Data Source Layer 16 | 17 | 18 | async def make_memory_data_source() -> DictMemoryDataSource: 19 | return DictMemoryDataSource() 20 | 21 | 22 | async def make_fallback_data_source() -> DictFallbackDataSource: 23 | return DictFallbackDataSource() 24 | 25 | 26 | # Domain Layer 27 | 28 | 29 | @dataclass 30 | class Person: 31 | id: str 32 | name: str 33 | age: int 34 | 35 | 36 | def make_person(name: str, age: int) -> Person: 37 | return Person(name.replace(' ', '_').lower(), name, age) 38 | 39 | 40 | class PersonRepository(HashRepository[Person, str]): 41 | ... 42 | 43 | 44 | person_service = asyncio.run( 45 | make_hash_service( 46 | PersonRepository, 47 | memory_data_source_factory=make_memory_data_source, 48 | fallback_data_source_factory=make_fallback_data_source, 49 | repository_expire_time=60, 50 | ) 51 | ) 52 | 53 | 54 | @dataclass 55 | class Playlist: 56 | person_id: str 57 | data: SortedSetData 58 | max_size: Optional[int] = None 59 | 60 | 61 | class PlaylistRepository( 62 | SortedSetRepository[Playlist, str], id_name='person_id' 63 | ): 64 | ... 65 | 66 | 67 | playlist_repository = PlaylistRepository( 68 | memory_data_source=DictMemoryDataSource(), 69 | fallback_data_source=DictFallbackDataSource(), 70 | expire_time=60, 71 | ) 72 | 73 | 74 | def make_playlist(person_id: str, *musics_ids: str) -> Playlist: 75 | return Playlist( 76 | person_id=person_id, 77 | data=[(id_, i) for i, id_ in enumerate(musics_ids)], 78 | ) 79 | 80 | 81 | # Application Layer 82 | 83 | 84 | async def main() -> None: 85 | person = make_person('John Doe', 33) 86 | playlist = make_playlist(person.id, 'm1', 'm2', 'm3') 87 | 88 | await person_service.add(person) 89 | await playlist_repository.add(playlist) 90 | 91 | goted_person = await person_service.get_one(person.id) 92 | goted_playlist = await playlist_repository.query(goted_person.id).entity 93 | 94 | print(goted_person) 95 | print(goted_playlist) 96 | 97 | 98 | asyncio.run(main()) 99 | -------------------------------------------------------------------------------- /docs/src/domain_model/tests.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | @test "should print domain model entities" { 4 | run coverage run -p docs/src/domain_model/simple.py 5 | [ "$status" -eq 0 ] 6 | [ "$output" = "$(cat docs/src/domain_model/simple.output)" ] 7 | } 8 | -------------------------------------------------------------------------------- /docs/src/index/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dutradda/dbdaora/5c87a3818e9d736bbf5e1438edc5929a2f5acd3f/docs/src/index/__init__.py -------------------------------------------------------------------------------- /docs/src/index/simple_hash.output: -------------------------------------------------------------------------------- 1 | Person(id='john_doe', name='John Doe', age=33) -------------------------------------------------------------------------------- /docs/src/index/simple_hash.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from dataclasses import dataclass 3 | 4 | from dbdaora import ( 5 | DictFallbackDataSource, 6 | DictMemoryDataSource, 7 | HashRepository, 8 | ) 9 | 10 | 11 | @dataclass 12 | class Person: 13 | id: str 14 | name: str 15 | age: int 16 | 17 | 18 | def make_person(name: str, age: int) -> Person: 19 | return Person(name.replace(' ', '_').lower(), name, age) 20 | 21 | 22 | class PersonRepository(HashRepository[Person, str]): 23 | key_attrs = ('id',) 24 | 25 | 26 | repository = PersonRepository( 27 | memory_data_source=DictMemoryDataSource(), 28 | fallback_data_source=DictFallbackDataSource(), 29 | expire_time=60, 30 | ) 31 | person = make_person('John Doe', 33) 32 | asyncio.run(repository.add(person)) 33 | 34 | geted_person = asyncio.run(repository.query(person.id).entity) 35 | print(geted_person) 36 | -------------------------------------------------------------------------------- /docs/src/index/simple_service.output: -------------------------------------------------------------------------------- 1 | Person(id='john_doe', name='John Doe', age=33) -------------------------------------------------------------------------------- /docs/src/index/simple_service.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from dataclasses import dataclass 3 | 4 | from dbdaora import ( 5 | DictFallbackDataSource, 6 | DictMemoryDataSource, 7 | HashRepository, 8 | make_hash_service, 9 | ) 10 | 11 | 12 | @dataclass 13 | class Person: 14 | id: str 15 | name: str 16 | age: int 17 | 18 | 19 | def make_person(name: str, age: int) -> Person: 20 | return Person(name.replace(' ', '_').lower(), name, age) 21 | 22 | 23 | class PersonRepository(HashRepository[Person, str]): 24 | ... 25 | 26 | 27 | async def make_memory_data_source() -> DictMemoryDataSource: 28 | return DictMemoryDataSource() 29 | 30 | 31 | async def make_fallback_data_source() -> DictFallbackDataSource: 32 | return DictFallbackDataSource() 33 | 34 | 35 | service = asyncio.run( 36 | make_hash_service( 37 | PersonRepository, 38 | memory_data_source_factory=make_memory_data_source, 39 | fallback_data_source_factory=make_fallback_data_source, 40 | repository_expire_time=60, 41 | ) 42 | ) 43 | person = make_person('John Doe', 33) 44 | asyncio.run(service.add(person)) 45 | 46 | geted_person = asyncio.run(service.get_one(person.id)) 47 | print(geted_person) 48 | -------------------------------------------------------------------------------- /docs/src/index/simple_sorted_set.output: -------------------------------------------------------------------------------- 1 | Playlist(id='my_plalist', data=[b'm1', b'm2', b'm3'], max_size=None) -------------------------------------------------------------------------------- /docs/src/index/simple_sorted_set.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from dbdaora import ( 4 | DictFallbackDataSource, 5 | DictMemoryDataSource, 6 | SortedSetEntity, 7 | SortedSetRepository, 8 | ) 9 | 10 | 11 | class Playlist(SortedSetEntity): 12 | id: str 13 | 14 | 15 | class PlaylistRepository(SortedSetRepository[Playlist, str]): 16 | ... 17 | 18 | 19 | repository = PlaylistRepository( 20 | memory_data_source=DictMemoryDataSource(), 21 | fallback_data_source=DictFallbackDataSource(), 22 | expire_time=60, 23 | ) 24 | data = [('m1', 1), ('m2', 2), ('m3', 3)] 25 | playlist = Playlist(id='my_plalist', data=data) 26 | asyncio.run(repository.add(playlist)) 27 | 28 | geted_playlist = asyncio.run(repository.query(playlist.id).entity) 29 | print(geted_playlist) 30 | -------------------------------------------------------------------------------- /docs/src/index/tests.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | @test "should print geted person" { 4 | run coverage run -p docs/src/index/simple_hash.py 5 | [ "$status" -eq 0 ] 6 | [ "$output" = "$(cat docs/src/index/simple_hash.output)" ] 7 | } 8 | 9 | @test "should print geted persons from service" { 10 | run coverage run -p docs/src/index/simple_service.py 11 | [ "$status" -eq 0 ] 12 | [ "$output" = "$(cat docs/src/index/simple_service.output)" ] 13 | } 14 | 15 | @test "should print geted playlist from repository" { 16 | run coverage run -p docs/src/index/simple_sorted_set.py 17 | [ "$status" -eq 0 ] 18 | [ "$output" = "$(cat docs/src/index/simple_sorted_set.output)" ] 19 | } 20 | -------------------------------------------------------------------------------- /docs/src/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | bats docs/src/index docs/src/domain_model docs/src 4 | -------------------------------------------------------------------------------- /docs/src/tests.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | 4 | @test "should print typed dict entities" { 5 | run coverage run -p docs/src/typed_dict.py 6 | [ "$status" -eq 0 ] 7 | [ "$output" = "$(cat docs/src/typed_dict.output)" ] 8 | } 9 | -------------------------------------------------------------------------------- /docs/src/typed_dict.output: -------------------------------------------------------------------------------- 1 | {'id': 'john_doe', 'name': 'John Doe', 'age': 33} 2 | {'data': [b'm1', b'm2', b'm3'], 'max_size': None, 'person_id': 'john_doe'} -------------------------------------------------------------------------------- /docs/src/typed_dict.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from typing import TypedDict 3 | 4 | from jsondaora import jsondaora 5 | 6 | from dbdaora import ( 7 | DictFallbackDataSource, 8 | DictMemoryDataSource, 9 | HashRepository, 10 | SortedSetDictEntity, 11 | SortedSetRepository, 12 | make_hash_service, 13 | ) 14 | 15 | 16 | # Data Source Layer 17 | 18 | 19 | async def make_memory_data_source() -> DictMemoryDataSource: 20 | return DictMemoryDataSource() 21 | 22 | 23 | async def make_fallback_data_source() -> DictFallbackDataSource: 24 | return DictFallbackDataSource() 25 | 26 | 27 | # Domain Layer 28 | 29 | 30 | @jsondaora 31 | class Person(TypedDict): 32 | id: str 33 | name: str 34 | age: int 35 | 36 | 37 | def make_person(name: str, age: int) -> Person: 38 | return Person(id=name.replace(' ', '_').lower(), name=name, age=age) 39 | 40 | 41 | class PersonRepository(HashRepository[Person, str]): 42 | ... 43 | 44 | 45 | person_service = asyncio.run( 46 | make_hash_service( 47 | PersonRepository, 48 | memory_data_source_factory=make_memory_data_source, 49 | fallback_data_source_factory=make_fallback_data_source, 50 | repository_expire_time=60, 51 | ) 52 | ) 53 | 54 | 55 | class Playlist(SortedSetDictEntity): 56 | person_id: str 57 | 58 | 59 | class PlaylistRepository(SortedSetRepository[Playlist, str]): 60 | id_name = 'person_id' 61 | 62 | 63 | playlist_repository = PlaylistRepository( 64 | memory_data_source=DictMemoryDataSource(), 65 | fallback_data_source=DictFallbackDataSource(), 66 | expire_time=60, 67 | ) 68 | 69 | 70 | def make_playlist(person_id: str, *musics_ids: str) -> Playlist: 71 | return Playlist( 72 | person_id=person_id, 73 | data=[(id_, i) for i, id_ in enumerate(musics_ids)], 74 | max_size=None, 75 | ) 76 | 77 | 78 | # Application Layer 79 | 80 | 81 | async def main() -> None: 82 | person = make_person('John Doe', 33) 83 | playlist = make_playlist(person['id'], 'm1', 'm2', 'm3') 84 | 85 | await person_service.add(person) 86 | await playlist_repository.add(playlist) 87 | 88 | goted_person = await person_service.get_one(person['id']) 89 | goted_playlist = await playlist_repository.query(goted_person['id']).entity 90 | 91 | print(goted_person) 92 | print(goted_playlist) 93 | 94 | 95 | asyncio.run(main()) 96 | -------------------------------------------------------------------------------- /docs/stylesheets/dark_theme.css: -------------------------------------------------------------------------------- 1 | /* 2 | ////////////////// 3 | // Main content // 4 | ////////////////// 5 | */ 6 | 7 | /* 8 | Default text color 9 | and background color 10 | */ 11 | .md-main { 12 | color: #F5F5F5 !important; 13 | background-color: #212121 !important; 14 | } 15 | 16 | /* 17 | Main headlines 18 | */ 19 | .md-main h1 { 20 | color: white !important; 21 | } 22 | 23 | /* 24 | Tables 25 | */ 26 | tbody { 27 | background-color: rgba(255, 255, 255, 0.05) !important; 28 | } 29 | 30 | .md-typeset table:not([class]) th { 31 | background-color: rgba(255,255,255,0.11) !important; 32 | } 33 | 34 | .md-typeset table:not([class]) tr:hover { 35 | box-shadow: none !important; 36 | background-color: rgba(255, 255, 255, 0.04) !important; 37 | } 38 | 39 | /* 40 | Blockquotes 41 | */ 42 | .md-typeset blockquote { 43 | color: rgba(255,255,255,0.8) !important; 44 | border-color: rgba(255,255,255,0.54) !important; 45 | } 46 | 47 | /* 48 | //////////////////// 49 | // Navigation bar // 50 | //////////////////// 51 | */ 52 | 53 | /* 54 | Left and right toc scrollbar 55 | */ 56 | .md-sidebar__scrollwrap::-webkit-scrollbar-thumb { 57 | background-color: #E0E0E0 !important; 58 | } 59 | 60 | 61 | 62 | .md-nav { 63 | color: #F5F5F5 !important; 64 | background-color: #212121 !important; 65 | } 66 | 67 | /* 68 | Arrow Left Icon 69 | */ 70 | html .md-nav--primary .md-nav__title:before { 71 | color: #FAFAFA !important; 72 | } 73 | 74 | .md-nav__title { 75 | color: rgba(255,255,255,1) !important; 76 | background-color: #212121 !important; 77 | } 78 | 79 | /* 80 | Arrow Right Icon 81 | */ 82 | .md-nav--primary .md-nav__link:after { 83 | color: #FAFAFA !important; 84 | } 85 | 86 | .md-nav__list { 87 | color: rgba(255,255,255,1) !important; 88 | background-color: #212121 !important; 89 | } 90 | 91 | .md-nav__item { 92 | color: rgba(255,255,255,1) !important; 93 | background-color: #212121 !important; 94 | } 95 | 96 | .md-nav__link[data-md-state=blur] { 97 | color: rgba(255,255,255,0.54) !important; 98 | } 99 | 100 | /* 101 | //////////// 102 | // Search // 103 | //////////// 104 | */ 105 | 106 | /* 107 | scroll bar 108 | 109 | attention: 110 | background is scroll handle color! 111 | */ 112 | .md-search__scrollwrap::-webkit-scrollbar-thumb { 113 | background-color: #E0E0E0 !important; 114 | } 115 | /* 116 | scroll bar background color 117 | */ 118 | .md-search__scrollwrap { 119 | background-color: #424242 !important; 120 | } 121 | 122 | /* 123 | Icon color 124 | */ 125 | .md-search-result__article--document:before { 126 | color: #EEEEEE !important; 127 | } 128 | 129 | /* 130 | headline color and 131 | result list background 132 | */ 133 | .md-search-result__list { 134 | color: #EEEEEE !important; 135 | background-color: #212121 !important; 136 | } 137 | 138 | /* 139 | result info/count 140 | */ 141 | .md-search-result__meta { 142 | background-color: #EEEEEE !important; 143 | } 144 | 145 | /* 146 | article preview text color 147 | */ 148 | .md-search-result__teaser { 149 | color: #BDBDBD !important; 150 | } 151 | -------------------------------------------------------------------------------- /docs/typed-dict.md: -------------------------------------------------------------------------------- 1 | # Typed Dict Example 2 | 3 | ```python 4 | {!./src/typed_dict.py!} 5 | ``` 6 | 7 | ```python 8 | {!./src/typed_dict.output!} 9 | ``` 10 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: dbdaora 2 | site_description: | 3 | Communicates with databases using repository and 4 | service patterns and python dataclasses 5 | 6 | theme: 7 | name: material 8 | custom_dir: theme 9 | palette: 10 | primary: deep-purple 11 | accent: grey 12 | logo: dbdaora.svg 13 | favicon: favicon.png 14 | 15 | repo_name: dutradda/dbdaora 16 | repo_url: https://github.com/dutradda/dbdaora 17 | edit_uri: '' 18 | 19 | nav: 20 | - dbdaora: index.md 21 | - Domain Model Example: domain-model/simple.md 22 | - Typed Dict Entity: typed-dict.md 23 | # - Features: 'features.md' 24 | # - Repository pattern intro: 'repository-pattern.md' 25 | # - Alternatives, Inspiration and Comparisons: 'alternatives.md' 26 | # - History, Design and Future: 'history-design-future.md' 27 | # - External Links and Articles: 'external-links.md' 28 | # - Benchmarks: 'benchmarks.md' 29 | - Development - Contributing: contributing.md 30 | - Changelog: changelog.md 31 | 32 | markdown_extensions: 33 | - markdown_include.include: 34 | base_path: docs 35 | - codehilite: 36 | guess_lang: false 37 | 38 | extra_css: 39 | - stylesheets/dark_theme.css 40 | - stylesheets/codehilite.css 41 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | [mypy-dbdaora.*.conftest] 3 | ignore_errors = True 4 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ['flit'] 3 | build-backend = 'flit.buildapi' 4 | 5 | [tool.flit.metadata] 6 | module = 'dbdaora' 7 | author = 'Diogo Dutra' 8 | author-email = 'diogodutradamata@gmail.com' 9 | home-page = 'https://github.com/dutradda/dbdaora' 10 | classifiers = [ 11 | 'License :: OSI Approved :: MIT License', 12 | 'Intended Audience :: Developers', 13 | 'Programming Language :: Python :: 3.8', 14 | 'Programming Language :: Python :: 3 :: Only', 15 | 'Topic :: Database', 16 | ] 17 | requires = [ 18 | 'circuitbreaker', 19 | 'cachetools', 20 | 'jsondaora', 21 | ] 22 | description-file = 'README.md' 23 | requires-python = '>=3.8' 24 | 25 | [tool.flit.metadata.urls] 26 | Documentation = 'https://dutradda.github.io/dbdaora/' 27 | 28 | [tool.flit.metadata.requires-extra] 29 | test = [ 30 | 'black', 31 | 'isort', 32 | 'ipython', 33 | 'mypy', 34 | 'pytest-asyncio', 35 | 'pytest-cov', 36 | 'pytest-mock', 37 | 'pytest', 38 | ] 39 | doc = [ 40 | 'mkdocs', 41 | 'mkdocs-material', 42 | 'markdown-include' 43 | ] 44 | datastore = ['google-cloud-datastore'] 45 | aioredis = ['aioredis'] 46 | mongodb = ['motor'] 47 | newrelic = ['newrelic'] 48 | 49 | [tool.flit.sdist] 50 | exclude = [ 51 | "dbdaora/conftest.py", 52 | "dbdaora/_tests", 53 | "dbdaora/*/_tests", 54 | "dbdaora/*/*/_tests", 55 | "Makefile", 56 | "Bakefile", 57 | "devtools", 58 | "docs", 59 | "stubs", 60 | "mypy.ini", 61 | "mkdocs.yml", 62 | "theme", 63 | ".bumpversion.cfg", 64 | ".coveragerc", 65 | ".gitignore", 66 | ".gitmodules", 67 | ".flake8", 68 | ] 69 | 70 | [tool.isort] 71 | case_sensitive= '1' 72 | use_parentheses = '1' 73 | line_length = '79' 74 | order_by_type = '1' 75 | multi_line_output = '3' 76 | include_trailing_comma = '1' 77 | lines_after_imports = '2' 78 | atomic = '1' 79 | 80 | [tool.black] 81 | exclude = ''' 82 | \.pyi 83 | ''' 84 | target-version = ['py38'] 85 | line-length = '79' 86 | skip-string-normalization = '1' 87 | -------------------------------------------------------------------------------- /stubs/aioredis/__init__.pyi: -------------------------------------------------------------------------------- 1 | from typing import ( 2 | Any, 3 | ClassVar, 4 | Dict, 5 | NamedTuple, 6 | Optional, 7 | Sequence, 8 | Tuple, 9 | Type, 10 | Union, 11 | ) 12 | 13 | 14 | rangeOutput = Sequence[bytes] 15 | rangeWithScoresOutput = Sequence[Tuple[bytes, float]] 16 | SortedSetData = Union[rangeOutput, rangeWithScoresOutput] 17 | 18 | 19 | class ConnectionsPool: 20 | ... 21 | 22 | 23 | class Redis: 24 | key_separator: ClassVar[str] = ':' 25 | _pool_or_conn: ConnectionsPool 26 | 27 | def make_key(self, *key_parts: str) -> str: ... 28 | 29 | async def get(self, key: str) -> Optional[bytes]: ... 30 | 31 | async def set(self, key: str, data: str) -> None: ... 32 | 33 | async def delete(self, key: str) -> None: ... 34 | 35 | async def expire(self, key: str, time: int) -> None: ... 36 | 37 | async def exists(self, key: str) -> int: ... 38 | 39 | async def zrevrange( 40 | self, key: str, start: int, stop: int, withscores: bool = False 41 | ) -> Optional[SortedSetData]: ... 42 | 43 | async def zrange( 44 | self, 45 | key: str, 46 | start: int = 0, 47 | stop: int = -1, 48 | withscores: bool = False, 49 | ) -> Optional[SortedSetData]: ... 50 | 51 | async def zadd( 52 | self, key: str, score: float, member: str, *pairs: Union[float, str] 53 | ) -> None: ... 54 | 55 | async def hmset( 56 | self, 57 | key: str, 58 | field: Union[str, bytes], 59 | value: Union[str, bytes], 60 | *pairs: Union[str, bytes], 61 | ) -> None: ... 62 | 63 | async def hmget( 64 | self, key: str, field: Union[str, bytes], *fields: Union[str, bytes] 65 | ) -> Sequence[Optional[bytes]]: ... 66 | 67 | async def hgetall(self, key: str) -> Dict[bytes, bytes]: ... 68 | 69 | def pipeline(self) -> Any: ... 70 | 71 | async def zrevrangebyscore( 72 | self, 73 | key: str, 74 | max: float = float('inf'), 75 | min: float = float('-inf'), 76 | withscores: bool = False, 77 | ) -> Optional[SortedSetData]: 78 | ... 79 | 80 | async def zrangebyscore( 81 | self, 82 | key: str, 83 | min: float = float('-inf'), 84 | max: float = float('inf'), 85 | withscores: bool = False, 86 | ) -> Optional[SortedSetData]: 87 | ... 88 | 89 | async def zcard(self, key: str) -> int: 90 | ... 91 | 92 | 93 | async def create_redis_pool( 94 | address: str, *, commands_factory: Optional[Type[Redis]] = None 95 | ) -> Redis: ... 96 | 97 | 98 | class GeoPoint(NamedTuple): 99 | longitude: float 100 | latitude: float 101 | 102 | 103 | class GeoMember(NamedTuple): 104 | member: bytes 105 | dist: Optional[float] 106 | coord: Optional[GeoPoint] 107 | -------------------------------------------------------------------------------- /stubs/aioredis/commands/__init__.pyi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dutradda/dbdaora/5c87a3818e9d736bbf5e1438edc5929a2f5acd3f/stubs/aioredis/commands/__init__.pyi -------------------------------------------------------------------------------- /stubs/aioredis/commands/transaction.pyi: -------------------------------------------------------------------------------- 1 | from typing import Any, Type, Union 2 | 3 | from aioredis import ConnectionsPool, Redis 4 | 5 | 6 | class MultiExec: 7 | def __init__( 8 | self, 9 | pool_or_connection: ConnectionsPool, 10 | commands_factory: Type[Redis], 11 | ): 12 | ... 13 | 14 | def delete(self, key: str) -> Any: 15 | ... 16 | 17 | def hmset( 18 | self, 19 | key: str, 20 | field: Union[str, bytes], 21 | value: Union[str, bytes], 22 | *pairs: Union[str, bytes], 23 | ) -> Any: 24 | ... 25 | 26 | def zadd( 27 | self, key: str, score: float, member: str, *pairs: Union[float, str] 28 | ) -> Any: 29 | ... 30 | 31 | async def execute(self, *, return_exceptions: bool = False) -> Any: 32 | ... 33 | -------------------------------------------------------------------------------- /stubs/bson/__init__.pyi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dutradda/dbdaora/5c87a3818e9d736bbf5e1438edc5929a2f5acd3f/stubs/bson/__init__.pyi -------------------------------------------------------------------------------- /stubs/bson/objectid.pyi: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | 3 | 4 | @dataclasses.dataclass 5 | class ObjectId: 6 | key: bytes 7 | -------------------------------------------------------------------------------- /stubs/cachetools/__init__.pyi: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict 2 | 3 | 4 | class Cache(Dict[Any, Any]): 5 | def __init__(self, *args: Any, **kwargs: Any): ... 6 | 7 | 8 | class LFUCache(Cache): 9 | def __init__(self, maxsize: int): ... 10 | 11 | 12 | class LRUCache(Cache): 13 | def __init__(self, maxsize: int): ... 14 | 15 | 16 | class TTLCache(Cache): 17 | def __init__(self, maxsize: int, ttl: int): ... 18 | -------------------------------------------------------------------------------- /stubs/circuitbreaker/__init__.pyi: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from typing import ( 3 | Any, 4 | Awaitable, 5 | Callable, 6 | Optional, 7 | Tuple, 8 | Type, 9 | TypeVar, 10 | Union, 11 | ) 12 | 13 | 14 | FuncReturn = TypeVar('FuncReturn') 15 | 16 | STATE_CLOSED = 'closed' 17 | STATE_OPEN = 'open' 18 | 19 | 20 | class CircuitBreaker: 21 | _expected_exception: Union[Type[Exception], Tuple[Type[Exception], ...]] 22 | _failure_threshold: int 23 | name: str 24 | FAILURE_THRESHOLD: int 25 | RECOVERY_TIMEOUT: int 26 | EXPECTED_EXCEPTION: Type[Exception] 27 | FALLBACK_FUNCTION: Optional[Callable[..., Awaitable[FuncReturn]]] 28 | 29 | def __init__( 30 | self, 31 | failure_threshold: Optional[int] = None, 32 | recovery_timeout: Optional[int] = None, 33 | expected_exception: Optional[Union[Type[Exception], Tuple[Type[Exception], ...]]] = None, 34 | name: Optional[str] = None, 35 | fallback_function: Optional[ 36 | Callable[..., Awaitable[FuncReturn]] 37 | ] = None, 38 | ): ... 39 | 40 | async def call( 41 | self, 42 | func: Callable[..., Awaitable[FuncReturn]], 43 | *args: Any, 44 | **kwargs: Any, 45 | ) -> FuncReturn: ... 46 | 47 | def __call__( 48 | self, wrapped: Callable[..., FuncReturn] 49 | ) -> Callable[..., FuncReturn]: ... 50 | 51 | @property 52 | def opened(self) -> bool: ... 53 | 54 | @property 55 | def fallback_function( 56 | self, 57 | ) -> Optional[Callable[..., Awaitable[FuncReturn]]]: ... 58 | 59 | @property 60 | def open_until(self) -> datetime: ... 61 | 62 | @property 63 | def open_remaining(self) -> int: ... 64 | 65 | @property 66 | def failure_count(self) -> int: ... 67 | 68 | @property 69 | def last_failure(self) -> Optional[Exception]: ... 70 | 71 | 72 | class CircuitBreakerError(Exception): 73 | def __init__( 74 | self, circuit_breaker: CircuitBreaker, *args: Any, **kwargs: Any 75 | ): ... 76 | -------------------------------------------------------------------------------- /stubs/google/__init__.pyi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dutradda/dbdaora/5c87a3818e9d736bbf5e1438edc5929a2f5acd3f/stubs/google/__init__.pyi -------------------------------------------------------------------------------- /stubs/google/cloud/__init__.pyi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dutradda/dbdaora/5c87a3818e9d736bbf5e1438edc5929a2f5acd3f/stubs/google/cloud/__init__.pyi -------------------------------------------------------------------------------- /stubs/google/cloud/datastore/__init__.pyi: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Any, Dict, Iterable, List, Optional 3 | 4 | 5 | class Key: 6 | kind: str 7 | 8 | 9 | class Query: 10 | def fetch(self) -> Iterable[Dict[str, Any]]: ... 11 | 12 | 13 | @dataclass 14 | class Entity(Dict[str, Any]): 15 | key: Key 16 | exclude_from_indexes: Iterable[str] = () 17 | 18 | 19 | class Client: 20 | def get(self, key: Key) -> Optional[Entity]: ... 21 | 22 | def put(self, data: Entity) -> None: ... 23 | 24 | def delete(self, key: Key) -> None: ... 25 | 26 | def get_multi(self, keys: Iterable[Key]) -> List[Entity]: ... 27 | 28 | def key(self, *key_parts: Any) -> Key: ... 29 | 30 | def query(self, kind: str, **kwargs: Any) -> Query: ... 31 | -------------------------------------------------------------------------------- /stubs/motor/__init__.pyi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dutradda/dbdaora/5c87a3818e9d736bbf5e1438edc5929a2f5acd3f/stubs/motor/__init__.pyi -------------------------------------------------------------------------------- /stubs/motor/motor_asyncio.pyi: -------------------------------------------------------------------------------- 1 | from typing import Any, AsyncGenerator, Dict, Optional 2 | 3 | 4 | class AsyncIOMotorCollection: 5 | async def replace_one( 6 | self, 7 | filter: Dict[str, Any], 8 | replacement: Dict[str, Any], 9 | upsert: bool = False, 10 | bypass_document_validation: bool = False, 11 | collation: Any = None, 12 | session: Any = None, 13 | ) -> Dict[str, Any]: ... 14 | 15 | async def delete_one( 16 | self, 17 | filter: Dict[str, Any], 18 | collation: Any = None, 19 | session: Any = None, 20 | ) -> Dict[str, Any]: ... 21 | 22 | async def find_one( 23 | self, 24 | filter: Optional[Dict[str, Any]] = None, 25 | *args: Any, 26 | **kwargs: Any, 27 | ) -> Dict[str, Any]: ... 28 | 29 | def find( 30 | self, 31 | *args: Any, 32 | **kwargs: Any, 33 | ) -> AsyncGenerator[Dict[str, Any], None]: ... 34 | 35 | async def create_index( 36 | self, name: str, expireAfterSeconds: Optional[int], 37 | ) -> None: ... 38 | 39 | async def drop_index(self, name: str) -> None: ... 40 | 41 | def list_indexes(self) -> AsyncGenerator[Dict[str, Any], None]: ... 42 | 43 | 44 | class AsyncIOMotorDatabase: 45 | def __getitem__(self, key: str) -> AsyncIOMotorCollection: ... 46 | 47 | 48 | class AsyncIOMotorClient: 49 | def __getitem__(self, key: str) -> AsyncIOMotorDatabase: ... 50 | -------------------------------------------------------------------------------- /stubs/newrelic/__init__.pyi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dutradda/dbdaora/5c87a3818e9d736bbf5e1438edc5929a2f5acd3f/stubs/newrelic/__init__.pyi -------------------------------------------------------------------------------- /stubs/newrelic/agent.pyi: -------------------------------------------------------------------------------- 1 | from typing import Any, Optional 2 | 3 | 4 | def wrap_datastore_trace( 5 | module: Any, 6 | object_path: str, 7 | product: str, 8 | target: Optional[str], 9 | operation: str, 10 | ) -> Any: ... 11 | -------------------------------------------------------------------------------- /stubs/pymongo/__init__.pyi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dutradda/dbdaora/5c87a3818e9d736bbf5e1438edc5929a2f5acd3f/stubs/pymongo/__init__.pyi -------------------------------------------------------------------------------- /stubs/pymongo/errors.pyi: -------------------------------------------------------------------------------- 1 | class OperationFailure(Exception): 2 | ... 3 | -------------------------------------------------------------------------------- /theme/main.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block extrahead %} 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | {% endblock %} -------------------------------------------------------------------------------- /theme/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "Wiki", 3 | "name": "Wiki\nMarkus Ressel", 4 | "icons": [ 5 | { 6 | "src": "open-book-2x.png", 7 | "sizes": "96x96", 8 | "type": "image/png" 9 | }, 10 | { 11 | "src": "open-book-3x.png", 12 | "sizes": "144x144", 13 | "type": "image/png" 14 | }, 15 | { 16 | "src": "open-book-4x.png", 17 | "sizes": "192x192", 18 | "type": "image/png" 19 | } 20 | ], 21 | "background_color": "#212121", 22 | "theme_color": "#212121", 23 | "start_url": "/", 24 | "display": "standalone" 25 | } 26 | -------------------------------------------------------------------------------- /theme/open-book-2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dutradda/dbdaora/5c87a3818e9d736bbf5e1438edc5929a2f5acd3f/theme/open-book-2x.png -------------------------------------------------------------------------------- /theme/open-book-3x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dutradda/dbdaora/5c87a3818e9d736bbf5e1438edc5929a2f5acd3f/theme/open-book-3x.png -------------------------------------------------------------------------------- /theme/open-book-4x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dutradda/dbdaora/5c87a3818e9d736bbf5e1438edc5929a2f5acd3f/theme/open-book-4x.png --------------------------------------------------------------------------------