├── .gitignore ├── .readthedocs.yml ├── .travis.yml ├── .vscode └── settings.json ├── CHANGELOG.rst ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── bloop ├── __init__.py ├── actions.py ├── conditions.py ├── engine.py ├── exceptions.py ├── ext │ ├── __init__.py │ ├── arrow.py │ ├── delorean.py │ └── pendulum.py ├── models.py ├── search.py ├── session.py ├── signals.py ├── stream │ ├── __init__.py │ ├── buffer.py │ ├── coordinator.py │ ├── shard.py │ └── stream.py ├── transactions.py ├── types.py └── util.py ├── docs ├── Makefile ├── _static │ ├── bloop.css │ └── favicon-cog.ico ├── _templates │ └── layout.html ├── api │ ├── internal.rst │ └── public.rst ├── conf.py ├── docutils.conf ├── index.rst ├── make.bat ├── meta │ ├── about.rst │ └── changelog.rst └── user │ ├── conditions.rst │ ├── engine.rst │ ├── extensions.rst │ ├── install.rst │ ├── models.rst │ ├── patterns.rst │ ├── quickstart.rst │ ├── signals.rst │ ├── streams.rst │ ├── transactions.rst │ └── types.rst ├── examples ├── documents.py ├── mixins.py ├── replication.py └── tweet.py ├── requirements.txt ├── scripts ├── graph-dependencies └── single-test ├── setup.py ├── tests ├── __init__.py ├── helpers │ ├── __init__.py │ ├── models.py │ └── utils.py ├── integ │ ├── __init__.py │ ├── conftest.py │ ├── models.py │ ├── test_basic.py │ ├── test_inheritance.py │ └── test_queries.py └── unit │ ├── __init__.py │ ├── conftest.py │ ├── test_actions.py │ ├── test_conditions.py │ ├── test_engine.py │ ├── test_ext │ ├── __init__.py │ ├── test_arrow.py │ ├── test_delorean.py │ └── test_pendulum.py │ ├── test_models.py │ ├── test_search.py │ ├── test_session.py │ ├── test_stream │ ├── __init__.py │ ├── conftest.py │ ├── test_buffer.py │ ├── test_coordinator.py │ ├── test_shard.py │ └── test_stream.py │ ├── test_transactions.py │ ├── test_types.py │ └── test_util.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | 25 | # PyInstaller 26 | # Usually these files are written by a python script from a template 27 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 28 | *.manifest 29 | *.spec 30 | 31 | # Installer logs 32 | pip-log.txt 33 | pip-delete-this-directory.txt 34 | 35 | # Unit test / coverage reports 36 | htmlcov/ 37 | .tox/ 38 | .coverage 39 | .cache 40 | nosetests.xml 41 | coverage.xml 42 | .pytest_cache 43 | 44 | # Translations 45 | *.mo 46 | *.pot 47 | 48 | # Django stuff: 49 | *.log 50 | 51 | # Sphinx documentation 52 | docs/_build/ 53 | 54 | # PyBuilder 55 | target/ 56 | 57 | # Subl 58 | *.sublime-project 59 | *.sublime-workspace 60 | 61 | # Versioning 62 | .python-version 63 | .idea/ 64 | .venv/ 65 | 66 | # Personal testing files 67 | t.py 68 | bloop.svg 69 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | build: 3 | os: ubuntu-lts-latest 4 | tools: 5 | python: "latest" 6 | python: 7 | install: 8 | - requirements: requirements.txt 9 | - method: pip 10 | path: . 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: trusty 2 | sudo: required 3 | language: python 4 | dist: trusty 5 | python: 3.6 6 | cache: pip 7 | services: 8 | - docker 9 | env: 10 | matrix: 11 | - TOXENV=unit 12 | - TOXENV=integ 13 | - TOXENV=docs 14 | install: pip install tox 15 | script: tox -e $TOXENV -- -s 16 | 17 | notifications: 18 | slack: 19 | secure: cXjzhmFI0oAWifBilGvTApiL8VSyM7/4v3Ve4TZeU+15EoYf05AhNoHs0FKVcj+ockYPqlj3p+SIUXnGfcI41QFlO09gGvFVB/FA/PHH2n4hb20+zxSx5Ic9ac3B2Nb5u2lWGsSsRTvCHtC0Wcxx878ML5UFrP5yu4vKIgj8AyE= 20 | webhooks: 21 | urls: 22 | - https://webhooks.gitter.im/e/12807f3b2c9083de2e36 23 | on_success: change 24 | on_failure: always 25 | on_start: never 26 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.testing.pytestArgs": [ 3 | "tests" 4 | ], 5 | "python.testing.unittestEnabled": false, 6 | "python.testing.pytestEnabled": true 7 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2021 Joe Cross 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst CHANGELOG.rst 2 | recursive-exclude tests * 3 | recursive-exclude examples * 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | .PHONY: cov docs publish 3 | 4 | cov: 5 | scripts/single-test 6 | 7 | docs: 8 | cd docs && $(MAKE) html 9 | firefox docs/_build/html/index.html 10 | 11 | publish: 12 | - rm -fr build dist .egg bloop.egg-info 13 | python setup.py sdist bdist_wheel 14 | twine check dist/* 15 | twine upload dist/* 16 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://readthedocs.org/projects/bloop/badge?style=flat-square 2 | :target: http://bloop.readthedocs.org/ 3 | .. image:: https://img.shields.io/travis/numberoverzero/bloop/master.svg?style=flat-square 4 | :target: https://travis-ci.org/numberoverzero/bloop 5 | .. image:: https://img.shields.io/gitter/room/numberoverzero/bloop.svg?style=flat-square 6 | :target: https://gitter.im/numberoverzero/bloop 7 | .. image:: https://img.shields.io/pypi/v/bloop.svg?style=flat-square 8 | :target: https://pypi.python.org/pypi/bloop 9 | 10 | Bloop is an object mapper for DynamoDB and DynamoDBStreams. 11 | 12 | Requires Python 3.6+ 13 | 14 | :: 15 | 16 | pip install bloop 17 | 18 | ======= 19 | Usage 20 | ======= 21 | 22 | First, we need to import all the things: 23 | 24 | .. code-block:: python 25 | 26 | >>> from bloop import ( 27 | ... BaseModel, Boolean, Column, String, UUID, 28 | ... GlobalSecondaryIndex, Engine 29 | ... ) 30 | 31 | Next we'll define the account model (with streaming enabled), and create the backing table: 32 | 33 | .. code-block:: python 34 | 35 | >>> class Account(BaseModel): 36 | ... class Meta: 37 | ... stream = { 38 | ... "include": {"old", "new"} 39 | ... } 40 | ... id = Column(UUID, hash_key=True) 41 | ... name = Column(String) 42 | ... email = Column(String) 43 | ... by_email = GlobalSecondaryIndex(projection='keys', hash_key='email') 44 | ... verified = Column(Boolean, default=False) 45 | ... 46 | >>> engine = Engine() 47 | >>> engine.bind(Account) 48 | 49 | Let's make a few users and persist them: 50 | 51 | .. code-block:: python 52 | 53 | >>> import uuid 54 | >>> admin = Account(id=uuid.uuid4(), email="admin@domain.com") 55 | >>> admin.name = "Admin McAdminFace" 56 | >>> support = Account(name="this-is-fine.jpg", email="help@domain.com") 57 | >>> support.id = uuid.uuid4() 58 | >>> engine.save(admin, support) 59 | 60 | Or do the same in a transaction: 61 | 62 | .. code-block:: python 63 | 64 | >>> with engine.transaction() as tx: 65 | ... tx.save(admin) 66 | ... tx.save(support) 67 | ... 68 | >>> 69 | 70 | And find them again: 71 | 72 | .. code-block:: python 73 | 74 | >>> q = engine.query( 75 | ... Account.by_email, 76 | ... key=Account.email=="help@domain.com" 77 | ... ) 78 | >>> q.first() 79 | Account(email='help@domain.com', 80 | id=UUID('d30e343f-f067-4fe5-bc5e-0b00cdeaf2ba'), 81 | verified=False) 82 | 83 | .. code-block:: python 84 | 85 | >>> s = engine.scan( 86 | ... Account, 87 | ... filter=Account.name.begins_with("Admin") 88 | ... ) 89 | >>> s.one() 90 | Account(email='admin@domain.com', 91 | id=UUID('08da44ac-5ff6-4f70-8a3f-b75cadb4dd79'), 92 | name='Admin McAdminFace', 93 | verified=False) 94 | 95 | Let's find them in the stream: 96 | 97 | .. code-block:: python 98 | 99 | >>> stream = engine.stream(Account, "trim_horizon") 100 | >>> next(stream) 101 | {'key': None, 102 | 'meta': {'created_at': datetime.datetime(...), 103 | 'event': {'id': 'cbb9a9b45eb0a98889b7da85913a5c65', 104 | 'type': 'insert', 105 | 'version': '1.1'}, 106 | 'sequence_number': '100000000000588052489'}, 107 | 'new': Account( 108 | email='help@domain.com', 109 | id=UUID('d30e343f-...-0b00cdeaf2ba'), 110 | name='this-is-fine.jpg', 111 | verified=False), 112 | 'old': None} 113 | >>> next(stream) 114 | {'key': None, 115 | 'meta': {'created_at': datetime.datetime(...), 116 | 'event': {'id': 'cbdfac5671ea38b99017c4b43a8808ce', 117 | 'type': 'insert', 118 | 'version': '1.1'}, 119 | 'sequence_number': '200000000000588052506'}, 120 | 'new': Account( 121 | email='admin@domain.com', 122 | id=UUID('08da44ac-...-b75cadb4dd79'), 123 | name='Admin McAdminFace', 124 | verified=False), 125 | 'old': None} 126 | >>> next(stream) 127 | >>> next(stream) 128 | >>> 129 | 130 | ============= 131 | What's Next 132 | ============= 133 | 134 | Check out the `User Guide`_ or `Public API Reference`_ to create your own nested types, overlapping models, 135 | set up cross-region replication in less than 20 lines, and more! 136 | 137 | .. _User Guide: https://bloop.readthedocs.io/en/latest/user/quickstart.html 138 | .. _Public API Reference: https://bloop.readthedocs.io/en/latest/api/public.html 139 | -------------------------------------------------------------------------------- /bloop/__init__.py: -------------------------------------------------------------------------------- 1 | from .conditions import Condition 2 | from .engine import Engine 3 | from .exceptions import ( 4 | BloopException, 5 | ConstraintViolation, 6 | MissingObjects, 7 | RecordsExpired, 8 | ShardIteratorExpired, 9 | TableMismatch, 10 | TransactionCanceled, 11 | ) 12 | from .models import BaseModel, Column, GlobalSecondaryIndex, LocalSecondaryIndex 13 | from .search import QueryIterator, ScanIterator 14 | from .signals import ( 15 | before_create_table, 16 | model_bound, 17 | model_created, 18 | model_validated, 19 | object_deleted, 20 | object_loaded, 21 | object_modified, 22 | object_saved, 23 | ) 24 | from .stream import Stream 25 | from .transactions import ReadTransaction, WriteTransaction 26 | from .types import ( 27 | UUID, 28 | Binary, 29 | Boolean, 30 | DateTime, 31 | DynamicList, 32 | DynamicMap, 33 | Integer, 34 | List, 35 | Map, 36 | Number, 37 | Set, 38 | String, 39 | Timestamp, 40 | ) 41 | from .util import missing 42 | 43 | 44 | __all__ = [ 45 | # Models 46 | "BaseModel", "Boolean", "Binary", "Column", "DateTime", "Engine", "GlobalSecondaryIndex", "Integer", 47 | "List", "LocalSecondaryIndex", "Map", "Number", "Set", "String", "UUID", 48 | 49 | # Exceptions 50 | "BloopException", "ConstraintViolation", "MissingObjects", 51 | "RecordsExpired", "ShardIteratorExpired", "TableMismatch", "TransactionCanceled", 52 | 53 | # Signals 54 | "before_create_table", "model_bound", "model_created", "model_validated", 55 | "object_deleted", "object_loaded", "object_modified", "object_saved", 56 | 57 | # Types 58 | "UUID", "Binary", "Boolean", "DateTime", "Integer", "List", "Map", "Number", "Set", "String", "Timestamp", 59 | "DynamicList", "DynamicMap", 60 | 61 | # Misc 62 | "Condition", "QueryIterator", "ReadTransaction", "ScanIterator", "Stream", "WriteTransaction", "missing", 63 | ] 64 | __version__ = "3.1.1" 65 | -------------------------------------------------------------------------------- /bloop/actions.py: -------------------------------------------------------------------------------- 1 | # https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.UpdateExpressions.html#Expressions.UpdateExpressions 2 | import enum 3 | from typing import Any, Union 4 | 5 | 6 | class ActionType(enum.Enum): 7 | """Represents how Dynamo should apply an update.""" 8 | Add = ("ADD", "{name_ref.name} {value_ref.name}", False) 9 | Delete = ("DELETE", "{name_ref.name} {value_ref.name}", False) 10 | Remove = ("REMOVE", "{name_ref.name}", True) 11 | Set = ("SET", "{name_ref.name}={value_ref.name}", True) 12 | 13 | def __init__(self, wire_key: str, fmt: str, nestable: bool): 14 | self.wire_key = wire_key 15 | self.fmt = fmt 16 | self.nestable = nestable 17 | 18 | def render(self, name_ref, value_ref): 19 | """name_ref, value_ref should be instances of ``bloop.conditions.Reference`` or None""" 20 | return self.fmt.format(name_ref=name_ref, value_ref=value_ref) 21 | 22 | def new_action(self, value) -> "Action": 23 | """Convenience function to instantiate an Action with this type""" 24 | if self is ActionType.Remove and value is None: 25 | return NONE_SENTINEL 26 | return Action(self, value) 27 | 28 | 29 | # O(1) __contains__ for Action.__new__ 30 | # noinspection PyTypeChecker 31 | _type_set = set(ActionType) 32 | 33 | 34 | class Action: 35 | # noinspection PyUnresolvedReferences 36 | """ 37 | Encapsulates an update value and how Dynamo should apply the update. 38 | 39 | Generally, you will only need to use the ``Action`` class if you are updating an atomic counter (ADD) 40 | or making additions and deletions from a set (ADD, DELETE). 41 | 42 | You do not need to use an ``Action`` for SET or REMOVE updates. 43 | 44 | .. code-block:: python 45 | 46 | >>> import bloop.actions 47 | >>> from my_models import Website, User 48 | >>> user = User() 49 | >>> website = Website() 50 | # SET and REMOVE don't need an explicit action 51 | >>> user.verified = True 52 | >>> del user.pw_hash 53 | # ADD and DELETE need explicit actions 54 | >>> website.view_count = bloop.actions.add(1) 55 | >>> website.remote_addrs = bloop.actions.delete({"::0", "localhost"}) 56 | """ 57 | def __new__(cls, action_type: ActionType, value): 58 | if action_type not in _type_set: 59 | raise ValueError(f"action_type must be one of {_type_set} but was {action_type}") 60 | return super().__new__(cls) 61 | 62 | def __init__(self, action_type: ActionType, value): 63 | self.__type = action_type 64 | self.__value = value 65 | 66 | @property 67 | def type(self): 68 | return self.__type 69 | 70 | @property 71 | def value(self): 72 | return self.__value 73 | 74 | def __repr__(self): 75 | return f"" 76 | 77 | def __eq__(self, other): 78 | return ( 79 | isinstance(other, Action) and 80 | (self.type is other.type) and 81 | (self.value == other.value) 82 | ) 83 | 84 | __hash__ = object.__hash__ 85 | 86 | 87 | def unwrap(x: Union[Action, Any]) -> Any: 88 | """return an action's inner value""" 89 | if isinstance(x, Action): 90 | return x.value 91 | return x 92 | 93 | 94 | def wrap(x: Any) -> Action: 95 | """return an action: REMOVE if x is None else SET""" 96 | if isinstance(x, Action): 97 | return x 98 | elif x is None: 99 | return NONE_SENTINEL 100 | return set(x) 101 | 102 | 103 | def add(value): 104 | # noinspection PyUnresolvedReferences 105 | """Create a new ADD action. 106 | 107 | The ADD action only supports Number and Set data types. 108 | In addition, ADD can only be used on top-level attributes, not nested attributes. 109 | 110 | .. code-block:: pycon 111 | 112 | >>> import bloop.actions 113 | >>> from my_models import Website 114 | >>> website = Website(...) 115 | >>> website.views = bloop.actions.add(1) 116 | >>> website.remote_addrs = bloop.actions.add({"::0", "localhost"}) 117 | """ 118 | return Action(ActionType.Add, value) 119 | 120 | 121 | def delete(value): 122 | # noinspection PyUnresolvedReferences 123 | """Create a new DELETE action. 124 | 125 | The DELETE action only supports Set data types. 126 | In addition, DELETE can only be used on top-level attributes, not nested attributes. 127 | 128 | .. code-block:: pycon 129 | 130 | >>> import bloop.actions 131 | >>> from my_models import Website 132 | >>> website = Website(...) 133 | >>> website.remote_addrs = bloop.actions.delete({"::0", "localhost"}) 134 | """ 135 | return Action(ActionType.Delete, value) 136 | 137 | 138 | def remove(value=None): 139 | # noinspection PyUnresolvedReferences 140 | """Create a new REMOVE action. 141 | 142 | Most types automatically create this action when you use ``del obj.some_attr`` or ``obj.some_attr = None`` 143 | 144 | .. code-block:: pycon 145 | 146 | >>> import bloop.actions 147 | >>> from my_models import User 148 | >>> user = User(...) 149 | # equivalent 150 | >>> user.shell = None 151 | >>> user.shell = bloop.actions.remove(None) 152 | """ 153 | if value is None: 154 | return NONE_SENTINEL 155 | return Action(ActionType.Remove, value) 156 | 157 | 158 | def set(value): 159 | # noinspection PyUnresolvedReferences 160 | """Create a new SET action. 161 | 162 | Most types automatically create this action when you use ``obj.some_attr = value`` 163 | 164 | .. code-block:: pycon 165 | 166 | >>> import bloop.actions 167 | >>> from my_models import User 168 | >>> user = User(...) 169 | # equivalent 170 | >>> user.shell = "/bin/sh" 171 | >>> user.shell = bloop.actions.set("/bin/sh") 172 | """ 173 | return Action(ActionType.Set, value) 174 | 175 | 176 | NONE_SENTINEL = Action(ActionType.Remove, None) 177 | -------------------------------------------------------------------------------- /bloop/exceptions.py: -------------------------------------------------------------------------------- 1 | class BloopException(Exception): 2 | """An unexpected exception occurred.""" 3 | 4 | 5 | class ConstraintViolation(BloopException): 6 | """A required condition was not met.""" 7 | 8 | 9 | class TransactionCanceled(BloopException): 10 | """The transaction was canceled. 11 | 12 | A WriteTransaction is canceled when: 13 | * A condition in one of the condition expressions is not met. 14 | * A table in the TransactWriteItems request is in a different account or region. 15 | * More than one action in the TransactWriteItems operation targets the same item. 16 | * There is insufficient provisioned capacity for the transaction to be completed. 17 | * An item size becomes too large (larger than 400 KB), or a local secondary index (LSI) 18 | becomes too large, or a similar validation error occurs because of changes made by the transaction. 19 | 20 | A ReadTransaction is canceled when: 21 | * There is an ongoing TransactGetItems operation that conflicts with a concurrent PutItem, 22 | UpdateItem, DeleteItem or TransactWriteItems request. 23 | * A table in the TransactGetItems request is in a different account or region. 24 | * There is insufficient provisioned capacity for the transaction to be completed. 25 | * There is a user error, such as an invalid data format. 26 | 27 | .. seealso:: 28 | 29 | The API reference for `TransactionCanceledException`_ 30 | 31 | .. _TransactionCanceledException: https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_TransactGetItems.html#API_TransactGetItems_Errors 32 | """ # noqa: E501 33 | 34 | 35 | class TransactionTokenExpired(BloopException): 36 | """The transaction's tx_id (ClientRequestToken) was first used more than 10 minutes ago""" 37 | 38 | 39 | class MissingObjects(BloopException): 40 | """Some objects were not found.""" 41 | 42 | #: The objects that failed to load 43 | objects: list 44 | 45 | def __init__(self, *args, objects=None): 46 | super().__init__(*args) 47 | self.objects = list(objects) if objects else [] 48 | 49 | 50 | class TableMismatch(BloopException): 51 | """The expected and actual tables for this Model do not match.""" 52 | 53 | 54 | class InvalidSearch(BloopException, ValueError): 55 | """The search was malformed""" 56 | 57 | 58 | class MissingKey(BloopException, ValueError): 59 | """The instance must provide values for its key columns.""" 60 | 61 | 62 | class RecordsExpired(BloopException): 63 | """The requested stream records are beyond the trim horizon.""" 64 | 65 | 66 | class ShardIteratorExpired(BloopException): 67 | """The shard iterator is past its expiration date.""" 68 | 69 | 70 | class InvalidModel(BloopException, ValueError): 71 | """This is not a valid Model.""" 72 | 73 | 74 | class InvalidTemplate(BloopException, ValueError): 75 | """This is not a valid template string.""" 76 | 77 | 78 | class InvalidStream(BloopException, ValueError): 79 | """This is not a valid stream definition.""" 80 | 81 | 82 | class InvalidShardIterator(BloopException, ValueError): 83 | """This is not a valid shard iterator.""" 84 | 85 | 86 | class InvalidCondition(BloopException, ValueError): 87 | """This is not a valid Condition.""" 88 | 89 | 90 | class InvalidPosition(BloopException, ValueError): 91 | """This is not a valid position for a Stream.""" 92 | -------------------------------------------------------------------------------- /bloop/ext/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numberoverzero/bloop/a875f268d876ed0834876ffa81b5065eab6d972b/bloop/ext/__init__.py -------------------------------------------------------------------------------- /bloop/ext/arrow.py: -------------------------------------------------------------------------------- 1 | import arrow 2 | 3 | from .. import types 4 | 5 | 6 | DEFAULT_TIMEZONE = "utc" 7 | 8 | 9 | class DateTime(types.DateTime): 10 | python_type = arrow.Arrow 11 | 12 | def __init__(self, timezone=DEFAULT_TIMEZONE): 13 | self.timezone = timezone 14 | super().__init__() 15 | 16 | def dynamo_dump(self, value, *, context, **kwargs): 17 | if value is None: 18 | return None 19 | value = value.to("utc").datetime 20 | return super().dynamo_dump(value, context=context, **kwargs) 21 | 22 | def dynamo_load(self, value, *, context, **kwargs): 23 | if value is None: 24 | return None 25 | dt = super().dynamo_load(value, context=context, **kwargs) 26 | return arrow.get(dt).to(self.timezone) 27 | 28 | 29 | class Timestamp(types.Timestamp): 30 | python_type = arrow.Arrow 31 | 32 | def __init__(self, timezone=DEFAULT_TIMEZONE): 33 | self.timezone = timezone 34 | super().__init__() 35 | 36 | def dynamo_dump(self, value, *, context, **kwargs): 37 | if value is None: 38 | return None 39 | value = value.to("utc").datetime 40 | return super().dynamo_dump(value, context=context, **kwargs) 41 | 42 | def dynamo_load(self, value, *, context, **kwargs): 43 | if value is None: 44 | return None 45 | dt = super().dynamo_load(value, context=context, **kwargs) 46 | return arrow.get(dt).to(self.timezone) 47 | -------------------------------------------------------------------------------- /bloop/ext/delorean.py: -------------------------------------------------------------------------------- 1 | import delorean 2 | 3 | from .. import types 4 | 5 | 6 | # https://github.com/myusuf3/delorean/issues/97 7 | DEFAULT_TIMEZONE = "utc" 8 | 9 | 10 | class DateTime(types.DateTime): 11 | python_type = delorean.Delorean 12 | 13 | def __init__(self, timezone=DEFAULT_TIMEZONE): 14 | self.timezone = timezone 15 | super().__init__() 16 | 17 | def dynamo_dump(self, value, *, context, **kwargs): 18 | if value is None: 19 | return None 20 | value = value.shift("utc").datetime 21 | return super().dynamo_dump(value, context=context, **kwargs) 22 | 23 | def dynamo_load(self, value, *, context, **kwargs): 24 | if value is None: 25 | return None 26 | dt = super().dynamo_load(value, context=context, **kwargs) 27 | return delorean.Delorean(dt).shift(self.timezone) 28 | 29 | 30 | class Timestamp(types.Timestamp): 31 | python_type = delorean.Delorean 32 | 33 | def __init__(self, timezone=DEFAULT_TIMEZONE): 34 | self.timezone = timezone 35 | super().__init__() 36 | 37 | def dynamo_dump(self, value, *, context, **kwargs): 38 | if value is None: 39 | return None 40 | value = value.shift("utc").datetime 41 | return super().dynamo_dump(value, context=context, **kwargs) 42 | 43 | def dynamo_load(self, value, *, context, **kwargs): 44 | if value is None: 45 | return None 46 | dt = super().dynamo_load(value, context=context, **kwargs) 47 | return delorean.Delorean(dt).shift(self.timezone) 48 | -------------------------------------------------------------------------------- /bloop/ext/pendulum.py: -------------------------------------------------------------------------------- 1 | import pendulum 2 | 3 | from .. import types 4 | 5 | 6 | # https://github.com/sdispater/pendulum/issues/97 7 | DEFAULT_TIMEZONE = "utc" 8 | 9 | 10 | class DateTime(types.DateTime): 11 | python_type = pendulum.DateTime 12 | 13 | def __init__(self, timezone=DEFAULT_TIMEZONE): 14 | self.timezone = timezone 15 | super().__init__() 16 | 17 | def dynamo_dump(self, value, *, context, **kwargs): 18 | if value is None: 19 | return None 20 | value = value.in_timezone("utc") 21 | return super().dynamo_dump(value, context=context, **kwargs) 22 | 23 | def dynamo_load(self, value, *, context, **kwargs): 24 | if value is None: 25 | return None 26 | dt = super().dynamo_load(value, context=context, **kwargs) 27 | return pendulum.instance(dt).in_timezone(self.timezone) 28 | 29 | 30 | class Timestamp(types.Timestamp): 31 | python_type = pendulum.DateTime 32 | 33 | def __init__(self, timezone=DEFAULT_TIMEZONE): 34 | self.timezone = timezone 35 | super().__init__() 36 | 37 | def dynamo_dump(self, value, *, context, **kwargs): 38 | if value is None: 39 | return None 40 | value = value.in_timezone("utc") 41 | return super().dynamo_dump(value, context=context, **kwargs) 42 | 43 | def dynamo_load(self, value, *, context, **kwargs): 44 | if value is None: 45 | return None 46 | dt = super().dynamo_load(value, context=context, **kwargs) 47 | return pendulum.instance(dt).in_timezone(self.timezone) 48 | -------------------------------------------------------------------------------- /bloop/signals.py: -------------------------------------------------------------------------------- 1 | import blinker 2 | 3 | 4 | __all__ = [ 5 | "before_create_table", 6 | "object_deleted", 7 | "object_loaded", 8 | "object_modified", 9 | "object_saved", 10 | "model_bound", 11 | "model_created", 12 | "model_validated", 13 | ] 14 | 15 | # Isolate to avoid collisions with other modules. 16 | # Don't expose the namespace. 17 | __signals = blinker.Namespace() 18 | signal = __signals.signal 19 | 20 | 21 | before_create_table = signal("before_create_table") 22 | before_create_table.__doc__ = """Sent by ``engine`` before a model's backing table is created. 23 | 24 | .. code-block:: python 25 | 26 | # Nonce table names to avoid testing collisions 27 | @before_create_table.connect 28 | def apply_table_nonce(_, model, **__): 29 | nonce = datetime.now().isoformat() 30 | model.Meta.table_name += "-test-{}".format(nonce) 31 | 32 | :param engine: :class:`~bloop.engine.Engine` creating the model's table. 33 | :param model: The :class:`~bloop.models.BaseModel` class to create a table for. 34 | """ 35 | 36 | object_loaded = signal("object_loaded") 37 | object_loaded.__doc__ = """Sent by ``engine`` after an object is loaded from DynamoDB. 38 | 39 | 40 | .. code-block:: python 41 | 42 | # Track objects "checked out" locally 43 | local_objects = {} 44 | 45 | def key(obj): 46 | meta = obj.Meta 47 | return (getattr(obj, k.name) for k in meta.keys) 48 | 49 | @object_loaded.connect 50 | def on_loaded(_, obj, **__): 51 | local_objects[key(obj)] = obj 52 | 53 | :param engine: The :class:`~bloop.engine.Engine` that loaded the object. 54 | :param obj: The :class:`~bloop.models.BaseModel` loaded from DynamoDB. 55 | """ 56 | 57 | object_saved = signal("object_saved") 58 | object_saved.__doc__ = """Sent by ``engine`` after an object is saved to DynamoDB. 59 | 60 | .. code-block:: python 61 | 62 | # Track objects "checked out" locally 63 | local_objects = {} 64 | 65 | def key(obj): 66 | meta = obj.Meta 67 | return (getattr(obj, k.name) for k in meta.keys) 68 | 69 | @object_saved.connect 70 | def on_saved(_, obj, **__): 71 | local_objects.pop(key(obj)) 72 | 73 | :param engine: The :class:`~bloop.engine.Engine` that saved the object. 74 | :param obj: The :class:`~bloop.models.BaseModel` saved to DynamoDB. 75 | """ 76 | 77 | object_deleted = signal("object_deleted") 78 | object_deleted.__doc__ = """Sent by ``engine`` after an object is deleted from DynamoDB. 79 | 80 | .. code-block:: python 81 | 82 | # Track objects "checked out" locally 83 | local_objects = {} 84 | 85 | def key(obj): 86 | meta = obj.Meta 87 | return (getattr(obj, k.name) for k in meta.keys) 88 | 89 | @object_deleted.connect 90 | def on_deleted(_, obj, **__): 91 | local_objects.pop(key(obj)) 92 | 93 | :param engine: The :class:`~bloop.engine.Engine` that deleted the object. 94 | :param obj: The :class:`~bloop.models.BaseModel` deleted from DynamoDB. 95 | """ 96 | 97 | object_modified = signal("object_modified") 98 | object_modified.__doc__ = """Sent by ``column`` after an object's attribute is set or deleted. 99 | 100 | This is sent on ``__set__`` if an exception isn't raised, 101 | and on ``__del__`` regardless of exceptions. 102 | 103 | .. code-block:: python 104 | 105 | # Account balance can't be less than 0 106 | 107 | @object_modified.connect 108 | def enforce_positive_balance(_, obj, column, value, **__): 109 | if column is Account.balance and value < 0: 110 | # Danger: careful around infinite loops! 111 | setattr(obj, column.name, 0) 112 | 113 | :param column: The :class:`~bloop.models.Column` that corresponds to the modified attribute. 114 | :param obj: The :class:`~bloop.models.BaseModel` that was modified. 115 | :param value: The new value of the attribute. 116 | """ 117 | 118 | 119 | model_bound = signal("model_bound") 120 | model_bound.__doc__ = """Sent by ``engine`` after a model is bound to that :class:`~bloop.engine.Engine`. 121 | 122 | This signal is sent after :data:`~bloop.signals.model_validated`. 123 | 124 | :param engine: The :class:`~bloop.engine.Engine` that the model was bound to. 125 | :param model: The :class:`~bloop.models.BaseModel` class that was bound. 126 | """ 127 | 128 | 129 | model_created = signal("model_created") 130 | model_created.__doc__ = """Sent by ``None`` after a new model is defined. 131 | 132 | While this signal is sent when the :class:`~bloop.models.BaseModel` is created, the BaseModel is created so 133 | early in Bloop's import order that no handlers will be connected when it occurs. 134 | 135 | You can manually send the BaseModel through your handler with: 136 | 137 | .. code-block:: python 138 | 139 | model_created.send(model=BaseModel) 140 | 141 | :param model: The subclass of :class:`~bloop.models.BaseModel` that was created. 142 | """ 143 | 144 | model_validated = signal("model_validated") 145 | model_validated.__doc__ = """Sent by ``engine`` after a model is validated. 146 | 147 | This signal is sent before :data:`~bloop.signals.model_bound`. 148 | 149 | :param engine: The :class:`~bloop.engine.Engine` that validated the model. 150 | :param model: The :class:`~bloop.models.BaseModel` class that was validated. 151 | """ 152 | -------------------------------------------------------------------------------- /bloop/stream/__init__.py: -------------------------------------------------------------------------------- 1 | from .stream import Stream 2 | 3 | 4 | __all__ = ["Stream"] 5 | -------------------------------------------------------------------------------- /bloop/stream/buffer.py: -------------------------------------------------------------------------------- 1 | import heapq 2 | 3 | 4 | def heap_item(clock, record, shard): 5 | """Create a tuple of (ordering, (record, shard)) for use in a RecordBuffer.""" 6 | # Primary ordering is by event creation time. 7 | # However, creation time is *approximate* and has whole-second resolution. 8 | # This means two events in the same shard within one second can't be ordered. 9 | ordering = record["meta"]["created_at"] 10 | # From testing, SequenceNumber isn't a guaranteed ordering either. However, 11 | # it is guaranteed to be unique within a shard. This will be tie-breaker 12 | # for multiple records within the same shard, within the same second. 13 | second_ordering = int(record["meta"]["sequence_number"]) 14 | # It's possible though unlikely, that sequence numbers will collide across 15 | # multiple shards, within the same second. The final tie-breaker is 16 | # a monotonically increasing integer from the buffer. 17 | total_ordering = (ordering, second_ordering, clock()) 18 | return total_ordering, record, shard 19 | 20 | 21 | class RecordBuffer: 22 | """Maintains a total ordering for records across any number of shards. 23 | 24 | Methods are thin wrappers around :mod:`heapq`. Buffer entries have the form: 25 | 26 | .. code-block: python 27 | 28 | (total_ordering, record, shard) 29 | 30 | where ``total_ordering`` is a tuple of ``(created_at, sequence_number, monotonic_clock)`` created from each 31 | record as it is inserted. 32 | """ 33 | def __init__(self): 34 | self.heap = [] 35 | 36 | # Used by the total ordering clock 37 | self.__monotonic_integer = 0 38 | 39 | def push(self, record, shard): 40 | """Push a new record into the buffer 41 | 42 | :param dict record: new record 43 | :param shard: Shard the record came from 44 | :type shard: :class:`~bloop.stream.shard.Shard` 45 | """ 46 | heapq.heappush(self.heap, heap_item(self.clock, record, shard)) 47 | 48 | def push_all(self, record_shard_pairs): 49 | """Push multiple (record, shard) pairs at once, with only one :meth:`heapq.heapify` call to maintain order. 50 | 51 | :param record_shard_pairs: list of ``(record, shard)`` tuples 52 | (see :func:`~bloop.stream.buffer.RecordBuffer.push`). 53 | """ 54 | # Faster than inserting one at a time; the heap is sorted once after all inserts. 55 | for record, shard in record_shard_pairs: 56 | item = heap_item(self.clock, record, shard) 57 | self.heap.append(item) 58 | heapq.heapify(self.heap) 59 | 60 | def pop(self): 61 | """Pop the oldest (lowest total ordering) record and the shard it came from. 62 | 63 | :return: Oldest ``(record, shard)`` tuple. 64 | """ 65 | return heapq.heappop(self.heap)[1:] 66 | 67 | def peek(self): 68 | """A :func:`~bloop.stream.buffer.RecordBuffer.pop` without removing the (record, shard) from the buffer. 69 | 70 | :return: Oldest ``(record, shard)`` tuple. 71 | """ 72 | return self.heap[0][1:] 73 | 74 | def clear(self): 75 | """Drop the entire buffer.""" 76 | self.heap.clear() 77 | 78 | def __len__(self): 79 | return len(self.heap) 80 | 81 | def clock(self): 82 | """Returns a monotonically increasing integer. 83 | 84 | **Do not rely on the clock using a fixed increment.** 85 | 86 | .. code-block:: python 87 | 88 | >>> buffer = RecordBuffer() 89 | >>> buffer.clock() 90 | 3 91 | >>> buffer.clock() 92 | 40 93 | >>> buffer.clock() 94 | 41 95 | >>> buffer.clock() 96 | 300 97 | 98 | :return: A unique clock value guaranteed to be larger than every previous value 99 | :rtype: int 100 | """ 101 | # Try to prevent collisions from someone accessing the underlying int. 102 | # This offset ensures _RecordBuffer__monotonic_integer will never have 103 | # the same value as any call to clock(). 104 | value = self.__monotonic_integer + 1 105 | self.__monotonic_integer += 2 106 | return value 107 | -------------------------------------------------------------------------------- /bloop/stream/coordinator.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import collections.abc 3 | import datetime 4 | import logging 5 | from typing import Dict, List 6 | 7 | from ..exceptions import InvalidPosition, InvalidStream, RecordsExpired 8 | from .buffer import RecordBuffer 9 | from .shard import Shard, unpack_shards 10 | 11 | 12 | logger = logging.getLogger("bloop.stream") 13 | 14 | 15 | class Coordinator: 16 | """Encapsulates the shard-level management for a whole Stream. 17 | 18 | :param session: Used to make DynamoDBStreams calls. 19 | :type session: :class:`~bloop.session.SessionWrapper` 20 | :param str stream_arn: Stream arn, usually from the model's ``Meta.stream["arn"]``. 21 | """ 22 | def __init__(self, *, session, stream_arn): 23 | 24 | self.session = session 25 | 26 | # The stream that's being coordinated 27 | self.stream_arn = stream_arn 28 | 29 | # The oldest shards in each shard tree (no parents) 30 | self.roots = [] 31 | 32 | # Shards being iterated right now 33 | self.active: List[Shard] = [] 34 | 35 | # Closed shards that still have buffered records 36 | # shard -> buffered count 37 | self.closed: Dict[Shard, int] = {} 38 | 39 | # Single buffer for the lifetime of the Coordinator, but mutates frequently 40 | # Records in the buffer aren't considered read. When a Record popped from the buffer is 41 | # consumed, the Coordinator MUST notify the Shard by updating the sequence_number and iterator_type. 42 | # The new values should be: 43 | # shard.sequence_number = record["meta"]["sequence_number"] 44 | # shard.iterator_type = "after_record" 45 | 46 | # Holds records from advancing all active shard iterators. 47 | # Shards aren't advanced again until the buffer drains completely. 48 | self.buffer = RecordBuffer() 49 | 50 | def __repr__(self): 51 | # 52 | return "<{}[{}]>".format(self.__class__.__name__, self.stream_arn) 53 | 54 | def __iter__(self): 55 | return self 56 | 57 | def __next__(self): 58 | if not self.buffer: 59 | self.advance_shards() 60 | 61 | if self.buffer: 62 | record, shard = self.buffer.pop() 63 | 64 | # Now that the record is "consumed", advance the shard's checkpoint 65 | shard.sequence_number = record["meta"]["sequence_number"] 66 | shard.iterator_type = "after_sequence" 67 | 68 | if shard in self.closed: 69 | self.closed[shard] -= 1 70 | if self.closed[shard] == 0: 71 | self.closed.pop(shard) 72 | return record 73 | 74 | # No records :( 75 | return None 76 | 77 | def advance_shards(self): 78 | """Poll active shards for records and insert them into the buffer. Rotate exhausted shards. 79 | 80 | Returns immediately if the buffer isn't empty. 81 | """ 82 | # Don't poll shards when there are pending records. 83 | if self.buffer: 84 | return 85 | 86 | # 0) Collect new records from all active shards. 87 | record_shard_pairs = [] 88 | for shard in self.active: 89 | records = next(shard) 90 | if records: 91 | record_shard_pairs.extend((record, shard) for record in records) 92 | self.buffer.push_all(record_shard_pairs) 93 | 94 | self.migrate_closed_shards() 95 | 96 | def heartbeat(self): 97 | """Keep active shards with "trim_horizon", "latest" iterators alive by advancing their iterators.""" 98 | for shard in self.active: 99 | if shard.sequence_number is None: 100 | records = next(shard) 101 | # Success! This shard now has an ``at_sequence`` iterator 102 | if records: 103 | self.buffer.push_all((record, shard) for record in records) 104 | self.migrate_closed_shards() 105 | 106 | def migrate_closed_shards(self): 107 | # 1) Clean up exhausted Shards. Can't modify the active list while iterating it. 108 | to_migrate = {shard for shard in self.active if shard.exhausted} 109 | 110 | # Early exit since this method will iterate the entire buffer. 111 | if not to_migrate: 112 | return 113 | 114 | # Build the count once, rather than look for each shard as we process it below. 115 | buffered_count = {} 116 | for *_, shard in self.buffer.heap: 117 | buffered_count.setdefault(shard, 0) 118 | buffered_count[shard] += 1 119 | 120 | for shard in to_migrate: 121 | shard.load_children() 122 | # This call also promotes children to the shard's previous roles 123 | self.remove_shard(shard) 124 | for child in shard.children: 125 | child.jump_to(iterator_type="trim_horizon") 126 | # May still need to track this shard 127 | if shard in buffered_count: 128 | self.closed[shard] = buffered_count[shard] 129 | 130 | @property 131 | def token(self): 132 | """JSON-serializable representation of the current Stream state. 133 | 134 | Use :func:`Engine.stream(YourModel, token) ` to create an identical stream, 135 | or :func:`stream.move_to(token) ` to move an existing stream to this position. 136 | 137 | :returns: Stream state as a json-friendly dict 138 | :rtype: dict 139 | """ 140 | # 0) Trace roots and active shards 141 | active_ids = [] 142 | shard_tokens = [] 143 | for root in self.roots: 144 | for shard in root.walk_tree(): 145 | shard_tokens.append(shard.token) 146 | # dedupe, stream_arn will be in the root token 147 | shard_tokens[-1].pop("stream_arn") 148 | active_ids.extend((shard.shard_id for shard in self.active)) 149 | 150 | # 1) Inject closed shards 151 | for shard in self.closed.keys(): 152 | active_ids.append(shard.shard_id) 153 | shard_tokens.append(shard.token) 154 | shard_tokens[-1].pop("stream_arn") 155 | 156 | return { 157 | "stream_arn": self.stream_arn, 158 | "active": active_ids, 159 | "shards": shard_tokens 160 | } 161 | 162 | def remove_shard(self, shard, drop_buffered_records=False): 163 | """Remove a Shard from the Coordinator. Drops all buffered records from the Shard. 164 | 165 | If the Shard is active or a root, it is removed and any children promoted to those roles. 166 | 167 | :param shard: The shard to remove 168 | :type shard: :class:`~bloop.stream.shard.Shard` 169 | :param bool drop_buffered_records: 170 | Whether records from this shard should be removed. 171 | Default is False. 172 | """ 173 | try: 174 | self.roots.remove(shard) 175 | except ValueError: 176 | # Wasn't a root Shard 177 | pass 178 | else: 179 | self.roots.extend(shard.children) 180 | 181 | try: 182 | self.active.remove(shard) 183 | except ValueError: 184 | # Wasn't an active Shard 185 | pass 186 | else: 187 | self.active.extend(shard.children) 188 | 189 | if drop_buffered_records: 190 | # TODO can this be improved? Gets expensive for high-volume streams with large buffers 191 | heap = self.buffer.heap 192 | # Clear buffered records from the shard. Each record is (ordering, record, shard) 193 | to_remove = [x for x in heap if x[2] is shard] 194 | for x in to_remove: 195 | heap.remove(x) 196 | 197 | def move_to(self, position): 198 | """Set the Coordinator to a specific endpoint or time, or load state from a token. 199 | 200 | :param position: "trim_horizon", "latest", :class:`~datetime.datetime`, or a 201 | :attr:`Coordinator.token ` 202 | """ 203 | if isinstance(position, collections.abc.Mapping): 204 | move = _move_stream_token 205 | elif hasattr(position, "timestamp") and callable(position.timestamp): 206 | move = _move_stream_time 207 | elif isinstance(position, str) and position.lower() in ["latest", "trim_horizon"]: 208 | move = _move_stream_endpoint 209 | else: 210 | raise InvalidPosition("Don't know how to move to position {!r}".format(position)) 211 | move(self, position) 212 | 213 | 214 | def _move_stream_endpoint(coordinator, position): 215 | """Move to the "trim_horizon" or "latest" of the entire stream.""" 216 | # 0) Everything will be rebuilt from DescribeStream. 217 | stream_arn = coordinator.stream_arn 218 | coordinator.roots.clear() 219 | coordinator.active.clear() 220 | coordinator.buffer.clear() 221 | 222 | # 1) Build a Dict[str, Shard] of the current Stream from a DescribeStream call 223 | current_shards = coordinator.session.describe_stream(stream_arn=stream_arn)["Shards"] 224 | current_shards = unpack_shards(current_shards, stream_arn, coordinator.session) 225 | 226 | # 2) Roots are any shards without parents. 227 | coordinator.roots.extend(shard for shard in current_shards.values() if not shard.parent) 228 | 229 | # 3.0) Stream trim_horizon is the combined trim_horizon of all roots. 230 | if position == "trim_horizon": 231 | for shard in coordinator.roots: 232 | shard.jump_to(iterator_type="trim_horizon") 233 | coordinator.active.extend(coordinator.roots) 234 | # 3.1) Stream latest is the combined latest of all shards without children. 235 | else: 236 | for root in coordinator.roots: 237 | for shard in root.walk_tree(): 238 | if not shard.children: 239 | shard.jump_to(iterator_type="latest") 240 | coordinator.active.append(shard) 241 | 242 | 243 | def _move_stream_time(coordinator, time): 244 | """Scan through the *entire* Stream for the first record after ``time``. 245 | 246 | This is an extremely expensive, naive algorithm that starts at trim_horizon and simply 247 | dumps records into the void until the first hit. General improvements in performance are 248 | tough; we can use the fact that Shards have a max life of 24hr to pick a pretty-good starting 249 | point for any Shard trees with 6 generations. Even then we can't know how close the oldest one 250 | is to rolling off so we either hit trim_horizon, or iterate an extra Shard more than we need to. 251 | 252 | The corner cases are worse; short trees, recent splits, trees with different branch heights. 253 | """ 254 | if time > datetime.datetime.now(datetime.timezone.utc): 255 | _move_stream_endpoint(coordinator, "latest") 256 | return 257 | 258 | _move_stream_endpoint(coordinator, "trim_horizon") 259 | shard_trees = collections.deque(coordinator.roots) 260 | while shard_trees: 261 | shard = shard_trees.popleft() 262 | records = shard.seek_to(time) 263 | 264 | # Success! This section of some Shard tree is at the desired time. 265 | if records: 266 | coordinator.buffer.push_all((record, shard) for record in records) 267 | 268 | # Closed shard, keep searching its children. 269 | elif shard.exhausted: 270 | coordinator.remove_shard(shard, drop_buffered_records=True) 271 | shard_trees.extend(shard.children) 272 | 273 | 274 | def _move_stream_token(coordinator, token): 275 | """Move to the Stream position described by the token. 276 | 277 | The following rules are applied when interpolation is required: 278 | - If a shard does not exist (past the trim_horizon) it is ignored. If that 279 | shard had children, its children are also checked against the existing shards. 280 | - If none of the shards in the token exist, then InvalidStream is raised. 281 | - If a Shard expects its iterator to point to a SequenceNumber that is now past 282 | that Shard's trim_horizon, the Shard instead points to trim_horizon. 283 | """ 284 | stream_arn = coordinator.stream_arn = token["stream_arn"] 285 | # 0) Everything will be rebuilt from the DescribeStream masked by the token. 286 | coordinator.roots.clear() 287 | coordinator.active.clear() 288 | coordinator.closed.clear() 289 | coordinator.buffer.clear() 290 | 291 | # Injecting the token gives us access to the standard shard management functions 292 | token_shards = unpack_shards(token["shards"], stream_arn, coordinator.session) 293 | coordinator.roots = [shard for shard in token_shards.values() if not shard.parent] 294 | coordinator.active.extend(token_shards[shard_id] for shard_id in token["active"]) 295 | 296 | # 1) Build a Dict[str, Shard] of the current Stream from a DescribeStream call 297 | current_shards = coordinator.session.describe_stream(stream_arn=stream_arn)["Shards"] 298 | current_shards = unpack_shards(current_shards, stream_arn, coordinator.session) 299 | 300 | # 2) Trying to find an intersection with the actual Stream by walking each root shard's tree. 301 | # Prune any Shard with no children that's not part of the actual Stream. 302 | # Raise InvalidStream if the entire token is pruned. 303 | unverified = collections.deque(coordinator.roots) 304 | while unverified: 305 | shard = unverified.popleft() 306 | if shard.shard_id not in current_shards: 307 | logger.info("Unknown or expired shard \"{}\" - pruning from stream token".format(shard.shard_id)) 308 | coordinator.remove_shard(shard, drop_buffered_records=True) 309 | unverified.extend(shard.children) 310 | 311 | # 3) Everything was pruned, so the token describes an unknown stream. 312 | if not coordinator.roots: 313 | raise InvalidStream("This token has no relation to the actual Stream.") 314 | 315 | # 4) Now that everything's verified, grab new iterators for the coordinator's active Shards. 316 | for shard in coordinator.active: 317 | try: 318 | if shard.iterator_type is None: 319 | # Descendant of an unknown shard 320 | shard.iterator_type = "trim_horizon" 321 | # Move back to the token's specified position 322 | shard.jump_to(iterator_type=shard.iterator_type, sequence_number=shard.sequence_number) 323 | except RecordsExpired: 324 | # This token shard's sequence_number is beyond the trim_horizon. 325 | # The next closest record is at trim_horizon. 326 | msg = "SequenceNumber \"{}\" in shard \"{}\" beyond trim horizon: jumping to trim_horizon" 327 | logger.info(msg.format(shard.sequence_number, shard.shard_id)) 328 | shard.jump_to(iterator_type="trim_horizon") 329 | -------------------------------------------------------------------------------- /bloop/stream/stream.py: -------------------------------------------------------------------------------- 1 | from ..models import unpack_from_dynamodb 2 | from ..signals import object_loaded 3 | from .coordinator import Coordinator 4 | 5 | 6 | class Stream: 7 | """Iterator over all records in a stream. 8 | 9 | :param model: The model to stream records from. 10 | :param engine: The engine to load model objects through. 11 | :type engine: :class:`~bloop.engine.Engine` 12 | """ 13 | def __init__(self, *, model, engine): 14 | 15 | self.model = model 16 | self.engine = engine 17 | self.coordinator = Coordinator( 18 | session=engine.session, 19 | stream_arn=model.Meta.stream["arn"]) 20 | 21 | def __repr__(self): 22 | # 23 | return "<{}[{}]>".format(self.__class__.__name__, self.model.__name__) 24 | 25 | def __iter__(self): 26 | return self 27 | 28 | def __next__(self): 29 | record = next(self.coordinator) 30 | if record: 31 | meta = self.model.Meta 32 | for key, expected in [("new", meta.columns), ("old", meta.columns), ("key", meta.keys)]: 33 | if key not in meta.stream["include"]: 34 | record[key] = None 35 | else: 36 | self._unpack(record, key, expected) 37 | return record 38 | 39 | def heartbeat(self): 40 | """Refresh iterators without sequence numbers so they don't expire. 41 | 42 | Call this at least every 14 minutes. 43 | """ 44 | self.coordinator.heartbeat() 45 | 46 | def move_to(self, position): 47 | """Move the Stream to a specific endpoint or time, or load state from a token. 48 | 49 | Moving to an endpoint with "trim_horizon" or "latest" and loading from a previous token are both 50 | very efficient. 51 | 52 | In contrast, seeking to a specific time requires iterating **all records in the stream up to that time**. 53 | This can be **very expensive**. Once you have moved a stream to a time, you should save the 54 | :attr:`Stream.token ` so reloading will be extremely fast. 55 | 56 | :param position: "trim_horizon", "latest", :class:`~datetime.datetime`, or a 57 | :attr:`Stream.token ` 58 | """ 59 | self.coordinator.move_to(position) 60 | 61 | @property 62 | def token(self): 63 | """JSON-serializable representation of the current Stream state. 64 | 65 | Use :func:`Engine.stream(YourModel, token) ` to create an identical stream, 66 | or :func:`stream.move_to(token) ` to move an existing stream to this position. 67 | 68 | :returns: Stream state as a json-friendly dict 69 | :rtype: dict 70 | """ 71 | return self.coordinator.token 72 | 73 | def _unpack(self, record, key, expected): 74 | """Replaces the attr dict at the given key with an instance of a Model""" 75 | attrs = record.get(key) 76 | if attrs is None: 77 | return 78 | obj = unpack_from_dynamodb( 79 | attrs=attrs, 80 | expected=expected, 81 | model=self.model, 82 | engine=self.engine 83 | ) 84 | object_loaded.send(self.engine, engine=self.engine, obj=obj) 85 | record[key] = obj 86 | -------------------------------------------------------------------------------- /bloop/transactions.py: -------------------------------------------------------------------------------- 1 | import enum 2 | import logging 3 | import uuid 4 | from datetime import datetime, timedelta, timezone 5 | from typing import Any, List, NamedTuple, Optional 6 | 7 | from .conditions import render 8 | from .exceptions import MissingObjects, TransactionTokenExpired 9 | from .models import unpack_from_dynamodb 10 | from .signals import object_deleted, object_loaded, object_saved 11 | from .util import dump_key, get_table_name 12 | 13 | 14 | __all__ = [ 15 | "PreparedTransaction", 16 | "ReadTransaction", 17 | "Transaction", 18 | "TxItem", "TxType", 19 | "WriteTransaction", 20 | ] 21 | logger = logging.getLogger("bloop.transactions") 22 | 23 | MAX_TRANSACTION_ITEMS = 10 24 | # per docs this is 10 minutes, minus a bit for clock skew guard 25 | MAX_TOKEN_LIFETIME = timedelta(minutes=9, seconds=30) 26 | 27 | 28 | class TxType(enum.Enum): 29 | """Enum whose value is the wire format of its name""" 30 | Get = "Get" 31 | Check = "CheckCondition" 32 | Delete = "Delete" 33 | Update = "Update" 34 | 35 | @classmethod 36 | def by_alias(cls, name: str) -> "TxType": 37 | """get a type by the common bloop operation name: get/check/delete/save""" 38 | return { 39 | "get": TxType.Get, 40 | "check": TxType.Check, 41 | "delete": TxType.Delete, 42 | "save": TxType.Update, 43 | }[name] 44 | 45 | 46 | class TxItem(NamedTuple): 47 | """ 48 | Includes the type, an object, and its condition settings. 49 | 50 | The common way to construct an item is through the ``new`` method: 51 | 52 | .. code-block:: pycon 53 | 54 | >>> get_item = TxItem.new("get", some_obj) 55 | >>> save_item = TxItem.new("save", some_obj) 56 | """ 57 | 58 | #: How this item will be used in a transaction 59 | type: TxType 60 | 61 | #: The object that will be modified, persisted, or referenced in a transaction 62 | obj: Any 63 | 64 | #: An optional condition that constrains an update 65 | condition: Optional[Any] 66 | 67 | @classmethod 68 | def new(cls, type_alias, obj, condition=None) -> "TxItem": 69 | return TxItem(type=TxType.by_alias(type_alias), obj=obj, condition=condition) 70 | 71 | @property 72 | def is_update(self): 73 | """Whether this should render an "UpdateExpression" in the TransactItem""" 74 | return self.type is TxType.Update 75 | 76 | @property 77 | def should_render_obj(self): 78 | """Whether the object values should be rendered in the TransactItem""" 79 | return self.type not in {TxType.Check, TxType.Get} 80 | 81 | 82 | # hack to get around NamedTuple field docstrings renaming: 83 | # https://stackoverflow.com/a/39320627 84 | TxItem.type.__doc__ = """How this item will be used in a transaction""" 85 | TxItem.obj.__doc__ = """The object that will be modified, persisted, or referenced in a transaction""" 86 | TxItem.condition.__doc__ = """An optional condition that constrains an update""" 87 | 88 | 89 | class Transaction: 90 | """ 91 | Holds a collection of transaction items to be rendered into a PreparedTransaction. 92 | 93 | If used as a context manager, calls prepare() and commit() when the outermost context exits. 94 | 95 | .. code-block:: pycon 96 | 97 | >>> engine = Engine() 98 | >>> tx = Transaction(engine) 99 | >>> tx.mode = "w" 100 | >>> p1 = tx.prepare() 101 | >>> p2 = tx.prepare() # different instances 102 | 103 | >>> with tx: 104 | ... pass 105 | >>> # tx.prepare().commit() is called here 106 | """ 107 | mode: str 108 | _items: List[TxItem] 109 | 110 | def __init__(self, engine): 111 | self.engine = engine 112 | self._items = [] 113 | self._ctx_depth = 0 114 | 115 | def __enter__(self): 116 | self._ctx_depth += 1 117 | return self 118 | 119 | def __exit__(self, exc_type, exc_value, exc_tb): 120 | self._ctx_depth -= 1 121 | if exc_type: 122 | return 123 | if self._ctx_depth == 0: 124 | self.prepare().commit() 125 | 126 | def _extend(self, items): 127 | if len(self._items) + len(items) > MAX_TRANSACTION_ITEMS: 128 | raise RuntimeError(f"transaction cannot exceed {MAX_TRANSACTION_ITEMS} items.") 129 | self._items += items 130 | 131 | def prepare(self): 132 | """ 133 | Create a new PreparedTransaction that can be committed. 134 | 135 | This is called automatically when exiting the transaction as a context: 136 | 137 | .. code-block:: python 138 | 139 | >>> engine = Engine() 140 | >>> tx = WriteTransaction(engine) 141 | >>> prepared = tx.prepare() 142 | >>> prepared.commit() 143 | 144 | # automatically calls commit when exiting 145 | >>> with WriteTransaction(engine) as tx: 146 | ... # modify the transaction here 147 | ... pass 148 | >>> # tx commits here 149 | 150 | :return: 151 | """ 152 | tx = PreparedTransaction() 153 | tx.prepare( 154 | engine=self.engine, 155 | mode=self.mode, 156 | items=self._items, 157 | ) 158 | return tx 159 | 160 | 161 | class PreparedTransaction: 162 | """ 163 | Transaction that can be committed once or more. 164 | 165 | Usually created from a :class:`~bloop.transactions.Transaction` instance. 166 | """ 167 | mode: str 168 | items: List[TxItem] 169 | 170 | #: Unique id used as the "ClientRequestToken" for write transactions. This is 171 | #: generated but not sent with a read transaction, since reads are not idempotent. 172 | tx_id: str 173 | 174 | #: When the transaction was first committed at. A prepared write transaction can only call commit 175 | #: again within 10 minutes of its first commit. This is ``None`` until commit() is called at least once. 176 | first_commit_at: Optional[datetime] = None 177 | 178 | def __init__(self): 179 | self.engine = None 180 | self._request = None 181 | 182 | def prepare(self, engine, mode, items) -> None: 183 | """ 184 | Create a unique transaction id and dumps the items into a cached request object. 185 | """ 186 | self.tx_id = str(uuid.uuid4()).replace("-", "") 187 | self.engine = engine 188 | self.mode = mode 189 | self.items = items 190 | self._prepare_request() 191 | 192 | def _prepare_request(self): 193 | self._request = [ 194 | { 195 | item.type.value: { 196 | "Key": dump_key(self.engine, item.obj), 197 | "TableName": get_table_name(self.engine, item.obj), 198 | **render( 199 | self.engine, 200 | obj=item.obj if item.should_render_obj else None, 201 | condition=item.condition, 202 | update=item.is_update), 203 | } 204 | } 205 | for item in self.items 206 | ] 207 | 208 | def commit(self) -> None: 209 | """ 210 | Commit the transaction with a fixed transaction id. 211 | 212 | A read transaction can call commit() any number of times, while a write transaction can only use the 213 | same tx_id for 10 minutes from the first call. 214 | """ 215 | now = datetime.now(timezone.utc) 216 | if self.first_commit_at is None: 217 | self.first_commit_at = now 218 | 219 | if self.mode == "r": 220 | response = self.engine.session.transaction_read(self._request) 221 | elif self.mode == "w": 222 | if now - self.first_commit_at > MAX_TOKEN_LIFETIME: 223 | raise TransactionTokenExpired 224 | response = self.engine.session.transaction_write(self._request, self.tx_id) 225 | else: 226 | raise ValueError(f"unrecognized mode {self.mode}") 227 | 228 | self._handle_response(response) 229 | 230 | def _handle_response(self, response: dict) -> None: 231 | if self.mode == "w": 232 | for item in self.items: 233 | obj = item.obj 234 | if item.type is TxType.Delete: 235 | object_deleted.send(self.engine, engine=self.engine, obj=obj) 236 | elif item.type is TxType.Update: 237 | object_saved.send(self.engine, engine=self.engine, obj=obj) 238 | else: 239 | blobs = response["Responses"] 240 | not_loaded = set() 241 | if len(self.items) != len(blobs): 242 | raise RuntimeError("malformed response from DynamoDb") 243 | for item, blob in zip(self.items, blobs): 244 | obj = item.obj 245 | if not blob: 246 | not_loaded.add(obj) 247 | continue 248 | unpack_from_dynamodb(attrs=blob["Item"], expected=obj.Meta.columns, engine=self.engine, obj=obj) 249 | object_loaded.send(self.engine, engine=self.engine, obj=obj) 250 | if not_loaded: 251 | logger.info("loaded {} of {} objects".format(len(self.items) - len(not_loaded), len(self.items))) 252 | raise MissingObjects("Failed to load some objects.", objects=not_loaded) 253 | logger.info("successfully loaded {} objects".format(len(self.items))) 254 | 255 | 256 | class ReadTransaction(Transaction): 257 | """ 258 | Loads all items in the same transaction. Items can be from different models and tables. 259 | """ 260 | mode = "r" 261 | 262 | def load(self, *objs) -> "ReadTransaction": 263 | """ 264 | Add one or more objects to be loaded in this transaction. 265 | 266 | At most 10 items can be loaded in the same transaction. All objects will be loaded each time you 267 | call commit(). 268 | 269 | 270 | :param objs: Objects to add to the set that are loaded in this transaction. 271 | :return: this transaction for chaining 272 | :raises bloop.exceptions.MissingObjects: if one or more objects aren't loaded. 273 | """ 274 | self._extend([TxItem.new("get", obj) for obj in objs]) 275 | return self 276 | 277 | 278 | class WriteTransaction(Transaction): 279 | """ 280 | Applies all updates in the same transaction. Items can be from different models and tables. 281 | 282 | As with an engine, you can apply conditions to each object that you save or delete, or a condition for the entire 283 | transaction that won't modify the specified object: 284 | 285 | .. code-block:: python 286 | 287 | # condition on some_obj 288 | >>> tx.save(some_obj, condition=SomeModel.name.begins_with("foo")) 289 | # condition on the tx, based on the values of some_other_obj 290 | >>> tx.check(some_other_obj, condition=ThatModel.capacity >= 100) 291 | 292 | """ 293 | mode = "w" 294 | 295 | def check(self, obj, condition) -> "WriteTransaction": 296 | """ 297 | Add a condition which must be met for the transaction to commit. 298 | 299 | While the condition is checked against the provided object, that object will not be modified. It is only 300 | used to provide the hash and range key to apply the condition to. 301 | 302 | At most 10 items can be checked, saved, or deleted in the same transaction. The same idempotency token will 303 | be used for a single prepared transaction, which allows you to safely call commit on the PreparedCommit object 304 | multiple times. 305 | 306 | 307 | :param obj: The object to use for the transaction condition. This object will not be modified. 308 | :param condition: A condition on an object which must hold for the transaction to commit. 309 | :return: this transaction for chaining 310 | """ 311 | self._extend([TxItem.new("check", obj, condition)]) 312 | return self 313 | 314 | def save(self, *objs, condition=None) -> "WriteTransaction": 315 | """ 316 | Add one or more objects to be saved in this transaction. 317 | 318 | At most 10 items can be checked, saved, or deleted in the same transaction. The same idempotency token will 319 | be used for a single prepared transaction, which allows you to safely call commit on the PreparedCommit object 320 | multiple times. 321 | 322 | :param objs: Objects to add to the set that are updated in this transaction. 323 | :param condition: A condition for these objects which must hold for the transaction to commit. 324 | :return: this transaction for chaining 325 | """ 326 | self._extend([TxItem.new("save", obj, condition) for obj in objs]) 327 | return self 328 | 329 | def delete(self, *objs, condition=None) -> "WriteTransaction": 330 | """ 331 | Add one or more objects to be deleted in this transaction. 332 | 333 | At most 10 items can be checked, saved, or deleted in the same transaction. The same idempotency token will 334 | be used for a single prepared transaction, which allows you to safely call commit on the PreparedCommit object 335 | multiple times. 336 | 337 | :param objs: Objects to add to the set that are deleted in this transaction. 338 | :param condition: A condition for these objects which must hold for the transaction to commit. 339 | :return: this transaction for chaining 340 | """ 341 | self._extend([TxItem.new("delete", obj, condition) for obj in objs]) 342 | return self 343 | -------------------------------------------------------------------------------- /bloop/util.py: -------------------------------------------------------------------------------- 1 | import collections.abc 2 | 3 | from .actions import ActionType 4 | from .exceptions import MissingKey 5 | 6 | 7 | __all__ = [ 8 | "Sentinel", 9 | "default_context", 10 | "dump_key", "extract_key", "get_table_name", 11 | "index_for", "missing", "ordered", 12 | "value_of", "walk_subclasses", 13 | ] 14 | 15 | # De-dupe dict for Sentinel 16 | _symbols = {} 17 | 18 | 19 | def index(objects, attr): 20 | """ 21 | Generate a mapping of a list of objects indexed by the given attr. 22 | 23 | Parameters 24 | ---------- 25 | objects : :class:`list`, iterable 26 | attr : string 27 | The attribute to index the list of objects by 28 | 29 | Returns 30 | ------- 31 | dictionary : dict 32 | keys are the value of each object's attr, and values are from objects 33 | 34 | Example 35 | ------- 36 | 37 | class Person(object): 38 | def __init__(self, name, email, age): 39 | self.name = name 40 | self.email = email 41 | self.age = age 42 | 43 | people = [ 44 | Person('one', 'one@people.com', 1), 45 | Person('two', 'two@people.com', 2), 46 | Person('three', 'three@people.com', 3) 47 | ] 48 | 49 | by_email = index(people, 'email') 50 | by_name = index(people, 'name') 51 | 52 | assert by_name['one'] is people[0] 53 | assert by_email['two@people.com'] is people[1] 54 | 55 | """ 56 | return {getattr(obj, attr): obj for obj in objects} 57 | 58 | 59 | def ordered(obj): 60 | """ 61 | Return sorted version of nested dicts/lists for comparing. 62 | 63 | Modified from: 64 | http://stackoverflow.com/a/25851972 65 | """ 66 | if isinstance(obj, collections.abc.Mapping): 67 | return sorted((k, ordered(v)) for k, v in obj.items()) 68 | # Special case str since it's a collections.abc.Iterable 69 | elif isinstance(obj, str): 70 | return obj 71 | elif isinstance(obj, collections.abc.Iterable): 72 | return sorted(ordered(x) for x in obj) 73 | else: 74 | return obj 75 | 76 | 77 | def walk_subclasses(root): 78 | """Does not yield the input class""" 79 | classes = [root] 80 | visited = set() 81 | while classes: 82 | cls = classes.pop() 83 | if cls is type or cls in visited: 84 | continue 85 | classes.extend(cls.__subclasses__()) 86 | visited.add(cls) 87 | if cls is not root: 88 | yield cls 89 | 90 | 91 | def value_of(column): 92 | """value_of({'S': 'Space Invaders'}) -> 'Space Invaders'""" 93 | return next(iter(column.values())) 94 | 95 | 96 | def index_for(key): 97 | """stable hashable tuple of object keys for indexing an item in constant time. 98 | 99 | usage:: 100 | 101 | index_for({'id': {'S': 'foo'}, 'range': {'S': 'bar'}}) -> ('bar', 'foo') 102 | """ 103 | return tuple(sorted(value_of(k) for k in key.values())) 104 | 105 | 106 | def extract_key(key_shape, item): 107 | """ 108 | construct a key according to key_shape for building an index 109 | 110 | usage:: 111 | 112 | key_shape = "foo", "bar" 113 | item = {"baz": 1, "bar": 2, "foo": 3} 114 | extract_key(key_shape, item) -> {"foo": 3, "bar": 2} 115 | """ 116 | return {field: item[field] for field in key_shape} 117 | 118 | 119 | def dump_key(engine, obj): 120 | """dump the hash (and range, if there is one) key(s) of an object into 121 | a dynamo-friendly format. 122 | 123 | returns {dynamo_name: {type: value} for dynamo_name in hash/range keys} 124 | """ 125 | key = {} 126 | context = default_context(engine) 127 | for key_column in obj.Meta.keys: 128 | key_value = getattr(obj, key_column.name, missing) 129 | if key_value is missing: 130 | raise MissingKey("{!r} is missing {}: {!r}".format( 131 | obj, "hash_key" if key_column.hash_key else "range_key", 132 | key_column.name 133 | )) 134 | # noinspection PyProtectedMember 135 | key_action = key_column.typedef._dump(key_value, context=context) 136 | if key_action.type is not ActionType.Set: 137 | raise ValueError( 138 | f"key value {key_value} for column {key_column} must be a SET action but was {key_action}") 139 | key[key_column.dynamo_name] = key_action.value 140 | return key 141 | 142 | 143 | def get_table_name(engine, obj): 144 | """return the table name for an object as seen by a given engine""" 145 | # noinspection PyProtectedMember 146 | return engine._compute_table_name(obj.__class__) 147 | 148 | 149 | def default_context(engine, context=None) -> dict: 150 | """Return a dict with an engine, using the existing values if provided""" 151 | if context is None: 152 | context = {} 153 | context.setdefault("engine", engine) 154 | return context 155 | 156 | 157 | class Sentinel: 158 | """Simple string-based placeholders for missing or special values. 159 | 160 | Names are unique, and instances are re-used for the same name: 161 | 162 | .. code-block:: pycon 163 | 164 | >>> from bloop.util import Sentinel 165 | >>> empty = Sentinel("empty") 166 | >>> empty 167 | 168 | >>> same_token = Sentinel("empty") 169 | >>> empty is same_token 170 | True 171 | 172 | This removes the need to import the same signal or placeholder value everywhere; two modules can create 173 | ``Sentinel("some-value")`` and refer to the same object. This is especially helpful where ``None`` is a possible 174 | value, and so can't be used to indicate omission of an optional parameter. 175 | 176 | Implements ``__repr__`` to render nicely in function signatures. Standard object-based sentinels: 177 | 178 | .. code-block:: pycon 179 | 180 | >>> missing = object() 181 | >>> def some_func(optional=missing): 182 | ... pass 183 | ... 184 | >>> help(some_func) 185 | Help on function some_func in module __main__: 186 | 187 | some_func(optional=) 188 | 189 | With the Sentinel class: 190 | 191 | .. code-block:: pycon 192 | 193 | >>> from bloop.util import Sentinel 194 | >>> missing = Sentinel("Missing") 195 | >>> def some_func(optional=missing): 196 | ... pass 197 | ... 198 | >>> help(some_func) 199 | Help on function some_func in module __main__: 200 | 201 | some_func(optional=) 202 | 203 | :param str name: The name for this sentinel. 204 | """ 205 | def __new__(cls, name, *args, **kwargs): 206 | name = name.lower() 207 | sentinel = _symbols.get(name, None) 208 | if sentinel is None: 209 | sentinel = _symbols[name] = super().__new__(cls) 210 | return sentinel 211 | 212 | def __init__(self, name): 213 | self.name = name 214 | 215 | def __repr__(self): 216 | return "".format(self.name) 217 | 218 | 219 | missing = Sentinel("missing") 220 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " applehelp to make an Apple Help Book" 34 | @echo " devhelp to make HTML files and a Devhelp project" 35 | @echo " epub to make an epub" 36 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 37 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 38 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 39 | @echo " text to make text files" 40 | @echo " man to make manual pages" 41 | @echo " texinfo to make Texinfo files" 42 | @echo " info to make Texinfo files and run them through makeinfo" 43 | @echo " gettext to make PO message catalogs" 44 | @echo " changes to make an overview of all changed/added/deprecated items" 45 | @echo " xml to make Docutils-native XML files" 46 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 47 | @echo " linkcheck to check all external links for integrity" 48 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 49 | @echo " coverage to run coverage check of the documentation (if enabled)" 50 | 51 | clean: 52 | rm -rf $(BUILDDIR)/* 53 | 54 | html: 55 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 56 | @echo 57 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 58 | 59 | dirhtml: 60 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 61 | @echo 62 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 63 | 64 | singlehtml: 65 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 66 | @echo 67 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 68 | 69 | pickle: 70 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 71 | @echo 72 | @echo "Build finished; now you can process the pickle files." 73 | 74 | json: 75 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 76 | @echo 77 | @echo "Build finished; now you can process the JSON files." 78 | 79 | htmlhelp: 80 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 81 | @echo 82 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 83 | ".hhp project file in $(BUILDDIR)/htmlhelp." 84 | 85 | qthelp: 86 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 87 | @echo 88 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 89 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 90 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/bloop.qhcp" 91 | @echo "To view the help file:" 92 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/bloop.qhc" 93 | 94 | applehelp: 95 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 96 | @echo 97 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 98 | @echo "N.B. You won't be able to view it unless you put it in" \ 99 | "~/Library/Documentation/Help or install it in your application" \ 100 | "bundle." 101 | 102 | devhelp: 103 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 104 | @echo 105 | @echo "Build finished." 106 | @echo "To view the help file:" 107 | @echo "# mkdir -p $$HOME/.local/share/devhelp/bloop" 108 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/bloop" 109 | @echo "# devhelp" 110 | 111 | epub: 112 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 113 | @echo 114 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 115 | 116 | latex: 117 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 118 | @echo 119 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 120 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 121 | "(use \`make latexpdf' here to do that automatically)." 122 | 123 | latexpdf: 124 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 125 | @echo "Running LaTeX files through pdflatex..." 126 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 127 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 128 | 129 | latexpdfja: 130 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 131 | @echo "Running LaTeX files through platex and dvipdfmx..." 132 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 133 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 134 | 135 | text: 136 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 137 | @echo 138 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 139 | 140 | man: 141 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 142 | @echo 143 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 144 | 145 | texinfo: 146 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 147 | @echo 148 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 149 | @echo "Run \`make' in that directory to run these through makeinfo" \ 150 | "(use \`make info' here to do that automatically)." 151 | 152 | info: 153 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 154 | @echo "Running Texinfo files through makeinfo..." 155 | make -C $(BUILDDIR)/texinfo info 156 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 157 | 158 | gettext: 159 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 160 | @echo 161 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 162 | 163 | changes: 164 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 165 | @echo 166 | @echo "The overview file is in $(BUILDDIR)/changes." 167 | 168 | linkcheck: 169 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 170 | @echo 171 | @echo "Link check complete; look for any errors in the above output " \ 172 | "or in $(BUILDDIR)/linkcheck/output.txt." 173 | 174 | doctest: 175 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 176 | @echo "Testing of doctests in the sources finished, look at the " \ 177 | "results in $(BUILDDIR)/doctest/output.txt." 178 | 179 | coverage: 180 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 181 | @echo "Testing of coverage in the sources finished, look at the " \ 182 | "results in $(BUILDDIR)/coverage/python.txt." 183 | 184 | xml: 185 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 186 | @echo 187 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 188 | 189 | pseudoxml: 190 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 191 | @echo 192 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 193 | -------------------------------------------------------------------------------- /docs/_static/bloop.css: -------------------------------------------------------------------------------- 1 | .caption-text, .bloop-gh { 2 | color: #2980B9; 3 | } 4 | 5 | .wy-menu-vertical p.caption { 6 | font-size: 90%; 7 | } 8 | 9 | .versionmodified { 10 | font-style: italic 11 | } 12 | -------------------------------------------------------------------------------- /docs/_static/favicon-cog.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numberoverzero/bloop/a875f268d876ed0834876ffa81b5065eab6d972b/docs/_static/favicon-cog.ico -------------------------------------------------------------------------------- /docs/_templates/layout.html: -------------------------------------------------------------------------------- 1 | {% extends "!layout.html" %} 2 | {% block menu %} 3 | {{ super() }} 4 | 11 | {% endblock %} 12 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import pkg_resources 3 | import sphinx_rtd_theme 4 | 5 | extensions = [ 6 | 'sphinx.ext.autodoc', 7 | 'sphinx.ext.intersphinx', 8 | 'sphinx.ext.viewcode', 9 | 'sphinx.ext.napoleon' 10 | ] 11 | 12 | templates_path = ['_templates'] 13 | 14 | master_doc = 'index' 15 | 16 | project = 'bloop' 17 | copyright = '2016, Joe Cross' 18 | author = 'Joe Cross' 19 | 20 | try: 21 | release = pkg_resources.get_distribution('bloop').version 22 | except pkg_resources.DistributionNotFound: 23 | print('To build the documentation, The distribution information of bloop') 24 | print('Has to be available. Either install the package into your') 25 | print('development environment or run "setup.py develop" to setup the') 26 | print('metadata. A virtualenv is recommended!') 27 | sys.exit(1) 28 | del pkg_resources 29 | version = '.'.join(release.split('.')[:2]) 30 | 31 | language = 'en' 32 | exclude_patterns = ['_build'] 33 | 34 | pygments_style = 'sphinx' 35 | html_static_path = ["_static"] 36 | html_theme = 'sphinx_rtd_theme' 37 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 38 | html_context = { 39 | "favicon": "favicon-cog.ico", 40 | "show_sphinx": False 41 | } 42 | 43 | intersphinx_mapping = { 44 | 'python': ('https://docs.python.org/3.6', None), 45 | 'arrow': ('https://arrow.readthedocs.io/en/latest/', None), 46 | 'PIL': ('https://pillow.readthedocs.io/en/latest', None), 47 | 'blinker': ('https://blinker.readthedocs.io/en/stable', None), 48 | 'boto3': ('https://boto3.amazonaws.com/v1/documentation/api/latest', None), 49 | 'delorean': ('https://delorean.readthedocs.io/en/latest/', None) 50 | } 51 | 52 | 53 | def setup(app): 54 | app.add_css_file("bloop.css") 55 | -------------------------------------------------------------------------------- /docs/docutils.conf: -------------------------------------------------------------------------------- 1 | [parsers] 2 | smart_quotes: false 3 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Bloop: DynamoDB Modeling 2 | ^^^^^^^^^^^^^^^^^^^^^^^^ 3 | 4 | DynamoDB's concurrency model is great, but using it correctly is tedious and unforgiving. 5 | `Bloop manages that complexity for you.`__ 6 | 7 | Requires Python 3.6+ 8 | 9 | __ https://gist.github.com/numberoverzero/9584cfc375de0e087c8e1ae35ab8559c 10 | 11 | ========== 12 | Features 13 | ========== 14 | 15 | * Simple declarative modeling 16 | * Stream interface that makes sense 17 | * Easy transactions 18 | * Extensible type system, useful built-in types 19 | * Secure expression-based wire format 20 | * Expressive conditions 21 | * Model composition 22 | * Diff-based saves 23 | * Server-Side Encryption 24 | * Time-To-Live 25 | * Continuous Backups 26 | * On-Demand Billing 27 | 28 | ============ 29 | Ergonomics 30 | ============ 31 | 32 | The basics: 33 | 34 | .. code-block:: python 35 | 36 | class Account(BaseModel): 37 | id = Column(UUID, hash_key=True) 38 | name = Column(String) 39 | email = Column(String) 40 | by_email = GlobalSecondaryIndex( 41 | projection='keys', hash_key='email') 42 | 43 | engine.bind(Account) 44 | 45 | some_account = Account(id=uuid.uuid4(), email='foo@bar.com') 46 | engine.save(some_account) 47 | 48 | q = engine.query(Account.by_email, key=Account.email == 'foo@bar.com') 49 | same_account = q.one() 50 | 51 | print(same_account.id) 52 | 53 | Iterate over a stream: 54 | 55 | .. code-block:: python 56 | 57 | template = "old: {old}\nnew: {new}\ndetails:{meta}" 58 | 59 | stream = engine.stream(User, 'trim_horizon') 60 | while True: 61 | record = next(stream) 62 | if not record: 63 | time.sleep(0.5) 64 | continue 65 | print(template.format(**record)) 66 | 67 | Use transactions: 68 | 69 | .. code-block:: python 70 | 71 | with engine.transaction() as tx: 72 | tx.save(account) 73 | tx.delete(update_token, condition=Token.until <= now()) 74 | 75 | ============= 76 | What's Next 77 | ============= 78 | 79 | Get started by :ref:`installing ` Bloop, or check out a :ref:`larger example `. 80 | 81 | .. toctree:: 82 | :maxdepth: 2 83 | :caption: User Guide 84 | :hidden: 85 | 86 | user/install 87 | user/quickstart 88 | user/models 89 | user/engine 90 | user/transactions 91 | user/streams 92 | user/types 93 | user/conditions 94 | user/signals 95 | user/patterns 96 | user/extensions 97 | 98 | .. toctree:: 99 | :maxdepth: 2 100 | :caption: API 101 | :hidden: 102 | 103 | api/public 104 | api/internal 105 | 106 | .. toctree:: 107 | :maxdepth: 2 108 | :caption: Project 109 | :hidden: 110 | 111 | meta/changelog 112 | meta/about 113 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | echo. coverage to run coverage check of the documentation if enabled 41 | goto end 42 | ) 43 | 44 | if "%1" == "clean" ( 45 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 46 | del /q /s %BUILDDIR%\* 47 | goto end 48 | ) 49 | 50 | 51 | REM Check if sphinx-build is available and fallback to Python version if any 52 | %SPHINXBUILD% 2> nul 53 | if errorlevel 9009 goto sphinx_python 54 | goto sphinx_ok 55 | 56 | :sphinx_python 57 | 58 | set SPHINXBUILD=python -m sphinx.__init__ 59 | %SPHINXBUILD% 2> nul 60 | if errorlevel 9009 ( 61 | echo. 62 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 63 | echo.installed, then set the SPHINXBUILD environment variable to point 64 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 65 | echo.may add the Sphinx directory to PATH. 66 | echo. 67 | echo.If you don't have Sphinx installed, grab it from 68 | echo.http://sphinx-doc.org/ 69 | exit /b 1 70 | ) 71 | 72 | :sphinx_ok 73 | 74 | 75 | if "%1" == "html" ( 76 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 77 | if errorlevel 1 exit /b 1 78 | echo. 79 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 80 | goto end 81 | ) 82 | 83 | if "%1" == "dirhtml" ( 84 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 85 | if errorlevel 1 exit /b 1 86 | echo. 87 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 88 | goto end 89 | ) 90 | 91 | if "%1" == "singlehtml" ( 92 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 93 | if errorlevel 1 exit /b 1 94 | echo. 95 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 96 | goto end 97 | ) 98 | 99 | if "%1" == "pickle" ( 100 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 101 | if errorlevel 1 exit /b 1 102 | echo. 103 | echo.Build finished; now you can process the pickle files. 104 | goto end 105 | ) 106 | 107 | if "%1" == "json" ( 108 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 109 | if errorlevel 1 exit /b 1 110 | echo. 111 | echo.Build finished; now you can process the JSON files. 112 | goto end 113 | ) 114 | 115 | if "%1" == "htmlhelp" ( 116 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 117 | if errorlevel 1 exit /b 1 118 | echo. 119 | echo.Build finished; now you can run HTML Help Workshop with the ^ 120 | .hhp project file in %BUILDDIR%/htmlhelp. 121 | goto end 122 | ) 123 | 124 | if "%1" == "qthelp" ( 125 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 129 | .qhcp project file in %BUILDDIR%/qthelp, like this: 130 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\bloop.qhcp 131 | echo.To view the help file: 132 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\bloop.ghc 133 | goto end 134 | ) 135 | 136 | if "%1" == "devhelp" ( 137 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 138 | if errorlevel 1 exit /b 1 139 | echo. 140 | echo.Build finished. 141 | goto end 142 | ) 143 | 144 | if "%1" == "epub" ( 145 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 146 | if errorlevel 1 exit /b 1 147 | echo. 148 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 149 | goto end 150 | ) 151 | 152 | if "%1" == "latex" ( 153 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 154 | if errorlevel 1 exit /b 1 155 | echo. 156 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 157 | goto end 158 | ) 159 | 160 | if "%1" == "latexpdf" ( 161 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 162 | cd %BUILDDIR%/latex 163 | make all-pdf 164 | cd %~dp0 165 | echo. 166 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 167 | goto end 168 | ) 169 | 170 | if "%1" == "latexpdfja" ( 171 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 172 | cd %BUILDDIR%/latex 173 | make all-pdf-ja 174 | cd %~dp0 175 | echo. 176 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 177 | goto end 178 | ) 179 | 180 | if "%1" == "text" ( 181 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 182 | if errorlevel 1 exit /b 1 183 | echo. 184 | echo.Build finished. The text files are in %BUILDDIR%/text. 185 | goto end 186 | ) 187 | 188 | if "%1" == "man" ( 189 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 190 | if errorlevel 1 exit /b 1 191 | echo. 192 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 193 | goto end 194 | ) 195 | 196 | if "%1" == "texinfo" ( 197 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 198 | if errorlevel 1 exit /b 1 199 | echo. 200 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 201 | goto end 202 | ) 203 | 204 | if "%1" == "gettext" ( 205 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 206 | if errorlevel 1 exit /b 1 207 | echo. 208 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 209 | goto end 210 | ) 211 | 212 | if "%1" == "changes" ( 213 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 214 | if errorlevel 1 exit /b 1 215 | echo. 216 | echo.The overview file is in %BUILDDIR%/changes. 217 | goto end 218 | ) 219 | 220 | if "%1" == "linkcheck" ( 221 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 222 | if errorlevel 1 exit /b 1 223 | echo. 224 | echo.Link check complete; look for any errors in the above output ^ 225 | or in %BUILDDIR%/linkcheck/output.txt. 226 | goto end 227 | ) 228 | 229 | if "%1" == "doctest" ( 230 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 231 | if errorlevel 1 exit /b 1 232 | echo. 233 | echo.Testing of doctests in the sources finished, look at the ^ 234 | results in %BUILDDIR%/doctest/output.txt. 235 | goto end 236 | ) 237 | 238 | if "%1" == "coverage" ( 239 | %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage 240 | if errorlevel 1 exit /b 1 241 | echo. 242 | echo.Testing of coverage in the sources finished, look at the ^ 243 | results in %BUILDDIR%/coverage/python.txt. 244 | goto end 245 | ) 246 | 247 | if "%1" == "xml" ( 248 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 249 | if errorlevel 1 exit /b 1 250 | echo. 251 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 252 | goto end 253 | ) 254 | 255 | if "%1" == "pseudoxml" ( 256 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 257 | if errorlevel 1 exit /b 1 258 | echo. 259 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 260 | goto end 261 | ) 262 | 263 | :end 264 | -------------------------------------------------------------------------------- /docs/meta/about.rst: -------------------------------------------------------------------------------- 1 | About 2 | ^^^^^ 3 | 4 | ============ 5 | Contributing 6 | ============ 7 | 8 | Thanks for contributing! Feel free to `open an issue`_ for any bugs, typos, unhelpful docs, or general unhappiness 9 | which you may encounter while using Bloop. If you want to `create a pull request`_, even more awesome! Please 10 | make sure all the tox environments pass. 11 | 12 | To start developing Bloop first `create a fork`_, then clone and run the tests:: 13 | 14 | git clone git@github.com:[YOU]/bloop.git 15 | cd bloop 16 | pip install tox -e . 17 | tox 18 | 19 | .. note:: 20 | 21 | The integration tests use `docker`_ to run a `local instance of DynamoDB`_. The tests automatically 22 | start and tear down an image named ``"ddb-local"`` that uses port ``8000``. You can use ``--skip-cleanup`` 23 | to leave the container running after tests finish. 24 | 25 | .. _docker: https://www.docker.com/get-started 26 | .. _local instance of DynamoDB: https://hub.docker.com/r/amazon/dynamodb-local/ 27 | 28 | .. _meta-versioning: 29 | 30 | ========== 31 | Versioning 32 | ========== 33 | 34 | .. _meta-versioning-public: 35 | 36 | ---------- 37 | Public API 38 | ---------- 39 | 40 | Bloop follows `Semantic Versioning 2.0.0`__ and a `draft appendix`__ for its :ref:`Public API `. 41 | 42 | The following are enforced: 43 | 44 | * Backwards incompatible changes in major version only 45 | * New features in minor version or higher 46 | * Backwards compatible bug fixes in patch version or higher (see `appendix`_) 47 | 48 | __ http://semver.org/spec/v2.0.0.html 49 | __ appendix_ 50 | .. _appendix: https://gist.github.com/numberoverzero/c5d0fc6dea624533d004239a27e545ad 51 | 52 | .. _versioning-internal: 53 | 54 | ------------ 55 | Internal API 56 | ------------ 57 | 58 | The :ref:`Internal API ` is not versioned, and may make backwards incompatible changes at any time. 59 | When a class or function is not explicitly documented as part on the public or internal api, 60 | it is part of the internal api. Still, please `open an issue`_ so it can be appropriately documented. 61 | 62 | .. _open an issue: https://github.com/numberoverzero/bloop/issues/new 63 | .. _create a pull request: https://github.com/numberoverzero/bloop/pull/new/master 64 | .. _create a fork: https://github.com/numberoverzero/bloop/network 65 | 66 | ======= 67 | License 68 | ======= 69 | 70 | .. include:: ../../LICENSE 71 | -------------------------------------------------------------------------------- /docs/user/conditions.rst: -------------------------------------------------------------------------------- 1 | .. _conditions: 2 | 3 | Conditions 4 | ^^^^^^^^^^ 5 | 6 | Conditions are used for: 7 | 8 | * Optimistic concurrency when :ref:`saving ` or :ref:`deleting ` objects 9 | * To specify a Query's :ref:`key condition ` 10 | * To :ref:`filter results ` from a Query or Scan 11 | 12 | ===================== 13 | Built-In Conditions 14 | ===================== 15 | 16 | There is no DynamoDB type that supports all of the conditions. For example, ``contains`` does not work with 17 | a numeric type ``"N"`` such as Number or Integer. DynamoDB's `ConditionExpression Reference`__ has the full 18 | specification. 19 | 20 | .. code-block:: python 21 | 22 | class Model(BaseModel): 23 | column = Column(SomeType) 24 | 25 | # Comparisons 26 | Model.column < value 27 | Model.column <= value 28 | Model.column == value 29 | Model.column >= value 30 | Model.column > value 31 | Model.column != value 32 | 33 | Model.column.begins_with(value) 34 | Model.column.between(low, high) 35 | Model.column.contains(value) 36 | Model.column.in_([foo, bar, baz]) 37 | Model.column.is_(None) 38 | Model.column.is_not(False) 39 | 40 | # bitwise operators combine conditions 41 | not_none = Model.column.is_not(None) 42 | in_the_future = Model.column > now 43 | 44 | in_the_past = ~in_the_future 45 | either = not_none | in_the_future 46 | both = not_none & in_the_future 47 | 48 | __ http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.SpecifyingConditions.html#ConditionExpressionReference 49 | 50 | ============================== 51 | Chained Conditions (AND, OR) 52 | ============================== 53 | 54 | Bloop overloads the ``&`` and ``|`` operators for conditions, allowing you to more easily construct compound 55 | conditions. Some libraries allow you to chain filters with ``.filter(c1).filter(c2)`` or pass a list of conditions 56 | ``.filter([c1, c2])`` but both of these forms struggle to express nested conditions, especially when expressing an 57 | OR operation. 58 | 59 | For example, consider a query to find popular articles. We want either new articles with more than 100 likes, 60 | recent articles with more than 500 likes, or older articles with more than 1000 likes. We're running a spotlight on 61 | editor of the month "Nancy Stevens" so let's include those as well. 62 | 63 | .. code-block:: python 64 | 65 | from datetime import datetime, timedelta, timezone 66 | now = datetime.now(timezone.utc) 67 | yesterday = now - timedelta(hours=12) 68 | last_week = now - timedelta(days=7) 69 | last_year = now - timedelta(weeks=52) 70 | 71 | popular = ( 72 | ((Article.likes >= 100) & (Article.publish_date >= yesterday)) | 73 | ((Article.likes >= 500) & (Article.publish_date >= last_week)) | 74 | ((Article.likes >= 1000) & (Article.publish_date >= last_year)) 75 | ) 76 | spotlight = Article.editor == "nstevens" 77 | 78 | articles = engine.scan(Article, filter=popular|spotlight) 79 | 80 | 81 | We can programmatically build conditions from a base of ``bloop.Condition``, which is an empty condition. In the 82 | following example, ``editors`` may have come from a query param or form submission: 83 | 84 | .. code-block:: python 85 | 86 | editors = ["nstevens", "jsmith", "bholly"] 87 | condition = bloop.Condition() 88 | 89 | for editor in editors: 90 | condition |= Article.editor == editor 91 | 92 | articles = engine.scan(Article, filter=condition) 93 | 94 | 95 | Although less frequently used, there is also the ``~`` operator to negate an existing condition. This is useful to 96 | flip a compound condition, rather than trying to invert all the intermediate operators. To find all the unpopular or 97 | non-spotlighted articles, we'll use the variables from the first example above: 98 | 99 | .. code-block:: python 100 | 101 | popular = (...) # see first example 102 | spotlight = ... 103 | 104 | popular_articles = engine.scan(Article, filter=popular|spotlight) 105 | unpopular_articles = engine.scan(Article, filter=~(popular|spotlight)) 106 | 107 | ================ 108 | Document Paths 109 | ================ 110 | 111 | You can construct conditions against individual elements of List and Map types with the usual indexing notation. 112 | 113 | .. code-block:: python 114 | 115 | Item = Map( 116 | name=String, 117 | price=Number, 118 | quantity=Integer) 119 | Metrics = Map(**{ 120 | "payment-duration": Number, 121 | "coupons.used"=Integer, 122 | "coupons.available"=Integer 123 | }) 124 | class Receipt(BaseModel): 125 | transaction_id = Column(UUID, column=True) 126 | total = Column(Integer) 127 | 128 | items = Column(List(Item)) 129 | metrics = Column(Metrics) 130 | 131 | Here are some basic conditions using paths: 132 | 133 | .. code-block:: python 134 | 135 | Receipt.metrics["payment-duration"] > 30000 136 | Receipt.items[0]["name"].begins_with("deli:salami:") 137 | -------------------------------------------------------------------------------- /docs/user/extensions.rst: -------------------------------------------------------------------------------- 1 | Bloop Extensions 2 | ^^^^^^^^^^^^^^^^ 3 | 4 | Extension dependencies aren't installed with Bloop, because they may include a huge number of libraries that Bloop 5 | does not depend on. For example, two extensions could provide automatic mapping to Django or SQLAlchemy models. 6 | Many users would never need either of these, since Bloop does not depend on them for normal usage. 7 | 8 | Bloop extensions are part of the :ref:`Public API `, and subject to 9 | :ref:`its versioning policy`. 10 | 11 | .. _user-extensions-datetime: 12 | 13 | ======================== 14 | DateTime and Timestamp 15 | ======================== 16 | 17 | Working with python's :class:`datetime.datetime` is tedious, but there are a number of popular libraries 18 | that improve the situation. Bloop includes drop-in replacements for the basic 19 | :class:`~bloop.types.DateTime` and :class:`~bloop.types.Timestamp` types for `arrow`_, `delorean`_, and `pendulum`_ 20 | through the :ref:`extensions module`. For example, let's swap out some code using the 21 | built-in DateTime: 22 | 23 | .. code-block:: python 24 | 25 | import datetime 26 | from bloop import DateTime 27 | from bloop import BaseModel, Column, Integer 28 | 29 | class User(BaseModel): 30 | id = Column(Integer, hash_key=True) 31 | created_on = Column(DateTime) 32 | 33 | utc = datetime.timezone.utc 34 | now = datetime.datetime.now(utc) 35 | 36 | user = User(id=0, created_on=now) 37 | 38 | Now, using pendulum: 39 | 40 | .. code-block:: python 41 | 42 | import pendulum 43 | from bloop.ext.pendulum import DateTime 44 | from bloop import BaseModel, Column, Integer 45 | 46 | class User(BaseModel): 47 | id = Column(Integer, hash_key=True) 48 | created_on = Column(DateTime) 49 | 50 | now = pendulum.now("utc") 51 | 52 | user = User(id=0, created_on=now) 53 | 54 | Now, using arrow: 55 | 56 | .. code-block:: python 57 | 58 | import arrow 59 | from bloop.ext.arrow import DateTime 60 | from bloop import BaseModel, Column, Integer 61 | 62 | class User(BaseModel): 63 | id = Column(Integer, hash_key=True) 64 | created_on = Column(DateTime) 65 | 66 | now = arrow.now("utc") 67 | 68 | user = User(id=0, created_on=now) 69 | 70 | .. _arrow: http://crsmithdev.com/arrow 71 | .. _delorean: https://delorean.readthedocs.io/en/latest/ 72 | .. _pendulum: https://pendulum.eustace.io 73 | -------------------------------------------------------------------------------- /docs/user/install.rst: -------------------------------------------------------------------------------- 1 | .. _user-install: 2 | 3 | Installation 4 | ^^^^^^^^^^^^ 5 | 6 | :: 7 | 8 | pip install bloop 9 | 10 | # or 11 | 12 | git clone git://github.com/numberoverzero/bloop.git 13 | cd bloop && python setup.py install 14 | -------------------------------------------------------------------------------- /docs/user/quickstart.rst: -------------------------------------------------------------------------------- 1 | .. _user-quickstart: 2 | 3 | Quickstart 4 | ^^^^^^^^^^ 5 | 6 | First define a model and create the backing table in DynamoDB: 7 | 8 | .. code-block:: pycon 9 | 10 | >>> import uuid 11 | >>> from bloop import ( 12 | ... BaseModel, Boolean, Column, Engine, 13 | ... GlobalSecondaryIndex, String, UUID) 14 | ... 15 | >>> class Account(BaseModel): 16 | ... id = Column(UUID, hash_key=True) 17 | ... name = Column(String) 18 | ... email = Column(String) 19 | ... by_email = GlobalSecondaryIndex( 20 | ... projection='keys', 21 | ... hash_key='email') 22 | ... verified = Column(Boolean, default=False) 23 | ... 24 | >>> engine = Engine() 25 | >>> engine.bind(Account) 26 | 27 | To create an instance and save it in DynamoDB: 28 | 29 | .. code-block:: pycon 30 | 31 | >>> account = Account( 32 | ... id=uuid.uuid4(), 33 | ... name='username', 34 | ... email='foo@bar.com') 35 | ... 36 | >>> engine.save(account) 37 | 38 | 39 | You can load the account by ``id``, or query the GSI by ``email``: 40 | 41 | .. code-block:: pycon 42 | 43 | >>> same_account = Account(id=account.id) 44 | >>> engine.load(same_account) 45 | >>> q = engine.query( 46 | ... Account.by_email, 47 | ... key=Account.email == 'foo@bar.com') 48 | ... 49 | >>> also_same_account = q.first() 50 | 51 | Kick it up a notch with conditional operations: 52 | 53 | .. code-block:: pycon 54 | 55 | # Only save if the account doesn't already exist 56 | >>> if_not_exist = Account.id.is_(None) 57 | >>> engine.save(account, condition=if_not_exist) 58 | 59 | # Only update the account if the name hasn't changed 60 | >>> account.email = 'new@email.com' 61 | >>> engine.save(account, condition=Account.name == 'username') 62 | 63 | # Only delete the account if the email hasn't changed since we last saved 64 | >>> engine.delete(account, condition=Account.email == "new@email.com") 65 | 66 | 67 | Or load the last state of an object before it was deleted: 68 | 69 | .. code-block:: pycon 70 | 71 | >>> engine.delete(account, sync="old") 72 | >>> print(f"last email was {account.email}") 73 | -------------------------------------------------------------------------------- /docs/user/signals.rst: -------------------------------------------------------------------------------- 1 | Signals 2 | ^^^^^^^ 3 | 4 | Signals (powered by `blinker`_) allow you to easily respond to events. Bloop exposes a number of signals during 5 | model creation, validation, and as objects are loaded and saved. 6 | 7 | .. code-block:: pycon 8 | 9 | >>> from bloop import model_created 10 | >>> @model_created.connect 11 | ... def on_new_model(_, *, model, **__): 12 | ... models.append(model) 13 | ... 14 | >>> models = [] 15 | 16 | To disconnect a receiver: 17 | 18 | .. code-block:: pycon 19 | 20 | >>> model_created.disconnect(on_new_model) 21 | 22 | You can specify a sender to restrict who you receive notifications from. This simplifies many cross-region 23 | tasks, where multiple engines are sending the same type of notifications. For example, you can 24 | automatically bind and save models to a second region: 25 | 26 | .. code-block:: pycon 27 | 28 | >>> @model_created.connect(sender=primary_engine) 29 | >>> def on_new_model(_, model, **__): 30 | ... secondary_engine.bind(model) 31 | ... 32 | >>> @object_saved.connect(sender=primary_engine) 33 | ... def on_save(_, obj, **__): 34 | ... secondary_engine.save(obj) 35 | 36 | .. _blinker: https://pythonhosted.org/blinker/ 37 | 38 | ========== 39 | Parameters 40 | ========== 41 | 42 | Your receiver must accept ``**kwargs``, and should only use ``_`` or ``sender`` for the positional argument. 43 | The following templates are recommended for all receivers: 44 | 45 | .. code-block:: python 46 | 47 | def receiver(_, *, kwarg1, kwarg2, **__): 48 | 49 | def receiver(sender, *, kwarg1, kwarg2, **__): 50 | 51 | Instead of forcing you to remember which parameter the sender is (engine? model?) Bloop sends **every** parameter 52 | as a kwarg. This means your receiver can always ignore the positional argument, and cherry pick the parameters you 53 | care about. The sender is accessed the same as all other parameters. 54 | 55 | You can still specify a sender when you connect, but you should not use that parameter name in your function signature. 56 | For example, :data:`~.signals.model_bound` is sent by ``engine`` and includes ``engine`` and ``model``. 57 | If you set up a receiver that names its first positional arg "engine", this causes a :exc:`TypeError`: 58 | 59 | .. code-block:: pycon 60 | 61 | >>> @model_bound.connect 62 | ... def wrong_receiver(engine, model, **__): 63 | ... pass 64 | ... 65 | >>> model_bound.send("engine", model="model", engine="engine") 66 | TypeError: wrong_receiver() got multiple values for argument 'engine' 67 | 68 | 69 | Here's the correct version, which also filters on sender: 70 | 71 | .. code-block:: pycon 72 | 73 | >>> @model_bound.connect(sender="engine") 74 | ... def correct_receiver(_, model, engine, **__): 75 | ... print("Called!") 76 | ... 77 | >>> model_bound.send("engine", model="model", engine="engine") 78 | Called! 79 | 80 | .. note:: 81 | 82 | * New parameters can be added in a minor version. 83 | * A sender can be added to an anonymous signal in a minor version. 84 | * A major version can remove a parameter and remove or replace a sender. 85 | 86 | 87 | ================ 88 | Built-in Signals 89 | ================ 90 | 91 | See the :ref:`Public API ` for a list of available signals. 92 | -------------------------------------------------------------------------------- /docs/user/streams.rst: -------------------------------------------------------------------------------- 1 | .. _user-streams: 2 | 3 | Streams 4 | ^^^^^^^ 5 | 6 | Bloop provides a simple, pythonic interface to DynamoDB's `complex`__ `Streams API`__. This abstracts away the 7 | minutiae of managing and refreshing iterators, tracking sequence numbers and shard splits, merging records from 8 | adjacent shards, and saving and loading processing state. 9 | 10 | .. warning:: 11 | 12 | **Chronological order is not guaranteed for high throughput streams.** 13 | 14 | DynamoDB guarantees ordering: 15 | 16 | * within any single shard 17 | * across shards for a single hash/range key 18 | 19 | There is no way to exactly order records from adjacent shards. High throughput streams 20 | provide approximate ordering using each record's "ApproximateCreationDateTime". 21 | 22 | Tables with a single partition guarantee order across all records. 23 | 24 | See :ref:`Stream Internals ` for details. 25 | 26 | 27 | __ http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Streams.html 28 | __ http://docs.aws.amazon.com/dynamodbstreams/latest/APIReference/Welcome.html 29 | 30 | ================ 31 | Enable Streaming 32 | ================ 33 | 34 | Add the following to a model's ``Meta`` to enable a stream with new and old objects in each record: 35 | 36 | .. code-block:: python 37 | 38 | class User(BaseModel): 39 | class Meta: 40 | stream = { 41 | "include": ["new", "old"] 42 | } 43 | id = Column(Integer, hash_key=True) 44 | email = Column(String) 45 | verified = Column(Boolean) 46 | 47 | engine.bind(User) 48 | 49 | ``"include"`` has four possible values, matching `StreamViewType`__: 50 | 51 | .. code-block:: python 52 | 53 | {"keys"}, {"new"}, {"old"}, {"new", "old"} 54 | 55 | __ http://docs.aws.amazon.com/dynamodbstreams/latest/APIReference/API_StreamDescription.html#DDB-Type-StreamDescription-StreamViewType 56 | 57 | 58 | .. _stream-create: 59 | 60 | =============== 61 | Create a Stream 62 | =============== 63 | 64 | Next, create a stream on the model. This example starts at "trim_horizon" to get all records from the last 65 | 24 hours, but could also be "latest" to only return records created after the stream was instantiated. 66 | 67 | .. code-block:: pycon 68 | 69 | >>> stream = engine.stream(User, "trim_horizon") 70 | 71 | If you want to start at a certain point in time, you can also use a :class:`datetime.datetime`. 72 | Creating streams at a specific time is **very expensive**, and will iterate all records since the stream's 73 | trim_horizon until the target time. 74 | 75 | .. code-block:: pycon 76 | 77 | >>> stream = engine.stream(User, datetime.now() - timedelta(hours=12)) 78 | 79 | If you are trying to resume processing from the same position as another stream, you should load from a persisted 80 | :data:`Stream.token ` instead of using a specific time. 81 | See :ref:`stream-resume` for an example of a stream token. 82 | 83 | .. code-block:: pycon 84 | 85 | >>> import json 86 | >>> original_stream = engine.stream(User, "trim_horizon") 87 | >>> with open("/tmp/state", "w") as f: 88 | ... json.dump(original_stream.token, f) 89 | ... 90 | # Some time later 91 | >>> with open("/tmp/state", "r") as f: 92 | ... token = json.load(f) 93 | ... 94 | >>> stream = engine.stream(User, token) 95 | 96 | ================ 97 | Retrieve Records 98 | ================ 99 | 100 | You only need to call :func:`next` on a Stream to get the next record: 101 | 102 | .. code-block:: pycon 103 | 104 | >>> record = next(stream) 105 | 106 | If there are no records at the current position, record will be ``None``. A common pattern is to poll immediately 107 | when a record is found, but to wait a small amount when no record is found. 108 | 109 | .. code-block:: pycon 110 | 111 | >>> while True: 112 | ... record = next(stream) 113 | ... if not record: 114 | ... time.sleep(0.2) 115 | ... else: 116 | ... process(record) 117 | 118 | ---------------- 119 | Record Structure 120 | ---------------- 121 | 122 | Each record is a dict with instances of the model in one or more of ``"key"``, ``"old"``, and ``"new"``. 123 | These are populated according to the stream's ``"include"`` above, as well as the event type. A key-only 124 | stream will never have new or old objects. If a stream includes new and old objects and the event type is delete, 125 | new will be ``None``. 126 | 127 | Save a new user, and then update the email address: 128 | 129 | .. code-block:: pycon 130 | 131 | >>> user = User(id=3, email="user@domain.com") 132 | >>> engine.save(user) 133 | >>> user.email = "admin@domain.com" 134 | >>> engine.save(user) 135 | 136 | The first record won't have an old value, since it was the first time this item was saved: 137 | 138 | .. code-block:: pycon 139 | 140 | >>> next(stream) 141 | {'key': None, 142 | 'old': None, 143 | 'new': User(email='user@domain.com', id=3, verified=None), 144 | 'meta': { 145 | 'created_at': datetime.datetime(2016, 10, 23, ...), 146 | 'event': { 147 | 'id': '3fe6d339b7cb19a1474b3d853972c12a', 148 | 'type': 'insert', 149 | 'version': '1.1'}, 150 | 'sequence_number': '700000000007366876916'} 151 | } 152 | 153 | The second record shows the change to email, and has both old and new objects: 154 | 155 | .. code-block:: pycon 156 | 157 | >>> next(stream) 158 | {'key': None, 159 | 'old': User(email='user@domain.com', id=3, verified=None), 160 | 'new': User(email='admin@domain.com', id=3, verified=None), 161 | 'meta': { 162 | 'created_at': datetime.datetime(2016, 10, 23, ...), 163 | 'event': { 164 | 'id': '73a4b8568a85a0bcac25799f806df239', 165 | 'type': 'modify', 166 | 'version': '1.1'}, 167 | 'sequence_number': '800000000007366876936'} 168 | } 169 | 170 | .. _periodic-heartbeats: 171 | 172 | ------------------- 173 | Periodic Heartbeats 174 | ------------------- 175 | 176 | You should call :func:`Stream.heartbeat() ` 177 | at least every 14 minutes in your processing loop. 178 | 179 | Iterators only last 15 minutes, and need to be refreshed periodically. There's no way to 180 | safely refresh an iterator that hasn't found a record. For example, refreshing an iterator at "latest" could miss 181 | records since the time that the previous iterator was at "latest". If you call this every 15 minutes, an iterator 182 | may expire due to clock skew or processing time. 183 | 184 | Only iterators without sequence numbers will be refreshed. Once a shard finds a record it's 185 | skipped on every subsequent heartbeat. For a moderately active stream, heartbeat will make about one call per shard. 186 | 187 | The following pattern will call heartbeat every 12 minutes (if record processing is quick): 188 | 189 | .. code-block:: pycon 190 | 191 | >>> from datetime import datetime, timedelta 192 | >>> now = datetime.now 193 | >>> future = lambda: datetime.now() + timedelta(minutes=12) 194 | >>> 195 | >>> next_heartbeat = now() 196 | >>> while True: 197 | ... record = next(stream) 198 | ... process(record) 199 | ... if now() > next_heartbeat: 200 | ... next_heartbeat = future() 201 | ... stream.heartbeat() 202 | 203 | .. _stream-resume: 204 | 205 | -------------------- 206 | Pausing and Resuming 207 | -------------------- 208 | 209 | Use :data:`Stream.token ` to save the current state and resume processing later: 210 | 211 | .. code-block:: pycon 212 | 213 | >>> with open("/tmp/stream-token", "w") as f: 214 | ... json.dump(stream.token, f, indent=2) 215 | >>> with open("/tmp/stream-token", "r") as f: 216 | ... token = json.load(f) 217 | >>> stream = engine.stream(User, token) 218 | 219 | When reloading from a token, Bloop will automatically prune shards that have expired, and extend the 220 | state to include new shards. Any iterators that fell behind the current trim_horizon will be moved 221 | to each of their children's trim_horizons. 222 | 223 | Here's a token from a new stream. After 8-12 hours there will be one active shard, but also a few 224 | closed shards that form the lineage of the stream. 225 | 226 | .. code-block:: python 227 | 228 | { 229 | "active": [ 230 | "shardId-00000001477207595861-d35d208d" 231 | ], 232 | "shards": [ 233 | { 234 | "iterator_type": "after_sequence", 235 | "sequence_number": "800000000007366876936", 236 | "shard_id": "shardId-00000001477207595861-d35d208d" 237 | } 238 | ], 239 | "stream_arn": "arn:.../stream/2016-10-23T07:26:33.312" 240 | } 241 | 242 | 243 | 244 | ------------- 245 | Moving Around 246 | ------------- 247 | 248 | This function takes the same ``position`` argument as :func:`Engine.stream `: 249 | 250 | .. code-block:: pycon 251 | 252 | # Any stream token; this one rebuilds the 253 | # stream in its current location 254 | >>> stream.move_to(stream.token) 255 | 256 | # Jump back in time 2 hours 257 | >>> stream.move_to(datetime.now() - timedelta(hours=2)) 258 | 259 | # Move to the oldest record in the stream 260 | >>> stream.move_to("trim_horizon") 261 | 262 | As noted :ref:`above `, moving to a specific time is **very expensive**. 263 | -------------------------------------------------------------------------------- /docs/user/transactions.rst: -------------------------------------------------------------------------------- 1 | .. _user-transactions: 2 | 3 | Transactions 4 | ^^^^^^^^^^^^ 5 | 6 | Bloop supports reading and updating items in `transactions`_ similar to the way you already 7 | load, save, and delete items using an engine. A single read or write transaction can have at most 10 items. 8 | 9 | To create a new transaction, call :func:`Engine.transaction(mode="w") ` and specify 10 | a mode: 11 | 12 | .. code-block:: python 13 | 14 | wx = engine.transaction(mode="w") 15 | rx = engine.transaction(mode="r") 16 | 17 | When used as a context manager the transaction will call 18 | :func:`commit() ` on exit if no exception occurs: 19 | 20 | 21 | .. code-block:: python 22 | 23 | # mode defaults to "w" 24 | with engine.transaction() as tx: 25 | tx.save(some_obj) 26 | tx.delete(other_obj) 27 | 28 | 29 | # read transaction loads all objects at once 30 | user = User(id="numberoverzero") 31 | meta = Metadata(id=to_load.id) 32 | with engine.transaction(mode="r") as tx: 33 | tx.load(user, meta) 34 | 35 | You may also call :func:`prepare() ` and 36 | :func:`commit() ` yourself: 37 | 38 | .. code-block:: python 39 | 40 | import bloop 41 | 42 | tx = engine.transaction() 43 | tx.save(some_obj) 44 | p = tx.prepare() 45 | try: 46 | p.commit() 47 | except bloop.TransactionCanceled: 48 | print("failed to commit") 49 | 50 | 51 | See :exc:`~bloop.exceptions.TransactionCanceled` for the conditions that can cause each type of transaction to fail. 52 | 53 | .. _transactions: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/transactions.html 54 | 55 | 56 | ==================== 57 | Write Transactions 58 | ==================== 59 | 60 | A write transaction can save and delete items, and specify additional conditions on objects not being modified. 61 | 62 | As with :ref:`Engine.save ` and :ref:`Engine.delete ` you can provide multiple 63 | objects to each :func:`WriteTransaction.save() ` or 64 | :func:`WriteTransaction.delete() ` call: 65 | 66 | .. code-block:: python 67 | 68 | with engine.transaction() as tx: 69 | tx.delete(*old_tweets) 70 | tx.save(new_user, new_tweet) 71 | 72 | ----------------- 73 | Item Conditions 74 | ----------------- 75 | 76 | You can specify a ``condition`` with each save or delete call: 77 | 78 | .. code-block:: python 79 | 80 | with engine.transaction() as tx: 81 | tx.delete(auth_token, condition=Token.last_used <= now()) 82 | 83 | ------------------------ 84 | Transaction Conditions 85 | ------------------------ 86 | 87 | In addition to specifying conditions on the objects being modified, you can also specify a condition for the 88 | transaction on an object that won't be modified. This can be useful if you want to check another table without 89 | changing its value: 90 | 91 | .. code-block:: python 92 | 93 | user_meta = Metadata(id="numberoverzero") 94 | 95 | with engine.transaction() as tx: 96 | tx.save(new_tweet) 97 | tx.check(user_meta, condition=Metadata.verified.is_(True)) 98 | 99 | In the above example the transaction doesn't modify the user metadata. If we want to modify that object we should 100 | instead use a condition on the object being modified: 101 | 102 | .. code-block:: python 103 | 104 | user_meta = Metadata(id="numberoverzero") 105 | engine.load(user_meta) 106 | user_meta.tweets += 1 107 | 108 | with engine.transaction() as tx: 109 | tx.save(new_tweet) 110 | tx.save(user_meta, condition=Metadata.tweets <= 500) 111 | 112 | ------------- 113 | Idempotency 114 | ------------- 115 | 116 | Bloop automatically generates timestamped unique tokens (:attr:`~bloop.transactions.PreparedTransaction.tx_id` and 117 | :attr:`~bloop.transactions.PreparedTransaction.first_commit_at`) 118 | to guard against committing a write transaction twice or accidentally committing a transaction that was prepared a 119 | long time ago. While these are generated for both read and write commits, only `TransactWriteItems`_ respects the 120 | `"ClientRequestToken"`_ stored in tx_id. 121 | 122 | When the :attr:`~bloop.transactions.PreparedTransaction.first_commit_at` value is too old, 123 | committing will raise :exc:`~bloop.exceptions.TransactionTokenExpired`. 124 | 125 | .. _TransactWriteItems: https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_TransactWriteItems.html 126 | .. _"ClientRequestToken": https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_TransactWriteItems.html#DDB-TransactWriteItems-request-ClientRequestToken 127 | 128 | =================== 129 | Read Transactions 130 | =================== 131 | 132 | By default :func:`engine.transaction(mode="w") ` will create a 133 | :class:`~bloop.transactions.WriteTransaction`. To create a :class:`~bloop.transactions.ReadTransaction` pass 134 | ``mode="r"``: 135 | 136 | .. code-block:: python 137 | 138 | with engine.transaction(mode="r") as rx: 139 | rx.load(user, tweet) 140 | rx.load(meta) 141 | 142 | All objects in the read transaction will be loaded at the same time, when 143 | :func:`commit() ` is called or the transaction context closes. 144 | 145 | ------------------ 146 | Multiple Commits 147 | ------------------ 148 | 149 | Every time you call commit on the prepared transaction, the objects will be loaded again: 150 | 151 | .. code-block:: python 152 | 153 | rx = engine.transaction(mode="r") 154 | rx.load(user, tweet) 155 | prepared = rx.prepare() 156 | 157 | prepared.commit() # first load 158 | prepared.commit() # second load 159 | 160 | ----------------- 161 | Missing Objects 162 | ----------------- 163 | 164 | As with :ref:`Engine.load ` if any objects in the transaction are missing when commit is called, 165 | bloop will raise :exc:`~bloop.exceptions.MissingObjects` with the list of objects that were not found: 166 | 167 | .. code-block:: python 168 | 169 | import bloop 170 | 171 | engine = bloop.Engine() 172 | ... 173 | 174 | 175 | def tx_load(*objs): 176 | with engine.transaction(mode="r") as rx: 177 | rx.load(*objs) 178 | 179 | ... 180 | 181 | try: 182 | tx_load(user, tweet) 183 | except bloop.MissingObjects as exc: 184 | missing = exc.objects 185 | print(f"failed to load {len(missing)} objects: {missing}") 186 | -------------------------------------------------------------------------------- /examples/documents.py: -------------------------------------------------------------------------------- 1 | import decimal 2 | import random 3 | import uuid 4 | from datetime import datetime, timezone 5 | 6 | from bloop import ( 7 | UUID, 8 | BaseModel, 9 | Column, 10 | DateTime, 11 | Engine, 12 | Integer, 13 | Map, 14 | Number, 15 | Set, 16 | String, 17 | ) 18 | 19 | 20 | # ================================================ 21 | # Model setup 22 | # ================================================ 23 | 24 | Product = Map(**{ 25 | 'Name': String, 26 | 'Rating': Number, 27 | 'Updated': DateTime, 28 | 'Description': Map(**{ 29 | 'Title': String, 30 | 'Body': String 31 | }), 32 | 'Sellers': Set(Integer) 33 | }) 34 | 35 | 36 | class Item(BaseModel): 37 | id = Column(UUID, hash_key=True) 38 | data = Column(Product) 39 | 40 | 41 | engine = Engine() 42 | engine.bind(BaseModel) 43 | 44 | 45 | # ================================================ 46 | # Usage 47 | # ================================================ 48 | 49 | item = Item(id=uuid.uuid4()) 50 | item.data = { 51 | 'Name': 'item-name', 52 | 'Rating': decimal.Decimal(str(random.random())), 53 | 'Updated': datetime.now(timezone.utc), 54 | 'Description': { 55 | 'Title': 'item-title', 56 | 'Body': 'item-body', 57 | }, 58 | 'Sellers': set() 59 | } 60 | 61 | for i in range(4): 62 | seller_id = 'seller-{}'.format(i) 63 | item.data['Sellers'].add(seller_id) 64 | 65 | engine.save(item) 66 | -------------------------------------------------------------------------------- /examples/mixins.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import datetime 3 | import hashlib 4 | 5 | import delorean 6 | 7 | from bloop import BaseModel, Binary, Column, Engine, Integer, String 8 | from bloop.ext.delorean import Timestamp 9 | 10 | 11 | DEFAULT_PASTE_LIFETIME_DAYS = 31 12 | 13 | 14 | def new_expiry(days=DEFAULT_PASTE_LIFETIME_DAYS): 15 | """Return an expiration `days` in the future""" 16 | now = delorean.Delorean() 17 | return now + datetime.timedelta(days=days) 18 | 19 | 20 | class SortByVersion: 21 | """Mixin for a string-based hash key and a version number for range_key""" 22 | id = Column(String, hash_key=True) 23 | version = Column(Integer, range_key=True, dynamo_name="v") 24 | 25 | 26 | class Paste(SortByVersion, BaseModel): 27 | class Meta: 28 | ttl = {"column": "not_after"} 29 | 30 | not_after = Column(Timestamp, default=new_expiry) 31 | bucket = Column(String, dynamo_name="b") 32 | key = Column(String, dynamo_name="k") 33 | 34 | 35 | class UserImage(SortByVersion, BaseModel): 36 | jpg = Column(Binary) 37 | 38 | 39 | engine = Engine() 40 | engine.bind(BaseModel) 41 | 42 | 43 | def s3_upload(content: str) -> (str, str): 44 | # TODO persist in s3 45 | return "bucket-id", "key-id" 46 | 47 | 48 | def b64sha256(content: str) -> str: 49 | hash = hashlib.sha256(content.encode()) 50 | return base64.b64encode(hash.digest()).decode() 51 | 52 | 53 | def new_paste(content: str) -> str: 54 | id = b64sha256(content) 55 | bucket, key = s3_upload(content) 56 | 57 | paste = Paste(bucket=bucket, key=key, id=id, version=0) 58 | engine.save(paste) 59 | return id 60 | 61 | 62 | def get_paste(id: str, version=None) -> Paste: 63 | if version: 64 | paste = Paste(id=id, version=version) 65 | engine.load(paste) 66 | return paste 67 | else: 68 | # Reverse ordering to get last value of version 69 | query = engine.query(Paste, key=Paste.id == id, forward=False) 70 | return query.first() 71 | -------------------------------------------------------------------------------- /examples/replication.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import pendulum 3 | 4 | from bloop import UUID, BaseModel, Column, Engine 5 | from bloop.ext.pendulum import DateTime 6 | 7 | 8 | def engine_for_region(region, table_name_template="{table_name}"): 9 | dynamodb = boto3.client("dynamodb", region_name=region) 10 | dynamodbstreams = boto3.client("dynamodbstreams", region_name=region) 11 | return Engine( 12 | dynamodb=dynamodb, 13 | dynamodbstreams=dynamodbstreams, 14 | table_name_template=table_name_template 15 | ) 16 | 17 | 18 | primary = engine_for_region("us-west-2", table_name_template="primary.{table_name}") 19 | replica = engine_for_region("us-east-1", table_name_template="replica.{table_name}") 20 | 21 | 22 | class SomeDataBlob(BaseModel): 23 | class Meta: 24 | stream = { 25 | "include": {"new", "old"} 26 | } 27 | 28 | id = Column(UUID, hash_key=True) 29 | uploaded = Column(DateTime, range_key=True) 30 | 31 | 32 | primary.bind(SomeDataBlob) 33 | replica.bind(SomeDataBlob) 34 | 35 | 36 | def scan_replicate(): 37 | """Bulk replicate existing data""" 38 | for obj in primary.scan(SomeDataBlob): 39 | replica.save(obj) 40 | 41 | 42 | def stream_replicate(): 43 | """Monitor changes in approximately real-time and replicate them""" 44 | stream = primary.stream(SomeDataBlob, "trim_horizon") 45 | next_heartbeat = pendulum.now() 46 | while True: 47 | now = pendulum.now() 48 | if now >= next_heartbeat: 49 | stream.heartbeat() 50 | next_heartbeat = now.add(minutes=10) 51 | 52 | record = next(stream) 53 | if record is None: 54 | continue 55 | if record["new"] is not None: 56 | replica.save(record["new"]) 57 | else: 58 | replica.delete(record["old"]) 59 | -------------------------------------------------------------------------------- /examples/tweet.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from datetime import datetime, timezone 3 | 4 | from bloop import ( 5 | UUID, 6 | BaseModel, 7 | Column, 8 | DateTime, 9 | Engine, 10 | GlobalSecondaryIndex, 11 | Integer, 12 | String, 13 | ) 14 | 15 | 16 | # ================================================ 17 | # Model setup 18 | # ================================================ 19 | 20 | class Account(BaseModel): 21 | class Meta: 22 | read_units = 5 23 | write_units = 2 24 | 25 | id = Column(UUID, hash_key=True) 26 | name = Column(String) 27 | email = Column(String) 28 | by_email = GlobalSecondaryIndex( 29 | hash_key='email', projection='keys', 30 | write_units=1, read_units=5) 31 | 32 | 33 | class Tweet(BaseModel): 34 | class Meta: 35 | write_units = 10 36 | account = Column(UUID, hash_key=True) 37 | id = Column(String, range_key=True) 38 | content = Column(String) 39 | date = Column(DateTime) 40 | favorites = Column(Integer) 41 | 42 | by_date = GlobalSecondaryIndex( 43 | hash_key='date', projection='keys') 44 | 45 | 46 | engine = Engine() 47 | engine.bind(BaseModel) 48 | 49 | 50 | # ================================================ 51 | # Usage 52 | # ================================================ 53 | 54 | account = Account( 55 | id=uuid.uuid4(), name='@garybernhardt', 56 | email='REDACTED') 57 | tweet = Tweet( 58 | account=account.id, id='616102582239399936', 59 | content='today, I wrote a type validator in Python, as you do', 60 | favorites=9, 61 | date=datetime.now(timezone.utc)) 62 | 63 | engine.save(account, tweet) 64 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | arrow 2 | blinker==1.4 3 | boto3==1.20.4 4 | coverage 5 | delorean 6 | flake8 7 | pendulum 8 | pytest 9 | pytz 10 | readme_renderer[md] 11 | setuptools 12 | sphinx==7.3.7 13 | sphinx-rtd-theme==2.0.0 14 | tox 15 | twine 16 | wheel 17 | -------------------------------------------------------------------------------- /scripts/graph-dependencies: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | pydeps ./bloop -x blinker boto3 botocore pendulum delorean arrow bloop.ext 3 | -------------------------------------------------------------------------------- /scripts/single-test: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import subprocess 4 | import sys 5 | 6 | from pathlib import Path 7 | 8 | CWD = Path(".").resolve() 9 | PROJECT_ROOT = Path(os.path.abspath(os.path.dirname(__file__))) / ".." 10 | SRC_ROOT = PROJECT_ROOT / "bloop" 11 | TEST_ROOT = PROJECT_ROOT / "tests" / "unit" 12 | 13 | 14 | def run(cmd: str, suppress=False) -> None: 15 | stdout = subprocess.PIPE if suppress else None 16 | stderr = subprocess.PIPE if suppress else None 17 | subprocess.run(cmd, shell=True, universal_newlines=True, stdout=stdout, stderr=stderr) 18 | 19 | 20 | def clear_coverage() -> None: 21 | try: 22 | (PROJECT_ROOT.resolve() / ".coverage").unlink() 23 | except FileNotFoundError: 24 | pass 25 | 26 | 27 | def src_path(name: str) -> Path: 28 | """stream/shard -> /some/path/bloop/stream/shard.py""" 29 | path = (SRC_ROOT.resolve() / name).with_suffix(".py") 30 | try: 31 | return path.resolve() 32 | except FileNotFoundError: 33 | sys.exit("ERROR {} does not exist.".format(path)) 34 | 35 | 36 | def test_path(name: str) -> Path: 37 | """stream/shard -> /some/path/tests/unit/test_stream/test_shard.py""" 38 | path = TEST_ROOT.resolve().joinpath( 39 | # foo / bar -> test_foo / test_bar 40 | *["test_" + x for x in Path(name).parts] 41 | ).with_suffix(".py") 42 | try: 43 | return path.resolve() 44 | except FileNotFoundError: 45 | pass 46 | 47 | 48 | def run_test(name: str, verbose=False, suppress=False, append=False) -> None: 49 | src = src_path(name) 50 | test = test_path(name) 51 | 52 | if not test: 53 | print(">>> NO COVERAGE FOR {} <<<".format(name)) 54 | return 55 | template = "coverage run {append} --branch --include={src} -m pytest {test} {verbose}" 56 | try: 57 | src = src.relative_to(CWD) 58 | except ValueError: 59 | pass 60 | try: 61 | test = test.relative_to(CWD) 62 | except ValueError: 63 | pass 64 | 65 | cmd = template.format( 66 | append="--append" if append else "", verbose="-v" if verbose else "", 67 | src=src, test=test) 68 | print(cmd) 69 | run(cmd, suppress=suppress) 70 | 71 | 72 | if __name__ == "__main__": 73 | if len(sys.argv) > 2: 74 | sys.exit("No args to run all tests, or module path to run one suite") 75 | 76 | clear_coverage() 77 | 78 | if len(sys.argv) == 2: 79 | filename = sys.argv[1] 80 | run_test(filename, verbose=True) 81 | 82 | else: 83 | # Suppress output since this is *only* to collect coverage info 84 | names = [str(p.relative_to(SRC_ROOT)) for p in SRC_ROOT.rglob("*.py") if p.stem != "__init__"] 85 | for filename in names: 86 | run_test(filename, suppress=True, append=True) 87 | print() 88 | 89 | run("coverage report -m") 90 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pathlib 3 | from setuptools import setup, find_packages 4 | 5 | HERE = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) 6 | README = (HERE / "README.rst").read_text() 7 | CHANGES = (HERE / "CHANGELOG.rst").read_text() 8 | VERSION = "VERSION-NOT-FOUND" 9 | for line in (HERE / "bloop" / "__init__.py").read_text().split("\n"): 10 | if line.startswith("__version__"): 11 | VERSION = eval(line.split("=")[-1]) 12 | 13 | REQUIREMENTS = [ 14 | "blinker==1.8.2", 15 | "boto3==1.34.131", 16 | ] 17 | 18 | if __name__ == "__main__": 19 | setup( 20 | name="bloop", 21 | version=VERSION, 22 | description="ORM for DynamoDB", 23 | long_description=README + "\n\n" + CHANGES, 24 | long_description_content_type="text/x-rst", 25 | classifiers=[ 26 | "Development Status :: 5 - Production/Stable", 27 | "Intended Audience :: Developers", 28 | "License :: OSI Approved :: MIT License", 29 | "Operating System :: OS Independent", 30 | "Programming Language :: Python", 31 | "Programming Language :: Python :: 3", 32 | "Programming Language :: Python :: 3.6", 33 | "Topic :: Software Development :: Libraries" 34 | ], 35 | author="Joe Cross", 36 | author_email="joe.mcross@gmail.com", 37 | url="https://github.com/numberoverzero/bloop", 38 | license="MIT", 39 | keywords="aws dynamo dynamodb dynamodbstreams orm", 40 | platforms="any", 41 | include_package_data=True, 42 | packages=find_packages(exclude=("docs", "examples", "scripts", "tests")), 43 | install_requires=REQUIREMENTS, 44 | ) 45 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numberoverzero/bloop/a875f268d876ed0834876ffa81b5065eab6d972b/tests/__init__.py -------------------------------------------------------------------------------- /tests/helpers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numberoverzero/bloop/a875f268d876ed0834876ffa81b5065eab6d972b/tests/helpers/__init__.py -------------------------------------------------------------------------------- /tests/helpers/models.py: -------------------------------------------------------------------------------- 1 | from bloop import ( 2 | UUID, 3 | BaseModel, 4 | Binary, 5 | Column, 6 | Condition, 7 | DateTime, 8 | GlobalSecondaryIndex, 9 | Integer, 10 | List, 11 | LocalSecondaryIndex, 12 | Map, 13 | Number, 14 | Set, 15 | String, 16 | ) 17 | 18 | 19 | DocumentType = Map(**{ 20 | 'Rating': Number(), 21 | 'Stock': Integer(), 22 | 'Description': Map(**{ 23 | 'Heading': String, 24 | 'Body': String, 25 | 'Specifications': String 26 | }), 27 | 'Id': UUID, 28 | 'Updated': DateTime 29 | }) 30 | 31 | 32 | class Document(BaseModel): 33 | id = Column(Integer, hash_key=True) 34 | data = Column(DocumentType) 35 | numbers = Column(List(Integer)) 36 | value = Column(Number) 37 | another_value = Column(Number) 38 | some_string = Column(String) 39 | nested_numbers = Column(List(List(Integer))) 40 | 41 | 42 | class User(BaseModel): 43 | id = Column(String, hash_key=True) 44 | age = Column(Integer) 45 | name = Column(String) 46 | email = Column(String) 47 | joined = Column(DateTime, dynamo_name="j") 48 | by_email = GlobalSecondaryIndex(hash_key="email", projection="all") 49 | 50 | 51 | class SimpleModel(BaseModel): 52 | class Meta: 53 | table_name = "Simple" 54 | id = Column(String, hash_key=True) 55 | 56 | 57 | class ComplexModel(BaseModel): 58 | class Meta: 59 | write_units = 2 60 | read_units = 3 61 | table_name = "CustomTableName" 62 | 63 | name = Column(UUID, hash_key=True) 64 | date = Column(String, range_key=True) 65 | email = Column(String) 66 | joined = Column(String) 67 | not_projected = Column(Integer) 68 | by_email = GlobalSecondaryIndex(hash_key="email", read_units=4, projection="all", write_units=5) 69 | by_joined = LocalSecondaryIndex(range_key="joined", projection=["email"]) 70 | 71 | 72 | class VectorModel(BaseModel): 73 | name = Column(String, hash_key=True) 74 | list_str = Column(List(String)) 75 | set_str = Column(Set(String)) 76 | map_nested = Column(Map(**{ 77 | "bytes": Binary, 78 | "str": String, 79 | "map": Map(**{ 80 | "int": Integer, 81 | "str": String 82 | }) 83 | })) 84 | some_int = Column(Integer) 85 | some_bytes = Column(Binary) 86 | 87 | 88 | # Provides a gsi and lsi with constrained projections for testing Filter.select validation 89 | class ProjectedIndexes(BaseModel): 90 | h = Column(Integer, hash_key=True) 91 | r = Column(Integer, range_key=True) 92 | both = Column(String) 93 | neither = Column(String) 94 | gsi_only = Column(String) 95 | lsi_only = Column(String) 96 | 97 | by_gsi = GlobalSecondaryIndex(hash_key="h", projection=["both", "gsi_only"]) 98 | by_lsi = LocalSecondaryIndex(range_key="r", projection=["both", "lsi_only"]) 99 | 100 | 101 | conditions = set() 102 | 103 | 104 | def _build_conditions(): 105 | """This is a method so that we can name each condition before adding it. 106 | 107 | This makes the conditions self-documenting; 108 | simplifies building compound conditions; 109 | eases extension for new test cases 110 | """ 111 | empty = Condition() 112 | lt = Document.id < 10 113 | gt = Document.id > 12 114 | 115 | path = Document.data["Rating"] == 3.4 116 | 117 | # Order doesn't matter for multi conditions 118 | basic_and = lt & gt 119 | swapped_and = gt & lt 120 | multiple_and = lt & lt & gt 121 | 122 | basic_or = lt | gt 123 | swapped_or = gt | lt 124 | multiple_or = lt | lt | gt 125 | 126 | not_lt = ~lt 127 | not_gt = ~gt 128 | 129 | not_exists_data = Document.data.is_(None) 130 | not_exists_id = Document.id.is_(None) 131 | exists_id = Document.id.is_not(None) 132 | 133 | begins_hello = Document.some_string.begins_with("hello") 134 | begins_world = Document.some_string.begins_with("world") 135 | 136 | contains_hello = Document.some_string.contains("hello") 137 | contains_world = Document.some_string.contains("world") 138 | contains_numbers = Document.numbers.contains(9) 139 | 140 | between_small = Document.id.between(5, 6) 141 | between_big = Document.id.between(100, 200) 142 | between_strings = Document.some_string.between("alpha", "zebra") 143 | 144 | in_small = Document.id.in_([3, 7, 11]) 145 | in_big = Document.id.in_([123, 456]) 146 | in_numbers = Document.numbers.in_([120, 450]) 147 | 148 | conditions.update(( 149 | empty, 150 | lt, gt, path, 151 | basic_and, swapped_and, multiple_and, 152 | basic_or, swapped_or, multiple_or, 153 | not_lt, not_gt, 154 | not_exists_data, not_exists_id, exists_id, 155 | begins_hello, begins_world, between_strings, 156 | contains_hello, contains_world, contains_numbers, 157 | between_small, between_big, between_strings, 158 | in_small, in_big, in_numbers 159 | )) 160 | 161 | 162 | _build_conditions() 163 | -------------------------------------------------------------------------------- /tests/helpers/utils.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | 4 | def get_tables(dynamodb): 5 | it = dynamodb.get_paginator("list_tables").paginate() 6 | tables = [response["TableNames"] for response in it] 7 | tables = itertools.chain(*tables) 8 | return tables 9 | -------------------------------------------------------------------------------- /tests/integ/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numberoverzero/bloop/a875f268d876ed0834876ffa81b5065eab6d972b/tests/integ/__init__.py -------------------------------------------------------------------------------- /tests/integ/conftest.py: -------------------------------------------------------------------------------- 1 | import secrets 2 | import subprocess 3 | 4 | import boto3 5 | import pytest 6 | from tests.helpers.utils import get_tables 7 | 8 | from bloop import BaseModel, BloopException, Engine 9 | from bloop.session import SessionWrapper 10 | from bloop.util import walk_subclasses 11 | 12 | 13 | DEFAULT_PORT = 8000 14 | DEFAULT_ENDPOINT = f"http://localhost:{DEFAULT_PORT}" 15 | DOCKER_START_COMMAND = [ 16 | "docker", "run", "-d", 17 | "-p", f"{DEFAULT_PORT}:{DEFAULT_PORT}", 18 | "--name", "ddb-local", "amazon/dynamodb-local" 19 | ] 20 | DOCKER_STOP_COMMAND = ["docker", "stop", "ddb-local"] 21 | DOCKER_RM_COMMAND = ["docker", "rm", "ddb-local"] 22 | 23 | 24 | class PatchedDynamoDBClient: 25 | def __init__(self, real_client): 26 | self.__client = real_client 27 | 28 | def describe_time_to_live(self, **_): 29 | return {"TimeToLiveDescription": {"TimeToLiveStatus": "DISABLED"}} 30 | 31 | def describe_continuous_backups(self, **_): 32 | return {"ContinuousBackupsDescription": {"ContinuousBackupsStatus": "DISABLED"}} 33 | 34 | def __getattr__(self, name): 35 | return getattr(self.__client, name) 36 | 37 | 38 | class DynamoDBLocal: 39 | def __init__(self, endpoint: str) -> None: 40 | self.access_key = secrets.token_hex(10) 41 | self.secret_access_key = secrets.token_hex(20) 42 | self.endpoint = endpoint 43 | self.running = False 44 | 45 | @property 46 | def session(self) -> boto3.Session: 47 | assert self.running 48 | return boto3.Session( 49 | region_name="local-not-a-region", 50 | aws_access_key_id=self.access_key, 51 | aws_secret_access_key=self.secret_access_key, 52 | ) 53 | 54 | @property 55 | def clients(self) -> tuple: 56 | session = self.session 57 | endpoint = self.endpoint 58 | return ( 59 | # TODO | have to patch dynamodb until DynamoDBLocal supports DescribeTimeToLive 60 | # TODO | otherwise, SessionWrapper.describe_table throws UnknownOperationException 61 | PatchedDynamoDBClient(session.client("dynamodb", endpoint_url=endpoint)), 62 | session.client("dynamodbstreams", endpoint_url=endpoint) 63 | ) 64 | 65 | def start(self) -> None: 66 | assert not self.running 67 | self.running = True 68 | subprocess.run(DOCKER_START_COMMAND, stdout=subprocess.PIPE, check=True) 69 | 70 | def stop(self) -> None: 71 | assert self.running 72 | subprocess.run(DOCKER_STOP_COMMAND, stdout=subprocess.PIPE, check=True) 73 | subprocess.run(DOCKER_RM_COMMAND, stdout=subprocess.PIPE, check=True) 74 | self.running = False 75 | 76 | 77 | def pytest_addoption(parser): 78 | default_nonce = f"-local-{secrets.token_hex(8)}" 79 | parser.addoption( 80 | "--nonce", action="store", default=default_nonce, 81 | help="make table names unique for parallel runs") 82 | parser.addoption( 83 | "--skip-cleanup", action="store_true", default=False, 84 | help="don't clean up the docker instance after tests run") 85 | parser.addoption( 86 | "--dynamodblocal-endpoint", action="store", default=None, 87 | help="endpoint for an existing dynamodblocal instance") 88 | 89 | 90 | @pytest.fixture(scope="session") 91 | def dynamodb_local(request): 92 | nonce = request.config.getoption("--nonce") 93 | skip_cleanup = request.config.getoption("--skip-cleanup") 94 | existing_local = request.config.getoption("--dynamodblocal-endpoint") 95 | 96 | dynamodb_local = DynamoDBLocal(endpoint=existing_local or DEFAULT_ENDPOINT) 97 | if existing_local: 98 | dynamodb_local.running = True 99 | else: 100 | dynamodb_local.start() 101 | 102 | yield dynamodb_local 103 | 104 | if skip_cleanup: 105 | print("Skipping cleanup, leaving docker image intact") 106 | return 107 | try: 108 | print("Cleaning up tables with nonce '{}'".format(nonce)) 109 | dynamodb, _ = dynamodb_local.clients 110 | tables = get_tables(dynamodb) 111 | for table in tables: 112 | if nonce not in table: 113 | continue 114 | # noinspection PyBroadException 115 | try: 116 | print("Removing table: {}".format(table)) 117 | dynamodb.delete_table(TableName=table) 118 | except Exception: 119 | print("Failed to clean up table '{}'".format(table)) 120 | finally: 121 | if not existing_local: 122 | print("Shutting down ddb-local") 123 | dynamodb_local.stop() 124 | 125 | 126 | @pytest.fixture 127 | def dynamodb(dynamodb_local): 128 | dynamodb, _ = dynamodb_local.clients 129 | return dynamodb 130 | 131 | 132 | @pytest.fixture 133 | def dynamodbstreams(dynamodb_local): 134 | _, dynamodbstreams = dynamodb_local.clients 135 | return dynamodbstreams 136 | 137 | 138 | @pytest.fixture 139 | def engine(dynamodb, dynamodbstreams, request): 140 | engine = Engine( 141 | dynamodb=dynamodb, dynamodbstreams=dynamodbstreams, 142 | table_name_template="{table_name}" + request.config.getoption("--nonce") 143 | ) 144 | yield engine 145 | 146 | # This collects all subclasses of BaseModel and are not abstract. We are trying to delete any data in 147 | # dynamodb-local between unit tests so we don't step on each other's toes. 148 | concrete = set(filter(lambda m: not m.Meta.abstract, walk_subclasses(BaseModel))) 149 | for model in concrete: 150 | # we can run into a situation where the class was created, but not bound in the engine (or table created), so 151 | # we only try. As the dynamodb-local process is only running in memory this isn't too much of a problem. 152 | try: 153 | objs = list(engine.scan(model)) 154 | if objs: 155 | engine.delete(*objs) 156 | except BloopException: 157 | pass 158 | 159 | 160 | @pytest.fixture 161 | def session(dynamodb, dynamodbstreams): 162 | return SessionWrapper(dynamodb=dynamodb, dynamodbstreams=dynamodbstreams) 163 | -------------------------------------------------------------------------------- /tests/integ/models.py: -------------------------------------------------------------------------------- 1 | import random 2 | import string 3 | from enum import Enum 4 | 5 | from bloop import ( 6 | UUID, 7 | BaseModel, 8 | Boolean, 9 | Column, 10 | DateTime, 11 | DynamicMap, 12 | GlobalSecondaryIndex, 13 | LocalSecondaryIndex, 14 | Set, 15 | String, 16 | ) 17 | 18 | 19 | # ================================================================================ 20 | # Types 21 | # ================================================================================ 22 | class StringEnum(String): 23 | def __init__(self, enum_cls): 24 | self.enum_cls = enum_cls 25 | super().__init__() 26 | 27 | def dynamo_dump(self, value, *, context, **kwargs): 28 | if value is None: 29 | return value 30 | value = value.name 31 | return super().dynamo_dump(value, context=context, **kwargs) 32 | 33 | def dynamo_load(self, value, *, context, **kwargs): 34 | if value is None: 35 | return value 36 | value = super().dynamo_load(value, context=context, **kwargs) 37 | return self.enum_cls[value] 38 | 39 | 40 | class Role(Enum): 41 | user = "user" 42 | curator = "curator" 43 | superuser = "super_user" 44 | admin = "admin" 45 | 46 | 47 | # ================================================================================ 48 | # Mixins 49 | # ================================================================================ 50 | class UUIDHashKey(object): 51 | id = Column(UUID, hash_key=True) 52 | 53 | 54 | class CreatedRangeKey(object): 55 | created = Column(DateTime, range_key=True) 56 | 57 | 58 | class IdentityMixin(object): 59 | roles = Column(Set(StringEnum(Role))) 60 | 61 | @property 62 | def is_active(self): 63 | return True 64 | 65 | @property 66 | def is_authenticated(self): 67 | return True 68 | 69 | @property 70 | def is_anonymous(self): 71 | return False 72 | 73 | def get_id(self): 74 | return self.id 75 | 76 | 77 | # ================================================================================ 78 | # Classes 79 | # ================================================================================ 80 | class MixinBase(BaseModel, UUIDHashKey, CreatedRangeKey): 81 | class Meta: 82 | abstract = True 83 | email = Column(String) 84 | updated = Column(DateTime) 85 | active = Column(Boolean) 86 | by_created = LocalSecondaryIndex(projection="all", range_key=CreatedRangeKey.created) 87 | 88 | 89 | class MixinUser(MixinBase, IdentityMixin): 90 | first_name = Column(String) 91 | last_name = Column(String) 92 | by_email = GlobalSecondaryIndex(projection='all', dynamo_name='email-index', hash_key='email') 93 | 94 | def __str__(self): 95 | return "{} {}: {}".format(self.first_name, self.last_name, self.email) 96 | 97 | 98 | class ExternalUser(MixinUser): 99 | company = Column(String) 100 | by_email = GlobalSecondaryIndex(projection='all', dynamo_name='email-index', hash_key=MixinUser.email) 101 | 102 | 103 | class User(BaseModel): 104 | class Meta: 105 | read_units = 1 106 | write_units = 3 107 | email = Column(String, hash_key=True) 108 | username = Column(String, range_key=True) 109 | by_username = GlobalSecondaryIndex(projection="keys", hash_key="username") 110 | 111 | profile = Column(String) 112 | data = Column(DynamicMap) 113 | extra = Column(String) 114 | 115 | 116 | def _letters(n): 117 | return "".join(random.choice(string.ascii_letters) for _ in range(n)) 118 | 119 | 120 | def valid_user(): 121 | email = "e-{}@{}".format(_letters(3), _letters(4)) 122 | username = "u-{}".format(_letters(7)) 123 | return User( 124 | email=email, 125 | username=username 126 | ) 127 | -------------------------------------------------------------------------------- /tests/integ/test_basic.py: -------------------------------------------------------------------------------- 1 | """Basic scenarios, symmetric tests""" 2 | import uuid 3 | from random import randint 4 | 5 | import pytest 6 | 7 | from bloop import ( 8 | UUID, 9 | BaseModel, 10 | Column, 11 | DynamicMap, 12 | GlobalSecondaryIndex, 13 | Integer, 14 | MissingObjects, 15 | String, 16 | ) 17 | 18 | from .models import User 19 | 20 | 21 | def test_crud(engine): 22 | engine.bind(User) 23 | 24 | user = User(email="user@domain.com", username="user", profile="first") 25 | engine.save(user) 26 | 27 | same_user = User(email=user.email, username=user.username) 28 | engine.load(same_user) 29 | assert user.profile == same_user.profile 30 | 31 | same_user.profile = "second" 32 | engine.save(same_user) 33 | 34 | engine.load(user, consistent=True) 35 | assert user.profile == same_user.profile 36 | 37 | engine.delete(user) 38 | 39 | with pytest.raises(MissingObjects) as excinfo: 40 | engine.load(same_user, consistent=True) 41 | assert [same_user] == excinfo.value.objects 42 | 43 | 44 | def test_model_defaults(engine): 45 | class ColumnDefaultsModel(BaseModel): 46 | hash = Column(Integer, hash_key=True, default=12) 47 | range = Column(Integer, range_key=True, default=24) 48 | other = Column(Integer, default=48) 49 | engine.bind(ColumnDefaultsModel) 50 | 51 | obj = ColumnDefaultsModel() 52 | assert obj.hash == 12 53 | assert obj.range == 24 54 | assert obj.other == 48 55 | 56 | engine.save(obj) 57 | 58 | same_obj = ColumnDefaultsModel(hash=12, range=24) 59 | engine.load(same_obj) 60 | 61 | assert same_obj.hash == 12 62 | assert same_obj.range == 24 63 | assert same_obj.other == 48 64 | 65 | 66 | def test_model_defaults_load(engine): 67 | class ColumnDefaultLoadModel(BaseModel): 68 | hash = Column(Integer, hash_key=True, default=12) 69 | range = Column(Integer, range_key=True, default=24) 70 | other = Column(Integer, default=48) 71 | engine.bind(ColumnDefaultLoadModel) 72 | 73 | obj = ColumnDefaultLoadModel(hash=333, range=333) 74 | engine.save(obj) 75 | 76 | same_obj = ColumnDefaultLoadModel(hash=obj.hash, range=obj.range) 77 | engine.load(same_obj) 78 | 79 | assert same_obj.hash == 333 80 | assert same_obj.range == 333 81 | assert same_obj.other == 48 82 | 83 | 84 | def test_model_default_func(engine): 85 | get_int_called = 0 86 | get_randint_called = 0 87 | 88 | def get_int(): 89 | nonlocal get_int_called 90 | get_int_called += 1 91 | return 404 92 | 93 | def random_int(): 94 | nonlocal get_randint_called 95 | get_randint_called += 1 96 | return randint(1, 100) 97 | 98 | class ColumnDefaultFuncModel(BaseModel): 99 | hash = Column(UUID, hash_key=True, default=uuid.uuid4()) 100 | range = Column(Integer, range_key=True, default=get_int) 101 | other = Column(Integer, default=random_int) 102 | engine.bind(ColumnDefaultFuncModel) 103 | 104 | obj = ColumnDefaultFuncModel() 105 | assert get_int_called == 1 106 | assert get_randint_called == 1 107 | assert isinstance(obj.hash, uuid.UUID) 108 | assert obj.range == 404 109 | 110 | engine.save(obj) 111 | 112 | # get_int shouldn't be called because we are passing that value in the constructor 113 | same_obj = ColumnDefaultFuncModel(hash=obj.hash, range=obj.range) 114 | engine.load(same_obj) 115 | 116 | assert get_int_called == 1 117 | assert get_randint_called == 2 118 | assert same_obj.hash == obj.hash 119 | assert same_obj.range == obj.range 120 | assert same_obj.other == obj.other 121 | 122 | 123 | def test_model_default_projection(engine): 124 | def token_hex(prefix=None): 125 | if prefix: 126 | return prefix + uuid.uuid4().hex 127 | return uuid.uuid4().hex 128 | 129 | class MyModel(BaseModel): 130 | id = Column(Integer, hash_key=True) 131 | email = Column(String) 132 | 133 | password = Column(String, default=token_hex) 134 | 135 | by_email = GlobalSecondaryIndex( 136 | projection="keys", 137 | hash_key="email" 138 | ) 139 | 140 | engine.bind(MyModel) 141 | 142 | expected_password = token_hex("RC_") 143 | instance = MyModel( 144 | id=3, email="u@d.com", 145 | password=expected_password 146 | ) 147 | engine.save(instance) 148 | 149 | q = engine.query(MyModel.by_email, key=MyModel.email == "u@d.com") 150 | same_instance = q.first() 151 | 152 | assert not hasattr(same_instance, 'password') 153 | 154 | q = engine.query(MyModel, key=MyModel.id == 3) 155 | same_instance = q.first() 156 | 157 | assert same_instance.password == expected_password 158 | 159 | 160 | def test_projection_overlap(engine): 161 | class ProjectionOverlap(BaseModel): 162 | hash = Column(Integer, hash_key=True) 163 | range = Column(Integer, range_key=True) 164 | other = Column(Integer) 165 | 166 | by_other = GlobalSecondaryIndex(projection=["other", "range"], hash_key="other") 167 | # by_other's projected attributes overlap with the model and its own keys 168 | engine.bind(ProjectionOverlap) 169 | 170 | 171 | def test_stream_creation(engine): 172 | class StreamCreation(BaseModel): 173 | class Meta: 174 | stream = { 175 | "include": ["keys"] 176 | } 177 | hash = Column(Integer, hash_key=True) 178 | engine.bind(StreamCreation) 179 | assert "arn" in StreamCreation.Meta.stream 180 | 181 | 182 | def test_stream_read(engine): 183 | class MyStreamReadModel(BaseModel): 184 | class Meta: 185 | stream = { 186 | "include": ["new", "old"] 187 | } 188 | id = Column(Integer, hash_key=True) 189 | data = Column(DynamicMap) 190 | engine.bind(MyStreamReadModel) 191 | 192 | stream = engine.stream(MyStreamReadModel, "trim_horizon") 193 | assert next(stream) is None 194 | 195 | obj = MyStreamReadModel(id=3, data={"foo": "hello, world"}) 196 | another = MyStreamReadModel(id=5, data={"bar": 3}) 197 | # Two calls to ensure ordering 198 | engine.save(obj) 199 | engine.save(another) 200 | for expected in obj, another: 201 | record = next(stream) 202 | assert record["new"].id == expected.id 203 | assert record["new"].data == expected.data 204 | assert record["old"] is None 205 | 206 | 207 | # TODO enable when/if DynamoDBLocal supports DescribeTimeToLive, UpdateTimeToLive 208 | # def test_ttl_enabled(engine): 209 | # class MyModel(BaseModel): 210 | # class Meta: 211 | # ttl = {"column": "expiry"} 212 | # id = Column(Integer, hash_key=True) 213 | # expiry = Column(Timestamp, dynamo_name='e') 214 | # engine.bind(MyModel) 215 | # assert MyModel.Meta.ttl["enabled"] == "enabled" 216 | 217 | 218 | def test_model_overlap(dynamodb, engine): 219 | """Two models backed by the same table, with different indexes""" 220 | class FirstOverlap(BaseModel): 221 | class Meta: 222 | table_name = "overlap-table" 223 | id = Column(Integer, hash_key=True) 224 | first_value = Column(Integer) 225 | first_index = GlobalSecondaryIndex(projection="keys", hash_key="first_value") 226 | 227 | class SecondOverlap(BaseModel): 228 | class Meta: 229 | table_name = "overlap-table" 230 | id = Column(Integer, hash_key=True) 231 | second_value = Column(Integer) 232 | second_index = GlobalSecondaryIndex(projection="keys", hash_key="second_value") 233 | 234 | # bloop won't modify the table to match the expected value, so we need to 235 | # emulate someone setting up a table in the console or by hand. 236 | combined_table = { 237 | "ProvisionedThroughput": {"WriteCapacityUnits": 1, "ReadCapacityUnits": 1}, 238 | "AttributeDefinitions": [ 239 | {"AttributeType": "N", "AttributeName": "id"}, 240 | {"AttributeType": "N", "AttributeName": "first_value"}, 241 | {"AttributeType": "N", "AttributeName": "second_value"}, 242 | ], 243 | "KeySchema": [{"KeyType": "HASH", "AttributeName": "id"}], 244 | "GlobalSecondaryIndexes": [ 245 | { 246 | "IndexName": "first_index", 247 | "Projection": {"ProjectionType": "KEYS_ONLY"}, 248 | "KeySchema": [{"KeyType": "HASH", "AttributeName": "first_value"}], 249 | "ProvisionedThroughput": {"WriteCapacityUnits": 1, "ReadCapacityUnits": 1} 250 | }, 251 | { 252 | "IndexName": "second_index", 253 | "Projection": {"ProjectionType": "KEYS_ONLY"}, 254 | "KeySchema": [{"KeyType": "HASH", "AttributeName": "second_value"}], 255 | "ProvisionedThroughput": {"WriteCapacityUnits": 1, "ReadCapacityUnits": 1} 256 | } 257 | ], 258 | # Can't use the fixed value above since it'll be modified by 259 | # the test framework to allow parallel runs 260 | "TableName": engine._compute_table_name(FirstOverlap) 261 | } 262 | dynamodb.create_table(**combined_table) 263 | 264 | # Now, both of these binds should see the particular subset of indexes/attribute names that they care about 265 | engine.bind(FirstOverlap) 266 | engine.bind(SecondOverlap) 267 | assert True 268 | 269 | 270 | def test_unknown_throughput(dynamodb, engine): 271 | """A model doesn't have to specify read_units or write_units but will take the existing value""" 272 | class ExplicitValues(BaseModel): 273 | class Meta: 274 | read_units = 10 275 | write_units = 1 276 | table_name = "throughput-test" 277 | id = Column(Integer, hash_key=True) 278 | other = Column(Integer) 279 | by_other = GlobalSecondaryIndex( 280 | projection="keys", hash_key=other, read_units=11, write_units=1) 281 | 282 | class ImplicitValues(BaseModel): 283 | class Meta: 284 | write_units = 1 285 | table_name = "throughput-test" 286 | id = Column(Integer, hash_key=True) 287 | other = Column(Integer) 288 | by_other = GlobalSecondaryIndex( 289 | projection="keys", hash_key=other, write_units=1) 290 | 291 | engine.bind(ExplicitValues) 292 | assert ImplicitValues.Meta.read_units is None 293 | assert ImplicitValues.by_other.read_units is None 294 | 295 | # Now binding to the same table but not specifying read_units should have the same value 296 | engine.bind(ImplicitValues) 297 | assert ImplicitValues.Meta.read_units == 10 298 | assert ImplicitValues.by_other.read_units == 11 299 | 300 | 301 | def test_partial_load_save(engine): 302 | engine.bind(User) 303 | 304 | obj = User( 305 | email="my-email@", 306 | username="my-username", 307 | profile="original-profile", 308 | data={"original": "data"}, 309 | extra="my-extra" 310 | ) 311 | engine.save(obj) 312 | 313 | partial = engine.query( 314 | User.by_username, 315 | key=User.username == "my-username").one() 316 | assert not hasattr(partial, "profile") 317 | 318 | partial.data = {"new": [1, 2, True]} 319 | partial.extra = None 320 | engine.save(partial) 321 | 322 | same = User( 323 | email="my-email@", 324 | username="my-username" 325 | ) 326 | engine.load(same) 327 | 328 | # never modified 329 | assert same.profile == "original-profile" 330 | # modified from partial 331 | assert same.data == {"new": [1, 2, True]} 332 | # deleted from partial, missing value for String is "" 333 | assert same.extra == "" 334 | -------------------------------------------------------------------------------- /tests/integ/test_inheritance.py: -------------------------------------------------------------------------------- 1 | import random 2 | import uuid 3 | from datetime import datetime, timezone 4 | from string import ascii_letters 5 | 6 | import pytest 7 | from tests.integ.models import ExternalUser, MixinUser, Role 8 | 9 | from bloop import ( 10 | UUID, 11 | BaseModel, 12 | Column, 13 | DateTime, 14 | GlobalSecondaryIndex, 15 | Integer, 16 | ) 17 | from bloop.exceptions import InvalidModel 18 | 19 | 20 | def test_inheritance_simple(engine): 21 | class NewBase(BaseModel): 22 | class Meta: 23 | abstract = True 24 | uuid = Column(UUID) 25 | 26 | class SimpleModel(NewBase): 27 | id = Column(Integer, hash_key=True) 28 | created_at = Column(DateTime) 29 | 30 | model = SimpleModel() 31 | assert len(model.Meta.columns) == 3 32 | assert len(model.Meta.keys) == 1 33 | assert list(model.Meta.keys)[0].name == 'id' 34 | 35 | 36 | def test_inheritance_base_hashkey(engine): 37 | class NewBase(BaseModel): 38 | class Meta: 39 | abstract = True 40 | uuid = Column(UUID, hash_key=True) 41 | 42 | class SimpleModel(NewBase): 43 | id = Column(Integer) 44 | created_at = Column(DateTime) 45 | 46 | model = SimpleModel() 47 | assert len(model.Meta.columns) == 3 48 | assert len(model.Meta.keys) == 1 49 | assert list(model.Meta.keys)[0].name == 'uuid' 50 | 51 | 52 | def test_inheritance_mixins(engine): 53 | model = MixinUser() 54 | assert len(model.Meta.columns) == 8 55 | assert len(model.Meta.keys) == 2 56 | assert model.Meta.hash_key.name == 'id' 57 | assert model.Meta.range_key.name == 'created' 58 | 59 | 60 | def _create_user(cls, **extra): 61 | now = datetime.now(timezone.utc) 62 | first_name = "".join([random.choice(ascii_letters) for _ in range(8)]) 63 | last_name = "".join([random.choice(ascii_letters) for _ in range(12)]) 64 | email = f"{first_name}.{last_name}@example.com" 65 | 66 | return cls( 67 | id=uuid.uuid4(), created=now, updated=now, active=True, 68 | first_name=first_name, last_name=last_name, email=email, 69 | **extra 70 | ) 71 | 72 | 73 | def gen_external_user(): 74 | extra = {'company': 'Acme', 'roles': {Role.user, Role.admin}} 75 | return _create_user(ExternalUser, **extra) 76 | 77 | 78 | def gen_mixin_user(): 79 | extra = {'roles': {Role.user}} 80 | return _create_user(MixinUser, **extra) 81 | 82 | 83 | @pytest.mark.parametrize("cls, factory", [ 84 | (MixinUser, gen_mixin_user), 85 | (ExternalUser, gen_external_user) 86 | ]) 87 | def test_inheritance_load(engine, cls, factory): 88 | engine.bind(BaseModel) 89 | 90 | obj = factory() 91 | engine.save(obj) 92 | 93 | same_obj = cls(id=obj.id, created=obj.created) 94 | engine.load(same_obj) 95 | 96 | assert same_obj.Meta.model is cls 97 | 98 | for attr in [col.name for col in obj.Meta.columns]: 99 | assert getattr(same_obj, attr) == getattr(obj, attr) 100 | 101 | 102 | def test_inheritance_lsi_from_baseclass(engine): 103 | engine.bind(BaseModel) 104 | 105 | first_group = [] 106 | for x in range(3): 107 | user = gen_mixin_user() 108 | engine.save(user) 109 | first_group.append(user) 110 | 111 | saved_date = datetime.now(timezone.utc) 112 | 113 | second_group = [] 114 | for x in range(3): 115 | user = gen_mixin_user() 116 | engine.save(user) 117 | second_group.append(user) 118 | 119 | # ensure that we won't find a user in the first group that has a created after our saved date. 120 | cond = (MixinUser.id == first_group[0].id) & (MixinUser.created > saved_date) 121 | q = engine.query(MixinUser.by_created, key=cond) 122 | assert len(list(q)) == 0 123 | 124 | # ensure that we *do* find a user in the second group that has a created after our saved date. 125 | cond = (MixinUser.id == second_group[-1].id) & (MixinUser.created > saved_date) 126 | q = engine.query(MixinUser.by_created, key=cond) 127 | items = list(q) 128 | assert len(items) == 1 129 | assert items[0].Meta.model is MixinUser 130 | 131 | 132 | def test_inheritance_lsi_from_concrete_subclass(engine): 133 | engine.bind(BaseModel) 134 | 135 | first_group = [] 136 | for x in range(3): 137 | user = gen_external_user() 138 | engine.save(user) 139 | first_group.append(user) 140 | 141 | saved_date = datetime.now(timezone.utc) 142 | 143 | second_group = [] 144 | for x in range(3): 145 | user = gen_external_user() 146 | engine.save(user) 147 | second_group.append(user) 148 | 149 | # ensure that we won't find a user in the first group that has a created after our saved date. 150 | cond = (ExternalUser.id == first_group[0].id) & (ExternalUser.created > saved_date) 151 | q = engine.query(ExternalUser.by_created, key=cond) 152 | assert len(list(q)) == 0 153 | 154 | # ensure that we *do* find a user in the second group that has a created after our saved date. 155 | cond = (ExternalUser.id == second_group[-1].id) & (ExternalUser.created > saved_date) 156 | q = engine.query(ExternalUser.by_created, key=cond) 157 | items = list(q) 158 | assert len(items) == 1 159 | assert items[0].Meta.model is ExternalUser 160 | 161 | 162 | def test_inheritance_gsi_to_baseclass(engine): 163 | engine.bind(BaseModel) 164 | 165 | user1 = gen_mixin_user() 166 | engine.save(user1) 167 | 168 | cond = MixinUser.email == user1.email 169 | user2 = engine.query(MixinUser.by_email, key=cond).one() 170 | 171 | assert user2.Meta.model is MixinUser 172 | for attr in [col.name for col in user1.Meta.columns]: 173 | assert getattr(user2, attr) == getattr(user1, attr) 174 | 175 | 176 | def test_inheritance_gsi_from_concrete_subclass(engine): 177 | engine.bind(BaseModel) 178 | 179 | user1 = gen_external_user() 180 | engine.save(user1) 181 | 182 | cond = ExternalUser.email == user1.email 183 | user2 = engine.query(ExternalUser.by_email, key=cond).one() 184 | 185 | assert user2.Meta.model is ExternalUser 186 | for attr in [col.name for col in user1.Meta.columns]: 187 | assert getattr(user2, attr) == getattr(user1, attr) 188 | 189 | 190 | def test_inheritance_overwrites_rangekey(engine): 191 | class NextGenUser(MixinUser): 192 | version = Column(Integer, range_key=True) 193 | 194 | 195 | def test_inheritance_overwrites_hashkey(engine): 196 | class NextGenUser(MixinUser): 197 | version = Column(Integer, hash_key=True) 198 | 199 | 200 | def test_inheritance_two_models_same_dynamo_index_name(engine): 201 | class NextGenUser(MixinUser): 202 | version = Column(Integer) 203 | next_by_email = GlobalSecondaryIndex(projection='all', dynamo_name='email-index', hash_key='email') 204 | 205 | 206 | def test_inheritance_two_models_same_dynamo_column_name(engine): 207 | with pytest.raises(InvalidModel): 208 | class NextGenUser(MixinUser): 209 | version = Column(Integer, dynamo_name='email') 210 | -------------------------------------------------------------------------------- /tests/integ/test_queries.py: -------------------------------------------------------------------------------- 1 | from .models import User, valid_user 2 | 3 | 4 | def test_query_with_projection(engine): 5 | engine.bind(User) 6 | user = valid_user() 7 | user.profile = "Hello, World" 8 | engine.save(user) 9 | 10 | query = engine.query( 11 | User, 12 | key=User.email == user.email, 13 | projection={User.email, User.username}) 14 | 15 | result = query.one() 16 | assert not hasattr(result, "profile") 17 | 18 | 19 | def test_scan_count(engine): 20 | engine.bind(User) 21 | scan = engine.scan(User, projection="count") 22 | 23 | for _ in range(7): 24 | engine.save(valid_user()) 25 | assert scan.count == 7 26 | 27 | scan.reset() 28 | engine.save(valid_user()) 29 | assert scan.count == 8 30 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numberoverzero/bloop/a875f268d876ed0834876ffa81b5065eab6d972b/tests/unit/__init__.py -------------------------------------------------------------------------------- /tests/unit/conftest.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from unittest.mock import Mock 3 | 4 | import pytest 5 | 6 | from bloop import Engine 7 | from bloop.session import SessionWrapper 8 | from bloop.signals import ( 9 | object_deleted, 10 | object_loaded, 11 | object_modified, 12 | object_saved, 13 | ) 14 | 15 | 16 | @pytest.fixture(autouse=True) 17 | def __set_log_level(caplog): 18 | """Always set caplog to capture DEBUG""" 19 | caplog.set_level(logging.DEBUG) 20 | 21 | 22 | @pytest.fixture 23 | def dynamodb(): 24 | return Mock() 25 | 26 | 27 | @pytest.fixture 28 | def dynamodbstreams(): 29 | return Mock() 30 | 31 | 32 | @pytest.fixture 33 | def session(): 34 | s = Mock(spec=SessionWrapper) 35 | 36 | # Most tests won't send a "ReturnValues" so simplify mock setup by 37 | # assuming we don't inspect the response. 38 | s.save_item.return_value = None 39 | s.delete_item.return_value = None 40 | 41 | return s 42 | 43 | 44 | @pytest.fixture 45 | def engine(session, dynamodb, dynamodbstreams): 46 | # HACK: These clients won't be used. We're going to replace the session immediately. 47 | engine = Engine(dynamodb=dynamodb, dynamodbstreams=dynamodbstreams) 48 | # Toss the clients above and hook up the mock session 49 | engine.session = session 50 | return engine 51 | 52 | 53 | @pytest.fixture 54 | def signals(): 55 | calls = { 56 | "object_deleted": [], 57 | "object_loaded": [], 58 | "object_modified": [], 59 | "object_saved": [] 60 | } 61 | 62 | @object_deleted.connect 63 | def on_deleted(**kwargs): 64 | calls["deleted"].append(kwargs) 65 | 66 | @object_loaded.connect 67 | def on_loaded(**kwargs): 68 | calls["loaded"].append(kwargs) 69 | 70 | @object_modified.connect 71 | def on_modified(**kwargs): 72 | calls["modified"].append(kwargs) 73 | 74 | @object_saved.connect 75 | def on_saved(**kwargs): 76 | calls["saved"].append(kwargs) 77 | 78 | return calls 79 | -------------------------------------------------------------------------------- /tests/unit/test_actions.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from bloop.actions import ( 4 | NONE_SENTINEL, 5 | Action, 6 | ActionType, 7 | add, 8 | delete, 9 | remove, 10 | set, 11 | unwrap, 12 | wrap, 13 | ) 14 | from bloop.conditions import Reference 15 | 16 | 17 | short_funcs = [add, delete, remove, set] 18 | 19 | 20 | # noinspection PyTypeChecker 21 | @pytest.mark.parametrize("action_type", ActionType) 22 | @pytest.mark.parametrize("value", [None, object(), 1, ActionType.Add, Action]) 23 | def test_valid_actions(action_type, value): 24 | action = Action(action_type, value) 25 | assert action.type is action_type 26 | assert action.value is value 27 | 28 | 29 | @pytest.mark.parametrize("action_type", ["set", "ADD", Action, ActionType]) 30 | def test_invalid_actions(action_type): 31 | with pytest.raises(ValueError): 32 | # noinspection PyTypeChecker 33 | Action(action_type, 3) 34 | 35 | 36 | @pytest.mark.parametrize("action_type, expected", [ 37 | (ActionType.Add, "my_name my_value"), 38 | (ActionType.Delete, "my_name my_value"), 39 | (ActionType.Remove, "my_name"), 40 | (ActionType.Set, "my_name=my_value"), 41 | ]) 42 | def test_render(action_type, expected): 43 | name_ref = Reference("my_name", object(), object()) 44 | value_ref = Reference("my_value", object(), object()) 45 | assert action_type.render(name_ref, value_ref) == expected 46 | 47 | 48 | @pytest.mark.parametrize("func, type", [ 49 | (add, ActionType.Add), 50 | (delete, ActionType.Delete), 51 | (set, ActionType.Set), 52 | (remove, ActionType.Remove), 53 | 54 | ]) 55 | def test_shorthand(func, type): 56 | assert func(3).type is type 57 | 58 | 59 | @pytest.mark.parametrize("value", [3, dict(), ActionType, None]) 60 | def test_unwrap_non_action(value): 61 | assert unwrap(value) is value 62 | 63 | 64 | @pytest.mark.parametrize("value", [3, dict(), ActionType, None]) 65 | @pytest.mark.parametrize("func", short_funcs) 66 | def test_unwrap_action(value, func): 67 | assert unwrap(func(value)) is value 68 | 69 | 70 | @pytest.mark.parametrize("value, type", [ 71 | (3, ActionType.Set), 72 | (dict(), ActionType.Set), 73 | (ActionType, ActionType.Set), 74 | ("", ActionType.Set), 75 | 76 | (None, ActionType.Remove) 77 | ]) 78 | def test_wrap_non_action(value, type): 79 | w = wrap(value) 80 | assert w.value is value 81 | assert w.type is type 82 | 83 | 84 | @pytest.mark.parametrize("value", [3, dict(), ActionType, None]) 85 | @pytest.mark.parametrize("func", short_funcs) 86 | def test_wrap_action(value, func): 87 | action = func(value) 88 | assert wrap(action) is action 89 | 90 | 91 | # noinspection PyTypeChecker 92 | @pytest.mark.parametrize("action_type", list(ActionType)) 93 | def test_new_action(action_type): 94 | value = object() 95 | action = action_type.new_action(value) 96 | assert isinstance(action, Action) 97 | assert action.type is action_type 98 | assert action.value is value 99 | 100 | 101 | # noinspection PyTypeChecker 102 | @pytest.mark.parametrize("action_type", list(ActionType)) 103 | def test_repr(action_type): 104 | action = action_type.new_action("testval") 105 | assert repr(action) == f"" 106 | 107 | 108 | @pytest.mark.parametrize("func", [ActionType.Remove.new_action, remove, wrap]) 109 | def test_sentinel(func): 110 | """Common methods of creating Remove""" 111 | assert func(None) is NONE_SENTINEL 112 | 113 | 114 | # noinspection PyTypeChecker 115 | @pytest.mark.parametrize("action_type", list(ActionType)) 116 | @pytest.mark.parametrize("value", [None, object(), 1, ActionType.Add, Action]) 117 | def test_eq(action_type, value): 118 | a1 = Action(action_type, value) 119 | a2 = Action(action_type, value) 120 | assert a1 == a2 121 | assert a2 == a1 122 | -------------------------------------------------------------------------------- /tests/unit/test_ext/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/numberoverzero/bloop/a875f268d876ed0834876ffa81b5065eab6d972b/tests/unit/test_ext/__init__.py -------------------------------------------------------------------------------- /tests/unit/test_ext/test_arrow.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | import arrow 4 | import pytest 5 | import pytz 6 | 7 | from bloop.ext.arrow import DateTime, Timestamp 8 | from bloop.types import FIXED_ISO8601_FORMAT 9 | 10 | 11 | now = datetime.now(pytz.utc).replace(microsecond=0) 12 | now_eastern = datetime.now(pytz.timezone("US/Eastern")) 13 | now_iso8601 = now.strftime(FIXED_ISO8601_FORMAT) 14 | now_timestamp = str(int(now.timestamp())) 15 | 16 | 17 | @pytest.mark.parametrize("timezone", ["utc", "US/Eastern"]) 18 | def test_datetime(timezone): 19 | arrow_now = arrow.get(now) 20 | typedef = DateTime(timezone) 21 | 22 | assert typedef.dynamo_dump(arrow_now, context={}) == now_iso8601 23 | assert typedef.dynamo_load(now_iso8601, context={}).to("utc").datetime == now 24 | 25 | 26 | @pytest.mark.parametrize("timezone", ["utc", "US/Eastern"]) 27 | def test_timestamp(timezone): 28 | arrow_now = arrow.get(now) 29 | typedef = Timestamp(timezone) 30 | 31 | assert typedef.dynamo_dump(arrow_now, context={}) == now_timestamp 32 | assert typedef.dynamo_load(now_timestamp, context={}).to("utc").datetime == now 33 | 34 | 35 | @pytest.mark.parametrize("typedef_cls", (DateTime, Timestamp)) 36 | def test_none(typedef_cls): 37 | typedef = typedef_cls() 38 | assert typedef.dynamo_dump(None, context={}) is None 39 | assert typedef.dynamo_load(None, context={}) is None 40 | -------------------------------------------------------------------------------- /tests/unit/test_ext/test_delorean.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | import delorean 4 | import pytest 5 | import pytz 6 | 7 | from bloop.ext.delorean import DateTime, Timestamp 8 | from bloop.types import FIXED_ISO8601_FORMAT 9 | 10 | 11 | now = datetime.now(pytz.utc).replace(microsecond=0) 12 | now_eastern = datetime.now(pytz.timezone("US/Eastern")) 13 | now_iso8601 = now.strftime(FIXED_ISO8601_FORMAT) 14 | now_timestamp = str(int(now.timestamp())) 15 | 16 | 17 | @pytest.mark.parametrize("timezone", ["utc", "US/Eastern"]) 18 | def test_datetime(timezone): 19 | delorean_now = delorean.Delorean(now) 20 | typedef = DateTime(timezone) 21 | 22 | assert typedef.dynamo_dump(delorean_now, context={}) == now_iso8601 23 | assert typedef.dynamo_load(now_iso8601, context={}).shift("utc").datetime == now 24 | 25 | 26 | @pytest.mark.parametrize("timezone", ["utc", "US/Eastern"]) 27 | def test_timestamp(timezone): 28 | delorean_now = delorean.Delorean(now) 29 | typedef = Timestamp(timezone) 30 | 31 | assert typedef.dynamo_dump(delorean_now, context={}) == now_timestamp 32 | assert typedef.dynamo_load(now_timestamp, context={}).shift("utc").datetime == now 33 | 34 | 35 | @pytest.mark.parametrize("typedef_cls", (DateTime, Timestamp)) 36 | def test_none(typedef_cls): 37 | typedef = typedef_cls() 38 | assert typedef.dynamo_dump(None, context={}) is None 39 | assert typedef.dynamo_load(None, context={}) is None 40 | -------------------------------------------------------------------------------- /tests/unit/test_ext/test_pendulum.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | import pendulum 4 | import pytest 5 | import pytz 6 | 7 | from bloop.ext.pendulum import DateTime, Timestamp 8 | from bloop.types import FIXED_ISO8601_FORMAT 9 | 10 | 11 | now = datetime.now(pytz.utc).replace(microsecond=0) 12 | now_eastern = datetime.now(pytz.timezone("US/Eastern")) 13 | now_iso8601 = now.strftime(FIXED_ISO8601_FORMAT) 14 | now_timestamp = str(int(now.timestamp())) 15 | 16 | 17 | @pytest.mark.parametrize("timezone", ["utc", "US/Eastern"]) 18 | def test_datetime(timezone): 19 | delorean_now = pendulum.instance(now) 20 | typedef = DateTime(timezone) 21 | 22 | assert typedef.dynamo_dump(delorean_now, context={}) == now_iso8601 23 | assert typedef.dynamo_load(now_iso8601, context={}).in_timezone("utc") == now 24 | 25 | 26 | @pytest.mark.parametrize("timezone", ["utc", "US/Eastern"]) 27 | def test_timestamp(timezone): 28 | delorean_now = pendulum.instance(now) 29 | typedef = Timestamp(timezone) 30 | 31 | assert typedef.dynamo_dump(delorean_now, context={}) == now_timestamp 32 | assert typedef.dynamo_load(now_timestamp, context={}).in_timezone("utc") == now 33 | 34 | 35 | @pytest.mark.parametrize("typedef_cls", (DateTime, Timestamp)) 36 | def test_none(typedef_cls): 37 | typedef = typedef_cls() 38 | assert typedef.dynamo_dump(None, context={}) is None 39 | assert typedef.dynamo_load(None, context={}) is None 40 | -------------------------------------------------------------------------------- /tests/unit/test_stream/__init__.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import random 3 | 4 | from bloop.stream.shard import Shard 5 | from bloop.util import Sentinel 6 | 7 | 8 | missing = Sentinel("missing") 9 | 10 | 11 | def build_shards(n, shape=None, session=None, stream_arn=None, shard_id_prefix=""): 12 | """Shape describes the parent/child relationships. 13 | 14 | a -> b -> c -> d 15 | -> e -> f 16 | 17 | is expressed as: 18 | 19 | build_shards(session, 6, {0: 1, 1: [2, 3], 2: 4, 3: 5}) 20 | """ 21 | # Default to flat shards, no hierarchy 22 | shape = shape or {} 23 | shard_id = lambda i: "{}shard-id-{}".format(shard_id_prefix + "-" if shard_id_prefix else "", i) 24 | shards = [ 25 | Shard(stream_arn=stream_arn, shard_id=shard_id(i), session=session) 26 | for i in range(n) 27 | ] 28 | for shard_index, child_indexes in shape.items(): 29 | if isinstance(child_indexes, int): 30 | shards[shard_index].children.append(shards[child_indexes]) 31 | shards[child_indexes].parent = shards[shard_index] 32 | else: 33 | for child_index in child_indexes: 34 | shards[shard_index].children.append(shards[child_index]) 35 | shards[child_index].parent = shards[shard_index] 36 | 37 | return shards 38 | 39 | 40 | def stream_description(n, shape=None, stream_arn=None): 41 | """Build a DescribeStream response with the given number of shards""" 42 | # Default to flat shards, no hierarchy 43 | shape = shape or {} 44 | 45 | shard_ids = ["shard-id-{}".format(i) for i in range(n)] 46 | template = { 47 | "SequenceNumberRange": { 48 | "EndingSequenceNumber": "820400000000000001192334", 49 | "StartingSequenceNumber": "820400000000000001192334" 50 | } 51 | } 52 | shards = [{**template, "ShardId": shard_id} for shard_id in shard_ids] 53 | 54 | for shard_index, child_indexes in shape.items(): 55 | if isinstance(child_indexes, int): 56 | shards[child_indexes]["ParentShardId"] = shard_ids[shard_index] 57 | else: 58 | for child_index in child_indexes: 59 | shards[child_index]["ParentShardId"] = shard_ids[shard_index] 60 | return { 61 | "Shards": shards, 62 | "StreamArn": stream_arn 63 | } 64 | 65 | 66 | def dynamodb_record_with(key=False, new=False, old=False, sequence_number=None, creation_time=None): 67 | if creation_time is None: 68 | creation_time = 1.46480527E9 69 | else: 70 | creation_time = creation_time.timestamp() 71 | if sequence_number is None: 72 | sequence_number = "400000000000000499660" 73 | sequence_number = str(sequence_number) 74 | creation_time = datetime.datetime.fromtimestamp(int(creation_time)) 75 | template = { 76 | "awsRegion": "us-west-2", 77 | "dynamodb": { 78 | "ApproximateCreationDateTime": creation_time, 79 | "SequenceNumber": sequence_number, 80 | "SizeBytes": 41, 81 | "StreamViewType": "KEYS_ONLY", 82 | 83 | "Keys": { 84 | "ForumName": {"S": "DynamoDB"}, 85 | "Subject": {"S": "DynamoDB Thread 1"}}, 86 | "NewImage": { 87 | "ForumName": {"S": "DynamoDB"}, 88 | "Subject": {"S": "DynamoDB Thread 1"}}, 89 | "OldImage": { 90 | "ForumName": {"S": "DynamoDB"}, 91 | "Subject": {"S": "DynamoDB Thread 1"}} 92 | }, 93 | "eventID": "4b25bd0da9a181a155114127e4837252", 94 | "eventName": "MODIFY", 95 | "eventSource": "aws:dynamodb", 96 | "eventVersion": "1.0" 97 | } 98 | if not key: 99 | del template["dynamodb"]["Keys"] 100 | if not new: 101 | del template["dynamodb"]["NewImage"] 102 | if not old: 103 | del template["dynamodb"]["OldImage"] 104 | return template 105 | 106 | 107 | def local_record(created_at=missing, sequence_number=None): 108 | if created_at is missing: 109 | created_at = datetime.datetime.now(datetime.timezone.utc) 110 | if sequence_number is None: 111 | sequence_number = str(random.randint(-100, 100)) 112 | return { 113 | "meta": { 114 | "created_at": created_at, 115 | "sequence_number": sequence_number 116 | } 117 | } 118 | 119 | 120 | def build_get_records_responses(*chain): 121 | """Return an iterable of responses for session.get_stream_records calls. 122 | 123 | Chain is the number of results to include in each page. 124 | For example: [0, 2, 1] expands into (0 results, proceed) -> (2 results, proceed) -> (1 result, stop). 125 | Very similar to the build_responses helper in test_search.py 126 | """ 127 | sequence_number = 0 128 | responses = [] 129 | for i, count in enumerate(chain): 130 | responses.append({ 131 | "Records": [ 132 | dynamodb_record_with(key=True, sequence_number=sequence_number + offset) 133 | for offset in range(count)], 134 | "NextShardIterator": "continue-from-response-{}".format(i) 135 | }) 136 | sequence_number += count 137 | # Last response doesn't point to a new iterator 138 | del responses[-1]["NextShardIterator"] 139 | 140 | return responses 141 | -------------------------------------------------------------------------------- /tests/unit/test_stream/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from bloop.stream.coordinator import Coordinator 4 | from bloop.stream.shard import Shard 5 | 6 | 7 | @pytest.fixture 8 | def stream_arn(): 9 | return "stream-arn" 10 | 11 | 12 | @pytest.fixture 13 | def shard_id(): 14 | return "shard-id" 15 | 16 | 17 | @pytest.fixture 18 | def shard(session, stream_arn, shard_id): 19 | return Shard(stream_arn=stream_arn, shard_id=shard_id, session=session) 20 | 21 | 22 | @pytest.fixture 23 | def coordinator(session, stream_arn): 24 | return Coordinator(session=session, stream_arn=stream_arn) 25 | -------------------------------------------------------------------------------- /tests/unit/test_stream/test_buffer.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from unittest.mock import Mock 3 | 4 | import pytest 5 | 6 | from bloop.stream.buffer import RecordBuffer, heap_item 7 | from bloop.stream.shard import Shard 8 | 9 | from . import local_record 10 | 11 | 12 | def new_clock(): 13 | x = 0 14 | 15 | def call(): 16 | nonlocal x 17 | x += 1 18 | return x 19 | return call 20 | 21 | 22 | def now(): 23 | return datetime.datetime.now(datetime.timezone.utc) 24 | 25 | 26 | def new_shard() -> Shard: 27 | return Mock(spec=Shard) 28 | 29 | 30 | @pytest.mark.parametrize("created_at", [None, now()]) 31 | @pytest.mark.parametrize("sequence_number", ["2", "140"]) 32 | def test_heap_item_clock(created_at, sequence_number): 33 | """heap_item guarantees total ordering, even for identical items.""" 34 | shard = new_shard() 35 | clock = new_clock() 36 | 37 | record = local_record(created_at, sequence_number) 38 | 39 | first_item = heap_item(clock, record, shard) 40 | second_item = heap_item(clock, record, shard) 41 | 42 | assert first_item < second_item 43 | # Same payload - heap_item returns tuple of (ordering, payload) 44 | assert first_item[1] == second_item[1] 45 | 46 | # Called twice before this 47 | assert clock() == 3 48 | 49 | 50 | @pytest.mark.parametrize("created_at", [None, now()]) 51 | @pytest.mark.parametrize("sequence_number", ["0", "12"]) 52 | def test_heap_item_broken_clock(created_at, sequence_number): 53 | """When the clock can return the same value, total ordering is lost.""" 54 | shard = new_shard() 55 | broken_clock = lambda: 4 56 | 57 | record = local_record(created_at, sequence_number) 58 | 59 | first_item = heap_item(broken_clock, record, shard) 60 | second_item = heap_item(broken_clock, record, shard) 61 | assert first_item == second_item 62 | 63 | 64 | def test_empty_buffer(): 65 | """Trying to access an empty buffer raises IndexError""" 66 | buffer = RecordBuffer() 67 | 68 | assert not buffer 69 | with pytest.raises(IndexError): 70 | buffer.pop() 71 | with pytest.raises(IndexError): 72 | buffer.peek() 73 | 74 | 75 | def test_single_record(): 76 | """Push a record, peek at it, then get the same thing back""" 77 | record = local_record(now(), "1") 78 | shard = new_shard() 79 | buffer = RecordBuffer() 80 | 81 | buffer.push(record, shard) 82 | assert buffer 83 | 84 | same_record, same_shard = buffer.peek() 85 | 86 | also_same_record, also_same_shard = buffer.pop() 87 | assert not buffer 88 | 89 | assert record is same_record is also_same_record 90 | assert shard is same_shard is also_same_shard 91 | 92 | 93 | def test_sort_every_push(): 94 | """Push high to low, retrieve low to high""" 95 | now_ = now() 96 | records = [local_record(now_, str(i)) for i in reversed(range(15))] 97 | shard = new_shard() 98 | buffer = RecordBuffer() 99 | 100 | for record in records: 101 | buffer.push(record, shard) 102 | # inserting high to low, every record should be at the front 103 | assert buffer.peek()[0] is record 104 | 105 | same_records = [ 106 | buffer.pop()[0] 107 | for _ in range(len(records)) 108 | ] 109 | same_records.reverse() 110 | assert records == same_records 111 | 112 | 113 | def test_push_all(): 114 | """Bulk push is slightly more efficient""" 115 | now_ = now() 116 | records = [local_record(now_, str(i)) for i in reversed(range(100))] 117 | shard = new_shard() 118 | buffer = RecordBuffer() 119 | 120 | pairs = [(record, shard) for record in records] 121 | buffer.push_all(pairs) 122 | 123 | same_records = [ 124 | buffer.pop()[0] 125 | for _ in range(len(records)) 126 | ] 127 | same_records.reverse() 128 | assert records == same_records 129 | 130 | 131 | def test_clear(): 132 | record = local_record(now(), "1") 133 | shard = new_shard() 134 | buffer = RecordBuffer() 135 | 136 | buffer.push(record, shard) 137 | assert buffer 138 | 139 | buffer.clear() 140 | assert not buffer 141 | 142 | 143 | def test_buffer_heap(): 144 | """RecordBuffer directly exposes its heap""" 145 | record = local_record(now(), "1") 146 | shard = new_shard() 147 | buffer = RecordBuffer() 148 | 149 | buffer.push(record, shard) 150 | 151 | # [(sort, record, shard)] 152 | assert buffer.heap[0][2] is shard 153 | -------------------------------------------------------------------------------- /tests/unit/test_stream/test_stream.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from unittest.mock import MagicMock 3 | 4 | import pytest 5 | 6 | from bloop.models import BaseModel, Column 7 | from bloop.stream.coordinator import Coordinator 8 | from bloop.stream.stream import Stream 9 | from bloop.types import Integer, String 10 | from bloop.util import ordered 11 | 12 | from . import build_shards 13 | 14 | 15 | @pytest.fixture 16 | def coordinator(): 17 | # MagicMock because we're testing __next__ 18 | return MagicMock(spec=Coordinator) 19 | 20 | 21 | @pytest.fixture 22 | def stream(coordinator, engine): 23 | stream = Stream(model=Email, engine=engine) 24 | stream.coordinator = coordinator 25 | return stream 26 | 27 | 28 | class Email(BaseModel): 29 | class Meta: 30 | stream = { 31 | "include": {"new", "old"}, 32 | "arn": "stream-arn" 33 | } 34 | id = Column(Integer, hash_key=True) 35 | data = Column(String) 36 | 37 | 38 | def test_repr(stream): 39 | assert repr(stream) == "" 40 | 41 | 42 | def test_iter(stream): 43 | """stream is both an Iterable and an Iterator""" 44 | assert iter(stream) is stream 45 | 46 | 47 | def test_token(engine): 48 | engine.bind(Email) 49 | shards = build_shards(3, {0: [1, 2]}, stream_arn=Email.Meta.stream["arn"]) 50 | shards[1].iterator_type = "latest" 51 | shards[2].iterator_type = "at_sequence" 52 | shards[2].sequence_number = "sequence-number" 53 | 54 | stream = Stream(model=Email, engine=engine) 55 | stream.coordinator.roots.append(shards[0]) 56 | stream.coordinator.active.extend(shards[1:]) 57 | 58 | assert ordered(stream.token) == ordered({ 59 | "stream_arn": "stream-arn", 60 | "active": ["shard-id-1", "shard-id-2"], 61 | "shards": [ 62 | {"shard_id": "shard-id-0"}, 63 | {"shard_id": "shard-id-1", "parent": "shard-id-0", "iterator_type": "latest"}, 64 | {"shard_id": "shard-id-2", "parent": "shard-id-0", 65 | "iterator_type": "at_sequence", "sequence_number": "sequence-number"}, 66 | ] 67 | }) 68 | 69 | 70 | def test_heartbeat(stream, coordinator): 71 | stream.heartbeat() 72 | coordinator.heartbeat.assert_called_once_with() 73 | 74 | 75 | def test_move_to(stream, coordinator): 76 | stream.move_to("latest") 77 | coordinator.move_to.assert_called_once_with("latest") 78 | 79 | 80 | def test_next_no_record(stream, coordinator): 81 | coordinator.__next__.return_value = None 82 | # Explicit marker so we don't get next's default value 83 | missing = object() 84 | 85 | record = next(stream, missing) 86 | assert record is None 87 | 88 | 89 | def test_next_unpacks(stream, coordinator): 90 | now = datetime.datetime.now(datetime.timezone.utc) 91 | meta = { 92 | "created_at": now, 93 | "sequence_number": "sequence-number", 94 | "event": { 95 | "id": "event-id", 96 | "type": "event-type", 97 | "version": "event-version" 98 | } 99 | } 100 | coordinator.__next__.return_value = { 101 | # Impossible to have old and key, but for the sake of testing 102 | # an object that's partially/fully loaded 103 | "old": { 104 | "id": {"N": "0"}, 105 | "data": {"S": "some-data"} 106 | }, 107 | "key": { 108 | # Omitted because the model only includes "new" 109 | "id": {"N": "343"} 110 | }, 111 | "new": None, 112 | "meta": meta 113 | } 114 | 115 | record = next(stream) 116 | 117 | assert record["old"].id == 0 118 | assert record["old"].data == "some-data" 119 | 120 | assert record["new"] is None 121 | 122 | assert record["key"] is None 123 | assert not hasattr(record["key"], "data") 124 | -------------------------------------------------------------------------------- /tests/unit/test_transactions.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta, timezone 2 | from unittest.mock import Mock 3 | 4 | import pytest 5 | from tests.helpers.models import User 6 | 7 | from bloop.exceptions import MissingObjects, TransactionTokenExpired 8 | from bloop.signals import object_deleted, object_loaded, object_saved 9 | from bloop.transactions import ( 10 | MAX_TOKEN_LIFETIME, 11 | MAX_TRANSACTION_ITEMS, 12 | PreparedTransaction, 13 | ReadTransaction, 14 | Transaction, 15 | TxItem, 16 | TxType, 17 | WriteTransaction, 18 | ) 19 | 20 | 21 | class NoopTransaction(Transaction): 22 | 23 | def __init__(self, engine): 24 | super().__init__(engine) 25 | self.prepared = Mock(spec=PreparedTransaction) 26 | 27 | def prepare(self): 28 | return self.prepared 29 | 30 | 31 | @pytest.fixture 32 | def wx(engine): 33 | """prepared write tx with one item""" 34 | user = User(id="numberoverzero") 35 | other = User(id="other") 36 | items = [ 37 | TxItem.new("save", user, condition=User.id.is_(None)), 38 | TxItem.new("delete", other), 39 | TxItem.new("check", other, condition=User.email.begins_with("foo")) 40 | ] 41 | tx = PreparedTransaction() 42 | tx.prepare(engine, "w", items) 43 | return tx 44 | 45 | 46 | @pytest.fixture 47 | def rx(engine): 48 | """prepared read tx with one item""" 49 | user = User(id="numberoverzero") 50 | items = [TxItem.new("get", user)] 51 | tx = PreparedTransaction() 52 | tx.prepare(engine, "r", items) 53 | return tx 54 | 55 | 56 | @pytest.mark.parametrize("type, expected", [ 57 | (TxType.Get, False), 58 | (TxType.Check, False), 59 | (TxType.Delete, False), 60 | (TxType.Update, True), 61 | ]) 62 | def test_is_update(type, expected): 63 | item = TxItem(type=type, obj=None, condition=None) 64 | assert item.is_update is expected 65 | 66 | 67 | @pytest.mark.parametrize("type, expected", [ 68 | (TxType.Get, False), 69 | (TxType.Check, False), 70 | (TxType.Delete, True), 71 | (TxType.Update, True), 72 | ]) 73 | def test_should_render_obj(type, expected): 74 | item = TxItem(type=type, obj=None, condition=None) 75 | assert item.should_render_obj is expected 76 | 77 | 78 | def test_nested_ctx(engine): 79 | """Transaction.__exit__ should only call commit on the outer context""" 80 | tx = NoopTransaction(engine) 81 | 82 | with tx: 83 | with tx: 84 | with tx: 85 | pass 86 | tx.prepared.commit.assert_called_once() 87 | 88 | 89 | def test_no_commit_during_exception(engine): 90 | """Transaction.__exit__ shouldn't commit if the block raised an exception""" 91 | tx = NoopTransaction(engine) 92 | with pytest.raises(ZeroDivisionError): 93 | with tx: 94 | raise ZeroDivisionError 95 | tx.prepared.commit.assert_not_called() 96 | 97 | 98 | def test_extend(engine): 99 | """Each Transaction can hold transactions.MAX_TRANSACTION_ITEMS items""" 100 | tx = Transaction(engine) 101 | for _ in range(MAX_TRANSACTION_ITEMS): 102 | tx._extend([object()]) 103 | with pytest.raises(RuntimeError): 104 | tx._extend([object()]) 105 | 106 | 107 | def test_read_item(engine): 108 | engine.bind(User) 109 | user = User(id="numberoverzero") 110 | tx = ReadTransaction(engine) 111 | 112 | tx.load(user) 113 | p = tx.prepare() 114 | 115 | expected_items = [TxItem.new("get", user, None)] 116 | assert tx._items == expected_items 117 | assert p.items == expected_items 118 | assert p.first_commit_at is None 119 | 120 | 121 | def test_check_complex_item(engine): 122 | engine.bind(User) 123 | user = User(id="numberoverzero") 124 | tx = WriteTransaction(engine) 125 | 126 | condition = User.id.begins_with("foo") 127 | tx.check(user, condition=condition) 128 | p = tx.prepare() 129 | 130 | expected_items = [TxItem.new("check", user, condition)] 131 | assert tx._items == expected_items 132 | assert p.items == expected_items 133 | assert p.first_commit_at is None 134 | assert len(p._request) == 1 135 | entry = p._request[0]["CheckCondition"] 136 | expected_fields = { 137 | "Key", "TableName", 138 | "ConditionExpression", 139 | "ExpressionAttributeNames", 140 | "ExpressionAttributeValues" 141 | } 142 | assert set(entry.keys()) == expected_fields 143 | 144 | 145 | def test_save_complex_item(engine): 146 | engine.bind(User) 147 | user = User(id="numberoverzero") 148 | tx = WriteTransaction(engine) 149 | 150 | condition = User.id.begins_with("foo") 151 | tx.save(user, condition=condition) 152 | p = tx.prepare() 153 | 154 | expected_items = [TxItem.new("save", user, condition)] 155 | assert tx._items == expected_items 156 | assert p.items == expected_items 157 | assert p.first_commit_at is None 158 | assert len(p._request) == 1 159 | entry = p._request[0]["Update"] 160 | expected_fields = { 161 | "Key", "TableName", 162 | "ConditionExpression", 163 | "ExpressionAttributeNames", 164 | "ExpressionAttributeValues" 165 | } 166 | assert set(entry.keys()) == expected_fields 167 | 168 | 169 | def test_delete_complex_item(engine): 170 | engine.bind(User) 171 | user = User(id="numberoverzero") 172 | tx = WriteTransaction(engine) 173 | 174 | condition = User.id.begins_with("foo") 175 | tx.delete(user, condition=condition) 176 | p = tx.prepare() 177 | 178 | expected_items = [TxItem.new("delete", user, condition)] 179 | assert tx._items == expected_items 180 | assert p.items == expected_items 181 | assert p.first_commit_at is None 182 | assert len(p._request) == 1 183 | entry = p._request[0]["Delete"] 184 | expected_fields = { 185 | "Key", "TableName", 186 | "ConditionExpression", 187 | "ExpressionAttributeNames", 188 | "ExpressionAttributeValues" 189 | } 190 | assert set(entry.keys()) == expected_fields 191 | 192 | 193 | def test_commit_bad_mode(rx): 194 | rx.mode = "j" 195 | with pytest.raises(ValueError): 196 | rx.commit() 197 | 198 | 199 | def test_write_commit_expired(wx, session): 200 | now = datetime.now(timezone.utc) 201 | offset = MAX_TOKEN_LIFETIME + timedelta(seconds=1) 202 | wx.first_commit_at = now - offset 203 | 204 | with pytest.raises(TransactionTokenExpired): 205 | wx.commit() 206 | 207 | session.transaction_write.assert_not_called() 208 | 209 | 210 | def test_read_commit(rx, session): 211 | """read commits don't expire""" 212 | calls = {"loaded": 0} 213 | 214 | @object_loaded.connect 215 | def on_loaded(*_, **__): 216 | calls["loaded"] += 1 217 | 218 | session.transaction_read.return_value = { 219 | "Responses": [ 220 | { 221 | "Item": { 222 | "id": {"S": "numberoverzero"}, 223 | "age": {"N": "3"} 224 | } 225 | } 226 | ] 227 | } 228 | 229 | now = datetime.now(timezone.utc) 230 | offset = MAX_TOKEN_LIFETIME + timedelta(seconds=1) 231 | rx.first_commit_at = now - offset 232 | rx.commit() 233 | 234 | session.transaction_read.assert_called_once_with(rx._request) 235 | assert rx.items[0].obj.age == 3 236 | assert calls["loaded"] == 1 237 | 238 | 239 | def test_write_commit(wx, session): 240 | calls = { 241 | "saved": 0, 242 | "deleted": 0 243 | } 244 | 245 | @object_saved.connect 246 | def on_saved(*_, **__): 247 | calls["saved"] += 1 248 | 249 | @object_deleted.connect 250 | def on_deleted(*_, **__): 251 | calls["deleted"] += 1 252 | 253 | now = datetime.now(timezone.utc) 254 | wx.commit() 255 | 256 | session.transaction_write.assert_called_once_with(wx._request, wx.tx_id) 257 | assert (wx.first_commit_at - now) <= timedelta(seconds=1) 258 | assert calls["saved"] == 1 259 | assert calls["deleted"] == 1 260 | 261 | 262 | def test_malformed_read_response(rx, session): 263 | session.transaction_read.return_value = {"Responses": []} 264 | with pytest.raises(RuntimeError): 265 | rx.commit() 266 | 267 | 268 | def test_read_missing_object(rx, session): 269 | session.transaction_read.return_value = {"Responses": [{}]} 270 | with pytest.raises(MissingObjects) as excinfo: 271 | rx.commit() 272 | 273 | obj = rx.items[0].obj 274 | assert excinfo.value.objects == [obj] 275 | -------------------------------------------------------------------------------- /tests/unit/test_util.py: -------------------------------------------------------------------------------- 1 | import collections 2 | 3 | import pytest 4 | from tests.helpers.models import User 5 | 6 | from bloop.actions import ActionType 7 | from bloop.engine import Engine 8 | from bloop.exceptions import MissingKey 9 | from bloop.models import BaseModel, Column 10 | from bloop.types import Integer 11 | from bloop.util import ( 12 | Sentinel, 13 | default_context, 14 | dump_key, 15 | extract_key, 16 | get_table_name, 17 | index, 18 | index_for, 19 | ordered, 20 | value_of, 21 | walk_subclasses, 22 | ) 23 | 24 | 25 | class HashAndRange(BaseModel): 26 | class Meta: 27 | abstract = True 28 | foo = Column(Integer, hash_key=True) 29 | bar = Column(Integer, range_key=True) 30 | 31 | 32 | def test_index(): 33 | """Index by each object's value for an attribute""" 34 | class Person: 35 | def __init__(self, name): 36 | self.name = name 37 | 38 | p1, p2, p3 = Person("foo"), Person("bar"), Person("baz") 39 | assert index([p1, p2, p3], "name") == { 40 | "foo": p1, 41 | "bar": p2, 42 | "baz": p3 43 | } 44 | 45 | 46 | def test_dump_key(engine): 47 | user = User(id="foo") 48 | user_key = {"id": {"S": "foo"}} 49 | assert dump_key(engine, user) == user_key 50 | 51 | obj = HashAndRange(foo=4, bar=5) 52 | obj_key = {"bar": {"N": "5"}, "foo": {"N": "4"}} 53 | assert dump_key(engine, obj) == obj_key 54 | 55 | 56 | def test_dump_key_missing(engine): 57 | obj = HashAndRange() 58 | with pytest.raises(MissingKey): 59 | dump_key(engine, obj) 60 | 61 | 62 | @pytest.mark.parametrize("action_type", [t for t in ActionType if t is not ActionType.Set]) 63 | def test_dump_invalid_key_action(engine, action_type): 64 | obj = HashAndRange( 65 | foo=action_type.new_action(2), 66 | bar=3 67 | ) 68 | with pytest.raises(ValueError): 69 | dump_key(engine, obj) 70 | 71 | 72 | def test_extract_key(): 73 | key_shape = "foo", "bar" 74 | item = {"baz": 1, "bar": 2, "foo": 3} 75 | expected = {"foo": 3, "bar": 2} 76 | assert extract_key(key_shape, item) == expected 77 | 78 | 79 | def test_get_table_name(dynamodb, dynamodbstreams): 80 | def transform_table_name(model): 81 | return f"transform.{model.Meta.table_name}" 82 | 83 | engine = Engine( 84 | dynamodb=dynamodb, dynamodbstreams=dynamodbstreams, 85 | table_name_template=transform_table_name) 86 | obj = HashAndRange() 87 | assert get_table_name(engine, obj) == "transform.HashAndRange" 88 | 89 | 90 | @pytest.mark.parametrize("obj", [None, object(), 2, False, "abc"]) 91 | def test_ordered_basic_objects(obj): 92 | """Things that don't need to be unpacked or flattened for comparison""" 93 | assert ordered(obj) is obj 94 | 95 | 96 | @pytest.mark.parametrize("it", [ 97 | iter(list("bac")), 98 | ["b", "c", "a"], 99 | ("c", "a", "b"), 100 | (x for x in "cba"), 101 | {"a", "c", "b"} 102 | ]) 103 | def test_ordered_iterable(it): 104 | """Any non-mapping iterable is sorted, even if it's consumable""" 105 | expected = ["a", "b", "c"] 106 | assert ordered(it) == expected 107 | 108 | 109 | @pytest.mark.parametrize("mapping", [ 110 | {"b": True, "a": "zebra", "c": None}, 111 | collections.OrderedDict([("c", None), ("b", True), ("a", "zebra")]) 112 | ]) 113 | def test_ordered_mapping(mapping): 114 | """Mappings are flattened into (key, value) tuples and then those tuples are sorted""" 115 | expected = [ 116 | ("a", "zebra"), 117 | ("b", True), 118 | ("c", None) 119 | ] 120 | assert ordered(mapping) == expected 121 | 122 | 123 | @pytest.mark.parametrize("obj, expected", [ 124 | # mapping int -> set(str) 125 | ({3: {"a", "b"}, 2: {"c", "b"}, 1: {"a", "c"}}, [(1, ["a", "c"]), (2, ["b", "c"]), (3, ["a", "b"])]), 126 | # mapping str -> list(int) 127 | ({"b": [1, 2], "a": [3, 2], "c": [1, 3]}, [("a", [2, 3]), ("b", [1, 2]), ("c", [1, 3])]), 128 | # list(set(bool)) 129 | ([{False}, {True}], [[False], [True]]), 130 | ]) 131 | def test_ordered_recursion(obj, expected): 132 | """Mappings and iterables inside each other are sorted and flattened""" 133 | assert ordered(obj) == expected 134 | 135 | 136 | def test_walk_subclasses(): 137 | class A: 138 | pass 139 | 140 | class B: # Not included 141 | pass 142 | 143 | class C(A): 144 | pass 145 | 146 | class D(A): 147 | pass 148 | 149 | class E(C, A): # would be visited twice without dedupe 150 | pass 151 | 152 | class F(D, A): # would be visited twice without dedupe 153 | pass 154 | 155 | # list instead of set ensures we don't false succeed on duplicates 156 | subclasses = sorted(walk_subclasses(A), key=lambda c: c.__name__) 157 | assert subclasses == [C, D, E, F] 158 | 159 | 160 | def test_value_of(): 161 | column = {"S": "Space Invaders"} 162 | assert value_of(column) == "Space Invaders" 163 | 164 | 165 | def test_index_for_sorts(): 166 | key = { 167 | "f": {"S": "foo"}, 168 | "b": {"S": "bar"}, 169 | } 170 | same_key = { 171 | "b": {"S": "bar"}, 172 | "f": {"S": "foo"}, 173 | } 174 | assert index_for(key) == index_for(same_key) 175 | 176 | 177 | def test_sentinel_uniqueness(): 178 | sentinel = Sentinel("name") 179 | same_sentinel = Sentinel("NAME") 180 | assert sentinel is same_sentinel 181 | 182 | 183 | def test_sentinel_repr(): 184 | foo = Sentinel("foo") 185 | assert repr(foo) == "" 186 | 187 | 188 | def test_default_context(engine): 189 | ctx = default_context(engine) 190 | assert ctx["engine"] is engine 191 | 192 | other_engine = object() 193 | other_ctx = {"engine": other_engine} 194 | ctx = default_context(engine, other_ctx) 195 | assert ctx is other_ctx 196 | assert ctx["engine"] is other_engine 197 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = unit, integ, docs 3 | 4 | [testenv] 5 | basepython = python3 6 | deps = -rrequirements.txt 7 | 8 | [testenv:unit] 9 | commands = 10 | coverage run --branch --source=bloop -m pytest tests/unit {posargs} 11 | coverage report -m 12 | flake8 bloop tests examples 13 | 14 | [testenv:integ] 15 | commands = pytest tests/integ -vv {posargs} 16 | 17 | [testenv:docs] 18 | changedir = docs 19 | commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html 20 | 21 | 22 | [flake8] 23 | ignore = E731,W504,Q000 24 | max-line-length = 119 25 | --------------------------------------------------------------------------------