├── .github └── workflows │ └── test.yml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README ├── README.rst ├── setup.py ├── tests.py ├── tinyrecord ├── __init__.py ├── changeset.py ├── operations.py ├── py.typed └── transaction.py └── tox.ini /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: [push, pull_request, workflow_dispatch] 4 | 5 | env: 6 | FORCE_COLOR: 1 7 | 8 | jobs: 9 | test: 10 | runs-on: ${{ matrix.os }} 11 | strategy: 12 | fail-fast: false 13 | matrix: 14 | python-version: ["pypy3.8", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11-dev"] 15 | os: [ubuntu-latest, macos-latest, windows-latest] 16 | 17 | steps: 18 | - uses: actions/checkout@v3 19 | 20 | - name: Set up Python ${{ matrix.python-version }} 21 | uses: actions/setup-python@v4 22 | with: 23 | python-version: ${{ matrix.python-version }} 24 | cache: pip 25 | cache-dependency-path: setup.py 26 | 27 | - name: Install dependencies 28 | run: | 29 | python -m pip install -U pip 30 | python -m pip install -U wheel 31 | python -m pip install -U tox 32 | 33 | - name: Tox tests 34 | run: | 35 | tox -e py 36 | 37 | - name: Tox MyPy 38 | if: "!startsWith(matrix.python-version, 'pypy')" 39 | run: | 40 | tox -e mypy 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | eggs/ 15 | lib/ 16 | lib64/ 17 | parts/ 18 | sdist/ 19 | var/ 20 | *.egg-info/ 21 | .installed.cfg 22 | *.egg 23 | 24 | # PyInstaller 25 | # Usually these files are written by a python script from a template 26 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 27 | *.manifest 28 | *.spec 29 | 30 | # Installer logs 31 | pip-log.txt 32 | pip-delete-this-directory.txt 33 | 34 | # Unit test / coverage reports 35 | htmlcov/ 36 | .tox/ 37 | .coverage 38 | .cache 39 | nosetests.xml 40 | coverage.xml 41 | 42 | # Translations 43 | *.mo 44 | *.pot 45 | 46 | # Django stuff: 47 | *.log 48 | 49 | # Sphinx documentation 50 | docs/_build/ 51 | 52 | # PyBuilder 53 | target/ 54 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2014 Eugene Eeo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | include LICENSE 3 | global-exclude *.orig *.py[co] *.log *.swp 4 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | README.rst -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | TinyDB __ _ __ 4 | / /_(_)__ __ _________ _______ _______/ / 5 | / __/ / _ \/ // / __/ -_) __/ _ \/ __/ _ / 6 | \__/_/_//_/\_, /_/ \__/\__/\___/_/ \_,_/ 7 | /___/ 8 | 9 | 10 | **Supported Pythons:** 3.6+ 11 | 12 | Tinyrecord is a library which implements atomic 13 | transaction support for the `TinyDB`_ NoSQL database. 14 | It uses a record-first then execute architecture which 15 | allows us to minimize the time that we are within a 16 | thread lock. Usage example: 17 | 18 | .. code-block:: python 19 | 20 | from tinydb import TinyDB, where 21 | from tinyrecord import transaction 22 | 23 | table = TinyDB('db.json').table('table') 24 | with transaction(table) as tr: 25 | # insert a new record 26 | tr.insert({'username': 'john'}) 27 | # update records matching a query 28 | tr.update({'invalid': True}, where('username') == 'john') 29 | # delete records 30 | tr.remove(where('invalid') == True) 31 | # update using a function 32 | tr.update(updater, where(...)) 33 | # insert many items 34 | tr.insert_multiple(documents) 35 | 36 | Note that due to performance reasons you cannot view 37 | the data within a transaction unless you've committed. 38 | You will have to call operations on the transaction 39 | object and not the database itself. Since tinyrecord 40 | works with dictionaries and the latest API, it will 41 | only support the latest version (**4.x**). 42 | 43 | Installation is as simple as ``pip install tinyrecord``. 44 | 45 | .. image:: https://github.com/eugene-eeo/tinyrecord/actions/workflows/test.yml/badge.svg 46 | :target: https://github.com/eugene-eeo/tinyrecord/actions/workflows/test.yml 47 | .. _TinyDB: https://github.com/msiemens/tinydb 48 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from setuptools import setup 3 | 4 | classifiers = """\ 5 | Development Status :: 5 - Production/Stable 6 | Intended Audience :: Developers 7 | License :: OSI Approved :: MIT License 8 | Programming Language :: Python 9 | Programming Language :: Python :: 3 10 | Programming Language :: Python :: 3.6 11 | Programming Language :: Python :: 3.7 12 | Programming Language :: Python :: 3.8 13 | Programming Language :: Python :: 3.9 14 | Programming Language :: Python :: 3.10 15 | Programming Language :: Python :: 3.11 16 | Programming Language :: Python :: 3 :: Only 17 | Topic :: Software Development :: Libraries :: Python Modules 18 | Operating System :: Microsoft :: Windows 19 | Operating System :: Unix 20 | Operating System :: MacOS :: MacOS X 21 | """ 22 | 23 | curr_path = os.path.abspath(os.path.dirname(__file__)) 24 | setup( 25 | name='tinyrecord', 26 | version='0.2.1', 27 | packages=['tinyrecord'], 28 | package_data={'tinyrecord': ['py.typed']}, 29 | python_requires='>=3.6', 30 | install_requires=['tinydb >= 4.0.0'], 31 | classifiers=filter(None, classifiers.split('\n')), 32 | zip_safe=True, 33 | author='Eugene Eeo', 34 | author_email='141bytes@gmail.com', 35 | long_description=open(os.path.join(curr_path, 'README.rst'), 'r').read(), 36 | long_description_content_type='text/x-rst', 37 | description='Atomic transactions for TinyDB', 38 | license='MIT', 39 | keywords='tinydb nosql database transaction', 40 | url='https://github.com/eugene-eeo/tinyrecord', 41 | ) 42 | -------------------------------------------------------------------------------- /tests.py: -------------------------------------------------------------------------------- 1 | from pytest import fixture, raises 2 | from threading import Thread 3 | from tinydb import where, TinyDB 4 | from tinydb.storages import MemoryStorage 5 | from tinyrecord import transaction, abort 6 | 7 | 8 | @fixture 9 | def db(): 10 | return TinyDB(storage=MemoryStorage).table('table') 11 | 12 | 13 | def test_insert_multiple(db): 14 | with transaction(db) as tr: 15 | tr.insert_multiple({} for x in range(5)) 16 | 17 | assert len(db) == 5 18 | 19 | 20 | def test_update_callable(db): 21 | match_all = lambda x: True # noqa: E731 22 | [db.insert({'x': {'u': 10}}) for i in range(5)] 23 | 24 | with transaction(db) as tr: 25 | def function(t): 26 | t['x']['u'] = 1 27 | tr.update(function, match_all) 28 | 29 | assert len(db) == 5 30 | assert all(x['x']['u'] == 1 for x in db.search(match_all)) 31 | 32 | 33 | def test_remove(db): 34 | [db.insert({}) for i in range(10)] 35 | anything = lambda x: True # noqa: E731 36 | db.search(anything) 37 | 38 | with transaction(db) as tr: 39 | tr.remove(anything) 40 | 41 | assert not db._query_cache 42 | assert len(db) == 0 43 | 44 | db.insert({}) 45 | assert db.get(anything) == {} 46 | 47 | 48 | def test_remove_doc_ids(db): 49 | doc_id = db.insert({'x': 1}) 50 | other_doc_id = db.insert({'x': 4}) 51 | 52 | with transaction(db) as tr: 53 | tr.remove(doc_ids=[doc_id]) 54 | 55 | assert not db.get(doc_id=doc_id) 56 | assert db.get(doc_id=other_doc_id) 57 | 58 | 59 | def test_update(db): 60 | doc_id = db.insert({'x': 1}) 61 | other_doc_id = db.insert({'x': 4}) 62 | 63 | with transaction(db) as tr: 64 | tr.update({'x': 2}, where('x') == 1) 65 | tr.update({'x': 3}, doc_ids=[doc_id]) 66 | 67 | assert db.get(where('x') == 3).doc_id == doc_id 68 | assert db.get(where('x') == 4).doc_id == other_doc_id 69 | 70 | 71 | def test_atomicity(db): 72 | with raises(ValueError): 73 | with transaction(db) as tr: 74 | tr.insert({}) 75 | tr.insert({'x': 1}) 76 | tr.update({'x': 2}, where('x') == 1) 77 | raise ValueError 78 | assert len(db) == 0 79 | 80 | 81 | def test_abort(db): 82 | with transaction(db) as tr: 83 | tr.insert({}) 84 | abort() 85 | 86 | assert len(db) == 0 87 | 88 | 89 | def test_insert(db): 90 | with transaction(db) as tr: 91 | tr.insert({}) 92 | assert len(db) == 1 93 | 94 | 95 | def test_concurrent(db): 96 | def callback(): 97 | with transaction(db) as tr: 98 | tr.insert({}) 99 | tr.insert({}) 100 | 101 | threads = [Thread(target=callback) for i in range(10)] 102 | [thread.start() for thread in threads] 103 | [thread.join() for thread in threads] 104 | 105 | ids = {x.doc_id for x in db.all()} 106 | assert len(ids) == 20 107 | 108 | 109 | def test_raise(db): 110 | db.insert({"1": 1}) 111 | db.insert({"2": 2}) 112 | values = db._storage.read() 113 | 114 | def bad(x): 115 | raise Exception("wtf!") 116 | 117 | with raises(Exception): 118 | with transaction(db) as tr: 119 | tr.insert({"3": 3}) 120 | tr.insert({"4": 4}) 121 | tr.update({"2": 3}, doc_ids=[2]) 122 | tr.update({}, bad) 123 | 124 | assert len(db) == 2 125 | assert values == db._storage.read() 126 | -------------------------------------------------------------------------------- /tinyrecord/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | tinyrecord 3 | ~~~~~~~~~~ 4 | 5 | Implements atomic transaction support for the 6 | embedded TinyDB NoSQL database. 7 | 8 | :copyright: (c) 2014 by Eugene Eeo. 9 | :license: MIT, see LICENSE for more details. 10 | """ 11 | 12 | 13 | from tinyrecord.transaction import transaction, abort 14 | -------------------------------------------------------------------------------- /tinyrecord/changeset.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, List 2 | 3 | from tinydb.table import Table 4 | 5 | from tinyrecord.operations import Operation 6 | 7 | 8 | class Changeset: 9 | """ 10 | A changeset represents a series of changes that 11 | can be applied to the *database*. 12 | 13 | :param table: The TinyDB table. 14 | """ 15 | 16 | def __init__(self, table: Table) -> None: 17 | self.table = table 18 | self.record: List[Operation] = [] 19 | 20 | def execute(self) -> None: 21 | """ 22 | Execute the changeset, applying every 23 | operation on the database. Note that this 24 | function is not idempotent, if you call 25 | it again and again it will be executed 26 | many times. 27 | """ 28 | def updater(docs: Dict[int, Any]) -> None: 29 | for op in self.record: 30 | op.perform(docs) 31 | 32 | self.table._update_table(updater) 33 | self.table._next_id = None 34 | 35 | def append(self, change: Operation) -> None: 36 | """ 37 | Append a *change* to the internal record 38 | of operations. 39 | 40 | :param change: The change to append. 41 | """ 42 | self.record.append(change) 43 | -------------------------------------------------------------------------------- /tinyrecord/operations.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, Dict, Iterable, Optional 2 | 3 | QueryFunction = Callable[[Any], bool] 4 | 5 | 6 | class Operation: 7 | """ 8 | An operation represents a single, atomic 9 | sequence of things to do to in-memory data. 10 | Every operation must implement the abstract 11 | ``perform`` method. 12 | """ 13 | def perform(self, data: Dict[int, Any]) -> None: 14 | raise NotImplementedError 15 | 16 | 17 | class InsertMultiple(Operation): 18 | """ 19 | Insert multiple records *iterable* into the 20 | database. 21 | 22 | :param iterable: The iterable of elements to 23 | be inserted into the DB. 24 | """ 25 | def __init__(self, iterable: Iterable[Any]) -> None: 26 | self.iterable = iterable 27 | 28 | def perform(self, data: Dict[int, Any]) -> None: 29 | doc_id = max(data) if data else 0 30 | for element in self.iterable: 31 | doc_id += 1 32 | data[doc_id] = element 33 | 34 | 35 | class Update(Operation): 36 | """ 37 | Mutate each of the records with a given 38 | *function* for all records that match a 39 | certain *query* (if specified). If *doc_ids* 40 | is specified, then the update is performed 41 | over those which have doc_id in *doc_ids*. 42 | 43 | :param function: Updator function, or a dictionary 44 | of fields to update. 45 | :param query: Query function. 46 | :param doc_ids: Iterable of document IDs. 47 | """ 48 | def __init__( 49 | self, 50 | function: Callable[[Any], Any], 51 | query: Optional[QueryFunction] = None, 52 | doc_ids: Optional[Iterable[int]] = None 53 | ) -> None: 54 | if query is None and doc_ids is None: 55 | raise TypeError("query or doc_ids must be specified") 56 | self.function = function if callable(function) else \ 57 | lambda x: x.update(function) 58 | self.query = query 59 | self.doc_ids = doc_ids 60 | 61 | def perform(self, data: Dict[int, Any]) -> None: 62 | if self.query is not None: 63 | for key in data: 64 | value = data[key] 65 | if self.query(value): 66 | self.function(value) 67 | else: 68 | assert self.doc_ids is not None 69 | for key, value in data.items(): 70 | if key in self.doc_ids: 71 | self.function(value) 72 | 73 | 74 | class Remove(Operation): 75 | """ 76 | Remove documents from the DB matching 77 | the given *query*, or alternatively if 78 | *doc_ids* is specified, then those which 79 | have the given doc_ids. 80 | 81 | :param query: Query. 82 | :param doc_ids: Document ids. 83 | """ 84 | def __init__( 85 | self, 86 | query: Optional[QueryFunction] = None, 87 | doc_ids: Optional[Iterable[int]] = None 88 | ) -> None: 89 | if query is None and doc_ids is None: 90 | raise TypeError("query or doc_ids must be specified") 91 | self.query = query 92 | self.doc_ids = set(doc_ids) if doc_ids is not None else None 93 | 94 | def perform(self, data: Dict[int, Any]) -> None: 95 | if self.query is not None: 96 | for key in list(data): 97 | if self.query(data[key]): 98 | del data[key] 99 | else: 100 | assert self.doc_ids is not None 101 | for key in self.doc_ids: 102 | data.pop(key) 103 | -------------------------------------------------------------------------------- /tinyrecord/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugene-eeo/tinyrecord/96223e9f472ef3353aa3c4a63d1e46501edd8452/tinyrecord/py.typed -------------------------------------------------------------------------------- /tinyrecord/transaction.py: -------------------------------------------------------------------------------- 1 | from functools import wraps 2 | from threading import Lock 3 | from types import TracebackType 4 | from typing import Any, Callable, MutableMapping, NoReturn, Optional, Type 5 | from weakref import WeakKeyDictionary 6 | 7 | from tinydb.table import Table 8 | 9 | from tinyrecord.changeset import Changeset 10 | from tinyrecord.operations import (Operation, 11 | Remove, 12 | InsertMultiple, 13 | Update) 14 | 15 | 16 | class AbortSignal(Exception): 17 | """ 18 | Signals the abortion of a transaction. It is 19 | ignored when raised within the body of a 20 | transaction. 21 | """ 22 | pass 23 | 24 | 25 | def abort() -> NoReturn: 26 | """ 27 | Aborts the transaction. All operations defined on 28 | the transaction will be ignored (discarded). 29 | Raises the ``AbortSignal``, to be called only 30 | within a transaction. 31 | """ 32 | raise AbortSignal 33 | 34 | 35 | def records(cls: Type[Operation]) -> Callable[..., None]: 36 | """ 37 | Helper method for creating a method that records 38 | another operation to the changeset. 39 | 40 | :param cls: The operation class. 41 | """ 42 | @wraps(cls) 43 | def proxy(self: "transaction", *args: Any, **kwargs: Any) -> None: 44 | # Too many arguments for "Operation" 45 | self.record.append(cls(*args, **kwargs)) 46 | return proxy 47 | 48 | 49 | class transaction: 50 | """ 51 | Create an atomic transaction for the given 52 | *table*. All IO actions during the transaction are 53 | executed within a database-local lock. 54 | 55 | :param table: A TinyDB table. 56 | """ 57 | 58 | _locks: MutableMapping[Table, Lock] = WeakKeyDictionary() 59 | 60 | def __init__(self, table: Table) -> None: 61 | self.record = Changeset(table) 62 | self.lock = (self._locks.get(table) or 63 | self._locks.setdefault(table, Lock())) 64 | 65 | insert_multiple = records(InsertMultiple) 66 | update = records(Update) 67 | remove = records(Remove) 68 | 69 | def insert(self, row: Any) -> None: 70 | self.insert_multiple((row,)) 71 | 72 | def __enter__(self) -> "transaction": 73 | """ 74 | Enter a transaction. 75 | """ 76 | return self 77 | 78 | def __exit__( 79 | self, 80 | type: Optional[Type[BaseException]], 81 | value: Optional[BaseException], 82 | traceback: Optional[TracebackType] 83 | ) -> bool: 84 | """ 85 | Commits the transaction and raises a traceback 86 | if it is not an ``AbortSignal``. All actions 87 | are executed within a lock. 88 | """ 89 | if not traceback: 90 | with self.lock: 91 | self.record.execute() 92 | return isinstance(value, AbortSignal) 93 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py{36,37,38,39,310,311,py3}, mypy 3 | 4 | [testenv] 5 | deps = 6 | pytest 7 | commands = 8 | pytest tests.py 9 | 10 | [testenv:mypy] 11 | deps = 12 | mypy 13 | commands = 14 | mypy --pretty --show-error-codes --strict tinyrecord 15 | --------------------------------------------------------------------------------