├── .bandit.yaml ├── .flake8 ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .prospector.yaml ├── .style.yapf ├── LICENSE.txt ├── MANIFEST.in ├── README.rst ├── black.toml ├── docs ├── Makefile ├── requirements.txt └── source │ ├── _static │ ├── custom.css │ └── logo.svg │ ├── apidoc.rst │ ├── conf.py │ ├── development.rst │ ├── examples │ ├── mapped-types.ipynb │ └── quick-start.ipynb │ ├── img │ ├── mincepy-gui.png │ └── mincepy.svg │ ├── index.rst │ └── storing-objects.rst ├── notebooks └── development.ipynb ├── pyproject.toml ├── release.sh ├── src └── mincepy │ ├── __init__.py │ ├── _autosave.py │ ├── archive_factory.py │ ├── archives.py │ ├── base_savable.py │ ├── builtins.py │ ├── cli │ ├── __init__.py │ ├── dev.py │ ├── main.py │ └── query.py │ ├── common_helpers.py │ ├── comparators.py │ ├── defaults.py │ ├── depositors.py │ ├── exceptions.py │ ├── expr.py │ ├── fields.py │ ├── files.py │ ├── frontend.py │ ├── helpers.py │ ├── hist │ ├── __init__.py │ ├── live_objects.py │ ├── metas.py │ ├── references.py │ └── snapshots.py │ ├── historians.py │ ├── history.py │ ├── migrate.py │ ├── migrations.py │ ├── mongo │ ├── __init__.py │ ├── aggregation.py │ ├── bulk.py │ ├── db.py │ ├── migrate.py │ ├── migrations.py │ ├── mongo_archive.py │ ├── queries.py │ ├── references.py │ ├── settings.py │ └── types.py │ ├── operations.py │ ├── plugins.py │ ├── process.py │ ├── provides.py │ ├── qops.py │ ├── records.py │ ├── refs.py │ ├── result_types.py │ ├── saving.py │ ├── staging.py │ ├── testing.py │ ├── tracking.py │ ├── transactions.py │ ├── type_ids.py │ ├── type_registry.py │ ├── types.py │ ├── typing.py │ ├── utils.py │ └── version.py └── test ├── __init__.py ├── archive └── test_reference_graph.py ├── cli ├── __init__.py └── test_migrate.py ├── common.py ├── conftest.py ├── historian ├── test_delete.py ├── test_meta.py ├── test_references.py └── test_type_registry.py ├── mongo └── test_mongo_archive.py ├── test_archive.py ├── test_autosave.py ├── test_base_savable.py ├── test_benchmarks.py ├── test_builtins.py ├── test_convenience.py ├── test_data.py ├── test_expr.py ├── test_fields_saving.py ├── test_file.py ├── test_find.py ├── test_frontend.py ├── test_global.py ├── test_helpers.py ├── test_historian.py ├── test_history.py ├── test_migrate.py ├── test_migrations.py ├── test_process.py ├── test_qops.py ├── test_refs.py ├── test_savables.py ├── test_snapshots.py ├── test_staging.py ├── test_transactions.py ├── test_type_registry.py ├── test_types.py └── utils.py /.bandit.yaml: -------------------------------------------------------------------------------- 1 | exclude_dirs: ["test"] 2 | skips: ["B101"] 3 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E203, E266, E501, W503, F403, F401, E704 3 | max-line-length = 100 4 | max-complexity = 18 5 | select = B,C,E,F,W,T4,B9 6 | exclude = 7 | .git 8 | __pycache__ 9 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: continuous-integration 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | pre-commit: 7 | runs-on: ubuntu-latest 8 | 9 | steps: 10 | - uses: actions/checkout@v3 11 | - name: Set up Python 3.12 12 | uses: actions/setup-python@v3 13 | with: 14 | python-version: '3.12' 15 | - uses: pre-commit/action@v3.0.1 16 | 17 | pylint: 18 | name: pylint 19 | runs-on: ubuntu-latest 20 | timeout-minutes: 10 21 | 22 | steps: 23 | - uses: actions/checkout@v2 24 | 25 | - name: Set up Python ${{ env.DEFAULT_PYTHON }} 26 | uses: actions/setup-python@v5.1.1 27 | with: 28 | python-version: ${{ env.DEFAULT_PYTHON }} 29 | 30 | - name: Install python dependencies 31 | run: pip install -e .[cli,gui,dev,docs,sci] 32 | 33 | - name: Run pylint checks 34 | run: | 35 | pre-commit run --hook-stage manual pylint-with-spelling --all-files 36 | 37 | tests: 38 | runs-on: ubuntu-latest 39 | 40 | strategy: 41 | fail-fast: false 42 | matrix: 43 | python-version: ['3.9', '3.12', '3.11', '3.12', '3.13'] 44 | include: 45 | - python-version: 3.12 46 | rabbitmq: 3.6 47 | - python-version: 3.12 48 | rabbitmq: 3.8 49 | 50 | steps: 51 | - uses: actions/checkout@v2 52 | 53 | - name: Set up Python ${{ matrix.python-version }} 54 | uses: actions/setup-python@v5.1.1 55 | with: 56 | python-version: ${{ matrix.python-version }} 57 | 58 | - name: Install python dependencies 59 | run: pip install -e .[cli,gui,dev,docs,sci] 60 | 61 | - name: Create MongoDB Docker container 62 | id: build_mongo_docker 63 | uses: DigiPie/mongo-action@v2.0.1 64 | with: 65 | image-version: latest 66 | port: 27017 67 | 68 | - name: Run pytest 69 | run: pytest --cov=mincepy -sv -p no:nb_regression test 70 | 71 | - name: Create xml coverage 72 | run: coverage xml 73 | 74 | - name: Upload coverage to Codecov 75 | if: github.repository == 'muhrin/mincepy' 76 | uses: codecov/codecov-action@v1 77 | with: 78 | file: ./coverage.xml 79 | name: mincepy 80 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | .pytest_cache 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | env/ 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *,cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | 56 | # Sphinx documentation 57 | docs/_build/ 58 | 59 | # PyBuilder 60 | target/ 61 | 62 | #Ipython Notebook 63 | .ipynb_checkpoints 64 | 65 | 66 | # JetBrains IDE stuff 67 | .idea/ 68 | 69 | # Virtual environment directories 70 | /venv*/ 71 | 72 | .benchmarks 73 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | ci: 2 | skip: [ pylint ] 3 | 4 | # See https://pre-commit.com for more information 5 | # See https://pre-commit.com/hooks.html for more hooks 6 | repos: 7 | - repo: https://github.com/pre-commit/pre-commit-hooks 8 | rev: v4.6.0 9 | hooks: 10 | - id: check-added-large-files 11 | args: [ '--maxkb=5000' ] 12 | - id: end-of-file-fixer 13 | - id: check-case-conflict 14 | - id: detect-private-key 15 | - id: check-docstring-first 16 | - id: fix-encoding-pragma 17 | exclude: &exclude_files > 18 | (?x)^( 19 | docs/.*| 20 | )$ 21 | args: [ --remove ] 22 | - id: trailing-whitespace 23 | 24 | - repo: https://github.com/ikamensh/flynt/ 25 | rev: '1.0.1' 26 | hooks: 27 | - id: flynt 28 | 29 | - repo: https://github.com/psf/black 30 | rev: 25.1.0 31 | hooks: 32 | - id: black 33 | exclude: (.*)/migrations 34 | 35 | - repo: https://github.com/pycqa/flake8 36 | rev: 7.1.2 37 | hooks: 38 | - id: flake8 39 | 40 | - repo: https://github.com/pycqa/isort 41 | rev: '5.13.2' 42 | hooks: 43 | - id: isort 44 | 45 | - repo: https://github.com/PyCQA/bandit 46 | rev: 1.8.3 47 | hooks: 48 | - id: bandit 49 | args: [ "-c", ".bandit.yaml" ] 50 | 51 | - repo: https://github.com/PyCQA/pylint 52 | # Configuration help can be found here: 53 | # https://pylint.pycqa.org/en/latest/user_guide/installation/pre-commit-integration.html 54 | rev: v3.3.6 55 | hooks: 56 | - id: pylint 57 | alias: pylint-with-spelling 58 | stages: [ manual ] 59 | language: system 60 | types: [ python ] 61 | require_serial: true 62 | exclude: 63 | (?x)^( 64 | docs/.*| 65 | test/.* 66 | )$ 67 | 68 | - repo: https://github.com/commitizen-tools/commitizen 69 | rev: v4.4.1 70 | hooks: 71 | - id: commitizen 72 | stages: [ commit-msg ] 73 | 74 | - repo: https://github.com/srstevenson/nb-clean 75 | rev: 4.0.1 76 | hooks: 77 | - id: nb-clean 78 | -------------------------------------------------------------------------------- /.prospector.yaml: -------------------------------------------------------------------------------- 1 | ignore-paths: 2 | - doc 3 | - examples 4 | - test 5 | - utils 6 | 7 | pylint: 8 | max-line-length: 100 9 | disable: 10 | - missing-docstring 11 | 12 | pyflakes: 13 | run: false 14 | 15 | pep8: 16 | run: false 17 | 18 | mccabe: 19 | run: false 20 | -------------------------------------------------------------------------------- /.style.yapf: -------------------------------------------------------------------------------- 1 | [style] 2 | based_on_style = google 3 | column_limit = 100 4 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # Licenses 2 | include LICENSE 3 | include GPL 4 | include MIT 5 | 6 | # Include readme manually (because it has md extension) 7 | include README.md 8 | 9 | # Force test subdirectories 10 | recursive-include test *.py 11 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. _documentation: https://mincepy.readthedocs.org/ 2 | .. _object-document mapper: https://en.wikipedia.org/wiki/Object%E2%80%93relational_mapping#Object-oriented_databases 3 | .. _data mapper pattern: https://en.wikipedia.org/wiki/Data_mapper_pattern 4 | 5 | mincePy 6 | ======= 7 | 8 | .. image:: https://codecov.io/gh/muhrin/mincepy/branch/develop/graph/badge.svg 9 | :target: https://codecov.io/gh/muhrin/mincepy 10 | :alt: Coverage 11 | 12 | .. image:: https://github.com/muhrin/mincepy/actions/workflows/ci.yml/badge.svg 13 | :target: https://github.com/muhrin/mincepy/actions/workflows/ci.yml 14 | :alt: Tests 15 | 16 | .. image:: https://img.shields.io/pypi/v/mincepy.svg 17 | :target: https://pypi.python.org/pypi/mincepy/ 18 | :alt: Latest Version 19 | 20 | .. image:: https://img.shields.io/pypi/wheel/mincepy.svg 21 | :target: https://pypi.python.org/pypi/mincepy/ 22 | 23 | .. image:: https://img.shields.io/pypi/pyversions/mincepy.svg 24 | :target: https://pypi.python.org/pypi/mincepy/ 25 | 26 | .. image:: https://img.shields.io/pypi/l/mincepy.svg 27 | :target: https://pypi.python.org/pypi/mincepy/ 28 | 29 | .. image:: https://img.shields.io/badge/code%20style-black-000000.svg 30 | :target: https://github.com/psf/black 31 | 32 | 33 | mincePy: move the database to one side and let your objects take centre stage. 34 | 35 | MincePy is an `object-document mapper`_ (ORM) using the `data mapper pattern`_ designed specifically for computational 36 | and data science. What does this all mean?? It's simple really, it means you can store, find, get the history of and 37 | annotate any of your python objects either in your local database or one shared with your collaborators. 38 | 39 | 40 | 41 | See `documentation`_ for more information. 42 | -------------------------------------------------------------------------------- /black.toml: -------------------------------------------------------------------------------- 1 | line-length = 100 2 | target-version = ['py37', 'py38', 'py39', 'py310'] 3 | include = '\.pyi?$' 4 | exclude = ''' 5 | 6 | ( 7 | /( 8 | \.eggs # exclude a few common directories in the 9 | | \.git # root of the project 10 | | \.hg 11 | | \.mypy_cache 12 | | \.tox 13 | | \.venv 14 | | _build 15 | | buck-out 16 | | build 17 | | (.*)/migrations 18 | | dist 19 | )/ 20 | ) 21 | ''' 22 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = mincepy 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | # ONLY FOR ReadTheDocs 2 | .[dev] 3 | autodoc 4 | nbsphinx 5 | -------------------------------------------------------------------------------- /docs/source/_static/custom.css: -------------------------------------------------------------------------------- 1 | @import url('https://fonts.googleapis.com/css?family=Roboto|Roboto+Condensed|Roboto+Mono|Roboto+Slab'); 2 | 3 | h1.logo { 4 | text-align: center !important; 5 | } 6 | -------------------------------------------------------------------------------- /docs/source/apidoc.rst: -------------------------------------------------------------------------------- 1 | API Reference 2 | ============= 3 | 4 | `mincepy` 5 | --------- 6 | 7 | .. automodule:: mincepy 8 | :members: 9 | 10 | 11 | `mincepy.hist` 12 | --------------- 13 | 14 | .. automodule:: mincepy.hist 15 | :members: 16 | 17 | 18 | `mincepy.mongo` 19 | --------------- 20 | 21 | .. automodule:: mincepy.mongo 22 | :members: 23 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # mincepy documentation build configuration file, created by 5 | # sphinx-quickstart on Fri Mar 31 17:03:20 2017. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | from importlib.machinery import SourceFileLoader 17 | 18 | # If extensions (or modules to document with autodoc) are in another directory, 19 | # add these directories to sys.path here. If the directory is relative to the 20 | # documentation root, use os.path.abspath to make it absolute, like shown here. 21 | # 22 | import os 23 | import sys 24 | 25 | sys.path.insert(0, os.path.abspath(os.path.dirname("__file__"))) 26 | 27 | module = SourceFileLoader( 28 | "version", 29 | os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "mincepy", "version.py"), 30 | ).load_module() 31 | 32 | autoclass_content = "both" 33 | 34 | # -- General configuration ------------------------------------------------ 35 | 36 | # If your documentation needs a minimal Sphinx version, state it here. 37 | # 38 | # needs_sphinx = '1.0' 39 | 40 | # Add any Sphinx extension module names here, as strings. They can be 41 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 42 | # ones. 43 | extensions = [ 44 | "sphinx.ext.autodoc", 45 | "sphinx.ext.doctest", 46 | "sphinx.ext.coverage", 47 | "sphinx.ext.viewcode", 48 | "nbsphinx", 49 | ] 50 | 51 | # Add any paths that contain templates here, relative to this directory. 52 | templates_path = ["_templates"] 53 | 54 | # The suffix(es) of source filenames. 55 | # You can specify multiple suffix as a list of string: 56 | # 57 | # source_suffix = ['.rst', '.md'] 58 | source_suffix = ".rst" 59 | 60 | # The master toctree document. 61 | master_doc = "index" 62 | 63 | # General information about the project. 64 | project = "mincePy" 65 | copyright = "2020, Martin Uhrin" 66 | author = "Martin Uhrin" 67 | 68 | # The version info for the project you're documenting, acts as replacement for 69 | # |version| and |release|, also used in various other places throughout the 70 | # built documents. 71 | # 72 | # The short X.Y version. 73 | version = ".".join(map(str, module.version_info[:-1])) 74 | # The full version, including alpha/beta/rc tags. 75 | release = ".".join(map(str, module.version_info)) 76 | 77 | # The language for content autogenerated by Sphinx. Refer to documentation 78 | # for a list of supported languages. 79 | # 80 | # This is also used if you do content translation via gettext catalogs. 81 | # Usually you set "language" from the command line for these cases. 82 | language = None 83 | 84 | # List of patterns, relative to source directory, that match files and 85 | # directories to ignore when looking for source files. 86 | # This patterns also effect to html_static_path and html_extra_path 87 | exclude_patterns = [] 88 | 89 | # The name of the Pygments (syntax highlighting) style to use. 90 | pygments_style = "sphinx" 91 | 92 | # If true, `todo` and `todoList` produce output, else they produce nothing. 93 | todo_include_todos = False 94 | 95 | # -- Options for HTML output ---------------------------------------------- 96 | 97 | # The theme to use for HTML and HTML Help pages. See the documentation for 98 | # a list of builtin themes. 99 | # 100 | html_theme = "alabaster" 101 | 102 | # Theme options are theme-specific and customize the look and feel of a theme 103 | # further. For a list of options available for each theme, see the 104 | # documentation. 105 | # 106 | # html_theme_options = {} 107 | html_theme_options = { 108 | "analytics_id": "UA-17296547-2", 109 | "codecov_button": True, 110 | "description": "Python object storage with versioning made simple", 111 | "github_button": True, 112 | "github_repo": "mincepy", 113 | "github_type": "star", 114 | "github_user": "muhrin", 115 | "travis_button": True, 116 | "logo": "logo.svg", 117 | "logo_name": True, 118 | } 119 | 120 | # Add any paths that contain custom static files (such as style sheets) here, 121 | # relative to this directory. They are copied after the builtin static files, 122 | # so a file named "default.css" will overwrite the builtin "default.css". 123 | html_static_path = ["_static"] 124 | 125 | # -- Options for HTMLHelp output ------------------------------------------ 126 | 127 | # Output file base name for HTML help builder. 128 | htmlhelp_basename = "mincepydoc" 129 | 130 | # -- Options for LaTeX output --------------------------------------------- 131 | 132 | latex_elements = { 133 | # The paper size ('letterpaper' or 'a4paper'). 134 | # 135 | # 'papersize': 'a4paper', 136 | # The font size ('10pt', '11pt' or '12pt'). 137 | # 138 | # 'pointsize': '12pt', 139 | # Additional stuff for the LaTeX preamble. 140 | # 141 | # 'preamble': '', 142 | # Latex figure (float) alignment 143 | # 144 | # 'figure_align': 'htbp', 145 | } 146 | 147 | # Grouping the document tree into LaTeX files. List of tuples 148 | # (source start file, target name, title, 149 | # author, documentclass [howto, manual, or own class]). 150 | latex_documents = [ 151 | (master_doc, "mincepy.tex", "mincePy Documentation", "Martin Uhrin", "manual"), 152 | ] 153 | 154 | # -- Options for manual page output --------------------------------------- 155 | 156 | # One entry per manual page. List of tuples 157 | # (source start file, name, description, authors, manual section). 158 | man_pages = [(master_doc, "mincepy", "mincePy Documentation", [author], 1)] 159 | 160 | # -- Options for Texinfo output ------------------------------------------- 161 | 162 | # Grouping the document tree into Texinfo files. List of tuples 163 | # (source start file, target name, title, author, 164 | # dir menu entry, description, category) 165 | texinfo_documents = [ 166 | ( 167 | master_doc, 168 | "mincepy", 169 | "mincePy Documentation", 170 | author, 171 | "mincepy", 172 | "One line description of project.", 173 | "Miscellaneous", 174 | ), 175 | ] 176 | 177 | # -- Options for Epub output ---------------------------------------------- 178 | 179 | # Bibliographic Dublin Core info. 180 | epub_title = project 181 | epub_author = author 182 | epub_publisher = author 183 | epub_copyright = copyright 184 | 185 | # The unique identifier of the text. This can be a ISBN number 186 | # or the project homepage. 187 | # 188 | # epub_identifier = '' 189 | 190 | # A unique identification for the text. 191 | # 192 | # epub_uid = '' 193 | 194 | # A list of files that should not be packed into the epub file. 195 | epub_exclude_files = ["search.html"] 196 | 197 | # 198 | # html_logo = 'logo.svg' 199 | # html_favicon = 'icon.png' 200 | # 201 | html_sidebars = { 202 | "**": [ 203 | "about.html", 204 | "navigation.html", 205 | "searchbox.html", 206 | ] 207 | } 208 | -------------------------------------------------------------------------------- /docs/source/development.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | .. _repository: https://github.com/muhrin/mincepy 4 | 5 | 6 | Development 7 | =========== 8 | 9 | The mincePy source code, issues, etc are all kept at our `repository`_. 10 | 11 | Clone the project: 12 | 13 | .. code-block:: shell 14 | 15 | git clone https://github.com/muhrin/mincepy.git 16 | cd mincepy 17 | 18 | 19 | Create a new virtualenv: 20 | 21 | .. code-block:: shell 22 | 23 | virtualenv -p python3 mincepy 24 | 25 | Install all requirements: 26 | 27 | .. code-block:: shell 28 | 29 | env/bin/pip install -e '.[dev,docs]' 30 | -------------------------------------------------------------------------------- /docs/source/img/mincepy-gui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muhrin/mincepy/6f40ac667e5025e5b76894c4fa7e9a2efeaf8501/docs/source/img/mincepy-gui.png -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. mincepy documentation master file, created by 2 | sphinx-quickstart on Fri Mar 31 17:03:20 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | .. _mincePy: https://github.com/muhrin/mincepy 7 | .. _scientific types: https://github.com/muhrin/mincepy_sci 8 | .. _object-document mapper: https://en.wikipedia.org/wiki/Object-relational_mapping#Object-oriented_databases 9 | .. _data mapper pattern: https://en.wikipedia.org/wiki/Data_mapper_pattern 10 | .. _python ORMs: https://en.wikipedia.org/wiki/List_of_object-relational_mapping_software#Python 11 | .. _gui: https://github.com/muhrin/mincepy_gui/ 12 | .. _store: examples/quick-start.ipynb#Storing-objects 13 | .. _find: examples/quick-start.ipynb#Finding-objects 14 | .. _annotate: examples/quick-start.ipynb#Annotating-objects 15 | .. _history: examples/quick-start.ipynb#Version-control 16 | .. _Django: https://www.djangoproject.com/ 17 | .. _SQLAlchemy: https://www.sqlalchemy.org/ 18 | .. _Storm: https://www.sqlalchemy.org/ 19 | .. _MongoEngine: http://mongoengine.org/ 20 | .. _identity map: examples/quick-start.ipynb#References 21 | 22 | Welcome to mincePy's documentation! 23 | =================================== 24 | 25 | .. image:: https://codecov.io/gh/muhrin/mincepy/branch/develop/graph/badge.svg 26 | :target: https://codecov.io/gh/muhrin/mincepy 27 | :alt: Coveralls 28 | 29 | .. image:: https://travis-ci.org/muhrin/mincepy.svg 30 | :target: https://travis-ci.org/muhrin/mincepy 31 | :alt: Travis CI 32 | 33 | .. image:: https://img.shields.io/pypi/v/mincepy.svg 34 | :target: https://pypi.python.org/pypi/mincepy/ 35 | :alt: Latest Version 36 | 37 | .. image:: https://img.shields.io/pypi/wheel/mincepy.svg 38 | :target: https://pypi.python.org/pypi/mincepy/ 39 | 40 | .. image:: https://img.shields.io/pypi/pyversions/mincepy.svg 41 | :target: https://pypi.python.org/pypi/mincepy/ 42 | 43 | .. image:: https://img.shields.io/pypi/l/mincepy.svg 44 | :target: https://pypi.python.org/pypi/mincepy/ 45 | 46 | 47 | `mincePy`_: move the database to one side and let your objects take centre stage. 48 | 49 | MincePy is an `object-document mapper`_ (ODM) designed to make any of your Python object storable and queryable in a MongoDB database. 50 | It is designed with machine learning and big-data computational and experimental science applications in mind but is entirely general and can be useful to anyone looking to organise, share, or process large amounts data with as little change to their current workflow as possible. 51 | 52 | Why was mincePy built? 53 | ++++++++++++++++++++++ 54 | 55 | For Python we already have `MongoEngine`_, `Django`_, `SQLAlchemy`_, `Storm`_ and a bunch of `other `_ great ORMs, so why do we need mincePy? 56 | Well, in a typical ORM you subclass some kind of model class thus every object *is a* database object *plus* whatever else the object is designed to do. 57 | This is great for applications where there is a tight coupling to the database but what if you can't or don't want to subclass from a model? 58 | What if you want to store a numpy array, a PyTorch neural network configuration or any arbitrary Python object and have it be queryable? 59 | This is where mincePy excels: You tell mincePy about the type you want to store and it takes care of the rest. 60 | 61 | The other big thing that differentiates mincePy is version control. 62 | Many of us are used to git and other VCS for code but what if we want to track changes to our data, made either by us a collaborator? 63 | MincePy achieves this by keeping a snapshot of your object each time you save it so you can always retrieve an older version and see how it mutated over time. 64 | 65 | Features 66 | ++++++++ 67 | 68 | * Out-of-the-box support for commonly used `scientific types`_, including: 69 | 70 | * NumPy 71 | * pandas 72 | * ASE 73 | 74 | * Ability to work locally or collaboratively on a shared database 75 | * Automatic tracking of in-memory objects (`identity map`_) 76 | * Easy addition of new Python types 77 | * Object version control 78 | * Tracking of references between objects 79 | * Optimistic locking 80 | * Plugin system makes it easy to extend mincePy to support new types 81 | * Python 3.5+ compatible 82 | * A responsive, Qt, `gui`_: 83 | 84 | .. image:: img/mincepy-gui.png 85 | :align: center 86 | :width: 600 87 | :alt: The mincePy GUI 88 | 89 | 90 | Installation 91 | ++++++++++++ 92 | 93 | Installation with pip: 94 | 95 | .. code-block:: shell 96 | 97 | pip install mincepy 98 | 99 | Or if you'd like to include plugins for storing common scientific types, use: 100 | 101 | .. code-block:: shell 102 | 103 | pip install mincepy[sci] 104 | 105 | Installation from git: 106 | 107 | .. code-block:: shell 108 | 109 | # via pip 110 | pip install https://github.com/muhrin/mincepy/archive/master.zip 111 | 112 | # manually 113 | git clone https://github.com/muhrin/mincepy.git 114 | cd mincepy 115 | python setup.py install 116 | 117 | 118 | Next you'll need MongoDB, in Ubuntu it's as simple as: 119 | 120 | 121 | .. code-block:: shell 122 | 123 | apt install mongodb 124 | 125 | see `here `_, for other platforms. 126 | 127 | 128 | Table Of Contents 129 | +++++++++++++++++ 130 | 131 | .. toctree:: 132 | :glob: 133 | :maxdepth: 3 134 | 135 | examples/quick-start.ipynb 136 | examples/mapped-types.ipynb 137 | storing-objects 138 | development 139 | apidoc 140 | 141 | 142 | Versioning 143 | ++++++++++ 144 | 145 | This software follows `Semantic Versioning`_ 146 | 147 | 148 | .. _Semantic Versioning: http://semver.org/ 149 | -------------------------------------------------------------------------------- /docs/source/storing-objects.rst: -------------------------------------------------------------------------------- 1 | Storing Objects 2 | +++++++++++++++ 3 | 4 | Unlike many ORMs, mincePy does not require that the object to be stored are subclasses of a mincePy type. 5 | This is a deliberate choice and means that it is possible to store objects that you cannot, or do not want to make inherit from anything in mincePy. This is all possible thanks to the :class:`mincepy.TypeHelper` which is used to tell mincePy all the things it needs to know to be able to work with them. That said, if you've got a type want to use specifically with mincePy you can subclass from :class:`mincepy.SavableObject` (or one of its subclasses). 6 | 7 | 8 | Concepts 9 | ======== 10 | 11 | Migrations 12 | ========== 13 | 14 | Decided you want to change the way you store your objects? Have a database with thousands of precious objects stored in the, soon to be, old format? Not ideal... 15 | 16 | Don't worry, we've got your back. Migrations are a way to tell mincePy how to go from the old version to the new. Imagine we have a ``Car`` object that looks like this: 17 | 18 | .. code-block:: python 19 | 20 | class Car(mincepy.SimpleSavable, type_id=uuid.UUID('297808e4-9bc7-4f0a-9f8d-850a5f558663')): 21 | ATTRS = ('colour', 'make') 22 | 23 | def __init__(self, colour: str, make: str): 24 | super(Car, self).__init__() 25 | self.colour = colour 26 | self.make = make 27 | 28 | def save_instance_state(self, saver: mincepy.Saver): 29 | super(Car, self).save_instance_state(saver) 30 | # Here, I decide to store as an array 31 | return [self.colour, self.make] 32 | 33 | def load_instance_state(self, saved_state, loader: mincepy.Loader): 34 | self.colour = saved_state[0] 35 | self.make = saved_state[1] 36 | 37 | Great, so now let's save some cars! 38 | 39 | 40 | .. code-block:: python 41 | 42 | Car('red', 'zonda').save() 43 | Car('black', 'bugatti').save() 44 | Car('white', 'ferrari').save() 45 | 46 | Ok, and now we decide that instead of storing the details as a list, we want to use a dictionary instead. No problem, let's just put in place a migration: 47 | 48 | .. code-block:: python 49 | 50 | class Car(mincepy.SimpleSavable, type_id=uuid.UUID('297808e4-9bc7-4f0a-9f8d-850a5f558663')): 51 | ATTRS = ('colour', 'make') 52 | 53 | class Migration1(mincepy.ObjectMigration): 54 | VERSION = 1 55 | 56 | @classmethod 57 | def upgrade(cls, saved_state, migrator): 58 | return dict(colour=saved_state[0], make=saved_state[1]) 59 | 60 | # Set the migration 61 | LATEST_MIGRATION = Migration1 62 | 63 | def save_instance_state(self, saver): 64 | # I've changed my mind, I'd like to store it as a dict 65 | return dict(colour=self.colour, make=self.make) 66 | 67 | def load_instance_state(self, saved_state, loader): 68 | self.colour = saved_state['colour'] 69 | self.make = saved_state['make'] 70 | 71 | 72 | Here, we've changed :meth:`~mincepy.Savable.save_instance_state` and :meth:`~mincepy.Savable.load_instance_state` as expected. 73 | Then, we create a subclass of :class:`mincepy.ObjectMigration` which implements the :func:`~mincepy.ObjectMigration.upgrade` class method. 74 | This method gets an old saved state and is tasked with returning a state in the new format. 75 | So, we take the old array and return it as a dictionary that will be understood by our new :meth:`~mincepy.Savable.load_instance_state`. 76 | 77 | Next, we set the ``VERSION`` number for our migration. This should be an integer that is higher than the last migration. As we have no migration, ``1`` will do. 78 | 79 | Finally, we tell the ``Car`` object what the latest migration is by setting the ``LATEST_MIGRATION`` class attribute to the migration class. 80 | 81 | All this will allow ``mincepy`` to load your objects by converting them to the current format as needed. The database, however, will not be touched unless you save the object again after making changes, in which case it will saved with the new form. 82 | 83 | Performing Migrations 84 | --------------------- 85 | 86 | To update the state of all objects in your database you can use the command line command: 87 | 88 | .. code-block:: python 89 | 90 | mince migrate mongodb://localhost/my-database 91 | 92 | Where the databases URI is supplied as the argument. This will inform you how many records are to be migrated and allow you to perform the migration. 93 | 94 | Adding Migrations 95 | ----------------- 96 | 97 | If you decide you want to change the format of ``Car`` again, say by adding a registration field, it can be done like this: 98 | 99 | .. code-block:: python 100 | 101 | class Car(mincepy.SimpleSavable, type_id=uuid.UUID('297808e4-9bc7-4f0a-9f8d-850a5f558663')): 102 | ATTRS = ('colour', 'make') 103 | 104 | class Migration1(mincepy.ObjectMigration): 105 | VERSION = 1 106 | 107 | @classmethod 108 | def upgrade(cls, saved_state, migrator): 109 | return dict(colour=saved_state[0], make=saved_state[1]) 110 | 111 | class Migration2(mincepy.ObjectMigration): 112 | VERSION = 2 113 | PREVIOUS = Migration1 114 | 115 | @classmethod 116 | def upgrade(cls, saved_state, migrator): 117 | # Augment the saved state 118 | saved_state['reg'] = 'unknown' 119 | return saved_state 120 | 121 | # Set the migration 122 | LATEST_MIGRATION = Migration2 123 | 124 | def __init__(self, colour: str, make: str, reg=None): 125 | super(Car, self).__init__() 126 | self.colour = colour 127 | self.make = make 128 | self.reg = reg 129 | 130 | def save_instance_state(self, saver: mincepy.Saver): 131 | # I've changed my mind, I'd like to store it as a dict 132 | return dict(colour=self.colour, make=self.make, reg=self.reg) 133 | 134 | def load_instance_state(self, saved_state, loader): 135 | self.colour = saved_state['colour'] 136 | self.make = saved_state['make'] 137 | self.reg = saved_state['reg'] 138 | 139 | This migration was added using the following steps: 140 | 141 | 1. Created ``Migration2`` with an `upgrade` method that adds the missing data, 142 | 2. Set ``Migration2.VERSION`` to a version number higher than the previous 143 | 3. Set the ``PREVIOUS`` class attribute to the previous migration. This way, mincePy can upgrade all the way from the original version to the latest. 144 | 4. Set ``Car``'s ``LATEST_MIGRATION`` to point to th enew migration 145 | 146 | Again, this is enough to load and save old versions, however to make the changes to the database records use the `migrate` tool described above. 147 | -------------------------------------------------------------------------------- /notebooks/development.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import pymongo\n", 10 | "import bson\n", 11 | "import random\n", 12 | "\n", 13 | "import mincepy\n", 14 | "from mincepy.testing import *" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "hist = mincepy.create_historian(\"mongodb://localhost/test\")\n", 24 | "mincepy.set_historian(hist)" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": null, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "hist.get_archive()._data_collection.drop()\n", 34 | "hist.get_archive()._meta_collection.drop()" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": null, 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "def populate_database(hist):\n", 44 | " colours = ('red' , 'orange', 'yellow', 'green', 'blue', 'indigo', 'violet')\n", 45 | " makes = ('honda', 'ferrari', 'zonda', 'fiat')\n", 46 | " \n", 47 | " cars = []\n", 48 | " \n", 49 | " for make in makes:\n", 50 | " for colour in colours:\n", 51 | " # Make some cars\n", 52 | " car = Car(make, colour)\n", 53 | " hist.save(car)\n", 54 | " cars.append(car)\n", 55 | " \n", 56 | " \n", 57 | " # Now randomly change some of them\n", 58 | " for _ in range(int(len(cars)/4)):\n", 59 | " car = random.choice(cars)\n", 60 | " car.colour = random.choice(colours)\n", 61 | " hist.save(car)\n", 62 | " \n", 63 | " # Now change one a number of times\n", 64 | " car = random.choice(cars)\n", 65 | " for colour in colours:\n", 66 | " car.colour = colour\n", 67 | " hist.save(car)\n", 68 | " " 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": null, 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | " \n", 78 | "for i in range(10):\n", 79 | " populate_database(hist)\n", 80 | "\n", 81 | "car = Car('ferrari')\n", 82 | "car_id = hist.save(car)\n", 83 | "\n" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": null, 89 | "metadata": {}, 90 | "outputs": [], 91 | "source": [ 92 | "garage = Garage(car)\n", 93 | "garage_id = hist.save(garage)" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": null, 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "del garage\n" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": null, 108 | "metadata": {}, 109 | "outputs": [], 110 | "source": [ 111 | "car.colour = 'yellow'\n", 112 | "hist.save(car)\n", 113 | "garage = hist.load(garage_id)\n", 114 | "print(garage.car.colour)" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": null, 120 | "metadata": {}, 121 | "outputs": [], 122 | "source": [ 123 | "\n" 124 | ] 125 | } 126 | ], 127 | "metadata": { 128 | "kernelspec": { 129 | "display_name": "Python 3", 130 | "language": "python", 131 | "name": "python3" 132 | }, 133 | "language_info": { 134 | "codemirror_mode": { 135 | "name": "ipython", 136 | "version": 2 137 | }, 138 | "file_extension": ".py", 139 | "mimetype": "text/x-python", 140 | "name": "python", 141 | "nbconvert_exporter": "python", 142 | "pygments_lexer": "ipython2" 143 | }, 144 | "pycharm": { 145 | "stem_cell": { 146 | "cell_type": "raw", 147 | "metadata": { 148 | "collapsed": false 149 | }, 150 | "source": [] 151 | } 152 | } 153 | }, 154 | "nbformat": 4, 155 | "nbformat_minor": 0 156 | } 157 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ['flit_core >=3.9,<4'] 3 | build-backend = 'flit_core.buildapi' 4 | 5 | [project] 6 | name = 'mincepy' 7 | dynamic = ["version", "description"] 8 | authors = [ 9 | { name = 'Martin Uhrin', email = 'martin.uhrin.10@ucl.ac.uk' }, 10 | ] 11 | readme = 'README.rst' 12 | license = { file = 'LICENSE.txt' } 13 | classifiers = [ 14 | 'Development Status :: 4 - Beta', 15 | 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)', 16 | 'Programming Language :: Python', 17 | 'Programming Language :: Python :: 3.9', 18 | 'Programming Language :: Python :: 3.10', 19 | 'Programming Language :: Python :: 3.11', 20 | 'Programming Language :: Python :: 3.12', 21 | 'Programming Language :: Python :: 3.13', 22 | ] 23 | keywords = ["database", "schemaless", "nosql", "orm", "object-store", "concurrent", "optimistic-locking"] 24 | requires-python = '>=3.9' 25 | dependencies = [ 26 | "deprecation", 27 | "dnspython", # Needed to be able to connect using domain name rather than IP 28 | "pymongo<5.0", 29 | "litemongo", 30 | "importlib-metadata<5.0", # see: https://stackoverflow.com/questions/73929564/entrypoints-object-has-no-attribute-get-digital-ocean 31 | "mongomock", 32 | "bidict", 33 | "networkx", # For reference graphs 34 | "pytray>=0.2.1", 35 | "stevedore", 36 | "click", 37 | "tabulate", 38 | "tqdm", 39 | "typing-extensions", 40 | ] 41 | 42 | [project.urls] 43 | Home = 'https://mincepy.readthedocs.io/en/latest/index.html' 44 | Source = 'https://github.com/muhrin/mincepy.git' 45 | 46 | [project.optional-dependencies] 47 | docs = [ 48 | "nbsphinx", 49 | "sphinx", 50 | "sphinx-autobuild", 51 | ] 52 | dev = [ 53 | "black", 54 | "flit", 55 | "ipython", 56 | "mongomock", 57 | "pip", 58 | "pylint", 59 | "pytest>4", 60 | "pytest-benchmark", 61 | "pytest-cov", 62 | "pre-commit", 63 | "yapf", 64 | ] 65 | cli = ["click", "tabulate"] 66 | gui = ["mincepy-gui"] 67 | sci = ["mincepy-sci"] 68 | 69 | [project.scripts] 70 | mince = "mincepy.cli:main" 71 | 72 | [project.entry-points."mincepy.plugins.types"] 73 | native = "mincepy.provides:get_types" 74 | 75 | 76 | [tool.flit.module] 77 | name = 'mincepy' 78 | 79 | [tool.flit.sdist] 80 | exclude = [ 81 | '.github/', 82 | 'docs/', 83 | 'examples/', 84 | 'test/', 85 | ] 86 | 87 | [tool.flynt] 88 | line-length = 100 89 | fail-on-change = true 90 | 91 | [tool.isort] 92 | profile = "black" 93 | force_sort_within_sections = true 94 | include_trailing_comma = true 95 | line_length = 100 96 | multi_line_output = 3 97 | 98 | [tool.pylint.format] 99 | max-line-length = 100 100 | 101 | [tool.black] 102 | line-length = 100 103 | 104 | [tool.pylint.messages_control] 105 | disable = [ 106 | # Unfortunately jaxtyping decorator creates a function that seems to mistakenly be identified as 107 | # not returning anything, so we have to disable the error below for now 108 | 'assignment-from-no-return', 109 | 'duplicate-code', 110 | 'import-outside-toplevel', 111 | 'missing-docstring', 112 | 'locally-disabled', 113 | 'too-few-public-methods', 114 | 'too-many-instance-attributes', 115 | 'use-dict-literal', 116 | ] 117 | 118 | [tool.pylint.design] 119 | max-args = 14 120 | max-locals = 20 121 | max-parents = 12 122 | max-positional-arguments = 8 123 | 124 | [pytest] 125 | log_cli = "True" 126 | log_cli_level = "DEBUG" 127 | 128 | [tool.pytest.ini_options] 129 | minversion = '6.0' 130 | testpaths = [ 131 | 'test', 132 | ] 133 | filterwarnings = [ 134 | 'ignore::DeprecationWarning:frozendict:', 135 | ] 136 | 137 | [tool.yapf] 138 | align_closing_bracket_with_visual_indent = true 139 | based_on_style = 'google' 140 | coalesce_brackets = true 141 | column_limit = 100 142 | dedent_closing_brackets = true 143 | indent_dictionary_value = false 144 | split_arguments_when_comma_terminated = true 145 | 146 | [tool.tox] 147 | legacy_tox_ini = """ 148 | [tox] 149 | envlist = py311 150 | 151 | [testenv] 152 | usedevelop = true 153 | 154 | [testenv:py{39,310,311,312}] 155 | description = Run the unit tests 156 | extras = 157 | dev 158 | commands = pytest {posargs} 159 | 160 | [testenv:pre-commit] 161 | description = Run the style checks and formatting 162 | extras = 163 | dev 164 | commands = pre-commit run {posargs} 165 | 166 | [pytest] 167 | filterwarnings = 168 | ignore::DeprecationWarning:distutils: 169 | """ 170 | -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | 2 | PACKAGE="mincepy" 3 | REMOTE="muhrin" 4 | VERSION_FILE=${PACKAGE}/__init__.py 5 | 6 | version=$1 7 | while true; do 8 | read -p "Release version ${version}? " yn 9 | case $yn in 10 | [Yy]* ) break;; 11 | [Nn]* ) exit;; 12 | * ) echo "Please answer yes or no.";; 13 | esac 14 | done 15 | 16 | set -x 17 | 18 | sed -i "/^__version__/c __version__ = ${version}" $VERSION_FILE 19 | 20 | current_branch=`git rev-parse --abbrev-ref HEAD` 21 | 22 | tag="v${version}" 23 | relbranch="release-${version}" 24 | 25 | echo Releasing version $version 26 | 27 | git checkout -b $relbranch 28 | git add ${VERSION_FILE} 29 | git commit --no-verify -m "Release ${version}" 30 | 31 | git tag -a $tag -m "Version $version" 32 | 33 | 34 | # Merge into master 35 | 36 | git checkout master 37 | git merge $relbranch 38 | 39 | # And back into the working branch (usually develop) 40 | git checkout $current_branch 41 | git merge $relbranch 42 | 43 | git branch -d $relbranch 44 | 45 | # Push everything 46 | git push --tags $REMOTE master $current_branch 47 | 48 | 49 | # Release on pypi 50 | flit publish 51 | -------------------------------------------------------------------------------- /src/mincepy/__init__.py: -------------------------------------------------------------------------------- 1 | """mincePy: move the database to one side and let your objects take centre stage.""" 2 | 3 | from . import ( 4 | archive_factory, 5 | archives, 6 | base_savable, 7 | builtins, 8 | common_helpers, 9 | depositors, 10 | exceptions, 11 | expr, 12 | fields, 13 | helpers, 14 | hist, 15 | historians, 16 | history, 17 | migrate, 18 | migrations, 19 | mongo, 20 | operations, 21 | process, 22 | ) 23 | from . import qops as q 24 | from . import records, refs, tracking, types, typing, utils, version 25 | from .archive_factory import * 26 | from .archives import * 27 | from .base_savable import * 28 | from .builtins import * 29 | from .comparators import * 30 | from .depositors import * 31 | from .exceptions import * # pylint: disable=redefined-builtin 32 | from .expr import * 33 | from .fields import * 34 | from .helpers import * 35 | from .hist import * 36 | from .historians import * 37 | from .history import * 38 | from .migrations import * 39 | from .process import * 40 | from .records import * 41 | from .refs import * 42 | from .tracking import * 43 | from .types import * 44 | from .version import __author__, __version__ 45 | 46 | _ADDITIONAL = ( 47 | "mongo", 48 | "builtins", 49 | "common_helpers", 50 | "migrate", 51 | "utils", 52 | "q", 53 | "operations", 54 | "typing", 55 | ) 56 | 57 | __all__ = ( 58 | archives.__all__ 59 | + depositors.__all__ 60 | + exceptions.__all__ 61 | + historians.__all__ 62 | + process.__all__ 63 | + types.__all__ 64 | + helpers.__all__ 65 | + history.__all__ 66 | + archive_factory.__all__ 67 | + refs.__all__ 68 | + records.__all__ 69 | + base_savable.__all__ 70 | + builtins.__all__ 71 | + migrations.__all__ 72 | + tracking.__all__ 73 | + hist.__all__ 74 | + fields.__all__ 75 | + expr.__all__ 76 | + _ADDITIONAL 77 | ) 78 | -------------------------------------------------------------------------------- /src/mincepy/_autosave.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING, Union 2 | 3 | from pytray import obj_load 4 | from typing_extensions import override 5 | 6 | from . import helpers 7 | 8 | if TYPE_CHECKING: 9 | import mincepy 10 | 11 | TYPE_ID_PREFIX = "autosave" 12 | 13 | 14 | def _create_helper(obj_type: type, obj_path: str) -> "mincepy.TypeHelper": 15 | """Create a type helper that uses the object path as the type id""" 16 | 17 | class AutoSavable( 18 | helpers.TypeHelper, obj_type=obj_type, type_id=f"{TYPE_ID_PREFIX}:{obj_path}" 19 | ): 20 | @override 21 | def save_instance_state(self, obj, saver: "mincepy.Saver", /) -> dict: 22 | saved_state = _get_state(obj) 23 | # Check if a superclass wants to save the state 24 | super_state = {} 25 | for super_type in obj_type.mro()[1:]: 26 | try: 27 | helper: "mincepy.TypeHelper" = ( 28 | saver.historian.type_registry.get_helper_from_obj_type(super_type) 29 | ) 30 | super_state = helper.save_instance_state(obj, saver) 31 | break 32 | except ValueError: 33 | pass 34 | saved_state.update(super_state) 35 | return saved_state 36 | 37 | @override 38 | def load_instance_state(self, obj, state: dict, loader: "mincepy.Loader", /) -> None: 39 | _set_state(obj, state) 40 | # Now see if there is a superclass that wants to load the instance state 41 | for super_type in obj_type.mro()[1:]: 42 | try: 43 | helper: "mincepy.TypeHelper" = ( 44 | loader.historian.type_registry.get_helper_from_obj_type(super_type) 45 | ) 46 | helper.load_instance_state(obj, state, loader) 47 | break 48 | except ValueError: 49 | pass 50 | 51 | return AutoSavable() 52 | 53 | 54 | def autosavable(obj_type_or_id: Union[type, str]) -> "mincepy.TypeHelper": 55 | if isinstance(obj_type_or_id, type): 56 | obj_type = obj_type_or_id 57 | obj_path = obj_load.full_name(obj_type) 58 | # Make sure that it's importable 59 | assert obj_load.load_obj(obj_path) is obj_type 60 | elif isinstance(obj_type_or_id, str) and obj_type_or_id.startswith(TYPE_ID_PREFIX): 61 | obj_path = obj_type_or_id[len(TYPE_ID_PREFIX) + 1 :] 62 | obj_type = obj_load.load_obj(obj_path) 63 | else: 64 | raise TypeError(f"Unknown object type or id: {obj_type_or_id}") 65 | 66 | return _create_helper(obj_type, obj_path) 67 | 68 | 69 | def _get_state(obj: object) -> dict: 70 | """ 71 | Get the writable attributes of an object. 72 | 73 | This will try to use vars() but this fails for object with __slots__ in which case it will fall 74 | back to that 75 | """ 76 | state = {} 77 | if hasattr(obj, "__slots__"): 78 | if not hasattr(obj, "__dict__") and "__weakref__" not in obj.__slots__: 79 | raise ValueError( 80 | f"Object `{obj}` is not compatible with the historian because it uses __slots__ " 81 | f"but does not have __weakref__. Add it to make it compatible." 82 | ) 83 | state.update( 84 | {name: getattr(obj, name) for name in obj.__slots__ if name not in ["__weakref__"]} 85 | ) 86 | if hasattr(obj, "__dict__"): 87 | state.update(obj.__dict__) 88 | 89 | return state 90 | 91 | 92 | def _set_state(obj: object, state: dict): 93 | for name, value in state.items(): 94 | setattr(obj, name, value) 95 | -------------------------------------------------------------------------------- /src/mincepy/archive_factory.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from typing import TYPE_CHECKING, Optional, Union 4 | from urllib import parse 5 | 6 | from . import mongo 7 | 8 | if TYPE_CHECKING: 9 | import mincepy 10 | 11 | __all__ = ( 12 | "create_archive", 13 | "DEFAULT_ARCHIVE_URI", 14 | "ENV_ARCHIVE_URI", 15 | "default_archive_uri", 16 | ) 17 | 18 | _LOGGER = logging.getLogger(__name__) 19 | 20 | DEFAULT_ARCHIVE_URI = "mongodb://localhost/mincepy" 21 | ENV_ARCHIVE_URI = "MINCEPY_ARCHIVE" 22 | 23 | 24 | def default_archive_uri() -> Optional[str]: 25 | """Returns the default archive URI. This is currently being taken from the environmental 26 | MINCEPY_ARCHIVE, however it may chance to include a config file in the future.""" 27 | return os.environ.get(ENV_ARCHIVE_URI, DEFAULT_ARCHIVE_URI) 28 | 29 | 30 | def create_archive(uri: Union[str, parse.ParseResult], connect_timeout=30000) -> "mincepy.Archive": 31 | """Create an archive type based on a URI string 32 | 33 | :param uri: the specification of where to connect to 34 | :param connect_timeout: a connection timeout (in milliseconds) 35 | """ 36 | archive = mongo.connect(uri, timeout=connect_timeout) 37 | 38 | _LOGGER.info("Connected to archive with uri: %s", uri) 39 | return archive 40 | -------------------------------------------------------------------------------- /src/mincepy/base_savable.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import typing 3 | from typing import TYPE_CHECKING, Optional, cast 4 | 5 | from typing_extensions import override 6 | 7 | from . import refs, types 8 | 9 | if TYPE_CHECKING: 10 | import mincepy 11 | 12 | __all__ = ( 13 | "BaseSavableObject", 14 | "ConvenienceMixin", 15 | "SimpleSavable", 16 | "AsRef", 17 | "ConvenientSavable", 18 | ) 19 | 20 | AttrSpec = collections.namedtuple("AttrSpec", "name as_ref") 21 | 22 | 23 | def AsRef(name: str) -> AttrSpec: # pylint: disable=invalid-name 24 | """Create an attribute specification for an attribute that should be stored by reference""" 25 | return AttrSpec(name, True) 26 | 27 | 28 | class BaseSavableObject(types.SavableObject): 29 | """A helper class that makes a class compatible with the historian by flagging certain 30 | attributes which will be saved/loaded/hashed and compared in __eq__. This should be an 31 | exhaustive list of all the attributes that define this class. If more complex functionality 32 | is needed, then the standard SavableComparable interface methods should be overwritten.""" 33 | 34 | ATTRS = tuple() 35 | # When loading ignore attributes that are missing in the record 36 | IGNORE_MISSING = True 37 | 38 | @override 39 | def __new__(cls, *_args, **_kwargs): 40 | new_instance = super(BaseSavableObject, cls).__new__(cls) 41 | attrs = {} 42 | for entry in cls.mro(): 43 | try: 44 | class_attrs = getattr(entry, "ATTRS") 45 | except AttributeError: 46 | pass 47 | else: 48 | for attr_spec in class_attrs: 49 | if isinstance(attr_spec, str): 50 | # If it's just a string then default to store by value 51 | attr_spec = AttrSpec(attr_spec, False) 52 | 53 | # Check that it's not already there so higher up in the MRO is always kept 54 | if attr_spec.name not in attrs: 55 | attrs[attr_spec.name] = attr_spec 56 | 57 | setattr(new_instance, "__attrs", tuple(attrs.values())) 58 | return new_instance 59 | 60 | @override 61 | def __eq__(self, other): 62 | if not isinstance(other, type(self)): 63 | return False 64 | 65 | return all( 66 | getattr(self, attr.name) == getattr(other, attr.name) for attr in self.__get_attrs() 67 | ) 68 | 69 | @override 70 | def yield_hashables(self, hasher, /): 71 | yield from super().yield_hashables(hasher) 72 | yield from hasher.yield_hashables([getattr(self, attr.name) for attr in self.__get_attrs()]) 73 | 74 | @override 75 | def save_instance_state(self, saver: "mincepy.Saver", /) -> dict: 76 | saved_state = super().save_instance_state(saver) 77 | for attr in self.__get_attrs(): 78 | item = getattr(self, attr.name) 79 | if attr.as_ref: 80 | item = refs.ObjRef(item) 81 | saved_state[attr.name] = item 82 | 83 | return saved_state 84 | 85 | @override 86 | def load_instance_state(self, saved_state, loader: "mincepy.Loader", /): 87 | super().load_instance_state(saved_state, loader) 88 | for attr in self.__get_attrs(): 89 | try: 90 | obj = saved_state[attr.name] 91 | except KeyError: 92 | if self.IGNORE_MISSING: 93 | # Set any missing attributes to None 94 | setattr(self, attr.name, None) 95 | else: 96 | raise 97 | else: 98 | if attr.as_ref: 99 | if obj: 100 | obj = obj() 101 | else: 102 | obj = None 103 | setattr(self, attr.name, obj) 104 | 105 | def __get_attrs(self) -> typing.Sequence[AttrSpec]: 106 | return getattr(self, "__attrs") 107 | 108 | 109 | class ConvenienceMixin: 110 | """A mixin that adds convenience methods to your savable object""" 111 | 112 | @property 113 | def obj_id(self): 114 | if self._historian is None: 115 | return None 116 | return self._historian.get_obj_id(self) 117 | 118 | def get_meta(self) -> Optional[dict]: 119 | """Get the metadata dictionary for this object""" 120 | if self._historian is None: 121 | return None 122 | return self._historian.meta.get(self) 123 | 124 | def set_meta(self, meta: Optional[dict]): 125 | """Set the metadata dictionary for this object""" 126 | if self._historian is None: 127 | raise RuntimeError("Object must be saved before the metadata can be set") 128 | 129 | self._historian.meta.set(self, meta) 130 | 131 | def update_meta(self, meta: dict): 132 | """Update the metadata dictionary for this object""" 133 | if self._historian is None: 134 | raise RuntimeError("Object must be saved before the metadata can be updated") 135 | 136 | self._historian.meta.update(self, meta) 137 | 138 | def is_saved(self) -> bool: 139 | """Returns True if this object is saved with a historian""" 140 | if self._historian is not None: 141 | return self._historian.is_saved(self) 142 | 143 | return False 144 | 145 | def save(self, meta: dict = None): 146 | """Save the object""" 147 | if self._historian is None: 148 | import mincepy # pylint: disable=import-outside-toplevel 149 | 150 | # We don't have a historian yet (we haven't been saved), so use the current global one 151 | historian = mincepy.get_historian() 152 | else: 153 | historian = self._historian 154 | 155 | return historian.save_one(self, meta=meta) 156 | 157 | def sync(self): 158 | """Update the state of this object by loading the latest version from the historian""" 159 | if self._historian is not None: 160 | self._historian.sync(self) 161 | 162 | def save_instance_state(self, saver: "mincepy.Saver", /): 163 | self._on_save(saver) 164 | return cast(types.Savable, super()).save_instance_state(saver) 165 | 166 | def load_instance_state(self, saved_state, loader: "mincepy.Loader", /): 167 | """Take the given object and load the instance state into it""" 168 | cast(types.Savable, super()).load_instance_state(saved_state, loader) 169 | self._on_load(loader) 170 | 171 | def _on_save(self, saver: "mincepy.Saver"): 172 | # Check if we've been assigned an object id, otherwise we're just being saved by value 173 | if saver.get_historian().get_obj_id(self) is not None: 174 | self._historian = saver.get_historian() 175 | 176 | def _on_load(self, loader: "mincepy.Loader"): 177 | if loader.get_historian().get_obj_id(self) is not None: 178 | self._historian = loader.get_historian() 179 | 180 | 181 | class SimpleSavable(ConvenienceMixin, BaseSavableObject): 182 | """A BaseSavableObject with convenience methods mixed in""" 183 | 184 | 185 | class ConvenientSavable(ConvenienceMixin, types.SavableObject): 186 | """A savable with convenience methods. 187 | 188 | See :py:class:`ConvenienceMixin` 189 | """ 190 | -------------------------------------------------------------------------------- /src/mincepy/cli/__init__.py: -------------------------------------------------------------------------------- 1 | from . import main 2 | from .dev import * 3 | from .main import * 4 | 5 | __all__ = main.__all__ 6 | -------------------------------------------------------------------------------- /src/mincepy/cli/dev.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | 4 | import click 5 | 6 | from . import main 7 | 8 | 9 | @main.mince.group() 10 | def dev(): 11 | """Commands to help with development and testing""" 12 | 13 | 14 | @dev.command() 15 | @click.argument("uri", type=str) 16 | def populate(uri): 17 | """Populate the database with testing data""" 18 | import mincepy.testing # pylint: disable=import-outside-toplevel 19 | 20 | main.set_print_logging(logging.INFO) 21 | try: 22 | hist = mincepy.create_historian(uri) 23 | except ValueError as exc: 24 | click.echo(exc) 25 | sys.exit(1) 26 | else: 27 | mincepy.testing.populate(hist) 28 | click.echo(f"Successfully populated database at '{uri}'") 29 | -------------------------------------------------------------------------------- /src/mincepy/cli/main.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | import uuid 4 | 5 | import click 6 | import pytray.pretty 7 | import tabulate 8 | 9 | import mincepy.plugins 10 | 11 | __all__ = ("mince",) 12 | 13 | # pylint: disable=import-outside-toplevel 14 | 15 | 16 | def set_print_logging(level=logging.WARNING): 17 | mince_logger = logging.getLogger("mincepy") 18 | mince_logger.setLevel(level) 19 | 20 | handler = logging.StreamHandler() 21 | handler.setLevel(logging.DEBUG) 22 | formatter = logging.Formatter("%(message)s") 23 | handler.setFormatter(formatter) 24 | mince_logger.addHandler(handler) 25 | 26 | 27 | @click.group() 28 | def mince(): 29 | pass 30 | 31 | 32 | @mince.command() 33 | @click.argument("uri", type=str) 34 | def gui(uri): 35 | """Start the mincepy gui (mincepy_gui package must be installed)""" 36 | try: 37 | import mincepy_gui 38 | except ImportError: 39 | click.echo("mincepy-gui not found, please install (e.g. via pip install mincepy-gui)") 40 | sys.exit(1) 41 | else: 42 | mincepy_gui.start(uri) 43 | 44 | 45 | @mince.command() 46 | def plugins(): 47 | type_plugins = mincepy.plugins.get_types() 48 | click.echo("Types:") 49 | 50 | headers = "Type", "Class" 51 | plugin_info = [] 52 | for plugin in type_plugins: 53 | if isinstance(plugin, mincepy.TypeHelper): 54 | row = "Type Helper", pytray.pretty.type_string(type(plugin)) 55 | elif isinstance(plugin, type) and issubclass(plugin, mincepy.SavableObject): 56 | row = "Savable Object", pytray.pretty.type_string(plugin) 57 | else: 58 | row = "Unrecognised!", pytray.pretty.type_string(plugin) 59 | plugin_info.append(row) 60 | 61 | click.echo(tabulate.tabulate(plugin_info, headers)) 62 | 63 | 64 | @mince.command() 65 | @click.option("--yes", is_flag=True, default=False, help="Yes to all prompts") 66 | @click.argument("uri", type=str) 67 | @click.pass_context 68 | def migrate(ctx, yes, uri): 69 | try: 70 | hist = mincepy.create_historian(uri) 71 | except ValueError as exc: 72 | click.echo(exc) 73 | sys.exit(1) 74 | else: 75 | if isinstance(ctx.obj, dict) and "helpers" in ctx.obj: 76 | hist.register_types(ctx.obj["helpers"]) 77 | 78 | click.echo("Looking for records to migrate...") 79 | records = tuple(hist.migrations.find_migratable_records()) 80 | if not records: 81 | click.echo("No migrations necessary") 82 | return 83 | 84 | click.echo(f"Found {len(records)} records to migrate") 85 | if yes or click.confirm("Migrate all?"): 86 | set_print_logging(logging.INFO) 87 | hist.migrations.migrate_records(records) 88 | else: 89 | sys.exit(2) 90 | 91 | 92 | @mince.command() 93 | def tid(): 94 | """Create a new type id""" 95 | click.echo(str(uuid.uuid4())) 96 | 97 | 98 | @mince.command() 99 | @click.argument("uri", type=str) 100 | @click.option("--deleted/--no-deleted", default=True, help="Purge all deleted objects") 101 | @click.option( 102 | "--unreferenced/--no-unreferenced", 103 | default=True, 104 | help="Purge all snapshots that are not referenced by live objects", 105 | ) 106 | @click.option("--yes", "-y", is_flag=True, default=False, help="Yes to all prompts") 107 | @click.option("-v", "--verbose", count=True) 108 | def purge(uri, deleted, unreferenced, yes, verbose): 109 | """Purge the snapshots collection of any unused objects""" 110 | if verbose: 111 | set_print_logging(logging.INFO if verbose == 1 else logging.DEBUG) 112 | 113 | try: 114 | hist = mincepy.create_historian(uri) 115 | except ValueError as exc: 116 | click.echo(exc) 117 | sys.exit(1) 118 | else: 119 | click.echo("Searching records...") 120 | res = hist.purge(deleted=deleted, unreferenced=unreferenced, dry_run=True) 121 | click.echo( 122 | f"Found {len(res.deleted_purged)} deleted " 123 | f"and {len(res.unreferenced_purged)} unreferenced snapshot(s)" 124 | ) 125 | 126 | to_delete = list(res.deleted_purged | res.unreferenced_purged) 127 | if to_delete and (yes or click.confirm("Do you want to delete them?")): 128 | click.echo("Deleting...", nl=False) 129 | hist.archive.bulk_write(list(map(mincepy.operations.Delete, to_delete))) 130 | click.echo("done") 131 | -------------------------------------------------------------------------------- /src/mincepy/cli/query.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | import typing 3 | 4 | import click 5 | import pymongo 6 | from tabulate import tabulate 7 | 8 | import mincepy 9 | import mincepy.records 10 | import mincepy.testing 11 | 12 | 13 | @click.command() 14 | @click.option("--obj-type", default=None, help="The type of object to find") 15 | @click.option("--filter", default=None, help="Filter on the state") 16 | @click.option("--limit", default=0, help="Limit the number of results") 17 | def query(obj_type, filter, limit): # pylint: disable=redefined-builtin 18 | historian = mincepy.get_historian() 19 | 20 | results = historian.records.find(obj_type, state=filter, limit=limit, version=-1) 21 | 22 | historian.register_types(mincepy.testing.HISTORIAN_TYPES) 23 | 24 | # Gather by object types 25 | gathered = {} 26 | for result in results: 27 | gathered.setdefault(result.type_id, []).append(result) 28 | 29 | for type_id, records in gathered.items(): 30 | print(f"type: {type_id}") 31 | print_records(records, historian) 32 | 33 | 34 | SCALAR_VALUE = "" 35 | UNSET = "" 36 | REF = "ref" 37 | 38 | 39 | def print_records(records: typing.Sequence[mincepy.records.DataRecord], historian): 40 | columns = OrderedDict() 41 | refs = [] 42 | for record in records: 43 | try: 44 | helper = historian.get_helper(record.type_id) 45 | type_str = get_type_name(helper.TYPE) + f"#{record.version}" 46 | except KeyError: 47 | type_str = str(record.snapshot_id) 48 | if record.is_deleted_record(): 49 | type_str += " [deleted]" 50 | refs.append(type_str) 51 | columns[REF] = refs 52 | 53 | for record in records: 54 | if not record.is_deleted_record(): 55 | for column_name in get_all_columns(record.state): 56 | columns[tuple(column_name)] = [] 57 | 58 | for column_name in columns.keys(): 59 | if column_name != REF: 60 | columns[column_name] = [get_value(column_name, record.state) for record in records] 61 | 62 | rows = [] 63 | for row in range(len(records)): 64 | rows.append([col[row] for col in columns.values()]) 65 | 66 | print( 67 | tabulate( 68 | rows, 69 | headers=[ 70 | ".".join(path) if isinstance(path, tuple) else path for path in columns.keys() 71 | ], 72 | ) 73 | ) 74 | 75 | 76 | def get_all_columns(state): 77 | if isinstance(state, dict): 78 | if "type_id" in state and "state" in state: 79 | yield [] 80 | else: 81 | for key, value in state.items(): 82 | key_path = [key] 83 | if isinstance(value, dict): 84 | for path in get_all_columns(value): 85 | yield key_path + path 86 | else: 87 | yield key_path 88 | elif isinstance(state, list): 89 | yield from [(idx,) for idx in range(len(state))] 90 | else: 91 | yield SCALAR_VALUE 92 | 93 | 94 | def get_value(title, state): 95 | if isinstance(state, (dict, list)): 96 | idx = title[0] 97 | value = state[idx] 98 | # Check for references 99 | if isinstance(value, dict) and set(value.keys()) == {"type_id", "state"}: 100 | return str(mincepy.records.SnapshotId(*value["state"])) 101 | 102 | if len(title) > 1: 103 | return get_value(title[1:], value) 104 | 105 | try: 106 | return state[idx] 107 | except (KeyError, IndexError): 108 | return UNSET 109 | 110 | if title == SCALAR_VALUE: 111 | return state 112 | 113 | return UNSET 114 | 115 | 116 | def get_type_name(obj_type): 117 | try: 118 | return f"{obj_type.__module__}.{obj_type.__name__}" 119 | except AttributeError: 120 | return str(obj_type) 121 | 122 | 123 | if __name__ == "__main__": 124 | client = pymongo.MongoClient() 125 | db = client.test_database 126 | mongo_archive = mincepy.mongo.MongoArchive(db) 127 | 128 | hist = mincepy.Historian(mongo_archive) 129 | mincepy.set_historian(hist) 130 | 131 | query() # pylint: disable=no-value-for-parameter 132 | -------------------------------------------------------------------------------- /src/mincepy/common_helpers.py: -------------------------------------------------------------------------------- 1 | # This module will be removed in 0.17.0 2 | from .builtins import NamespaceHelper, PathHelper, TupleHelper 3 | 4 | __all__ = "PathHelper", "TupleHelper", "NamespaceHelper" 5 | 6 | HISTORIAN_TYPES = NamespaceHelper, TupleHelper, PathHelper 7 | -------------------------------------------------------------------------------- /src/mincepy/comparators.py: -------------------------------------------------------------------------------- 1 | import collections.abc 2 | import datetime 3 | import numbers 4 | from operator import itemgetter 5 | import uuid 6 | 7 | from typing_extensions import override 8 | 9 | from . import helpers 10 | 11 | __all__ = ("SimpleHelper", "BytesEquator") 12 | 13 | 14 | class SimpleHelper(helpers.TypeHelper, obj_type=None, type_id=None): 15 | @override 16 | def yield_hashables(self, obj, hasher, /): 17 | yield from obj.yield_hashables(hasher) 18 | 19 | @override 20 | def eq(self, one, other, /) -> bool: # pylint: disable=invalid-name 21 | return one == other 22 | 23 | @override 24 | def save_instance_state(self, obj, saver, /): 25 | return obj.save_instance_state(saver) 26 | 27 | @override 28 | def load_instance_state(self, obj, saved_state, loader, /): 29 | return obj.load(saved_state, loader) 30 | 31 | 32 | class BytesEquator(SimpleHelper, obj_type=(bytes, bytearray), type_id=None): 33 | @override 34 | def yield_hashables(self, obj, hasher, /): 35 | yield obj 36 | 37 | 38 | class StrEquator(SimpleHelper, obj_type=str, type_id=None): 39 | @override 40 | def yield_hashables(self, obj: str, hasher, /): 41 | yield obj.encode("utf-8") 42 | 43 | 44 | class SequenceEquator(SimpleHelper, obj_type=collections.abc.Sequence, type_id=None): 45 | @override 46 | def yield_hashables(self, obj: collections.abc.Sequence, hasher, /): 47 | for entry in obj: 48 | yield from hasher.yield_hashables(entry) 49 | 50 | 51 | class SetEquator(SimpleHelper, obj_type=collections.abc.Set, type_id=None): 52 | @override 53 | def yield_hashables(self, obj: collections.abc.Set, hasher, /): 54 | for entry in sorted(obj): 55 | yield from hasher.yield_hashables(entry) 56 | 57 | 58 | class MappingEquator(SimpleHelper, obj_type=collections.abc.Mapping, type_id=None): 59 | @override 60 | def yield_hashables(self, obj: collections.abc.Mapping, hasher, /): 61 | def hashed_key_mapping(mapping): 62 | for key, value in mapping.items(): 63 | yield tuple(hasher.yield_hashables(key)), value 64 | 65 | for key_hashables, value in sorted(hashed_key_mapping(obj), key=itemgetter(0)): 66 | # Yield all the key hashables 67 | yield from key_hashables 68 | # And now all the value hashables for that entry 69 | yield from hasher.yield_hashables(value) 70 | 71 | 72 | class OrderedDictEquator(SimpleHelper, obj_type=collections.OrderedDict, type_id=None): 73 | @override 74 | def yield_hashables(self, obj: collections.OrderedDict, hasher, /): 75 | for key, val in sorted(obj, key=itemgetter(0)): 76 | yield from hasher.yield_hashables(key) 77 | yield from hasher.yield_hashables(val) 78 | 79 | 80 | class RealEquator(SimpleHelper, obj_type=numbers.Real, type_id=None): 81 | @override 82 | def yield_hashables(self, obj, hasher, /): 83 | yield from hasher.yield_hashables(hasher.float_to_str(obj)) 84 | 85 | 86 | class ComplexEquator(SimpleHelper, obj_type=numbers.Complex, type_id=None): 87 | @override 88 | def yield_hashables(self, obj: numbers.Complex, hasher, /): 89 | yield from hasher.yield_hashables(obj.real) 90 | yield from hasher.yield_hashables(obj.imag) 91 | 92 | 93 | class IntegerEquator(SimpleHelper, obj_type=numbers.Integral, type_id=None): 94 | @override 95 | def yield_hashables(self, obj: numbers.Integral, hasher, /): 96 | yield from hasher.yield_hashables(f"{obj}") 97 | 98 | 99 | class BoolEquator(SimpleHelper, obj_type=bool, type_id=None): 100 | @override 101 | def yield_hashables(self, obj, hasher, /): 102 | yield b"\x01" if obj else b"\x00" 103 | 104 | 105 | class NoneEquator(SimpleHelper, obj_type=type(None), type_id=None): 106 | @override 107 | def yield_hashables(self, obj, hasher, /): 108 | yield from hasher.yield_hashables("None") 109 | 110 | 111 | class TupleEquator(SimpleHelper, obj_type=tuple, type_id=None): 112 | @override 113 | def yield_hashables(self, obj, hasher, /): 114 | yield from hasher.yield_hashables(obj) 115 | 116 | 117 | class UuidEquator(SimpleHelper, obj_type=uuid.UUID, type_id=None): 118 | @override 119 | def yield_hashables(self, obj: uuid.UUID, hasher, /): 120 | yield obj.bytes 121 | 122 | 123 | class DatetimeEquator(SimpleHelper, obj_type=datetime.datetime, type_id=None): 124 | @override 125 | def yield_hashables(self, obj: datetime.datetime, hasher, /): 126 | yield str(obj).encode("utf-8") 127 | -------------------------------------------------------------------------------- /src/mincepy/defaults.py: -------------------------------------------------------------------------------- 1 | from . import comparators 2 | 3 | 4 | def get_default_equators(): 5 | return ( 6 | comparators.SequenceEquator(), 7 | comparators.BytesEquator(), 8 | comparators.StrEquator(), 9 | comparators.SetEquator(), 10 | comparators.OrderedDictEquator(), 11 | comparators.MappingEquator(), 12 | comparators.ComplexEquator(), 13 | comparators.RealEquator(), 14 | comparators.IntegerEquator(), 15 | comparators.BoolEquator(), 16 | comparators.NoneEquator(), 17 | comparators.TupleEquator(), 18 | comparators.UuidEquator(), 19 | comparators.DatetimeEquator(), 20 | ) 21 | -------------------------------------------------------------------------------- /src/mincepy/exceptions.py: -------------------------------------------------------------------------------- 1 | __all__ = ( 2 | "NotFound", 3 | "ModificationError", 4 | "ObjectDeleted", 5 | "DuplicateKeyError", 6 | "MigrationError", 7 | "VersionError", 8 | "IntegrityError", 9 | "ReferenceError", 10 | "ConnectionError", 11 | "MergeError", 12 | ) 13 | 14 | 15 | class ConnectionError(Exception): # pylint: disable=redefined-builtin 16 | """Raise when there is an error in connecting to the backend""" 17 | 18 | 19 | class NotFound(Exception): 20 | """Raised when something can not be found in the history""" 21 | 22 | 23 | class ModificationError(Exception): 24 | """Raised when a modification of the history encountered a problem""" 25 | 26 | 27 | class ObjectDeleted(NotFound): 28 | """Raise when the user tries to interact with a deleted object""" 29 | 30 | 31 | class DuplicateKeyError(ModificationError): 32 | """Indicates that a uniqueness constraint was violated""" 33 | 34 | 35 | class MigrationError(Exception): 36 | """Indicates that an error occurred during migration""" 37 | 38 | 39 | class VersionError(Exception): 40 | """Indicates a version mismatch between the code and the database""" 41 | 42 | 43 | class IntegrityError(Exception): 44 | """Indicates an error that occurred because of an operation that would conflict with a database 45 | constraint""" 46 | 47 | 48 | class MergeError(Exception): 49 | """Indicates that an error occurred when trying to merge""" 50 | 51 | 52 | class ReferenceError(IntegrityError): # pylint: disable=redefined-builtin 53 | """Raised when there is an operation that causes a problem with references for example if 54 | you try to delete an object that is referenced by another this exception will be raised. The 55 | objects ids being referenced will be found in .references. 56 | """ 57 | 58 | def __init__(self, msg, references: set): 59 | super().__init__(msg) 60 | self.references = references 61 | 62 | 63 | class UnorderedError(Exception): 64 | """Raised when an operation is attempted that assumed an underlying ordering but the data is 65 | unordered""" 66 | 67 | 68 | class NotOneError(Exception): 69 | """Raised when a singlar result is expected but there are in fact more""" 70 | -------------------------------------------------------------------------------- /src/mincepy/files.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import shutil 3 | import tempfile 4 | from typing import BinaryIO, Optional, TextIO, Union 5 | 6 | from typing_extensions import override 7 | 8 | from . import base_savable, fields, type_ids 9 | 10 | __all__ = "File", "BaseFile" 11 | 12 | 13 | class File(base_savable.SimpleSavable, type_id=type_ids.FILE_TYPE_ID): 14 | """ 15 | A mincePy file object. These should not be instantiated directly but using 16 | `Historian.create_file()` 17 | """ 18 | 19 | READ_SIZE = 256 # The number of bytes to read at a time 20 | 21 | def __init__(self, file_store, filename: str = None, encoding=None): 22 | super().__init__() 23 | self._file_store = file_store 24 | self._filename = filename 25 | self._encoding = encoding 26 | self._file_id = None 27 | self._buffer_file = _create_buffer_file() 28 | 29 | @fields.field("_filename") 30 | def filename(self) -> Optional[str]: 31 | return self._filename 32 | 33 | @fields.field("_encoding") 34 | def encoding(self) -> Optional[str]: 35 | return self._encoding 36 | 37 | @fields.field("_file_id") 38 | def file_id(self): 39 | return self._file_id 40 | 41 | def open(self, mode="r", **kwargs) -> Union[BinaryIO, TextIO]: 42 | """Open returning a file like object that supports close() and read()""" 43 | self._ensure_buffer() 44 | if "b" not in mode: 45 | kwargs.setdefault("encoding", self.encoding) 46 | return open(self._buffer_file, mode, **kwargs) # pylint: disable=unspecified-encoding 47 | 48 | def from_disk(self, path): 49 | """Copy the contents of a disk file to this file""" 50 | with open(str(path), "r", encoding=self.encoding) as disk_file: 51 | with self.open("w") as this: 52 | shutil.copyfileobj(disk_file, this) 53 | 54 | def to_disk(self, path: [str, pathlib.Path]): 55 | """Copy the contents of this file to disk. 56 | 57 | :param path: the path can be either a folder in which case the file contents are written to 58 | `path / self.filename` or path can be a full file path in which case that will be used. 59 | """ 60 | file_path = pathlib.Path(str(path)) 61 | if file_path.is_dir(): 62 | file_path /= self.filename 63 | 64 | with open(str(file_path), "w", encoding=self._encoding) as disk_file: 65 | with self.open("r") as this: 66 | shutil.copyfileobj(this, disk_file) 67 | 68 | def write_text(self, text: str, encoding=None): 69 | encoding = encoding or self._encoding 70 | with self.open("w", encoding=encoding) as fileobj: 71 | fileobj.write(text) 72 | 73 | def read_text(self, encoding=None) -> str: 74 | """Read the contents of the file as text. 75 | This function is named as to mirror pathlib.Path""" 76 | encoding = encoding or self._encoding 77 | with self.open("r", encoding=encoding) as fileobj: 78 | return fileobj.read() 79 | 80 | @override 81 | def save_instance_state(self, saver, /): 82 | filename = self.filename or "" 83 | with open(self._buffer_file, "rb") as fstream: 84 | self._file_id = self._file_store.upload_from_stream(filename, fstream) 85 | 86 | return super().save_instance_state(saver) 87 | 88 | @override 89 | def load_instance_state(self, saved_state, loader, /): 90 | super().load_instance_state(saved_state, loader) 91 | self._file_store = loader.get_archive().file_store 92 | # Don't copy the file over now, do it lazily when the file is first opened 93 | self._buffer_file = None 94 | 95 | def _ensure_buffer(self): 96 | if self._buffer_file is None: 97 | if self._file_id is not None: 98 | self._update_buffer() 99 | else: 100 | _create_buffer_file() 101 | 102 | def _update_buffer(self): 103 | self._buffer_file = _create_buffer_file() 104 | with open(self._buffer_file, "wb") as fstream: 105 | self._file_store.download_to_stream(self._file_id, fstream) 106 | 107 | @override 108 | def __str__(self): 109 | contents = [str(self._filename)] 110 | if self._encoding is not None: 111 | contents.append(f"({self._encoding})") 112 | return " ".join(contents) 113 | 114 | @override 115 | def __eq__(self, other) -> bool: 116 | """Compare the contents of two files 117 | 118 | If both files do not exist they are considered equal. 119 | """ 120 | if ( # pylint: disable=comparison-with-callable 121 | not isinstance(other, File) or self.filename != other.filename 122 | ): 123 | return False 124 | 125 | try: 126 | with self.open() as my_file: 127 | try: 128 | with other.open() as other_file: 129 | while True: 130 | my_line = my_file.readline(self.READ_SIZE) 131 | other_line = other_file.readline(self.READ_SIZE) 132 | if my_line != other_line: 133 | return False 134 | if my_line == "" and other_line == "": 135 | return True 136 | except FileNotFoundError: 137 | return False 138 | except FileNotFoundError: 139 | # Our file doesn't exist, make sure the other doesn't either 140 | try: 141 | with other.open(): 142 | return False 143 | except FileNotFoundError: 144 | return True 145 | 146 | @override 147 | def yield_hashables(self, hasher, /): 148 | """Hash the contents of the file""" 149 | try: 150 | with self.open("rb") as opened: 151 | while True: 152 | line = opened.read(self.READ_SIZE) 153 | if line == b"": 154 | return 155 | yield line 156 | except FileNotFoundError: 157 | yield from hasher.yield_hashables(None) 158 | 159 | 160 | def _create_buffer_file(): 161 | with tempfile.NamedTemporaryFile(delete=False) as tmp_file: 162 | tmp_path = tmp_file.name 163 | 164 | return tmp_path 165 | 166 | 167 | BaseFile = File # Here just for legacy reasons. Deprecate in 1.0 168 | -------------------------------------------------------------------------------- /src/mincepy/hist/__init__.py: -------------------------------------------------------------------------------- 1 | from . import live_objects 2 | from .live_objects import * 3 | from .metas import Meta 4 | from .references import References 5 | from .snapshots import SnapshotsCollection 6 | 7 | __all__ = live_objects.__all__ + ("Meta", "References", "SnapshotsCollection") 8 | -------------------------------------------------------------------------------- /src/mincepy/hist/live_objects.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING, Callable, Optional 2 | 3 | from mincepy import frontend 4 | from mincepy import records as records_ 5 | 6 | if TYPE_CHECKING: 7 | import mincepy 8 | 9 | __all__ = ("LiveObjectsCollection",) 10 | 11 | 12 | class LiveObjectsCollection(frontend.ObjectCollection): 13 | def __init__( 14 | self, historian: "mincepy.Historian", archive_collection: "mincepy.archives.Collection" 15 | ): 16 | super().__init__( 17 | historian, 18 | archive_collection, 19 | record_factory=lambda record_dict: LoadableRecord( 20 | record_dict, 21 | historian.load_snapshot_from_record, 22 | # pylint: disable=protected-access 23 | historian._load_object_from_record, 24 | ), 25 | obj_loader=historian._load_object_from_record, 26 | ) 27 | 28 | 29 | class LoadableRecord(records_.DataRecord): 30 | __slots__ = () 31 | _obj_loader: Optional[Callable[["mincepy.DataRecord"], object]] = None 32 | _snapshot_loader: Optional[Callable[["mincepy.DataRecord"], object]] = None 33 | 34 | def __new__( 35 | cls, 36 | record_dict: dict, 37 | snapshot_loader: Callable[["mincepy.DataRecord"], object], 38 | obj_loader: Callable[["mincepy.DataRecord"], object], 39 | ): 40 | loadable = super().__new__(cls, **record_dict) 41 | loadable._obj_loader = obj_loader 42 | loadable._snapshot_loader = snapshot_loader 43 | return loadable 44 | 45 | def load_snapshot(self) -> object: 46 | return self._snapshot_loader(self) # pylint: disable=not-callable 47 | 48 | def load(self) -> object: 49 | return self._obj_loader(self) # pylint: disable=not-callable 50 | -------------------------------------------------------------------------------- /src/mincepy/hist/metas.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Hashable 2 | from typing import TYPE_CHECKING, Any, Generic, Iterator, Mapping, Optional, TypeVar 3 | 4 | from mincepy import exceptions 5 | 6 | if TYPE_CHECKING: 7 | import mincepy 8 | 9 | __all__ = ("Meta",) 10 | 11 | 12 | IdT = TypeVar("IdT", bound=Hashable) 13 | 14 | 15 | class Meta(Generic[IdT]): 16 | """A class for grouping metadata related methods""" 17 | 18 | # Meta is a 'friend' of Historian and so can access privates pylint: disable=protected-access 19 | 20 | def __init__( 21 | self, historian: "mincepy.Historian[IdT]", archive: "mincepy.Archive[IdT]" 22 | ) -> None: 23 | self._hist: "mincepy.Historian[IdT]" = historian 24 | self._archive = archive 25 | self._sticky = {} 26 | 27 | @property 28 | def sticky(self) -> dict: 29 | return self._sticky 30 | 31 | def get(self, obj_or_identifier) -> Optional[dict]: 32 | """Get the metadata for an object 33 | 34 | :param obj_or_identifier: either the object instance, an object ID or a snapshot reference 35 | """ 36 | results = self.get_many((obj_or_identifier,)) 37 | assert len(results) == 1 38 | meta = tuple(results.values())[0] 39 | return meta 40 | 41 | def get_many(self, obj_or_identifiers) -> dict[Any, dict]: 42 | obj_ids = set(map(self._hist._ensure_obj_id, obj_or_identifiers)) 43 | trans = self._hist.current_transaction() 44 | if trans: 45 | # First, get what we can from the transaction 46 | found = {} 47 | for obj_id in obj_ids: 48 | try: 49 | found[obj_id] = trans.get_meta(obj_id) 50 | except exceptions.NotFound: 51 | pass 52 | 53 | # Now get anything else from the archive 54 | obj_ids -= found.keys() 55 | if obj_ids: 56 | from_archive = self._archive.meta_get_many(obj_ids) 57 | # Now put into the transaction so it doesn't look it up again. 58 | for obj_id in obj_ids: 59 | trans.set_meta(obj_id, from_archive[obj_id]) 60 | found.update(from_archive) 61 | 62 | return found 63 | 64 | # No transaction 65 | return self._archive.meta_get_many(obj_ids) 66 | 67 | def set(self, obj_or_identifier, meta: Optional[Mapping]): 68 | """Set the metadata for an object 69 | 70 | :param obj_or_identifier: either the object instance, an object ID or a snapshot reference 71 | :param meta: the metadata dictionary 72 | """ 73 | obj_id = self._hist._ensure_obj_id(obj_or_identifier) 74 | trans = self._hist.current_transaction() 75 | if trans: 76 | return trans.set_meta(obj_id, meta) 77 | 78 | return self._archive.meta_set(obj_id, meta) 79 | 80 | def set_many(self, metas: Mapping[Any, Optional[dict]]): 81 | mapped = {self._hist._ensure_obj_id(ident): meta for ident, meta in metas.items()} 82 | trans = self._hist.current_transaction() 83 | if trans: 84 | for entry in mapped.items(): 85 | trans.set_meta(*entry) 86 | else: 87 | self._archive.meta_set_many(mapped) 88 | 89 | def update(self, obj_or_identifier, meta: Mapping): 90 | """Update the metadata for an object 91 | 92 | :param obj_or_identifier: either the object instance, an object ID or a snapshot reference 93 | :param meta: the metadata dictionary 94 | """ 95 | obj_id = self._hist._ensure_obj_id(obj_or_identifier) 96 | trans = self._hist.current_transaction() 97 | if trans: 98 | # Update the metadata in the transaction 99 | try: 100 | current = trans.get_meta(obj_id) 101 | except exceptions.NotFound: 102 | current = self._archive.meta_get(obj_id) # Try the archive 103 | if current is None: 104 | current = {} # Ok, no meta 105 | 106 | current.update(meta) 107 | trans.set_meta(obj_id, current) 108 | else: 109 | self._archive.meta_update(obj_id, meta) 110 | 111 | def update_many(self, metas: Mapping[Any, Optional[dict]]): 112 | mapped = {self._hist._ensure_obj_id(ident): meta for ident, meta in metas.items()} 113 | trans = self._hist.current_transaction() 114 | if trans: 115 | for entry in mapped.items(): 116 | self.update(*entry) 117 | else: 118 | self._archive.meta_update_many(mapped) 119 | 120 | def find( 121 | self, filter, obj_id=None # pylint: disable=redefined-builtin 122 | ) -> Iterator["mincepy.Archive.MetaEntry"]: 123 | """Find metadata matching the given criteria. Each returned result is a tuple containing 124 | the corresponding object id and the metadata dictionary itself""" 125 | return self._archive.meta_find(filter=filter, obj_id=obj_id) 126 | 127 | def distinct( 128 | self, 129 | key: str, 130 | filter: dict = None, # pylint: disable=redefined-builtin 131 | obj_id=None, 132 | ) -> Iterator: 133 | """Yield distinct values found for 'key' within metadata documents, optionally matching a 134 | search filter. 135 | 136 | The search can optionally be restricted to a set of passed object ids. 137 | 138 | :param key: the document key to get distinct values for 139 | :param filter: a query filter for the search 140 | :param obj_id: an optional restriction on the object ids to search. This ben be either: 141 | 1. a single object id 142 | 2. an iterable of object ids in which is treated as {'$in': list(obj_ids)} 143 | 3. a general query filter to be applied to the object ids 144 | """ 145 | return self._archive.meta_distinct(key, filter=filter, obj_id=obj_id) 146 | 147 | def create_index(self, keys, unique=False, where_exist=False): 148 | """Create an index on the metadata. Takes either a single key or list of (key, direction) 149 | pairs 150 | 151 | :param keys: the key or keys to create the index on 152 | :param unique: if True, create a uniqueness constraint on this index 153 | :param where_exist: if True, only apply this index on documents that contain the key(s) 154 | """ 155 | self._archive.meta_create_index(keys, unique=unique, where_exist=where_exist) 156 | -------------------------------------------------------------------------------- /src/mincepy/hist/references.py: -------------------------------------------------------------------------------- 1 | from typing import Generic, Set, TypeVar, overload 2 | 3 | import networkx 4 | from networkx.algorithms import dag 5 | 6 | from mincepy import archives, operations, records, transactions 7 | 8 | __all__ = ("References",) 9 | 10 | IdT = TypeVar("IdT") # The archive ID type 11 | 12 | 13 | class References(Generic[IdT]): 14 | """ 15 | A class that can provide reference graph information about objects stored in the archive. 16 | 17 | .. note:: 18 | It is deliberately not possible to pass an object directly to methods in this class as what 19 | is returned is reference information from the archive and _not_ reference information about 20 | the in-memory python object. 21 | """ 22 | 23 | def __init__(self, historian): 24 | self._historian = historian 25 | self._archive = historian.archive # type: archives.Archive 26 | 27 | SnapshotId = records.SnapshotId[IdT] 28 | 29 | @overload 30 | def references(self, identifier: IdT) -> Set[IdT]: ... 31 | 32 | @overload 33 | def references(self, identifier: "SnapshotId") -> "Set[SnapshotId]": ... 34 | 35 | def references(self, identifier): 36 | """Get the ids of the objects referred to by the passed identifier.""" 37 | if isinstance(identifier, records.SnapshotId): 38 | graph = self.get_snapshot_ref_graph(identifier, max_dist=1) 39 | elif isinstance(identifier, self._archive.get_id_type()): 40 | graph = self.get_obj_ref_graph(identifier, max_dist=1) 41 | else: 42 | raise TypeError(identifier) 43 | 44 | return set(edge[1] for edge in graph.edges) 45 | 46 | @overload 47 | def referenced_by(self, identifier: IdT) -> "Set[IdT]": ... 48 | 49 | @overload 50 | def referenced_by(self, identifier: "SnapshotId") -> "Set[SnapshotId]": ... 51 | 52 | def referenced_by(self, identifier): 53 | """Get the ids of the objects that refer to the passed object""" 54 | if isinstance(identifier, records.SnapshotId): 55 | graph = self.get_snapshot_ref_graph(identifier, direction=archives.INCOMING, max_dist=1) 56 | elif isinstance(identifier, self._archive.get_id_type()): 57 | graph = self.get_obj_ref_graph(identifier, direction=archives.INCOMING, max_dist=1) 58 | else: 59 | raise TypeError(identifier) 60 | 61 | return set(edge[0] for edge in graph.edges) 62 | 63 | def get_snapshot_ref_graph( 64 | self, *snapshot_ids: SnapshotId, direction=archives.OUTGOING, max_dist: int = None 65 | ) -> networkx.DiGraph: 66 | 67 | return self._archive.get_snapshot_ref_graph( 68 | *snapshot_ids, direction=direction, max_dist=max_dist 69 | ) 70 | 71 | def get_obj_ref_graph( 72 | self, *obj_ids: IdT, direction=archives.OUTGOING, max_dist: int = None 73 | ) -> networkx.DiGraph: 74 | obj_ids = set(obj_ids) 75 | graph = self._archive.get_obj_ref_graph(*obj_ids, direction=direction, max_dist=max_dist) 76 | 77 | # If there is a transaction then we should fix up the graph to contain information from that 78 | # too 79 | trans = self._historian.current_transaction() # type: transactions.Transaction 80 | if trans is not None: 81 | _update_from_transaction(graph, trans) 82 | 83 | # Now cull all the nodes not reachable from the nodes of interest 84 | 85 | # Now, get the subgraph we're interested in 86 | reachable = set() 87 | for obj_id in obj_ids: 88 | if direction == archives.OUTGOING: 89 | reachable.update(dag.descendants(graph, obj_id)) 90 | else: 91 | reachable.update(dag.ancestors(graph, obj_id)) 92 | 93 | # Remove all non-reachable nodes except obj_ids as these can stay even if they have no 94 | # edges 95 | graph.remove_nodes_from(set(graph.nodes) - obj_ids - reachable) 96 | 97 | return graph 98 | 99 | 100 | def _update_from_transaction(graph: networkx.DiGraph, transaction: transactions.Transaction): 101 | """Given a transaction update the reference graph to reflect the insertion of any new records""" 102 | for op in transaction.staged: # pylint: disable=invalid-name 103 | if isinstance(op, operations.Insert): 104 | # Modify the graph to reflect the insertion 105 | obj_id = op.obj_id 106 | if obj_id in graph.nodes: 107 | # Incoming edges (things referencing this object) can stay, as they haven't 108 | # changed but outgoing edges may have 109 | out_edges = tuple(graph.out_edges(obj_id)) 110 | graph.remove_edges_from(out_edges) 111 | 112 | # And add in the current ones 113 | for refs in op.record.get_references(): 114 | graph.add_edge(obj_id, refs[1].obj_id) 115 | -------------------------------------------------------------------------------- /src/mincepy/hist/snapshots.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Callable 3 | 4 | from mincepy import archives, frontend, operations, result_types 5 | import mincepy.records as records_ 6 | 7 | __all__ = ("SnapshotsCollection",) 8 | 9 | logger = logging.getLogger(__name__) # pylint: disable=invalid-name 10 | 11 | 12 | class SnapshotsCollection(frontend.ObjectCollection): 13 | def __init__(self, historian, archive_collection: archives.Collection): 14 | super().__init__( 15 | historian, 16 | archive_collection, 17 | record_factory=lambda record_dict: SnapshotLoadableRecord( 18 | record_dict, historian.load_snapshot_from_record 19 | ), 20 | obj_loader=historian.load_snapshot_from_record, 21 | ) 22 | 23 | def purge(self, deleted=True, dry_run=True) -> result_types.PurgeResult: 24 | """Function to delete various unused objects from the database. 25 | 26 | This function cannot and will never delete data the is currently in use.""" 27 | if deleted: 28 | # First find all the object ids of those that have been deleted 29 | # pylint: disable=protected-access 30 | res = self.records.find(records_.DataRecord.state == records_.DELETED)._project( 31 | records_.OBJ_ID 32 | ) 33 | obj_ids = [entry[records_.OBJ_ID] for entry in res] # DB HIT 34 | 35 | logging.debug("Found %i objects that have been deleted", len(obj_ids)) 36 | 37 | # Need the object id and version to create the snapshot ids 38 | # pylint: disable=protected-access 39 | res = self.records.find(records_.DataRecord.obj_id.in_(*obj_ids))._project( 40 | records_.OBJ_ID, records_.VERSION 41 | ) 42 | snapshot_ids = [records_.SnapshotId(**entry) for entry in res] # DB HIT 43 | 44 | logging.info( 45 | "Found %i objects with %i snapshots that are deleted, removing.", 46 | len(obj_ids), 47 | len(snapshot_ids), 48 | ) 49 | 50 | if snapshot_ids and not dry_run: 51 | # Commit the changes 52 | self._historian.archive.bulk_write([operations.Delete(sid) for sid in snapshot_ids]) 53 | logging.info("Deleted %i snapshots", len(snapshot_ids)) 54 | 55 | return result_types.PurgeResult(set(snapshot_ids)) 56 | 57 | 58 | class SnapshotLoadableRecord(records_.DataRecord): 59 | __slots__ = () 60 | 61 | def __new__(cls, record_dict: dict, snapshot_loader: Callable[[records_.DataRecord], object]): 62 | loadable = super().__new__(cls, **record_dict) 63 | loadable._snapshot_loader = snapshot_loader 64 | return loadable 65 | 66 | def load(self) -> object: 67 | return self._snapshot_loader(self) 68 | -------------------------------------------------------------------------------- /src/mincepy/history.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module exposes some global functionality for connecting to and interacting with the current 3 | historian 4 | """ 5 | 6 | from typing import TYPE_CHECKING, Optional, Union 7 | from urllib import parse 8 | 9 | from . import archive_factory, helpers, historians, plugins 10 | 11 | if TYPE_CHECKING: 12 | import mincepy 13 | 14 | __all__ = ( 15 | "connect", 16 | "create_historian", 17 | "get_historian", 18 | "set_historian", 19 | "load", 20 | "save", 21 | "find", 22 | "delete", 23 | "db", 24 | ) 25 | 26 | CURRENT_HISTORIAN = None 27 | 28 | 29 | def connect( 30 | uri: Union[str, parse.ParseResult] = "", use_globally=False, timeout=30000 31 | ) -> "mincepy.Historian": 32 | """Connect to an archive and return a corresponding historian 33 | 34 | :param uri: the URI of the archive to connect to 35 | :param use_globally: if True sets the newly create historian as the current global historian 36 | :param timeout: a connection timeout (in milliseconds) 37 | """ 38 | uri = uri or archive_factory.default_archive_uri() 39 | hist = create_historian(uri, apply_plugins=True, connect_timeout=timeout) 40 | if use_globally: 41 | set_historian(hist, apply_plugins=False) 42 | return hist 43 | 44 | 45 | def create_historian( 46 | archive_uri: Union[str, parse.ParseResult], apply_plugins=True, connect_timeout=30000 47 | ) -> "mincepy.Historian": 48 | """Convenience function to create a standard historian directly from an archive URI 49 | 50 | :param archive_uri: the specification of where to connect to 51 | :param apply_plugins: register the plugin types with the new historian 52 | :param connect_timeout: a connection timeout (in milliseconds) 53 | """ 54 | historian = historians.Historian( 55 | archive_factory.create_archive(archive_uri, connect_timeout=connect_timeout) 56 | ) 57 | if apply_plugins: 58 | historian.register_types(plugins.get_types()) 59 | 60 | return historian 61 | 62 | 63 | # region Globals 64 | 65 | 66 | def get_historian(create=True) -> Optional["mincepy.Historian"]: 67 | """Get the currently set global historian. If one doesn't exist and create is True then this 68 | call will attempt to create a new default historian using connect()""" 69 | global CURRENT_HISTORIAN # pylint: disable=global-statement, global-variable-not-assigned 70 | if CURRENT_HISTORIAN is None and create: 71 | # Try creating a new one, use globally otherwise a new one will be created each time which 72 | # is unlikely to be what users want 73 | connect(use_globally=True) 74 | 75 | return CURRENT_HISTORIAN 76 | 77 | 78 | def set_historian(new_historian: Optional["mincepy.Historian"], apply_plugins=True): 79 | """Set the current global historian. Optionally load all plugins. 80 | To reset the historian pass None. 81 | """ 82 | global CURRENT_HISTORIAN # pylint: disable=global-statement 83 | if new_historian is not None and apply_plugins: 84 | new_historian.register_types(plugins.get_types()) 85 | CURRENT_HISTORIAN = new_historian 86 | 87 | 88 | def load(*obj_ids_or_refs): 89 | """Load one or more objects using the current global historian""" 90 | return get_historian().load(*obj_ids_or_refs) 91 | 92 | 93 | def save(*objs): 94 | """Save one or more objects. See :py:meth:`mincepy.Historian.save`""" 95 | return get_historian().save(*objs) 96 | 97 | 98 | def find(*args, **kwargs): 99 | """Find objects. See :py:meth:`mincepy.Historian.find`""" 100 | yield from get_historian().find(*args, **kwargs) 101 | 102 | 103 | def delete(*obj_or_identifier): 104 | """Delete an object. See :py:meth:`mincepy.Historian.delete`""" 105 | return get_historian().delete(*obj_or_identifier) 106 | 107 | 108 | def db( 109 | type_id_or_type: "mincepy.typing.TypeIdOrType", 110 | ) -> helpers.TypeHelper: # pylint: disable=invalid-name 111 | """Get the database type helper for a type. See :py:meth:`mincepy.Historian.get_helper`""" 112 | return get_historian().get_helper(type_id_or_type) 113 | 114 | 115 | # endregion 116 | -------------------------------------------------------------------------------- /src/mincepy/migrate.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING, Iterable, Iterator, Sequence 2 | 3 | from . import depositors, helpers, records 4 | from .qops import elem_match_, lt_, or_ 5 | 6 | if TYPE_CHECKING: 7 | import mincepy 8 | 9 | __all__ = ("Migrations",) 10 | 11 | 12 | def _state_types_migration_condition(helper: helpers.TypeHelper) -> dict: 13 | return elem_match_( 14 | **{ 15 | "1": {"$eq": helper.TYPE_ID}, # Type id has to match, and, 16 | **or_( 17 | # version has to be less than the current, or, 18 | {"2": lt_(helper.get_version())}, 19 | # there is no version number 20 | {"2": None}, 21 | {"2": {"$exists": False}}, 22 | ), 23 | } 24 | ) 25 | 26 | 27 | class Migrations: 28 | """The historian migrations namespace""" 29 | 30 | def __init__(self, historian: "mincepy.Historian"): 31 | self._historian = historian 32 | 33 | def find_migratable_records(self) -> Iterator[records.DataRecord]: 34 | """Find archive records that can be migrated""" 35 | type_registry = self._historian.type_registry 36 | # Find all the types in the registry that have migrations 37 | have_migrations = [ 38 | helper 39 | for helper in type_registry.type_helpers.values() 40 | if helper.get_version() is not None 41 | ] 42 | 43 | if not have_migrations: 44 | return iter([]) 45 | 46 | # Now, let's look for those records that would need migrating 47 | archive = self._historian.archive 48 | query = or_(*list(map(_state_types_migration_condition, have_migrations))) 49 | return archive.find(state_types=query) 50 | 51 | def migrate_all(self) -> Sequence[records.DataRecord]: 52 | """Migrate all records that can be updated""" 53 | to_migrate = self.find_migratable_records() 54 | return self.migrate_records(to_migrate) 55 | 56 | def migrate_records( 57 | self, to_migrate: Iterable[records.DataRecord] 58 | ) -> Sequence[records.DataRecord]: 59 | """Migrate the given records (if possible). Returns all the records that were actually 60 | migrated.""" 61 | migrator = depositors.Migrator(self._historian) 62 | return migrator.migrate_records(to_migrate) 63 | -------------------------------------------------------------------------------- /src/mincepy/migrations.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING, Any, Optional 2 | 3 | import pytray.pretty 4 | 5 | if TYPE_CHECKING: 6 | import mincepy 7 | 8 | 9 | __all__ = "ObjectMigration", "ObjectMigrationMeta" 10 | 11 | 12 | class ObjectMigrationMeta(type): 13 | PARENT = None 14 | 15 | def __init__(cls, name, bases, dct): 16 | super().__init__(name, bases, dct) 17 | 18 | if cls.PARENT is None or cls == cls.PARENT: 19 | # This one is the parent class for others and doesn't need to pass the checks 20 | return 21 | 22 | if cls.VERSION is None: 23 | raise RuntimeError("Migration version not set") 24 | 25 | if cls.PREVIOUS is not None and cls.VERSION <= cls.PREVIOUS.VERSION: 26 | raise RuntimeError( 27 | f"A migration must have a version number higher than the previous migration. " 28 | f"{pytray.pretty.type_string(cls.PREVIOUS)}.VERSION is {cls.PREVIOUS.VERSION} " 29 | f"while {pytray.pretty.type_string(cls)}.VERSION is {cls.VERSION}" 30 | ) 31 | 32 | if cls.NAME is None: 33 | cls.NAME = cls.__class__.__name__ 34 | 35 | 36 | class ObjectMigration(metaclass=ObjectMigrationMeta): 37 | NAME = None # type: Optional[str] 38 | VERSION = None # type: Optional[int] 39 | PREVIOUS = None # type: Optional[ObjectMigration] 40 | 41 | @classmethod 42 | def upgrade(cls, saved_state, loader: "mincepy.Loader") -> Any: 43 | """ 44 | This method should take the saved state, which will have been created with the previous 45 | version, and return a new saved state that is compatible with this version. 46 | 47 | :raises mincepy.MigrationError: if a problem is encountered during migration 48 | """ 49 | raise NotImplementedError 50 | 51 | 52 | ObjectMigrationMeta.PARENT = ObjectMigration 53 | -------------------------------------------------------------------------------- /src/mincepy/mongo/__init__.py: -------------------------------------------------------------------------------- 1 | from . import mongo_archive 2 | from .mongo_archive import * 3 | 4 | __all__ = mongo_archive.__all__ 5 | -------------------------------------------------------------------------------- /src/mincepy/mongo/aggregation.py: -------------------------------------------------------------------------------- 1 | """Module that contains aggregation operations""" 2 | 3 | 4 | def and_(*conditions) -> dict: 5 | """Helper that produces query dict for AND of multiple conditions""" 6 | if len(conditions) == 1: 7 | return conditions[0] 8 | 9 | return {"$and": list(conditions)} 10 | 11 | 12 | def eq_(one, other) -> dict: 13 | """Helper that produces mongo aggregation dict for two items being equal""" 14 | return {"$eq": [one, other]} 15 | 16 | 17 | def in_(*possibilities) -> dict: 18 | """Tests if a value is one of the possibilities""" 19 | return {"$in": possibilities} 20 | -------------------------------------------------------------------------------- /src/mincepy/mongo/bulk.py: -------------------------------------------------------------------------------- 1 | import functools 2 | 3 | import pymongo 4 | 5 | from mincepy import operations 6 | import mincepy.qops as q 7 | 8 | from . import db 9 | 10 | # To prevent 'op' being flagged 11 | # pylint: disable=invalid-name 12 | 13 | 14 | @functools.singledispatch 15 | def to_mongo_op(op: operations.Operation): 16 | """Convert a mincepy operation to a mongodb one. Returns a tuple of the data operation and 17 | history operation that need to be bulk written""" 18 | raise NotImplementedError 19 | 20 | 21 | @to_mongo_op.register(operations.Insert) 22 | def _(op: operations.Insert): 23 | """Insert""" 24 | record = op.record 25 | document = db.to_document(record) 26 | document["_id"] = record.obj_id 27 | 28 | if record.is_deleted_record(): 29 | data_op = pymongo.operations.DeleteOne( 30 | filter={db.OBJ_ID: record.obj_id, db.VERSION: q.lt_(record.version)}, 31 | ) 32 | else: 33 | data_op = pymongo.operations.UpdateOne( 34 | filter={db.OBJ_ID: record.obj_id, db.VERSION: q.lt_(record.version)}, 35 | update={"$set": document.copy()}, 36 | upsert=True, 37 | ) 38 | 39 | # History uses the sid as the document id 40 | document["_id"] = str(record.snapshot_id) 41 | history_op = pymongo.operations.InsertOne(document) 42 | 43 | return [data_op], [history_op] 44 | 45 | 46 | @to_mongo_op.register(operations.Update) 47 | def _(op: operations.Update): 48 | """Update""" 49 | sid = op.snapshot_id 50 | update = db.to_document(op.update) 51 | update = {"$set": update} 52 | 53 | # It's fine if either (or both) of these fail to find anything to update 54 | data_op = pymongo.operations.UpdateOne( 55 | filter={db.OBJ_ID: sid.obj_id, db.VERSION: sid.version}, update=update 56 | ) 57 | history_op = pymongo.operations.UpdateOne(filter={"_id": str(sid)}, update=update) 58 | 59 | return [data_op], [history_op] 60 | 61 | 62 | @to_mongo_op.register(operations.Delete) 63 | def _(op: operations.Delete): 64 | """Delete""" 65 | sid = op.snapshot_id 66 | 67 | # It's fine if either (or both) of these fail to find anything to update 68 | data_op = pymongo.operations.DeleteOne( 69 | filter={db.OBJ_ID: sid.obj_id, db.VERSION: sid.version}, 70 | ) 71 | history_op = pymongo.operations.DeleteOne( 72 | filter={"_id": str(sid)}, 73 | ) 74 | 75 | return [data_op], [history_op] 76 | 77 | 78 | @to_mongo_op.register(operations.Merge) 79 | def _(op: operations.Merge): 80 | """Merge""" 81 | record = op.record 82 | document = db.to_document(record) 83 | document["_id"] = record.obj_id 84 | 85 | data_ops = [] 86 | 87 | if record.is_deleted_record(): 88 | # Delete record, so expunge the record from the current objects 89 | data_ops.append( 90 | pymongo.operations.DeleteOne( 91 | filter={db.OBJ_ID: record.obj_id, db.VERSION: q.lt_(record.version)}, 92 | ) 93 | ) 94 | else: 95 | data_ops.append( 96 | pymongo.operations.DeleteOne( 97 | filter={db.OBJ_ID: record.obj_id, db.VERSION: q.lt_(record.version)} 98 | ) 99 | ) 100 | data_ops.append( 101 | pymongo.operations.UpdateOne( 102 | filter={db.OBJ_ID: record.obj_id}, 103 | update={"$setOnInsert": document.copy()}, 104 | upsert=True, 105 | ) 106 | ) 107 | 108 | # History uses the sid as the document id 109 | document["_id"] = str(record.snapshot_id) 110 | history_op = pymongo.operations.InsertOne(document) 111 | 112 | return data_ops, [history_op] 113 | -------------------------------------------------------------------------------- /src/mincepy/mongo/db.py: -------------------------------------------------------------------------------- 1 | """This module contains the names of the keys in the various collections used by the mongo 2 | archive and methods to convert mincepy types to mongo collection entries and back""" 3 | 4 | import functools 5 | from typing import Optional 6 | 7 | from bidict import bidict 8 | import bson 9 | import pymongo.collection 10 | import pymongo.errors 11 | 12 | import mincepy.qops as q 13 | import mincepy.records 14 | 15 | SETTINGS_COLLECTION = "settings" 16 | GLOBAL_SETTINGS = "global" 17 | 18 | # region Data collection 19 | OBJ_ID = "obj_id" 20 | VERSION = "ver" 21 | TYPE_ID = mincepy.records.TYPE_ID 22 | CREATION_TIME = "ctime" 23 | STATE = "state" 24 | STATE_TYPES = "state_types" 25 | SNAPSHOT_HASH = "hash" 26 | SNAPSHOT_TIME = "stime" 27 | EXTRAS = mincepy.records.EXTRAS 28 | REFERENCES = "refs" 29 | META = "meta" # The metadata field 30 | 31 | # Here we map the data record property names onto ones in our entry format. 32 | # If a record property doesn't appear here it means the name says the same 33 | KEY_MAP = bidict( 34 | { 35 | mincepy.records.OBJ_ID: OBJ_ID, 36 | mincepy.records.TYPE_ID: TYPE_ID, 37 | mincepy.records.CREATION_TIME: CREATION_TIME, 38 | mincepy.records.VERSION: VERSION, 39 | mincepy.records.STATE: STATE, 40 | mincepy.records.STATE_TYPES: STATE_TYPES, 41 | mincepy.records.SNAPSHOT_HASH: SNAPSHOT_HASH, 42 | mincepy.records.SNAPSHOT_TIME: SNAPSHOT_TIME, 43 | mincepy.records.EXTRAS: EXTRAS, 44 | } 45 | ) 46 | 47 | # endregion 48 | 49 | 50 | def to_record(entry) -> "mincepy.DataRecord": 51 | """Convert a MongoDB data collection entry to a DataRecord""" 52 | record_dict = mincepy.DataRecord.defaults() 53 | 54 | record_dict[mincepy.OBJ_ID] = entry[OBJ_ID] 55 | record_dict[mincepy.VERSION] = entry[VERSION] 56 | 57 | # Invert our mapping of keys back to the data record property names and update over any 58 | # defaults 59 | record_dict.update( 60 | {recordkey: entry[dbkey] for recordkey, dbkey in KEY_MAP.items() if dbkey in entry} 61 | ) 62 | 63 | return mincepy.DataRecord(**record_dict) 64 | 65 | 66 | @functools.singledispatch 67 | def to_document(record, exclude_defaults=False) -> dict: 68 | """Convert mincepy record information to a MongoDB document. Optionally exclude entries that 69 | have the same value as the default""" 70 | raise TypeError(record.__class__) 71 | 72 | 73 | @to_document.register(mincepy.records.DataRecord) 74 | def _(record: mincepy.records.DataRecord, exclude_defaults=False) -> dict: 75 | """Convert a DataRecord to a MongoDB document with our keys""" 76 | defaults = mincepy.DataRecord.defaults() 77 | entry = {} 78 | for key, item in record.__dict__.items(): 79 | db_key = KEY_MAP[key] # pylint: disable=unsubscriptable-object 80 | # Exclude entries that have the default value 81 | if not (exclude_defaults and key in defaults and defaults[key] == item): 82 | entry[db_key] = item 83 | 84 | return entry 85 | 86 | 87 | @to_document.register(dict) 88 | def _(record: dict, exclude_defaults=False) -> dict: 89 | """Convert a dictionary containing record keys to a MongoDB document with our keys""" 90 | defaults = mincepy.DataRecord.defaults() 91 | entry = {} 92 | for key, item in record.items(): 93 | db_key = KEY_MAP[key] # pylint: disable=unsubscriptable-object 94 | # Exclude entries that have the default value 95 | if not (exclude_defaults and key in defaults and defaults[key] == item): 96 | entry[db_key] = item 97 | 98 | return entry 99 | 100 | 101 | def remap(record_dict: Optional[dict]) -> Optional[dict]: 102 | """Given a dictionary return a new dictionary with the key names that we use""" 103 | if record_dict is None: 104 | return None 105 | 106 | remapped = {} 107 | for key, value in record_dict.items(): 108 | remapped[remap_key(key)] = value 109 | return remapped 110 | 111 | 112 | def remap_back(entry_dict: dict) -> dict: 113 | remapped = {} 114 | inverse_map = KEY_MAP.inverse 115 | for key, value in entry_dict.items(): 116 | if key in inverse_map: # pylint: disable=unsupported-membership-test 117 | remapped[inverse_map[key]] = value # pylint: disable=unsubscriptable-object 118 | return remapped 119 | 120 | 121 | def remap_key(key: str) -> str: 122 | """Given a key remap it to the names that we use, even if it as a path e.g. state.colour""" 123 | split_key = key.split(".") 124 | base = KEY_MAP[split_key[0]] # pylint: disable=unsubscriptable-object 125 | split_key[0] = base 126 | return ".".join(split_key) 127 | 128 | 129 | def to_id_dict(sid: mincepy.records.SnapshotId) -> dict: 130 | return {OBJ_ID: sid.obj_id, VERSION: sid.version} 131 | 132 | 133 | def sid_from_dict(record: dict): 134 | return mincepy.SnapshotId(record[OBJ_ID], record[VERSION]) 135 | 136 | 137 | def sid_from_str(sid_str: str): 138 | parts = sid_str.split("#") 139 | return mincepy.SnapshotId(bson.ObjectId(parts[0]), int(parts[1])) 140 | 141 | 142 | def safe_bulk_delete(collection: pymongo.collection.Collection, ids, id_key="_id"): 143 | """ 144 | Sometimes when you want to delete a bunch of documents using an identifier the 'delete document' 145 | itself exceeds the 16MB Mongo limit. This function will catch such cases and break up the 146 | command into suitably batches 147 | """ 148 | ids = list(set(ids)) # No needs to repeat ourselves 149 | try: 150 | collection.delete_many({id_key: q.in_(*ids)}) 151 | except pymongo.errors.DocumentTooLarge: 152 | # Use bulk operation instead 153 | # Note, this could be spead up further by batching the deletes but for now it's not worth it 154 | bulk_ops = [pymongo.DeleteOne({id_key: entry_id}) for entry_id in ids] 155 | collection.bulk_write(bulk_ops) 156 | -------------------------------------------------------------------------------- /src/mincepy/mongo/migrate.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import contextlib 3 | import random 4 | import string 5 | from typing import List, Optional, Type 6 | 7 | import pymongo.database 8 | 9 | from . import settings 10 | 11 | VERSION = "version" 12 | MIGRATIONS = "migrations" 13 | 14 | 15 | class MigrationError(RuntimeError): 16 | """An error that occurred during migration""" 17 | 18 | 19 | class Migration(metaclass=abc.ABCMeta): 20 | """Migration class to be used when making changes to the database. The implementer should 21 | provide a version number that higher than the previous as well as a reference to the previous 22 | migration. 23 | 24 | Migrations should avoid using constants from the codebase as these could change. Ideally each 25 | migration should be self-contained and not refer to any other parts of the code. 26 | """ 27 | 28 | # pylint: disable=invalid-name 29 | NAME = None 30 | VERSION = None 31 | PREVIOUS = None 32 | 33 | def __init__(self): 34 | if self.NAME is None: 35 | self.NAME = self.__class__.__name__ 36 | 37 | if self.VERSION is None: 38 | raise RuntimeError("Migration version not set") 39 | 40 | @abc.abstractmethod 41 | def upgrade(self, database: pymongo.database.Database): 42 | """Apply the transformations to bring the database up to this version""" 43 | current = settings.get_settings(database) or {} 44 | # Update the version 45 | current[VERSION] = self.VERSION 46 | # and migrations 47 | current.setdefault(MIGRATIONS, []).append(self.NAME) 48 | settings.set_settings(database, current) 49 | 50 | 51 | class MigrationManager: 52 | def __init__(self, latest: Type[Migration]): 53 | self.latest = latest 54 | 55 | def migration_required(self, database: pymongo.database.Database) -> bool: 56 | return len(self._get_required_migrations(database)) > 0 57 | 58 | def migrate(self, database: pymongo.database.Database) -> int: 59 | """Migrate the database and returns the number of migrations applied""" 60 | migrations = self._get_required_migrations(database) 61 | 62 | if migrations: 63 | for migration in reversed(migrations): 64 | migration().upgrade(database) 65 | 66 | # Recheck the settings to make sure the migration has been applied 67 | current = settings.get_settings(database) 68 | if current[MIGRATIONS][-1] != migrations[0].NAME: 69 | raise RuntimeError( 70 | "After applying migrations, latest migration doesn't seem to be applied" 71 | ) 72 | 73 | return len(migrations) 74 | 75 | def get_migration_sequence(self) -> List[Type[Migration]]: 76 | """Get the sequence of migrations in REVERSE order i.e. the 0th entry is the latest""" 77 | # Establish the sequence of migrations 78 | migrations = [self.latest] 79 | while migrations[-1].PREVIOUS is not None: 80 | migrations.append(migrations[-1].PREVIOUS) 81 | 82 | return migrations 83 | 84 | def _get_required_migrations( 85 | self, database: pymongo.database.Database 86 | ) -> List[Type[Migration]]: 87 | current = settings.get_settings(database) or {} 88 | migrations = self.get_migration_sequence() 89 | # Find out where we're at 90 | applied = current.get(MIGRATIONS, []) 91 | 92 | if applied: 93 | # Find the last one that was applied 94 | while migrations and applied[-1] != migrations.pop().NAME: 95 | pass 96 | 97 | # Anything left in the migrations list needs to be applied 98 | return migrations 99 | 100 | 101 | def ensure_up_to_date(database: pymongo.database.Database, latest: Type[Migration]): 102 | """Apply any necessary migrations to the database""" 103 | current = settings.get_settings(database) or {} 104 | current_version = current.get(VERSION, None) 105 | if current_version and current_version > latest.VERSION: 106 | raise MigrationError( 107 | f"The current database version ({current_version}) is higher than the code version " 108 | f"({latest.VERSION}) you may need to update your version of the code" 109 | ) 110 | 111 | migrator = MigrationManager(latest) 112 | return migrator.migrate(database) 113 | 114 | 115 | def get_version(database) -> Optional[int]: 116 | current = settings.get_settings(database) or {} 117 | return current.get(VERSION, None) 118 | 119 | 120 | @contextlib.contextmanager 121 | def temporary_collection(database: pymongo.database.Database, coll_name=None): 122 | coll_name = coll_name or "".join(random.choices(string.ascii_letters, k=10)) # nosec 123 | coll = database[coll_name] 124 | yield coll 125 | database.drop_collection(coll_name) 126 | -------------------------------------------------------------------------------- /src/mincepy/mongo/queries.py: -------------------------------------------------------------------------------- 1 | import functools 2 | from typing import Mapping 3 | 4 | from . import db 5 | from .aggregation import and_, eq_ 6 | 7 | 8 | def pipeline_latest_version(data_collection: str) -> list: 9 | """Returns a pipeline that will take the incoming data record documents and for each one find 10 | the latest version.""" 11 | oid_var = f"${db.OBJ_ID}" 12 | ver_var = f"${db.VERSION}" 13 | 14 | pipeline = [] 15 | pipeline.extend( 16 | [ 17 | # Group by object id the maximum version 18 | {"$group": {"_id": oid_var, "max_ver": {"$max": ver_var}}}, 19 | # Then do a lookup against the same collection to get the records 20 | { 21 | "$lookup": { 22 | "from": data_collection, 23 | "let": {"obj_id": "$_id", "max_ver": "$max_ver"}, 24 | "pipeline": [ 25 | { 26 | "$match": { 27 | "$expr": and_( 28 | # Match object id and version 29 | eq_(oid_var, "$$obj_id"), 30 | eq_(ver_var, "$$max_ver"), 31 | ), 32 | } 33 | } 34 | ], 35 | "as": "latest", 36 | } 37 | }, 38 | # Now unwind and promote the 'latest' field 39 | {"$unwind": {"path": "$latest"}}, 40 | {"$replaceRoot": {"newRoot": "$latest"}}, 41 | ] 42 | ) 43 | 44 | return pipeline 45 | 46 | 47 | class QueryBuilder: 48 | """Simple MongoDB query builder. Creates a compound query of one or more terms""" 49 | 50 | def __init__(self, *terms): 51 | self._terms = [] 52 | for term in terms: 53 | self.and_(term) 54 | 55 | def and_(self, *term: dict): 56 | for entry in term: 57 | if not isinstance(entry, dict): 58 | raise TypeError(entry) 59 | 60 | self._terms.extend(term) 61 | 62 | def build(self): 63 | if not self._terms: 64 | return {} 65 | if len(self._terms) == 1: 66 | return self._terms[0] 67 | 68 | return {"$and": self._terms.copy()} 69 | 70 | 71 | def flatten_filter(entry_name: str, query) -> list: 72 | """Expand nested search criteria, e.g. state={'color': 'red'} -> {'state.colour': 'red'}""" 73 | if isinstance(query, dict): 74 | transformed = _transform_query_keys(query, entry_name) 75 | flattened = [{key: value} for key, value in transformed.items()] 76 | else: 77 | flattened = [{entry_name: query}] 78 | 79 | return flattened 80 | 81 | 82 | def expand_filter(entry_name: str, query: Mapping) -> dict: 83 | if not query: 84 | return {} 85 | return {f"{entry_name}.{key}": value for key, value in query.items()} 86 | 87 | 88 | @functools.singledispatch 89 | def _transform_query_keys(entry, prefix: str = ""): # pylint: disable=unused-argument 90 | """Transform a query entry into the correct syntax given a global prefix and the entry itself""" 91 | return entry 92 | 93 | 94 | @_transform_query_keys.register(list) 95 | def _(entry: list, prefix: str = ""): 96 | return [_transform_query_keys(value, prefix) for value in entry] 97 | 98 | 99 | @_transform_query_keys.register(dict) 100 | def _(entry: dict, prefix: str = ""): 101 | transformed = {} 102 | for key, value in entry.items(): 103 | if key.startswith("$"): 104 | if key in ("$and", "$not", "$nor", "$or"): 105 | transformed[key] = _transform_query_keys(value, prefix) 106 | else: 107 | update = {prefix: {key: value}} if prefix else {key: value} 108 | transformed.update(update) 109 | else: 110 | to_join = [prefix, key] if prefix else [key] 111 | # Don't pass the prefix on, we've consumed it here 112 | transformed[".".join(to_join)] = _transform_query_keys(value) 113 | 114 | return transformed 115 | -------------------------------------------------------------------------------- /src/mincepy/mongo/settings.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | import pymongo.database 4 | 5 | from . import db 6 | 7 | 8 | def get_settings( 9 | database: pymongo.database.Database, name: str = db.GLOBAL_SETTINGS 10 | ) -> Optional[dict]: 11 | return database[db.SETTINGS_COLLECTION].find_one(name) 12 | 13 | 14 | def set_settings( 15 | database: pymongo.database.Database, 16 | settings: Optional[dict], 17 | name: str = db.GLOBAL_SETTINGS, 18 | ): 19 | coll = database[db.SETTINGS_COLLECTION] 20 | 21 | if settings is None: 22 | coll.delete_one({"_id": name}) 23 | else: 24 | coll.replace_one({"_id": name}, settings, upsert=True) 25 | -------------------------------------------------------------------------------- /src/mincepy/mongo/types.py: -------------------------------------------------------------------------------- 1 | import bson 2 | 3 | from mincepy import archives 4 | 5 | # pylint: disable=invalid-name 6 | 7 | # Problem with subscripting archive, bug report here: 8 | # https://github.com/PyCQA/pylint/issues/2822 9 | _Archive = archives.Archive[bson.ObjectId] # pylint: disable=unsubscriptable-object 10 | SnapshotId = _Archive.SnapshotId 11 | -------------------------------------------------------------------------------- /src/mincepy/operations.py: -------------------------------------------------------------------------------- 1 | """Module containing record operations that can be performed sent to the archive to perform""" 2 | 3 | import abc 4 | 5 | from . import records 6 | 7 | __all__ = "Operation", "Insert", "Update", "Delete" 8 | 9 | 10 | class Operation(metaclass=abc.ABCMeta): 11 | """Base class for all operations""" 12 | 13 | @property 14 | @abc.abstractmethod 15 | def obj_id(self): 16 | """The is of the object being operated on""" 17 | 18 | @property 19 | @abc.abstractmethod 20 | def snapshot_id(self) -> records.SnapshotId: 21 | """The snapshot id of the object being operated on""" 22 | 23 | 24 | class Insert(Operation): 25 | """Insert a record into the archive 26 | 27 | For use with :meth:`~mincepy.Archive.bulk_write` 28 | """ 29 | 30 | def __init__(self, record: records.DataRecord): 31 | self._record = record 32 | 33 | @property 34 | def obj_id(self): 35 | return self._record.obj_id 36 | 37 | @property 38 | def snapshot_id(self) -> records.SnapshotId: 39 | return self._record.snapshot_id 40 | 41 | @property 42 | def record(self) -> records.DataRecord: 43 | return self._record 44 | 45 | 46 | class Update(Operation): 47 | """Update a record currently in the archive. This takes the snapshot id and a dictionary 48 | containing the fields to be updated. The update operation behaves like a dict.update()""" 49 | 50 | def __init__(self, sid: records.SnapshotId, update: dict): 51 | diff = set(update.keys()) - set(records.DataRecord._fields) 52 | if diff: 53 | raise ValueError(f"Invalid keys found in the update operation: {diff}") 54 | 55 | self._sid = sid 56 | self._update = update 57 | 58 | @property 59 | def obj_id(self): 60 | return self._sid.obj_id 61 | 62 | @property 63 | def snapshot_id(self) -> records.SnapshotId: 64 | """The snapshot being updated""" 65 | return self._sid 66 | 67 | @property 68 | def update(self) -> dict: 69 | """The update that will be performed""" 70 | return self._update 71 | 72 | 73 | class Delete(Operation): 74 | """Delete a record from the archive""" 75 | 76 | def __init__(self, sid: records.SnapshotId): 77 | self._sid = sid 78 | 79 | @property 80 | def obj_id(self): 81 | return self._sid.obj_id 82 | 83 | @property 84 | def snapshot_id(self) -> records.SnapshotId: 85 | """The snapshot being deleted""" 86 | return self._sid 87 | 88 | 89 | class Merge(Operation): 90 | """Merge a record into the archive. This could be: 91 | * An entirely new snapshot, i.e. the object id doesn't exist in the archive at all 92 | * A new version of a record, i.e. the object id does exist but this version is newer than 93 | any other 94 | * An old version of a record, i.e. the object id does exist but this version is older than 95 | the latest 96 | 97 | In any case the snapshot id should not exist in the database already. 98 | """ 99 | 100 | def __init__(self, record: records.DataRecord): 101 | self._record = record 102 | 103 | @property 104 | def obj_id(self): 105 | return self._record.obj_id 106 | 107 | @property 108 | def snapshot_id(self) -> records.SnapshotId: 109 | return self._record.snapshot_id 110 | 111 | @property 112 | def record(self) -> records.DataRecord: 113 | return self._record 114 | -------------------------------------------------------------------------------- /src/mincepy/plugins.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import List 3 | 4 | import stevedore 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | def load_failed(_manager, entrypoint, exception): 10 | logger.warning("Error loading mincepy plugin from entrypoing '%s':\n%s", entrypoint, exception) 11 | 12 | 13 | def get_types() -> List: 14 | """Get all mincepy types and type helper instances registered as extensions""" 15 | mgr = stevedore.extension.ExtensionManager( 16 | namespace="mincepy.plugins.types", 17 | invoke_on_load=False, 18 | on_load_failure_callback=load_failed, 19 | ) 20 | 21 | all_types = [] 22 | 23 | def get_type(extension: stevedore.extension.Extension): 24 | try: 25 | all_types.extend(extension.plugin()) 26 | except Exception: # pylint: disable=broad-except 27 | logger.exception("Failed to get types plugin from %s", extension.name) 28 | 29 | mgr.map(get_type) 30 | 31 | return all_types 32 | -------------------------------------------------------------------------------- /src/mincepy/process.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import uuid 3 | 4 | from . import base_savable, tracking 5 | 6 | __all__ = ("Process",) 7 | 8 | 9 | class Process( 10 | base_savable.SimpleSavable, type_id=uuid.UUID("bcf03171-a1f1-49c7-b890-b7f9d9f9e5a2") 11 | ): 12 | STACK = [] 13 | ATTRS = "_name", "_running" 14 | 15 | def __init__(self, name: str): 16 | super().__init__() 17 | self._name = name 18 | self._running = 0 19 | 20 | def __eq__(self, other): 21 | if not isinstance(other, Process): 22 | return False 23 | 24 | return self.name == other.name 25 | 26 | @property 27 | def is_running(self): 28 | return self._running != 0 29 | 30 | @property 31 | def name(self) -> str: 32 | return self._name 33 | 34 | @contextlib.contextmanager 35 | def running(self): 36 | try: 37 | self._running += 1 38 | with tracking.track(self): 39 | yield 40 | finally: 41 | self._running -= 1 42 | -------------------------------------------------------------------------------- /src/mincepy/provides.py: -------------------------------------------------------------------------------- 1 | from . import testing 2 | 3 | 4 | def get_types(): 5 | """Provide a list of all historian types""" 6 | types = [] 7 | types.extend(testing.HISTORIAN_TYPES) 8 | 9 | return types 10 | -------------------------------------------------------------------------------- /src/mincepy/qops.py: -------------------------------------------------------------------------------- 1 | """Module containing functions to generate query operations. To prevent clashes with python 2 | builtins we append underscores to the function names. This also makes it safer to import this 3 | module as a wildcard import. 4 | """ 5 | 6 | __all__ = ( 7 | "and_", 8 | "or_", 9 | "eq_", 10 | "exists_", 11 | "elem_match_", 12 | "gt_", 13 | "gte_", 14 | "in_", 15 | "lt_", 16 | "lte_", 17 | "ne_", 18 | "nin_", 19 | ) 20 | 21 | # region Logical operators 22 | 23 | 24 | def and_(*conditions) -> dict: 25 | """Helper that produces query dict for AND of multiple conditions""" 26 | if len(conditions) == 1: 27 | return conditions[0] 28 | 29 | return {"$and": list(conditions)} 30 | 31 | 32 | def or_(*conditions) -> dict: 33 | """Helper that produces query dict for OR of multiple conditions""" 34 | if len(conditions) == 1: 35 | return conditions[0] 36 | 37 | return {"$or": list(conditions)} 38 | 39 | 40 | # endregion 41 | 42 | # region Element operators 43 | 44 | 45 | def exists_(key, value: bool = True) -> dict: 46 | """Return condition for the existence of a key. If True, matches if key exists, if False 47 | matches if it does not.""" 48 | return {key: {"$exists": value}} 49 | 50 | 51 | # endregion 52 | 53 | # region Array operators 54 | 55 | 56 | def elem_match_(**conditions) -> dict: 57 | """Match an element that is an array and has at least one member that matches all the specified 58 | conditions""" 59 | return {"$elemMatch": conditions} 60 | 61 | 62 | # endregion 63 | 64 | # region Comparison operators 65 | 66 | 67 | def eq_(value) -> dict: 68 | """Helper that produces mongo query dict for two items being equal""" 69 | return {"$eq": value} 70 | 71 | 72 | def gt_(quantity) -> dict: 73 | """Match values greater than quantity""" 74 | return {"$gt": quantity} 75 | 76 | 77 | def gte_(quantity) -> dict: 78 | """Match values greater than or equal to quantity""" 79 | return {"$gte": quantity} 80 | 81 | 82 | def in_(*possibilities) -> dict: 83 | """Match values that are equal to any of the possibilities""" 84 | if len(possibilities) == 1: 85 | return possibilities[0] 86 | 87 | return {"$in": list(possibilities)} 88 | 89 | 90 | def lt_(quantity) -> dict: 91 | """Match values less than quantity""" 92 | return {"$lt": quantity} 93 | 94 | 95 | def lte_(quantity) -> dict: 96 | """Match values less than or equal to quantity""" 97 | return {"$lte": quantity} 98 | 99 | 100 | def ne_(value) -> dict: 101 | """Match values not equal to to value""" 102 | return {"$ne": value} 103 | 104 | 105 | def nin_(*possibilities) -> dict: 106 | """Match values that are not equal to any of the possibilities""" 107 | if len(possibilities) == 1: 108 | return ne_(possibilities[0]) 109 | 110 | return {"$nin": list(possibilities)} 111 | 112 | 113 | # endregion 114 | -------------------------------------------------------------------------------- /src/mincepy/refs.py: -------------------------------------------------------------------------------- 1 | """Module for mincepy object references""" 2 | 3 | from collections.abc import Hashable 4 | from typing import TYPE_CHECKING, Iterator, Optional 5 | 6 | from typing_extensions import override 7 | 8 | from . import exceptions, records, type_ids, types 9 | 10 | if TYPE_CHECKING: 11 | import mincepy 12 | 13 | __all__ = ("ObjRef", "ref") 14 | 15 | 16 | class ObjRef(types.SavableObject, type_id=type_ids.OBJ_REF_TYPE_ID, immutable=True): 17 | """A reference to an object instance""" 18 | 19 | _obj = None 20 | _sid: Optional["mincepy.SnapshotId"] = None 21 | _loader = None 22 | 23 | def __init__(self, obj=None): 24 | super().__init__() 25 | assert not ( 26 | obj is not None and types.is_primitive(obj) 27 | ), "Can't create a reference to a primitive type" 28 | self._obj = obj 29 | 30 | def __bool__(self) -> bool: 31 | """Test if this is a null reference""" 32 | return self._obj is not None or self._sid is not None 33 | 34 | def __str__(self) -> str: 35 | desc = ["ObjRef('"] 36 | if self._obj is not None: 37 | desc.append(str(self._obj)) 38 | else: 39 | desc.append(str(self._sid)) 40 | desc.append("')") 41 | return "".join(desc) 42 | 43 | def __repr__(self) -> str: 44 | return f"ObjRef({self._obj if self._obj is not None else self._sid})" 45 | 46 | def __call__(self, update=False) -> object: 47 | """Get the object being referenced. If update is called then the latest version 48 | will be loaded from the historian""" 49 | if self._obj is None: 50 | # This means we were loaded and need to load the object 51 | if self._sid is None: 52 | raise RuntimeError("Cannot dereference a None reference") 53 | # Cache the object 54 | self._obj = self._loader.load(self._sid) 55 | assert ( # nosec: intentional internal assert 56 | self._obj is not None 57 | ), f"Loader did not load object using SID {self._sid}" 58 | self._sid = None 59 | self._loader = None 60 | elif update: 61 | try: 62 | self._historian.sync(self._obj) 63 | except exceptions.NotFound: 64 | pass # Object must have never been saved and is therefore up-to-date 65 | 66 | return self._obj 67 | 68 | def __eq__(self, other) -> bool: 69 | if not isinstance(other, ObjRef): 70 | return False 71 | 72 | if self._obj is not None: 73 | return id(self._obj) == id(other._obj) 74 | 75 | return self._sid == other._sid 76 | 77 | @override 78 | def yield_hashables(self, hasher, /) -> Iterator[Hashable]: 79 | if self._obj is not None: 80 | yield from hasher.yield_hashables(id(self._obj)) 81 | else: 82 | # This will also work if ref is None 83 | yield from hasher.yield_hashables(self._sid) 84 | 85 | @override 86 | def save_instance_state(self, saver: "mincepy.Saver", /): 87 | if self._obj is not None: 88 | sid = saver.get_snapshot_id(self._obj) 89 | else: 90 | sid = self._sid 91 | 92 | if sid is not None: 93 | return sid.to_dict() 94 | 95 | return None 96 | 97 | @override 98 | def load_instance_state(self, saved_state, loader: "mincepy.Loader", /): 99 | super().load_instance_state(saved_state, loader) 100 | # Rely on class default values for members 101 | if saved_state is not None: 102 | if isinstance(saved_state, list): 103 | # Legacy version 104 | self._sid = records.SnapshotId(*saved_state) 105 | else: 106 | # New version is dict 107 | self._sid = records.SnapshotId(**saved_state) 108 | 109 | self._loader = loader 110 | 111 | 112 | def ref(obj=None) -> ObjRef: 113 | """Create an object reference""" 114 | return ObjRef(obj) 115 | 116 | 117 | HISTORIAN_TYPES = (ObjRef,) 118 | -------------------------------------------------------------------------------- /src/mincepy/result_types.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | from typing import TYPE_CHECKING, Iterable, List, Optional, Sequence, Set, Tuple 3 | 4 | if TYPE_CHECKING: 5 | import mincepy 6 | 7 | __all__ = "MergeResult", "DeleteResult" 8 | 9 | 10 | class MergeResult: 11 | """Information about the results from a merge operation. 12 | `all` contains all the IDs that were considered in the merge which means not only those that 13 | were passed but also all those that they reference. 14 | `merged` contains the ids of all the records that were actually merged (the rest were already 15 | present) 16 | """ 17 | 18 | __slots__ = "all", "merged" 19 | 20 | def __init__( 21 | self, 22 | all_snapshots: Sequence["mincepy.SnapshotId"] = None, 23 | merged_snapshots: Sequence["mincepy.SnapshotId"] = None, 24 | ): 25 | self.all: List["mincepy.SnapshotId"] = [] 26 | self.merged: List["mincepy.SnapshotId"] = [] 27 | if all_snapshots: 28 | self.all.extend(all_snapshots) 29 | if merged_snapshots: 30 | self.merged.extend(merged_snapshots) 31 | 32 | def update(self, result: "MergeResult"): 33 | self.all.extend(result.all) 34 | self.merged.extend(result.merged) 35 | 36 | 37 | class DeleteResult: 38 | """Information about results from a delete operation.""" 39 | 40 | __slots__ = "_deleted", "_not_found", "_files_transferred" 41 | 42 | def __init__(self, deleted: list, not_found: Iterable = None): 43 | self._deleted = tuple(deleted) 44 | self._not_found = tuple() if not_found is None else tuple(not_found) 45 | 46 | @property 47 | def deleted(self) -> Tuple: 48 | return self._deleted 49 | 50 | @property 51 | def not_found(self) -> Tuple: 52 | return self._not_found 53 | 54 | 55 | @dataclasses.dataclass 56 | class PurgeResult: 57 | """Information about results from a purge operation""" 58 | 59 | deleted_purged: Optional[Set["mincepy.SnapshotId"]] = None 60 | unreferenced_purged: Optional[Set["mincepy.SnapshotId"]] = None 61 | -------------------------------------------------------------------------------- /src/mincepy/saving.py: -------------------------------------------------------------------------------- 1 | """Module for methods related to saving and loading objects to/from records""" 2 | 3 | from typing import TYPE_CHECKING, Union 4 | 5 | if TYPE_CHECKING: 6 | import mincepy 7 | 8 | 9 | def save_instance_state(obj, db_type: type["mincepy.fields.WithFields"] = None) -> dict: 10 | """Save the instance state of an object. 11 | 12 | Given an object this function takes a DbType specifying the attributes to be saved and will use 13 | these to return a saved sate. Note, that for regular Savable objects, the db_type is the object 14 | itself in which case this argument can be omitted. 15 | """ 16 | import mincepy # pylint: disable=import-outside-toplevel 17 | 18 | if db_type is None: 19 | assert issubclass( 20 | type(obj), mincepy.fields.WithFields 21 | ), "A DbType wasn't passed and obj isn't a DbType instance other" 22 | db_type = type(obj) 23 | 24 | field_properties = mincepy.fields.get_field_properties(db_type).values() 25 | state = {} 26 | 27 | for properties in field_properties: 28 | attr_val = getattr(obj, properties.attr_name) 29 | if properties.ref and attr_val is not None: 30 | attr_val = mincepy.ObjRef(attr_val) 31 | 32 | # Check if it's still a field because otherwise it hasn't been set yet 33 | if attr_val is not properties: 34 | state[properties.store_as] = attr_val 35 | 36 | return state 37 | 38 | 39 | def load_instance_state( 40 | obj, 41 | state: Union[list, dict], 42 | db_type: type["mincepy.fields.WithFields"] = None, 43 | ignore_missing=True, 44 | ): 45 | import mincepy # pylint: disable=import-outside-toplevel 46 | 47 | if db_type is None: 48 | assert issubclass( 49 | type(obj), mincepy.fields.WithFields 50 | ), "A DbType wasn't passed and obj isn't a DbType instance other" 51 | db_type = type(obj) 52 | 53 | to_set = {} 54 | if isinstance(state, dict): 55 | for properties in mincepy.fields.get_field_properties(db_type).values(): 56 | try: 57 | value = state[properties.store_as] 58 | except KeyError: 59 | if ignore_missing: 60 | value = None 61 | else: 62 | raise ValueError(f"Saved state missing '{properties.store_as}'") from None 63 | 64 | if properties.ref and value is not None: 65 | assert isinstance(value, mincepy.ObjRef), ( 66 | f"Expected to see a reference in the saved state for key " 67 | f"'{properties.store_as}' but got '{value}'" 68 | ) 69 | value = value() # Dereference it 70 | 71 | to_set[properties.attr_name] = value 72 | 73 | for attr_name, value in to_set.items(): 74 | setattr(obj, attr_name, value) 75 | -------------------------------------------------------------------------------- /src/mincepy/staging.py: -------------------------------------------------------------------------------- 1 | from typing import MutableMapping, Optional 2 | 3 | from . import utils 4 | 5 | __all__ = "get_info", "remove" 6 | 7 | 8 | class StagingArea: 9 | """This global singleton stores information about objects before they have been saved by the 10 | historian. This is useful when there is global information that may or may not be used if the 11 | object is indeed eventually saved. If not, the information is simply discarded when the object 12 | is destructed""" 13 | 14 | # pylint: disable=too-few-public-methods 15 | 16 | staged_obj_info: MutableMapping[object, dict] = utils.WeakObjectIdDict[dict]() 17 | 18 | def __init__(self): 19 | raise RuntimeError("Cannot be instantiated") 20 | 21 | 22 | def get_info(obj, create=True) -> Optional[dict]: 23 | """Get the information dictionary for a given staged object. If create is True, the 24 | dictionary will be created if it doesn't already exist. If False and the object is not 25 | staged then None will be returned. 26 | """ 27 | if create: 28 | return StagingArea.staged_obj_info.setdefault(obj, {}) 29 | 30 | try: 31 | return StagingArea.staged_obj_info[obj] 32 | except KeyError: 33 | return None 34 | 35 | 36 | def remove(obj): 37 | """Remove the information dictionary for a staged object. If the object is not staged 38 | then this function does nothing""" 39 | StagingArea.staged_obj_info.pop(obj) 40 | 41 | 42 | def replace(old, new): 43 | """Move over the staging information from object 'old' to object 'new'""" 44 | try: 45 | staging_info = StagingArea.staged_obj_info.pop(old) 46 | except KeyError: 47 | # Delete any information present for new 48 | StagingArea.staged_obj_info.pop(new, None) 49 | else: 50 | StagingArea.staged_obj_info[new] = staging_info 51 | -------------------------------------------------------------------------------- /src/mincepy/tracking.py: -------------------------------------------------------------------------------- 1 | import copy as python_copy 2 | import functools 3 | from typing import Callable 4 | 5 | from . import records, staging 6 | 7 | __all__ = "track", "copy", "deepcopy", "mark_as_copy" 8 | 9 | 10 | class TrackStack: 11 | """Stack to keep track of functions being called""" 12 | 13 | _stack = [] 14 | 15 | def __init__(self): 16 | raise RuntimeError("Cannot be instantiated") 17 | 18 | @classmethod 19 | def peek(cls): 20 | if not cls._stack: 21 | return None 22 | return cls._stack[-1] 23 | 24 | @classmethod 25 | def push(cls, obj): 26 | """Push an object onto the stack""" 27 | cls._stack.append(obj) 28 | 29 | @classmethod 30 | def pop(cls, obj): 31 | """Pop an object off of the stack""" 32 | if cls._stack[-1] != obj: 33 | raise RuntimeError( 34 | f"Someone has corrupted the process stack!\n" 35 | f"Expected to find '{obj}' on top but found '{cls._stack[-1]}'" 36 | ) 37 | 38 | cls._stack.pop() 39 | 40 | 41 | class TrackContext: 42 | """Context manager that pushes and pops the object to/from the track stack""" 43 | 44 | def __init__(self, obj): 45 | self._obj = obj 46 | 47 | def __enter__(self): 48 | TrackStack.push(self._obj) 49 | 50 | def __exit__(self, exc_type, exc_val, exc_tb): 51 | TrackStack.pop(self._obj) 52 | 53 | 54 | def track(obj_or_fn): 55 | """Allows object creation to be tracked. When an object is created within this context, the 56 | creator of the object will be saved in the database record. 57 | 58 | This can be used either as a decorator to a class method, in which case the object instance will 59 | be the creator. Or it can be used as a context in which case the creator should be passed as 60 | the argument. 61 | """ 62 | if isinstance(obj_or_fn, Callable): # pylint: disable=isinstance-second-argument-not-valid-type 63 | # We're acting as a decorator 64 | @functools.wraps(obj_or_fn) 65 | def wrapper(self, *args, **kwargs): 66 | with TrackContext(self): 67 | return obj_or_fn(self, *args, **kwargs) 68 | 69 | return wrapper 70 | 71 | # We're acting as a context 72 | return TrackContext(obj_or_fn) 73 | 74 | 75 | def obj_created(obj): 76 | creator = TrackStack.peek() 77 | if creator is not None: 78 | staging.get_info(obj)[records.ExtraKeys.CREATED_BY] = creator 79 | 80 | 81 | def mark_as_copy(obj, copied_from): 82 | staging.get_info(obj)[records.ExtraKeys.COPIED_FROM] = copied_from 83 | 84 | 85 | def copy(obj): 86 | """Create a shallow copy of the object. Using this method allows the historian to inject 87 | information about where the object was copied from into the record if saved.""" 88 | obj_copy = python_copy.copy(obj) 89 | mark_as_copy(obj_copy, obj) 90 | return obj_copy 91 | 92 | 93 | def deepcopy(obj): 94 | """Create a shallow copy of the object. Using this method allows the historian to inject 95 | information about where the object was copied from into the record if saved.""" 96 | obj_copy = python_copy.deepcopy(obj) 97 | mark_as_copy(obj_copy, obj) 98 | return obj_copy 99 | -------------------------------------------------------------------------------- /src/mincepy/type_ids.py: -------------------------------------------------------------------------------- 1 | """Module for storing type ids of common mincePy types""" 2 | 3 | import uuid 4 | 5 | FILE_TYPE_ID = uuid.UUID("3bf3c24e-f6c8-4f70-956f-bdecd7aed091") 6 | OBJ_REF_TYPE_ID = uuid.UUID("633c7035-64fe-4d87-a91e-3b7abd8a6a28") 7 | SNAPSHOT_ID_TYPE_ID = uuid.UUID("05fe092b-07b3-4ffc-8cf2-cee27aa37e81") 8 | -------------------------------------------------------------------------------- /src/mincepy/typing.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Hashable 2 | from typing import Union 3 | 4 | TypeId = Hashable 5 | TypeIdOrType = Union[TypeId, type] # pylint: disable=invalid-name 6 | -------------------------------------------------------------------------------- /src/mincepy/utils.py: -------------------------------------------------------------------------------- 1 | import collections.abc 2 | from contextlib import nullcontext 3 | import functools 4 | from typing import Generic, MutableMapping, Type, TypeVar 5 | import weakref 6 | 7 | V = TypeVar("V") 8 | 9 | 10 | class WeakObjectIdDict(Generic[V], collections.abc.MutableMapping): 11 | """ 12 | Like `weakref.WeakKeyDict` but internally uses object ids (from `id(obj)`) instead of the 13 | object reference itself thereby avoiding the need for the object to be hashable 14 | (and therefore immutable). 15 | """ 16 | 17 | def __init__(self, seq=None, **kwargs): 18 | self._refs: MutableMapping[int, weakref.ReferenceType] = {} 19 | self._values: MutableMapping[int, V] = {} 20 | if seq: 21 | if isinstance(seq, collections.abc.Mapping): 22 | for key, value in seq.items(): 23 | self[key] = value 24 | elif isinstance(seq, collections.abc.Iterable): 25 | for key, value in seq: 26 | self[key] = value 27 | if kwargs: 28 | for key, value in kwargs.items(): 29 | self[key] = value 30 | 31 | def __copy__(self): 32 | return WeakObjectIdDict(self) 33 | 34 | def __getitem__(self, item: object) -> V: 35 | try: 36 | return self._values[id(item)] 37 | except KeyError: 38 | raise KeyError(item) from None 39 | 40 | def __setitem__(self, key: object, value: V): 41 | obj_id = id(key) 42 | wref = weakref.ref(key, functools.partial(self._finalised, obj_id)) 43 | self._refs[obj_id] = wref 44 | self._values[obj_id] = value 45 | 46 | def __delitem__(self, key: object): 47 | obj_id = id(key) 48 | del self._values[obj_id] 49 | del self._refs[obj_id] 50 | 51 | def __len__(self): 52 | return len(self._values) 53 | 54 | def __iter__(self): 55 | for ref in self._refs.values(): 56 | yield ref() 57 | 58 | def _finalised(self, obj_id, _wref): 59 | # Delete both the object values and the reference itself 60 | del self._values[obj_id] 61 | del self._refs[obj_id] 62 | 63 | 64 | T = TypeVar("T") # Declare type variable pylint: disable=invalid-name 65 | 66 | 67 | class DefaultFromCall: 68 | """Can be used as a default that is generated from a callable when needed""" 69 | 70 | def __init__(self, default_fn): 71 | assert callable(default_fn), "Must supply callable" 72 | self._callable = default_fn 73 | 74 | def __call__(self, *args, **kwargs): 75 | return self._callable(*args, **kwargs) 76 | 77 | 78 | class NamedTupleBuilder(Generic[T]): 79 | """A builder that allows namedtuples to be build step by step""" 80 | 81 | def __init__(self, tuple_type: Type[T], defaults=None): 82 | # Have to do it this way because we overwrite __setattr__ 83 | defaults = defaults or {} 84 | diff = set(defaults.keys()) - set(tuple_type._fields) 85 | if diff: 86 | raise RuntimeError(f"Can't supply defaults that are not in the namedtuple: '{diff}'") 87 | 88 | super().__setattr__("_tuple_type", tuple_type) 89 | super().__setattr__("_values", defaults) 90 | 91 | def __getattr__(self, item): 92 | """Read a key as an attribute. 93 | 94 | :raises AttributeError: if the attribute does not correspond to an existing key. 95 | """ 96 | if item == "_tuple_type": 97 | return self._tuple_type 98 | try: 99 | return self._values[item] 100 | except KeyError: 101 | errmsg = f"'{self.__class__.__name__}' object has no attribute '{item}'" 102 | raise AttributeError(errmsg) from None 103 | 104 | def __setattr__(self, attr, value): 105 | """Set a key as an attribute.""" 106 | if attr not in super().__getattribute__("_tuple_type")._fields: 107 | raise AttributeError( 108 | f"AttributeError: '{attr}' is not a valid attribute of the object " 109 | f"'{self.__class__.__name__}'" 110 | ) 111 | 112 | self._values[attr] = value 113 | 114 | def __repr__(self): 115 | """Representation of the object.""" 116 | return f"{self.__class__.__name__}({dict.__repr__(self._values)})" 117 | 118 | def __dir__(self): 119 | return self._tuple_type._fields 120 | 121 | def update(self, new_values: dict): 122 | for key, value in new_values.items(): 123 | setattr(self, key, value) 124 | 125 | def build(self) -> T: 126 | build_from = { 127 | key: value if not isinstance(value, DefaultFromCall) else value() 128 | for key, value in self._values.items() 129 | } 130 | return self._tuple_type(**build_from) 131 | 132 | 133 | def to_slice(specifier) -> slice: 134 | """ 135 | Turn the specifier into a slice object. Accepts either: 136 | 1. a slice, which is just returned. 137 | 2. a concrete index (positive or negative) e.g. to_slice(5) will generate a slice that 138 | returns the 5th element 139 | 3. a string containing '*' or ':' to get all entries. 140 | """ 141 | if isinstance(specifier, slice): 142 | return specifier 143 | if isinstance(specifier, int): 144 | sign = -1 if specifier < 0 else 1 145 | return slice(specifier, specifier + sign, sign) 146 | if isinstance(specifier, str) and specifier == ":" or specifier == "*": 147 | return slice(None) 148 | 149 | raise ValueError(f"Unknown slice specifier: {specifier}") 150 | 151 | 152 | def sync(save=False): 153 | """ 154 | Decorator that will call sync before the method is invoked to make sure the object is up-to-date 155 | with what's in the database. 156 | """ 157 | 158 | def inner(obj_method): 159 | @functools.wraps(obj_method) 160 | def wrapper(self, *args, **kwargs): 161 | # pylint: disable=protected-access 162 | ctx = nullcontext if self._historian is None else self._historian.transaction 163 | with ctx(): 164 | try: 165 | self.__sync += 1 166 | except AttributeError: 167 | self.__sync = 1 168 | if self.is_saved(): 169 | self.sync() 170 | try: 171 | retval = obj_method(self, *args, **kwargs) 172 | if self.is_saved() and save: 173 | self.save() 174 | return retval 175 | finally: 176 | self.__sync -= 1 177 | 178 | return wrapper 179 | 180 | return inner 181 | 182 | 183 | class Progress: 184 | """Gives information about progress of a discreet number of entries""" 185 | 186 | __slots__ = "done", "_total" 187 | 188 | def __str__(self) -> str: 189 | return f"{self.done}/{self._total} done" 190 | 191 | def __init__(self, total: int): 192 | self.done = 0 193 | self._total = total 194 | 195 | @property 196 | def total(self) -> int: 197 | return self._total 198 | -------------------------------------------------------------------------------- /src/mincepy/version.py: -------------------------------------------------------------------------------- 1 | __author__ = "Martin Uhrin " 2 | __version__ = "0.18.0" 3 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muhrin/mincepy/6f40ac667e5025e5b76894c4fa7e9a2efeaf8501/test/__init__.py -------------------------------------------------------------------------------- /test/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/muhrin/mincepy/6f40ac667e5025e5b76894c4fa7e9a2efeaf8501/test/cli/__init__.py -------------------------------------------------------------------------------- /test/cli/test_migrate.py: -------------------------------------------------------------------------------- 1 | from click.testing import CliRunner 2 | 3 | import mincepy 4 | import mincepy.cli.main 5 | 6 | from ..common import CarV1, CarV2, StoreByRef, StoreByValue 7 | 8 | 9 | def test_simple_migrate(historian: mincepy.Historian, archive_uri): 10 | car = CarV1("white", "lada") 11 | car.save() 12 | by_val = StoreByValue(car) 13 | by_val_id = by_val.save() 14 | 15 | # Register a new version of the car 16 | historian.register_type(CarV2) 17 | 18 | # Now both car and by_val should need migration (because by_val stores a car) 19 | migratable = tuple(historian.migrations.find_migratable_records()) 20 | assert len(migratable) == 2 21 | 22 | # Now migrate 23 | runner = CliRunner() 24 | result = runner.invoke( 25 | mincepy.cli.main.migrate, 26 | ["--yes", archive_uri], 27 | obj={"helpers": [CarV2, StoreByValue]}, 28 | ) 29 | assert result.exit_code == 0 30 | 31 | migratable = tuple(historian.migrations.find_migratable_records()) 32 | assert len(migratable) == 0 33 | 34 | # Now register a new version of StoreByVal 35 | historian.register_type(StoreByRef) 36 | 37 | # There should still be the same to migratables as before 38 | migratable = tuple(historian.migrations.find_migratable_records()) 39 | assert len(migratable) == 1 40 | ids = [record.obj_id for record in migratable] 41 | assert by_val_id in ids 42 | 43 | # Now migrate 44 | runner = CliRunner() 45 | result = runner.invoke( 46 | mincepy.cli.main.migrate, 47 | ["--yes", archive_uri], 48 | obj={"helpers": [CarV2, StoreByRef]}, 49 | ) 50 | assert result.exit_code == 0 51 | 52 | migratable = tuple(historian.migrations.find_migratable_records()) 53 | assert len(migratable) == 0 54 | -------------------------------------------------------------------------------- /test/common.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | import mincepy 4 | 5 | # pylint: disable=invalid-name 6 | 7 | 8 | class CarV0(mincepy.ConvenientSavable, type_id=uuid.UUID("297808e4-9bc7-4f0a-9f8d-850a5f558663")): 9 | colour = mincepy.field() 10 | make = mincepy.field() 11 | 12 | def __init__(self, colour: str, make: str): 13 | super().__init__() 14 | self.colour = colour 15 | self.make = make 16 | 17 | def save_instance_state(self, saver: mincepy.Saver): 18 | super().save_instance_state(saver) 19 | # Here, I decide to store as an array 20 | return [self.colour, self.make] 21 | 22 | def load_instance_state(self, saved_state, loader: mincepy.Loader): 23 | self.colour = saved_state[0] 24 | self.make = saved_state[1] 25 | 26 | 27 | class CarV1(mincepy.ConvenientSavable, type_id=uuid.UUID("297808e4-9bc7-4f0a-9f8d-850a5f558663")): 28 | colour = mincepy.field() 29 | make = mincepy.field() 30 | 31 | class V0toV1(mincepy.ObjectMigration): 32 | VERSION = 1 33 | 34 | @classmethod 35 | def upgrade(cls, saved_state, loader: "mincepy.Loader") -> dict: 36 | return dict(colour=saved_state[0], make=saved_state[1]) 37 | 38 | # Set the migration 39 | LATEST_MIGRATION = V0toV1 40 | 41 | def __init__(self, colour: str, make: str): 42 | super().__init__() 43 | self.colour = colour 44 | self.make = make 45 | 46 | # This time we don't overwrite save/load instance state and let the SavableObject save the 47 | # attributes as a dictionary 48 | 49 | 50 | class CarV2(mincepy.ConvenientSavable, type_id=uuid.UUID("297808e4-9bc7-4f0a-9f8d-850a5f558663")): 51 | colour = mincepy.field() 52 | make = mincepy.field() 53 | reg = mincepy.field() # New attribute 54 | 55 | class V1toV2(mincepy.ObjectMigration): 56 | VERSION = 2 57 | PREVIOUS = CarV1.V0toV1 58 | 59 | @classmethod 60 | def upgrade(cls, saved_state, loader: "mincepy.Loader") -> dict: 61 | # Augment the saved state 62 | saved_state["reg"] = "unknown" 63 | return saved_state 64 | 65 | # Set the migration 66 | LATEST_MIGRATION = V1toV2 67 | 68 | def __init__(self, colour: str, make: str, reg=None): 69 | super().__init__() 70 | self.colour = colour 71 | self.make = make 72 | self.reg = reg 73 | 74 | 75 | class HatchbackCarV0(CarV0, type_id=uuid.UUID("d4131d3c-c140-4959-a545-21082dae9f1b")): 76 | """A hatchback that inherits from CarV0""" 77 | 78 | 79 | class HatchbackCarV1(CarV1, type_id=uuid.UUID("d4131d3c-c140-4959-a545-21082dae9f1b")): 80 | """A hatchback that inherits from CarV1, simulating what would have happened if parent was 81 | migrated""" 82 | 83 | 84 | class StoreByValue( 85 | mincepy.ConvenientSavable, type_id=uuid.UUID("40377bfc-901c-48bb-a85c-1dd692cddcae") 86 | ): 87 | ref = mincepy.field() 88 | 89 | def __init__(self, ref): 90 | super().__init__() 91 | self.ref = ref 92 | 93 | 94 | class StoreByRef( 95 | mincepy.ConvenientSavable, type_id=uuid.UUID("40377bfc-901c-48bb-a85c-1dd692cddcae") 96 | ): 97 | ref = mincepy.field(ref=True) 98 | 99 | class ToRefMigration(mincepy.ObjectMigration): 100 | VERSION = 1 101 | 102 | @classmethod 103 | def upgrade(cls, saved_state, loader: "mincepy.Loader") -> dict: 104 | # Replace the value stored version with a reference 105 | saved_state["ref"] = mincepy.ObjRef(saved_state["ref"]) 106 | return saved_state 107 | 108 | # Changed my mind, want to store by value now 109 | LATEST_MIGRATION = ToRefMigration 110 | 111 | def __init__(self, ref): 112 | super().__init__() 113 | self.ref = ref 114 | 115 | 116 | class A(mincepy.ConvenientSavable, type_id=uuid.UUID("a50f21bc-899e-445f-baf7-0a1a373e51fc")): 117 | migrations = mincepy.field() 118 | 119 | class Migration(mincepy.ObjectMigration): 120 | VERSION = 11 121 | 122 | @classmethod 123 | def upgrade(cls, saved_state, loader: "mincepy.Loader"): 124 | saved_state["migrations"].append("A V11") 125 | 126 | LATEST_MIGRATION = Migration 127 | 128 | def __init__(self): 129 | super().__init__() 130 | self.migrations = [] 131 | 132 | 133 | class B(A, type_id=uuid.UUID("f1c07f5f-bf64-441d-8dc7-bbde65eb6fa2")): 134 | 135 | class Migration(mincepy.ObjectMigration): 136 | VERSION = 2 137 | 138 | @classmethod 139 | def upgrade(cls, saved_state, loader: "mincepy.Loader"): 140 | saved_state["migrations"].append("B V2") 141 | 142 | LATEST_MIGRATION = Migration 143 | 144 | 145 | class BV3(A, type_id=uuid.UUID("f1c07f5f-bf64-441d-8dc7-bbde65eb6fa2")): 146 | 147 | class Migration(mincepy.ObjectMigration): 148 | VERSION = 3 149 | PREVIOUS = B.Migration 150 | 151 | @classmethod 152 | def upgrade(cls, saved_state, loader: "mincepy.Loader"): 153 | pass 154 | 155 | LATEST_MIGRATION = Migration 156 | 157 | 158 | class C(B, type_id=uuid.UUID("c76153c1-82d0-4048-bdbe-937889c7fac9")): 159 | pass 160 | 161 | 162 | class C_BV3(BV3, type_id=uuid.UUID("c76153c1-82d0-4048-bdbe-937889c7fac9")): 163 | pass 164 | -------------------------------------------------------------------------------- /test/conftest.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=unused-import, redefined-outer-name 2 | import random 3 | 4 | import pytest 5 | 6 | import mincepy 7 | from mincepy import testing 8 | from mincepy.testing import archive_base_uri, archive_uri, historian, mongodb_archive 9 | 10 | from . import utils 11 | 12 | 13 | @pytest.fixture 14 | def standard_dataset(historian: mincepy.Historian): # noqa: F811 15 | with historian.transaction(): 16 | # Put in some cars 17 | ferrari = testing.Car("ferrari", "red") 18 | ferrari.save() 19 | honda = testing.Car("honda", "white") 20 | honda.save() 21 | fiat = testing.Car("fiat", "green") 22 | fiat.save() 23 | 24 | # Put in some identical yellow cars 25 | for _ in range(4): 26 | testing.Car(make="renault", colour="yellow").save() 27 | 28 | # Put in some people 29 | testing.Person("sonia", 30, ferrari).save() 30 | testing.Person("martin", 35, honda).save() 31 | testing.Person("gavin", 34).save() 32 | testing.Person("upul", 35, fiat).save() 33 | 34 | # Put in some people with the same name and different age 35 | for _ in range(100): 36 | testing.Person(name="mike", age=random.randint(0, 4)).save() 37 | 38 | 39 | @pytest.fixture 40 | def large_dataset(historian: mincepy.Historian): # noqa: F811 41 | with historian.transaction(): 42 | # Put in some cars 43 | ferrari = testing.Car("ferrari", "red") 44 | ferrari.save() 45 | honda = testing.Car("honda", "white") 46 | honda.save() 47 | fiat = testing.Car("fiat", "green") 48 | fiat.save() 49 | 50 | # Put in some yellow cars 51 | for _ in range(100): 52 | testing.Car(make="renault", colour="yellow").save() 53 | 54 | # Put in some purely random 55 | for _ in range(100): 56 | testing.Car(make=utils.random_str(5), colour=utils.random_str(3)).save() 57 | 58 | # Put in some people 59 | testing.Person("sonia", 30, ferrari).save() 60 | testing.Person("martin", 35, honda).save() 61 | testing.Person("gavin", 34).save() 62 | testing.Person("upul", 35, fiat).save() 63 | 64 | # Put in some with the same age 65 | for _ in range(100): 66 | testing.Person(name=utils.random_str(5), age=54).save() 67 | 68 | # Put in some with the same name 69 | for _ in range(100): 70 | testing.Person(name="mike", age=random.randint(0, 100)).save() 71 | 72 | # Put in some purely random 73 | for _ in range(100): 74 | testing.Person(name=utils.random_str(5), age=random.randint(0, 100)).save() 75 | -------------------------------------------------------------------------------- /test/historian/test_delete.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import mincepy 4 | from mincepy import testing 5 | 6 | # pylint: disable=invalid-name 7 | 8 | 9 | def test_delete(historian: mincepy.Historian): 10 | """Test deleting and then attempting to load an object""" 11 | car = testing.Car("lada") 12 | car_id = historian.save(car) 13 | historian.delete(car) 14 | with pytest.raises(mincepy.NotFound): 15 | historian.load(car_id) 16 | 17 | records = historian.history(car_id, as_objects=False) 18 | assert len(records) == 2, "There should be two record, the initial and the delete" 19 | assert records[-1].is_deleted_record() 20 | 21 | # Check imperative deleting 22 | with pytest.raises(mincepy.NotFound): 23 | historian.delete(car_id) 24 | # This, should not raise: 25 | result = historian.delete(car_id, imperative=False) 26 | assert not result.deleted 27 | assert car_id in result.not_found 28 | 29 | 30 | def test_delete_from_obj_id(historian: mincepy.Historian): 31 | """Test deleting an object using it's object id""" 32 | car = testing.Car("skoda") 33 | car_id = car.save() 34 | del car 35 | 36 | historian.delete(car_id) 37 | 38 | with pytest.raises(mincepy.NotFound): 39 | historian.load(car_id) 40 | 41 | 42 | def test_delete_in_transaction(historian: mincepy.Historian): 43 | saved_outside = testing.Car("fiat") 44 | outside_id = saved_outside.save() 45 | 46 | with historian.transaction(): 47 | saved_inside = testing.Car("bmw") 48 | inside_id = saved_inside.save() 49 | historian.delete(inside_id) 50 | historian.delete(outside_id) 51 | 52 | with pytest.raises(mincepy.ObjectDeleted): 53 | historian.get_obj(obj_id=inside_id) 54 | with pytest.raises(mincepy.ObjectDeleted): 55 | historian.get_obj(obj_id=outside_id) 56 | 57 | with pytest.raises(mincepy.ObjectDeleted): 58 | historian.load(inside_id) 59 | with pytest.raises(mincepy.ObjectDeleted): 60 | historian.load(outside_id) 61 | 62 | with pytest.raises(mincepy.NotFound): 63 | historian.get_obj(obj_id=inside_id) 64 | with pytest.raises(mincepy.NotFound): 65 | historian.get_obj(obj_id=outside_id) 66 | 67 | with pytest.raises(mincepy.NotFound): 68 | historian.load(inside_id) 69 | with pytest.raises(mincepy.NotFound): 70 | historian.load(outside_id) 71 | 72 | 73 | def test_delete_find(historian: mincepy.Historian): 74 | car = testing.Car("trabant") 75 | car_id = car.save() 76 | 77 | historian.delete(car_id) 78 | assert historian.find(obj_id=car_id).count() == 0 79 | 80 | # Now check the archive 81 | assert historian.snapshots.records.find(obj_id=car_id).count() == 2 82 | assert historian.snapshots.records.find(obj_id=car_id, state=mincepy.DELETED).count() == 1 83 | 84 | 85 | def test_delete_multiple_versions(historian: mincepy.Historian): 86 | car = testing.Car("skoda", "green") 87 | car.save() 88 | car.colour = "red" 89 | car.save() 90 | 91 | with historian.transaction(): 92 | historian.delete(car) 93 | 94 | 95 | def test_delete_twice(historian: mincepy.Historian): 96 | car = testing.Car("trabant") 97 | car_id = car.save() 98 | historian.delete(car_id) 99 | 100 | with pytest.raises(mincepy.NotFound): 101 | historian.delete(car_id) 102 | 103 | with pytest.raises(mincepy.NotFound): 104 | historian.delete(car) 105 | 106 | 107 | def test_delete_twice_in_transaction(historian: mincepy.Historian): 108 | car = testing.Car("trabant") 109 | car_id = car.save() 110 | 111 | with historian.transaction(): 112 | historian.delete(car_id) 113 | 114 | with pytest.raises(mincepy.NotFound): 115 | historian.delete(car_id) 116 | 117 | with pytest.raises(mincepy.NotFound): 118 | historian.delete(car) 119 | 120 | 121 | def test_delete_referenced_by(historian: mincepy.Historian): 122 | car = testing.Car() 123 | person = testing.Person("martin", 35, car) 124 | person.save() 125 | 126 | with pytest.raises(mincepy.ReferenceError): 127 | historian.delete(car) 128 | 129 | # Check you can delete it in a transaction 130 | with historian.transaction(): 131 | historian.delete(car) 132 | historian.delete(person) 133 | -------------------------------------------------------------------------------- /test/historian/test_references.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import mincepy 4 | from mincepy.testing import Car, Garage, Person 5 | 6 | # pylint: disable=invalid-name 7 | 8 | 9 | def test_references_wrong_type(historian: mincepy.Historian): 10 | with pytest.raises(TypeError): 11 | historian.references.references(1234) 12 | 13 | 14 | def test_referenced_by_wrong_type(historian: mincepy.Historian): 15 | with pytest.raises(TypeError): 16 | historian.references.referenced_by(1234) 17 | 18 | 19 | def test_references_simple(historian: mincepy.Historian): 20 | address_book = mincepy.RefList() 21 | for i in range(3): 22 | address_book.append(Person(name="test", age=i)) 23 | address_book.save() 24 | 25 | refs = historian.references.references(address_book.obj_id) 26 | assert len(refs) == len(address_book) 27 | assert not set(person.obj_id for person in address_book) - set(refs) 28 | 29 | refs = historian.references.referenced_by(address_book[0].obj_id) 30 | assert len(refs) == 1 31 | assert address_book.obj_id in refs 32 | 33 | 34 | def test_remove_references_in_transaction(historian: mincepy.Historian): 35 | car = Car() 36 | garage = Garage(mincepy.ObjRef(car)) 37 | gid = historian.save(garage) 38 | 39 | graph = historian.references.get_obj_ref_graph(gid) 40 | assert len(graph.nodes) == 2 41 | assert len(graph.edges) == 1 42 | assert (gid, car.obj_id) in graph.edges 43 | 44 | reffed_by = historian.references.referenced_by(car.obj_id) 45 | assert len(reffed_by) == 1 46 | assert gid in reffed_by 47 | 48 | # Now, change the reference in a transaction 49 | with historian.transaction(): 50 | garage.car = None 51 | garage.save() 52 | # Still in a transaction, check the references are correct, i.e. None 53 | graph = historian.references.get_obj_ref_graph(gid) 54 | 55 | assert len(graph.nodes) == 1 56 | assert gid in graph.nodes 57 | assert len(graph.edges) == 0 58 | 59 | reffed_by = historian.references.referenced_by(car.obj_id) 60 | assert len(reffed_by) == 0 61 | 62 | assert len(historian.references.referenced_by(car.obj_id)) == 0 63 | assert len(historian.references.references(gid)) == 0 64 | 65 | 66 | def test_add_references_in_transaction(historian: mincepy.Historian): 67 | car = Car() 68 | garage = mincepy.RefList() 69 | garage.append(car) 70 | historian.save(garage) 71 | 72 | refs = historian.references.references(garage.obj_id) 73 | assert len(refs) == 1 74 | assert car.obj_id in refs 75 | 76 | # Now add a car 77 | with historian.transaction(): 78 | car2 = Car() 79 | garage.append(car2) 80 | garage.save() 81 | 82 | # Still in a transaction, check the references are correct 83 | refs = historian.references.references(garage.obj_id) 84 | assert len(refs) == 2 85 | assert car.obj_id in refs 86 | assert car2.obj_id in refs 87 | 88 | reffed_by = historian.references.referenced_by(car2.obj_id) 89 | assert len(reffed_by) == 1 90 | assert garage.obj_id in reffed_by 91 | 92 | refs = historian.references.references(garage.obj_id) 93 | assert len(refs) == 2 94 | assert car.obj_id in refs 95 | assert car2.obj_id in refs 96 | 97 | reffed_by = historian.references.referenced_by(car2.obj_id) 98 | assert len(reffed_by) == 1 99 | assert garage.obj_id in reffed_by 100 | 101 | 102 | def test_snapshot_references(historian: mincepy.Historian): 103 | address_book = mincepy.RefList() 104 | for i in range(3): 105 | address_book.append(Person(name="test", age=i)) 106 | address_book.save() 107 | sid = historian.get_snapshot_id(address_book) 108 | 109 | refs = historian.references.references(sid) 110 | assert len(refs) == len(address_book) 111 | assert not set(historian.get_snapshot_id(person) for person in address_book) - set(refs) 112 | 113 | 114 | def test_snapshot_referenced_by(historian: mincepy.Historian): 115 | address_book = mincepy.RefList() 116 | for i in range(3): 117 | address_book.append(Person(name="test", age=i)) 118 | address_book.save() 119 | sid = historian.get_snapshot_id(address_book) 120 | 121 | refs = historian.references.referenced_by(historian.get_snapshot_id(address_book[0])) 122 | assert len(refs) == 1 123 | assert sid in refs 124 | -------------------------------------------------------------------------------- /test/historian/test_type_registry.py: -------------------------------------------------------------------------------- 1 | from test import common # pylint: disable=wrong-import-order 2 | 3 | import pytest 4 | 5 | from mincepy import type_registry 6 | 7 | # pylint: disable=invalid-name 8 | 9 | 10 | def test_get_version_info(): 11 | registry = type_registry.TypeRegistry() 12 | registry.register_type(common.A) 13 | registry.register_type(common.B) 14 | registry.register_type(common.C) 15 | 16 | c_info = registry.get_version_info(common.C) 17 | # Check versions 18 | assert c_info[common.A.TYPE_ID] == common.A.LATEST_MIGRATION.VERSION 19 | assert c_info[common.B.TYPE_ID] == common.B.LATEST_MIGRATION.VERSION 20 | assert c_info[common.C.TYPE_ID] == common.C.LATEST_MIGRATION.VERSION 21 | 22 | 23 | def test_automatic_registration_of_parent(): 24 | """Test that if a historian type with historian type ancestor(s) is registered then so are its 25 | ancestors""" 26 | registry = type_registry.TypeRegistry() 27 | registry.register_type(common.C) 28 | assert common.A in registry 29 | assert common.B in registry 30 | 31 | 32 | def test_get_type_id(): 33 | registry = type_registry.TypeRegistry() 34 | with pytest.raises(ValueError): 35 | registry.get_type_id(common.A) 36 | 37 | registry.register_type(common.A) 38 | # This should return the superclass as we haven't registered B yet 39 | assert registry.get_type_id(common.B) == common.A.TYPE_ID 40 | 41 | registry.register_type(common.B) 42 | assert registry.get_type_id(common.B) == common.B.TYPE_ID 43 | 44 | 45 | def test_get_helper_from_type_id(): 46 | registry = type_registry.TypeRegistry() 47 | registry.register_type(common.A) 48 | helper = registry.get_helper_from_type_id(common.A.TYPE_ID) 49 | assert helper.TYPE is common.A 50 | assert helper.TYPE_ID == common.A.TYPE_ID 51 | -------------------------------------------------------------------------------- /test/mongo/test_mongo_archive.py: -------------------------------------------------------------------------------- 1 | """Specific tests for the MongoDB archive""" 2 | 3 | import bson 4 | import gridfs 5 | import pytest 6 | 7 | import mincepy 8 | from mincepy import testing 9 | import mincepy.mongo.migrations 10 | 11 | 12 | def test_schema_version(historian: mincepy.Historian): 13 | archive: mincepy.mongo.MongoArchive = historian.archive 14 | assert archive.schema_version == mincepy.mongo.migrations.LATEST.VERSION 15 | 16 | 17 | def test_get_gridfs_bucket(historian: mincepy.Historian): 18 | archive: mincepy.mongo.MongoArchive = historian.archive 19 | assert isinstance(archive.get_gridfs_bucket(), gridfs.GridFSBucket) 20 | 21 | 22 | def test_load(historian: mincepy.Historian): 23 | archive: mincepy.mongo.MongoArchive = historian.archive 24 | with pytest.raises(TypeError): 25 | archive.load("invalid") 26 | 27 | with pytest.raises(mincepy.NotFound): 28 | archive.load(mincepy.SnapshotId(bson.ObjectId(), 0)) 29 | 30 | 31 | def test_distinct(historian: mincepy.Historian): 32 | archive: mincepy.mongo.MongoArchive = historian.archive 33 | testing.Car(colour="blue").save() 34 | testing.Car(colour="red").save() 35 | 36 | assert set(archive.distinct("state.colour")) == {"red", "blue"} 37 | assert set(archive.distinct("state.colour", {"state": {"colour": "red"}})) == {"red"} 38 | -------------------------------------------------------------------------------- /test/test_autosave.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import mincepy 4 | from mincepy import _autosave, testing 5 | 6 | 7 | def test_autosave_helper(): 8 | helper = _autosave.autosavable(testing.Car) 9 | assert isinstance(helper, mincepy.TypeHelper) 10 | assert helper.TYPE is testing.Car 11 | 12 | 13 | def test_autosave_historian(historian): 14 | with pytest.raises(ValueError): 15 | historian.type_registry.get_helper(testing.Sphere) 16 | 17 | new_obj = testing.Sphere(3.14) 18 | obj_id = historian.save(new_obj) 19 | helper = historian.type_registry.get_helper(testing.Sphere) 20 | assert isinstance(helper, mincepy.TypeHelper) 21 | type_id = helper.TYPE_ID 22 | 23 | del new_obj, helper 24 | 25 | historian.type_registry.unregister_type(type_id) 26 | with pytest.raises(ValueError): 27 | historian.type_registry.get_helper(testing.Sphere) 28 | 29 | loaded = historian.load(obj_id) 30 | assert isinstance(loaded, testing.Sphere) 31 | assert loaded.radius == 3.14 32 | 33 | 34 | def test_autosave_subclassing(historian): 35 | # By default, a subclass of a registered type (i.e. `Car`) is not, by default, savable 36 | with pytest.raises(ValueError): 37 | historian.type_registry.get_helper(testing.NamedSolarSystem) 38 | 39 | earth_sun = testing.NamedSolarSystem("earth-sun", testing.Sphere(10)) 40 | sun = earth_sun.sun 41 | historian.save(sun) 42 | obj_id = historian.save(earth_sun) 43 | del earth_sun 44 | 45 | loaded = historian.load(obj_id) 46 | assert isinstance(loaded, testing.NamedSolarSystem) 47 | assert loaded.sun is sun 48 | -------------------------------------------------------------------------------- /test/test_base_savable.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | import mincepy 4 | import mincepy.builtins 5 | from mincepy.testing import Car 6 | 7 | 8 | def test_save_as_ref(historian: mincepy.Historian): 9 | """Test the 'AsRef' functionality of BaseSavableObject""" 10 | 11 | class Person(mincepy.SimpleSavable, type_id=uuid.UUID("692429b6-a08b-489a-aa09-6eb3174b6405")): 12 | ATTRS = (mincepy.AsRef("car"), "name") # Save the car by reference 13 | 14 | def __init__(self, name: str, car): 15 | super().__init__() 16 | self.name = name 17 | self.car = car 18 | 19 | car = Car() 20 | # Both martin and sonia have the same car 21 | martin = Person("martin", car) 22 | sonia = Person("sonia", car) 23 | martin_id, sonia_id = historian.save(martin, sonia) 24 | del martin, sonia, car 25 | 26 | # No reload and check they still have the same car 27 | martin, sonia = historian.load(martin_id, sonia_id) 28 | assert martin.car is not None 29 | assert martin.name == "martin" 30 | assert martin.car is sonia.car 31 | assert sonia.name == "sonia" 32 | -------------------------------------------------------------------------------- /test/test_benchmarks.py: -------------------------------------------------------------------------------- 1 | try: 2 | from contextlib import nullcontext 3 | except ImportError: 4 | from contextlib2 import nullcontext 5 | 6 | import pytest 7 | 8 | import mincepy 9 | import mincepy.testing 10 | from mincepy.testing import Car 11 | 12 | from . import utils 13 | 14 | # pylint: disable=invalid-name 15 | 16 | 17 | def insert_cars(historian: mincepy.Historian, num=100, in_transaction=False): 18 | """Insert a number of cars into the database, optionally in a transaction so they all get 19 | inserted in one go.""" 20 | if in_transaction: 21 | ctx = historian.transaction() 22 | else: 23 | ctx = nullcontext() 24 | 25 | with ctx: 26 | for _ in range(num): 27 | historian.save(mincepy.testing.Car()) 28 | 29 | 30 | def find(historian, **kwargs): 31 | # Have to wrap the find method like this because it returns a generator and won't necessarily 32 | # fetch from the db unless we iterate it 33 | return tuple(historian.find(**kwargs)) 34 | 35 | 36 | def test_benchmark_insertions_individual(historian: mincepy.Historian, benchmark): 37 | benchmark(insert_cars, historian, in_transaction=False) 38 | 39 | 40 | def test_benchmark_insertions_transaction(historian: mincepy.Historian, benchmark): 41 | benchmark(insert_cars, historian, in_transaction=True) 42 | 43 | 44 | @pytest.mark.parametrize("num", [10**i for i in range(5)]) 45 | def test_find_cars(historian: mincepy.Historian, benchmark, num): 46 | """Test finding a car as a function of the number of entries in the database""" 47 | # Put in the one we want to find 48 | historian.save(Car("honda", "green")) 49 | 50 | # Put in the correct number of random other entries 51 | for _ in range(num): 52 | historian.save(Car(utils.random_str(10), utils.random_str(5))) 53 | 54 | result = benchmark(find, historian, state=dict(make="honda", colour="green")) 55 | assert len(result) == 1 56 | 57 | 58 | @pytest.mark.parametrize("num", [10**i for i in range(5)]) 59 | def test_find_many_cars(historian: mincepy.Historian, benchmark, num): 60 | """Test finding a car as a function of the number of entries in the database""" 61 | # Put in the correct number of random other entries 62 | for _ in range(num): 63 | historian.save(Car(utils.random_str(10), utils.random_str(5))) 64 | 65 | result = benchmark(find, historian) 66 | assert len(result) == num 67 | 68 | 69 | @pytest.mark.parametrize("num", [5**i for i in range(1, 4)]) 70 | def test_load_cars(historian: mincepy.Historian, benchmark, num): 71 | """Test finding a car as a function of the number of entries in the database""" 72 | # Put in the correct number of random other entries 73 | car_ids = [] 74 | for _ in range(num): 75 | car_ids.append(historian.save(Car(utils.random_str(10), utils.random_str(5)))) 76 | 77 | result = benchmark(historian.load, *car_ids) 78 | assert len(result) == len(car_ids) 79 | -------------------------------------------------------------------------------- /test/test_convenience.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import mincepy 4 | from mincepy.testing import Car 5 | 6 | 7 | def test_load(historian: mincepy.Historian): 8 | car = Car() 9 | car_id = historian.save(car) 10 | 11 | assert mincepy.load(car_id) is car 12 | 13 | # Check loading from 'cold' 14 | del car 15 | car = mincepy.load(car_id) 16 | assert car._historian is historian # pylint: disable=protected-access 17 | 18 | 19 | def test_save(historian: mincepy.Historian): 20 | car = Car() 21 | mincepy.save(car) 22 | assert car._historian is historian # pylint: disable=protected-access 23 | 24 | 25 | def test_invalid_connect(): 26 | """Check we get the right error when attempting to connect to invalid archive""" 27 | with pytest.raises(mincepy.ConnectionError): 28 | mincepy.connect("mongodb://unknown-server/db", timeout=5) 29 | 30 | with pytest.raises(ValueError): 31 | mincepy.connect("unknown-protocol://nowhere", timeout=5) 32 | -------------------------------------------------------------------------------- /test/test_expr.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from mincepy import expr 4 | 5 | # pylint: disable=pointless-statement, invalid-name 6 | 7 | 8 | def test_expr_types_and_filters(): 9 | """Test the query filters for expressions""" 10 | name_eq = expr.Comparison("name", expr.Eq("frank")) 11 | age_gt = expr.Comparison("age", expr.Gt(38)) 12 | 13 | # Check all operators 14 | for expression in expr.SimpleOperator.__subclasses__(): 15 | assert expression.oper.startswith("$") 16 | assert expression(True).dict() == {expression.oper: True} 17 | 18 | # Check all logicals 19 | for list_expr in expr.Logical.__subclasses__(): 20 | assert list_expr.oper.startswith("$") 21 | if issubclass(list_expr, expr.WithListOperand): 22 | assert list_expr([name_eq, age_gt]).dict() == { 23 | list_expr.oper: [name_eq.dict(), age_gt.dict()] 24 | } 25 | 26 | # Check that the the passthrough for list expressions work 27 | assert list_expr([name_eq]).dict() == name_eq.dict() 28 | 29 | with pytest.raises(TypeError): 30 | list_expr("non list") 31 | with pytest.raises(TypeError): 32 | list_expr(["non expression"]) 33 | else: 34 | assert list_expr(name_eq).dict() == {list_expr.oper: name_eq.dict()} 35 | with pytest.raises(TypeError): 36 | list_expr(["non expression"]) 37 | 38 | assert expr.Empty().dict() == {} 39 | 40 | with pytest.raises(TypeError): 41 | # Comparison takes an operator, not a string 42 | expr.Comparison("my_field", "oper") 43 | 44 | 45 | def test_expr_and_or(): 46 | """Test expression and/or methods""" 47 | name_eq = expr.Comparison("name", expr.Eq("frank")) 48 | age_gt = expr.Comparison("age", expr.Gt(38)) 49 | 50 | anded = name_eq & age_gt 51 | assert anded.dict() == {"$and": [name_eq.dict(), age_gt.dict()]} 52 | 53 | ored = name_eq | age_gt 54 | assert ored.dict() == {"$or": [name_eq.dict(), age_gt.dict()]} 55 | 56 | with pytest.raises(TypeError): 57 | name_eq & "hello" 58 | 59 | with pytest.raises(TypeError): 60 | name_eq | "goodbye" 61 | 62 | # Test fusing 63 | assert (anded & anded).operand == [name_eq, age_gt, name_eq, age_gt] 64 | assert (anded | anded).operand != [name_eq, age_gt, name_eq, age_gt] 65 | assert (ored | ored).operand == [name_eq, age_gt, name_eq, age_gt] 66 | assert (ored & ored).operand != [name_eq, age_gt, name_eq, age_gt] 67 | 68 | 69 | def test_build_expr(): 70 | """Test building an expression from a query dictionary""" 71 | name_eq = expr.Comparison("name", expr.Eq("frank")) 72 | assert expr.build_expr(name_eq) is name_eq 73 | 74 | assert isinstance(expr.build_expr({}), expr.Empty) 75 | 76 | assert isinstance(expr.build_expr({"name": "tom", "age": 54}), expr.And) 77 | 78 | 79 | def test_query_overlapping_filter_keys(): 80 | gt_24 = expr.Comparison("age", expr.Gt(24)) 81 | lt_38 = expr.Comparison("age", expr.Lt(38)) 82 | compound1 = gt_24 & lt_38 83 | compound2 = gt_24 & lt_38 84 | query_filter = expr.Query(compound1, compound2).get_filter() 85 | assert query_filter == {"$and": [expr.query_expr(compound1), expr.query_expr(compound2)]} 86 | 87 | 88 | def test_queryable(): 89 | """Test queryable operators result in MongoDB expressions that we expect""" 90 | field_name = "test" 91 | value = "value" 92 | list_value = "val1", "val2" 93 | 94 | class TestQueryable(expr.Queryable): 95 | field = field_name 96 | 97 | def get_path(self) -> str: 98 | return self.field 99 | 100 | queryable = TestQueryable() 101 | 102 | # Check that the field name cannot be None 103 | with pytest.raises(ValueError): 104 | queryable.field = None 105 | queryable == value 106 | 107 | queryable.field = field_name 108 | 109 | # Special case for equals which drops the operator 110 | assert expr.query_expr(queryable == value) == {field_name: value} 111 | 112 | # Check that 'simple' operators (i.e. field value) 113 | simple_operators = { 114 | "__ne__": "$ne", 115 | "__gt__": "$gt", 116 | "__ge__": "$gte", 117 | "__lt__": "$lt", 118 | "__le__": "$lte", 119 | } 120 | for attr, op in simple_operators.items(): 121 | query_expr = expr.query_expr(getattr(queryable, attr)(value)) 122 | assert query_expr == {field_name: {op: value}} 123 | 124 | # Check operators that take a list of values 125 | list_operators = { 126 | "in_": "$in", 127 | "nin_": "$nin", 128 | } 129 | for attr, op in list_operators.items(): 130 | query_expr = expr.query_expr(getattr(queryable, attr)(*list_value)) 131 | assert query_expr == {field_name: {op: list_value}} 132 | 133 | # Test exists 134 | assert expr.query_expr(queryable.exists_(True)) == {field_name: {"$exists": True}} 135 | with pytest.raises(ValueError): 136 | expr.query_expr(queryable.exists_("true")) 137 | 138 | # Test regex 139 | assert expr.query_expr(queryable.regex_(value)) == {field_name: {"$regex": value}} 140 | assert expr.query_expr(queryable.regex_(value, "i")) == { 141 | field_name: {"$regex": value, "$options": "i"} 142 | } 143 | with pytest.raises(ValueError): 144 | queryable.regex_(True) 145 | 146 | # Test starts_with 147 | assert expr.query_expr(queryable.starts_with_(value)) == {field_name: {"$regex": f"^{value}"}} 148 | 149 | 150 | def test_query_expr(): 151 | field_name = "test" 152 | value = "value" 153 | 154 | # If you pass in a dictionary it is just returned 155 | assert expr.query_expr({field_name: value}) == {field_name: value} 156 | 157 | with pytest.raises(TypeError): 158 | expr.query_expr([]) 159 | 160 | class FaultyFilterLike(expr.FilterLike): 161 | def __query_expr__(self) -> dict: 162 | return "hello" 163 | 164 | with pytest.raises(TypeError): 165 | expr.query_expr(FaultyFilterLike()) 166 | -------------------------------------------------------------------------------- /test/test_file.py: -------------------------------------------------------------------------------- 1 | import io 2 | import shutil 3 | 4 | import mincepy 5 | 6 | # pylint: disable=invalid-name 7 | 8 | 9 | def test_file_basics(historian: mincepy.Historian): 10 | ENCODING = "utf-8" 11 | INITIAL_DATA = "hello there" 12 | file = historian.create_file(ENCODING) 13 | with file.open("w") as stream: 14 | stream.write(INITIAL_DATA) 15 | 16 | file_id = historian.save(file) 17 | del file 18 | 19 | loaded = historian.load(file_id) 20 | with loaded.open("r") as file: 21 | buffer = io.StringIO() 22 | shutil.copyfileobj(file, buffer) 23 | assert buffer.getvalue() == INITIAL_DATA 24 | 25 | 26 | def test_file_changing(tmp_path, historian: mincepy.Historian): # pylint: disable=unused-argument 27 | encoding = "utf-8" 28 | INITIAL_DATA = "Initial string" 29 | mince_file = historian.create_file(encoding=encoding) 30 | 31 | with mince_file.open("w") as file: 32 | file.write(INITIAL_DATA) 33 | 34 | historian.save(mince_file) 35 | 36 | # Now let's append to the file 37 | NEW_DATA = "Second string" 38 | with mince_file.open("a") as file: 39 | file.write(NEW_DATA) 40 | 41 | historian.save(mince_file) 42 | history = historian.history(mince_file) 43 | assert len(history) == 2 44 | 45 | with history[0].obj.open() as file: 46 | buffer = io.StringIO() 47 | shutil.copyfileobj(file, buffer) 48 | assert INITIAL_DATA == buffer.getvalue() 49 | 50 | with history[1].obj.open() as file: 51 | buffer = io.StringIO() 52 | shutil.copyfileobj(file, buffer) 53 | assert INITIAL_DATA + NEW_DATA == buffer.getvalue() 54 | 55 | 56 | def test_nested_files_in_list(historian: mincepy.Historian): 57 | file = historian.create_file() 58 | my_list = mincepy.builtins.List() 59 | my_list.append(file) 60 | 61 | list_id = historian.save(my_list) 62 | del my_list 63 | 64 | loaded = historian.load(list_id) 65 | assert len(loaded) == 1 66 | assert loaded[0].filename is None 67 | 68 | 69 | def test_nested_files_in_dict(historian: mincepy.Historian): 70 | file = historian.create_file() 71 | my_dict = mincepy.builtins.Dict() 72 | my_dict["file"] = file 73 | 74 | list_id = historian.save(my_dict) 75 | del my_dict 76 | 77 | loaded = historian.load(list_id) 78 | assert len(loaded) == 1 79 | assert loaded["file"].filename is None 80 | 81 | 82 | def test_nested_files_in_list_mutating( 83 | tmp_path, historian: mincepy.Historian 84 | ): # pylint: disable=unused-argument 85 | encoding = "utf-8" 86 | INITIAL_DATA = "First string".encode(encoding) 87 | my_file = historian.create_file() 88 | with my_file.open("wb") as file: 89 | file.write(INITIAL_DATA) 90 | 91 | my_list = mincepy.builtins.List() 92 | my_list.append(my_file) 93 | 94 | list_id = historian.save(my_list) 95 | 96 | # Now let's append to the file 97 | NEW_DATA = "Second string".encode(encoding) 98 | with my_file.open("ab") as file: 99 | file.write(NEW_DATA) 100 | 101 | # Save the list again 102 | historian.save(my_list) 103 | del my_list 104 | 105 | loaded = historian.load(list_id) 106 | with loaded[0].open("rb") as contents: 107 | buffer = io.BytesIO() 108 | shutil.copyfileobj(contents, buffer) 109 | assert buffer.getvalue() == INITIAL_DATA + NEW_DATA 110 | 111 | 112 | def test_file_eq(historian: mincepy.Historian): 113 | file1 = historian.create_file("file1") 114 | file1_again = historian.create_file("file1") 115 | file3 = historian.create_file("file3") 116 | 117 | file1.write_text("hello 1!") 118 | file1_again.write_text("hello 1!") # Same again 119 | file3.write_text("hello 3!") # Different 120 | 121 | assert file1 == file1_again 122 | assert file1 != file3 123 | assert file1_again != file3 124 | -------------------------------------------------------------------------------- /test/test_frontend.py: -------------------------------------------------------------------------------- 1 | import mincepy 2 | from mincepy import frontend, testing 3 | 4 | # pylint: disable=invalid-name 5 | 6 | 7 | def test_collection(historian): 8 | def identity(x): 9 | return x 10 | 11 | coll = frontend.EntriesCollection(historian, historian.archive.objects, entry_factory=identity) 12 | p1 = testing.Person("martin", 35) 13 | p1.save() 14 | p2 = testing.Person("john", 5) 15 | p2.save() 16 | 17 | p1.age = 36 18 | p1.save() 19 | 20 | # Find by obj ID 21 | records = coll.find(obj_id=p1.obj_id).one() 22 | assert isinstance(records, dict) 23 | assert records["obj_id"] == p1.obj_id 24 | 25 | # Find using multiple obj IDs 26 | records = list(coll.find(obj_id=[p1.obj_id, p2.obj_id])) 27 | assert len(records) == 2 28 | assert {records[0]["obj_id"], records[1]["obj_id"]} == {p1.obj_id, p2.obj_id} 29 | 30 | c1 = testing.Car() 31 | c1.save() 32 | 33 | records = list(coll.find(obj_type=[testing.Car.TYPE_ID, testing.Person.TYPE_ID])) 34 | assert len(records) == 3 35 | assert {records[0]["type_id"], records[1]["type_id"], records[2]["type_id"]} == { 36 | testing.Car.TYPE_ID, 37 | testing.Person.TYPE_ID, 38 | } 39 | 40 | 41 | def test_distinct(historian): 42 | """Test that .distinct() works on collections""" 43 | honda = testing.Car("honda", "green") 44 | porsche = testing.Car("porsche", "black") 45 | red_honda = testing.Car("honda", "red") 46 | fiat = testing.Car("fiat", "green") 47 | red_porsche = testing.Car("porsche", "red") 48 | historian.save(honda, porsche, red_honda, fiat, red_porsche) 49 | 50 | assert set(historian.objects.distinct("state.colour")) == {"green", "black", "red"} 51 | # Now using the field 52 | assert set(historian.objects.distinct(testing.Car.colour)) == { 53 | "green", 54 | "black", 55 | "red", 56 | } 57 | assert len(list(historian.objects.distinct(mincepy.DataRecord.obj_id))) == 5 58 | -------------------------------------------------------------------------------- /test/test_global.py: -------------------------------------------------------------------------------- 1 | """Test global functions in mincepy""" 2 | 3 | import mincepy 4 | from mincepy import testing 5 | 6 | 7 | def test_default_archive_uri(monkeypatch): 8 | monkeypatch.delenv(mincepy.ENV_ARCHIVE_URI, raising=False) 9 | assert mincepy.default_archive_uri() == mincepy.DEFAULT_ARCHIVE_URI 10 | monkeypatch.setenv(mincepy.ENV_ARCHIVE_URI, "mongodb://example.com") 11 | assert mincepy.default_archive_uri() == "mongodb://example.com" 12 | 13 | 14 | def test_save_load(): 15 | car = testing.Car() 16 | car_id = mincepy.save(car) 17 | del car 18 | # Now try loading 19 | mincepy.load(car_id) 20 | 21 | 22 | def test_find(): 23 | car = testing.Car() 24 | car_id = car.save() 25 | assert list(mincepy.find(obj_id=car_id))[0] is car 26 | 27 | 28 | def test_delete(): 29 | car = testing.Car() 30 | car_id = car.save() 31 | mincepy.delete(car) 32 | assert not list(mincepy.find(obj_id=car_id)) 33 | -------------------------------------------------------------------------------- /test/test_helpers.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | import pytest 4 | 5 | import mincepy 6 | from mincepy import testing 7 | 8 | # pylint: disable=too-few-public-methods 9 | 10 | 11 | def test_type_helper(historian: mincepy.Historian): 12 | """Check that a type helper can be used to make a non-historian compatible type compatible""" 13 | 14 | # Prevent the historian from automatically making classes savable 15 | historian._autosave = False 16 | 17 | class Bird: 18 | def __init__(self, specie="hoopoe"): 19 | self.specie = specie 20 | 21 | class BirdHelper( 22 | mincepy.TypeHelper, obj_type=Bird, type_id=uuid.UUID("5cc59e03-ea5d-43ff-8814-3b6f2e22cd76") 23 | ): 24 | specie = mincepy.field() 25 | 26 | bird = Bird() 27 | with pytest.raises(TypeError): 28 | historian.save(bird) 29 | 30 | # Now register the helper... 31 | historian.register_type(BirdHelper()) 32 | # ...and we should be able to save 33 | assert historian.save(bird) is not None 34 | 35 | 36 | def test_transaction_snapshots(historian: mincepy.Historian): 37 | class ThirdPartyPerson: 38 | """A class from a third party library""" 39 | 40 | def __init__(self, name): 41 | self.name = name 42 | 43 | class PersonHelper( 44 | mincepy.TypeHelper, 45 | obj_type=ThirdPartyPerson, 46 | type_id=uuid.UUID("62d8c767-14bc-4437-a9a3-ca5d0ce65d9b"), 47 | ): 48 | INJECT_CREATION_TRACKING = True 49 | 50 | def yield_hashables(self, obj, hasher): 51 | yield from hasher.yield_hashables(obj.name) 52 | 53 | def eq(self, one, other) -> bool: 54 | return one.name == other.name 55 | 56 | def save_instance_state(self, obj, saver): 57 | return obj.name 58 | 59 | def load_instance_state(self, obj, saved_state, loader): 60 | obj.name = saved_state.name 61 | 62 | person_helper = PersonHelper() 63 | historian.register_type(person_helper) 64 | 65 | person_maker = mincepy.Process("person maker") 66 | 67 | with person_maker.running(): 68 | martin = ThirdPartyPerson("Martin") 69 | 70 | historian.save(martin) 71 | assert historian.created_by(martin) == historian.get_obj_id(person_maker) 72 | 73 | 74 | class Boat: 75 | def __init__(self, make: str, length: float, owner: testing.Person = None): 76 | self.make = make 77 | self.length = length 78 | self.owner = owner 79 | 80 | 81 | class BoatHelper( 82 | mincepy.TypeHelper, obj_type=Boat, type_id=uuid.UUID("4d82b67a-dbcb-4388-b20e-8542c70491d1") 83 | ): 84 | # Describe how to store the properties 85 | make = mincepy.field() 86 | length = mincepy.field() 87 | owner = mincepy.field(ref=True) 88 | 89 | 90 | def test_simple_helper(historian: mincepy.Historian): 91 | historian.register_type(BoatHelper()) 92 | 93 | jenneau = Boat("jenneau", 38.9) 94 | jenneau_id = historian.save(jenneau) 95 | del jenneau 96 | 97 | jenneau = historian.load(jenneau_id) 98 | assert jenneau.make == "jenneau" 99 | assert jenneau.length == 38.9 100 | 101 | # Now check that references work 102 | martin = testing.Person("martin", 35) 103 | jenneau.owner = martin 104 | historian.save(jenneau) 105 | del jenneau 106 | 107 | jenneau = historian.load(jenneau_id) 108 | assert jenneau.owner is martin 109 | 110 | 111 | class Powerboat(Boat): 112 | horsepower = mincepy.field() 113 | 114 | def __init__(self, make: str, length: float, horsepower: float, owner: testing.Person = None): 115 | super().__init__(make, length, owner) 116 | self.horsepower = horsepower 117 | 118 | 119 | class PowerboatHelper( 120 | BoatHelper, obj_type=Powerboat, type_id=uuid.UUID("924ef5b2-ce20-40b0-8c98-4da470f6c2c3") 121 | ): 122 | 123 | horsepower = mincepy.field() 124 | 125 | 126 | def test_subclass_helper(historian: mincepy.Historian): 127 | historian.register_type(PowerboatHelper()) 128 | 129 | quicksilver = Powerboat("quicksilver", length=7.0, horsepower=115) 130 | quicksilver_id = historian.save(quicksilver) 131 | del quicksilver 132 | 133 | quicksilver = historian.load(quicksilver_id) 134 | assert quicksilver.make == "quicksilver" 135 | assert quicksilver.length == 7.0 136 | assert quicksilver.horsepower == 115 137 | 138 | martin = testing.Person("martin", 35) 139 | quicksilver.owner = martin 140 | historian.save(quicksilver) 141 | del quicksilver 142 | 143 | quicksilver = historian.load(quicksilver_id) 144 | assert quicksilver.owner is martin 145 | -------------------------------------------------------------------------------- /test/test_history.py: -------------------------------------------------------------------------------- 1 | import mincepy 2 | from mincepy import testing 3 | 4 | 5 | def test_db(): 6 | """Test using the db() free-function to get type helpers""" 7 | helper = mincepy.db(testing.Car) 8 | assert isinstance(helper, mincepy.TypeHelper) 9 | assert helper.TYPE_ID == testing.Car.TYPE_ID 10 | -------------------------------------------------------------------------------- /test/test_migrate.py: -------------------------------------------------------------------------------- 1 | """Tests of migration""" 2 | 3 | import gc 4 | 5 | import mincepy 6 | from mincepy import testing 7 | 8 | from .common import CarV1, CarV2, StoreByRef, StoreByValue 9 | 10 | 11 | def test_find_migratable(historian: mincepy.Historian): 12 | car = CarV1("white", "lada") 13 | car_id = car.save() 14 | by_val = StoreByValue(car) 15 | by_val_id = by_val.save() 16 | 17 | # Register a new version of the car 18 | historian.register_type(CarV2) 19 | 20 | # Now both car and by_val should need migration (because by_val stores a car) 21 | migratable = tuple(historian.migrations.find_migratable_records()) 22 | assert len(migratable) == 2 23 | ids = [record.obj_id for record in migratable] 24 | assert car_id in ids 25 | assert by_val_id in ids 26 | 27 | # Now register a new version of StoreByVal 28 | historian.register_type(StoreByRef) 29 | 30 | # There should still be the same to migratables as before 31 | migratable = tuple(historian.migrations.find_migratable_records()) 32 | assert len(migratable) == 2 33 | ids = [record.obj_id for record in migratable] 34 | assert car_id in ids 35 | assert by_val_id in ids 36 | 37 | 38 | def test_migrate_with_saved(historian: mincepy.Historian): 39 | """Test migrating an object that has saved references""" 40 | 41 | class V3(mincepy.ConvenientSavable, type_id=StoreByRef.TYPE_ID): 42 | ref = mincepy.field(ref=True) 43 | description = mincepy.field() 44 | 45 | class Migration(mincepy.ObjectMigration): 46 | VERSION = 2 47 | PREVIOUS = StoreByRef.ToRefMigration 48 | 49 | @classmethod 50 | def upgrade(cls, saved_state, loader: "mincepy.Loader"): 51 | saved_state["description"] = None 52 | return saved_state 53 | 54 | LATEST_MIGRATION = Migration 55 | 56 | def __init__(self, ref): 57 | super().__init__() 58 | self.ref = ref 59 | self.description = None 60 | 61 | obj = StoreByRef(testing.Car()) 62 | obj_id = obj.save() 63 | del obj 64 | gc.collect() 65 | 66 | historian.register_type(V3) 67 | migrated = historian.migrations.migrate_all() 68 | assert len(migrated) == 1 69 | assert migrated[0].obj_id == obj_id 70 | 71 | obj = historian.load(obj_id) 72 | 73 | assert isinstance(obj, V3) 74 | assert hasattr(obj, "description") 75 | assert obj.description is None 76 | -------------------------------------------------------------------------------- /test/test_process.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | import pytest 4 | 5 | import mincepy 6 | from mincepy import testing 7 | 8 | # pylint: disable=invalid-name 9 | 10 | 11 | def test_basic_save_process(historian: mincepy.Historian): 12 | proc = mincepy.Process("test_basic_save") 13 | pid = historian.save(proc) 14 | with proc.running(): 15 | car = testing.Car("nissan", "white") 16 | car_id = historian.save(car) 17 | assert historian.created_by(car) == pid 18 | assert historian.get_creator(car) is proc 19 | assert historian.get_creator(car_id) is proc 20 | 21 | second_car = testing.Car("ford") 22 | historian.save(second_car) 23 | assert historian.created_by(second_car) is None 24 | 25 | # Now check we can get the creator id from the object id 26 | del car 27 | assert historian.created_by(car_id) == pid 28 | assert historian.get_creator(car_id) is proc 29 | 30 | 31 | def test_save_after_creation(historian: mincepy.Historian): 32 | """ 33 | Test saving an object that was created inside a process context but then saved 34 | outside it. The creator should still be correctly set 35 | """ 36 | proc = mincepy.Process("test_delayed_save") 37 | proc.save() 38 | with proc.running(): 39 | # Create the car 40 | car = testing.Car("nissan", "white") 41 | 42 | # Save it 43 | historian.save(car) 44 | created_in = historian.created_by(car) 45 | assert created_in is not None 46 | assert created_in == historian.get_current_record(proc).obj_id 47 | 48 | 49 | def test_process_nested_running(historian: mincepy.Historian): 50 | proc = mincepy.Process("test_nested_exception") 51 | with proc.running(): 52 | with proc.running(): 53 | pass 54 | assert proc.is_running 55 | historian.save(proc) 56 | 57 | # Now check that nested exceptions are correctly handled 58 | with pytest.raises(TypeError): 59 | with proc.running(): 60 | with pytest.raises(RuntimeError): 61 | with proc.running(): 62 | raise RuntimeError("Failed yo") 63 | assert proc.is_running 64 | raise TypeError("New error") 65 | assert proc.is_running 66 | proc_id = historian.save(proc) 67 | del proc 68 | 69 | loaded = historian.load(proc_id) 70 | assert not loaded.is_running 71 | 72 | 73 | def test_saving_while_running(historian: mincepy.Historian): 74 | proc = mincepy.Process("test_nested_exception") 75 | with proc.running(): 76 | historian.save(proc) 77 | 78 | 79 | def test_saving_creator_that_owns_child(historian: mincepy.Historian): 80 | class TestProc(mincepy.Process, type_id=uuid.UUID("21b75757-d5dc-4dd6-9329-af56d0ddb879")): 81 | ATTRS = ("child",) 82 | 83 | def __init__(self): 84 | super().__init__("test_proc") 85 | self.child = None 86 | 87 | test_proc = TestProc() 88 | with test_proc.running(): 89 | test_proc.child = mincepy.ObjRef(testing.Car()) 90 | historian.save(test_proc) 91 | 92 | 93 | def test_process_track(historian: mincepy.Historian): 94 | class TestProc(mincepy.Process, type_id=uuid.UUID("90a867f1-1d7b-4591-8b35-5ff457987a89")): 95 | @mincepy.track 96 | def execute(self): 97 | return mincepy.builtins.RefList([testing.Car()]) 98 | 99 | proc = TestProc("test process") 100 | proc.save() 101 | car_list = proc.execute() 102 | historian.save(car_list) 103 | proc_id = historian.get_obj_id(proc) 104 | 105 | assert proc_id is not None 106 | assert historian.get_current_record(car_list).created_by == proc_id 107 | assert historian.get_current_record(car_list[0]).created_by == proc_id 108 | -------------------------------------------------------------------------------- /test/test_qops.py: -------------------------------------------------------------------------------- 1 | from mincepy import qops 2 | 3 | 4 | def test_and(): 5 | # If there's only one condition then there is no need to 'and' things together 6 | assert qops.and_("single") == "single" 7 | 8 | 9 | def test_or(): 10 | assert qops.or_("single") == "single" 11 | 12 | 13 | def test_eq(): 14 | assert qops.eq_("value") == {"$eq": "value"} 15 | 16 | 17 | def test_gt(): 18 | assert qops.gt_("value") == {"$gt": "value"} 19 | 20 | 21 | def test_gte(): 22 | assert qops.gte_("value") == {"$gte": "value"} 23 | 24 | 25 | def test_lte(): 26 | assert qops.lte_("value") == {"$lte": "value"} 27 | 28 | 29 | def test_ne(): 30 | assert qops.ne_("value") == {"$ne": "value"} 31 | 32 | 33 | def test_nin(): 34 | assert qops.nin_("value") == {"$ne": "value"} 35 | assert qops.nin_("value", "value2") == {"$nin": ["value", "value2"]} 36 | -------------------------------------------------------------------------------- /test/test_refs.py: -------------------------------------------------------------------------------- 1 | """Module for testing object references""" 2 | 3 | from argparse import Namespace 4 | import gc 5 | 6 | import mincepy 7 | from mincepy import testing 8 | import mincepy.records 9 | 10 | # pylint: disable=invalid-name 11 | 12 | 13 | def test_obj_ref_simple(historian: mincepy.Historian): 14 | a = testing.Cycle() 15 | a.ref = mincepy.ObjRef(a) 16 | aid = historian.save(a) 17 | del a 18 | 19 | loaded = historian.load(aid) 20 | assert loaded.ref() is loaded 21 | 22 | 23 | def test_obj_ref_snapshot(historian: mincepy.Historian): 24 | """Check that a historic snapshot still works with references""" 25 | ns = Namespace() 26 | car = testing.Car("honda", "white") 27 | ns.car = mincepy.ObjRef(car) 28 | historian.save(ns) 29 | honda_ns_sid = historian.get_snapshot_id(ns) 30 | 31 | car.make = "fiat" 32 | historian.save(ns) 33 | fiat_ns_sid = historian.get_snapshot_id(ns) 34 | del ns 35 | 36 | assert fiat_ns_sid.version == honda_ns_sid.version + 1 37 | 38 | loaded = historian.load(honda_ns_sid) 39 | assert loaded.car().make == "honda" 40 | 41 | loaded2 = historian.load(fiat_ns_sid) 42 | assert loaded2.car().make == "fiat" 43 | assert loaded2.car() is not car 44 | 45 | # Load the 'live' namespace 46 | loaded3 = historian.load(honda_ns_sid.obj_id) 47 | assert loaded3.car() is car 48 | 49 | 50 | def test_obj_sid_complex(historian: mincepy.Historian): 51 | honda = testing.Car("honda") 52 | nested1 = Namespace() 53 | nested2 = Namespace() 54 | parent = Namespace() 55 | 56 | # Both the nested refer to the same car 57 | nested1.car = mincepy.ObjRef(honda) 58 | nested2.car = mincepy.ObjRef(honda) 59 | 60 | # Now put them in their containers 61 | parent.ns1 = mincepy.ObjRef(nested1) 62 | parent.ns2 = mincepy.ObjRef(nested2) 63 | 64 | parent_id = historian.save(parent) 65 | del parent 66 | 67 | loaded = historian.load(parent_id) 68 | assert loaded.ns1() is nested1 69 | assert loaded.ns2() is nested2 70 | assert loaded.ns1().car() is loaded.ns2().car() 71 | 72 | fiat = testing.Car("fiat") 73 | loaded.ns2().car = mincepy.ObjRef(fiat) 74 | historian.save(loaded) 75 | parent_sid = historian.get_snapshot_id(loaded) 76 | del loaded 77 | 78 | loaded2 = historian.load_snapshot(mincepy.records.SnapshotId(parent_id, 0)) 79 | assert loaded2.ns1().car().make == "honda" 80 | assert loaded2.ns2().car().make == "honda" 81 | del loaded2 82 | 83 | loaded3 = historian.load_snapshot(parent_sid) 84 | assert loaded3.ns1().car().make == "honda" 85 | assert loaded3.ns2().car().make == "fiat" 86 | 87 | 88 | def test_null_ref(historian: mincepy.Historian): 89 | null = mincepy.ObjRef() 90 | null2 = mincepy.ObjRef() 91 | 92 | assert null == null2 93 | 94 | nid1, _nid2 = historian.save(null, null2) 95 | del null 96 | loaded = historian.load(nid1) 97 | assert loaded == null2 98 | 99 | 100 | def test_ref_load_save_load(historian: mincepy.Historian): 101 | """This is here to catch a bug that manifested when a reference was saved, loaded and then 102 | re-saved without being dereferenced in-between. This would result in the second saved state 103 | being that of a null reference. This can only be tested if the reference is stored by value 104 | as otherwise the historian will not re-save a reference that has not been mutated.""" 105 | ref_list = mincepy.List((mincepy.ObjRef(testing.Car()),)) 106 | assert isinstance(ref_list[0](), testing.Car) 107 | 108 | list_id = ref_list.save() # pylint: disable=no-member 109 | del ref_list 110 | 111 | loaded = historian.load(list_id) 112 | # Re-save 113 | loaded.save() 114 | del loaded 115 | 116 | # Re-load 117 | reloaded = historian.load(list_id) 118 | # Should still be our car but because of a bug this was a None reference 119 | assert isinstance(reloaded[0](), testing.Car) 120 | 121 | 122 | def test_load_changed_ref(historian: mincepy.Historian, archive_uri): 123 | """Test what happens when you dereference a reference to an object that was mutated since the 124 | reference was loaded""" 125 | historian2 = mincepy.connect(archive_uri) 126 | 127 | car = testing.Car(make="skoda") 128 | car_id = historian.save(car) 129 | car_ref = mincepy.ref(car) 130 | ref_id = historian.save(car_ref) 131 | ref_sid = historian.get_snapshot_id(car_ref) 132 | del car, car_ref 133 | gc.collect() 134 | 135 | # Now, load the reference but don't dereference it yet 136 | loaded = historian.load(ref_id) 137 | 138 | # Now, mutate the car that is being referenced 139 | loaded_car = historian2.load(car_id) 140 | loaded_car.make = "honda" 141 | historian2.save(loaded_car) 142 | 143 | # Finally, dereference and check that it is as expected 144 | assert loaded().make == "honda" 145 | 146 | # Now, check the snapshot still points to the original 147 | loaded_snapshot: mincepy.ObjRef = historian.load_snapshot(ref_sid) 148 | assert loaded_snapshot().make == "skoda" 149 | -------------------------------------------------------------------------------- /test/test_snapshots.py: -------------------------------------------------------------------------------- 1 | """Module for testing saved snapshots""" 2 | 3 | import time 4 | 5 | import pytest 6 | 7 | import mincepy 8 | from mincepy.testing import Car, Cycle 9 | 10 | # pylint: disable=invalid-name 11 | 12 | 13 | def test_list_basics(historian: mincepy.Historian): 14 | parking_lot = mincepy.builtins.List() 15 | for i in range(1000): 16 | parking_lot.append(Car(str(i))) 17 | 18 | historian.save(parking_lot) 19 | list_sid = historian.get_snapshot_id(parking_lot) 20 | 21 | # Change one element 22 | parking_lot[0].make = "ferrari" 23 | historian.save(parking_lot) 24 | new_list_sid = historian.get_snapshot_id(parking_lot) 25 | 26 | assert list_sid != new_list_sid 27 | 28 | old_list = historian.load_snapshot(list_sid) 29 | assert old_list is not parking_lot 30 | 31 | assert old_list[0].make == str(0) 32 | 33 | 34 | def test_save_snapshot_change_load(historian: mincepy.Historian): 35 | car = Car() 36 | 37 | # Saving twice without changing should produce the same snapshot id 38 | historian.save(car) 39 | ferrari_sid = historian.get_snapshot_id(car) 40 | historian.save(car) 41 | assert ferrari_sid == historian.get_snapshot_id(car) 42 | 43 | car.make = "fiat" 44 | car.color = "white" 45 | 46 | historian.save(car) 47 | fiat_sid = historian.get_snapshot_id(car) 48 | 49 | assert fiat_sid != ferrari_sid 50 | 51 | ferrari = historian.load_snapshot(ferrari_sid) 52 | 53 | assert ferrari.make == "ferrari" 54 | assert ferrari.colour == "red" 55 | 56 | 57 | def test_transaction_snapshots(historian: mincepy.Historian): 58 | ferrari = Car("ferrari") 59 | historian.save(ferrari) 60 | ferrari_sid = historian.get_snapshot_id(ferrari) 61 | 62 | with historian.transaction(): 63 | ferrari_snapshot_1 = historian.load_snapshot(ferrari_sid) 64 | with historian.transaction(): 65 | ferrari_snapshot_2 = historian.load_snapshot(ferrari_sid) 66 | # Reference wise they should be unequal 67 | assert ferrari_snapshot_1 is not ferrari_snapshot_2 68 | assert ferrari is not ferrari_snapshot_1 69 | assert ferrari is not ferrari_snapshot_2 70 | 71 | # Value wise they should be equal 72 | assert ferrari == ferrari_snapshot_1 73 | assert ferrari == ferrari_snapshot_2 74 | 75 | # Now check within the same transaction the result is the same 76 | ferrari_snapshot_2 = historian.load_snapshot(ferrari_sid) 77 | # Reference wise they should be unequal 78 | assert ferrari_snapshot_1 is not ferrari_snapshot_2 79 | assert ferrari is not ferrari_snapshot_1 80 | assert ferrari is not ferrari_snapshot_2 81 | 82 | # Value wise they should be equal 83 | assert ferrari == ferrari_snapshot_1 84 | assert ferrari == ferrari_snapshot_2 85 | 86 | 87 | def test_record_times(historian: mincepy.Historian): 88 | car = Car("honda", "red") 89 | historian.save(car) 90 | time.sleep(0.001) 91 | 92 | car.colour = "white" 93 | historian.save(car) 94 | time.sleep(0.01) 95 | 96 | car.colour = "yellow" 97 | historian.save(car) 98 | 99 | history = historian.history(car, as_objects=False) 100 | assert len(history) == 3 101 | for idx, record in zip(range(1, 2), history[1:]): 102 | assert record.creation_time == history[0].creation_time 103 | assert record.snapshot_time > history[0].creation_time 104 | assert record.snapshot_time > history[idx - 1].snapshot_time 105 | 106 | 107 | def test_get_latest(historian: mincepy.Historian): 108 | # Save the car 109 | car = Car() 110 | car_id = historian.save(car) 111 | 112 | # Change it and save getting a snapshot 113 | car.make = "fiat" 114 | car.colour = "white" 115 | historian.save(car) 116 | fiat_sid = historian.get_snapshot_id(car) 117 | assert car_id != fiat_sid 118 | 119 | # Change it again... 120 | car.make = "honda" 121 | car.colour = "wine red" 122 | historian.save(car) 123 | honda_sid = historian.get_snapshot_id(car) 124 | assert honda_sid != fiat_sid 125 | assert honda_sid != car_id 126 | 127 | # Now delete and reload 128 | del car 129 | latest = historian.load(car_id) 130 | assert latest == historian.load_snapshot(honda_sid) 131 | 132 | 133 | def test_history(historian: mincepy.Historian): 134 | rainbow = ["red", "orange", "yellow", "green", "blue", "indigo", "violet"] 135 | 136 | car = Car() 137 | car_id = None 138 | for colour in rainbow: 139 | car.colour = colour 140 | car_id = historian.save(car) 141 | 142 | car_history = historian.history(car_id) 143 | assert len(car_history) == len(rainbow) 144 | for i, entry in enumerate(car_history): 145 | assert entry[1].colour == rainbow[i] 146 | 147 | # Test loading directly from snapshot id 148 | assert historian.load_snapshot(car_history[2].ref) == car_history[2].obj 149 | 150 | # Test slicing 151 | assert historian.history(car_id, -1)[0].obj.colour == rainbow[-1] 152 | 153 | # Try changing history 154 | old_version = car_history[2].obj 155 | old_version.colour = "black" 156 | with pytest.raises(mincepy.ModificationError): 157 | historian.save(old_version) 158 | 159 | 160 | def test_loading_snapshot(historian: mincepy.Historian): 161 | honda = Car("honda", "white") 162 | historian.save(honda) 163 | white_honda_sid = historian.get_snapshot_id(honda) 164 | honda.colour = "red" 165 | historian.save(honda) 166 | del honda 167 | 168 | with historian.transaction(): 169 | white_honda = historian.load_snapshot(white_honda_sid) 170 | assert white_honda.colour == "white" 171 | # Make sure that if we load it again we get a different object instance 172 | assert white_honda is not historian.load_snapshot(white_honda_sid) 173 | 174 | 175 | def test_loading_snapshot_cycle(historian: mincepy.Historian): 176 | a = Cycle() 177 | a.ref = a # Close the cycle 178 | historian.save(a) 179 | a_sid = historian.get_snapshot_id(a) 180 | del a 181 | 182 | loaded = historian.load_snapshot(a_sid) 183 | assert loaded.ref is loaded 184 | 185 | 186 | def test_snapshot_id_eq(): 187 | sid1 = mincepy.SnapshotId("sid", 1) 188 | sid1_again = mincepy.SnapshotId("sid", 1) 189 | sid2 = mincepy.SnapshotId("sid", 2) 190 | 191 | assert sid1 == sid1_again 192 | assert sid1 != sid2 193 | assert sid1_again != sid2 194 | 195 | 196 | def test_purge(historian: mincepy.Historian): 197 | """Test that snapshots.purge() removes snapshots corresponding to deleted objects""" 198 | assert historian.snapshots.purge().deleted_purged == set() 199 | 200 | car = Car() 201 | car_id = car.save() 202 | car.colour = "yellow" 203 | car.save() 204 | 205 | # Insert something else just to check that it doesn't get accidentally purged 206 | car2 = Car() 207 | car2.save() 208 | 209 | historian.delete(car) 210 | records_count = historian.records.find().count() 211 | res = historian.snapshots.purge(dry_run=False) 212 | 213 | assert records_count == historian.records.find().count() 214 | assert res.deleted_purged == { 215 | mincepy.SnapshotId(car_id, 0), 216 | mincepy.SnapshotId(car_id, 1), 217 | mincepy.SnapshotId(car_id, 2), # The deleted record 218 | } 219 | -------------------------------------------------------------------------------- /test/test_staging.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from mincepy import staging 4 | 5 | 6 | def test_basics(): 7 | with pytest.raises(RuntimeError): 8 | staging.StagingArea() 9 | 10 | assert staging.get_info(object(), create=False) is None 11 | 12 | with pytest.raises(KeyError): 13 | staging.remove(object()) 14 | 15 | # These calls should do nothing 16 | staging.replace(object(), object()) 17 | -------------------------------------------------------------------------------- /test/test_transactions.py: -------------------------------------------------------------------------------- 1 | """Module for testing saved snapshots""" 2 | 3 | import mincepy 4 | from mincepy import testing 5 | 6 | 7 | def test_snapshot_id_in_transaction(historian: mincepy.Historian): 8 | """An object saved during a transaction should already have its new snapshot id available during 9 | the transaction.""" 10 | car = testing.Car("ferrari", "red") 11 | car.save() 12 | sid = historian.get_snapshot_id(car) 13 | 14 | with historian.transaction(): 15 | car.make = "honda" 16 | car.save() 17 | assert historian.get_snapshot_id(car) != sid 18 | -------------------------------------------------------------------------------- /test/test_type_registry.py: -------------------------------------------------------------------------------- 1 | import mincepy.testing 2 | import mincepy.type_registry 3 | 4 | 5 | def test_basics(): 6 | registry = mincepy.type_registry.TypeRegistry() 7 | 8 | # Register using the object type 9 | registry.register_type(mincepy.testing.Car) 10 | assert mincepy.testing.Car in registry 11 | registry.unregister_type(mincepy.testing.Car) 12 | assert mincepy.testing.Car not in registry 13 | -------------------------------------------------------------------------------- /test/test_types.py: -------------------------------------------------------------------------------- 1 | import mincepy 2 | from mincepy.testing import Car 3 | 4 | 5 | def test_savable_object(historian: mincepy.Historian): 6 | """Test basic properties and functions of a savable object""" 7 | car = Car("smart", "black") 8 | car_id = car.save() 9 | car.set_meta({"reg": "VD395"}) 10 | del car 11 | 12 | loaded = historian.load(car_id) # type: Car 13 | assert loaded.make == "smart" 14 | assert loaded.colour == "black" 15 | assert loaded.get_meta() == {"reg": "VD395"} 16 | 17 | loaded.update_meta({"driver": "martin"}) 18 | assert loaded.get_meta() == {"reg": "VD395", "driver": "martin"} 19 | 20 | assert loaded.obj_id == car_id 21 | 22 | loaded.make = "honda" 23 | loaded.save() 24 | del loaded 25 | 26 | reloaded = historian.load(car_id) 27 | assert reloaded.make == "honda" 28 | -------------------------------------------------------------------------------- /test/utils.py: -------------------------------------------------------------------------------- 1 | import random 2 | import string 3 | 4 | 5 | def random_str(length=10): 6 | letters = string.ascii_lowercase 7 | return "".join(random.choice(letters) for _ in range(length)) 8 | --------------------------------------------------------------------------------