├── testproject ├── testapp │ ├── __init__.py │ ├── models.py │ └── tests.py ├── testproject │ ├── __init__.py │ └── settings.py └── manage.py ├── src └── modelsubquery │ ├── __init__.py │ └── functions.py ├── .github └── workflows │ ├── isort.yml │ └── black.yml ├── README.md ├── pyproject.toml ├── LICENSE └── .gitignore /testproject/testapp/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /testproject/testproject/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/modelsubquery/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tools to build subqueries that produce model instances 3 | """ 4 | 5 | __version__ = "0.1.1" 6 | 7 | from .functions import JSONModelField, ModelSubquery, model_to_json 8 | -------------------------------------------------------------------------------- /.github/workflows/isort.yml: -------------------------------------------------------------------------------- 1 | name: Isort 2 | 3 | on: [pull_request] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v3 10 | - uses: isort/isort-action@v1 11 | -------------------------------------------------------------------------------- /.github/workflows/black.yml: -------------------------------------------------------------------------------- 1 | name: Black 2 | 3 | on: [pull_request] 4 | 5 | jobs: 6 | lint: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v3 10 | - uses: actions/setup-python@v5 11 | with: 12 | python-version: '3.12' # 3.11 needed for black's use_pyproject 13 | - uses: psf/black@stable 14 | with: 15 | use_pyproject: true 16 | -------------------------------------------------------------------------------- /testproject/testproject/settings.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | BASE_DIR = Path(__file__).resolve().parent.parent 4 | 5 | SECRET_KEY = "insecure" 6 | DEBUG = True 7 | 8 | INSTALLED_APPS = [ 9 | "testapp", 10 | ] 11 | 12 | DATABASES = { 13 | "default": { 14 | "ENGINE": "django.db.backends.sqlite3", 15 | "NAME": BASE_DIR / "db.sqlite3", 16 | } 17 | } 18 | 19 | TIME_ZONE = "UTC" 20 | USE_TZ = True 21 | 22 | DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # django-model-subquery 2 | Get model instances back from your subqueries 3 | 4 | Read the origin story on [my blog](https://blog.bmispelon.rocks/articles/2024/2024-05-09-django-getting-a-full-model-instance-from-a-subquery.html). 5 | 6 | 7 | # Test project 8 | This repository comes with a test project to show off some of the capabilities. 9 | 10 | To install it, use the `[testproject]` extra requirement (preferrably in a virtualenv): 11 | ``` 12 | pip install -e .[testproject] 13 | ``` 14 | 15 | Then you can run the project's test suite: 16 | ``` 17 | cd testproject && python manage.py test 18 | ``` 19 | -------------------------------------------------------------------------------- /testproject/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Django's command-line utility for administrative tasks.""" 3 | import os 4 | import sys 5 | 6 | 7 | def main(): 8 | """Run administrative tasks.""" 9 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "testproject.settings") 10 | try: 11 | from django.core.management import execute_from_command_line 12 | except ImportError as exc: 13 | raise ImportError( 14 | "Couldn't import Django. Are you sure it's installed and " 15 | "available on your PYTHONPATH environment variable? Did you " 16 | "forget to activate a virtual environment?" 17 | ) from exc 18 | execute_from_command_line(sys.argv) 19 | 20 | 21 | if __name__ == "__main__": 22 | main() 23 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["flit_core >=3.2,<4"] 3 | build-backend = "flit_core.buildapi" 4 | 5 | [project] 6 | name = "modelsubquery" 7 | authors = [{name = "Baptiste Mispelon", email = "hello@bmispelon.rocks"}] 8 | readme = "README.md" 9 | license = {file = "LICENSE"} 10 | classifiers = [ 11 | "Development Status :: 4 - Beta", 12 | "Programming Language :: Python :: 3", 13 | "Framework :: Django", 14 | "License :: OSI Approved :: MIT License" 15 | ] 16 | dynamic = ["version", "description"] 17 | 18 | [project.urls] 19 | Home = "https://github.com/bmispelon/django-model-subquery" 20 | 21 | [project.optional-dependencies] 22 | testproject = [ 23 | "Django", 24 | "model-bakery", 25 | "time-machine", 26 | ] 27 | dev = [ 28 | "black<24.5.0", 29 | "isort", 30 | ] 31 | 32 | [tool.isort] 33 | profile = "black" 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Baptiste Mispelon 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /testproject/testapp/models.py: -------------------------------------------------------------------------------- 1 | from datetime import date 2 | 3 | from django.db import models 4 | from django.db.models.functions import ExtractYear 5 | 6 | from modelsubquery import ModelSubquery 7 | 8 | 9 | class Book(models.Model): 10 | title = models.CharField(max_length=100) 11 | rating = models.IntegerField(blank=True, null=True) 12 | published = models.DateField(default=date.today) 13 | has_cover = models.BooleanField(default=True) 14 | author = models.ForeignKey( 15 | "Person", on_delete=models.CASCADE, related_name="books", null=True 16 | ) 17 | 18 | 19 | class PersonQuerySet(models.QuerySet): 20 | def with_book_of_the_year(self, fields=None): 21 | """ 22 | Annotate each person in the queryset with the best rated book of the 23 | year they were born. 24 | """ 25 | year = ExtractYear(models.OuterRef("birth")) 26 | all_books = Book.objects.filter(published__year=year).order_by("-rating") 27 | return self.annotate(book_of_year=ModelSubquery(all_books, fields=fields)) 28 | 29 | 30 | class Person(models.Model): 31 | name = models.CharField(max_length=100) 32 | birth = models.DateField(default=date.today) 33 | 34 | objects = PersonQuerySet.as_manager() 35 | 36 | 37 | class Shelve(models.Model): 38 | title = models.CharField(max_length=100) 39 | books = models.ManyToManyField("Book", related_name="shelves") 40 | -------------------------------------------------------------------------------- /src/modelsubquery/functions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tools to build subqueries that produce model instances 3 | """ 4 | 5 | from django.db import models 6 | from django.db.models.constants import LOOKUP_SEP 7 | from django.db.models.functions import JSONObject 8 | 9 | 10 | def _model_fields(model, fields): 11 | """ 12 | Return a set of the field names for the given model. If fields is __all__ 13 | then return all the declared fields. 14 | """ 15 | # TODO: the pk/id field should always be returned 16 | declared = {f.column for f in model._meta.local_concrete_fields} 17 | if fields is None: 18 | return declared 19 | 20 | assert declared.issuperset(fields) 21 | return declared.intersection(fields) 22 | 23 | 24 | class JSONModelField(models.JSONField): 25 | """ 26 | Instantiate an actual model instance from a JSON object containing its fields. 27 | Any missing fields in the JSON object are marked as "deferred" (and will be 28 | loaded from the db on access). 29 | """ 30 | 31 | # TODO: support the "pk" alias 32 | # TODO: support fk fields (recursive __ access) 33 | def __init__(self, model, *args, **kwargs): 34 | super().__init__(*args, **kwargs) 35 | self._model_class = model 36 | 37 | def from_db_value(self, value, expression, connection): 38 | value = super().from_db_value(value, expression, connection) 39 | if value is None: 40 | return None 41 | return self.instantiate_model_from_db_dict(value) 42 | 43 | def instantiate_model_from_db_dict(self, dbdict): 44 | declared = _model_fields(self._model_class, None) 45 | missing = declared.difference(dbdict.keys()) 46 | 47 | kwargs = { 48 | k: self._model_class._meta.get_field(k).to_python(v) 49 | for k, v in dbdict.items() 50 | } 51 | deferred = dict.fromkeys(missing, models.DEFERRED) 52 | return self._model_class(**kwargs, **deferred) 53 | 54 | 55 | def model_to_json(model, path="", fields=None): 56 | """ 57 | Return a JSONObject containing all the fields from the given model accessible 58 | at the given path (an empty path corresponds to the current queryset's model). 59 | """ 60 | prefix = "" if not path else (path + LOOKUP_SEP) 61 | fields = _model_fields(model, fields) 62 | return JSONObject(**{field: prefix + field for field in fields}) 63 | 64 | 65 | def ModelSubquery(queryset, fields=None): 66 | """ 67 | Return a subquery that when annotated will produce an actual instance of the 68 | queryset's model. 69 | All fields on the model are loaded by default, use `fields=...` to specify 70 | and iterable of the field names you'd like to load. The missing fields will 71 | be marked as deferred. 72 | """ 73 | jsonobj = model_to_json(queryset.model, fields=fields) 74 | return models.Subquery( 75 | queryset.values_list(jsonobj), output_field=JSONModelField(queryset.model) 76 | ) 77 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | -------------------------------------------------------------------------------- /testproject/testapp/tests.py: -------------------------------------------------------------------------------- 1 | from datetime import date 2 | 3 | import time_machine 4 | from django.test import TestCase 5 | from model_bakery import baker 6 | 7 | from modelsubquery.functions import _model_fields 8 | 9 | from .models import Book, Person, Shelve 10 | 11 | 12 | class ModelFieldTest(TestCase): 13 | def test_model_fields_book(self): 14 | fields = _model_fields(Book, None) 15 | self.assertEqual( 16 | fields, {"id", "title", "author_id", "published", "rating", "has_cover"} 17 | ) 18 | 19 | def test_model_fields_shelve(self): 20 | fields = _model_fields(Shelve, None) 21 | self.assertEqual(fields, {"id", "title"}) 22 | 23 | 24 | @time_machine.travel("2000-01-01") 25 | class DBFunctionTestCase(TestCase): 26 | def test_instance_is_returned(self): 27 | book = baker.make(Book) 28 | baker.make(Person) 29 | 30 | person = Person.objects.with_book_of_the_year().get() 31 | self.assertEqual(person.book_of_year, book) 32 | 33 | def test_field_type_int(self): 34 | baker.make(Book, rating=5) 35 | baker.make(Person) 36 | 37 | person = Person.objects.with_book_of_the_year().get() 38 | self.assertEqual(person.book_of_year.rating, 5) 39 | 40 | def test_field_type_date(self): 41 | baker.make(Book, published="2005-01-01") 42 | baker.make(Person, birth="2005-06-01") 43 | 44 | person = Person.objects.with_book_of_the_year().get() 45 | self.assertEqual(person.book_of_year.published, date(2005, 1, 1)) 46 | 47 | def test_field_type_bool(self): 48 | baker.make(Book, has_cover=False) 49 | baker.make(Person) 50 | 51 | person = Person.objects.with_book_of_the_year().get() 52 | self.assertIs(person.book_of_year.has_cover, False) 53 | 54 | def test_field_null(self): 55 | baker.make(Book, rating=None) 56 | baker.make(Person) 57 | 58 | person = Person.objects.with_book_of_the_year().get() 59 | self.assertIsNone(person.book_of_year.rating) 60 | 61 | def test_no_matching_book(self): 62 | baker.make(Person) 63 | 64 | person = Person.objects.with_book_of_the_year().get() 65 | self.assertIsNone(person.book_of_year) 66 | 67 | def test_multiple_matching_books(self): 68 | baker.make(Book, rating=10, title="best") 69 | baker.make(Book, rating=1, title="worst") 70 | baker.make(Person) 71 | 72 | person = Person.objects.with_book_of_the_year().get() 73 | self.assertEqual(person.book_of_year.title, "best") 74 | 75 | def test_returned_instance_is_updateable(self): 76 | book = baker.make(Book, title="test") 77 | baker.make(Person) 78 | 79 | person = Person.objects.with_book_of_the_year().get() 80 | person.book_of_year.title = "updated" 81 | person.book_of_year.save() 82 | book.refresh_from_db() 83 | self.assertEqual(book.title, "updated") 84 | 85 | def test_deffered_fields(self): 86 | book = baker.make(Book, title="test", rating=5) 87 | baker.make(Person) 88 | 89 | person = Person.objects.with_book_of_the_year(fields=["id"]).get() 90 | with self.assertNumQueries(0): 91 | self.assertEqual(person.book_of_year.pk, book.pk) 92 | with self.assertNumQueries(1): 93 | self.assertEqual(person.book_of_year.title, "test") 94 | with self.assertNumQueries(1): 95 | self.assertEqual(person.book_of_year.rating, 5) 96 | 97 | def test_with_related_field(self): 98 | author = baker.make(Person) 99 | book = baker.make(Book, author=author) 100 | 101 | # Yes he wrote a book in his birthyear! ;-) 102 | 103 | person = Person.objects.with_book_of_the_year().get() 104 | self.assertEqual(person.book_of_year, book) 105 | 106 | # The relation is there, but it's lazy! 107 | with self.assertNumQueries(1): 108 | self.assertEqual(person.book_of_year.author, author) 109 | --------------------------------------------------------------------------------