├── .github └── workflows │ ├── mysql_test.yml │ ├── postgres_test.yml │ └── sqlite_test.yml ├── LICENSE ├── Readme.rst ├── pyproject.toml ├── runtests.py └── sql_util ├── __init__.py ├── aggregates.py ├── apps.py ├── debug.py ├── tests ├── __init__.py ├── models.py ├── test_exists.py ├── test_mysql_settings.py ├── test_postgres_settings.py ├── test_sqlite_settings.py └── test_subquery.py └── utils.py /.github/workflows/mysql_test.yml: -------------------------------------------------------------------------------- 1 | name: Mysql Test 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | max-parallel: 4 14 | matrix: 15 | python-version: [ 3.8, 3.9, 3.10.13, 3.11 ] 16 | django-version: [ 3.2.21, 4.1.11, 4.2.5 ] 17 | exclude: 18 | - python-version: 3.8 19 | django-version: 4.1.11 20 | - python-version: 3.8 21 | django-version: 4.2.5 22 | - python-version: 3.11 23 | django-version: 3.2.21 24 | - python-version: 3.11 25 | django-version: 4.1.11 26 | services: 27 | mysql: 28 | image: mysql 29 | ports: 30 | - 3306:3306 31 | env: 32 | MYSQL_PASSWORD: mysql 33 | MYSQL_USER: mysql 34 | MYSQL_ROOT_PASSWORD: mysql 35 | MYSQL_DATABASE: mysql 36 | MYSQL_HOST: 127.0.0.1 37 | options: >- 38 | --health-cmd="mysqladmin ping" 39 | --health-interval=10s 40 | --health-timeout=5s 41 | --health-retries=5 42 | steps: 43 | - uses: actions/checkout@v2 44 | - name: Set up Python ${{ matrix.python-version }} 45 | uses: actions/setup-python@v2 46 | with: 47 | python-version: ${{ matrix.python-version }} 48 | - name: Install Dependencies 49 | run: | 50 | python -m pip install --upgrade pip 51 | pip install coverage 52 | pip install codecov 53 | pip install mysqlclient 54 | pip install sqlparse 55 | pip install -q Django==${{ matrix.django-version }} 56 | - name: Run Tests 57 | run: | 58 | coverage run --omit="*site-packages*","*test*" runtests.py --settings=sql_util.tests.test_mysql_settings 59 | codecov -------------------------------------------------------------------------------- /.github/workflows/postgres_test.yml: -------------------------------------------------------------------------------- 1 | name: Postgres Test 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | max-parallel: 4 14 | matrix: 15 | python-version: [ 3.8, 3.9, 3.10.13, 3.11 ] 16 | django-version: [ 3.2.21, 4.1.11, 4.2.5 ] 17 | exclude: 18 | - python-version: 3.8 19 | django-version: 4.1.11 20 | - python-version: 3.8 21 | django-version: 4.2.5 22 | - python-version: 3.11 23 | django-version: 3.2.21 24 | - python-version: 3.11 25 | django-version: 4.1.11 26 | services: 27 | postgres: 28 | image: postgres 29 | ports: 30 | - 5432:5432 31 | env: 32 | POSTGRES_PASSWORD: postgres 33 | POSTGRES_USER: postgres 34 | options: >- 35 | --health-cmd pg_isready 36 | --health-interval 10s 37 | --health-timeout 5s 38 | --health-retries 5 39 | steps: 40 | - uses: actions/checkout@v2 41 | - name: Set up Python ${{ matrix.python-version }} 42 | uses: actions/setup-python@v2 43 | with: 44 | python-version: ${{ matrix.python-version }} 45 | - name: Install Dependencies 46 | run: | 47 | python -m pip install --upgrade pip 48 | pip install coverage 49 | pip install codecov 50 | pip install psycopg2 51 | pip install sqlparse 52 | pip install -q Django==${{ matrix.django-version }} 53 | - name: Run Tests 54 | run: | 55 | coverage run --omit="*site-packages*","*test*" runtests.py --settings=sql_util.tests.test_postgres_settings 56 | codecov -------------------------------------------------------------------------------- /.github/workflows/sqlite_test.yml: -------------------------------------------------------------------------------- 1 | name: Sqlite Test 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | max-parallel: 4 14 | matrix: 15 | python-version: [ 3.8, 3.9, 3.10.13, 3.11 ] 16 | django-version: [ 3.2.21, 4.1.11, 4.2.5 ] 17 | exclude: 18 | - python-version: 3.8 19 | django-version: 4.1.11 20 | - python-version: 3.8 21 | django-version: 4.2.5 22 | - python-version: 3.11 23 | django-version: 3.2.21 24 | - python-version: 3.11 25 | django-version: 4.1.11 26 | steps: 27 | - uses: actions/checkout@v2 28 | - name: Set up Python ${{ matrix.python-version }} 29 | uses: actions/setup-python@v2 30 | with: 31 | python-version: ${{ matrix.python-version }} 32 | - name: Install Dependencies 33 | run: | 34 | python -m pip install --upgrade pip 35 | pip install coverage 36 | pip install codecov 37 | pip install sqlparse 38 | pip install -q Django==${{ matrix.django-version }} 39 | - name: Run Tests 40 | run: | 41 | coverage run --omit="*site-packages*","*test*" runtests.py --settings=sql_util.tests.test_sqlite_settings 42 | codecov -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Brad Martsberger 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Readme.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://codecov.io/gh/martsberger/django-sql-utils/branch/master/graph/badge.svg 2 | :target: https://codecov.io/gh/martsberger/django-sql-utils 3 | 4 | .. image:: https://img.shields.io/pypi/dm/django-sql-utils.svg 5 | :target: https://pypistats.org/packages/django-sql-utils 6 | 7 | 8 | Django SQL Utils 9 | ================ 10 | 11 | This package provides utilities for working with Django querysets so that 12 | you can generate the SQL that you want, with an API you enjoy. 13 | 14 | Subquery Aggregates 15 | ------------------- 16 | 17 | The `Count` aggregation in Django:: 18 | 19 | Parent.objects.annotate(child_count=Count('child')) 20 | 21 | generates SQL like the following:: 22 | 23 | SELECT parent.*, Count(child.id) as child_count 24 | FROM parent 25 | JOIN child on child.parent_id = parent.id 26 | GROUP BY parent.id 27 | 28 | In many cases, this is not as performant as doing the count in a SUBQUERY 29 | instead of with a JOIN:: 30 | 31 | SELECT parent.*, 32 | (SELECT Count(id) 33 | FROM child 34 | WHERE parent_id = parent.id) as child_count 35 | FROM parent 36 | 37 | Django allows us to generate this SQL using The Subquery and OuterRef classes:: 38 | 39 | 40 | subquery = Subquery(Child.objects.filter(parent_id=OuterRef('id')).order_by() 41 | .values('parent').annotate(count=Count('pk')) 42 | .values('count'), output_field=IntegerField()) 43 | Parent.objects.annotate(child_count=Coalesce(subquery, 0)) 44 | 45 | Holy cow! It's not trivial to figure what everything is doing in the above 46 | code and it's not particularly good for maintenance. SubqueryAggregates allow 47 | you to forget all that complexity and generate the subquery count like this:: 48 | 49 | Parent.objects.annotate(child_count=SubqueryCount('child')) 50 | 51 | Phew! Much easier to read and understand. It's the same API as the original `Count` 52 | just specifying the Subquery version. 53 | 54 | Easier API for Exists 55 | --------------------- 56 | If you have a Parent/Child relationship (Child has a ForeignKey to Parent), you can annotate a queryset 57 | of Parent objects with a boolean indicating whether or not the parent has children:: 58 | 59 | from django.db.models import Exists 60 | 61 | parents = Parent.objects.annotate( 62 | has_children=Exists(Child.objects.filter(parent=OuterRef('pk')) 63 | ) 64 | 65 | That's a bit more boilerplate than should be necessary, so we provide a simpler API for Exists:: 66 | 67 | from sql_util.utils import Exists 68 | 69 | parents = Parent.objects.annotate( 70 | has_children=Exists('child') 71 | ) 72 | 73 | The child queryset can be filtered with the keyword argument `filter`. E.g.,:: 74 | 75 | parents = Parent.objects.annotate( 76 | has_child_named_John = Exists('child', filter=Q(name='John')) 77 | ) 78 | 79 | The `sql_util` version of `Exists` can also take a queryset as the first parameter and behave just like 80 | the Django `Exists` class, so you are able to use it everywhere without worrying about name confusion. 81 | 82 | Installation and Usage 83 | ---------------------- 84 | 85 | Install from PyPI:: 86 | 87 | pip install django-sql-utils 88 | 89 | Then you can:: 90 | 91 | from sql_util.utils import SubqueryCount 92 | 93 | And use that as shown above. 94 | 95 | In addition to `SubqueryCount`, this package provides 96 | 97 | * `SubqueryMin` 98 | * `SubqueryMax` 99 | * `SubquerySum` 100 | * `SubqueryAvg` 101 | 102 | If you want to use other aggregates, you can use the 103 | generic `SubqueryAggregate` class. For example, if you want to use Postgres' `ArrayAgg` 104 | to get an array of `Child.name` for each `Parent`:: 105 | 106 | from django.contrib.postgres.aggregates import ArrayAgg 107 | 108 | aggregate = SubqueryAggregate('child__name', aggregate=ArrayAgg) 109 | Parent.objects.annotate(child_names=aggregate) 110 | 111 | Or subclass SubqueryAggregate:: 112 | 113 | from django.contrib.postgres.aggregates import ArrayAgg 114 | 115 | class SubqueryArrayAgg(SubqueryAggregate) 116 | aggregate = ArrayAgg 117 | unordered = True 118 | 119 | Parent.objects.annotate(avg_child_age=SubqueryArrayAgg('child__age')) 120 | 121 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "django-sql-utils" 7 | version = "0.7.0" 8 | authors = [ 9 | { name="Brad Martsberger", email="bradley.marts@gmail.com" }, 10 | ] 11 | description = "Improved API for aggregating using Subquery" 12 | readme = "Readme.rst" 13 | requires-python = ">=3.8" 14 | classifiers = [ 15 | "Programming Language :: Python :: 3", 16 | "License :: OSI Approved :: MIT License", 17 | "Operating System :: OS Independent", 18 | ] 19 | dependencies = [ 20 | "django>=3.2", 21 | "sqlparse" 22 | ] 23 | 24 | [project.urls] 25 | "Homepage" = "https://github.com/martsberger/django-sql-utils" 26 | "Download" = "https://github.com/martsberger/django-sql-utils/archive/0.7.0.tar.gz" -------------------------------------------------------------------------------- /runtests.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from optparse import OptionParser 4 | 5 | 6 | def parse_args(): 7 | parser = OptionParser() 8 | parser.add_option('-s', '--settings', help='Define settings.', default='sql_util.tests.test_sqlite_settings') 9 | parser.add_option('-v', '--verbosity', help='Set the verbosity level.', default=1, type=int) 10 | return parser.parse_args() 11 | 12 | 13 | if __name__ == '__main__': 14 | options, tests = parse_args() 15 | os.environ['DJANGO_SETTINGS_MODULE'] = options.settings 16 | 17 | # Local imports because DJANGO_SETTINGS_MODULE needs to be set first 18 | import django 19 | from django.test.utils import get_runner 20 | from django.conf import settings 21 | 22 | if hasattr(django, 'setup'): 23 | django.setup() 24 | 25 | TestRunner = get_runner(settings) 26 | runner = TestRunner(verbosity=options.verbosity, interactive=True, failfast=False) 27 | sys.exit(runner.run_tests(tests)) 28 | -------------------------------------------------------------------------------- /sql_util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/martsberger/django-sql-utils/3df890aa08f0e28b9f48489c5e1f7c5955d0daec/sql_util/__init__.py -------------------------------------------------------------------------------- /sql_util/aggregates.py: -------------------------------------------------------------------------------- 1 | from django.core.exceptions import FieldError 2 | from django.db.models import Q, F, QuerySet, BooleanField, Sum, Avg, ForeignKey 3 | from django.db.models import Subquery as DjangoSubquery, OuterRef, IntegerField, Min, Max, Count 4 | from django.db.models.constants import LOOKUP_SEP 5 | 6 | 7 | class Subquery(DjangoSubquery): 8 | def __init__(self, queryset_or_expression, **extra): 9 | if isinstance(queryset_or_expression, QuerySet): 10 | self.queryset = queryset_or_expression 11 | self.query = self.queryset.query 12 | super(Subquery, self).__init__(queryset_or_expression, **extra) 13 | else: 14 | expression = queryset_or_expression 15 | if not hasattr(expression, 'resolve_expression'): 16 | expression = F(expression) 17 | self.expression = expression 18 | self.query = None 19 | self.queryset = None 20 | self.output_field = extra.get('output_field') 21 | self.extra = extra 22 | self.filter = extra.pop('filter', Q()) 23 | self.distinct = extra.pop('distinct', None) 24 | self.outer_ref = extra.pop('outer_ref', None) 25 | self.unordered = extra.pop('unordered', self.unordered) 26 | 27 | def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False): 28 | # The parent class, Subquery, takes queryset as an initialization parameter 29 | # so self.queryset needs to be set before we call `resolve_expression`. 30 | # We can set it here because we now have access to the outer query object, 31 | # which is the first parameter of this method. 32 | if self.query is None or self.queryset is None: 33 | # Don't pass allow_joins = False here 34 | queryset = self.get_queryset(query.clone(), True, reuse, summarize) 35 | self.queryset = queryset 36 | self.query = queryset.query 37 | return super(Subquery, self).resolve_expression(query, allow_joins, reuse, summarize, for_save) 38 | 39 | def get_queryset(self, query, allow_joins, reuse, summarize): 40 | # This is a customization hook for child classes to override the base queryset computed automatically 41 | return self._get_base_queryset(query, allow_joins, reuse, summarize) 42 | 43 | def _get_base_queryset(self, query, allow_joins, reuse, summarize): 44 | resolved_expression = self.expression.resolve_expression(query, allow_joins, reuse, summarize) 45 | model = self._get_model_from_resolved_expression(resolved_expression) 46 | 47 | reverse, outer_ref = self._get_reverse_outer_ref_from_expression(model, query) 48 | 49 | outer_ref = self.outer_ref or outer_ref 50 | q = self.filter & Q(**{reverse: OuterRef(outer_ref)}) 51 | queryset = model._default_manager.filter(q) 52 | if self.unordered: 53 | queryset = queryset.order_by() 54 | return queryset.values(reverse) 55 | 56 | def _get_model_from_resolved_expression(self, resolved_expression): 57 | """ 58 | Retrieve the correct model from the resolved_expression. 59 | 60 | For simple expressions like F('child__field_name'), both of these are equivalent and correct: 61 | resolved_expression.field.model 62 | resolved_expression.target.model 63 | 64 | For many to many relations, resolved_expression.field.model goes one table deeper than 65 | necessary. We get more efficient SQL only going as far as we need. In this case only 66 | resolved_expression.target.model is correct. 67 | 68 | For functions of multiple columns like Coalesce, there is no resolved_expression.target, 69 | we have to recursively go through the source_expressions until we get to the bottom and 70 | get the target from there. 71 | """ 72 | def get_target(res_expr): 73 | for expression in res_expr.get_source_expressions(): 74 | return get_target(expression) 75 | return res_expr.field if res_expr.target.null else res_expr.target 76 | return get_target(resolved_expression).model 77 | 78 | def _get_fields_model_from_path(self, path, model, target_model): 79 | fields = [] 80 | 81 | # We want the paths reversed because we have the forward join info 82 | # and we need the string that tells us how to go back 83 | paths = list(reversed(path)) 84 | for p in paths: 85 | if p.to_opts.model == model and ((p.from_opts.model != target_model or p.m2m) or not fields): 86 | if getattr(p.join_field, 'related_query_name', None) and isinstance(p.join_field, ForeignKey): 87 | try: 88 | fields.append(p.join_field.related_query_name()) 89 | except TypeError: # Sometimes related_query_name is a string instead of a callable that returns a string 90 | fields.append(p.join_field.related_query_name) 91 | elif hasattr(p.join_field, 'field'): 92 | fields.append(p.join_field.field.name) 93 | model = p.from_opts.model 94 | 95 | return fields, model 96 | 97 | def _get_reverse_outer_ref_from_expression(self, model, query): 98 | source = self.expression 99 | while hasattr(source, 'get_source_expressions'): 100 | source = source.get_source_expressions()[0] 101 | field_list = source.name.split(LOOKUP_SEP) 102 | path, _, _, _ = query.names_to_path(field_list, query.get_meta(), allow_many=True, fail_on_missing=True) 103 | 104 | fields, model = self._get_fields_model_from_path(path, model, query.model) 105 | reverse = LOOKUP_SEP.join(fields) 106 | 107 | join_field = path[0].join_field 108 | if model == query.model or len(path) == 1: 109 | try: 110 | outer_ref = join_field.get_related_field().name 111 | except AttributeError: 112 | outer_ref = 'pk' 113 | else: 114 | outer_ref = join_field.name 115 | 116 | return reverse, outer_ref 117 | 118 | 119 | class SubqueryAggregate(Subquery): 120 | """ 121 | The intention of this class is to provide an API similar to other aggregate 122 | classes like Count, Min, Max, Sum, etc but generate SQL that performs the 123 | calculation in a subquery instead of adding joins to the outer query. This 124 | is commonly a performance improvement. It also reduces the risk of 125 | forgetting to add `distinct` when the joins duplicate data. 126 | 127 | E.g., 128 | queryset.annotate(min_field=Min('field')) 129 | 130 | is replaced by 131 | 132 | queryset.annotate(min_field=SubqueryAggregate('field', aggregate=Min)) 133 | 134 | A child class of SubqueryAggregate with `aggregate=Min` allows: 135 | 136 | queryset.annotate(min_field=SubqueryMin('field')) 137 | 138 | """ 139 | aggregate = None # Must be set by the subclass, or passed as kwarg 140 | unordered = None 141 | 142 | def __init__(self, *args, **extra): 143 | self.aggregate = extra.pop('aggregate', self.aggregate) 144 | self.ordering = extra.pop('ordering', None) 145 | assert self.aggregate is not None, "Error: Attempt to instantiate a " \ 146 | "SubqueryAggregate with no aggregate function" 147 | super(SubqueryAggregate, self).__init__(*args, **extra) 148 | 149 | def get_queryset(self, query, allow_joins, reuse, summarize): 150 | queryset = self._get_base_queryset(query, allow_joins, reuse, summarize) 151 | annotation = self._get_annotation(query, allow_joins, reuse, summarize) 152 | return queryset.annotate(**annotation).values('aggregation') 153 | 154 | def aggregate_kwargs(self): 155 | aggregate_kwargs = dict() 156 | if self.distinct: 157 | aggregate_kwargs['distinct'] = self.distinct 158 | if self.ordering: 159 | aggregate_kwargs['ordering'] = self.ordering 160 | 161 | return aggregate_kwargs 162 | 163 | def _get_annotation(self, query, allow_joins, reuse, summarize): 164 | resolved_expression = self.expression.resolve_expression(query, allow_joins, reuse, summarize) 165 | model = self._get_model_from_resolved_expression(resolved_expression) 166 | queryset = model._default_manager.all() 167 | # resolved_expression was resolved in the outer query to get the model 168 | # target_expression is resolved in the subquery to get the field to aggregate 169 | target_expression = self._resolve_to_target(resolved_expression, queryset.query, allow_joins, reuse, 170 | summarize) 171 | 172 | # Add test for output_field, distinct, and when resolved_expression.field.name isn't what we're aggregating 173 | 174 | if not self.output_field: 175 | self._output_field = self.output_field = target_expression.field 176 | 177 | kwargs = self.aggregate_kwargs() 178 | 179 | aggregation = self.aggregate(target_expression, **kwargs) 180 | 181 | annotation = { 182 | 'aggregation': aggregation 183 | } 184 | 185 | return annotation 186 | 187 | def _resolve_to_target(self, resolved_expression, query, allow_joins, reuse, summarize): 188 | if resolved_expression.get_source_expressions(): 189 | c = resolved_expression.copy() 190 | c.is_summary = summarize 191 | new_source_expressions = [self._resolve_to_target(source_expressions, query, allow_joins, reuse, summarize) 192 | for source_expressions in resolved_expression.get_source_expressions()] 193 | c.set_source_expressions(new_source_expressions) 194 | return c 195 | 196 | else: 197 | try: 198 | return F(resolved_expression.target.name).resolve_expression(query, allow_joins, reuse, summarize) 199 | except (FieldError, AttributeError): 200 | return resolved_expression 201 | 202 | 203 | class SubqueryCount(SubqueryAggregate): 204 | template = 'COALESCE((%(subquery)s), 0)' 205 | aggregate = Count 206 | unordered = True 207 | 208 | def __init__(self, expression, reverse='', *args, **kwargs): 209 | kwargs['output_field'] = kwargs.get('output_field', IntegerField()) 210 | super(SubqueryCount, self).__init__(expression, reverse=reverse, *args, **kwargs) 211 | 212 | 213 | class SubqueryMin(SubqueryAggregate): 214 | aggregate = Min 215 | unordered = True 216 | 217 | 218 | class SubqueryMax(SubqueryAggregate): 219 | aggregate = Max 220 | unordered = True 221 | 222 | 223 | class SubquerySum(SubqueryAggregate): 224 | aggregate = Sum 225 | unordered = True 226 | 227 | 228 | class SubqueryAvg(SubqueryAggregate): 229 | aggregate = Avg 230 | unordered = True 231 | 232 | 233 | class Exists(Subquery): 234 | unordered = True 235 | template = 'EXISTS(%(subquery)s)' 236 | 237 | def __init__(self, *args, **kwargs): 238 | self.negated = kwargs.pop('negated', False) 239 | super(Exists, self).__init__(*args, **kwargs) 240 | self.output_field = BooleanField() 241 | 242 | def __invert__(self): 243 | # Be careful not to evaluate self.queryset on this line 244 | return type(self)(self.queryset if self.queryset is not None else self.expression, negated=(not self.negated), **self.extra) 245 | 246 | def as_sql(self, compiler, connection, template=None, **extra_context): 247 | sql, params = super(Exists, self).as_sql(compiler, connection, template, **extra_context) 248 | if self.negated: 249 | sql = 'NOT {}'.format(sql) 250 | return sql, params 251 | 252 | def as_oracle(self, compiler, connection, template=None, **extra_context): 253 | # Oracle doesn't allow EXISTS() in the SELECT list, so wrap it with a 254 | # CASE WHEN expression. Change the template since the When expression 255 | # requires a left hand side (column) to compare against. 256 | sql, params = self.as_sql(compiler, connection, template, **extra_context) 257 | sql = 'CASE WHEN {} THEN 1 ELSE 0 END'.format(sql) 258 | return sql, params 259 | 260 | def get_queryset(self, query, allow_joins, reuse, summarize): 261 | return self._get_base_queryset(query, allow_joins, reuse, summarize) 262 | -------------------------------------------------------------------------------- /sql_util/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class SqlUtilConfig(AppConfig): 5 | name = 'sql_util' 6 | -------------------------------------------------------------------------------- /sql_util/debug.py: -------------------------------------------------------------------------------- 1 | import sqlparse 2 | 3 | 4 | def pretty_print_sql(queryset, **options): 5 | print(sqlparse.format(str(queryset.query), reindent=True, indent_width=4, **options)) 6 | -------------------------------------------------------------------------------- /sql_util/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/martsberger/django-sql-utils/3df890aa08f0e28b9f48489c5e1f7c5955d0daec/sql_util/tests/__init__.py -------------------------------------------------------------------------------- /sql_util/tests/models.py: -------------------------------------------------------------------------------- 1 | from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation 2 | from django.contrib.contenttypes.models import ContentType 3 | from django.db import models 4 | from django.db.models import CASCADE 5 | 6 | 7 | # Simple parent-child model 8 | class Parent(models.Model): 9 | name = models.CharField(max_length=32) 10 | 11 | class Meta: 12 | ordering = ['name'] 13 | 14 | 15 | class Child(models.Model): 16 | name = models.CharField(max_length=32) 17 | parent = models.ForeignKey(Parent, on_delete=CASCADE, related_name='a_child', related_query_name='da_child') 18 | timestamp = models.DateTimeField() 19 | other_timestamp = models.DateTimeField(null=True) 20 | 21 | 22 | # Books, Authors, Editors, there are more than 1 way Book/Author are m2m 23 | # With publisher, there are multiple depths of traversal 24 | class Author(models.Model): 25 | name = models.CharField(max_length=32) 26 | 27 | class Meta: 28 | ordering = ['name'] 29 | 30 | class Publisher(models.Model): 31 | name = models.CharField(max_length=32) 32 | number = models.IntegerField() 33 | 34 | class Meta: 35 | ordering = ['name'] 36 | 37 | 38 | class Book(models.Model): 39 | title = models.CharField(max_length=128) 40 | authors = models.ManyToManyField(Author, through='BookAuthor', related_name='authored_books') 41 | editors = models.ManyToManyField(Author, through='BookEditor', related_name='edited_books') 42 | publisher = models.ForeignKey(Publisher, on_delete=CASCADE, null=True) 43 | 44 | class Meta: 45 | ordering = ['title'] 46 | 47 | 48 | class BookAuthor(models.Model): 49 | book = models.ForeignKey(Book, on_delete=CASCADE) 50 | author = models.ForeignKey(Author, on_delete=CASCADE) 51 | 52 | 53 | class BookEditor(models.Model): 54 | book = models.ForeignKey(Book, on_delete=CASCADE) 55 | editor = models.ForeignKey(Author, on_delete=CASCADE) 56 | 57 | 58 | class Catalog(models.Model): 59 | number = models.CharField(max_length=50) 60 | 61 | 62 | class CatalogInfo(models.Model): 63 | catalog = models.ForeignKey(Catalog, on_delete=CASCADE) 64 | info = models.TextField() 65 | 66 | 67 | class Package(models.Model): 68 | name = models.CharField(max_length=12) 69 | quantity = models.SmallIntegerField() 70 | catalog = models.ForeignKey(Catalog, on_delete=CASCADE) 71 | 72 | 73 | class Purchase(models.Model): 74 | price = models.DecimalField(decimal_places=2, max_digits=10) 75 | pack = models.ForeignKey(Package, on_delete=CASCADE) 76 | 77 | 78 | # This section deliberately has some weird names to make sure we correctly compute forward 79 | # and backward joins 80 | 81 | class Category(models.Model): 82 | name = models.CharField(max_length=12) 83 | 84 | 85 | class Bit(models.Model): 86 | name = models.CharField(max_length=12) 87 | 88 | 89 | # A collection of items. An Item can be in many Collections 90 | # A collection is in a single category, or no categories (nullable) 91 | class Collection(models.Model): 92 | name = models.CharField(max_length=12) 93 | the_category = models.ForeignKey(Category, null=True, on_delete=CASCADE) 94 | bits = models.ManyToManyField(Bit) 95 | 96 | 97 | class Item(models.Model): 98 | name = models.CharField(max_length=12) 99 | collection_key = models.ManyToManyField(Collection, through='ItemCollectionM2M') 100 | 101 | 102 | class ItemCollectionM2M(models.Model): 103 | thing = models.ForeignKey(Item, on_delete=CASCADE) 104 | collection_key = models.ForeignKey(Collection, on_delete=CASCADE) 105 | 106 | 107 | # These models will make sure this works with GenericForeignKey/GenericRelation 108 | 109 | class Dog(models.Model): 110 | name = models.CharField(max_length=12) 111 | 112 | 113 | class Cat(models.Model): 114 | name = models.CharField(max_length=12) 115 | owner = GenericRelation('Owner', related_query_name='owner') 116 | 117 | 118 | class Owner(models.Model): 119 | name = models.CharField(max_length=12) 120 | content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) 121 | object_id = models.PositiveIntegerField() 122 | pet = GenericForeignKey('content_type', 'object_id') 123 | 124 | 125 | # These models will make sure this works with to_field ForeignKeys 126 | 127 | class Brand(models.Model): 128 | name = models.CharField(max_length=12) 129 | company_id = models.IntegerField(unique=True) 130 | 131 | 132 | class Product(models.Model): 133 | num_purchases = models.IntegerField() 134 | brand = models.ForeignKey( 135 | Brand, 136 | to_field='company_id', 137 | related_name='products', 138 | on_delete=models.CASCADE 139 | ) 140 | 141 | # Aggregate on computed column 142 | 143 | 144 | class Store(models.Model): 145 | name = models.CharField(max_length=12) 146 | 147 | 148 | class Seller(models.Model): 149 | name = models.CharField(max_length=12) 150 | store = models.ForeignKey(Store, on_delete=CASCADE) 151 | average_revenue = models.FloatField(default=0) 152 | total_sales = models.IntegerField(default=0) 153 | 154 | 155 | class Sale(models.Model): 156 | date = models.DateField() 157 | revenue = models.FloatField() 158 | expenses = models.FloatField() 159 | seller = models.ForeignKey(Seller, on_delete=CASCADE) 160 | 161 | 162 | # Want to test the case where a model has two foreign keys to the same model 163 | class Player(models.Model): 164 | nickname = models.CharField(max_length=128) 165 | 166 | 167 | class Team(models.Model): 168 | name = models.CharField(max_length=128) 169 | players = models.ManyToManyField(Player) 170 | 171 | 172 | class Game(models.Model): 173 | played = models.DateField() 174 | team1 = models.ForeignKey(Team, on_delete=CASCADE, related_name='team1_game') 175 | team2 = models.ForeignKey(Team, on_delete=CASCADE, related_name='team2_game') 176 | -------------------------------------------------------------------------------- /sql_util/tests/test_exists.py: -------------------------------------------------------------------------------- 1 | from django.db.models import Q, OuterRef 2 | from django.test import TestCase 3 | 4 | from sql_util.tests.models import (Parent, Child, Author, Book, BookAuthor, BookEditor, Publisher, 5 | Category, Collection, Item, ItemCollectionM2M, Bit, Dog, Cat, 6 | Owner) 7 | from sql_util.utils import Exists 8 | 9 | class TestExists(TestCase): 10 | 11 | @classmethod 12 | def setUpClass(cls): 13 | super(TestExists, cls).setUpClass() 14 | 15 | parents = [ 16 | Parent.objects.create(name='John'), 17 | Parent.objects.create(name='Jane') 18 | ] 19 | 20 | children = [ 21 | Child.objects.create(parent=parents[0], name='Joe', timestamp='2017-06-01', other_timestamp=None), 22 | Child.objects.create(parent=parents[0], name='Jan', timestamp='2017-07-01', other_timestamp=None), 23 | Child.objects.create(parent=parents[0], name='Jan', timestamp='2017-05-01', other_timestamp='2017-08-01') 24 | ] 25 | 26 | def test_original_exists(self): 27 | ps = Parent.objects.annotate(has_children=Exists(Child.objects.filter(parent=OuterRef('pk')))).order_by('pk') 28 | ps = list(ps) 29 | 30 | self.assertEqual(ps[0].has_children, True) 31 | self.assertEqual(ps[1].has_children, False) 32 | 33 | def test_easy_exists(self): 34 | ps = Parent.objects.annotate(has_children=Exists('da_child')).order_by('pk') 35 | ps = list(ps) 36 | 37 | self.assertEqual(ps[0].has_children, True) 38 | self.assertEqual(ps[1].has_children, False) 39 | 40 | def test_negated_exists(self): 41 | ps = Parent.objects.annotate(has_children=~Exists(Child.objects.filter(parent=OuterRef('pk')))).order_by('pk') 42 | ps = list(ps) 43 | 44 | self.assertEqual(ps[0].has_children, False) 45 | self.assertEqual(ps[1].has_children, True) 46 | 47 | def test_easy_negated_exists(self): 48 | ps = Parent.objects.annotate(has_children=~Exists('da_child')).order_by('pk') 49 | ps = list(ps) 50 | 51 | self.assertEqual(ps[0].has_children, False) 52 | self.assertEqual(ps[1].has_children, True) 53 | 54 | 55 | class TestExistsFilter(TestCase): 56 | @classmethod 57 | def setUpClass(cls): 58 | super(TestExistsFilter, cls).setUpClass() 59 | publishers = [ 60 | Publisher.objects.create(name='Publisher 1', number=1), 61 | Publisher.objects.create(name='Publisher 2', number=2) 62 | ] 63 | 64 | authors = [ 65 | Author.objects.create(name='Author 1'), 66 | Author.objects.create(name='Author 2'), 67 | Author.objects.create(name='Author 3'), 68 | Author.objects.create(name='Author 4'), 69 | Author.objects.create(name='Author 5'), 70 | ] 71 | 72 | books = [ 73 | Book.objects.create(title='Book 1', publisher=publishers[0]), 74 | Book.objects.create(title='Book 2', publisher=publishers[0]), 75 | Book.objects.create(title='Book 3', publisher=publishers[1]), 76 | Book.objects.create(title='Book 4', publisher=publishers[1]), 77 | ] 78 | 79 | book_authors = [ 80 | BookAuthor.objects.create(author=authors[0], book=books[0], id=1), 81 | BookAuthor.objects.create(author=authors[1], book=books[1], id=2), 82 | BookAuthor.objects.create(author=authors[2], book=books[1], id=3), 83 | BookAuthor.objects.create(author=authors[2], book=books[2], id=4), 84 | ] 85 | 86 | book_editors = [ 87 | BookEditor.objects.create(editor=authors[4], book=books[3]), 88 | ] 89 | 90 | def test_filter_on_exists(self): 91 | exists = Exists('authored_books') 92 | authors = Author.objects.filter(exists) 93 | 94 | self.assertEqual({author.id for author in authors}, 95 | {1, 2, 3}) 96 | 97 | def test_filter_on_negated_exists(self): 98 | exists = ~Exists('authored_books') 99 | authors = Author.objects.filter(exists) 100 | 101 | self.assertEqual({author.id for author in authors}, 102 | {4, 5}) 103 | 104 | def test_filter_exists_with_or(self): 105 | exists = Exists('authored_books') | Exists('edited_books') 106 | authors = Author.objects.filter(exists) 107 | 108 | self.assertEqual({author.id for author in authors}, 109 | {1, 2, 3, 5}) 110 | 111 | 112 | class TestManyToManyExists(TestCase): 113 | 114 | @classmethod 115 | def setUpClass(cls): 116 | super(TestManyToManyExists, cls).setUpClass() 117 | publishers = [ 118 | Publisher.objects.create(name='Publisher 1', number=1), 119 | Publisher.objects.create(name='Publisher 2', number=2) 120 | ] 121 | 122 | authors = [ 123 | Author.objects.create(name='Author 1'), 124 | Author.objects.create(name='Author 2'), 125 | Author.objects.create(name='Author 3'), 126 | Author.objects.create(name='Author 4'), 127 | Author.objects.create(name='Author 5'), 128 | Author.objects.create(name='Author 6') 129 | ] 130 | 131 | books = [ 132 | Book.objects.create(title='Book 1', publisher=publishers[0]), 133 | Book.objects.create(title='Book 2', publisher=publishers[0]), 134 | Book.objects.create(title='Book 3', publisher=publishers[1]), 135 | Book.objects.create(title='Book 4', publisher=publishers[1]) 136 | ] 137 | 138 | book_authors = [ 139 | BookAuthor.objects.create(author=authors[0], book=books[0], id=1), 140 | BookAuthor.objects.create(author=authors[1], book=books[1], id=2), 141 | BookAuthor.objects.create(author=authors[2], book=books[1], id=3), 142 | BookAuthor.objects.create(author=authors[2], book=books[2], id=4), 143 | BookAuthor.objects.create(author=authors[3], book=books[2], id=5), 144 | BookAuthor.objects.create(author=authors[4], book=books[3], id=6), 145 | ] 146 | 147 | book_editors = [ 148 | BookEditor.objects.create(editor=authors[5], book=books[3]), 149 | BookEditor.objects.create(editor=authors[5], book=books[3]), 150 | ] 151 | 152 | def test_forward(self): 153 | books = Book.objects.annotate(has_authors=Exists('authors')).order_by('id') 154 | for book in books: 155 | self.assertTrue(book.has_authors) 156 | 157 | # Only book 4 has editors 158 | books = Book.objects.annotate(has_editors=Exists('editors')).order_by('id') 159 | editors = {book.title: book.has_editors for book in books} 160 | 161 | self.assertEqual(editors, {'Book 1': False, 162 | 'Book 2': False, 163 | 'Book 3': False, 164 | 'Book 4': True}) 165 | 166 | def test_reverse(self): 167 | authors = Author.objects.annotate(has_books=Exists('authored_books')).order_by('id') 168 | books = {author.name: author.has_books for author in authors} 169 | 170 | self.assertEqual(books, {'Author 1': True, 171 | 'Author 2': True, 172 | 'Author 3': True, 173 | 'Author 4': True, 174 | 'Author 5': True, 175 | 'Author 6': False}) 176 | 177 | def test_two_joins(self): 178 | authors = Author.objects.annotate(has_editors=Exists('authored_books__editors')).order_by('id') 179 | 180 | # Only author 5 has written a book with editors 181 | 182 | editors = {author.name: author.has_editors for author in authors} 183 | 184 | self.assertEqual(editors, {'Author 1': False, 185 | 'Author 2': False, 186 | 'Author 3': False, 187 | 'Author 4': False, 188 | 'Author 5': True, 189 | 'Author 6': False}) 190 | 191 | def test_filter(self): 192 | publisher_id = Publisher.objects.get(name='Publisher 1').id 193 | authors = Author.objects.annotate(published_by_1=Exists('authored_books', filter=Q(book__publisher_id=publisher_id))) 194 | 195 | authors = {author.name: author.published_by_1 for author in authors} 196 | 197 | self.assertEqual(authors, {'Author 1': True, 198 | 'Author 2': True, 199 | 'Author 3': True, 200 | 'Author 4': False, 201 | 'Author 5': False, 202 | 'Author 6': False}) 203 | 204 | def test_filter_last_join(self): 205 | publisher_id = Publisher.objects.get(name='Publisher 1').id 206 | authors = Author.objects.annotate( 207 | published_by_1=Exists('authored_books__publisher', filter=Q(id=publisher_id))) 208 | 209 | authors = {author.name: author.published_by_1 for author in authors} 210 | 211 | self.assertEqual(authors, {'Author 1': True, 212 | 'Author 2': True, 213 | 'Author 3': True, 214 | 'Author 4': False, 215 | 'Author 5': False, 216 | 'Author 6': False}) 217 | 218 | 219 | class TestExistsReverseNames(TestCase): 220 | @classmethod 221 | def setUpClass(cls): 222 | super(TestExistsReverseNames, cls).setUpClass() 223 | categories = [ 224 | Category.objects.create(name='cat one'), 225 | Category.objects.create(name='cat two'), 226 | Category.objects.create(name='cat three'), 227 | ] 228 | 229 | collections = [ 230 | Collection.objects.create(name='coll one', the_category=categories[0]), 231 | Collection.objects.create(name='coll two', the_category=categories[0]), 232 | Collection.objects.create(name='coll three', the_category=categories[1]), 233 | Collection.objects.create(name='coll four', the_category=categories[1]), 234 | Collection.objects.create(name='coll five', the_category=categories[2]), 235 | ] 236 | 237 | items = [ 238 | Item.objects.create(name='item one'), 239 | Item.objects.create(name='item two'), 240 | Item.objects.create(name='item three'), 241 | Item.objects.create(name='item four'), 242 | Item.objects.create(name='item five'), 243 | Item.objects.create(name='item six'), 244 | ] 245 | 246 | m2ms = [ 247 | ItemCollectionM2M.objects.create(thing=items[0], collection_key=collections[0]), 248 | ItemCollectionM2M.objects.create(thing=items[1], collection_key=collections[1]) 249 | ] 250 | 251 | bits = [ 252 | Bit.objects.create(name="bit one") 253 | ] 254 | 255 | collections[0].bits.add(bits[0]) 256 | 257 | def test_name_doesnt_match(self): 258 | annotation = { 259 | 'has_category': Exists('collection_key__the_category') 260 | } 261 | 262 | items = Item.objects.annotate(**annotation) 263 | 264 | items = {item.name: item.has_category for item in items} 265 | 266 | self.assertEqual(items, {'item one': True, 267 | 'item two': True, 268 | 'item three': False, 269 | 'item four': False, 270 | 'item five': False, 271 | 'item six': False, 272 | }) 273 | 274 | def test_name_doesnt_match_m2m(self): 275 | annotation = { 276 | 'has_bits': Exists('collection_key__bits') 277 | } 278 | 279 | items = Item.objects.annotate(**annotation) 280 | 281 | items = {item.name: item.has_bits for item in items} 282 | 283 | self.assertEqual(items, {'item one': True, 284 | 'item two': False, 285 | 'item three': False, 286 | 'item four': False, 287 | 'item five': False, 288 | 'item six': False, 289 | }) 290 | 291 | 292 | class TestGenericForeignKey(TestCase): 293 | @classmethod 294 | def setUpClass(cls): 295 | super(TestGenericForeignKey, cls).setUpClass() 296 | dogs = [ 297 | Dog.objects.create(name="Fido"), 298 | Dog.objects.create(name="Snoopy"), 299 | Dog.objects.create(name="Otis") 300 | ] 301 | 302 | cats = [ 303 | Cat.objects.create(name="Muffin"), 304 | Cat.objects.create(name="Grumpy"), 305 | Cat.objects.create(name="Garfield") 306 | ] 307 | 308 | owners = [ 309 | Owner.objects.create(name="Jon", pet=cats[2]) 310 | ] 311 | 312 | def test_exists(self): 313 | annotation = {'has_an_owner': Exists('owner')} 314 | 315 | cats = Cat.objects.annotate(**annotation) 316 | 317 | cats = {cat.name: cat.has_an_owner for cat in cats} 318 | 319 | self.assertEqual(cats, {'Muffin': False, 320 | 'Grumpy': False, 321 | 'Garfield': True}) 322 | -------------------------------------------------------------------------------- /sql_util/tests/test_mysql_settings.py: -------------------------------------------------------------------------------- 1 | BACKEND = 'mysql' 2 | 3 | DATABASES = { 4 | 'default': { 5 | 'ENGINE': 'django.db.backends.mysql', 6 | 'NAME': 'sqlutil', 7 | 'USER': 'root', 8 | 'PASSWORD': 'mysql', 9 | 'HOST': '127.0.0.1', 10 | 'PORT': '3306' 11 | } 12 | } 13 | 14 | INSTALLED_APPS = ( 15 | 'django.contrib.contenttypes', 16 | 'sql_util.tests', 17 | ) 18 | 19 | MIGRATION_MODULES = {'tests': None, 20 | 'contenttypes': None 21 | } 22 | 23 | SITE_ID = 1, 24 | 25 | SECRET_KEY = 'secret' 26 | 27 | MIDDLEWARE_CLASSES = ( 28 | 'django.middleware.common.CommonMiddleware', 29 | 'django.middleware.csrf.CsrfViewMiddleware', 30 | ) 31 | -------------------------------------------------------------------------------- /sql_util/tests/test_postgres_settings.py: -------------------------------------------------------------------------------- 1 | BACKEND = 'postgres' 2 | 3 | DATABASES = { 4 | 'default': { 5 | 'ENGINE': 'django.db.backends.postgresql_psycopg2', 6 | 'NAME': 'sqlutil', 7 | 'USER': 'postgres', 8 | 'PASSWORD': 'postgres', 9 | 'HOST': 'localhost' 10 | } 11 | } 12 | 13 | INSTALLED_APPS = ( 14 | 'django.contrib.contenttypes', 15 | 'sql_util.tests', 16 | ) 17 | 18 | MIGRATION_MODULES = {'tests': None, 19 | 'contenttypes': None 20 | } 21 | 22 | SITE_ID = 1, 23 | 24 | SECRET_KEY = 'secret' 25 | 26 | MIDDLEWARE_CLASSES = ( 27 | 'django.middleware.common.CommonMiddleware', 28 | 'django.middleware.csrf.CsrfViewMiddleware', 29 | ) 30 | -------------------------------------------------------------------------------- /sql_util/tests/test_sqlite_settings.py: -------------------------------------------------------------------------------- 1 | BACKEND = 'sqlite' 2 | 3 | DATABASES = { 4 | 'default': { 5 | 'ENGINE': 'django.db.backends.sqlite3', 6 | 'NAME': ':memory:', 7 | } 8 | } 9 | 10 | INSTALLED_APPS = ( 11 | 'django.contrib.contenttypes', 12 | 'sql_util.tests', 13 | ) 14 | 15 | MIGRATION_MODULES = {'tests': None, 16 | 'contenttypes': None 17 | } 18 | 19 | SITE_ID = 1, 20 | 21 | SECRET_KEY = 'secret' 22 | 23 | MIDDLEWARE_CLASSES = ( 24 | 'django.middleware.common.CommonMiddleware', 25 | 'django.middleware.csrf.CsrfViewMiddleware', 26 | ) 27 | 28 | DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' 29 | -------------------------------------------------------------------------------- /sql_util/tests/test_subquery.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.db.models import DateTimeField, Q 3 | from django.db.models.functions import Coalesce, Cast 4 | from django.test import TestCase 5 | 6 | from sql_util.aggregates import SubqueryAvg, SubquerySum 7 | from sql_util.tests.models import (Parent, Child, Author, Book, BookAuthor, BookEditor, Publisher, Catalog, Package, 8 | Purchase, CatalogInfo, Store, Seller, Sale, Player, Team, Game, Brand, Product) 9 | from sql_util.utils import SubqueryMin, SubqueryMax, SubqueryCount 10 | 11 | 12 | class TestParentChild(TestCase): 13 | 14 | @classmethod 15 | def setUpClass(cls): 16 | super(TestParentChild, cls).setUpClass() 17 | parents = [ 18 | Parent.objects.create(name='John'), 19 | Parent.objects.create(name='Jane') 20 | ] 21 | 22 | children = [ 23 | Child.objects.create(parent=parents[0], name='Joe', timestamp='2017-06-01', other_timestamp=None), 24 | Child.objects.create(parent=parents[0], name='Jan', timestamp='2017-07-01', other_timestamp=None), 25 | Child.objects.create(parent=parents[0], name='Jan', timestamp='2017-05-01', other_timestamp='2017-08-01') 26 | ] 27 | 28 | def test_subquery_min(self): 29 | annotation = { 30 | 'oldest_child_timestamp': SubqueryMin('da_child__timestamp', 31 | output_field=DateTimeField()) 32 | } 33 | 34 | parents = Parent.objects.filter(name='John').annotate(**annotation) 35 | 36 | oldest_child = Child.objects.filter(parent__name='John').order_by('timestamp')[0] 37 | 38 | self.assertEqual(parents[0].oldest_child_timestamp, oldest_child.timestamp) 39 | 40 | def test_subquery_max(self): 41 | annotation = { 42 | 'youngest_child_timestamp': SubqueryMax('da_child__timestamp', 43 | output_field=DateTimeField()) 44 | } 45 | 46 | parents = Parent.objects.filter(name='John').annotate(**annotation) 47 | 48 | youngest_child = Child.objects.filter(parent__name='John').order_by('-timestamp')[0] 49 | 50 | self.assertEqual(parents[0].youngest_child_timestamp, youngest_child.timestamp) 51 | 52 | def test_subquery_count(self): 53 | annotation = { 54 | 'child_count': SubqueryCount('da_child') 55 | } 56 | 57 | parents = Parent.objects.annotate(**annotation) 58 | 59 | counts = {parent.name: parent.child_count for parent in parents} 60 | 61 | self.assertEqual(counts, {'John': 3, 'Jane': 0}) 62 | 63 | def test_subquery_count_filtered(self): 64 | annotation = { 65 | 'child_count': SubqueryCount('da_child', filter=Q(name='Jan')) 66 | } 67 | 68 | parents = Parent.objects.annotate(**annotation) 69 | 70 | counts = {parent.name: parent.child_count for parent in parents} 71 | 72 | self.assertEqual(counts, {'John': 2, 'Jane': 0}) 73 | 74 | def test_function(self): 75 | if settings.BACKEND == 'mysql': 76 | # Explicit cast for MySQL with Coalesce and Datetime 77 | # https://docs.djangoproject.com/en/2.1/ref/models/database-functions/#coalesce 78 | annotation = { 79 | 'oldest_child_with_other': Cast(SubqueryMin(Coalesce('da_child__other_timestamp', 'da_child__timestamp'), 80 | output_field=DateTimeField()), DateTimeField()) 81 | } 82 | else: 83 | annotation = { 84 | 'oldest_child_with_other': SubqueryMin(Coalesce('da_child__other_timestamp', 'da_child__timestamp'), 85 | output_field=DateTimeField()) 86 | } 87 | 88 | parents = Parent.objects.filter(name='John').annotate(**annotation) 89 | 90 | oldest_child = Child.objects.filter(parent__name='John').order_by(Coalesce('other_timestamp', 'timestamp').asc())[0] 91 | 92 | self.assertEqual(parents[0].oldest_child_with_other, oldest_child.other_timestamp or oldest_child.timestamp) 93 | 94 | 95 | class TestManyToMany(TestCase): 96 | 97 | @classmethod 98 | def setUpClass(cls): 99 | super(TestManyToMany, cls).setUpClass() 100 | publishers = [ 101 | Publisher.objects.create(name='Publisher 1', number=1), 102 | Publisher.objects.create(name='Publisher 2', number=2) 103 | ] 104 | 105 | authors = [ 106 | Author.objects.create(name='Author 1'), 107 | Author.objects.create(name='Author 2'), 108 | Author.objects.create(name='Author 3'), 109 | Author.objects.create(name='Author 4'), 110 | Author.objects.create(name='Author 5'), 111 | Author.objects.create(name='Author 6') 112 | ] 113 | 114 | books = [ 115 | Book.objects.create(title='Book 1', publisher=publishers[0]), 116 | Book.objects.create(title='Book 2', publisher=publishers[0]), 117 | Book.objects.create(title='Book 3', publisher=publishers[1]), 118 | Book.objects.create(title='Book 4', publisher=publishers[1]) 119 | ] 120 | 121 | book_authors = [ 122 | BookAuthor.objects.create(author=authors[0], book=books[0], id=1), 123 | BookAuthor.objects.create(author=authors[1], book=books[1], id=2), 124 | BookAuthor.objects.create(author=authors[2], book=books[1], id=3), 125 | BookAuthor.objects.create(author=authors[2], book=books[2], id=4), 126 | BookAuthor.objects.create(author=authors[3], book=books[2], id=5), 127 | BookAuthor.objects.create(author=authors[4], book=books[3], id=6), 128 | ] 129 | 130 | book_editors = [ 131 | BookEditor.objects.create(editor=authors[5], book=books[3]), 132 | BookEditor.objects.create(editor=authors[5], book=books[3]), 133 | ] 134 | 135 | def test_subquery_count_forward(self): 136 | annotation = { 137 | 'author_count': SubqueryCount('authors') 138 | } 139 | books = Book.objects.annotate(**annotation).order_by('id') 140 | 141 | counts = {book.title: book.author_count for book in books} 142 | self.assertEqual(counts, {'Book 1': 1, 'Book 2': 2, 'Book 3': 2, 'Book 4': 1}) 143 | 144 | def test_subquery_count_reverse(self): 145 | annotation = { 146 | 'book_count': SubqueryCount('authored_books') 147 | } 148 | authors = Author.objects.annotate(**annotation).order_by('id') 149 | 150 | counts = {author.name: author.book_count for author in authors} 151 | self.assertEqual(counts, {'Author 1': 1, 152 | 'Author 2': 1, 153 | 'Author 3': 2, 154 | 'Author 4': 1, 155 | 'Author 5': 1, 156 | 'Author 6': 0}) 157 | 158 | def test_subquery_count_reverse_explicit(self): 159 | # The two queries are the same, one just passes a long version of joining from author to books, 160 | # this test verifies that the automatic reverse of the joins handles both cases. 161 | # The annotation is a bit non-sensical, taking the Max over titles, but that isn't the point 162 | annotation = { 163 | 'max_book_title': SubqueryMax('bookauthor__book__title') 164 | } 165 | authors = Author.objects.annotate(**annotation).order_by('id') 166 | 167 | titles = {author.name: author.max_book_title for author in authors} 168 | self.assertEqual(titles, {'Author 1': 'Book 1', 169 | 'Author 2': 'Book 2', 170 | 'Author 3': 'Book 3', 171 | 'Author 4': 'Book 3', 172 | 'Author 5': 'Book 4', 173 | 'Author 6': None}) 174 | 175 | annotation = { 176 | 'max_book_title': SubqueryMax('authored_books__title') 177 | } 178 | authors = Author.objects.annotate(**annotation).order_by('id') 179 | 180 | titles = {author.name: author.max_book_title for author in authors} 181 | self.assertEqual(titles, {'Author 1': 'Book 1', 182 | 'Author 2': 'Book 2', 183 | 'Author 3': 'Book 3', 184 | 'Author 4': 'Book 3', 185 | 'Author 5': 'Book 4', 186 | 'Author 6': None}) 187 | 188 | def test_subquery_min_through_m2m_and_foreign_key(self): 189 | 190 | annotation = { 191 | 'max_publisher_number': SubqueryMax('authored_books__publisher__number') 192 | } 193 | authors = Author.objects.annotate(**annotation) 194 | 195 | numbers = {author.name: author.max_publisher_number for author in authors} 196 | self.assertEqual(numbers, {'Author 1': 1, 197 | 'Author 2': 1, 198 | 'Author 3': 2, 199 | 'Author 4': 2, 200 | 'Author 5': 2, 201 | 'Author 6': None}) 202 | 203 | def test_self_join(self): 204 | annotation = { 205 | 'book_author_count': SubqueryCount('book__bookauthor') 206 | } 207 | 208 | book_authors = BookAuthor.objects.annotate(**annotation) 209 | 210 | counts = {ba.id: ba.book_author_count for ba in book_authors} 211 | 212 | self.assertEqual(counts, {1: 1, 213 | 2: 2, 214 | 3: 2, 215 | 4: 2, 216 | 5: 2, 217 | 6: 1}) 218 | 219 | 220 | class TestForeignKey(TestCase): 221 | @classmethod 222 | def setUpClass(cls): 223 | super(TestForeignKey, cls).setUpClass() 224 | publishers = [ 225 | Publisher.objects.create(name='Publisher 1', number=1), 226 | Publisher.objects.create(name='Publisher 2', number=2) 227 | ] 228 | 229 | authors = [ 230 | Author.objects.create(name='Author 1'), 231 | Author.objects.create(name='Author 2'), 232 | Author.objects.create(name='Author 3'), 233 | Author.objects.create(name='Author 4'), 234 | Author.objects.create(name='Author 5'), 235 | Author.objects.create(name='Author 6') 236 | ] 237 | 238 | books = [ 239 | Book.objects.create(title='Book 1', publisher=publishers[0]), 240 | Book.objects.create(title='Book 2', publisher=publishers[0]), 241 | Book.objects.create(title='Book 3', publisher=publishers[1]), 242 | Book.objects.create(title='Book 4', publisher=publishers[1]) 243 | ] 244 | 245 | book_authors = [ 246 | BookAuthor.objects.create(author=authors[0], book=books[0], id=1), 247 | BookAuthor.objects.create(author=authors[1], book=books[1], id=2), 248 | BookAuthor.objects.create(author=authors[2], book=books[1], id=3), 249 | BookAuthor.objects.create(author=authors[2], book=books[2], id=4), 250 | BookAuthor.objects.create(author=authors[3], book=books[2], id=5), 251 | BookAuthor.objects.create(author=authors[4], book=books[3], id=6), 252 | ] 253 | 254 | def test_aggregate_foreign_key(self): 255 | bookauthors = BookAuthor.objects.annotate(min_publisher_id=SubqueryMin('book__publisher_id')) 256 | 257 | bookauthors = {bookauthor.id: bookauthor.min_publisher_id for bookauthor in bookauthors} 258 | 259 | publisher1_id = Publisher.objects.get(name='Publisher 1').id 260 | publisher2_id = Publisher.objects.get(name='Publisher 2').id 261 | 262 | self.assertEqual(bookauthors, {1: publisher1_id, 263 | 2: publisher1_id, 264 | 3: publisher1_id, 265 | 4: publisher2_id, 266 | 5: publisher2_id, 267 | 6: publisher2_id}) 268 | 269 | 270 | class TestReverseForeignKey(TestCase): 271 | 272 | @classmethod 273 | def setUpClass(cls): 274 | super(TestReverseForeignKey, cls).setUpClass() 275 | catalogs = [ 276 | Catalog.objects.create(number='A'), 277 | Catalog.objects.create(number='B') 278 | ] 279 | 280 | infos = [ 281 | CatalogInfo.objects.create(catalog=catalogs[0], info='cat A info', id=3), 282 | CatalogInfo.objects.create(catalog=catalogs[1], info='cat B info', id=4) 283 | ] 284 | 285 | packages = [ 286 | Package.objects.create(name='Box', quantity=10, catalog=catalogs[0]), 287 | Package.objects.create(name='Case', quantity=24, catalog=catalogs[1]), 288 | ] 289 | 290 | purchases = [ 291 | Purchase.objects.create(price=5, pack=packages[0]), 292 | Purchase.objects.create(price=6, pack=packages[0]), 293 | Purchase.objects.create(price=4, pack=packages[0]), 294 | Purchase.objects.create(price=11, pack=packages[1]), 295 | Purchase.objects.create(price=12, pack=packages[1]), 296 | ] 297 | 298 | def test_reverse_foreign_key(self): 299 | annotations = { 300 | 'max_price': SubqueryMax('package__purchase__price'), 301 | 'min_price': SubqueryMin('package__purchase__price') 302 | } 303 | catalogs = Catalog.objects.annotate(**annotations) 304 | 305 | prices = {catalog.number: (catalog.max_price, catalog.min_price) for catalog in catalogs} 306 | 307 | self.assertEqual(prices, {'A': (6, 4), 308 | 'B': (12, 11)}) 309 | 310 | def test_forward_and_reverse_foreign_keys(self): 311 | annotations = { 312 | 'max_price': SubqueryMax('catalog__package__purchase__price'), 313 | 'min_price': SubqueryMin('catalog__package__purchase__price') 314 | } 315 | 316 | catalog_infos = CatalogInfo.objects.annotate(**annotations) 317 | 318 | extremes = {info.info: (info.max_price, info.min_price) for info in catalog_infos} 319 | 320 | self.assertEqual(extremes, {'cat A info': (6, 4), 321 | 'cat B info': (12, 11)}) 322 | 323 | 324 | class TestUpdate(TestCase): 325 | @classmethod 326 | def setUpClass(cls): 327 | super().setUpClass() 328 | 329 | store = Store.objects.create(name='A Store') 330 | 331 | sellers = [ 332 | Seller.objects.create(store=store, name='Seller 1'), 333 | Seller.objects.create(store=store, name='Seller 2'), 334 | ] 335 | 336 | sales = [ 337 | Sale.objects.create(seller=sellers[0], date='2020-01-01', revenue=1.1, expenses=0.2), 338 | Sale.objects.create(seller=sellers[0], date='2020-01-03', revenue=2.3, expenses=0.3), 339 | Sale.objects.create(seller=sellers[0], date='2020-01-06', revenue=1.7, expenses=0.4), 340 | Sale.objects.create(seller=sellers[0], date='2020-01-08', revenue=5.4, expenses=0.1), 341 | Sale.objects.create(seller=sellers[1], date='2020-01-08', revenue=1.4, expenses=0.6), 342 | Sale.objects.create(seller=sellers[1], date='2020-01-08', revenue=2.4, expenses=0.5), 343 | ] 344 | 345 | def test_update_count(self): 346 | Seller.objects.update(total_sales=SubqueryCount('sale')) 347 | 348 | sellers = list(Seller.objects.values('name', 'total_sales').order_by('name')) 349 | 350 | self.assertEqual(sellers, 351 | [{'name': 'Seller 1', 'total_sales': 4}, 352 | {'name': 'Seller 2', 'total_sales': 2} 353 | ]) 354 | 355 | def test_update_avg(self): 356 | Seller.objects.update(average_revenue=Coalesce(SubqueryAvg('sale__revenue'), 0.0)) 357 | 358 | sellers = list(Seller.objects.values('name', 'average_revenue').order_by('name')) 359 | 360 | self.assertEqual(sellers, 361 | [{'name': 'Seller 1', 'average_revenue': 2.625}, 362 | {'name': 'Seller 2', 'average_revenue': 1.9} 363 | ]) 364 | 365 | class TestForeignKeyToField(TestCase): 366 | 367 | @classmethod 368 | def setUpClass(cls): 369 | super(TestForeignKeyToField, cls).setUpClass() 370 | brand = Brand.objects.create(name='Python', company_id=1337) 371 | products = [ 372 | Product.objects.create(brand=brand, num_purchases=1), 373 | Product.objects.create(brand=brand, num_purchases=3) 374 | ] 375 | 376 | def test_foreign_key_to_field(self): 377 | brands = Brand.objects.annotate( 378 | purchase_sum=SubquerySum('products__num_purchases') 379 | ) 380 | self.assertEqual(brands.first().purchase_sum, 4) 381 | 382 | 383 | class TestMultipleForeignKeyToTheSameModel(TestCase): 384 | 385 | @classmethod 386 | def setUpClass(cls): 387 | super(TestMultipleForeignKeyToTheSameModel, cls).setUpClass() 388 | 389 | player1 = Player.objects.create(nickname='Player 1') 390 | player2 = Player.objects.create(nickname='Player 2') 391 | player3 = Player.objects.create(nickname='Player 3') 392 | player4 = Player.objects.create(nickname='Player 4') 393 | player5 = Player.objects.create(nickname='Player 5') 394 | player6 = Player.objects.create(nickname='Player 6') 395 | 396 | team1 = Team.objects.create(name='Team 1') 397 | team2 = Team.objects.create(name='Team 2') 398 | team3 = Team.objects.create(name='Team 3') 399 | 400 | team1.players.add(player1, player2, player3) 401 | team2.players.add(player4, player5) 402 | team3.players.add(player6) 403 | 404 | game1 = Game.objects.create(team1=team1, team2=team2, played='2021-02-10') 405 | game2 = Game.objects.create(team1=team1, team2=team3, played='2021-02-13') 406 | game3 = Game.objects.create(team1=team1, team2=team2, played='2021-02-16') 407 | game4 = Game.objects.create(team1=team2, team2=team3, played='2021-02-19') 408 | game5 = Game.objects.create(team1=team2, team2=team3, played='2021-02-22') 409 | 410 | def test_player_count(self): 411 | team1_count_subquery_count = SubqueryCount('team1__players') 412 | team2_count_subquery_count = SubqueryCount('team2__players') 413 | 414 | games = Game.objects.annotate(team1_count=team1_count_subquery_count, 415 | team2_count=team2_count_subquery_count) 416 | 417 | for g in games: 418 | self.assertEqual(g.team1_count, g.team1.players.count()) 419 | self.assertEqual(g.team2_count, g.team2.players.count()) 420 | -------------------------------------------------------------------------------- /sql_util/utils.py: -------------------------------------------------------------------------------- 1 | from sql_util.aggregates import SubqueryAggregate, SubqueryCount, SubqueryAvg, SubqueryMax, SubqueryMin, SubquerySum, Exists --------------------------------------------------------------------------------