├── .flake8 ├── .github └── workflows │ ├── linting.yml │ ├── pypi.yml │ └── tox.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .vscode └── settings.json ├── LICENSE ├── README.md ├── docker-compose.yml ├── postgres_composite_types ├── __init__.py ├── caster.py ├── composite_type.py ├── fields.py ├── forms.py ├── operations.py ├── quoting.py ├── signals.py └── templates │ └── postgres_composite_types │ └── forms │ └── widgets │ └── composite_type.html ├── pyproject.toml ├── tests ├── __init__.py ├── fields.py ├── migrations │ ├── 0001_initial.py │ ├── 0002_models.py │ └── __init__.py ├── models.py ├── settings.py ├── test_field.py ├── test_forms.py └── test_more.py └── tox.ini /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = .git,__pycache__,migrations 3 | max-complexity = 10 4 | max-line-length = 88 5 | -------------------------------------------------------------------------------- /.github/workflows/linting.yml: -------------------------------------------------------------------------------- 1 | name: Linting 2 | 3 | on: 4 | - push 5 | - pull_request 6 | 7 | jobs: 8 | linting: 9 | runs-on: ubuntu-latest 10 | 11 | steps: 12 | - uses: actions/checkout@v3 13 | - uses: actions/setup-python@v4 14 | with: 15 | python-version: "3.10" 16 | 17 | - name: Install poetry 18 | run: curl -sSL https://install.python-poetry.org | python3 - 19 | 20 | - name: Install package 21 | run: poetry install --with dev 22 | 23 | - name: Run pre-commit 24 | run: poetry run pre-commit run --all-files --show-diff-on-failure 25 | -------------------------------------------------------------------------------- /.github/workflows/pypi.yml: -------------------------------------------------------------------------------- 1 | name: pypi 2 | 3 | on: 4 | release: 5 | types: 6 | - created 7 | 8 | jobs: 9 | release: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - uses: actions/checkout@v3 14 | - uses: actions/setup-python@v4 15 | with: 16 | python-version: "3.10" 17 | 18 | - name: Install poetry 19 | run: curl -sSL https://install.python-poetry.org | python3 - 20 | 21 | # - name: Verify tag matches what's in pyproject.toml 22 | # run: test "v$(poetry version --short)" = "$(git describe)" 23 | 24 | - name: Publish package 25 | run: poetry publish --build -n 26 | env: 27 | POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/tox.yml: -------------------------------------------------------------------------------- 1 | name: Tox 2 | 3 | on: 4 | - push 5 | - pull_request 6 | 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | strategy: 11 | matrix: 12 | python-version: ['3.9', '3.10', '3.11'] 13 | 14 | services: 15 | postgres: 16 | image: postgres 17 | 18 | env: 19 | POSTGRES_PASSWORD: postgres 20 | 21 | options: >- 22 | --health-cmd pg_isready 23 | --health-interval 10s 24 | --health-timeout 5s 25 | --health-retries 5 26 | 27 | ports: 28 | - 5432:5432 29 | 30 | steps: 31 | - uses: actions/checkout@v3 32 | - uses: actions/setup-python@v4 33 | with: 34 | python-version: ${{ matrix.python-version }} 35 | 36 | - name: Install poetry 37 | run: curl -sSL https://install.python-poetry.org | python3 - 38 | 39 | - name: Install Tox 40 | run: python -m pip install tox tox-gh-actions 41 | 42 | - name: Test with Tox 43 | run: tox -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /poetry.lock 2 | /dist/ 3 | __pycache__ 4 | /.tox 5 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: ".git|.tox|.pytest_cache" 2 | default_stages: [commit] 3 | fail_fast: true 4 | 5 | repos: 6 | - repo: https://github.com/pre-commit/pre-commit-hooks 7 | rev: "v4.3.0" 8 | hooks: 9 | - id: check-builtin-literals 10 | - id: check-case-conflict 11 | - id: check-merge-conflict 12 | - id: check-toml 13 | - id: check-yaml 14 | - id: end-of-file-fixer 15 | - id: fix-byte-order-marker 16 | - id: mixed-line-ending 17 | - id: trailing-whitespace 18 | 19 | - repo: https://github.com/psf/black 20 | rev: "22.8.0" 21 | hooks: 22 | - id: black 23 | 24 | - repo: https://github.com/asottile/pyupgrade 25 | rev: v3.2.2 26 | hooks: 27 | - id: pyupgrade 28 | 29 | - repo: https://github.com/dosisod/refurb 30 | rev: "v1.8.0" 31 | hooks: 32 | - id: refurb 33 | 34 | - repo: https://github.com/adamchainz/django-upgrade 35 | rev: "1.12.0" 36 | hooks: 37 | - id: django-upgrade 38 | args: [--target-version, "3.2"] 39 | 40 | - repo: https://github.com/timothycrosley/isort 41 | rev: "5.10.1" 42 | hooks: 43 | - id: isort 44 | 45 | - repo: https://github.com/pycqa/flake8 46 | rev: "5.0.4" 47 | hooks: 48 | - id: flake8 49 | 50 | - repo: local 51 | hooks: 52 | - id: pylint 53 | name: pylint 54 | entry: env DJANGO_SETTINGS_MODULE="tests.settings" pylint 55 | language: system 56 | types: [python] 57 | args: ["--ignore-paths=tests/"] 58 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.formatting.provider": "black", 3 | "python.linting.flake8Enabled": true, 4 | "python.testing.pytestEnabled": true 5 | } 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | (c) 2016, Danielle Madeley 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of the copyright holder nor the names of its contributors 15 | may be used to endorse or promote products derived from this software 16 | without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Django Postgres composite types 2 | 3 | An implementation of Postgres' [composite types](http://www.postgresql.org/docs/current/static/rowtypes.html) 4 | for [Django](https://docs.djangoproject.com/en/1.9/). 5 | 6 | ## Usage 7 | 8 | Install with: 9 | 10 | pip install django-postgres-composite-types 11 | 12 | Then add 'postgres_composite_types' to your `INSTALLED_APPS`: 13 | 14 | INSTALLED_APPS = [ 15 | # ... Other apps 16 | 'postgres_composite_types', 17 | ] 18 | 19 | Define a type and add it to a model: 20 | 21 | ```python 22 | from django.db import models 23 | from postgres_composite_types import CompositeType 24 | 25 | class Address(CompositeType): 26 | """An address.""" 27 | 28 | address_1 = models.CharField(max_length=255) 29 | address_2 = models.CharField(max_length=255) 30 | 31 | suburb = models.CharField(max_length=50) 32 | state = models.CharField(max_length=50) 33 | 34 | postcode = models.CharField(max_length=10) 35 | country = models.CharField(max_length=50) 36 | 37 | class Meta: 38 | db_type = 'x_address' # Required 39 | 40 | 41 | class Person(models.Model): 42 | """A person.""" 43 | 44 | address = Address.Field() 45 | ``` 46 | 47 | An operation needs to be prepended to your migration: 48 | 49 | ```python 50 | import address 51 | from django.db import migrations 52 | 53 | 54 | class Migration(migrations.Migration): 55 | 56 | operations = [ 57 | # Registers the type 58 | address.Address.Operation(), 59 | migrations.AddField( 60 | model_name='person', 61 | name='address', 62 | field=address.Address.Field(blank=True, null=True), 63 | ), 64 | ] 65 | ``` 66 | 67 | ## Examples 68 | 69 | Array fields: 70 | 71 | ```python 72 | class Card(CompositeType): 73 | """A playing card.""" 74 | 75 | suit = models.CharField(max_length=1) 76 | rank = models.CharField(max_length=2) 77 | 78 | class Meta: 79 | db_type = 'card' 80 | 81 | 82 | class Hand(models.Model): 83 | """A hand of cards.""" 84 | cards = ArrayField(base_field=Card.Field()) 85 | ``` 86 | 87 | Nested types: 88 | 89 | ```python 90 | class Point(CompositeType): 91 | """A point on the cartesian plane.""" 92 | 93 | x = models.IntegerField() 94 | y = models.IntegerField() 95 | 96 | class Meta: 97 | db_type = 'x_point' # Postgres already has a point type 98 | 99 | 100 | class Box(CompositeType): 101 | """An axis-aligned box on the cartesian plane.""" 102 | class Meta: 103 | db_type = 'x_box' # Postgres already has a box type 104 | 105 | top_left = Point.Field() 106 | bottom_right = Point.Field() 107 | ``` 108 | 109 | ## Gotchas and Caveats 110 | 111 | The migration operation currently loads the _current_ state of the type, not 112 | the state when the migration was written. A generic `CreateType` operation 113 | which takes the fields of the type would be possible, but it would still 114 | require manual handling still as Django's `makemigrations` is not currently 115 | extensible. 116 | 117 | Changes to types are possible using `RawSQL`, for example: 118 | 119 | ```python 120 | operations = [ 121 | migrations.RunSQL([ 122 | "ALTER TYPE x_address DROP ATTRIBUTE country", 123 | "ALTER TYPE x_address ADD ATTRIBUTE country integer", 124 | ], [ 125 | "ALTER TYPE x_address DROP ATTRIBUTE country", 126 | "ALTER TYPE x_address ADD ATTRIBUTE country varchar(50)", 127 | ]), 128 | ] 129 | ``` 130 | 131 | However, be aware that if your earlier operations were run using current DB 132 | code, you will already have the right types 133 | ([bug #8](https://github.com/danni/django-postgres-composite-types/issues/8)). 134 | 135 | It is recommended to that you namespace your custom types to avoid conflict 136 | with future PostgreSQL types. 137 | 138 | Lookups and indexes are not implemented yet 139 | ([bug #9](https://github.com/danni/django-postgres-composite-types/issues/9), 140 | [bug #10](https://github.com/danni/django-postgres-composite-types/issues/10)). 141 | 142 | ## Running Tests 143 | 144 | Clone the repository, go to it's base directory and run the following commands. 145 | 146 | pip install tox 147 | tox 148 | 149 | Or if you want a specific environment 150 | 151 | tox -e py311-dj41 152 | 153 | ## Authors 154 | 155 | - Danielle Madeley 156 | - Tim Heap 157 | 158 | ## License 159 | 160 | This project is licensed under the BSD license. 161 | See the LICENSE file for the full text of the license. 162 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Docker compose file to help with testing. 2 | # 3 | # Run: 4 | # docker-compose up 5 | # tox 6 | # 7 | # This setup is replicated in the Github Action 8 | 9 | version: "3.4" 10 | 11 | services: 12 | database: 13 | image: postgres 14 | 15 | environment: 16 | POSTGRES_PASSWORD: postgres 17 | 18 | ports: 19 | - "5432:5432" 20 | -------------------------------------------------------------------------------- /postgres_composite_types/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Implementation of Postgres composite types in Django. 3 | 4 | Takes inspiration from: 5 | - django-pgfields 6 | - django-postgres 7 | """ 8 | 9 | from .composite_type import CompositeType 10 | 11 | __all__ = ["CompositeType"] 12 | -------------------------------------------------------------------------------- /postgres_composite_types/caster.py: -------------------------------------------------------------------------------- 1 | from psycopg2.extras import CompositeCaster 2 | 3 | __all__ = ["BaseCaster"] 4 | 5 | 6 | class BaseCaster(CompositeCaster): 7 | """ 8 | Base caster to transform a tuple of values from postgres to a model 9 | instance. 10 | """ 11 | 12 | Meta = None 13 | 14 | def make(self, values): 15 | return self.Meta.model(*values) 16 | -------------------------------------------------------------------------------- /postgres_composite_types/composite_type.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import logging 3 | import sys 4 | 5 | from django.db import models 6 | from django.db.backends.postgresql.base import ( 7 | DatabaseWrapper as PostgresDatabaseWrapper, 8 | ) 9 | from django.db.backends.signals import connection_created 10 | from psycopg2 import ProgrammingError 11 | from psycopg2.extensions import ISQLQuote, register_adapter 12 | from psycopg2.extras import CompositeCaster, register_composite 13 | 14 | from .caster import BaseCaster 15 | from .fields import BaseField 16 | from .operations import BaseOperation 17 | from .quoting import QuotedCompositeType 18 | 19 | LOGGER = logging.getLogger(__name__) 20 | 21 | __all__ = ["CompositeType"] 22 | 23 | 24 | def _add_class_to_module(cls, module_name): 25 | cls.__module__ = module_name 26 | module = sys.modules[module_name] 27 | setattr(module, cls.__name__, cls) 28 | 29 | 30 | class CompositeTypeMeta(type): 31 | """Metaclass for Type.""" 32 | 33 | @classmethod 34 | def __prepare__(cls, name, bases): 35 | """ 36 | Guarantee the ordering of the declared attrs. 37 | 38 | We need this so that our type doesn't change ordering between 39 | invocations. 40 | """ 41 | return {} 42 | 43 | def __new__(cls, name, bases, attrs): 44 | # Only apply the metaclass to our subclasses 45 | if name == "CompositeType": 46 | return super().__new__(cls, name, bases, attrs) 47 | 48 | # retrieve any fields from our declaration 49 | fields = [] 50 | for field_name, value in attrs.copy().items(): 51 | if isinstance(value, models.fields.related.RelatedField): 52 | raise TypeError("Composite types cannot contain " "related fields") 53 | 54 | if isinstance(value, models.Field): 55 | field = attrs.pop(field_name) 56 | field.set_attributes_from_name(field_name) 57 | fields.append((field_name, field)) 58 | 59 | # retrieve the Meta from our declaration 60 | try: 61 | meta_obj = attrs.pop("Meta") 62 | except KeyError as exc: 63 | raise TypeError(f'{name} has no "Meta" class') from exc 64 | 65 | try: 66 | meta_obj.db_type 67 | except AttributeError as exc: 68 | raise TypeError(f"{name}.Meta.db_type is required.") from exc 69 | 70 | meta_obj.fields = fields 71 | 72 | # create the field for this Type 73 | attrs["Field"] = type(f"{name}Field", (BaseField,), {"Meta": meta_obj}) 74 | 75 | # add field class to the module in which the composite type class lives 76 | # this is required for migrations to work 77 | _add_class_to_module(attrs["Field"], attrs["__module__"]) 78 | 79 | # create the database operation for this type 80 | attrs["Operation"] = type( 81 | f"Create{name}Type", (BaseOperation,), {"Meta": meta_obj} 82 | ) 83 | 84 | # create the caster for this type 85 | attrs["Caster"] = type(f"{name}Caster", (BaseCaster,), {"Meta": meta_obj}) 86 | 87 | new_cls = super().__new__(cls, name, bases, attrs) 88 | new_cls._meta = meta_obj 89 | 90 | for _, field in new_cls._meta.fields: 91 | new_cls.Field.register_lookup( 92 | type( 93 | f"{name}{field.attname}Lookup", 94 | (models.Transform,), 95 | { 96 | "lookup_name": field.attname, 97 | "arity": 1, 98 | "template": f'(%(expressions)s)."{field.column}"', 99 | }, 100 | ) 101 | ) 102 | 103 | meta_obj.model = new_cls 104 | 105 | return new_cls 106 | 107 | def __init__(cls, name, bases, attrs): 108 | super().__init__(name, bases, attrs) 109 | if name == "CompositeType": 110 | return 111 | 112 | cls._capture_descriptors() # pylint:disable=no-value-for-parameter 113 | 114 | # Register the type on the first database connection 115 | connection_created.connect( 116 | receiver=cls.database_connected, dispatch_uid=cls._meta.db_type 117 | ) 118 | 119 | def _capture_descriptors(cls): 120 | """Work around for not being able to call contribute_to_class. 121 | 122 | Too much code to fake in our meta objects etc to be able to call 123 | contribute_to_class directly, but we still want fields to be able 124 | to set custom type descriptors. So we fake a model instead, with the 125 | same fields as the composite type, and extract any custom descriptors 126 | on that. 127 | """ 128 | 129 | attrs = dict(cls._meta.fields) 130 | 131 | # we need to build a unique app label and model name combination for 132 | # every composite type so django doesn't complain about model reloads 133 | class Meta: 134 | app_label = cls.__module__ 135 | 136 | attrs["__module__"] = cls.__module__ 137 | attrs["Meta"] = Meta 138 | model_name = f"_Fake{cls.__name__}Model" 139 | 140 | fake_model = type(model_name, (models.Model,), attrs) 141 | for field_name, _ in cls._meta.fields: 142 | attr = getattr(fake_model, field_name) 143 | if inspect.isdatadescriptor(attr): 144 | setattr(cls, field_name, attr) 145 | 146 | def database_connected(cls, signal, sender, connection, **kwargs): 147 | """ 148 | Register this type with the database the first time a connection is 149 | made. 150 | """ 151 | if isinstance(connection, PostgresDatabaseWrapper): 152 | # Try to register the type. If the type has not been created in a 153 | # migration, the registration will fail. The type will be 154 | # registered as part of the migration, so hopefully the migration 155 | # will run soon. 156 | try: 157 | cls.register_composite(connection) 158 | except ProgrammingError as exc: 159 | LOGGER.warning( 160 | "Failed to register composite %r. " 161 | "The migration to register it may not have run yet. " 162 | "Error details: %s", 163 | cls.__name__, 164 | exc, 165 | ) 166 | 167 | # Disconnect the signal now - only need to register types on the 168 | # initial connection 169 | connection_created.disconnect( 170 | cls.database_connected, dispatch_uid=cls._meta.db_type 171 | ) 172 | 173 | 174 | class CompositeType(metaclass=CompositeTypeMeta): 175 | """ 176 | A new composite type stored in Postgres. 177 | """ 178 | 179 | _meta = None 180 | 181 | # The database connection this type is registered with 182 | registered_connection = None 183 | 184 | def __init__(self, *args, **kwargs): 185 | if args and kwargs: 186 | raise RuntimeError("Specify either args or kwargs but not both.") 187 | 188 | # Initialise blank values for anyone expecting them 189 | for name, _ in self._meta.fields: 190 | setattr(self, name, None) 191 | 192 | # Unpack any args as if they came from the type 193 | for (name, _), arg in zip(self._meta.fields, args): 194 | setattr(self, name, arg) 195 | 196 | for name, value in kwargs.items(): 197 | setattr(self, name, value) 198 | 199 | def __repr__(self): 200 | args = ", ".join(f"{k}={v}" for k, v in self.__to_dict__().items()) 201 | return f"<{type(self).__name__}({args})>" 202 | 203 | def __to_tuple__(self): 204 | return tuple( 205 | field.get_prep_value(getattr(self, name)) 206 | for name, field in self._meta.fields 207 | ) 208 | 209 | def __to_dict__(self): 210 | return { 211 | name: field.get_prep_value(getattr(self, name)) 212 | for name, field in self._meta.fields 213 | } 214 | 215 | def __eq__(self, other): 216 | if not isinstance(other, CompositeType): 217 | return False 218 | if self._meta.model != other._meta.model: 219 | return False 220 | for name, _ in self._meta.fields: 221 | if getattr(self, name) != getattr(other, name): 222 | return False 223 | return True 224 | 225 | @classmethod 226 | def register_composite(cls, connection): 227 | """ 228 | Register this CompositeType with Postgres. 229 | 230 | If the CompositeType does not yet exist in the database, this will 231 | fail. Hopefully a migration will come along shortly and create the 232 | type in the database. If `retry` is True, this CompositeType will try 233 | to register itself again after the type is created. 234 | """ 235 | 236 | LOGGER.debug( 237 | "Registering composite type %s on connection %s", cls.__name__, connection 238 | ) 239 | cls.registered_connection = connection 240 | 241 | with connection.temporary_connection() as cur: 242 | # This is what to do when the type is coming out of the database 243 | register_composite( 244 | cls._meta.db_type, cur, globally=True, factory=cls.Caster 245 | ) 246 | # This is what to do when the type is going in to the database 247 | register_adapter(cls, QuotedCompositeType) 248 | 249 | def __conform__(self, protocol): 250 | """ 251 | CompositeTypes know how to conform to the ISQLQuote protocol, by 252 | wrapping themselves in a QuotedCompositeType. The ISQLQuote protocol 253 | is all about formatting custom types for use in SQL statements. 254 | 255 | Returns None if it can not conform to the requested protocol. 256 | """ 257 | if protocol is ISQLQuote: 258 | return QuotedCompositeType(self) 259 | 260 | return None 261 | 262 | class Field(BaseField): 263 | """ 264 | Placeholder for the field that will be produced for this type. 265 | """ 266 | 267 | class Operation(BaseOperation): 268 | """ 269 | Placeholder for the DB operation that will be produced for this type. 270 | """ 271 | 272 | class Caster(CompositeCaster): 273 | """ 274 | Placeholder for the caster that will be produced for this type 275 | """ 276 | -------------------------------------------------------------------------------- /postgres_composite_types/fields.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django.core.exceptions import ValidationError 4 | from django.db.backends.postgresql.base import ( 5 | DatabaseWrapper as PostgresDatabaseWrapper, 6 | ) 7 | from django.db.models import Field 8 | 9 | __all__ = ["BaseField"] 10 | 11 | 12 | class BaseField(Field): 13 | """Base class for the field that relates to this type.""" 14 | 15 | Meta = None 16 | 17 | default_error_messages = { 18 | "bad_json": "to_python() received a string that was not valid JSON", 19 | } 20 | 21 | def db_type(self, connection): 22 | if not isinstance(connection, PostgresDatabaseWrapper): 23 | raise RuntimeError("Composite types are only available for postgres") 24 | 25 | return self.Meta.db_type 26 | 27 | def formfield(self, **kwargs): # pylint:disable=arguments-differ 28 | """Form field for address.""" 29 | from .forms import CompositeTypeField 30 | 31 | defaults = { 32 | "form_class": CompositeTypeField, 33 | "model": self.Meta.model, 34 | } 35 | defaults.update(kwargs) 36 | 37 | return super().formfield(**defaults) 38 | 39 | def to_python(self, value): 40 | """ 41 | Convert a value to the correct type for this field. Values from the 42 | database will already be of the correct type, due to the the caster 43 | registered with psycopg2. The field can also be serialized as a string 44 | via value_to_string, where it is encoded as a JSON object. 45 | """ 46 | # Composite types are serialized as JSON blobs. If BaseField.to_python 47 | # is called with a string, assume it was produced by value_to_string 48 | # and decode it 49 | if isinstance(value, str): 50 | try: 51 | value = json.loads(value) 52 | except ValueError as exc: 53 | raise ValidationError( 54 | self.error_messages["bad_json"], 55 | code="bad_json", 56 | ) from exc 57 | 58 | return self.Meta.model( 59 | **{ 60 | name: field.to_python(value.get(name)) 61 | for name, field in self.Meta.fields 62 | } 63 | ) 64 | 65 | return super().to_python(value) 66 | 67 | def value_to_string(self, obj): 68 | """ 69 | Serialize this as a JSON object {name: field.value_to_string(...)} for 70 | each child field. 71 | """ 72 | value = self.value_from_object(obj) 73 | return json.dumps( 74 | {name: field.value_to_string(value) for name, field in self.Meta.fields} 75 | ) 76 | -------------------------------------------------------------------------------- /postgres_composite_types/forms.py: -------------------------------------------------------------------------------- 1 | """ 2 | Form fields for composite types 3 | 4 | Takes inspiration from django.forms.MultiValueField/MultiWidget. 5 | """ 6 | 7 | import copy 8 | import logging 9 | 10 | from django import forms 11 | from django.contrib.postgres.utils import prefix_validation_error 12 | from django.utils.translation import gettext as _ 13 | 14 | from . import CompositeType 15 | 16 | LOGGER = logging.getLogger(__name__) 17 | 18 | 19 | class CompositeBoundField(forms.BoundField): 20 | """ 21 | Allow access to nested BoundFields for fields. Useful for customising the 22 | rendering of a CompositeTypeField: 23 | 24 | 25 | {{ form.address.address_1 }} 26 | {{ form.address.address_2 }} 27 | 28 | {{ form.address.suburb }} 29 | """ 30 | 31 | def __init__(self, *args, **kwargs): 32 | super().__init__(*args, **kwargs) 33 | self._bound_fields_cache = {} 34 | 35 | initial = self.form.initial.get(self.name, self.field.initial) 36 | if isinstance(initial, CompositeType): 37 | initial = initial.__to_dict__() 38 | 39 | if self.form.is_bound: 40 | data = self.form.data 41 | else: 42 | data = None 43 | 44 | self.composite_form = forms.Form( 45 | data=data, initial=initial, prefix=self.form.add_prefix(self.name) 46 | ) 47 | self.composite_form.fields = copy.deepcopy(self.field.fields) 48 | 49 | def __getitem__(self, name): 50 | "Returns a BoundField with the given name." 51 | return self.composite_form[name] 52 | 53 | 54 | class CompositeTypeField(forms.Field): 55 | """ 56 | Takes an ordered dict of fields to produce a composite form field 57 | """ 58 | 59 | default_error_messags = { 60 | "field_invalid": _("%s: "), 61 | } 62 | 63 | def __init__(self, *args, fields=None, model=None, **kwargs): 64 | if fields is None: 65 | fields = {name: field.formfield() for name, field in model._meta.fields} 66 | else: 67 | fields = dict(fields) 68 | 69 | widget = CompositeTypeWidget( 70 | widgets=[(name, field.widget) for name, field in fields.items()] 71 | ) 72 | 73 | super().__init__(*args, widget=widget, **kwargs) 74 | self.fields = fields 75 | self.model = model 76 | 77 | for field, widget in zip(fields.values(), self.widget.widgets.values()): 78 | widget.attrs["placeholder"] = field.label 79 | 80 | def prepare_value(self, value): 81 | """ 82 | Prepare the field data for the CompositeTypeWidget, which expects data 83 | as a dict. 84 | """ 85 | if isinstance(value, CompositeType): 86 | return value.__to_dict__() 87 | 88 | if value is None: 89 | return {} 90 | 91 | return value 92 | 93 | def validate(self, value): 94 | pass 95 | 96 | def clean(self, value): 97 | LOGGER.debug("clean: > %s", value) 98 | 99 | if all( 100 | value.get(name) in field.empty_values for name, field in self.fields.items() 101 | ): 102 | value = None 103 | if self.required: 104 | raise forms.ValidationError( 105 | "This section is required", code="incomplete" 106 | ) 107 | 108 | else: 109 | cleaned_data = {} 110 | errors = [] 111 | 112 | for name, field in self.fields.items(): 113 | try: 114 | cleaned_data[name] = field.clean(value.get(name)) 115 | except forms.ValidationError as error: 116 | prefix = "%(label)s:" 117 | errors.append( 118 | prefix_validation_error( 119 | error, 120 | code="field_invalid", 121 | prefix=prefix, 122 | params={"label": field.label}, 123 | ) 124 | ) 125 | if errors: 126 | raise forms.ValidationError(errors) 127 | value = self.model(**cleaned_data) 128 | 129 | LOGGER.debug("clean: < %s", value) 130 | 131 | return value 132 | 133 | def has_changed(self, initial, data): 134 | return initial != data 135 | 136 | def get_bound_field(self, form, field_name): 137 | """ 138 | Return a CompositeBoundField instance that will be used when accessing 139 | the fields in a template. 140 | """ 141 | return CompositeBoundField(form, self, field_name) 142 | 143 | 144 | class CompositeTypeWidget(forms.Widget): 145 | """ 146 | Takes an ordered dict of widgets to produce a composite form widget. This 147 | widget knows nothing about CompositeTypes, and works only with dicts for 148 | initial and output data. 149 | """ 150 | 151 | template_name = "postgres_composite_types/forms/widgets/composite_type.html" 152 | 153 | def __init__(self, widgets, **kwargs): 154 | self.widgets = { 155 | name: widget() if isinstance(widget, type) else widget 156 | for name, widget in dict(widgets).items() 157 | } 158 | 159 | super().__init__(**kwargs) 160 | 161 | @property 162 | def is_hidden(self): 163 | return all(w.is_hidden for w in self.widgets.values()) 164 | 165 | def get_context(self, name, value, attrs): 166 | context = super().get_context(name, value, attrs) 167 | final_attrs = context["widget"]["attrs"] 168 | id_ = context["widget"]["attrs"].get("id") 169 | 170 | if self.is_localized: 171 | for widget in self.widgets.values(): 172 | widget.is_localized = self.is_localized 173 | 174 | subwidgets = {} 175 | for subname, widget in self.widgets.items(): 176 | widget_attrs = final_attrs.copy() 177 | if id_: 178 | widget_attrs["id"] = f"{id_}-{subname}" 179 | 180 | widget_context = widget.get_context( 181 | f"{name}-{subname}", value.get(subname), widget_attrs 182 | ) 183 | subwidgets[subname] = widget_context["widget"] 184 | 185 | context["widget"]["subwidgets"] = subwidgets 186 | return context 187 | 188 | def value_from_datadict(self, data, files, name): 189 | return { 190 | subname: widget.value_from_datadict(data, files, f"{name}-{subname}") 191 | for subname, widget in self.widgets.items() 192 | } 193 | 194 | def value_omitted_from_data(self, data, files, name): 195 | prefix = f"{name}-" 196 | return not any(key.startswith(prefix) for key in data) 197 | 198 | def id_for_label(self, id_): 199 | """ 200 | Wrapper around the field widget's `id_for_label` method. 201 | Useful, for example, for focusing on this field regardless of whether 202 | it has a single widget or a MultiWidget. 203 | """ 204 | if id_: 205 | name = next(iter(self.widgets.keys())) 206 | return f"{id_}-{name}" 207 | 208 | return id_ 209 | -------------------------------------------------------------------------------- /postgres_composite_types/operations.py: -------------------------------------------------------------------------------- 1 | from django.db.migrations.operations.base import Operation 2 | 3 | from .signals import composite_type_created 4 | 5 | __all__ = ["BaseOperation"] 6 | 7 | 8 | def sql_field_definition(field_name, field, schema_editor): 9 | quoted_name = schema_editor.quote_name(field_name) 10 | db_type = field.db_type(schema_editor.connection) 11 | return f"{quoted_name} {db_type}" 12 | 13 | 14 | def sql_create_type(type_name, fields, schema_editor): 15 | fields_list = ", ".join( 16 | sql_field_definition(field.column, field, schema_editor) for _, field in fields 17 | ) 18 | quoted_name = schema_editor.quote_name(type_name) 19 | return f"CREATE TYPE {quoted_name} AS ({fields_list})" 20 | 21 | 22 | def sql_drop_type(type_name, schema_editor): 23 | quoted_name = schema_editor.quote_name(type_name) 24 | return f"DROP TYPE {quoted_name}" 25 | 26 | 27 | class BaseOperation(Operation): 28 | """Base class for the DB operation that relates to this type.""" 29 | 30 | reversible = True 31 | Meta = None 32 | 33 | def state_forwards(self, app_label, state): 34 | pass 35 | 36 | def describe(self): 37 | return f"Creates type {self.Meta.db_type}" 38 | 39 | def database_forwards(self, app_label, schema_editor, from_state, to_state): 40 | schema_editor.execute( 41 | sql_create_type(self.Meta.db_type, self.Meta.fields, schema_editor) 42 | ) 43 | self.Meta.model.register_composite(schema_editor.connection) 44 | composite_type_created.send( 45 | self.Meta.model, connection=schema_editor.connection 46 | ) 47 | 48 | def database_backwards(self, app_label, schema_editor, from_state, to_state): 49 | schema_editor.execute( 50 | sql_drop_type(self.Meta.db_type, schema_editor=schema_editor) 51 | ) 52 | -------------------------------------------------------------------------------- /postgres_composite_types/quoting.py: -------------------------------------------------------------------------------- 1 | from psycopg2.extensions import ISQLQuote, adapt 2 | 3 | __all__ = ["QuotedCompositeType"] 4 | 5 | 6 | class QuotedCompositeType: 7 | """ 8 | A wrapper for CompositeTypes that knows how to convert itself into a safe 9 | postgres representation. Created from CompositeType.__conform__ 10 | """ 11 | 12 | value = None 13 | prepared = False 14 | 15 | def __init__(self, obj): 16 | self.obj = obj 17 | self.model = obj._meta.model 18 | 19 | self.value = adapt( 20 | tuple( 21 | field.get_db_prep_value( 22 | field.value_from_object(self.obj), self.model.registered_connection 23 | ) 24 | for _, field in self.model._meta.fields 25 | ) 26 | ) 27 | 28 | def __conform__(self, protocol): 29 | """ 30 | QuotedCompositeType conform to the ISQLQuote protocol all by 31 | themselves. This is required for nested composite types. 32 | 33 | Returns None if it can not conform to the requested protocol. 34 | """ 35 | if protocol is ISQLQuote: 36 | return self 37 | 38 | return None 39 | 40 | def prepare(self, connection): 41 | """ 42 | Prepare anything that depends on the database connection, such as 43 | strings with encodings. 44 | """ 45 | self.value.prepare(connection) 46 | self.prepared = True 47 | 48 | def getquoted(self): 49 | """ 50 | Format composite types as the correct Postgres snippet, including 51 | casts, for queries. 52 | 53 | Returns something like ``b"(value1, value2)::type_name"`` 54 | """ 55 | if not self.prepared: 56 | name = type(self).__name__ 57 | raise RuntimeError( 58 | f"{name}.prepare() must be called before {name}.getquoted()" 59 | ) 60 | 61 | db_type = self.model._meta.db_type.encode("ascii") 62 | return self.value.getquoted() + b"::" + db_type 63 | -------------------------------------------------------------------------------- /postgres_composite_types/signals.py: -------------------------------------------------------------------------------- 1 | from django.dispatch import Signal 2 | 3 | __all__ = ["composite_type_created"] 4 | 5 | composite_type_created = Signal() 6 | -------------------------------------------------------------------------------- /postgres_composite_types/templates/postgres_composite_types/forms/widgets/composite_type.html: -------------------------------------------------------------------------------- 1 | {% spaceless %}{% for subname, widget in widget.subwidgets.items %}{% include widget.template_name %}{% endfor %}{% endspaceless %} 2 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "django-postgres-composite-types" 3 | version = "1.0.0-beta.0" 4 | description = "Postgres composite types support for Django" 5 | authors = ["Danielle Madeley "] 6 | license = "BSD" 7 | readme = "README.md" 8 | packages = [{include = "postgres_composite_types"}] 9 | 10 | [tool.poetry.dependencies] 11 | python = "^3.7" 12 | Django = ">=3.2.16" 13 | psycopg2 = ">=2.8.4" 14 | 15 | [tool.poetry.group.dev.dependencies] 16 | flake8 = "^5.0.4" 17 | pylint = "^2.13.9" 18 | pylint-django = ">=2.5.3" 19 | isort = "^5.10.1" 20 | pre-commit = "^2.9.2" 21 | black = "^22.10.0" 22 | pytest = "^7.2.0" 23 | pytest-django = "^4.5.2" 24 | 25 | [build-system] 26 | requires = ["poetry-core"] 27 | build-backend = "poetry.core.masonry.api" 28 | 29 | [tool.pytest.ini_options] 30 | DJANGO_SETTINGS_MODULE = "tests.settings" 31 | 32 | [tool.isort] 33 | profile = "black" 34 | 35 | [tool.pylint.MASTER] 36 | load-plugins = "pylint_django" 37 | 38 | [tool.pylint.REPORTS] 39 | output-format = "colorized" 40 | 41 | [tool.pylint."MESSAGES CONTROL"] 42 | disable = [ 43 | "import-outside-toplevel", 44 | "missing-function-docstring", 45 | "missing-module-docstring", 46 | "too-few-public-methods", 47 | "unused-argument", 48 | "R0801", 49 | ] 50 | 51 | [tool.pylint.BASIC] 52 | function-rgx = "[a-z_][a-z0-9_]{2,50}$|test_[a-zA-Z_][a-zA-Z0-9_]{2,100}$|setUp$|tearDown$" 53 | method-rgx = "[a-z_][a-z0-9_]{2,30}$|test_[a-zA-Z_][a-zA-Z0-9_]{2,100}$" 54 | attr-rgx = "[a-z_][a-z0-9_]{2,30}$|maxDiff$" 55 | exclude-protected = "_asdict,_fields,_replace,_source,_make,_meta" 56 | no-docstring-rgx = "^Meta$|^_" 57 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danni/django-postgres-composite-types/e5a48c9ade0fccd0467229b7514b17dd13b2884d/tests/__init__.py -------------------------------------------------------------------------------- /tests/fields.py: -------------------------------------------------------------------------------- 1 | """ 2 | Custom fields for the tests. 3 | """ 4 | 5 | from django.db import models 6 | 7 | 8 | class TripleOnAssignDescriptor: 9 | """A descriptor that multiplies the assigned value by 3.""" 10 | 11 | def __init__(self, field): 12 | self.field = field 13 | 14 | def __get__(self, instance, owner=None): 15 | if instance is None: 16 | return self 17 | return instance.__dict__[self.field.name] 18 | 19 | def __set__(self, instance, value): 20 | # Allow NULL 21 | if value is not None: 22 | value *= 3 23 | instance.__dict__[self.field.name] = value 24 | 25 | 26 | class TriplingIntegerField(models.IntegerField): 27 | """Field that triples assigned value.""" 28 | 29 | # pylint:disable=arguments-differ 30 | def contribute_to_class(self, cls, name, **kwargs): 31 | super().contribute_to_class(cls, name, **kwargs) 32 | setattr(cls, self.name, TripleOnAssignDescriptor(self)) 33 | -------------------------------------------------------------------------------- /tests/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | """ 2 | Migration to create custom types 3 | """ 4 | 5 | from django.db import migrations 6 | 7 | from ..models import ( 8 | Box, 9 | Card, 10 | DateRange, 11 | DescriptorType, 12 | OptionalBits, 13 | Point, 14 | RenamedMemberType, 15 | SimpleType, 16 | ) 17 | 18 | 19 | class Migration(migrations.Migration): 20 | """Migration.""" 21 | 22 | dependencies = [] 23 | 24 | operations = [ 25 | SimpleType.Operation(), 26 | OptionalBits.Operation(), 27 | Card.Operation(), 28 | Point.Operation(), 29 | Box.Operation(), 30 | DateRange.Operation(), 31 | DescriptorType.Operation(), 32 | RenamedMemberType.Operation(), 33 | ] 34 | -------------------------------------------------------------------------------- /tests/migrations/0002_models.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.2 on 2018-03-04 23:12 2 | 3 | import django.contrib.postgres.fields 4 | from django.db import migrations, models 5 | 6 | import tests.models 7 | 8 | 9 | class Migration(migrations.Migration): 10 | initial = True 11 | 12 | dependencies = [ 13 | ("tests", "0001_initial"), 14 | ] 15 | 16 | operations = [ 17 | migrations.CreateModel( 18 | name="DescriptorModel", 19 | fields=[ 20 | ( 21 | "id", 22 | models.BigAutoField( 23 | auto_created=True, 24 | primary_key=True, 25 | serialize=False, 26 | verbose_name="ID", 27 | ), 28 | ), 29 | ("field", tests.models.DescriptorTypeField()), 30 | ], 31 | ), 32 | migrations.CreateModel( 33 | name="Hand", 34 | fields=[ 35 | ( 36 | "id", 37 | models.BigAutoField( 38 | auto_created=True, 39 | primary_key=True, 40 | serialize=False, 41 | verbose_name="ID", 42 | ), 43 | ), 44 | ( 45 | "cards", 46 | django.contrib.postgres.fields.ArrayField( 47 | base_field=tests.models.CardField(), size=None 48 | ), 49 | ), 50 | ], 51 | ), 52 | migrations.CreateModel( 53 | name="Item", 54 | fields=[ 55 | ( 56 | "id", 57 | models.BigAutoField( 58 | auto_created=True, 59 | primary_key=True, 60 | serialize=False, 61 | verbose_name="ID", 62 | ), 63 | ), 64 | ("name", models.CharField(max_length=20)), 65 | ("bounding_box", tests.models.BoxField()), 66 | ], 67 | ), 68 | migrations.CreateModel( 69 | name="NamedDateRange", 70 | fields=[ 71 | ( 72 | "id", 73 | models.BigAutoField( 74 | auto_created=True, 75 | primary_key=True, 76 | serialize=False, 77 | verbose_name="ID", 78 | ), 79 | ), 80 | ("name", models.TextField()), 81 | ("date_range", tests.models.DateRangeField()), 82 | ], 83 | ), 84 | migrations.CreateModel( 85 | name="OptionalModel", 86 | fields=[ 87 | ( 88 | "id", 89 | models.BigAutoField( 90 | auto_created=True, 91 | primary_key=True, 92 | serialize=False, 93 | verbose_name="ID", 94 | ), 95 | ), 96 | ( 97 | "optional_field", 98 | tests.models.OptionalBitsField(blank=True, null=True), 99 | ), 100 | ], 101 | ), 102 | migrations.CreateModel( 103 | name="SimpleModel", 104 | fields=[ 105 | ( 106 | "id", 107 | models.BigAutoField( 108 | auto_created=True, 109 | primary_key=True, 110 | serialize=False, 111 | verbose_name="ID", 112 | ), 113 | ), 114 | ("test_field", tests.models.SimpleTypeField()), 115 | ], 116 | ), 117 | migrations.CreateModel( 118 | name="RenamedMemberModel", 119 | fields=[ 120 | ("field", tests.models.RenamedMemberTypeField()), 121 | ], 122 | ), 123 | ] 124 | -------------------------------------------------------------------------------- /tests/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/danni/django-postgres-composite-types/e5a48c9ade0fccd0467229b7514b17dd13b2884d/tests/migrations/__init__.py -------------------------------------------------------------------------------- /tests/models.py: -------------------------------------------------------------------------------- 1 | """Models and types for the tests""" 2 | from django.contrib.postgres.fields.array import ArrayField 3 | from django.db import models 4 | 5 | from postgres_composite_types import CompositeType 6 | 7 | from .fields import TriplingIntegerField 8 | 9 | 10 | class SimpleType(CompositeType): 11 | """A test type.""" 12 | 13 | class Meta: 14 | db_type = "test_type" 15 | 16 | a = models.IntegerField(verbose_name="A number") 17 | b = models.CharField(verbose_name="A name", max_length=32) 18 | c = models.DateTimeField(verbose_name="A date") 19 | 20 | 21 | class SimpleModel(models.Model): 22 | """A test model.""" 23 | 24 | test_field = SimpleType.Field() 25 | 26 | 27 | class OptionalBits(CompositeType): 28 | """A type with an optional field""" 29 | 30 | required = models.CharField(max_length=32) 31 | optional = models.CharField(max_length=32, null=True, blank=True) 32 | 33 | class Meta: 34 | db_type = "optional_type" 35 | 36 | 37 | class OptionalModel(models.Model): 38 | """A model with an optional composity type""" 39 | 40 | optional_field = OptionalBits.Field(null=True, blank=True) 41 | 42 | 43 | class Card(CompositeType): 44 | """A playing card.""" 45 | 46 | class Meta: 47 | db_type = "card" 48 | 49 | suit = models.CharField(max_length=1) 50 | rank = models.CharField(max_length=2) 51 | 52 | 53 | class Hand(models.Model): 54 | """A hand of cards.""" 55 | 56 | cards = ArrayField(base_field=Card.Field()) 57 | 58 | 59 | class Point(CompositeType): 60 | """A point on the cartesian plane.""" 61 | 62 | class Meta: 63 | db_type = "test_point" # Postgres already has a point type 64 | 65 | x = models.IntegerField() 66 | y = models.IntegerField() 67 | 68 | 69 | class Box(CompositeType): 70 | """An axis-aligned box on the cartesian plane.""" 71 | 72 | class Meta: 73 | db_type = "test_box" # Postgres already has a box type 74 | 75 | top_left = Point.Field() 76 | bottom_right = Point.Field() 77 | 78 | @property 79 | def bottom_left(self): 80 | """The bottom-left corner of the box.""" 81 | return Point(x=self.top_left.x, y=self.bottom_right.y) 82 | 83 | @property 84 | def top_right(self): 85 | """The top-right corner of the box.""" 86 | return Point(x=self.bottom_right.x, y=self.top_left.y) 87 | 88 | 89 | class Item(models.Model): 90 | """An item that exists somewhere on a cartesian plane.""" 91 | 92 | name = models.CharField(max_length=20) 93 | bounding_box = Box.Field() 94 | 95 | 96 | class DateRange(CompositeType): 97 | """A date range with start and end.""" 98 | 99 | class Meta: 100 | db_type = "test_date_range" 101 | 102 | start = models.DateTimeField() 103 | end = models.DateTimeField() # uses reserved keyword 104 | 105 | 106 | class NamedDateRange(models.Model): 107 | """A date-range with a name""" 108 | 109 | name = models.TextField() 110 | date_range = DateRange.Field() 111 | 112 | 113 | class DescriptorType(CompositeType): 114 | """Has a field implementing a custom descriptor""" 115 | 116 | class Meta: 117 | db_type = "test_custom_descriptor" 118 | 119 | value = TriplingIntegerField() 120 | 121 | 122 | class DescriptorModel(models.Model): 123 | """Has a composite type with a field implementing a custom descriptor""" 124 | 125 | field = DescriptorType.Field() 126 | 127 | 128 | class RenamedMemberType(CompositeType): 129 | """Has a field with a different name in ORM vs db""" 130 | 131 | class Meta: 132 | db_type = "test_renamed_member" 133 | 134 | orm_name = models.CharField(db_column="db_name", max_length=32) 135 | other = models.BooleanField(default=False) 136 | 137 | 138 | class RenamedMemberModel(models.Model): 139 | """Has a composite type with a member attr name that differs from the column name""" 140 | 141 | field = RenamedMemberType.Field() 142 | -------------------------------------------------------------------------------- /tests/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for tests project. 3 | 4 | Generated by 'django-admin startproject' using Django 1.9.6. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.9/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/1.9/ref/settings/ 11 | """ 12 | 13 | import os 14 | 15 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 16 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 17 | SECRET_KEY = "SECRET" 18 | 19 | INSTALLED_APPS = [ 20 | "postgres_composite_types", 21 | "tests", 22 | ] 23 | 24 | # Database 25 | # https://docs.djangoproject.com/en/1.9/ref/settings/#databases 26 | DATABASES = { 27 | "default": { 28 | "ENGINE": "django.db.backends.postgresql", 29 | "NAME": "postgres", 30 | "USER": "postgres", 31 | "HOST": "localhost", 32 | "PASSWORD": "postgres", 33 | } 34 | } 35 | 36 | DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" 37 | -------------------------------------------------------------------------------- /tests/test_field.py: -------------------------------------------------------------------------------- 1 | """Tests for composite field.""" 2 | 3 | import datetime 4 | import json 5 | from unittest import mock 6 | 7 | from django.core import serializers 8 | from django.core.exceptions import ValidationError 9 | from django.db import connection 10 | from django.db.migrations.executor import MigrationExecutor 11 | from django.test import TestCase, TransactionTestCase 12 | from psycopg2.extensions import adapt 13 | 14 | from postgres_composite_types.signals import composite_type_created 15 | 16 | from .models import ( 17 | Box, 18 | DateRange, 19 | Item, 20 | NamedDateRange, 21 | OptionalBits, 22 | OptionalModel, 23 | Point, 24 | SimpleModel, 25 | SimpleType, 26 | ) 27 | 28 | 29 | def does_type_exist(type_name): 30 | """ 31 | Check if a composite type exists in the database 32 | """ 33 | sql = "select exists (select 1 from pg_type where typname = %s);" 34 | with connection.cursor() as cursor: 35 | cursor.execute(sql, [type_name]) 36 | row = cursor.fetchone() 37 | return row[0] 38 | 39 | 40 | def migrate(targets): 41 | """ 42 | Migrate to a new state. 43 | 44 | MigrationExecutors can not be reloaded, as they cache the state of the 45 | migrations when created. Attempting to reuse one might make some 46 | migrations not run, as it thinks they have already been run. 47 | """ 48 | executor = MigrationExecutor(connection) 49 | executor.migrate(targets) 50 | 51 | # Cant load state for apps in the initial empty state 52 | state_nodes = [node for node in targets if node[1] is not None] 53 | return executor.loader.project_state(state_nodes).apps 54 | 55 | 56 | class TestMigrations(TransactionTestCase): 57 | """ 58 | Taken from 59 | https://www.caktusgroup.com/blog/2016/02/02/writing-unit-tests-django-migrations/ 60 | 61 | FIXME: the ordering of the tests is totally broken. 62 | """ 63 | 64 | app = "tests" 65 | migrate_from = [("tests", None)] # Before the first migration 66 | migrate_to = [("tests", "0001_initial")] 67 | 68 | def test_migration(self): 69 | """Data data migration.""" 70 | 71 | # The migrations have already been run, and the type already exists in 72 | # the database 73 | self.assertTrue(does_type_exist(SimpleType._meta.db_type)) 74 | 75 | # Run the migration backwards to check the type is deleted 76 | migrate(self.migrate_from) 77 | 78 | # The type should now not exist 79 | self.assertFalse(does_type_exist(SimpleType._meta.db_type)) 80 | 81 | # A signal is fired when the migration creates the type 82 | signal_func = mock.Mock() 83 | composite_type_created.connect(receiver=signal_func, sender=SimpleType) 84 | 85 | # Run the migration forwards to create the type again 86 | migrate(self.migrate_to) 87 | self.assertTrue(does_type_exist(SimpleType._meta.db_type)) 88 | 89 | # The signal should have been sent 90 | self.assertEqual(signal_func.call_count, 1) 91 | self.assertEqual( 92 | signal_func.call_args, 93 | ( 94 | (), 95 | { 96 | "sender": SimpleType, 97 | "signal": composite_type_created, 98 | "connection": connection, 99 | }, 100 | ), 101 | ) 102 | 103 | # The type should now exist again 104 | self.assertTrue(does_type_exist(SimpleType._meta.db_type)) 105 | 106 | def test_migration_quoting(self): 107 | """Test that migration SQL is generated with correct quoting""" 108 | 109 | # The migrations have already been run, and the type already exists in 110 | # the database 111 | migrate(self.migrate_to) 112 | self.assertTrue(does_type_exist(DateRange._meta.db_type)) 113 | 114 | 115 | class FieldTests(TestCase): 116 | """Tests for composite field.""" 117 | 118 | def test_field_save_and_load(self): 119 | """Save and load a test model.""" 120 | t = SimpleType(a=1, b="β ☃", c=datetime.datetime(1985, 10, 26, 9, 0)) 121 | m = SimpleModel(test_field=t) 122 | m.save() 123 | 124 | # Retrieve from DB 125 | m = SimpleModel.objects.get(id=1) 126 | self.assertIsNotNone(m.test_field) 127 | self.assertIsInstance(m.test_field, SimpleType) 128 | self.assertEqual(m.test_field.a, 1) 129 | self.assertEqual(m.test_field.b, "β ☃") 130 | self.assertEqual(m.test_field.c, datetime.datetime(1985, 10, 26, 9, 0)) 131 | 132 | cursor = connection.connection.cursor() 133 | cursor.execute(f"SELECT (test_field).a FROM {SimpleModel._meta.db_table}") 134 | (result,) = cursor.fetchone() 135 | 136 | self.assertEqual(result, 1) 137 | 138 | cursor = connection.connection.cursor() 139 | cursor.execute(f"SELECT (test_field).b FROM {SimpleModel._meta.db_table}") 140 | (result,) = cursor.fetchone() 141 | 142 | self.assertEqual(result, "β ☃") 143 | 144 | def test_field_save_and_load_with_reserved_names(self): 145 | """Test save/load of a composite type with reserved field names""" 146 | start = datetime.datetime.now() 147 | end = datetime.datetime.now() + datetime.timedelta(days=1) 148 | date_range = DateRange(start=start, end=end) 149 | model = NamedDateRange(name="foobar", date_range=date_range) 150 | model.save() 151 | 152 | model = NamedDateRange.objects.get() 153 | self.assertEqual(model.date_range, date_range) 154 | 155 | def test_adapted_sql(self): 156 | """ 157 | Check that the value is serialised to the correct SQL string, including 158 | a type cast 159 | """ 160 | value = SimpleType(a=1, b="b", c=datetime.datetime(1985, 10, 26, 9, 0)) 161 | 162 | adapted = adapt(value) 163 | adapted.prepare(connection.connection) 164 | 165 | self.assertEqual( 166 | b"(1, 'b', '1985-10-26T09:00:00'::timestamp)::test_type", 167 | adapted.getquoted(), 168 | ) 169 | 170 | def test_serialize(self): 171 | """ 172 | Check that composite values are correctly handled through Django's 173 | serialize/deserialize helpers, used for dumpdata/loaddata. 174 | """ 175 | old = Item( 176 | name="table", 177 | bounding_box=Box(top_left=Point(x=1, y=1), bottom_right=Point(x=4, y=2)), 178 | ) 179 | out = serializers.serialize("json", [old]) 180 | new = next(serializers.deserialize("json", out)).object 181 | 182 | self.assertEqual(old.bounding_box, new.bounding_box) 183 | 184 | def test_to_python(self): 185 | """ 186 | Test the Field.to_python() method interprets strings as JSON data. 187 | """ 188 | start = datetime.datetime.now() 189 | end = datetime.datetime.now() + datetime.timedelta(days=1) 190 | 191 | field = NamedDateRange._meta.get_field("date_range") 192 | out = field.to_python( 193 | json.dumps( 194 | { 195 | "start": start.isoformat(), 196 | "end": end.isoformat(), 197 | } 198 | ) 199 | ) 200 | 201 | self.assertEqual(out, DateRange(start=start, end=end)) 202 | 203 | def test_to_python_bad_json(self): 204 | """ 205 | Test the Field.to_python() handles bad JSON data by raising 206 | a ValidationError 207 | """ 208 | field = NamedDateRange._meta.get_field("date_range") 209 | 210 | with self.assertRaises(ValidationError) as context: 211 | field.to_python("bogus JSON") 212 | 213 | exception = context.exception 214 | self.assertEqual(exception.code, "bad_json") 215 | 216 | 217 | class TestOptionalFields(TestCase): 218 | """ 219 | Test optional composite type fields, and optional fields on composite types 220 | """ 221 | 222 | def test_null_field_save_and_load(self): 223 | """Save and load a null composite field""" 224 | model = OptionalModel(optional_field=None) 225 | model.save() 226 | 227 | model = OptionalModel.objects.get() 228 | self.assertIsNone(model.optional_field) 229 | 230 | def test_null_subfield_save_and_load(self): 231 | """Save and load a null composite field""" 232 | model = OptionalModel( 233 | optional_field=OptionalBits(required="foo", optional=None) 234 | ) 235 | model.save() 236 | 237 | model = OptionalModel.objects.get() 238 | self.assertIsNotNone(model.optional_field) 239 | self.assertEqual( 240 | model.optional_field, OptionalBits(required="foo", optional=None) 241 | ) 242 | 243 | def test_all_filled(self): 244 | """ 245 | Save and load an optional composite field with all its optional fields 246 | filled in 247 | """ 248 | model = OptionalModel( 249 | optional_field=OptionalBits(required="foo", optional="bar") 250 | ) 251 | model.save() 252 | 253 | model = OptionalModel.objects.get(id=1) 254 | self.assertIsNotNone(model.optional_field) 255 | self.assertEqual( 256 | model.optional_field, OptionalBits(required="foo", optional="bar") 257 | ) 258 | -------------------------------------------------------------------------------- /tests/test_forms.py: -------------------------------------------------------------------------------- 1 | """Tests for CompositeTypeField and CompositeTypeWidget.""" 2 | import datetime 3 | 4 | from django import forms 5 | from django.test import SimpleTestCase 6 | from django.test.testcases import assert_and_parse_html 7 | 8 | from postgres_composite_types.forms import CompositeTypeField 9 | 10 | from .test_field import SimpleType 11 | 12 | 13 | class TestField(SimpleTestCase): 14 | """ 15 | Test the CompositeTypeField 16 | """ 17 | 18 | class SimpleForm(forms.Form): 19 | """Test form with CompositeTypeField""" 20 | 21 | simple_field = CompositeTypeField(model=SimpleType) 22 | 23 | simple_valid_data = { 24 | "simple_field-a": "1", 25 | "simple_field-b": "foo", 26 | "simple_field-c": "2016-05-24 17:38:32", 27 | } 28 | 29 | def test_composite_field(self): 30 | """Test that a composite field can create an instance of its model""" 31 | 32 | form = self.SimpleForm(data=self.simple_valid_data) 33 | 34 | self.assertTrue(form.is_valid()) 35 | 36 | out = form.cleaned_data["simple_field"] 37 | self.assertIsInstance(out, SimpleType) 38 | self.assertEqual( 39 | out, SimpleType(a=1, b="foo", c=datetime.datetime(2016, 5, 24, 17, 38, 32)) 40 | ) 41 | 42 | def test_validation(self): 43 | """ 44 | Test that a composite field validates its input, throwing errors for 45 | bad data 46 | """ 47 | form = self.SimpleForm( 48 | data={ 49 | "simple_field-a": "one", 50 | "simple_field-b": "", 51 | "simple_field-c": "yesterday, 10 oclock", 52 | } 53 | ) 54 | # CompositeTypeFields should fail validation if any of their fields 55 | # fail validation 56 | self.assertFalse(form.is_valid()) 57 | self.assertIn("simple_field", form.errors) 58 | # All three fields should be incorrect 59 | self.assertEqual(len(form.errors["simple_field"]), 3) 60 | # Errors should be formatted like 'Label: Error message' 61 | self.assertEqual( 62 | str(form.errors["simple_field"][0]), "A number: Enter a whole number." 63 | ) 64 | 65 | # Fields with validation errors should render with their invalid input 66 | self.assertHTMLContains( 67 | """ 68 | 70 | """, 71 | str(form["simple_field"]), 72 | ) 73 | 74 | def test_subfield_validation(self): 75 | """Errors on subfields should be accessible""" 76 | form = self.SimpleForm( 77 | data={ 78 | "simple_field-a": "one", 79 | } 80 | ) 81 | self.assertFalse(form.is_valid()) 82 | self.assertEqual( 83 | str(form["simple_field"]["a"].errors[0]), "Enter a whole number." 84 | ) 85 | 86 | def test_subfields(self): 87 | """Test accessing bound subfields""" 88 | form = self.SimpleForm(data=self.simple_valid_data) 89 | a_bound_field = form["simple_field"]["a"] 90 | 91 | self.assertIsInstance(a_bound_field.field, forms.IntegerField) 92 | self.assertEqual(a_bound_field.html_name, "simple_field-a") 93 | 94 | def test_nested_prefix(self): 95 | """Test forms with a prefix""" 96 | form = self.SimpleForm(data=self.simple_valid_data, prefix="step1") 97 | 98 | composite_bound_field = form["simple_field"] 99 | self.assertEqual(composite_bound_field.html_name, "step1-simple_field") 100 | 101 | a_bound_field = composite_bound_field["a"] 102 | self.assertEqual(a_bound_field.html_name, "step1-simple_field-a") 103 | 104 | def test_initial_data(self): 105 | """ 106 | Check that forms with initial data render with the fields prepopulated. 107 | """ 108 | initial = SimpleType(a=1, b="foo", c=datetime.datetime(2016, 5, 24, 17, 38, 32)) 109 | form = self.SimpleForm(initial={"simple_field": initial}) 110 | 111 | self.assertHTMLContains( 112 | """ 113 | 115 | """, 116 | str(form["simple_field"]), 117 | ) 118 | 119 | def test_null_initial_data(self): 120 | """ 121 | Check that forms with null initial data render with the fields. 122 | """ 123 | form = self.SimpleForm(initial={"simple_field": None}) 124 | 125 | self.assertHTMLContains( 126 | """ 127 | 129 | """, 130 | str(form["simple_field"]), 131 | ) 132 | 133 | def test_value_omission_check_inside_widget(self): 134 | """ 135 | Assert that CompositeTypeWidget.value_omitted_from_data function 136 | will return False when passing valid data. 137 | """ 138 | form = self.SimpleForm() 139 | widget = form.fields["simple_field"].widget 140 | self.assertFalse( 141 | widget.value_omitted_from_data( 142 | data=self.simple_valid_data, 143 | files=[], 144 | name="simple_field", 145 | ) 146 | ) 147 | 148 | def assertHTMLContains(self, text, content, count=None, msg=None): 149 | """ 150 | Assert that the HTML snippet ``text`` is found within the HTML snippet 151 | ``content``. Like assertContains, but works with plain strings instead 152 | of Response instances. 153 | """ 154 | content = assert_and_parse_html( 155 | self, content, None, "HTML content to search in is not valid:" 156 | ) 157 | text = assert_and_parse_html( 158 | self, text, None, "HTML content to search for is not valid:" 159 | ) 160 | 161 | matches = content.count(text) 162 | if count is None: 163 | self.assertTrue(matches > 0, msg=msg or "Could not find HTML snippet") 164 | else: 165 | self.assertEqual( 166 | matches, 167 | count, 168 | msg=msg or f"Found {matches} matches, expecting {count}", 169 | ) 170 | 171 | 172 | class OptionalFieldTests(SimpleTestCase): 173 | """ 174 | CompundTypeFields should handle being optional sensibly 175 | """ 176 | 177 | class OptionalSimpleForm(forms.Form): 178 | """Test form with optional CompositeTypeField""" 179 | 180 | optional_field = CompositeTypeField(model=SimpleType, required=False) 181 | 182 | simple_valid_data = { 183 | "optional_field-a": "1", 184 | "optional_field-b": "foo", 185 | "optional_field-c": "2016-05-24 17:38:32", 186 | } 187 | 188 | def test_blank_fields(self): 189 | """Test leaving all the fields blank""" 190 | 191 | form = self.OptionalSimpleForm( 192 | data={ 193 | "simple_field-a": "", 194 | "simple_field-b": "", 195 | "simple_field-c": "", 196 | } 197 | ) 198 | 199 | # The form should be valid, but simple_field should be None 200 | self.assertTrue(form.is_valid()) 201 | self.assertIsNone(form.cleaned_data["optional_field"]) 202 | 203 | def test_missing_fields(self): 204 | """Test not even submitting the fields""" 205 | 206 | form = self.OptionalSimpleForm(data={}) 207 | 208 | # The form should be valid, but simple_field should be None 209 | self.assertTrue(form.is_valid()) 210 | self.assertIsNone(form.cleaned_data["optional_field"]) 211 | 212 | def test_filling_out_fields(self): 213 | """Test filling out the fields normally still works""" 214 | 215 | form = self.OptionalSimpleForm(data=self.simple_valid_data) 216 | 217 | self.assertTrue(form.is_valid()) 218 | out = form.cleaned_data["optional_field"] 219 | self.assertIsInstance(out, SimpleType) 220 | self.assertEqual( 221 | out, SimpleType(a=1, b="foo", c=datetime.datetime(2016, 5, 24, 17, 38, 32)) 222 | ) 223 | 224 | def test_some_valid_some_empty(self): 225 | """Test with some fields filled in, some required fields blank""" 226 | 227 | form = self.OptionalSimpleForm( 228 | data={ 229 | "optional_field-a": "1", 230 | "optional_field-b": "foo", 231 | "optional_field-c": "", 232 | } 233 | ) 234 | 235 | self.assertFalse(form.is_valid()) 236 | self.assertIn("optional_field", form.errors) 237 | # Only the one field should fail validation 238 | self.assertEqual(len(form.errors["optional_field"]), 1) 239 | # Errors should be formatted like 'Label: Error message' 240 | self.assertEqual( 241 | "A date: This field is required.", str(form.errors["optional_field"][0]) 242 | ) 243 | -------------------------------------------------------------------------------- /tests/test_more.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for composite fields in combination with other interesting fields. 3 | """ 4 | import datetime 5 | 6 | from django.db.migrations.writer import MigrationWriter 7 | from django.test import TestCase 8 | 9 | from .models import ( 10 | Box, 11 | Card, 12 | DescriptorModel, 13 | DescriptorType, 14 | Hand, 15 | Item, 16 | Point, 17 | RenamedMemberModel, 18 | RenamedMemberType, 19 | SimpleModel, 20 | SimpleType, 21 | ) 22 | 23 | 24 | class TestArrayFields(TestCase): 25 | """ 26 | Test ArrayFields combined with CompositeType.Fields 27 | """ 28 | 29 | def test_saving_and_loading_array_field(self): 30 | """ 31 | Test saving and loading an ArrayField of a CompositeType 32 | """ 33 | # Nice hand 34 | hand = Hand( 35 | cards=[ 36 | Card("♡", "1"), 37 | Card("♡", "K"), 38 | Card("♡", "Q"), 39 | Card("♡", "J"), 40 | Card("♡", "10"), 41 | ] 42 | ) 43 | hand.save() 44 | 45 | hand = Hand.objects.get() 46 | self.assertEqual( 47 | hand.cards, 48 | [ 49 | Card("♡", "1"), 50 | Card("♡", "K"), 51 | Card("♡", "Q"), 52 | Card("♡", "J"), 53 | Card("♡", "10"), 54 | ], 55 | ) 56 | 57 | def test_querying_array_field_contains(self): 58 | """ 59 | Test using some array__contains=[CompositeType] 60 | """ 61 | hand = Hand( 62 | cards=[ 63 | Card("♡", "1"), 64 | Card("♡", "K"), 65 | Card("♡", "Q"), 66 | Card("♡", "J"), 67 | Card("♡", "10"), 68 | ] 69 | ) 70 | hand.save() 71 | 72 | queen_of_hearts = Card("♡", "Q") 73 | jack_of_spades = Card("♠", "J") 74 | self.assertTrue(Hand.objects.filter(cards__contains=[queen_of_hearts]).exists()) 75 | self.assertFalse(Hand.objects.filter(cards__contains=[jack_of_spades]).exists()) 76 | 77 | def test_generate_migrations(self): 78 | """Test deconstruction of composite type as a base field""" 79 | field = Hand._meta.get_field("cards") 80 | text, _ = MigrationWriter.serialize(field) 81 | # build the expected full path of the nested composite type class 82 | models_module = Hand.__module__ 83 | composite_field_cls = field.base_field.__class__.__name__ 84 | expected_path = ".".join((models_module, composite_field_cls)) 85 | # check that the expected path is the one used by deconstruct 86 | expected_deconstruction = f"base_field={expected_path}()" 87 | self.assertIn(expected_deconstruction, text) 88 | 89 | 90 | class TestNestedCompositeTypes(TestCase): 91 | """ 92 | Test CompositeTypes within CompositeTypes 93 | """ 94 | 95 | def test_saving_and_loading_nested_composite_types(self): 96 | """ 97 | Test saving and loading an Item with nested CompositeTypes 98 | """ 99 | item = Item( 100 | name="table", 101 | bounding_box=Box(top_left=Point(x=1, y=1), bottom_right=Point(x=4, y=2)), 102 | ) 103 | item.save() 104 | 105 | item = Item.objects.get() 106 | self.assertEqual(item.name, "table") 107 | self.assertEqual( 108 | item.bounding_box, 109 | Box(top_left=Point(x=1, y=1), bottom_right=Point(x=4, y=2)), 110 | ) 111 | 112 | self.assertEqual(item.bounding_box.bottom_left, Point(x=1, y=2)) 113 | self.assertEqual(item.bounding_box.top_right, Point(x=4, y=1)) 114 | 115 | 116 | class TestCustomDescriptors(TestCase): 117 | """ 118 | Test CompositeTypes with Fields with custom descriptors. 119 | """ 120 | 121 | def test_create(self): 122 | """Test descriptor used on creation""" 123 | model = DescriptorModel(field=DescriptorType(value=1)) 124 | self.assertEqual(model.field.value, 3) 125 | 126 | def test_set(self): 127 | """Test descriptor used on assign""" 128 | model = DescriptorModel(field=DescriptorType(value=0)) 129 | model.field.value = 14 130 | self.assertEqual(model.field.value, 42) 131 | 132 | 133 | class TestMemberLookup(TestCase): 134 | """ 135 | Test filtering queryset on composite type members. 136 | """ 137 | 138 | def test_value(self): 139 | """Test finding a record by composite member value.""" 140 | t = SimpleType(a=1, b="β ☃", c=datetime.datetime(1985, 10, 26, 9, 0)) 141 | m = SimpleModel(test_field=t) 142 | m.save() 143 | 144 | self.assertEqual(SimpleModel.objects.get(test_field__a=1), m) 145 | 146 | def test_comparison(self): 147 | """Test finding a record by comparison on a composite member value.""" 148 | t = SimpleType(a=1, b="β ☃", c=datetime.datetime(1985, 10, 26, 9, 0)) 149 | m = SimpleModel(test_field=t) 150 | m.save() 151 | t.a = 3 152 | SimpleModel.objects.create(test_field=t) 153 | 154 | self.assertEqual(SimpleModel.objects.get(test_field__a__lt=2), m) 155 | 156 | def test_renamed(self): 157 | """Test finding a record by composite member value with a different ORM name.""" 158 | t = RenamedMemberType(orm_name="foo") 159 | m = RenamedMemberModel(field=t) 160 | m.save() 161 | 162 | self.assertEqual(RenamedMemberModel.objects.get(field__orm_name="foo"), m) 163 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py{39,310,311}-dj{32,40,41} 3 | isolated_build = true 4 | 5 | [testenv] 6 | setenv = 7 | PYTHONPATH = {toxinidir} 8 | DJANGO_SETTINGS_MODULE = tests.settings 9 | 10 | allowlist_externals = pwd 11 | 12 | commands = 13 | pwd 14 | pytest tests/ 15 | 16 | deps = 17 | pytest 18 | pytest-django 19 | dj32: Django >= 3.2.16 20 | dj40: Django >= 4.0.8 21 | dj41: Django >= 4.1.4 22 | 23 | [gh-actions] 24 | python = 25 | 3.9: py39 26 | 3.10: py310 27 | 3.11: py311 28 | --------------------------------------------------------------------------------