├── app ├── app │ ├── __init__.py │ ├── asgi.py │ ├── wsgi.py │ ├── urls.py │ └── settings.py ├── rooms │ ├── middleware │ │ ├── __init__.py │ │ └── log_execution_time.py │ ├── migrations │ │ ├── __init__.py │ │ ├── 0002_alter_room_decoration_alter_room_door_and_more.py │ │ ├── 0007_roomwithrelatedobjsv3.py │ │ ├── 0003_alter_bed_rooms_alter_chair_rooms_and_more.py │ │ ├── 0006_roomwithrelatedobjsrebuildinapp.py │ │ ├── 0005_roomsrelatedobjectsmaterializedview_and_more.py │ │ ├── 0004_roomsrelatedobjects.py │ │ ├── 0001_initial.py │ │ └── 0008_bed_room_v3_bed_chair_room_v3_chair_and_more.py │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── models │ │ ├── __init__.py │ │ ├── common_info.py │ │ ├── rooms_related_v3.py │ │ ├── room_related_v2.py │ │ ├── room_related_view.py │ │ ├── door.py │ │ ├── decoration.py │ │ ├── room.py │ │ ├── souvenir.py │ │ ├── window.py │ │ ├── window_fittings.py │ │ └── furniture.py │ ├── urls.py │ ├── utils.py │ ├── views.py │ ├── management │ │ └── commands │ │ │ └── fill_db.py │ ├── serializers.py │ ├── signals.py │ └── tests.py ├── setup.cfg ├── manage.py └── conftest.py ├── .dockerignore ├── .gitignore ├── misc ├── db_diagram.png └── log_processing_output.png ├── requirements.txt ├── Dockerfile ├── Dockerfile-pg ├── execute_after_db_dump_restore.sh ├── docker-compose.yml ├── supplementary_scripts ├── create_view.sql └── compare_requests_time.py ├── LICENSE └── README.md /app/app/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/rooms/middleware/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/rooms/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | app/.venv/**/* 2 | app/.venv -------------------------------------------------------------------------------- /app/rooms/__init__.py: -------------------------------------------------------------------------------- 1 | default_app_config = 'rooms.apps.RoomsConfig' 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | app/.venv/ 3 | pgadmin-data/ 4 | data/ 5 | *.log 6 | -------------------------------------------------------------------------------- /app/rooms/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /misc/db_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/s-kust/django-postgresql-view/HEAD/misc/db_diagram.png -------------------------------------------------------------------------------- /misc/log_processing_output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/s-kust/django-postgresql-view/HEAD/misc/log_processing_output.png -------------------------------------------------------------------------------- /app/setup.cfg: -------------------------------------------------------------------------------- 1 | [tool:pytest] 2 | DJANGO_SETTINGS_MODULE = app.settings 3 | python_files = tests.py test_*.py *_tests.py 4 | addopts = --reuse-db -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | django 2 | django-pgtrigger 3 | model-bakery 4 | psycopg2-binary 5 | djangorestframework 6 | pytest-django 7 | pytest-xdist 8 | pytest-random-order 9 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3-alpine 2 | ENV PYTHONDONTWRITEBYTECODE=1 3 | ENV PYTHONUNBUFFERED=1 4 | WORKDIR /app 5 | COPY ./requirements.txt /app/ 6 | RUN pip install -r requirements.txt 7 | COPY ./app /app -------------------------------------------------------------------------------- /Dockerfile-pg: -------------------------------------------------------------------------------- 1 | FROM postgres:14-alpine 2 | 3 | COPY db_dump.sql /docker-entrypoint-initdb.d/ 4 | COPY execute_after_db_dump_restore.sh /docker-entrypoint-initdb.d/ 5 | RUN chmod 755 /docker-entrypoint-initdb.d/execute_after_db_dump_restore.sh -------------------------------------------------------------------------------- /app/rooms/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class RoomsConfig(AppConfig): 5 | default_auto_field = 'django.db.models.BigAutoField' 6 | name = 'rooms' 7 | 8 | def ready(self): 9 | import rooms.signals 10 | -------------------------------------------------------------------------------- /app/app/asgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | ASGI config for app project. 3 | 4 | It exposes the ASGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/4.0/howto/deployment/asgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.asgi import get_asgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'app.settings') 15 | 16 | application = get_asgi_application() 17 | -------------------------------------------------------------------------------- /app/app/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for app project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/4.0/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'app.settings') 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /app/rooms/models/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .common_info import CommonInfo 3 | from .decoration import Decoration 4 | from .door import Door 5 | from .furniture import Bed, Chair, Table 6 | from .room import Room 7 | from .room_related_v2 import RoomWithRelatedObjsRebuildInApp 8 | from .room_related_view import RoomsRelatedObjectsMaterializedView 9 | from .rooms_related_v3 import RoomWithRelatedObjsV3 10 | from .souvenir import Souvenir 11 | from .window import Window 12 | from .window_fittings import WindowFittings 13 | -------------------------------------------------------------------------------- /app/rooms/middleware/log_execution_time.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | 4 | logger = logging.getLogger(__name__) 5 | 6 | 7 | class ExecutionTimeLogMiddleware: 8 | 9 | def __init__(self, get_response): 10 | self.get_response = get_response 11 | 12 | def __call__(self, request): 13 | start_time = time.time() 14 | response = self.get_response(request) 15 | end_time = time.time() 16 | time_diff = end_time - start_time 17 | logger.info(msg=str(";" + request.get_full_path()) + 18 | ";" + str(time_diff)) 19 | return response 20 | -------------------------------------------------------------------------------- /app/rooms/models/common_info.py: -------------------------------------------------------------------------------- 1 | from django.core.validators import MaxValueValidator, MinValueValidator 2 | from django.db import models 3 | 4 | 5 | class CommonInfo(models.Model): 6 | name = models.CharField(max_length=30, unique=True) 7 | width = models.FloatField(validators=[MinValueValidator(1.0), MaxValueValidator(100.0)], default=1.0) 8 | length = models.FloatField(validators=[MinValueValidator(1.0), MaxValueValidator(100.0)], default=1.0) 9 | height = models.FloatField(validators=[MinValueValidator(1.0), MaxValueValidator(100.0)], default=1.0) 10 | 11 | class Meta: 12 | abstract = True 13 | -------------------------------------------------------------------------------- /execute_after_db_dump_restore.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | # just scaffolding, experiment 5 | # for possible later use 6 | 7 | psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL 8 | CREATE SCHEMA IF NOT EXISTS my_extensions; 9 | CREATE EXTENSION IF NOT EXISTS "uuid-ossp" SCHEMA my_extensions; 10 | CREATE EXTENSION IF NOT EXISTS "pg_stat_statements" SCHEMA my_extensions; 11 | SELECT extname AS new_extension_created FROM pg_extension; 12 | DROP EXTENSION IF EXISTS "uuid-ossp"; 13 | DROP SCHEMA IF EXISTS my_extensions; 14 | SELECT extname AS new_extension_deleted FROM pg_extension; 15 | EOSQL -------------------------------------------------------------------------------- /app/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Django's command-line utility for administrative tasks.""" 3 | import os 4 | import sys 5 | 6 | 7 | def main(): 8 | """Run administrative tasks.""" 9 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'app.settings') 10 | try: 11 | from django.core.management import execute_from_command_line 12 | except ImportError as exc: 13 | raise ImportError( 14 | "Couldn't import Django. Are you sure it's installed and " 15 | "available on your PYTHONPATH environment variable? Did you " 16 | "forget to activate a virtual environment?" 17 | ) from exc 18 | execute_from_command_line(sys.argv) 19 | 20 | 21 | if __name__ == '__main__': 22 | main() 23 | -------------------------------------------------------------------------------- /app/rooms/models/rooms_related_v3.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class RoomWithRelatedObjsV3(models.Model): 5 | id = models.BigIntegerField(primary_key=True) 6 | door = models.JSONField(blank=True, null=True) 7 | decoration = models.JSONField(blank=True, null=True) 8 | windows = models.JSONField(blank=True, null=True) 9 | name = models.CharField(max_length=30, blank=True, null=True) 10 | width = models.FloatField(blank=True, null=True) 11 | length = models.FloatField(blank=True, null=True) 12 | height = models.FloatField(blank=True, null=True) 13 | type = models.CharField(max_length=5, blank=True, null=True) 14 | beds = models.JSONField(blank=True, null=True) 15 | chairs = models.JSONField(blank=True, null=True) 16 | tables = models.JSONField(blank=True, null=True) 17 | -------------------------------------------------------------------------------- /app/rooms/models/room_related_v2.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class RoomWithRelatedObjsRebuildInApp(models.Model): 5 | id = models.BigIntegerField(primary_key=True) 6 | door = models.JSONField(blank=True, null=True) 7 | decoration = models.JSONField(blank=True, null=True) 8 | windows = models.JSONField(blank=True, null=True) 9 | name = models.CharField(max_length=30, blank=True, null=True) 10 | width = models.FloatField(blank=True, null=True) 11 | length = models.FloatField(blank=True, null=True) 12 | height = models.FloatField(blank=True, null=True) 13 | type = models.CharField(max_length=5, blank=True, null=True) 14 | beds = models.JSONField(blank=True, null=True) 15 | chairs = models.JSONField(blank=True, null=True) 16 | tables = models.JSONField(blank=True, null=True) 17 | -------------------------------------------------------------------------------- /app/app/urls.py: -------------------------------------------------------------------------------- 1 | """app URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/4.0/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.urls import include, path 14 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 15 | """ 16 | from django.contrib import admin 17 | from django.urls import include, path 18 | 19 | urlpatterns = [ 20 | path('admin/', admin.site.urls), 21 | path('', include('rooms.urls')), 22 | ] 23 | -------------------------------------------------------------------------------- /app/rooms/models/room_related_view.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class RoomsRelatedObjectsMaterializedView(models.Model): 5 | id = models.BigIntegerField(primary_key=True) 6 | door = models.JSONField(blank=True, null=True) 7 | decoration = models.JSONField(blank=True, null=True) 8 | windows = models.JSONField(blank=True, null=True) 9 | name = models.CharField(max_length=30, blank=True, null=True) 10 | width = models.FloatField(blank=True, null=True) 11 | length = models.FloatField(blank=True, null=True) 12 | height = models.FloatField(blank=True, null=True) 13 | type = models.CharField(max_length=5, blank=True, null=True) 14 | beds = models.JSONField(blank=True, null=True) 15 | chairs = models.JSONField(blank=True, null=True) 16 | tables = models.JSONField(blank=True, null=True) 17 | 18 | class Meta: 19 | managed = False # Created from a view. Don't remove. 20 | db_table = "rooms_related_objects" 21 | -------------------------------------------------------------------------------- /app/rooms/migrations/0002_alter_room_decoration_alter_room_door_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.3 on 2022-03-31 08:14 2 | 3 | import django.db.models.deletion 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('rooms', '0001_initial'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='room', 16 | name='decoration', 17 | field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='rooms', to='rooms.decoration'), 18 | ), 19 | migrations.AlterField( 20 | model_name='room', 21 | name='door', 22 | field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='rooms', to='rooms.door'), 23 | ), 24 | migrations.AlterField( 25 | model_name='window', 26 | name='room', 27 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='windows', to='rooms.room'), 28 | ), 29 | ] 30 | -------------------------------------------------------------------------------- /app/conftest.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import psycopg2 4 | import pytest 5 | from django.db import connections 6 | from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | 11 | def run_sql(sql): 12 | conn = psycopg2.connect(database='postgres', user='postgres', 13 | password='postgres', host='host.docker.internal') 14 | conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) 15 | cur = conn.cursor() 16 | cur.execute(sql) 17 | conn.close() 18 | 19 | 20 | @pytest.fixture(scope='session') 21 | def django_db_setup(): 22 | from django.conf import settings 23 | 24 | settings.DATABASES['default']['NAME'] = 'the_copied_db' 25 | logger.info(msg="; Started creating test DB...") 26 | run_sql('DROP DATABASE IF EXISTS the_copied_db') 27 | run_sql( 28 | "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = 'postgres' AND pid <> pg_backend_pid()" 29 | ) 30 | run_sql('CREATE DATABASE the_copied_db TEMPLATE postgres') 31 | logger.info(msg="; Creating test DB finished OK") 32 | 33 | yield 34 | 35 | for connection in connections.all(): 36 | connection.close() 37 | 38 | # run_sql('DROP DATABASE the_copied_db') 39 | -------------------------------------------------------------------------------- /app/rooms/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import include, path 2 | from rest_framework.routers import DefaultRouter 3 | 4 | from rooms import views 5 | 6 | # Create a router and register our viewsets with it. 7 | router = DefaultRouter() 8 | router.register(r"windows", views.WindowViewSet, basename="windows") 9 | router.register(r"window_fittings", views.WindowFittingsViewSet, basename="window_fittings") 10 | router.register(r"rooms_native", views.RoomViewSet, basename="rooms_native") 11 | router.register(r"rooms_v2", views.RoomsRelatedV2ViewSet, basename="rooms_v2") 12 | router.register(r"rooms_v3", views.RoomsRelatedV3ViewSet, basename="rooms_v3") 13 | router.register(r"rooms_mat_view", views.RoomsRelatedObjectsViewSet, basename="rooms_mat_view") 14 | router.register(r"souvenirs", views.SouvenirViewSet, basename="souvenirs") 15 | router.register(r"tables", views.TableViewSet, basename="tables") 16 | router.register(r"doors", views.DoorViewSet, basename="doors") 17 | router.register(r"chairs", views.ChairViewSet, basename="chairs") 18 | router.register(r"beds", views.BedViewSet, basename="beds") 19 | router.register(r"decorations", views.DecorationViewSet, basename="decorations") 20 | 21 | # The API URLs are now determined automatically by the router. 22 | urlpatterns = [ 23 | path("", include(router.urls)), 24 | ] 25 | -------------------------------------------------------------------------------- /app/rooms/migrations/0007_roomwithrelatedobjsv3.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.5 on 2022-06-09 15:05 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rooms', '0006_roomwithrelatedobjsrebuildinapp'), 10 | ] 11 | 12 | operations = [ 13 | migrations.CreateModel( 14 | name='RoomWithRelatedObjsV3', 15 | fields=[ 16 | ('id', models.BigIntegerField(primary_key=True, serialize=False)), 17 | ('door', models.JSONField(blank=True, null=True)), 18 | ('decoration', models.JSONField(blank=True, null=True)), 19 | ('windows', models.JSONField(blank=True, null=True)), 20 | ('name', models.CharField(blank=True, max_length=30, null=True)), 21 | ('width', models.FloatField(blank=True, null=True)), 22 | ('length', models.FloatField(blank=True, null=True)), 23 | ('height', models.FloatField(blank=True, null=True)), 24 | ('type', models.CharField(blank=True, max_length=5, null=True)), 25 | ('beds', models.JSONField(blank=True, null=True)), 26 | ('chairs', models.JSONField(blank=True, null=True)), 27 | ('tables', models.JSONField(blank=True, null=True)), 28 | ], 29 | ), 30 | ] 31 | -------------------------------------------------------------------------------- /app/rooms/models/door.py: -------------------------------------------------------------------------------- 1 | import pgtrigger 2 | from django.db import models 3 | 4 | from .common_info import CommonInfo 5 | from .rooms_related_v3 import RoomWithRelatedObjsV3 6 | 7 | 8 | @pgtrigger.register( 9 | pgtrigger.Trigger( 10 | name="room_v3_door", 11 | level=pgtrigger.Row, 12 | when=pgtrigger.After, 13 | operation=pgtrigger.Update | pgtrigger.Insert | pgtrigger.Delete, 14 | func=f""" 15 | INSERT INTO {RoomWithRelatedObjsV3._meta.db_table}(id, door) 16 | SELECT 17 | room.id id, 18 | jsonb_build_object( 19 | 'id', door.id, 'name', door.name, 'width', 20 | door.width, 'length', door.length, 21 | 'height', door.height, 'type', door.type 22 | ) door 23 | FROM 24 | rooms_room room 25 | LEFT JOIN rooms_door door ON room.door_id = door.id 26 | WHERE room.door_id = OLD.id 27 | ON CONFLICT (id) 28 | DO 29 | UPDATE SET door = EXCLUDED.door; 30 | RETURN NULL; 31 | """, 32 | ) 33 | ) 34 | class Door(CommonInfo): 35 | CHOICES = ( 36 | ("DT1", "Door Type 1"), 37 | ("DT2", "Door Type 2"), 38 | ("DT3", "Door Type 3"), 39 | ("DT4", "Door Type 4"), 40 | ) 41 | type = models.CharField(max_length=5, choices=CHOICES) 42 | -------------------------------------------------------------------------------- /app/rooms/migrations/0003_alter_bed_rooms_alter_chair_rooms_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.3 on 2022-03-31 08:18 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rooms', '0002_alter_room_decoration_alter_room_door_and_more'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='bed', 15 | name='rooms', 16 | field=models.ManyToManyField(related_name='beds', to='rooms.room'), 17 | ), 18 | migrations.AlterField( 19 | model_name='chair', 20 | name='rooms', 21 | field=models.ManyToManyField(related_name='chairs', to='rooms.room'), 22 | ), 23 | migrations.AlterField( 24 | model_name='decoration', 25 | name='souvenirs', 26 | field=models.ManyToManyField(related_name='decorations', to='rooms.souvenir'), 27 | ), 28 | migrations.AlterField( 29 | model_name='table', 30 | name='rooms', 31 | field=models.ManyToManyField(related_name='tables', to='rooms.room'), 32 | ), 33 | migrations.AlterField( 34 | model_name='windowfittings', 35 | name='windows', 36 | field=models.ManyToManyField(related_name='fittings', to='rooms.window'), 37 | ), 38 | ] 39 | -------------------------------------------------------------------------------- /app/rooms/migrations/0006_roomwithrelatedobjsrebuildinapp.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.3 on 2022-04-12 09:27 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rooms', '0005_roomsrelatedobjectsmaterializedview_and_more'), 10 | ] 11 | 12 | operations = [ 13 | migrations.CreateModel( 14 | name='RoomWithRelatedObjsRebuildInApp', 15 | fields=[ 16 | ('id', models.BigIntegerField(primary_key=True, serialize=False)), 17 | ('door', models.JSONField(blank=True, null=True)), 18 | ('decoration', models.JSONField(blank=True, null=True)), 19 | ('windows', models.JSONField(blank=True, null=True)), 20 | ('name', models.CharField(blank=True, max_length=30, null=True)), 21 | ('width', models.FloatField(blank=True, null=True)), 22 | ('length', models.FloatField(blank=True, null=True)), 23 | ('height', models.FloatField(blank=True, null=True)), 24 | ('type', models.CharField(blank=True, max_length=5, null=True)), 25 | ('beds', models.JSONField(blank=True, null=True)), 26 | ('chairs', models.JSONField(blank=True, null=True)), 27 | ('tables', models.JSONField(blank=True, null=True)), 28 | ], 29 | ), 30 | ] 31 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Warning! This docker-compose file is for educational use 2 | # and local testing only. It is very simplified and unsafe. 3 | # Not for production! 4 | 5 | version: '3' 6 | 7 | services: 8 | db: 9 | build: 10 | context: . 11 | dockerfile: Dockerfile-pg 12 | container_name: db 13 | volumes: 14 | - ./data/db:/var/lib/postgresql/data 15 | environment: 16 | - POSTGRES_DB=postgres 17 | - POSTGRES_USER=postgres 18 | - POSTGRES_PASSWORD=postgres 19 | ports: 20 | - "5432:5432" 21 | networks: 22 | - djangonetwork 23 | pgadmin: 24 | image: dpage/pgadmin4 25 | environment: 26 | PGADMIN_DEFAULT_EMAIL: admin@admin.com 27 | PGADMIN_DEFAULT_PASSWORD: secret 28 | PGADMIN_LISTEN_PORT: 80 29 | ports: 30 | - "8080:80" 31 | volumes: 32 | - ./pgadmin-data:/var/lib/pgadmin 33 | networks: 34 | - djangonetwork 35 | app: 36 | build: . 37 | ports: 38 | - "8000:8000" 39 | volumes: 40 | - ./app:/app 41 | command: > 42 | sh -c "python3 manage.py makemigrations && 43 | python3 manage.py migrate && 44 | pytest --random-order -s && 45 | python3 manage.py runserver 0.0.0.0:8000" 46 | environment: 47 | - POSTGRES_NAME=postgres 48 | - POSTGRES_USER=postgres 49 | - POSTGRES_PASSWORD=postgres 50 | depends_on: 51 | - db 52 | links: 53 | - db:db 54 | networks: 55 | - djangonetwork 56 | 57 | networks: 58 | djangonetwork: 59 | driver: bridge 60 | -------------------------------------------------------------------------------- /app/rooms/migrations/0005_roomsrelatedobjectsmaterializedview_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.3 on 2022-04-12 08:33 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rooms', '0004_roomsrelatedobjects'), 10 | ] 11 | 12 | operations = [ 13 | migrations.CreateModel( 14 | name='RoomsRelatedObjectsMaterializedView', 15 | fields=[ 16 | ('id', models.BigIntegerField(primary_key=True, serialize=False)), 17 | ('door', models.JSONField(blank=True, null=True)), 18 | ('decoration', models.JSONField(blank=True, null=True)), 19 | ('windows', models.JSONField(blank=True, null=True)), 20 | ('name', models.CharField(blank=True, max_length=30, null=True)), 21 | ('width', models.FloatField(blank=True, null=True)), 22 | ('length', models.FloatField(blank=True, null=True)), 23 | ('height', models.FloatField(blank=True, null=True)), 24 | ('type', models.CharField(blank=True, max_length=5, null=True)), 25 | ('beds', models.JSONField(blank=True, null=True)), 26 | ('chairs', models.JSONField(blank=True, null=True)), 27 | ('tables', models.JSONField(blank=True, null=True)), 28 | ], 29 | options={ 30 | 'db_table': 'rooms_related_objects', 31 | 'managed': False, 32 | }, 33 | ), 34 | migrations.DeleteModel( 35 | name='RoomsRelatedObjects', 36 | ), 37 | ] 38 | -------------------------------------------------------------------------------- /app/rooms/models/decoration.py: -------------------------------------------------------------------------------- 1 | import pgtrigger 2 | from django.db import models 3 | 4 | from .rooms_related_v3 import RoomWithRelatedObjsV3 5 | from .souvenir import Souvenir 6 | 7 | 8 | @pgtrigger.register( 9 | pgtrigger.Trigger( 10 | name="room_v3_decoration", 11 | level=pgtrigger.Row, 12 | when=pgtrigger.After, 13 | operation=pgtrigger.Update | pgtrigger.Insert | pgtrigger.Delete, 14 | func=f""" 15 | INSERT INTO {RoomWithRelatedObjsV3._meta.db_table}(id, decoration) 16 | SELECT 17 | room.id id, 18 | jsonb_build_object( 19 | 'id', 20 | decoration.id, 21 | 'name', 22 | decoration.name, 23 | 'souvenirs', 24 | jsonb_agg(souvenir) 25 | ) decoration 26 | FROM 27 | rooms_room room 28 | LEFT JOIN rooms_decoration decoration ON room.decoration_id = decoration.id 29 | LEFT JOIN rooms_decoration_souvenirs ds ON decoration.id = ds.decoration_id 30 | LEFT JOIN rooms_souvenir souvenir ON ds.souvenir_id = souvenir.id 31 | WHERE room.decoration_id = OLD.id 32 | GROUP BY 33 | room.id, 34 | decoration.id 35 | ON CONFLICT (id) 36 | DO 37 | UPDATE SET decoration = EXCLUDED.decoration; 38 | RETURN NULL; 39 | """, 40 | ) 41 | ) 42 | class Decoration(models.Model): 43 | name = models.CharField(max_length=30, unique=True) 44 | souvenirs = models.ManyToManyField(Souvenir, related_name="decorations") 45 | -------------------------------------------------------------------------------- /app/rooms/models/room.py: -------------------------------------------------------------------------------- 1 | import pgtrigger 2 | from django.db import models 3 | 4 | from .common_info import CommonInfo 5 | from .decoration import Decoration 6 | from .door import Door 7 | from .rooms_related_v3 import RoomWithRelatedObjsV3 8 | 9 | 10 | @pgtrigger.register( 11 | pgtrigger.Trigger( 12 | name="room_v3_common_info", 13 | level=pgtrigger.Row, 14 | when=pgtrigger.After, 15 | operation=pgtrigger.Update | pgtrigger.Insert | pgtrigger.Delete, 16 | func=f""" 17 | INSERT INTO {RoomWithRelatedObjsV3._meta.db_table}(id, name, width, length, height, type) 18 | SELECT 19 | room.id id, 20 | room.name name, 21 | room.width width, 22 | room.length length, 23 | room.height height, 24 | room.type type 25 | FROM 26 | rooms_room room 27 | WHERE room.id = OLD.id 28 | ON CONFLICT (id) 29 | DO 30 | UPDATE SET name = EXCLUDED.name, 31 | width = EXCLUDED.width, 32 | length = EXCLUDED.length, 33 | height = EXCLUDED.height, 34 | type = EXCLUDED.type; 35 | RETURN NULL; 36 | """, 37 | ) 38 | ) 39 | class Room(CommonInfo): 40 | CHOICES = ( 41 | ("KTCH", "Kitchen"), 42 | ("BDR", "Bedroom"), 43 | ("LBB", "Lobby"), 44 | ("LRM", "Living Room"), 45 | ) 46 | type = models.CharField(max_length=5, choices=CHOICES) 47 | door = models.ForeignKey(Door, on_delete=models.PROTECT, related_name="rooms") 48 | decoration = models.ForeignKey(Decoration, null=True, on_delete=models.SET_NULL, related_name="rooms") 49 | -------------------------------------------------------------------------------- /app/rooms/models/souvenir.py: -------------------------------------------------------------------------------- 1 | import pgtrigger 2 | from django.db import models 3 | 4 | from .rooms_related_v3 import RoomWithRelatedObjsV3 5 | 6 | 7 | @pgtrigger.register( 8 | pgtrigger.Trigger( 9 | name="room_v3_souvenir", 10 | level=pgtrigger.Row, 11 | when=pgtrigger.After, 12 | operation=pgtrigger.Update | pgtrigger.Insert | pgtrigger.Delete, 13 | func=f""" 14 | INSERT INTO {RoomWithRelatedObjsV3._meta.db_table}(id, decoration) 15 | WITH decoraition_ids AS( 16 | SELECT decoration_id from rooms_decoration_souvenirs where souvenir_id=OLD.id 17 | ) 18 | SELECT 19 | room.id id, 20 | jsonb_build_object( 21 | 'id', 22 | decoration.id, 23 | 'name', 24 | decoration.name, 25 | 'souvenirs', 26 | jsonb_agg(souvenir) 27 | ) decoration 28 | FROM 29 | rooms_room room 30 | LEFT JOIN rooms_decoration decoration ON room.decoration_id = decoration.id 31 | LEFT JOIN rooms_decoration_souvenirs ds ON decoration.id = ds.decoration_id 32 | LEFT JOIN rooms_souvenir souvenir ON ds.souvenir_id = souvenir.id 33 | WHERE room.decoration_id IN (SELECT decoration_id from decoraition_ids) 34 | GROUP BY 35 | room.id, 36 | decoration.id 37 | ON CONFLICT (id) 38 | DO 39 | UPDATE SET decoration = EXCLUDED.decoration; 40 | RETURN NULL; 41 | """, 42 | ) 43 | ) 44 | class Souvenir(models.Model): 45 | name = models.CharField(max_length=30, unique=True) 46 | -------------------------------------------------------------------------------- /app/rooms/utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from django.apps import apps 4 | 5 | from rooms.models import Room 6 | from rooms.serializers import (DecorationSerializer, DoorSerializer, 7 | RoomSerializer) 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | 12 | def create_room_with_related_objs(room_id: int, model_name: str = "RoomWithRelatedObjsRebuildInApp") -> None: 13 | logger.info(msg="; create_room_with_related_objs: " + str(room_id)) 14 | 15 | # Make sure the given room ID is valid 16 | source_room = Room.objects.filter(id=room_id).first() 17 | if not source_room: 18 | logger.error(msg="; not found room with ID " + str(room_id)) 19 | return 20 | 21 | # All the work of building up-to-date data 22 | # about related objects is performed by the RoomSerializer, 23 | # which is called here. 24 | source_room_data = RoomSerializer(source_room).data 25 | model = apps.get_model("rooms", model_name) 26 | room_with_related_objs = model(id=room_id) 27 | 28 | # add door data 29 | door_data = DoorSerializer(source_room.door).data 30 | room_with_related_objs.door = door_data 31 | 32 | # add decoration (souvenirs set) data 33 | decoration_data = DecorationSerializer(source_room.decoration).data 34 | room_with_related_objs.decoration = decoration_data 35 | 36 | # add chairs, beds, and tables 37 | room_with_related_objs.chairs = source_room_data["chairs"] 38 | room_with_related_objs.beds = source_room_data["beds"] 39 | room_with_related_objs.tables = source_room_data["tables"] 40 | 41 | # add native parameters 42 | room_with_related_objs.name = source_room_data["name"] 43 | room_with_related_objs.width = source_room_data["width"] 44 | room_with_related_objs.length = source_room_data["length"] 45 | room_with_related_objs.height = source_room_data["height"] 46 | room_with_related_objs.type = source_room_data["type"] 47 | 48 | # add windows and window fittings 49 | room_with_related_objs.windows = source_room_data["windows"] 50 | 51 | room_with_related_objs.save() 52 | -------------------------------------------------------------------------------- /app/rooms/models/window.py: -------------------------------------------------------------------------------- 1 | import pgtrigger 2 | from django.db import models 3 | 4 | from .common_info import CommonInfo 5 | from .room import Room 6 | from .rooms_related_v3 import RoomWithRelatedObjsV3 7 | 8 | 9 | @pgtrigger.register( 10 | pgtrigger.Trigger( 11 | name="room_v3_windows", 12 | level=pgtrigger.Row, 13 | when=pgtrigger.After, 14 | operation=pgtrigger.Update | pgtrigger.Insert | pgtrigger.Delete, 15 | func=f""" 16 | INSERT INTO {RoomWithRelatedObjsV3._meta.db_table}(id, windows) 17 | WITH room_window_fittings AS ( 18 | SELECT 19 | room_id, 20 | json_build_object( 21 | 'id', 22 | win.id, 23 | 'name', 24 | win.name, 25 | 'width', 26 | win.width, 27 | 'length', 28 | win.length, 29 | 'height', 30 | win.height, 31 | 'type', 32 | win.type, 33 | 'fittings', 34 | jsonb_agg(wf) 35 | ) window_in_room 36 | FROM rooms_window win 37 | LEFT JOIN rooms_windowfittings_windows rwfw ON rwfw.window_id = win.id 38 | LEFT JOIN rooms_windowfittings wf ON wf.id = rwfw.windowfittings_id 39 | WHERE win.id = OLD.id 40 | GROUP BY room_id, win.id 41 | ) 42 | SELECT 43 | room_window_fittings.room_id AS id, 44 | jsonb_agg( 45 | room_window_fittings.window_in_room 46 | ) windows 47 | FROM room_window_fittings 48 | GROUP BY id 49 | ON CONFLICT (id) 50 | DO 51 | UPDATE SET windows = EXCLUDED.windows; 52 | RETURN NULL; 53 | """, 54 | ) 55 | ) 56 | class Window(CommonInfo): 57 | CHOICES = ( 58 | ("WT1", "Window Type 1"), 59 | ("WT2", "Window Type 2"), 60 | ("WT3", "Window Type 3"), 61 | ) 62 | type = models.CharField(max_length=5, choices=CHOICES) 63 | room = models.ForeignKey(Room, on_delete=models.CASCADE, related_name="windows") 64 | -------------------------------------------------------------------------------- /app/rooms/models/window_fittings.py: -------------------------------------------------------------------------------- 1 | import pgtrigger 2 | from django.db import models 3 | 4 | from .common_info import CommonInfo 5 | from .rooms_related_v3 import RoomWithRelatedObjsV3 6 | from .window import Window 7 | 8 | 9 | @pgtrigger.register( 10 | pgtrigger.Trigger( 11 | name="room_v3_window_fittings", 12 | level=pgtrigger.Row, 13 | when=pgtrigger.After, 14 | operation=pgtrigger.Update | pgtrigger.Insert | pgtrigger.Delete, 15 | func=f""" 16 | INSERT INTO {RoomWithRelatedObjsV3._meta.db_table}(id, windows) 17 | WITH room_ids AS ( 18 | SELECT 19 | win.room_id 20 | FROM rooms_window win 21 | LEFT JOIN rooms_windowfittings_windows rwfw ON rwfw.window_id = win.id 22 | LEFT JOIN rooms_windowfittings wf ON wf.id = rwfw.windowfittings_id 23 | WHERE wf.id = OLD.id 24 | ), 25 | room_window_fittings AS ( 26 | SELECT 27 | room_id, 28 | json_build_object( 29 | 'id', 30 | win.id, 31 | 'name', 32 | win.name, 33 | 'width', 34 | win.width, 35 | 'length', 36 | win.length, 37 | 'height', 38 | win.height, 39 | 'type', 40 | win.type, 41 | 'fittings', 42 | jsonb_agg(wf) 43 | ) window_in_room 44 | FROM rooms_window win 45 | LEFT JOIN rooms_windowfittings_windows rwfw ON rwfw.window_id = win.id 46 | LEFT JOIN rooms_windowfittings wf ON wf.id = rwfw.windowfittings_id 47 | WHERE win.room_id IN (SELECT room_id FROM room_ids) 48 | GROUP BY room_id, win.id 49 | ) 50 | SELECT 51 | room_window_fittings.room_id AS id, 52 | jsonb_agg( 53 | room_window_fittings.window_in_room 54 | ) windows 55 | FROM room_window_fittings 56 | GROUP BY id 57 | ON CONFLICT (id) 58 | DO 59 | UPDATE SET windows = EXCLUDED.windows; 60 | RETURN NULL; 61 | """, 62 | ) 63 | ) 64 | class WindowFittings(CommonInfo): 65 | WF_CHOICES = ( 66 | ("WFT1", "Window Fittings Type 1"), 67 | ("WFT2", "Window Fittings Type 2"), 68 | ("WFT3", "Window Fittings Type 3"), 69 | ) 70 | type = models.CharField(max_length=5, choices=WF_CHOICES) 71 | windows = models.ManyToManyField(Window, related_name="fittings") 72 | -------------------------------------------------------------------------------- /app/rooms/views.py: -------------------------------------------------------------------------------- 1 | from rest_framework import viewsets 2 | 3 | from rooms.models import (Bed, Chair, Decoration, Door, Room, 4 | RoomsRelatedObjectsMaterializedView, 5 | RoomWithRelatedObjsRebuildInApp, 6 | RoomWithRelatedObjsV3, Souvenir, Table, Window, 7 | WindowFittings) 8 | from rooms.serializers import (BedSerializer, ChairSerializer, 9 | DecorationSerializer, DoorSerializer, 10 | RoomSerializer, RoomsRelatedObjectsSerializer, 11 | RoomsRelatedV2Serializer, 12 | RoomsRelatedV3Serializer, SouvenirSerializer, 13 | TableSerializer, WindowFittingsSerializer, 14 | WindowSerializer) 15 | 16 | 17 | class RoomViewSet(viewsets.ModelViewSet): 18 | serializer_class = RoomSerializer 19 | 20 | def get_queryset(self): 21 | qs = Room.objects.all() 22 | qs = self.serializer_class.setup_eager_loading(qs) 23 | return qs 24 | 25 | 26 | class RoomsRelatedObjectsViewSet(viewsets.ReadOnlyModelViewSet): 27 | serializer_class = RoomsRelatedObjectsSerializer 28 | queryset = RoomsRelatedObjectsMaterializedView.objects.all() 29 | 30 | 31 | class RoomsRelatedV2ViewSet(viewsets.ReadOnlyModelViewSet): 32 | serializer_class = RoomsRelatedV2Serializer 33 | queryset = RoomWithRelatedObjsRebuildInApp.objects.all() 34 | 35 | 36 | class RoomsRelatedV3ViewSet(viewsets.ReadOnlyModelViewSet): 37 | serializer_class = RoomsRelatedV3Serializer 38 | queryset = RoomWithRelatedObjsV3.objects.all() 39 | 40 | 41 | # The following classes are to test 42 | # if the post_save signal works, 43 | # i.e. PostgreSQL view is updated automatically 44 | # after the model instance changes 45 | 46 | 47 | class WindowViewSet(viewsets.ModelViewSet): 48 | serializer_class = WindowSerializer 49 | 50 | def get_queryset(self): 51 | qs = Window.objects.all() 52 | qs = self.serializer_class.setup_eager_loading(qs) 53 | return qs 54 | 55 | 56 | class SouvenirViewSet(viewsets.ModelViewSet): 57 | serializer_class = SouvenirSerializer 58 | queryset = Souvenir.objects.all() 59 | 60 | 61 | class TableViewSet(viewsets.ModelViewSet): 62 | serializer_class = TableSerializer 63 | queryset = Table.objects.all() 64 | 65 | 66 | class DecorationViewSet(viewsets.ModelViewSet): 67 | serializer_class = DecorationSerializer 68 | queryset = Decoration.objects.all() 69 | 70 | 71 | class DoorViewSet(viewsets.ModelViewSet): 72 | serializer_class = DoorSerializer 73 | queryset = Door.objects.all() 74 | 75 | 76 | class ChairViewSet(viewsets.ModelViewSet): 77 | serializer_class = ChairSerializer 78 | queryset = Chair.objects.all() 79 | 80 | 81 | class BedViewSet(viewsets.ModelViewSet): 82 | serializer_class = BedSerializer 83 | queryset = Bed.objects.all() 84 | 85 | 86 | class WindowFittingsViewSet(viewsets.ModelViewSet): 87 | serializer_class = WindowFittingsSerializer 88 | queryset = WindowFittings.objects.all() 89 | -------------------------------------------------------------------------------- /supplementary_scripts/create_view.sql: -------------------------------------------------------------------------------- 1 | CREATE materialized VIEW rooms_related_objects AS WITH room_id_door_decoration AS ( 2 | SELECT 3 | room.id id, 4 | jsonb_build_object( 5 | 'id', door.id, 'name', door.name, 'width', 6 | door.width, 'length', door.length, 7 | 'height', door.height, 'type', door.type 8 | ) door, 9 | jsonb_build_object( 10 | 'id', 11 | decoration.id, 12 | 'name', 13 | decoration.name, 14 | 'souvenirs', 15 | jsonb_agg(souvenir) 16 | ) decoration 17 | FROM 18 | rooms_room room 19 | LEFT JOIN rooms_door door ON room.door_id = door.id 20 | LEFT JOIN rooms_decoration decoration ON room.decoration_id = decoration.id 21 | LEFT JOIN rooms_decoration_souvenirs ds ON decoration.id = ds.decoration_id 22 | LEFT JOIN rooms_souvenir souvenir ON ds.souvenir_id = souvenir.id 23 | GROUP BY 24 | room.id, 25 | door.id, 26 | decoration.id 27 | ), 28 | windows_in_room_by_id AS ( 29 | WITH room_window_fittings AS ( 30 | SELECT 31 | room_id, 32 | json_build_object( 33 | 'id', 34 | win.id, 35 | 'name', 36 | win.name, 37 | 'width', 38 | win.width, 39 | 'length', 40 | win.length, 41 | 'height', 42 | win.height, 43 | 'type', 44 | win.type, 45 | 'fittings', 46 | jsonb_agg(wf) 47 | ) window_in_room 48 | FROM 49 | rooms_window win 50 | LEFT JOIN rooms_windowfittings_windows rwfw ON rwfw.window_id = win.id 51 | LEFT JOIN rooms_windowfittings wf ON wf.id = rwfw.windowfittings_id 52 | GROUP BY 53 | room_id, 54 | win.id 55 | ) 56 | SELECT 57 | room_window_fittings.room_id AS id, 58 | jsonb_agg( 59 | room_window_fittings.window_in_room 60 | ) windows 61 | FROM 62 | room_window_fittings 63 | GROUP BY 64 | id 65 | ), 66 | parameters_and_beds_in_room_by_id AS ( 67 | SELECT 68 | room.id id, 69 | room.name name, 70 | room.width width, 71 | room.length length, 72 | room.height height, 73 | room.type type, 74 | jsonb_agg(bed) beds 75 | FROM 76 | rooms_room room 77 | LEFT JOIN rooms_bed_rooms bed_i ON room.id = bed_i.room_id 78 | LEFT JOIN rooms_bed bed ON bed.id = bed_i.bed_id 79 | GROUP BY 80 | room.id 81 | ), 82 | chairs_in_room_by_id AS ( 83 | SELECT 84 | room.id id, 85 | jsonb_agg(chair) chairs 86 | FROM 87 | rooms_room room 88 | LEFT JOIN rooms_chair_rooms chair_i ON room.id = chair_i.room_id 89 | LEFT JOIN rooms_chair chair ON chair.id = chair_i.chair_id 90 | GROUP BY 91 | room.id 92 | ), 93 | tables_in_room_by_id AS ( 94 | SELECT 95 | room.id id, 96 | jsonb_agg(table_f) tables 97 | FROM 98 | rooms_room room 99 | LEFT JOIN rooms_table_rooms table_i ON room.id = table_i.room_id 100 | LEFT JOIN rooms_table table_f ON table_f.id = table_i.table_id 101 | GROUP BY 102 | room.id 103 | ) 104 | SELECT 105 | COALESCE( 106 | room_id_door_decoration.id, windows_in_room_by_id.id 107 | ) AS id, 108 | room_id_door_decoration.door, 109 | room_id_door_decoration.decoration, 110 | windows_in_room_by_id.windows, 111 | parameters_and_beds_in_room_by_id.name, 112 | parameters_and_beds_in_room_by_id.width, 113 | parameters_and_beds_in_room_by_id.length, 114 | parameters_and_beds_in_room_by_id.height, 115 | parameters_and_beds_in_room_by_id.type, 116 | parameters_and_beds_in_room_by_id.beds, 117 | chairs_in_room_by_id.chairs, 118 | tables_in_room_by_id.tables 119 | FROM 120 | room_id_door_decoration FULL 121 | OUTER JOIN windows_in_room_by_id ON room_id_door_decoration.id = windows_in_room_by_id.id FULL 122 | OUTER JOIN parameters_and_beds_in_room_by_id ON windows_in_room_by_id.id = parameters_and_beds_in_room_by_id.id FULL 123 | OUTER JOIN chairs_in_room_by_id ON parameters_and_beds_in_room_by_id.id = chairs_in_room_by_id.id FULL 124 | OUTER JOIN tables_in_room_by_id ON chairs_in_room_by_id.id = tables_in_room_by_id.id WITH DATA; 125 | CREATE UNIQUE INDEX ON rooms_related_objects (id); 126 | REFRESH MATERIALIZED VIEW CONCURRENTLY rooms_related_objects; -------------------------------------------------------------------------------- /supplementary_scripts/compare_requests_time.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import asyncio 3 | import io 4 | import random 5 | from pathlib import Path 6 | from typing import List 7 | 8 | import aiohttp 9 | import pandas as pd 10 | import psycopg2 11 | from aiohttp import ClientSession 12 | 13 | LOG_FILE_PATH = Path(__file__).parent.parent / "app/logs_all_here.log" 14 | DB_CONNECTION_DATA = ( 15 | "dbname='postgres' user='postgres' host='localhost' password='postgres'" 16 | ) 17 | URL_NATIVE_STUB = "http://localhost:8000/rooms_native/" 18 | URL_MAT_VIEW_STUB = "http://localhost:8000/rooms_mat_view/" 19 | URL_SIGNALS_STUB = "http://localhost:8000/rooms_v2/" 20 | URL_TRIGGERS_STUB = "http://localhost:8000/rooms_v3/" 21 | 22 | 23 | def _obtain_rooms_ids() -> List: 24 | conn = psycopg2.connect(DB_CONNECTION_DATA) 25 | db_cursor = conn.cursor() 26 | query_str = "SELECT id FROM public.rooms_room" 27 | db_cursor.execute(query_str) 28 | ids_obtained = db_cursor.fetchall() 29 | conn.close() 30 | room_ids = [] 31 | for elem in ids_obtained: 32 | room_ids.append(elem[0]) 33 | print("Obtained list of all room IDs - OK...") 34 | return room_ids 35 | 36 | 37 | async def _make_get_request_to_url(session: ClientSession, url: str) -> None: 38 | async with session.get(url): 39 | pass 40 | 41 | 42 | async def _make_requests_to_all_rooms( 43 | rooms_ids: List, iterations_count: int 44 | ) -> None: 45 | async with aiohttp.ClientSession() as session: 46 | for i in range(iterations_count): 47 | room_id = random.choice(rooms_ids) 48 | all_urls = [ 49 | URL_NATIVE_STUB + str(room_id), 50 | URL_MAT_VIEW_STUB + str(room_id), 51 | URL_SIGNALS_STUB + str(room_id), 52 | URL_TRIGGERS_STUB + str(room_id), 53 | ] 54 | request_tasks = [ 55 | _make_get_request_to_url(session, url) for url in all_urls 56 | ] 57 | await asyncio.gather(*request_tasks) 58 | print("Request iteration", i + 1, "of", iterations_count) 59 | 60 | 61 | def _log_lines_stage_1_determine_type() -> List: 62 | 63 | with open(LOG_FILE_PATH) as file: 64 | lines = file.read().splitlines() 65 | 66 | lines_processed = [] 67 | for line in lines: 68 | _, _, end = line.partition("/rooms_") 69 | if not end: 70 | continue 71 | if end.startswith("mat_view/"): 72 | line = line + "; VIEW" 73 | lines_processed.append(line) 74 | continue 75 | if end.startswith("native/"): 76 | line = line + "; NATIVE_ROOM" 77 | lines_processed.append(line) 78 | continue 79 | if end.startswith("v2/"): 80 | line = line + "; SIGNALS" 81 | lines_processed.append(line) 82 | continue 83 | if end.startswith("v3/"): 84 | line = line + "; TRIGGERS" 85 | lines_processed.append(line) 86 | continue 87 | line = line + "; OTHER" 88 | lines_processed.append(line) 89 | return lines_processed 90 | 91 | 92 | def _log_lines_stage_2_get_avg_time(log_lines: List) -> None: 93 | df = pd.read_csv( 94 | io.StringIO("\n".join(log_lines)), 95 | delimiter=";", 96 | delim_whitespace=False, 97 | header=None, 98 | ) 99 | df.columns = ["MsgType", "Path", "ExecutionTime", "Type"] 100 | print("Number of rows (log records) - ", df.shape[0]) 101 | print() 102 | print("Mean execution time by request type:") 103 | print(df.groupby("Type")["ExecutionTime"].mean()) 104 | 105 | 106 | if __name__ == "__main__": 107 | parser = argparse.ArgumentParser() 108 | parser.add_argument( 109 | "--count", 110 | type=int, 111 | help="how many rooms data to request, default 20", 112 | nargs="?", 113 | default=20, 114 | const=20, 115 | ) 116 | args = parser.parse_args() 117 | room_ids = _obtain_rooms_ids() 118 | print("Start making requests to log time...") 119 | asyncio.run(_make_requests_to_all_rooms(room_ids, args.count)) 120 | log_lines_with_types = _log_lines_stage_1_determine_type() 121 | _log_lines_stage_2_get_avg_time(log_lines_with_types) 122 | -------------------------------------------------------------------------------- /app/rooms/models/furniture.py: -------------------------------------------------------------------------------- 1 | import pgtrigger 2 | from django.db import models 3 | 4 | from .common_info import CommonInfo 5 | from .room import Room 6 | from .rooms_related_v3 import RoomWithRelatedObjsV3 7 | 8 | 9 | @pgtrigger.register( 10 | pgtrigger.Trigger( 11 | name="room_v3_chair", 12 | level=pgtrigger.Row, 13 | when=pgtrigger.After, 14 | operation=pgtrigger.Update | pgtrigger.Insert | pgtrigger.Delete, 15 | func=f""" 16 | INSERT INTO {RoomWithRelatedObjsV3._meta.db_table}(id, chairs) 17 | WITH room_ids AS ( 18 | SELECT 19 | rooms_chair_rooms.room_id 20 | FROM rooms_chair 21 | LEFT JOIN rooms_chair_rooms ON rooms_chair_rooms.chair_id = rooms_chair.id 22 | WHERE rooms_chair.id = OLD.id 23 | ) 24 | SELECT 25 | room.id id, 26 | jsonb_agg(chair) chairs 27 | FROM rooms_room room 28 | LEFT JOIN rooms_chair_rooms chair_i ON room.id = chair_i.room_id 29 | LEFT JOIN rooms_chair chair ON chair.id = chair_i.chair_id 30 | WHERE room.id IN (SELECT room_id FROM room_ids) 31 | GROUP BY room.id 32 | ON CONFLICT (id) 33 | DO 34 | UPDATE SET chairs = EXCLUDED.chairs; 35 | RETURN NULL; 36 | """, 37 | ) 38 | ) 39 | class Chair(CommonInfo): 40 | CHAIR_TYPE_CHOICES = ( 41 | ("CHT1", "Chair Type 1"), 42 | ("CHT2", "Chair Type 2"), 43 | ("CHT3", "Chair Type 3"), 44 | ) 45 | type = models.CharField(max_length=5, choices=CHAIR_TYPE_CHOICES) 46 | rooms = models.ManyToManyField(Room, related_name="chairs") 47 | 48 | 49 | @pgtrigger.register( 50 | pgtrigger.Trigger( 51 | name="room_v3_bed", 52 | level=pgtrigger.Row, 53 | when=pgtrigger.After, 54 | operation=pgtrigger.Update | pgtrigger.Insert | pgtrigger.Delete, 55 | func=f""" 56 | INSERT INTO {RoomWithRelatedObjsV3._meta.db_table}(id, beds) 57 | WITH room_ids AS ( 58 | SELECT 59 | rooms_bed_rooms.room_id 60 | FROM rooms_bed 61 | LEFT JOIN rooms_bed_rooms ON rooms_bed_rooms.bed_id = rooms_bed.id 62 | WHERE rooms_bed.id = OLD.id 63 | ) 64 | SELECT 65 | room.id id, 66 | jsonb_agg(bed) beds 67 | FROM rooms_room room 68 | LEFT JOIN rooms_bed_rooms bed_i ON room.id = bed_i.room_id 69 | LEFT JOIN rooms_bed bed ON bed.id = bed_i.bed_id 70 | WHERE room.id IN (SELECT room_id FROM room_ids) 71 | GROUP BY room.id 72 | ON CONFLICT (id) 73 | DO 74 | UPDATE SET beds = EXCLUDED.beds; 75 | RETURN NULL; 76 | """, 77 | ) 78 | ) 79 | class Bed(CommonInfo): 80 | BED_TYPE_CHOICES = ( 81 | ("BT1", "Bed Type 1"), 82 | ("BT2", "Bed Type 2"), 83 | ("BT3", "Bed Type 3"), 84 | ) 85 | type = models.CharField(max_length=5, choices=BED_TYPE_CHOICES) 86 | rooms = models.ManyToManyField(Room, related_name="beds") 87 | 88 | 89 | @pgtrigger.register( 90 | pgtrigger.Trigger( 91 | name="room_v3_table", 92 | level=pgtrigger.Row, 93 | when=pgtrigger.After, 94 | operation=pgtrigger.Update | pgtrigger.Insert | pgtrigger.Delete, 95 | func=f""" 96 | INSERT INTO {RoomWithRelatedObjsV3._meta.db_table}(id, tables) 97 | WITH room_ids AS ( 98 | SELECT 99 | rooms_table_rooms.room_id 100 | FROM rooms_table 101 | LEFT JOIN rooms_table_rooms ON rooms_table_rooms.table_id = rooms_table.id 102 | WHERE rooms_table.id = OLD.id 103 | ) 104 | SELECT 105 | room.id id, 106 | jsonb_agg(table_model) tables 107 | FROM rooms_room room 108 | LEFT JOIN rooms_table_rooms table_i ON room.id = table_i.room_id 109 | LEFT JOIN rooms_table table_model ON table_model.id = table_i.table_id 110 | WHERE room.id IN (SELECT room_id FROM room_ids) 111 | GROUP BY room.id 112 | ON CONFLICT (id) 113 | DO 114 | UPDATE SET tables = EXCLUDED.tables; 115 | RETURN NULL; 116 | """, 117 | ) 118 | ) 119 | class Table(CommonInfo): 120 | TABLE_TYPE_CHOICES = ( 121 | ("TBLT1", "Table Type 1"), 122 | ("TBLT2", "Table Type 2"), 123 | ("TBLT3", "Table Type 3"), 124 | ) 125 | type = models.CharField(max_length=5, choices=TABLE_TYPE_CHOICES) 126 | rooms = models.ManyToManyField(Room, related_name="tables") 127 | -------------------------------------------------------------------------------- /app/app/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for app project. 3 | 4 | Generated by 'django-admin startproject' using Django 4.0.3. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/4.0/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/4.0/ref/settings/ 11 | """ 12 | 13 | import os 14 | from pathlib import Path 15 | from typing import List 16 | 17 | # Build paths inside the project like this: BASE_DIR / 'subdir'. 18 | BASE_DIR = Path(__file__).resolve().parent.parent 19 | 20 | 21 | # Quick-start development settings - unsuitable for production 22 | # See https://docs.djangoproject.com/en/4.0/howto/deployment/checklist/ 23 | 24 | # SECURITY WARNING: keep the secret key used in production secret! 25 | SECRET_KEY = "django-insecure-1@5u5k283!&!)ibd1inm(!x6ici@et91$5mk4bd5x!v@tid!o_" 26 | 27 | # SECURITY WARNING: don't run with debug turned on in production! 28 | DEBUG = True 29 | 30 | ALLOWED_HOSTS: List[str] = [] 31 | 32 | 33 | # Application definition 34 | 35 | INSTALLED_APPS = [ 36 | "django.contrib.admin", 37 | "django.contrib.auth", 38 | "django.contrib.contenttypes", 39 | "django.contrib.sessions", 40 | "django.contrib.messages", 41 | "django.contrib.staticfiles", 42 | "rest_framework", 43 | "rooms", 44 | "pgtrigger", 45 | ] 46 | 47 | REST_FRAMEWORK = {"DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.PageNumberPagination", "PAGE_SIZE": 10} 48 | 49 | MIDDLEWARE = [ 50 | "django.middleware.security.SecurityMiddleware", 51 | "django.contrib.sessions.middleware.SessionMiddleware", 52 | "django.middleware.common.CommonMiddleware", 53 | "django.middleware.csrf.CsrfViewMiddleware", 54 | "django.contrib.auth.middleware.AuthenticationMiddleware", 55 | "django.contrib.messages.middleware.MessageMiddleware", 56 | "django.middleware.clickjacking.XFrameOptionsMiddleware", 57 | "rooms.middleware.log_execution_time.ExecutionTimeLogMiddleware", 58 | ] 59 | 60 | ROOT_URLCONF = "app.urls" 61 | 62 | TEMPLATES = [ 63 | { 64 | "BACKEND": "django.template.backends.django.DjangoTemplates", 65 | "DIRS": [], 66 | "APP_DIRS": True, 67 | "OPTIONS": { 68 | "context_processors": [ 69 | "django.template.context_processors.debug", 70 | "django.template.context_processors.request", 71 | "django.contrib.auth.context_processors.auth", 72 | "django.contrib.messages.context_processors.messages", 73 | ], 74 | }, 75 | }, 76 | ] 77 | 78 | WSGI_APPLICATION = "app.wsgi.application" 79 | 80 | 81 | # Database 82 | # https://docs.djangoproject.com/en/4.0/ref/settings/#databases 83 | 84 | DATABASES = { 85 | "default": { 86 | "ENGINE": "django.db.backends.postgresql", 87 | "NAME": os.environ.get("POSTGRES_NAME"), 88 | "USER": os.environ.get("POSTGRES_USER"), 89 | "PASSWORD": os.environ.get("POSTGRES_PASSWORD"), 90 | "HOST": "db", 91 | "PORT": 5432, 92 | } 93 | } 94 | 95 | LOGGING = { 96 | "version": 1, # the dictConfig format version 97 | "disable_existing_loggers": False, # retain the default loggers 98 | "handlers": { 99 | "file": { 100 | "class": "logging.FileHandler", 101 | "filename": "logs_all_here.log", 102 | "formatter": "simple", 103 | 'mode': 'w', 104 | }, 105 | }, 106 | "loggers": { 107 | "": { 108 | "level": "DEBUG", 109 | "handlers": ["file"], 110 | }, 111 | }, 112 | "formatters": { 113 | "verbose": { 114 | "format": "{name} {levelname} {asctime} {module} {process:d} {thread:d} {message}", 115 | "style": "{", 116 | }, 117 | "simple": { 118 | "format": "{levelname} {name} {message}", 119 | "style": "{", 120 | }, 121 | }, 122 | } 123 | 124 | 125 | # Password validation 126 | # https://docs.djangoproject.com/en/4.0/ref/settings/#auth-password-validators 127 | 128 | AUTH_PASSWORD_VALIDATORS = [ 129 | { 130 | "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", 131 | }, 132 | { 133 | "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", 134 | }, 135 | { 136 | "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", 137 | }, 138 | { 139 | "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", 140 | }, 141 | ] 142 | 143 | 144 | # Internationalization 145 | # https://docs.djangoproject.com/en/4.0/topics/i18n/ 146 | 147 | LANGUAGE_CODE = "en-us" 148 | 149 | TIME_ZONE = "UTC" 150 | 151 | USE_I18N = True 152 | 153 | USE_TZ = True 154 | 155 | 156 | # Static files (CSS, JavaScript, Images) 157 | # https://docs.djangoproject.com/en/4.0/howto/static-files/ 158 | 159 | STATIC_URL = "static/" 160 | 161 | # Default primary key field type 162 | # https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field 163 | 164 | DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" 165 | -------------------------------------------------------------------------------- /app/rooms/management/commands/fill_db.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | from django.core.management.base import BaseCommand 4 | from django.db import connection 5 | from model_bakery import baker 6 | 7 | from rooms.models import (Bed, Chair, Decoration, Door, Room, 8 | RoomWithRelatedObjsRebuildInApp, 9 | RoomWithRelatedObjsV3, Souvenir, Table, Window, 10 | WindowFittings) 11 | from rooms.utils import create_room_with_related_objs 12 | 13 | 14 | class Command(BaseCommand): 15 | help = "Fill the DB with fake data about rooms, their decorations and furniture." 16 | 17 | def add_arguments(self, parser): 18 | parser.add_argument( 19 | "items_quantity", type=int, help="How many instances to create: rooms, windows, chairs, beds etc." 20 | ) 21 | 22 | def handle(self, *args, **kwargs): 23 | items_quantity = kwargs["items_quantity"] 24 | self._remove_all_items_from_db() 25 | self._create_supplementary_objects(how_many=items_quantity) 26 | souvenirs_all = list(Souvenir.objects.all()) 27 | doors_all = list(Door.objects.all()) 28 | self._create_rooms_and_windows(doors_all, souvenirs_all, how_many=items_quantity) 29 | chairs_all = list(Chair.objects.all()) 30 | beds_all = list(Bed.objects.all()) 31 | tables_all = list(Table.objects.all()) 32 | win_fittings_all = list(WindowFittings.objects.all()) 33 | rooms = list(Room.objects.all()) 34 | windows = list(Window.objects.all()) 35 | self._create_m2m_for_rooms_and_windows( 36 | rooms, windows, chairs_all, beds_all, tables_all, win_fittings_all, how_many=items_quantity 37 | ) 38 | for i in range(items_quantity): 39 | create_room_with_related_objs(rooms[i].id) 40 | self.stdout.write( 41 | self.style.SUCCESS("Created RoomWithRelatedObjsRebuildInApp %s of %s - OK" % (i + 1, items_quantity)) 42 | ) 43 | with connection.cursor() as cursor: 44 | cursor.execute( 45 | """INSERT INTO rooms_roomwithrelatedobjsv3 46 | SELECT * 47 | FROM rooms_related_objects 48 | ON CONFLICT (ID) 49 | DO UPDATE 50 | SET door = EXCLUDED.door, 51 | decoration = EXCLUDED.decoration, 52 | windows = EXCLUDED.windows, 53 | name = EXCLUDED.name, 54 | width = EXCLUDED.width, 55 | length = EXCLUDED.length, 56 | height = EXCLUDED.height, 57 | type = EXCLUDED.type, 58 | beds = EXCLUDED.beds, 59 | chairs = EXCLUDED.chairs, 60 | tables = EXCLUDED.tables;""" 61 | ) 62 | 63 | def _remove_all_items_from_db(self): 64 | RoomWithRelatedObjsRebuildInApp.objects.all().delete() 65 | RoomWithRelatedObjsV3.objects.all().delete() 66 | Room.objects.all().delete() 67 | Door.objects.all().delete() 68 | Souvenir.objects.all().delete() 69 | Decoration.objects.all().delete() 70 | WindowFittings.objects.all().delete() 71 | Window.objects.all().delete() 72 | Chair.objects.all().delete() 73 | Bed.objects.all().delete() 74 | Table.objects.all().delete() 75 | self.stdout.write(self.style.SUCCESS("Initial data deletion - OK")) 76 | 77 | def _create_supplementary_objects(self, how_many=500): 78 | baker.make(Souvenir, _quantity=how_many) 79 | baker.make(Door, _quantity=how_many) 80 | baker.make(Chair, _quantity=how_many) 81 | baker.make(Bed, _quantity=how_many) 82 | baker.make(Table, _quantity=how_many) 83 | baker.make(WindowFittings, _quantity=how_many) 84 | self.stdout.write(self.style.SUCCESS("Created souvenirs, doors and furniture - OK")) 85 | 86 | def _create_rooms_and_windows(self, doors_list, souvenirs_list, how_many=500): 87 | for i in range(how_many): 88 | door_prepared = random.choice(doors_list) 89 | souvenirs_sample = random.sample(souvenirs_list, 5) 90 | decoration_prepared = baker.make(Decoration, souvenirs=souvenirs_sample) 91 | room = baker.make(Room, decoration=decoration_prepared, door=door_prepared) 92 | baker.make(Window, _quantity=2, room=room) 93 | self.stdout.write(self.style.SUCCESS("Created room and windows %s of %s - OK" % (i + 1, how_many))) 94 | 95 | def _create_m2m_for_rooms_and_windows(self, rooms_all, windows_all, chairs, beds, tables, fittings, how_many=500): 96 | for i in range(how_many): 97 | rooms_items = random.sample(rooms_all, 5) 98 | windows_items = random.sample(windows_all, 5) 99 | chairs[i].rooms.add(*rooms_items) 100 | beds[i].rooms.add(*rooms_items) 101 | tables[i].rooms.add(*rooms_items) 102 | fittings[i].windows.add(*windows_items) 103 | self.stdout.write( 104 | self.style.SUCCESS("Created relations for rooms and windows %s of %s - OK" % (i + 1, how_many)) 105 | ) 106 | -------------------------------------------------------------------------------- /app/rooms/serializers.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from rest_framework import serializers 4 | 5 | from rooms.models import (Bed, Chair, Decoration, Door, Room, 6 | RoomsRelatedObjectsMaterializedView, 7 | RoomWithRelatedObjsRebuildInApp, 8 | RoomWithRelatedObjsV3, Souvenir, Table, Window, 9 | WindowFittings) 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | class EagerLoadingMixin: 15 | @classmethod 16 | def setup_eager_loading(cls, queryset): 17 | """ 18 | This function allow dynamic addition of the related objects to 19 | the provided query. 20 | @parameter param1: queryset 21 | """ 22 | 23 | if hasattr(cls, "select_related_fields"): 24 | queryset = queryset.select_related(*cls.select_related_fields) 25 | if hasattr(cls, "prefetch_related_fields"): 26 | queryset = queryset.prefetch_related(*cls.prefetch_related_fields) 27 | return queryset 28 | 29 | 30 | class WindowFittingsSerializer(serializers.ModelSerializer): 31 | class Meta: 32 | model = WindowFittings 33 | fields = "__all__" 34 | extra_kwargs = { 35 | "id": {"read_only": False}, 36 | "name": {"validators": []}, 37 | } 38 | 39 | 40 | class WindowSerializer(serializers.ModelSerializer, EagerLoadingMixin): 41 | fittings = WindowFittingsSerializer(many=True, read_only=True) 42 | select_related_fields = () 43 | # Only necessary if you have fields to prefetch 44 | prefetch_related_fields = ("fittings",) 45 | 46 | class Meta: 47 | model = Window 48 | fields = "__all__" 49 | 50 | 51 | class BedSerializer(serializers.ModelSerializer): 52 | class Meta: 53 | model = Bed 54 | fields = "__all__" 55 | 56 | 57 | class TableSerializer(serializers.ModelSerializer): 58 | class Meta: 59 | model = Table 60 | fields = "__all__" 61 | 62 | 63 | class ChairSerializer(serializers.ModelSerializer): 64 | class Meta: 65 | model = Chair 66 | fields = "__all__" 67 | 68 | 69 | class DoorSerializer(serializers.ModelSerializer): 70 | class Meta: 71 | model = Door 72 | fields = "__all__" 73 | 74 | 75 | class SouvenirSerializer(serializers.ModelSerializer): 76 | class Meta: 77 | model = Souvenir 78 | fields = "__all__" 79 | extra_kwargs = { 80 | "id": {"read_only": False}, 81 | "name": {"validators": []}, 82 | } 83 | 84 | 85 | class DecorationSerializer(serializers.ModelSerializer, EagerLoadingMixin): 86 | souvenirs = SouvenirSerializer(many=True) 87 | select_related_fields = () 88 | prefetch_related_fields = ("souvenirs",) 89 | 90 | def get_or_create_souvenirs(self, souvenirs): 91 | logger.info(msg="; DecorationSerializer get_or_create_souvenirs...") 92 | souvenir_ids = [] 93 | for souvenir in souvenirs: 94 | souvenir_instance, created = Souvenir.objects.get_or_create(pk=souvenir.get("id"), defaults=souvenir) 95 | souvenir_ids.append(souvenir_instance.pk) 96 | return souvenir_ids 97 | 98 | def create_or_update_souvenirs(self, souvenirs): 99 | logger.info(msg="; DecorationSerializer create_or_update_souvenirs...") 100 | souvenir_ids = [] 101 | for souvenir in souvenirs: 102 | souvenir_instance, created = Souvenir.objects.update_or_create(pk=souvenir.get("id"), defaults=souvenir) 103 | souvenir_ids.append(souvenir_instance.pk) 104 | return souvenir_ids 105 | 106 | def create(self, validated_data): 107 | logger.info(msg="; DecorationSerializer create...") 108 | souvenirs = validated_data.pop("souvenirs", []) 109 | decoration = Decoration.objects.create(**validated_data) 110 | decoration.souvenirs.set(self.get_or_create_souvenirs(souvenirs)) 111 | return decoration 112 | 113 | def update(self, instance, validated_data): 114 | logger.info(msg="; DecorationSerializer update...") 115 | fields = ["name"] 116 | for field in fields: 117 | try: 118 | setattr(instance, field, validated_data[field]) 119 | except KeyError: # validated_data may not contain all fields during HTTP PATCH 120 | pass 121 | instance.save() 122 | return instance 123 | 124 | class Meta: 125 | model = Decoration 126 | fields = "__all__" 127 | 128 | 129 | class RoomSerializer(serializers.ModelSerializer, EagerLoadingMixin): 130 | door = DoorSerializer(many=False, read_only=True) 131 | decoration = DecorationSerializer(many=False, read_only=True) 132 | windows = WindowSerializer(many=True, read_only=True) 133 | chairs = ChairSerializer(many=True, read_only=True) 134 | beds = BedSerializer(many=True, read_only=True) 135 | tables = TableSerializer(many=True, read_only=True) 136 | select_related_fields = ("door", "decoration") 137 | prefetch_related_fields = ( 138 | "windows", 139 | "chairs", 140 | "beds", 141 | "tables", 142 | ) 143 | 144 | class Meta: 145 | model = Room 146 | fields = "__all__" 147 | 148 | 149 | class RoomsRelatedObjectsSerializer(serializers.ModelSerializer): 150 | class Meta: 151 | model = RoomsRelatedObjectsMaterializedView 152 | fields = "__all__" 153 | 154 | 155 | class RoomsRelatedV2Serializer(serializers.ModelSerializer): 156 | class Meta: 157 | model = RoomWithRelatedObjsRebuildInApp 158 | fields = "__all__" 159 | 160 | 161 | class RoomsRelatedV3Serializer(serializers.ModelSerializer): 162 | class Meta: 163 | model = RoomWithRelatedObjsV3 164 | fields = "__all__" 165 | -------------------------------------------------------------------------------- /app/rooms/migrations/0004_roomsrelatedobjects.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.3 on 2022-04-01 10:18 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('rooms', '0003_alter_bed_rooms_alter_chair_rooms_and_more'), 10 | ] 11 | 12 | operations = [ 13 | migrations.CreateModel( 14 | name='RoomsRelatedObjects', 15 | fields=[ 16 | ('id', models.BigIntegerField(primary_key=True, serialize=False)), 17 | ('door', models.JSONField(blank=True, null=True)), 18 | ('decoration', models.JSONField(blank=True, null=True)), 19 | ('windows', models.JSONField(blank=True, null=True)), 20 | ('name', models.CharField(blank=True, max_length=30, null=True)), 21 | ('width', models.FloatField(blank=True, null=True)), 22 | ('length', models.FloatField(blank=True, null=True)), 23 | ('height', models.FloatField(blank=True, null=True)), 24 | ('type', models.CharField(blank=True, max_length=5, null=True)), 25 | ('beds', models.JSONField(blank=True, null=True)), 26 | ('chairs', models.JSONField(blank=True, null=True)), 27 | ('tables', models.JSONField(blank=True, null=True)), 28 | ], 29 | options={ 30 | 'db_table': 'rooms_related_objects', 31 | 'managed': False, 32 | }, 33 | ), 34 | migrations.RunSQL( 35 | """ 36 | CREATE materialized VIEW rooms_related_objects AS WITH room_id_door_decoration AS ( 37 | SELECT 38 | room.id id, 39 | jsonb_build_object( 40 | 'id', door.id, 'name', door.name, 'width', 41 | door.width, 'length', door.length, 42 | 'height', door.height, 'type', door.type 43 | ) door, 44 | jsonb_build_object( 45 | 'id', 46 | decoration.id, 47 | 'name', 48 | decoration.name, 49 | 'souvenirs', 50 | jsonb_agg(souvenir) 51 | ) decoration 52 | FROM 53 | rooms_room room 54 | LEFT JOIN rooms_door door ON room.door_id = door.id 55 | LEFT JOIN rooms_decoration decoration ON room.decoration_id = decoration.id 56 | LEFT JOIN rooms_decoration_souvenirs ds ON decoration.id = ds.decoration_id 57 | LEFT JOIN rooms_souvenir souvenir ON ds.souvenir_id = souvenir.id 58 | GROUP BY 59 | room.id, 60 | door.id, 61 | decoration.id 62 | ), 63 | windows_in_room_by_id AS ( 64 | WITH room_window_fittings AS ( 65 | SELECT 66 | room_id, 67 | json_build_object( 68 | 'id', 69 | win.id, 70 | 'name', 71 | win.name, 72 | 'width', 73 | win.width, 74 | 'length', 75 | win.length, 76 | 'height', 77 | win.height, 78 | 'type', 79 | win.type, 80 | 'fittings', 81 | jsonb_agg(wf) 82 | ) window_in_room 83 | FROM 84 | rooms_window win 85 | LEFT JOIN rooms_windowfittings_windows rwfw ON rwfw.window_id = win.id 86 | LEFT JOIN rooms_windowfittings wf ON wf.id = rwfw.windowfittings_id 87 | GROUP BY 88 | room_id, 89 | win.id 90 | ) 91 | SELECT 92 | room_window_fittings.room_id AS id, 93 | jsonb_agg( 94 | room_window_fittings.window_in_room 95 | ) windows 96 | FROM 97 | room_window_fittings 98 | GROUP BY 99 | id 100 | ), 101 | parameters_and_beds_in_room_by_id AS ( 102 | SELECT 103 | room.id id, 104 | room.name name, 105 | room.width width, 106 | room.length length, 107 | room.height height, 108 | room.type type, 109 | jsonb_agg(bed) beds 110 | FROM 111 | rooms_room room 112 | LEFT JOIN rooms_bed_rooms bed_i ON room.id = bed_i.room_id 113 | LEFT JOIN rooms_bed bed ON bed.id = bed_i.bed_id 114 | GROUP BY 115 | room.id 116 | ), 117 | chairs_in_room_by_id AS ( 118 | SELECT 119 | room.id id, 120 | jsonb_agg(chair) chairs 121 | FROM 122 | rooms_room room 123 | LEFT JOIN rooms_chair_rooms chair_i ON room.id = chair_i.room_id 124 | LEFT JOIN rooms_chair chair ON chair.id = chair_i.chair_id 125 | GROUP BY 126 | room.id 127 | ), 128 | tables_in_room_by_id AS ( 129 | SELECT 130 | room.id id, 131 | jsonb_agg(table_f) tables 132 | FROM 133 | rooms_room room 134 | LEFT JOIN rooms_table_rooms table_i ON room.id = table_i.room_id 135 | LEFT JOIN rooms_table table_f ON table_f.id = table_i.table_id 136 | GROUP BY 137 | room.id 138 | ) 139 | SELECT 140 | COALESCE( 141 | room_id_door_decoration.id, windows_in_room_by_id.id 142 | ) AS id, 143 | room_id_door_decoration.door, 144 | room_id_door_decoration.decoration, 145 | windows_in_room_by_id.windows, 146 | parameters_and_beds_in_room_by_id.name, 147 | parameters_and_beds_in_room_by_id.width, 148 | parameters_and_beds_in_room_by_id.length, 149 | parameters_and_beds_in_room_by_id.height, 150 | parameters_and_beds_in_room_by_id.type, 151 | parameters_and_beds_in_room_by_id.beds, 152 | chairs_in_room_by_id.chairs, 153 | tables_in_room_by_id.tables 154 | FROM 155 | room_id_door_decoration FULL 156 | OUTER JOIN windows_in_room_by_id ON room_id_door_decoration.id = windows_in_room_by_id.id FULL 157 | OUTER JOIN parameters_and_beds_in_room_by_id ON windows_in_room_by_id.id = parameters_and_beds_in_room_by_id.id FULL 158 | OUTER JOIN chairs_in_room_by_id ON parameters_and_beds_in_room_by_id.id = chairs_in_room_by_id.id FULL 159 | OUTER JOIN tables_in_room_by_id ON chairs_in_room_by_id.id = tables_in_room_by_id.id WITH DATA; 160 | """ 161 | ), 162 | migrations.RunSQL( 163 | """ 164 | CREATE UNIQUE INDEX ON rooms_related_objects (id); 165 | """ 166 | ), 167 | migrations.RunSQL( 168 | """ 169 | REFRESH MATERIALIZED VIEW CONCURRENTLY rooms_related_objects; 170 | """ 171 | ) 172 | ] 173 | -------------------------------------------------------------------------------- /app/rooms/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.3 on 2022-03-31 02:35 2 | 3 | import django.core.validators 4 | import django.db.models.deletion 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | initial = True 11 | 12 | dependencies = [ 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name='Decoration', 18 | fields=[ 19 | ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 20 | ('name', models.CharField(max_length=30, unique=True)), 21 | ], 22 | ), 23 | migrations.CreateModel( 24 | name='Door', 25 | fields=[ 26 | ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 27 | ('name', models.CharField(max_length=30, unique=True)), 28 | ('width', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 29 | ('length', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 30 | ('height', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 31 | ('type', models.CharField(choices=[('DT1', 'Door Type 1'), ('DT2', 'Door Type 2'), ('DT3', 'Door Type 3'), ('DT4', 'Door Type 4')], max_length=5)), 32 | ], 33 | options={ 34 | 'abstract': False, 35 | }, 36 | ), 37 | migrations.CreateModel( 38 | name='Room', 39 | fields=[ 40 | ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 41 | ('name', models.CharField(max_length=30, unique=True)), 42 | ('width', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 43 | ('length', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 44 | ('height', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 45 | ('type', models.CharField(choices=[('KTCH', 'Kitchen'), ('BDR', 'Bedroom'), ('LBB', 'Lobby'), ('LRM', 'Living Room')], max_length=5)), 46 | ('decoration', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='rooms.decoration')), 47 | ('door', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='rooms.door')), 48 | ], 49 | options={ 50 | 'abstract': False, 51 | }, 52 | ), 53 | migrations.CreateModel( 54 | name='Souvenir', 55 | fields=[ 56 | ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 57 | ('name', models.CharField(max_length=30, unique=True)), 58 | ], 59 | ), 60 | migrations.CreateModel( 61 | name='Window', 62 | fields=[ 63 | ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 64 | ('name', models.CharField(max_length=30, unique=True)), 65 | ('width', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 66 | ('length', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 67 | ('height', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 68 | ('type', models.CharField(choices=[('WT1', 'Window Type 1'), ('WT2', 'Window Type 2'), ('WT3', 'Window Type 3')], max_length=5)), 69 | ('room', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rooms.room')), 70 | ], 71 | options={ 72 | 'abstract': False, 73 | }, 74 | ), 75 | migrations.CreateModel( 76 | name='WindowFittings', 77 | fields=[ 78 | ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 79 | ('name', models.CharField(max_length=30, unique=True)), 80 | ('width', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 81 | ('length', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 82 | ('height', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 83 | ('type', models.CharField(choices=[('WFT1', 'Window Fittings Type 1'), ('WFT2', 'Window Fittings Type 2'), ('WFT3', 'Window Fittings Type 3')], max_length=5)), 84 | ('windows', models.ManyToManyField(to='rooms.window')), 85 | ], 86 | options={ 87 | 'abstract': False, 88 | }, 89 | ), 90 | migrations.CreateModel( 91 | name='Table', 92 | fields=[ 93 | ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 94 | ('name', models.CharField(max_length=30, unique=True)), 95 | ('width', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 96 | ('length', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 97 | ('height', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 98 | ('type', models.CharField(choices=[('TBLT1', 'Table Type 1'), ('TBLT2', 'Table Type 2'), ('TBLT3', 'Table Type 3')], max_length=5)), 99 | ('rooms', models.ManyToManyField(to='rooms.room')), 100 | ], 101 | options={ 102 | 'abstract': False, 103 | }, 104 | ), 105 | migrations.AddField( 106 | model_name='decoration', 107 | name='souvenirs', 108 | field=models.ManyToManyField(to='rooms.souvenir'), 109 | ), 110 | migrations.CreateModel( 111 | name='Chair', 112 | fields=[ 113 | ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 114 | ('name', models.CharField(max_length=30, unique=True)), 115 | ('width', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 116 | ('length', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 117 | ('height', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 118 | ('type', models.CharField(choices=[('CHT1', 'Chair Type 1'), ('CHT2', 'Chair Type 2'), ('CHT3', 'Chair Type 3')], max_length=5)), 119 | ('rooms', models.ManyToManyField(to='rooms.room')), 120 | ], 121 | options={ 122 | 'abstract': False, 123 | }, 124 | ), 125 | migrations.CreateModel( 126 | name='Bed', 127 | fields=[ 128 | ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 129 | ('name', models.CharField(max_length=30, unique=True)), 130 | ('width', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 131 | ('length', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 132 | ('height', models.FloatField(default=1.0, validators=[django.core.validators.MinValueValidator(1.0), django.core.validators.MaxValueValidator(100.0)])), 133 | ('type', models.CharField(choices=[('BT1', 'Bed Type 1'), ('BT2', 'Bed Type 2'), ('BT3', 'Bed Type 3')], max_length=5)), 134 | ('rooms', models.ManyToManyField(to='rooms.room')), 135 | ], 136 | options={ 137 | 'abstract': False, 138 | }, 139 | ), 140 | ] 141 | -------------------------------------------------------------------------------- /app/rooms/signals.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Union 3 | 4 | from django.apps import apps 5 | from django.db import connection 6 | from django.db.models.signals import m2m_changed, post_save 7 | from django.dispatch import receiver 8 | 9 | from rooms.models import (Bed, Chair, Decoration, Door, Room, 10 | RoomWithRelatedObjsRebuildInApp, 11 | RoomWithRelatedObjsV3, Souvenir, Table, Window, 12 | WindowFittings) 13 | from rooms.serializers import (DecorationSerializer, DoorSerializer, 14 | RoomSerializer) 15 | from rooms.utils import create_room_with_related_objs 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | 20 | ############################################# 21 | # Approach 1 - PostgreSQL Materialized View # 22 | ############################################# 23 | 24 | 25 | @receiver(post_save, sender=WindowFittings) 26 | @receiver(post_save, sender=Window) 27 | @receiver(post_save, sender=Door) 28 | @receiver(post_save, sender=Souvenir) 29 | @receiver(post_save, sender=Decoration) 30 | @receiver(post_save, sender=Room) 31 | @receiver(post_save, sender=Chair) 32 | @receiver(post_save, sender=Bed) 33 | @receiver(post_save, sender=Table) 34 | @receiver(m2m_changed, sender=Decoration.souvenirs.through) 35 | @receiver(m2m_changed, sender=WindowFittings.windows.through) 36 | @receiver(m2m_changed, sender=Chair.rooms.through) 37 | @receiver(m2m_changed, sender=Table.rooms.through) 38 | @receiver(m2m_changed, sender=Bed.rooms.through) 39 | def update_view_rooms_related_objects(sender, **kwargs): 40 | # simple code but costly execution 41 | # because it rebuilds all rooms everytime, 42 | # including those that don't need to be recalculated. 43 | with connection.cursor() as cursor: 44 | cursor.execute("REFRESH MATERIALIZED VIEW CONCURRENTLY rooms_related_objects;") 45 | 46 | 47 | ################################################ 48 | # Approach 2 - new 'artificial' model # 49 | # RoomWithRelatedObjsRebuildInApp # 50 | # that stores rooms parameters # 51 | # together with all their related objects data # 52 | ################################################ 53 | 54 | 55 | def get_or_create_room_with_related_objs( 56 | room_id: int, model_name: str = "RoomWithRelatedObjsRebuildInApp" 57 | ) -> Union[RoomWithRelatedObjsV3, RoomWithRelatedObjsRebuildInApp, None]: 58 | """ 59 | If Room with related objects V2 (signals) or V3 (triggers) object does not exist for a given room ID, 60 | call the create_room_with_related_objs function to create it from scratch 61 | and build all its fields that store JSON data about related objects. 62 | Otherwise, return the found object, so the calling functiion will rebuild 63 | only the fields that have changed, and not all of them. 64 | """ 65 | model = apps.get_model("rooms", model_name) 66 | room_with_related_objs = model.objects.filter(id=room_id).first() 67 | if room_with_related_objs: 68 | logger.info(msg="; Room with related objects with ID " + str(room_id) + " already exists") 69 | return room_with_related_objs 70 | else: 71 | create_room_with_related_objs(room_id, model_name) 72 | return None 73 | 74 | 75 | # Whenever something changes (window, souvenirs, furniture, etc.), 76 | # we have to rebuild all the rooms that contain the changed object. 77 | 78 | 79 | @receiver(post_save, sender=Room) 80 | def room_changes_update_related_room(sender, instance, **kwargs): 81 | logger.info(msg="; room_changes_update_related_room: " + str(instance.id)) 82 | room_with_related_objs = get_or_create_room_with_related_objs(instance.id) 83 | if room_with_related_objs: 84 | source_room_data = RoomSerializer(instance).data 85 | room_with_related_objs.name = source_room_data["name"] 86 | room_with_related_objs.width = source_room_data["width"] 87 | room_with_related_objs.length = source_room_data["length"] 88 | room_with_related_objs.height = source_room_data["height"] 89 | room_with_related_objs.type = source_room_data["type"] 90 | room_with_related_objs.save() 91 | 92 | 93 | def _window_changed_process_related_rooms(window_instance, model_name: str = "RoomWithRelatedObjsRebuildInApp"): 94 | logger.info(msg="; _window_changed_process_related_rooms:") 95 | logger.info(msg="; window ID:" + str(window_instance.id)) 96 | # every window has only one room, so simplified processing 97 | room = window_instance.room 98 | logger.info(msg="; room ID:" + str(room.id)) 99 | room_with_related_objs = get_or_create_room_with_related_objs(room.id, model_name) 100 | if room_with_related_objs: 101 | source_room_data = RoomSerializer(room).data 102 | room_with_related_objs.windows = source_room_data["windows"] 103 | room_with_related_objs.save() 104 | 105 | 106 | @receiver(m2m_changed, sender=WindowFittings.windows.through) 107 | def windows_fitting_m2m_changed_update_related_rooms(sender, pk_set, action, **kwargs): 108 | logger.info(msg="; windows_fitting_m2m_changed_update_related_rooms - action: " + action) 109 | if action != "post_add" and action != "post_remove": 110 | return 111 | for window_id in pk_set: 112 | window_by_id = Window.objects.filter(id=window_id).first() 113 | _window_changed_process_related_rooms(window_by_id, "RoomWithRelatedObjsRebuildInApp") 114 | _window_changed_process_related_rooms(window_by_id, "RoomWithRelatedObjsV3") 115 | 116 | 117 | @receiver(post_save, sender=WindowFittings) 118 | def windows_fitting_changes_update_related_rooms(sender, instance, **kwargs): 119 | logger.info(msg="; windows_fitting_changes_update_related_rooms: " + str(instance.id)) 120 | windows = instance.windows.all() 121 | for window in windows: 122 | _window_changed_process_related_rooms(window) 123 | 124 | 125 | @receiver(post_save, sender=Window) 126 | def window_changes_update_related_room(sender, instance, **kwargs): 127 | logger.info(msg="; window_changes_update_related_room: " + str(instance.id)) 128 | _window_changed_process_related_rooms(instance) 129 | 130 | 131 | @receiver(post_save, sender=Door) 132 | def door_changed_update_related_rooms(sender, instance, **kwargs): 133 | logger.info(msg="; door_changed_update_related_rooms: " + str(instance.id)) 134 | rooms = Room.objects.filter(door=instance) 135 | for room in rooms: 136 | room_with_related_objs = get_or_create_room_with_related_objs(room.id) 137 | if not room_with_related_objs: 138 | continue 139 | door_data = DoorSerializer(instance).data 140 | room_with_related_objs.door = door_data 141 | room_with_related_objs.save() 142 | 143 | 144 | def _decoration_changed_process_related_rooms(decoration_instance, model_name: str = "RoomWithRelatedObjsRebuildInApp"): 145 | decoration_data = DecorationSerializer(decoration_instance).data 146 | rooms = Room.objects.filter(decoration=decoration_instance) 147 | for room in rooms: 148 | room_with_related_objs = get_or_create_room_with_related_objs(room.id, model_name) 149 | if not room_with_related_objs: 150 | continue 151 | room_with_related_objs.decoration = decoration_data 152 | room_with_related_objs.save() 153 | 154 | 155 | @receiver(post_save, sender=Souvenir) 156 | def souvenir_changed_update_related_rooms(sender, instance, **kwargs): 157 | logger.info(msg="; souvenir_changed_update_related_rooms: " + str(instance.id)) 158 | decorations_related = Decoration.objects.filter(souvenirs__in=[instance.pk]) 159 | for decoration_instance in decorations_related: 160 | _decoration_changed_process_related_rooms(decoration_instance) 161 | 162 | 163 | @receiver(post_save, sender=Decoration) 164 | def decoration_changed_update_related_rooms(sender, instance, **kwargs): 165 | logger.info(msg="; decoration_changed_update_related_rooms:" + str(instance.id)) 166 | _decoration_changed_process_related_rooms(instance) 167 | 168 | 169 | @receiver(m2m_changed, sender=Decoration.souvenirs.through) 170 | def decoration_souvenir_m2m_changed_update_related_rooms(sender, instance, action, **kwargs): 171 | logger.info(msg="; decoration_souvenir_m2m_changed_update_related_rooms - action: " + action) 172 | if action == "post_add" or action == "post_remove": 173 | _decoration_changed_process_related_rooms(instance, "RoomWithRelatedObjsRebuildInApp") 174 | _decoration_changed_process_related_rooms(instance, "RoomWithRelatedObjsV3") 175 | 176 | 177 | def _check_item_is_furniture(item): 178 | is_bed = item.__class__.__name__ == "Bed" 179 | is_table = item.__class__.__name__ == "Table" 180 | is_chair = item.__class__.__name__ == "Chair" 181 | if not is_bed and not is_table and not is_chair: 182 | logger.error(msg="; chair_bed_table_m2m_chng - wrong instance.__class__.__name__ - " + item.__class__.__name__) 183 | return False 184 | return True 185 | 186 | 187 | def _furniture_update_room_related_data(room_id, class_name, model_name: str = "RoomWithRelatedObjsRebuildInApp"): 188 | room_with_related_data = get_or_create_room_with_related_objs(room_id, model_name) 189 | if not room_with_related_data: 190 | # room_with_related_data was not obtained 191 | # but created from scratch 192 | # and fully built, with all fields, 193 | # nothing to do here 194 | return 195 | source_room = Room.objects.filter(id=room_id).first() 196 | source_room_data = RoomSerializer(source_room).data 197 | if class_name == "Bed": 198 | room_with_related_data.beds = source_room_data["beds"] 199 | if class_name == "Chair": 200 | room_with_related_data.chairs = source_room_data["chairs"] 201 | if class_name == "Table": 202 | room_with_related_data.tables = source_room_data["tables"] 203 | room_with_related_data.save() 204 | 205 | 206 | @receiver(post_save, sender=Chair) 207 | @receiver(post_save, sender=Bed) 208 | @receiver(post_save, sender=Table) 209 | def chair_bed_table_changed_update_related_rooms(sender, instance, **kwargs): 210 | logger.info( 211 | msg="; chair_bed_table_changed_update_related_rooms - " 212 | + ", class - " 213 | + instance.__class__.__name__ 214 | + ", instance ID - " 215 | + str(instance.id) 216 | ) 217 | if not _check_item_is_furniture(instance): 218 | return 219 | rooms = instance.rooms.all() 220 | for room in rooms: 221 | _furniture_update_room_related_data(room.id, instance.__class__.__name__) 222 | 223 | 224 | @receiver(m2m_changed, sender=Table.rooms.through) 225 | @receiver(m2m_changed, sender=Bed.rooms.through) 226 | @receiver(m2m_changed, sender=Chair.rooms.through) 227 | def chair_bed_table_m2m_chng_update_related_rooms(sender, instance, pk_set, action, **kwargs): 228 | logger.info( 229 | msg="; chair_bed_table_m2m_chng_update_related_rooms - rooms pk_set: " 230 | + str(pk_set) 231 | + ", action: " 232 | + action 233 | + ", class - " 234 | + instance.__class__.__name__ 235 | ) 236 | if not _check_item_is_furniture(instance): 237 | return 238 | if action != "post_add" and action != "post_remove": 239 | return 240 | for room_id in pk_set: 241 | _furniture_update_room_related_data(room_id, instance.__class__.__name__, "RoomWithRelatedObjsRebuildInApp") 242 | _furniture_update_room_related_data(room_id, instance.__class__.__name__, "RoomWithRelatedObjsV3") 243 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /app/rooms/migrations/0008_bed_room_v3_bed_chair_room_v3_chair_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.1.7 on 2023-03-15 20:13 2 | 3 | from django.db import migrations 4 | import pgtrigger.compiler 5 | import pgtrigger.migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('rooms', '0007_roomwithrelatedobjsv3'), 12 | ] 13 | 14 | operations = [ 15 | pgtrigger.migrations.AddTrigger( 16 | model_name='bed', 17 | trigger=pgtrigger.compiler.Trigger(name='room_v3_bed', sql=pgtrigger.compiler.UpsertTriggerSql(func='\n INSERT INTO rooms_roomwithrelatedobjsv3(id, beds)\n WITH room_ids AS (\n SELECT\n rooms_bed_rooms.room_id\n FROM rooms_bed\n LEFT JOIN rooms_bed_rooms ON rooms_bed_rooms.bed_id = rooms_bed.id\n WHERE rooms_bed.id = OLD.id\n )\n SELECT\n room.id id,\n jsonb_agg(bed) beds\n FROM rooms_room room\n LEFT JOIN rooms_bed_rooms bed_i ON room.id = bed_i.room_id\n LEFT JOIN rooms_bed bed ON bed.id = bed_i.bed_id\n WHERE room.id IN (SELECT room_id FROM room_ids)\n GROUP BY room.id\n ON CONFLICT (id)\n DO\n UPDATE SET beds = EXCLUDED.beds;\n RETURN NULL;\n ', hash='cd313a2ea26c7d3a2dbc561734353d866598a81e', operation='UPDATE OR INSERT OR DELETE', pgid='pgtrigger_room_v3_bed_512d8', table='rooms_bed', when='AFTER')), 18 | ), 19 | pgtrigger.migrations.AddTrigger( 20 | model_name='chair', 21 | trigger=pgtrigger.compiler.Trigger(name='room_v3_chair', sql=pgtrigger.compiler.UpsertTriggerSql(func='\n INSERT INTO rooms_roomwithrelatedobjsv3(id, chairs)\n WITH room_ids AS (\n SELECT\n rooms_chair_rooms.room_id\n FROM rooms_chair\n LEFT JOIN rooms_chair_rooms ON rooms_chair_rooms.chair_id = rooms_chair.id\n WHERE rooms_chair.id = OLD.id\n )\n SELECT\n room.id id,\n jsonb_agg(chair) chairs\n FROM rooms_room room\n LEFT JOIN rooms_chair_rooms chair_i ON room.id = chair_i.room_id\n LEFT JOIN rooms_chair chair ON chair.id = chair_i.chair_id\n WHERE room.id IN (SELECT room_id FROM room_ids)\n GROUP BY room.id\n ON CONFLICT (id)\n DO\n UPDATE SET chairs = EXCLUDED.chairs;\n RETURN NULL;\n ', hash='0bf996580913420f4c519f483f8df125030cc9f6', operation='UPDATE OR INSERT OR DELETE', pgid='pgtrigger_room_v3_chair_dee7b', table='rooms_chair', when='AFTER')), 22 | ), 23 | pgtrigger.migrations.AddTrigger( 24 | model_name='decoration', 25 | trigger=pgtrigger.compiler.Trigger(name='room_v3_decoration', sql=pgtrigger.compiler.UpsertTriggerSql(func="\n INSERT INTO rooms_roomwithrelatedobjsv3(id, decoration)\n SELECT\n room.id id,\n jsonb_build_object(\n 'id',\n decoration.id,\n 'name',\n decoration.name,\n 'souvenirs',\n jsonb_agg(souvenir)\n ) decoration\n FROM\n rooms_room room\n LEFT JOIN rooms_decoration decoration ON room.decoration_id = decoration.id\n LEFT JOIN rooms_decoration_souvenirs ds ON decoration.id = ds.decoration_id\n LEFT JOIN rooms_souvenir souvenir ON ds.souvenir_id = souvenir.id\n WHERE room.decoration_id = OLD.id\n GROUP BY\n room.id,\n decoration.id\n ON CONFLICT (id)\n DO\n UPDATE SET decoration = EXCLUDED.decoration;\n RETURN NULL;\n ", hash='3afde91a63247eb8681b0df94ef3fcbc9c6dffd3', operation='UPDATE OR INSERT OR DELETE', pgid='pgtrigger_room_v3_decoration_91dce', table='rooms_decoration', when='AFTER')), 26 | ), 27 | pgtrigger.migrations.AddTrigger( 28 | model_name='door', 29 | trigger=pgtrigger.compiler.Trigger(name='room_v3_door', sql=pgtrigger.compiler.UpsertTriggerSql(func="\n INSERT INTO rooms_roomwithrelatedobjsv3(id, door)\n SELECT\n room.id id,\n jsonb_build_object(\n 'id', door.id, 'name', door.name, 'width',\n door.width, 'length', door.length,\n 'height', door.height, 'type', door.type\n ) door\n FROM\n rooms_room room\n LEFT JOIN rooms_door door ON room.door_id = door.id\n WHERE room.door_id = OLD.id\n ON CONFLICT (id)\n DO\n UPDATE SET door = EXCLUDED.door;\n RETURN NULL;\n ", hash='f2ebd3f8f713c60f222a1a7952448c26fc58a787', operation='UPDATE OR INSERT OR DELETE', pgid='pgtrigger_room_v3_door_02aea', table='rooms_door', when='AFTER')), 30 | ), 31 | pgtrigger.migrations.AddTrigger( 32 | model_name='room', 33 | trigger=pgtrigger.compiler.Trigger(name='room_v3_common_info', sql=pgtrigger.compiler.UpsertTriggerSql(func='\n INSERT INTO rooms_roomwithrelatedobjsv3(id, name, width, length, height, type)\n SELECT\n room.id id,\n room.name name,\n room.width width,\n room.length length,\n room.height height,\n room.type type\n FROM\n rooms_room room\n WHERE room.id = OLD.id\n ON CONFLICT (id)\n DO\n UPDATE SET name = EXCLUDED.name,\n width = EXCLUDED.width,\n length = EXCLUDED.length,\n height = EXCLUDED.height,\n type = EXCLUDED.type;\n RETURN NULL;\n ', hash='ca9775a820068d71d5de790ffadc209c8df6e04b', operation='UPDATE OR INSERT OR DELETE', pgid='pgtrigger_room_v3_common_info_29562', table='rooms_room', when='AFTER')), 34 | ), 35 | pgtrigger.migrations.AddTrigger( 36 | model_name='souvenir', 37 | trigger=pgtrigger.compiler.Trigger(name='room_v3_souvenir', sql=pgtrigger.compiler.UpsertTriggerSql(func="\n INSERT INTO rooms_roomwithrelatedobjsv3(id, decoration)\n WITH decoraition_ids AS(\n SELECT decoration_id from rooms_decoration_souvenirs where souvenir_id=OLD.id\n )\n SELECT\n room.id id,\n jsonb_build_object(\n 'id',\n decoration.id,\n 'name',\n decoration.name,\n 'souvenirs',\n jsonb_agg(souvenir)\n ) decoration\n FROM\n rooms_room room\n LEFT JOIN rooms_decoration decoration ON room.decoration_id = decoration.id\n LEFT JOIN rooms_decoration_souvenirs ds ON decoration.id = ds.decoration_id\n LEFT JOIN rooms_souvenir souvenir ON ds.souvenir_id = souvenir.id\n WHERE room.decoration_id IN (SELECT decoration_id from decoraition_ids)\n GROUP BY\n room.id,\n decoration.id\n ON CONFLICT (id)\n DO\n UPDATE SET decoration = EXCLUDED.decoration;\n RETURN NULL;\n ", hash='1f3f69bf4c3c6fb79d6e8455e558316ef773f834', operation='UPDATE OR INSERT OR DELETE', pgid='pgtrigger_room_v3_souvenir_24f94', table='rooms_souvenir', when='AFTER')), 38 | ), 39 | pgtrigger.migrations.AddTrigger( 40 | model_name='table', 41 | trigger=pgtrigger.compiler.Trigger(name='room_v3_table', sql=pgtrigger.compiler.UpsertTriggerSql(func='\n INSERT INTO rooms_roomwithrelatedobjsv3(id, tables)\n WITH room_ids AS (\n SELECT\n rooms_table_rooms.room_id\n FROM rooms_table\n LEFT JOIN rooms_table_rooms ON rooms_table_rooms.table_id = rooms_table.id\n WHERE rooms_table.id = OLD.id\n )\n SELECT\n room.id id,\n jsonb_agg(table_model) tables\n FROM rooms_room room\n LEFT JOIN rooms_table_rooms table_i ON room.id = table_i.room_id\n LEFT JOIN rooms_table table_model ON table_model.id = table_i.table_id\n WHERE room.id IN (SELECT room_id FROM room_ids)\n GROUP BY room.id\n ON CONFLICT (id)\n DO\n UPDATE SET tables = EXCLUDED.tables;\n RETURN NULL;\n ', hash='7c51283ee83795ac747d920288cd3e93288cc192', operation='UPDATE OR INSERT OR DELETE', pgid='pgtrigger_room_v3_table_ece0f', table='rooms_table', when='AFTER')), 42 | ), 43 | pgtrigger.migrations.AddTrigger( 44 | model_name='window', 45 | trigger=pgtrigger.compiler.Trigger(name='room_v3_windows', sql=pgtrigger.compiler.UpsertTriggerSql(func="\n INSERT INTO rooms_roomwithrelatedobjsv3(id, windows)\n WITH room_window_fittings AS (\n SELECT\n room_id,\n json_build_object(\n 'id',\n win.id,\n 'name',\n win.name,\n 'width',\n win.width,\n 'length',\n win.length,\n 'height',\n win.height,\n 'type',\n win.type,\n 'fittings',\n jsonb_agg(wf)\n ) window_in_room\n FROM rooms_window win\n LEFT JOIN rooms_windowfittings_windows rwfw ON rwfw.window_id = win.id\n LEFT JOIN rooms_windowfittings wf ON wf.id = rwfw.windowfittings_id\n WHERE win.id = OLD.id\n GROUP BY room_id, win.id\n )\n SELECT\n room_window_fittings.room_id AS id,\n jsonb_agg(\n room_window_fittings.window_in_room\n ) windows\n FROM room_window_fittings\n GROUP BY id\n ON CONFLICT (id)\n DO\n UPDATE SET windows = EXCLUDED.windows;\n RETURN NULL;\n ", hash='7602d3988323fac7d0fe33a44a0bcd4b62679dad', operation='UPDATE OR INSERT OR DELETE', pgid='pgtrigger_room_v3_windows_2d5dd', table='rooms_window', when='AFTER')), 46 | ), 47 | pgtrigger.migrations.AddTrigger( 48 | model_name='windowfittings', 49 | trigger=pgtrigger.compiler.Trigger(name='room_v3_window_fittings', sql=pgtrigger.compiler.UpsertTriggerSql(func="\n INSERT INTO rooms_roomwithrelatedobjsv3(id, windows)\n WITH room_ids AS (\n SELECT\n win.room_id\n FROM rooms_window win\n LEFT JOIN rooms_windowfittings_windows rwfw ON rwfw.window_id = win.id\n LEFT JOIN rooms_windowfittings wf ON wf.id = rwfw.windowfittings_id\n WHERE wf.id = OLD.id\n ),\n room_window_fittings AS (\n SELECT\n room_id,\n json_build_object(\n 'id',\n win.id,\n 'name',\n win.name,\n 'width',\n win.width,\n 'length',\n win.length,\n 'height',\n win.height,\n 'type',\n win.type,\n 'fittings',\n jsonb_agg(wf)\n ) window_in_room\n FROM rooms_window win\n LEFT JOIN rooms_windowfittings_windows rwfw ON rwfw.window_id = win.id\n LEFT JOIN rooms_windowfittings wf ON wf.id = rwfw.windowfittings_id\n WHERE win.room_id IN (SELECT room_id FROM room_ids)\n GROUP BY room_id, win.id\n )\n SELECT\n room_window_fittings.room_id AS id,\n jsonb_agg(\n room_window_fittings.window_in_room\n ) windows\n FROM room_window_fittings\n GROUP BY id\n ON CONFLICT (id)\n DO\n UPDATE SET windows = EXCLUDED.windows;\n RETURN NULL;\n ", hash='46e1a5de9cebd1ec7f4d66e26a545e8589a8b7e7', operation='UPDATE OR INSERT OR DELETE', pgid='pgtrigger_room_v3_window_fittings_af150', table='rooms_windowfittings', when='AFTER')), 50 | ), 51 | ] 52 | -------------------------------------------------------------------------------- /app/rooms/tests.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import random 3 | import string 4 | 5 | import pytest 6 | from django.apps import apps 7 | from django.db.models import Q 8 | from django.urls import reverse 9 | from rest_framework import status 10 | from rest_framework.test import APIClient 11 | from rooms.models import Room 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | def _get_random_item_of_class(model_class_name): 17 | model = apps.get_model("rooms", model_class_name) 18 | if hasattr(model, 'rooms'): 19 | pks = model.objects.exclude(rooms=None).values_list("pk", flat=True) 20 | else: 21 | pks = model.objects.values_list("pk", flat=True) 22 | random_pk = random.choice(pks) 23 | instance = model.objects.get(pk=random_pk) 24 | return instance 25 | 26 | 27 | @pytest.mark.django_db 28 | @pytest.mark.parametrize( 29 | "room_model", ["RoomsRelatedObjectsMaterializedView", "RoomWithRelatedObjsRebuildInApp", "RoomWithRelatedObjsV3"] 30 | ) 31 | def test_door_change_reflected_in_its_rooms(room_model): 32 | """ 33 | Every room has one door type (ForeignKey). 34 | Every Door (door type) may be installed in many rooms. 35 | Ensure that Door (door type) changes are reflected in all its rooms. 36 | """ 37 | # NOTE Door model is about door types, not door instances 38 | 39 | door_instance = _get_random_item_of_class('Door') 40 | model = apps.get_model("rooms", room_model) 41 | 42 | # check before name change, just in case 43 | for room in door_instance.rooms.all(): 44 | room_model_instance = model.objects.get(pk=room.id) 45 | assert door_instance.name == room_model_instance.door["name"] 46 | 47 | # change door type's name 48 | letters = string.ascii_letters 49 | door_instance.name = "".join(random.choice(letters) for i in range(10)) 50 | door_instance.save() 51 | 52 | # check after change 53 | for room in door_instance.rooms.all(): 54 | room_model_instance = model.objects.get(pk=room.id) 55 | room_model_instance.refresh_from_db() 56 | assert room_model_instance.door["name"] == door_instance.name 57 | 58 | 59 | def _search_fitting_in_room_view_or_v2(fitting_id, room_with_related_data): 60 | found_fitting = False 61 | fitting_name = None 62 | # Rooms -> Windows -> Fittings 63 | for window in room_with_related_data.windows: 64 | if found_fitting: 65 | break 66 | for fitting in window["fittings"]: 67 | if not fitting: 68 | break 69 | if fitting["id"] == fitting_id: 70 | found_fitting = True 71 | fitting_name = fitting["name"] 72 | break 73 | return found_fitting, fitting_name 74 | 75 | 76 | @pytest.mark.django_db 77 | @pytest.mark.parametrize( 78 | "room_model", ["RoomsRelatedObjectsMaterializedView", "RoomWithRelatedObjsRebuildInApp", "RoomWithRelatedObjsV3"] 79 | ) 80 | def test_window_fitting_change_reflected_in_its_rooms(room_model): 81 | """ 82 | Rooms are related with WindowFittings through Windows. 83 | Ensure that window fitting change is reflected in all its rooms. 84 | """ 85 | # NOTE fittings have several windows, every window has one room 86 | 87 | # get random window fitting and change it 88 | fitting = _get_random_item_of_class("WindowFittings") 89 | model = apps.get_model("rooms", room_model) 90 | letters = string.ascii_letters 91 | new_wf_name = "".join(random.choice(letters) for i in range(10)) 92 | fitting.name = new_wf_name 93 | fitting.save() 94 | 95 | # get all rooms of the changed Window Fitting through its windows. 96 | rooms_set = set() 97 | for window in fitting.windows.all(): 98 | rooms_set.add(window.room) 99 | 100 | for room in rooms_set: 101 | room_model_instance = model.objects.get(pk=room.id) 102 | found_fitting_in_view, found_name_view = _search_fitting_in_room_view_or_v2(fitting.id, room_model_instance) 103 | assert found_fitting_in_view 104 | assert found_name_view == new_wf_name 105 | 106 | 107 | @pytest.mark.django_db 108 | @pytest.mark.parametrize( 109 | "room_model", ["RoomsRelatedObjectsMaterializedView", "RoomWithRelatedObjsRebuildInApp", "RoomWithRelatedObjsV3"] 110 | ) 111 | def test_room_change_reflected_in_its_related_models(room_model): 112 | """ 113 | Rooms have many related objects. Often, users need to get detailed information about a room 114 | and the properties of its associated objects. 115 | 116 | To avoid executing slow and heavy JOIN queries, their results are prepared in advance 117 | and stored in RoomsRelatedObjectsMaterializedView, RoomWithRelatedObjsRebuildInApp, and RoomWithRelatedObjsV3. 118 | 119 | Ensure that when we change the Room instance, these changes are reflected 120 | in PostgreSQL materialized view, solutions V2 (signals) and V3 (triggers). 121 | """ 122 | room = _get_random_item_of_class("Room") 123 | model = apps.get_model("rooms", room_model) 124 | room_model_instance = model.objects.get(pk=room.id) 125 | 126 | # check before name change, just in case 127 | assert room.name == room_model_instance.name 128 | 129 | letters = string.ascii_letters 130 | new_room_name = "".join(random.choice(letters) for i in range(10)) 131 | room.name = new_room_name 132 | room.save() 133 | 134 | room_model_instance.refresh_from_db() 135 | assert room.name == room_model_instance.name == new_room_name 136 | 137 | 138 | def _check_item_in_room_view_and_v2(item_id, furniture_class_name, room_model_obj): 139 | if furniture_class_name == "Bed": 140 | furniture_in_room = room_model_obj.beds 141 | if furniture_class_name == "Chair": 142 | furniture_in_room = room_model_obj.chairs 143 | if furniture_class_name == "Table": 144 | furniture_in_room = room_model_obj.tables 145 | item_in_room_model_object_found = False 146 | if len(furniture_in_room) == 0: 147 | return item_in_room_model_object_found 148 | for elem in furniture_in_room: 149 | if elem["id"] == item_id: 150 | item_in_room_model_object_found = True 151 | return item_in_room_model_object_found 152 | 153 | 154 | @pytest.mark.django_db 155 | @pytest.mark.parametrize("furniture_class", ["Bed", "Chair", "Table"]) 156 | @pytest.mark.parametrize( 157 | "room_model", ["RoomsRelatedObjectsMaterializedView", "RoomWithRelatedObjsRebuildInApp", "RoomWithRelatedObjsV3"] 158 | ) 159 | def test_furniture_rooms_set_remove_reflected(furniture_class, room_model): 160 | """ 161 | The Bed, Chair, Table models are about furniture types, not instances. 162 | Each type of furniture can be placed in one or more rooms. 163 | Ensure that removing a type of furniture from a room works as expected. 164 | It should be reflected not only in Room, 165 | but also in PostgreSQL materialized view, solutions V2 (signals) and V3 (triggers). 166 | """ 167 | item = _get_random_item_of_class(furniture_class) 168 | model = apps.get_model("rooms", room_model) 169 | room = item.rooms.first() 170 | room_model_instance = model.objects.get(pk=room.id) 171 | item_in_room_instance_found = _check_item_in_room_view_and_v2(item.id, furniture_class, room_model_instance) 172 | 173 | # check before furniture type removal from set, just in case 174 | assert item_in_room_instance_found 175 | 176 | item.rooms.remove(room) 177 | room_model_instance.refresh_from_db() 178 | item_in_room_instance_found = _check_item_in_room_view_and_v2(item.id, furniture_class, room_model_instance) 179 | assert not item_in_room_instance_found 180 | 181 | 182 | @pytest.mark.django_db 183 | @pytest.mark.parametrize("furniture_class", ["Bed", "Chair", "Table"]) 184 | @pytest.mark.parametrize( 185 | "room_model", ["RoomsRelatedObjectsMaterializedView", "RoomWithRelatedObjsRebuildInApp", "RoomWithRelatedObjsV3"] 186 | ) 187 | def test_furniture_rooms_set_add_reflected(furniture_class, room_model): 188 | """ 189 | The Bed, Chair, Table models are about furniture types, not instances. 190 | Each type of furniture can be placed in one or more rooms. 191 | Ensure that adding a type of furniture to the room works as expected. 192 | It should be reflected not only in Room, 193 | but also in PostgreSQL materialized view, solutions V2 (signals) and V3 (triggers). 194 | """ 195 | # Get random Bed, Chair, or Table as well as a room that does NOT have this type of furniture. 196 | item = _get_random_item_of_class(furniture_class) 197 | pks = Room.objects.filter(~Q(id__in=[o.id for o in item.rooms.all()])).values_list("pk", flat=True) 198 | random_pk = random.choice(pks) 199 | room = Room.objects.get(pk=random_pk) 200 | 201 | model = apps.get_model("rooms", room_model) 202 | room_model_instance = model.objects.get(pk=room.id) 203 | # check before item addition to the set of the room's furniture, just in case 204 | item_in_room_instance_found = _check_item_in_room_view_and_v2(item.id, furniture_class, room_model_instance) 205 | assert not item_in_room_instance_found 206 | 207 | item.rooms.add(room) 208 | room_model_instance.refresh_from_db() 209 | item_in_room_instance_found = _check_item_in_room_view_and_v2(item.id, furniture_class, room_model_instance) 210 | assert item_in_room_instance_found 211 | 212 | 213 | @pytest.mark.django_db 214 | @pytest.mark.parametrize( 215 | "model_name,model_url", 216 | [ 217 | ("Door", "doors"), 218 | ("Room", "rooms_native"), 219 | ("Window", "windows"), 220 | ("WindowFittings", "window_fittings"), 221 | ("Chair", "chairs"), 222 | ("Bed", "beds"), 223 | ("Table", "tables"), 224 | ], 225 | ) 226 | def test_common_info_models_api_get(model_name, model_url): 227 | """ 228 | Furniture types have their own URLs where you can get information about them. 229 | Ensure that the GET requests work as expected. 230 | """ 231 | client = APIClient() 232 | item = _get_random_item_of_class(model_name) 233 | url = reverse(model_url + "-list") + str(item.pk) + "/" 234 | response = client.get(url, format="json") 235 | assert response.status_code == status.HTTP_200_OK 236 | assert response.data["name"] == item.name 237 | assert "width" in response.data.keys() 238 | assert "length" in response.data.keys() 239 | assert "height" in response.data.keys() 240 | 241 | 242 | @pytest.mark.django_db 243 | @pytest.mark.parametrize( 244 | "room_model_name,room_model_url", 245 | [ 246 | ("RoomsRelatedObjectsMaterializedView", "rooms_mat_view"), 247 | ("RoomWithRelatedObjsRebuildInApp", "rooms_v2"), 248 | ("RoomWithRelatedObjsV3", "rooms_v3"), 249 | ], 250 | ) 251 | def test_window_fittings_change_reflected_in_all_room_models(room_model_name, room_model_url): 252 | """ 253 | Modify the name of the WindowFittings instance using PATCH call 254 | to its API URL. Ensure this change is reflected in related rooms - 255 | PostgreSQL materialized view, solutions V2 (signals) and V3 (triggers). 256 | Obtain the data for checks using GET calls to the URLs of the room models. 257 | """ 258 | 259 | # arrange 260 | wf = _get_random_item_of_class("WindowFittings") 261 | window = wf.windows.first() 262 | room = window.room 263 | room_model = apps.get_model("rooms", room_model_name) 264 | room_instance = room_model.objects.get(pk=room.pk) 265 | url_for_patch_request = reverse("window_fittings-list") + str(wf.pk) + "/" 266 | letters = string.ascii_letters 267 | new_wf_name = "".join(random.choice(letters) for i in range(10)) 268 | data = {"name": new_wf_name} 269 | client = APIClient() 270 | 271 | # act 272 | response = client.patch(url_for_patch_request, data=data, format="json") 273 | room_instance.refresh_from_db() 274 | 275 | # assertions 276 | url_for_get_request = reverse(room_model_url + "-list") + str(room.pk) + "/" 277 | response = client.get(url_for_get_request, format="json") 278 | assert response.status_code == status.HTTP_200_OK 279 | window_filtered = [obj for obj in response.data["windows"] if obj["id"] == window.pk] 280 | wf_filtered = [obj for obj in window_filtered[0]["fittings"] if obj["id"] == wf.pk] 281 | wf_name_obtained = wf_filtered[0]["name"] 282 | assert wf_name_obtained == new_wf_name 283 | 284 | 285 | @pytest.mark.django_db 286 | @pytest.mark.parametrize( 287 | "room_model_name,room_model_url", 288 | [ 289 | ("RoomsRelatedObjectsMaterializedView", "rooms_mat_view"), 290 | ("RoomWithRelatedObjsRebuildInApp", "rooms_v2"), 291 | ("RoomWithRelatedObjsV3", "rooms_v3"), 292 | ], 293 | ) 294 | @pytest.mark.parametrize( 295 | "furniture_model_name,furniture_model_url,furniture_related_name", 296 | [ 297 | ("Chair", "chairs", "chairs"), 298 | ("Bed", "beds", "beds"), 299 | ("Table", "tables", "tables"), 300 | ], 301 | ) 302 | def test_furniture_change_reflected_in_all_room_models( 303 | room_model_name, room_model_url, furniture_model_name, furniture_model_url, furniture_related_name 304 | ): 305 | """ 306 | Modify the name of the furniture instance using PATCH call 307 | to its API URL. Ensure this change is reflected in related rooms - 308 | PostgreSQL materialized view, solutions V2 (signals) and V3 (triggers). 309 | Obtain the data for checks using GET calls to the URLs of the room models. 310 | """ 311 | 312 | # arrange 313 | furniture_item = _get_random_item_of_class(furniture_model_name) 314 | room = furniture_item.rooms.first() 315 | room_model = apps.get_model("rooms", room_model_name) 316 | room_instance = room_model.objects.get(pk=room.pk) 317 | url_for_patch_request = reverse(furniture_model_url + "-list") + str(furniture_item.pk) + "/" 318 | letters = string.ascii_letters 319 | new_furniture_name = "".join(random.choice(letters) for i in range(10)) 320 | data = {"name": new_furniture_name} 321 | client = APIClient() 322 | 323 | # act 324 | response = client.patch(url_for_patch_request, data=data, format="json") 325 | room_instance.refresh_from_db() 326 | 327 | # assertions 328 | url_for_get_request = reverse(room_model_url + "-list") + str(room.pk) + "/" 329 | response = client.get(url_for_get_request, format="json") 330 | assert response.status_code == status.HTTP_200_OK 331 | furniture_filtered = [obj for obj in response.data[furniture_related_name] if obj["id"] == furniture_item.pk] 332 | furniture_name_obtained = furniture_filtered[0]["name"] 333 | assert furniture_name_obtained == new_furniture_name 334 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Django PostgreSQL Materialized View Example 2 | The complex JOIN queries can exert a heavy load on the database engine and cause frustration to users due to their long processing times. The solution to this problem involves preparing the results of these queries in advance and storing them in the database. This repository showcases an implementation of such a solution for Django and PostgreSQL. 3 | 4 | Here you can learn and practice three ways to solve the problem: 5 | 1. PostgreSQL materialized view. See the files `/app/rooms/migrations/0004_roomsrelatedobjects.py` and `/app/rooms/models/room_related_view.py`. This solution is the simplest. Unfortunately, it has limitations that make it rarely suitable for practice. 6 | 1. Django signals. See the file `/app/rooms/signals.py`. Most likely, in practice, you will use this solution. 7 | 1. PostgreSQL triggers. They are slightly more efficient than Django signals but much more difficult to set up and maintain. See the examples in the `/app/rooms/models` directory. Also, take a look at the `django-pgtrigger` plugin. 8 | 9 | The proposed solution involves denormalizing data and consuming more disk space. Also, it leads to more complex and long-lasting INSERT, UPDATE, and DELETE queries. Nevertheless, it results in a significant acceleration of the execution of complex SELECT requests involving JOIN operations. 10 | 11 | In real-world workloads, SELECT queries far outnumber data modifications. As a result, the overall speed gain is substantial. Also, the database server can get by with cheaper hardware. Consequently, this approach is widely utilized in practical applications. 12 | 13 | ## How This Repository Can Help You 14 | 15 | In this repository, you can find working examples of the following: 16 | 1. PostgreSQL materialized view in Django. 17 | 1. `EagerLoadingMixin` in Django REST framework serializer classes to solve the N+1 queries problem. 18 | 1. Updating the instances that have nested serializers. You can explicitly write update methods in the serializer class. Or you can simply mark nested serializers as read-only. The `app/rooms/serializers.py` file contains examples of both approaches. 19 | 1. Usage of the most generic `viewsets.ModelViewSet` to build views in Django REST framework fast and easy. 20 | 1. Advanced PostgreSQL SQL query with several Common Table Expressions and JSON functions. See the `migrations.RunSQL` code in the `app/rooms/migrations/0004_roomsrelatedobjects.py` file. 21 | 1. PostgreSQL triggers to keep the data in different tables in sync. See numerous examples in the `/app/rooms/models` directory. The triggers are configured not inside the database but in the Django application layer. The `django-pgtrigger` plugin provides you with such an opportunity. 22 | 1. Usage of Django `post_save` and `m2m_changed` signals, see the `app/rooms/signals.py` file. 23 | 1. Processing of the actions during the `m2m_changed` signal handling. 24 | 1. Populating the Django database with fake generated data for testing. See the `app/rooms/management/commands/fill_db.py` file. 25 | 1. Logging configuration and usage in Django, see files `app/app/settings.py` and `app/rooms/signals.py`. 26 | 1. Usage of the `get_model` function to perform the same operation on multiple Django models. See the examples in the `app/rooms/tests.py` file. 27 | 28 | ## Task 29 | Our primary entity is the room. Each room has one door. Also, the rooms have many-to-many relationships with different types of furniture: beds, tables, and chairs. Each room has one or more windows. In turn, windows have many-to-many relationships with window fittings. Among other things, the room may have a set of souvenirs, which is called decoration. 30 | 31 | Our API endpoint provides the users with JSON data about the rooms and all their associated entities. We are dealing with a complex, bloated database schema. It includes many redundant models, and we are not allowed to get rid of them. Therefore, SELECT requests are complex and resource-intensive. Such a problem often occurs on legacy projects. 32 | 33 | Here is a partial database diagram. 34 | 35 | ![database diagram](/misc/db_diagram.png) 36 | 37 | And here is the code of some models. 38 | 39 | ```python 40 | from django.db import models 41 | from django.core.validators import MaxValueValidator, MinValueValidator 42 | 43 | 44 | class CommonInfo(models.Model): 45 | name = models.CharField(max_length=30, unique=True) 46 | width = models.FloatField( 47 | validators=[MinValueValidator(1.0), MaxValueValidator(100.0)], default=1.0 48 | ) 49 | length = models.FloatField( 50 | validators=[MinValueValidator(1.0), MaxValueValidator(100.0)], default=1.0 51 | ) 52 | height = models.FloatField( 53 | validators=[MinValueValidator(1.0), MaxValueValidator(100.0)], default=1.0 54 | ) 55 | 56 | class Meta: 57 | abstract = True 58 | 59 | 60 | class Door(CommonInfo): 61 | CHOICES = ( 62 | ("DT1", "Door Type 1"), 63 | ("DT2", "Door Type 2"), 64 | ("DT3", "Door Type 3"), 65 | ("DT4", "Door Type 4"), 66 | ) 67 | type = models.CharField( 68 | max_length=5, choices=CHOICES 69 | ) 70 | 71 | 72 | class Souvenir(models.Model): 73 | name = models.CharField(max_length=30, unique=True) 74 | 75 | 76 | class Decoration(models.Model): 77 | name = models.CharField(max_length=30, unique=True) 78 | souvenirs = models.ManyToManyField(Souvenir, related_name="decorations") 79 | 80 | 81 | class Room(CommonInfo): 82 | CHOICES = ( 83 | ("KTCH", "Kitchen"), 84 | ("BDR", "Bedroom"), 85 | ("LBB", "Lobby"), 86 | ("LRM", "Living Room"), 87 | ) 88 | type = models.CharField( 89 | max_length=5, choices=CHOICES 90 | ) 91 | door = models.ForeignKey( 92 | Door, on_delete=models.PROTECT, related_name="rooms") 93 | decoration = models.ForeignKey( 94 | Decoration, null=True, on_delete=models.SET_NULL, related_name="rooms") 95 | 96 | 97 | class Window(CommonInfo): 98 | CHOICES = ( 99 | ("WT1", "Window Type 1"), 100 | ("WT2", "Window Type 2"), 101 | ("WT3", "Window Type 3"), 102 | ) 103 | type = models.CharField( 104 | max_length=5, choices=CHOICES 105 | ) 106 | room = models.ForeignKey( 107 | Room, on_delete=models.CASCADE, related_name="windows") 108 | 109 | 110 | class WindowFittings (CommonInfo): 111 | CHOICES = ( 112 | ("WFT1", "Window Fittings Type 1"), 113 | ("WFT2", "Window Fittings Type 2"), 114 | ("WFT3", "Window Fittings Type 3"), 115 | ) 116 | type = models.CharField( 117 | max_length=5, choices=CHOICES 118 | ) 119 | windows = models.ManyToManyField(Window, related_name="fittings") 120 | 121 | 122 | class Chair (CommonInfo): 123 | CHOICES = ( 124 | ("CHT1", "Chair Type 1"), 125 | ("CHT2", "Chair Type 2"), 126 | ("CHT3", "Chair Type 3"), 127 | ) 128 | type = models.CharField( 129 | max_length=5, choices=CHOICES 130 | ) 131 | rooms = models.ManyToManyField(Room, related_name="chairs") 132 | 133 | 134 | class Bed (CommonInfo): 135 | CHOICES = ( 136 | ("BT1", "Bed Type 1"), 137 | ("BT2", "Bed Type 2"), 138 | ("BT3", "Bed Type 3"), 139 | ) 140 | type = models.CharField( 141 | max_length=5, choices=CHOICES 142 | ) 143 | rooms = models.ManyToManyField(Room, related_name="beds") 144 | 145 | 146 | class Table (CommonInfo): 147 | CHOICES = ( 148 | ("TBLT1", "Table Type 1"), 149 | ("TBLT2", "Table Type 2"), 150 | ("TBLT3", "Table Type 3"), 151 | ) 152 | type = models.CharField( 153 | max_length=5, choices=CHOICES 154 | ) 155 | rooms = models.ManyToManyField(Room, related_name="tables") 156 | ``` 157 | 158 | The test database contains 500 rooms and the same number of furniture items, as well as doors and souvenirs. See the details in the `app/rooms/management/commands/fill_db.py` file. In practice, the number of objects can be in the millions, and the models can be even more complex. 159 | 160 | ## Starting the system 161 | 162 | You can easily start the system using the `docker-compose up` command. After that, run the command `docker-compose run app sh -c "python manage.py fill_db 50"` to fill the database. Here 50 is the number of rooms and other items to create. After the command succeeds, you can play around with the API endpoints. 163 | 164 | In a separate terminal, you can run tests with the `docker-compose -f docker-compose.yml run app sh -c "pytest --random-order -s"` command. Also, explore the database using `pgadmin`. It is located at the `localhost:8080` address. 165 | 166 | There are several nuances: 167 | 1. The `pgadmin-data` directory must have owner and group 5050. Otherwise, `pgadmin` won't work. So, you'll have to run something like `sudo chown -R 5050:5050 ./pgadmin-data/`. 168 | 1. Also, the first time you set up the database access for `pgadmin`, specify the hostname `host.docker.internal`. 169 | 170 | ## The Django PostgreSQL Materialized View Solution 171 | First, you can explore how the required data is returned by the usual means of the Django REST framework. 172 | 173 | Take a look at the `app/rooms/serializers.py` and `app/rooms/views.py` files. In the serializers file, the `EagerLoadingMixin` class deserves attention. It effectively solves the common N+1 queries problem. 174 | 175 | The most important view is the `RoomViewSet` class. It overrides the standard `get_queryset` method to take advantage of the `EagerLoadingMixin` functionality. 176 | 177 | ```python 178 | class RoomViewSet(viewsets.ModelViewSet): 179 | serializer_class = RoomSerializer 180 | 181 | def get_queryset(self): 182 | qs = Room.objects.all() 183 | qs = self.serializer_class.setup_eager_loading(qs) 184 | return qs 185 | ``` 186 | 187 | Run the system in Docker containers using the supplied `docker-compose.yml` file. Please note the `pgadmin` container. It will make it easier for you to explore the database and experiment with it. Use the `localhost:8080/` endpoint to access the utility. 188 | 189 | After a successful start of the system, you can view information about all rooms at `localhost:8000/rooms/`. Also, it shows all the objects associated with the room. You can request information about individual rooms by their IDs, e.g., `localhost:8000/rooms/311/`. 190 | 191 | If you use the `localhost:8000/rooms/` endpoint, the system each time rebuilds all the data about the associated objects. The test database is small, so these queries are reasonably fast. In practice, they can be extremely slow and annoying for users. They can be optimized using a PostgreSQL materialized view or other methods. 192 | 193 | The `localhost:8000/rooms_mat_view/` endpoint returns the same information from the PostgreSQL materialized view. You can test the speed of query execution. One of the following sections describes how to do that. If you run the tests, you will see that this endpoint works much faster. 194 | 195 | ## The SQL Query That Creates the PostgreSQL Materialized View 196 | 197 | The database schema is quite complex, similar to those encountered in practice. We can easily prepare complex queries to retrieve data from the database using the standard Django REST framework tools. See the `app/rooms/serializers.py` file as an example. 198 | 199 | Achieving the same result with pure SQL code can be difficult. However, in some cases, speeding up queries execution may justify the effort. It depends on the relative frequency of requests to read and modify the data. 200 | 201 | ```SQL 202 | CREATE materialized VIEW rooms_related_objects AS WITH room_id_door_decoration AS ( 203 | SELECT 204 | room.id id, 205 | jsonb_build_object( 206 | 'id', door.id, 'name', door.name, 'width', 207 | door.width, 'length', door.length, 208 | 'height', door.height, 'type', door.type 209 | ) door, 210 | jsonb_build_object( 211 | 'id', 212 | decoration.id, 213 | 'name', 214 | decoration.name, 215 | 'souvenirs', 216 | jsonb_agg(souvenir) 217 | ) decoration 218 | FROM 219 | rooms_room room 220 | LEFT JOIN rooms_door door ON room.door_id = door.id 221 | LEFT JOIN rooms_decoration decoration ON room.decoration_id = decoration.id 222 | LEFT JOIN rooms_decoration_souvenirs ds ON decoration.id = ds.decoration_id 223 | LEFT JOIN rooms_souvenir souvenir ON ds.souvenir_id = souvenir.id 224 | GROUP BY 225 | room.id, 226 | door.id, 227 | decoration.id 228 | ), 229 | windows_in_room_by_id AS ( 230 | WITH room_window_fittings AS ( 231 | SELECT 232 | room_id, 233 | json_build_object( 234 | 'id', 235 | win.id, 236 | 'name', 237 | win.name, 238 | 'width', 239 | win.width, 240 | 'length', 241 | win.length, 242 | 'height', 243 | win.height, 244 | 'type', 245 | win.type, 246 | 'fittings', 247 | jsonb_agg(wf) 248 | ) window_in_room 249 | FROM 250 | rooms_window win 251 | LEFT JOIN rooms_windowfittings_windows rwfw ON rwfw.window_id = win.id 252 | LEFT JOIN rooms_windowfittings wf ON wf.id = rwfw.windowfittings_id 253 | GROUP BY 254 | room_id, 255 | win.id 256 | ) 257 | SELECT 258 | room_window_fittings.room_id AS id, 259 | jsonb_agg( 260 | room_window_fittings.window_in_room 261 | ) windows 262 | FROM 263 | room_window_fittings 264 | GROUP BY 265 | id 266 | ), 267 | parameters_and_beds_in_room_by_id AS ( 268 | SELECT 269 | room.id id, 270 | room.name name, 271 | room.width width, 272 | room.length length, 273 | room.height height, 274 | room.type type, 275 | jsonb_agg(bed) beds 276 | FROM 277 | rooms_room room 278 | LEFT JOIN rooms_bed_rooms bed_i ON room.id = bed_i.room_id 279 | LEFT JOIN rooms_bed bed ON bed.id = bed_i.bed_id 280 | GROUP BY 281 | room.id 282 | ), 283 | chairs_in_room_by_id AS ( 284 | SELECT 285 | room.id id, 286 | jsonb_agg(chair) chairs 287 | FROM 288 | rooms_room room 289 | LEFT JOIN rooms_chair_rooms chair_i ON room.id = chair_i.room_id 290 | LEFT JOIN rooms_chair chair ON chair.id = chair_i.chair_id 291 | GROUP BY 292 | room.id 293 | ), 294 | tables_in_room_by_id AS ( 295 | SELECT 296 | room.id id, 297 | jsonb_agg(table_f) tables 298 | FROM 299 | rooms_room room 300 | LEFT JOIN rooms_table_rooms table_i ON room.id = table_i.room_id 301 | LEFT JOIN rooms_table table_f ON table_f.id = table_i.table_id 302 | GROUP BY 303 | room.id 304 | ) 305 | SELECT 306 | COALESCE( 307 | room_id_door_decoration.id, windows_in_room_by_id.id 308 | ) AS id, 309 | room_id_door_decoration.door, 310 | room_id_door_decoration.decoration, 311 | windows_in_room_by_id.windows, 312 | parameters_and_beds_in_room_by_id.name, 313 | parameters_and_beds_in_room_by_id.width, 314 | parameters_and_beds_in_room_by_id.length, 315 | parameters_and_beds_in_room_by_id.height, 316 | parameters_and_beds_in_room_by_id.type, 317 | parameters_and_beds_in_room_by_id.beds, 318 | chairs_in_room_by_id.chairs, 319 | tables_in_room_by_id.tables 320 | FROM 321 | room_id_door_decoration FULL 322 | OUTER JOIN windows_in_room_by_id ON room_id_door_decoration.id = windows_in_room_by_id.id FULL 323 | OUTER JOIN parameters_and_beds_in_room_by_id ON windows_in_room_by_id.id = parameters_and_beds_in_room_by_id.id FULL 324 | OUTER JOIN chairs_in_room_by_id ON parameters_and_beds_in_room_by_id.id = chairs_in_room_by_id.id FULL 325 | OUTER JOIN tables_in_room_by_id ON chairs_in_room_by_id.id = tables_in_room_by_id.id WITH DATA; 326 | CREATE UNIQUE INDEX ON rooms_related_objects (id); 327 | REFRESH MATERIALIZED VIEW CONCURRENTLY rooms_related_objects; 328 | ``` 329 | 330 | Notes to the SQL query text: 331 | 1. To obtain the data from the many-to-many relationship, we use the aggregation function `jsonb_agg`. We cannot utilize this function multiple times in a row. So we have to generate several rather small WITH statements. For example, the statements `chairs_in_room_by_id` and `tables_in_room_by_id` cannot be combined. 332 | 1. It is desirable to create an index of the view. Otherwise, it will not be possible to update the view asynchronously (CONCURRENTLY). To create an index, we run the query `CREATE UNIQUE INDEX ON rooms_related_objects (id)`. 333 | 1. Unless you have a good reason to use the regular JSON, use the binary JSON. In other words, use the `jsonb_build_object` function instead of the `json_build_object` function. Otherwise, the `TypeError: JSON object must be str, bytes or bytearray, not dict` may occur, and you will have to tinker to get rid of it. 334 | 335 | You can explore the `rooms_related_objects` materialized view directly in the database using the `pgadmin` utility. The SQL code listed above is automatically executed during the initial database migration. See the file `app/rooms/migrations/0004_roomsrelatedobjects.py` for details. 336 | 337 | ## Using Django Signals to Automatically Update the View 338 | The SQL code `REFRESH MATERIALIZED VIEW CONCURRENTLY rooms_related_objects;` is automatically executed whenever data in the database changes. The Django signals `post_save` and `m2m_changed` are used for that. Here is the code from the file `app/rooms/signals.py`. 339 | 340 | ```python 341 | from django.db import connection 342 | from django.db.models.signals import post_save, m2m_changed 343 | from django.dispatch import receiver 344 | from rooms.models import WindowFittings, Window, Door, Souvenir, Decoration, Room, Chair, Bed, Table 345 | 346 | 347 | @receiver(post_save, sender=WindowFittings) 348 | @receiver(post_save, sender=Window) 349 | @receiver(post_save, sender=Door) 350 | @receiver(post_save, sender=Souvenir) 351 | @receiver(post_save, sender=Decoration) 352 | @receiver(post_save, sender=Room) 353 | @receiver(post_save, sender=Chair) 354 | @receiver(post_save, sender=Bed) 355 | @receiver(post_save, sender=Table) 356 | @receiver(m2m_changed, sender=Decoration.souvenirs.through) 357 | @receiver(m2m_changed, sender=WindowFittings.windows.through) 358 | @receiver(m2m_changed, sender=Chair.rooms.through) 359 | @receiver(m2m_changed, sender=Table.rooms.through) 360 | @receiver(m2m_changed, sender=Bed.rooms.through) 361 | def update_view_rooms_related_objects(sender, **kwargs): 362 | with connection.cursor() as cursor: 363 | cursor.execute( 364 | "REFRESH MATERIALIZED VIEW CONCURRENTLY rooms_related_objects;") 365 | ``` 366 | 367 | Also, in the file `app/rooms/apps.py` we instruct the system that that the signals have to be processed. 368 | 369 | ```python 370 | from django.apps import AppConfig 371 | 372 | 373 | class RoomsConfig(AppConfig): 374 | default_auto_field = 'django.db.models.BigAutoField' 375 | name = 'rooms' 376 | 377 | def ready(self): 378 | import rooms.signals 379 | ``` 380 | 381 | Below you will learn how to test the functioning of the signals. 382 | 383 | ## Creating a Django Model from the PostgreSQL Materialized View 384 | 385 | Our goal is to acquire the cached data from the PostgreSQL materialized view instead of re-executing heavy SQL queries. To accomplish that, we have to create one more Django model. It will work with the materialized view as with a regular database table, except for minor details. 386 | 387 | We don't have to build that model manually. To create models based on existing database tables, Django has a utility `inspectdb`. By default, it does not work with views. It needs to get the name of the view as a parameter. Save the results of its work by redirecting them to some file. 388 | 389 | You will probably need to make small changes manually in the resulting code. In particular, indicate that the `id` field is the primary key. In the end, you get something like the following model. 390 | 391 | ```python 392 | class RoomsRelatedObjects(models.Model): 393 | id = models.BigIntegerField(primary_key=True) 394 | door = models.JSONField(blank=True, null=True) 395 | decoration = models.JSONField(blank=True, null=True) 396 | windows = models.JSONField(blank=True, null=True) 397 | name = models.CharField(max_length=30, blank=True, null=True) 398 | width = models.FloatField(blank=True, null=True) 399 | length = models.FloatField(blank=True, null=True) 400 | height = models.FloatField(blank=True, null=True) 401 | type = models.CharField(max_length=5, blank=True, null=True) 402 | beds = models.JSONField(blank=True, null=True) 403 | chairs = models.JSONField(blank=True, null=True) 404 | tables = models.JSONField(blank=True, null=True) 405 | 406 | class Meta: 407 | managed = False # Created from a view. Don't remove. 408 | db_table = 'rooms_related_objects' 409 | ``` 410 | 411 | The property `managed = False` is essential in this case. It instructs Django not to try to create a table in the database. 412 | 413 | You can create the PostgreSQL materialized view directly in the database by executing the SQL code using the `pgadmin` utility. It is probably the best way. Or add the SQL code to the migration file. It will instruct Django to create the materialized view during the execution of the `migrate` command. You can find an example of that in the `app/rooms/migrations/0004_roomsrelatedobjects.py` file. 414 | 415 | To use the newly created model, we have to build its serializer and view. Also, register the view in the `urls.py` file. See the code of the classes `RoomsRelatedObjectsSerializer` and `RoomsRelatedObjectsViewSet`. There is nothing special about them. Note that the `RoomsRelatedObjectsViewSet` class is a `ReadOnlyModelViewSet` class inheritor, and not a `ModelViewSet` inheritor. 416 | 417 | ## Testing the Optimization Effect Manually 418 | You can get the rooms' data using the PostgreSQL materialized view at the address `localhost:8000/rooms_mat_view/`. You can see the complete data about all rooms at once. Also, try to request data about individual rooms using their IDs. Make sure the data matches the one at the address `localhost:8000/rooms/`. 419 | 420 | The `rooms` app has a simple middleware that logs all requests. It lives in the `app/rooms/middleware/log_execution_time.py` file. 421 | 422 | ```python 423 | import time 424 | import logging 425 | 426 | logger = logging.getLogger(__name__) 427 | 428 | 429 | class ExecutionTimeLogMiddleware: 430 | 431 | def __init__(self, get_response): 432 | self.get_response = get_response 433 | 434 | def __call__(self, request): 435 | start_time = time.time() 436 | response = self.get_response(request) 437 | end_time = time.time() 438 | time_diff = end_time - start_time 439 | logger.info(msg=str(";" + request.get_full_path()) + 440 | ";" + str(time_diff)) 441 | return response 442 | ``` 443 | 444 | The logs contain the paths as well as the execution time. They are in a file `logs_all_here.log`. 445 | 446 | ## Comparing the Average Request Execution Time 447 | This project currently contains several solutions to the data denormalization task: 448 | 1. PostgreSQL materialized view. 449 | 1. Django signals `post_save` and `m2m_changed`. 450 | 1. PostgreSQL triggers. 451 | 452 | You can compare the effectiveness of all these solutions by running just one Python script. 453 | 454 | The `supplementary_scripts` folder contains the file `compare_requests_time.py`. This script works as follows: 455 | 1. It connects to the database and gets a list of all the rooms IDs. 456 | 1. On each iteration of the loop, using that list, it makes requests to random rooms in the usual way and through all the solutions listed above. The logs of these requests are stored in the `/app/logs_all_here.log` file. 457 | 1. After performing the desired number of rooms data requests, the script splits the entries in the log file into several groups and calculates the average execution time for each of them. To do that, it uses the `pandas` library. 458 | 459 | Run the `compare_requests_time.py` script. You can change the number of request iterations by modifying the `--count` command line parameter. Its default value is 20. As an output, you will hopefully get something like the following. 460 | 461 | ![log processing output](/misc/log_processing_output.png) 462 | 463 | When using the optimized solutions, the mean query execution time decreases significantly. In real-world conditions, the difference is even more substantial. It grows exponentially as the number of rooms instances in the database increases. 464 | 465 | ## PostgreSQL Materialized View and Its Alternatives 466 | 467 | The PostgreSQL materialized view solution has a serious problem. Every time it receives a `post_save` or `m2m_changed` signal, it rebuilds *all* the fields for all rooms. It is redundant. 468 | 469 | Only some rooms are associated with the changed object. It is enough to rebuild the data on them. A real-world database may contain millions of rooms. So the inefficiency may waste significant resources. 470 | 471 | To solve the assignment effectively, I have created one more model. It is called `RoomWithRelatedObjsRebuildInApp`. It is very similar to the `RoomsRelatedObjectsMaterializedView` model. However, it is fully controlled by Django. It has a default `managed = True` property. 472 | 473 | In the `app/rooms/signals.py` file, you can find several functions that update the fields of instances of this model as needed. For example, the `@receiver(post_save, sender=Bed)` signal is generated when the bed changes. It triggers the corresponding function. First, that function gets a list of IDs for all the rooms related to that bed. Then it rebuilds the `beds` field in `RoomWithRelatedObjsRebuildInApp` model instances that have IDs from the list. 474 | 475 | The `m2m_changed` signal is also processed. For example, the `@receiver(m2m_changed, sender=Decoration.souvenirs.through)` signal triggers the `decoration_souvenir_m2m_changed_update_related_rooms` function. 476 | 477 | Such an approach is flexible. Also, it helps save computing resources. Although, it required the developer to write and maintain more functions. Some of those functions are complicated because the database schema is bloated. However, the alternative is to write pure SQL code, which is not an easy task either. 478 | 479 | You can use a PostgreSQL materialized view if you don't need to recalculate its data immediately after one of its related objects changes. This solution may be suitable if the requirements for the freshness of the cached data are not very strict. Perhaps in your case, it is enough to update the materialized view, for example, once a day or once a week. 480 | 481 | Perhaps you have strict requirements for the freshness of cached data. The boss says that the cached data must be 100% up to date at all times. In such a case, use the alternative solution described in this section. You can find examples of the functions that process signals in the `app/rooms/signals.py` file. 482 | 483 | ## Testing the Django Signals 484 | 485 | The PostgreSQL materialized view should be updated automatically every time the data in the database changes. The Django signals `post_save` and `m2m_changed` are used for that, as described above. The project contains several `pytest` tests. Also, you can manually check the functioning of the signals. 486 | 487 | To test manually, update the names of some souvenirs or tables in the browser. Use the endpoint `localhost:8000/souvenirs/` or `localhost:8000/tables/` to send the PUT or PATCH requests. After that, ensure that the information on the rooms at the `localhost:8000/rooms_mat_view/` and `localhost:8000/rooms_v2/` endpoints has also been updated. --------------------------------------------------------------------------------