├── pggraph ├── __init__.py ├── db │ ├── __init__.py │ ├── base.py │ ├── build_references.py │ └── archiver.py ├── tests │ ├── __init__.py │ └── test_api.py ├── utils │ ├── __init__.py │ ├── classes │ │ ├── __init__.py │ │ ├── foreign_key.py │ │ └── base.py │ ├── action_enum.py │ └── funcs.py ├── config.py ├── main.py └── api.py ├── requirements.txt ├── config.test.ini ├── Dockerfile ├── CHANGELOG.md ├── docker-compose.test.yml ├── LICENSE.md ├── setup.py ├── .gitignore ├── .dockerignore ├── conftest.py └── README.md /pggraph/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pggraph/db/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pggraph/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pggraph/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pggraph/utils/classes/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | psycopg2-binary~=2.8 2 | dataclasses>=0.5 3 | pytest~=5.4 4 | pytest-cov~=2.10 5 | -------------------------------------------------------------------------------- /config.test.ini: -------------------------------------------------------------------------------- 1 | [db] 2 | host = localhost 3 | port = 54321 4 | user = postgres 5 | password = postgres 6 | dbname = books 7 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright Ⓒ 2020 "Sberbank Real Estate Center" Limited Liability Company. Licensed under the MIT license. 2 | # Please, see the LICENSE.md file in project's root for full licensing information. 3 | FROM python:3.8-slim 4 | 5 | # Copy all necessary files 6 | COPY ./ /app/ 7 | WORKDIR /app/ 8 | 9 | # Install python requirements 10 | RUN pip3 install -U pip && pip3 install --no-cache-dir -r requirements.txt 11 | -------------------------------------------------------------------------------- /pggraph/utils/classes/foreign_key.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright Ⓒ 2020 "Sberbank Real Estate Center" Limited Liability Company. Licensed under the MIT license. 3 | Please, see the LICENSE.md file in project's root for full licensing information. 4 | """ 5 | from dataclasses import dataclass 6 | 7 | 8 | @dataclass 9 | class ForeignKey: 10 | pk_main: str # Primary Key 11 | pk_ref: str # referring table Primary Key 12 | fk_ref: str # referring table Foreign Key 13 | fk_name: str # foreign key name 14 | 15 | -------------------------------------------------------------------------------- /pggraph/utils/action_enum.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright Ⓒ 2020 "Sberbank Real Estate Center" Limited Liability Company. Licensed under the MIT license. 3 | Please, see the LICENSE.md file in project's root for full licensing information. 4 | """ 5 | from enum import Enum 6 | 7 | 8 | class ActionEnum(Enum): 9 | archive_table = 'archive_table' 10 | get_table_references = 'get_table_references' 11 | get_rows_references = 'get_rows_references' 12 | 13 | @classmethod 14 | def list_values(cls): 15 | return [k.value for k in cls] 16 | -------------------------------------------------------------------------------- /pggraph/utils/funcs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright Ⓒ 2020 "Sberbank Real Estate Center" Limited Liability Company. Licensed under the MIT license. 3 | Please, see the LICENSE.md file in project's root for full licensing information. 4 | """ 5 | from distutils.util import strtobool 6 | 7 | 8 | def chunks(elems: list, step_size: int): 9 | """Yield successive n-sized chunks from l.""" 10 | for i in range(0, len(elems), step_size): 11 | yield elems[i:i + step_size] 12 | 13 | 14 | def arg_to_bool(value: str, default_value: bool = False) -> bool: 15 | return bool(strtobool(value or str(default_value))) 16 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 0.1.7 (22 июля 2024) 2 | 3 | Исправление записи списков словарей при переносе строк в архвиную таблицу 4 | 5 | # 0.1.6 (22 декабря 2020) 6 | 7 | Блокировка на уровне строк при архивации, исправление получения foreign keys 8 | 9 | # 0.1.5 (1 декабря 2020) 10 | 11 | Исправление для старых версий psycopg2 12 | 13 | # 0.1.4 (1 декабря 2020) 14 | 15 | Исправление получения primary keys для таблиц 16 | 17 | # 0.1.3 (1 декабря 2020) 18 | 19 | Добавление возможности создания Config из словаря 20 | 21 | # 0.1.2 (29 июля 2020) 22 | 23 | Исправление запроса для получения Foreign Keys (функция get_all_fk) 24 | 25 | # 0.1.1 (29 июля 2020) 26 | 27 | Исправление ошибки KeyError в build_references 28 | 29 | # 0.1.0 (10 июля 2020) 30 | 31 | Первая версия 32 | -------------------------------------------------------------------------------- /docker-compose.test.yml: -------------------------------------------------------------------------------- 1 | # Copyright Ⓒ 2020 "Sberbank Real Estate Center" Limited Liability Company. Licensed under the MIT license. 2 | # Please, see the LICENSE.md file in project's root for full licensing information. 3 | version: "3" 4 | 5 | services: 6 | test: 7 | image: pggraph 8 | build: 9 | context: . 10 | dockerfile: ./Dockerfile 11 | command: ["pytest", "-s"] 12 | depends_on: 13 | - test_db 14 | networks: 15 | - test_db_nw 16 | 17 | test_db: 18 | image: postgres:12 19 | command: 20 | postgres -c max_connections=300 21 | environment: 22 | POSTGRES_HOST_AUTH_METHOD: trust 23 | expose: 24 | - 5432 25 | networks: 26 | - test_db_nw 27 | 28 | networks: 29 | test_db_nw: 30 | driver: bridge 31 | -------------------------------------------------------------------------------- /pggraph/db/base.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright Ⓒ 2020 "Sberbank Real Estate Center" Limited Liability Company. Licensed under the MIT license. 3 | Please, see the LICENSE.md file in project's root for full licensing information. 4 | """ 5 | import logging 6 | 7 | import psycopg2 8 | from psycopg2._psycopg import connection 9 | from psycopg2.extras import DictCursor, LoggingConnection 10 | 11 | from pggraph.config import Config 12 | 13 | 14 | def get_db_conn(config: Config, with_db: bool = True, with_schema: bool = False) -> connection: 15 | db_config_dict = config.db_config.as_dict().copy() 16 | if not with_db: 17 | db_config_dict.pop('dbname') 18 | if not with_schema: 19 | db_config_dict.pop('schema') 20 | 21 | conn = psycopg2.connect(**db_config_dict, cursor_factory=DictCursor, connection_factory=LoggingConnection) 22 | conn.initialize(logging.getLogger()) 23 | 24 | return conn 25 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright Ⓒ 2020 "Sberbank Real Estate Center" Limited Liability Company. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright Ⓒ 2020 "Sberbank Real Estate Center" Limited Liability Company. Licensed under the MIT license. 3 | Please, see the LICENSE.md file in project's root for full licensing information. 4 | """ 5 | from setuptools import setup, find_packages 6 | 7 | # read the contents of your README file 8 | from os import path 9 | this_directory = path.abspath(path.dirname(__file__)) 10 | with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: 11 | long_description = f.read() 12 | 13 | 14 | setup( 15 | name="pggraph", 16 | description="Утилита для работы с зависимостями таблиц в PostgreSQL", 17 | long_description=long_description, 18 | long_description_content_type='text/markdown', 19 | version="0.1.7", 20 | author='"Sberbank Real Estate Center" Limited Liability Company omborzov@domclick.ru', 21 | author_email='omborzov@domclick.ru', 22 | url='https://github.com/domclick/pggraph', 23 | license='MIT', 24 | classifiers=[ 25 | 'License :: OSI Approved :: MIT License', 26 | 'Intended Audience :: Developers', 27 | 'Operating System :: OS Independent', 28 | 'Programming Language :: Python', 29 | 'Programming Language :: Python :: 3', 30 | 'Programming Language :: Python :: 3.6', 31 | 'Programming Language :: Python :: 3.7', 32 | 'Programming Language :: Python :: 3.8', 33 | 'Topic :: Database', 34 | 'Topic :: Utilities' 35 | ], 36 | packages=find_packages(), 37 | install_requires=[ 38 | "psycopg2-binary>=2.8", 39 | "dataclasses>=0.5", 40 | ], 41 | entry_points={'console_scripts': ['pggraph=pggraph.main:main']} 42 | ) 43 | -------------------------------------------------------------------------------- /pggraph/utils/classes/base.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright Ⓒ 2020 "Sberbank Real Estate Center" Limited Liability Company. Licensed under the MIT license. 3 | Please, see the LICENSE.md file in project's root for full licensing information. 4 | """ 5 | from configparser import ConfigParser, NoSectionError, NoOptionError 6 | 7 | 8 | class BaseConfig: 9 | def as_dict(self): 10 | return self.__dict__ 11 | 12 | @classmethod 13 | def from_config(cls, config: ConfigParser, section: str): 14 | fields = {} 15 | for field in cls.__annotations__: 16 | try: 17 | fields[field] = config.get(section, field) 18 | except (NoSectionError, NoOptionError): 19 | if hasattr(cls, field): 20 | fields[field] = getattr(cls, field) 21 | else: 22 | raise KeyError(f'{field} is required') 23 | 24 | return cls(**fields) 25 | 26 | @classmethod 27 | def from_dict(cls, config_data: dict): 28 | fields = {} 29 | for field, field_type in cls.__annotations__.items(): 30 | if field in config_data: 31 | value = config_data.get(field) 32 | elif hasattr(cls, field): 33 | value = getattr(cls, field) 34 | else: 35 | raise KeyError(f'{field} is required') 36 | 37 | if not isinstance(value, field_type): 38 | value_type = value.__class__.__name__ 39 | correct_type = field_type.__name__ 40 | raise ValueError(f'{field} value has incorrect type {value_type}, correct type - {correct_type}') 41 | 42 | fields[field] = value 43 | 44 | return cls(**fields) 45 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Custom 2 | *.DS_Store 3 | *.ini 4 | !config.test.ini 5 | .idea/ 6 | 7 | # Archives 8 | *.tar 9 | *.gz 10 | *.zip 11 | *.rar 12 | 13 | # Byte-compiled / optimized / DLL files 14 | *__pycache__/ 15 | *.idea/ 16 | *.py[cod] 17 | *$py.class 18 | *.pyc 19 | 20 | # C extensions 21 | *.so 22 | 23 | # Distribution / packaging 24 | .Python 25 | env/ 26 | build/ 27 | develop-eggs/ 28 | dist/ 29 | downloads/ 30 | eggs/ 31 | .eggs/ 32 | lib/ 33 | lib64/ 34 | parts/ 35 | !*/templates/**/parts/ 36 | sdist/ 37 | var/ 38 | wheels/ 39 | *.egg-info/ 40 | .installed.cfg 41 | *.egg 42 | 43 | # PyInstaller 44 | # Usually these files are written by a python script from a template 45 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 46 | *.manifest 47 | *.spec 48 | 49 | # Installer logs 50 | pip-log.txt 51 | pip-delete-this-directory.txt 52 | 53 | # Unit test / coverage reports 54 | htmlcov/ 55 | .tox/ 56 | .coverage 57 | .coverage.* 58 | .cache 59 | nosetests.xml 60 | coverage.xml 61 | *.cover 62 | .hypothesis/ 63 | .pytest_cache/ 64 | 65 | # Translations 66 | *.mo 67 | *.pot 68 | 69 | # Django stuff: 70 | *.log 71 | local_settings.py 72 | 73 | # Flask stuff: 74 | instance/ 75 | .webassets-cache 76 | 77 | # Scrapy stuff: 78 | .scrapy 79 | 80 | # Sphinx documentation 81 | docs/_build/ 82 | 83 | # PyBuilder 84 | target/ 85 | 86 | # Jupyter Notebook 87 | .ipynb_checkpoints 88 | 89 | # pyenv 90 | .python-version 91 | 92 | # celery beat schedule file 93 | celerybeat-schedule 94 | 95 | # SageMath parsed files 96 | *.sage.py 97 | 98 | # dotenv 99 | .env 100 | 101 | # virtualenv 102 | .venv 103 | venv/ 104 | ENV/ 105 | 106 | # Spyder project settings 107 | .spyderproject 108 | .spyproject 109 | 110 | # Rope project settings 111 | .ropeproject 112 | 113 | # mkdocs documentation 114 | /site 115 | 116 | # mypy 117 | .mypy_cache/ 118 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # Custom 2 | *.db 3 | *.DS_Store 4 | .* 5 | !.ci-cd 6 | *.txt 7 | !requirements.txt 8 | Dockerfile 9 | venv*/ 10 | tests/ 11 | *.yml 12 | *.DS_Store 13 | *.ini 14 | .idea/ 15 | 16 | 17 | # Archives 18 | *.tar 19 | *.gz 20 | *.zip 21 | *.rar 22 | 23 | # Byte-compiled / optimized / DLL files 24 | *__pycache__/ 25 | *.idea/ 26 | *.py[cod] 27 | *$py.class 28 | *.pyc 29 | 30 | # C extensions 31 | *.so 32 | 33 | # Distribution / packaging 34 | .Python 35 | env/ 36 | build/ 37 | develop-eggs/ 38 | dist/ 39 | downloads/ 40 | eggs/ 41 | .eggs/ 42 | lib/ 43 | lib64/ 44 | parts/ 45 | !templates/parts/ 46 | sdist/ 47 | var/ 48 | wheels/ 49 | *.egg-info/ 50 | .installed.cfg 51 | *.egg 52 | 53 | # PyInstaller 54 | # Usually these files are written by a python script from a template 55 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 56 | *.manifest 57 | *.spec 58 | 59 | # Installer logs 60 | pip-log.txt 61 | pip-delete-this-directory.txt 62 | 63 | # Unit test / coverage reports 64 | htmlcov/ 65 | .tox/ 66 | .coverage 67 | .coverage.* 68 | .cache 69 | nosetests.xml 70 | coverage.xml 71 | *.cover 72 | .hypothesis/ 73 | .pytest_cache/ 74 | 75 | # Translations 76 | *.mo 77 | *.pot 78 | 79 | # Django stuff: 80 | *.log 81 | local_settings.py 82 | 83 | # Flask stuff: 84 | instance/ 85 | .webassets-cache 86 | 87 | # Scrapy stuff: 88 | .scrapy 89 | 90 | # Sphinx documentation 91 | docs/_build/ 92 | 93 | # PyBuilder 94 | target/ 95 | 96 | # Jupyter Notebook 97 | .ipynb_checkpoints 98 | 99 | # pyenv 100 | .python-version 101 | 102 | # celery beat schedule file 103 | celerybeat-schedule 104 | 105 | # SageMath parsed files 106 | *.sage.py 107 | 108 | # dotenv 109 | .env 110 | 111 | # virtualenv 112 | .venv 113 | venv/ 114 | ENV/ 115 | 116 | # Spyder project settings 117 | .spyderproject 118 | .spyproject 119 | 120 | # Rope project settings 121 | .ropeproject 122 | 123 | # mkdocs documentation 124 | /site 125 | 126 | # mypy 127 | .mypy_cache/ 128 | -------------------------------------------------------------------------------- /pggraph/config.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright Ⓒ 2020 "Sberbank Real Estate Center" Limited Liability Company. Licensed under the MIT license. 3 | Please, see the LICENSE.md file in project's root for full licensing information. 4 | """ 5 | from configparser import ConfigParser 6 | from dataclasses import dataclass 7 | 8 | from pggraph.utils.classes.base import BaseConfig 9 | from pggraph.utils.funcs import arg_to_bool 10 | 11 | 12 | class Config: 13 | db_config: "DBConfig" 14 | archiver_config: "ArchiverConfig" 15 | 16 | def __init__(self, config_path: str = None, config_data: dict = None): 17 | if config_data: 18 | self.from_dict(config_data) 19 | elif config_path: 20 | self.from_ini(config_path) 21 | else: 22 | raise ValueError('config_path or config_data should be set') 23 | 24 | def from_ini(self, config_path: str): 25 | config = ConfigParser() 26 | config.read(config_path) 27 | self.db_config = DBConfig.from_config(config, 'db') 28 | self.archiver_config = ArchiverConfig.from_config(config, 'archive') 29 | 30 | def from_dict(self, config_data: dict): 31 | if not isinstance(config_data, dict): 32 | raise ValueError(f'config_data has incorrect type {config_data.__class__.__name__} (should be dict)') 33 | 34 | if 'db' in config_data: 35 | self.db_config = DBConfig.from_dict(config_data['db']) 36 | else: 37 | raise KeyError('config_data should contain db settings') 38 | 39 | self.archiver_config = ArchiverConfig.from_dict(config_data.get('archive', {})) 40 | 41 | 42 | @dataclass 43 | class DBConfig(BaseConfig): 44 | host: str 45 | port: int 46 | user: str 47 | password: str 48 | dbname: str 49 | schema: str = 'public' 50 | 51 | 52 | @dataclass 53 | class ArchiverConfig(BaseConfig): 54 | is_debug: bool = False 55 | chunk_size: int = 1000 56 | max_depth: int = 20 57 | to_archive: bool = True 58 | archive_suffix: str = 'archive' 59 | 60 | @classmethod 61 | def from_config(cls, config: ConfigParser, section: str): 62 | conf = super().from_config(config, section) 63 | conf.is_debug = arg_to_bool(str(conf.is_debug), default_value=cls.is_debug) 64 | conf.chunk_size = int(conf.chunk_size) 65 | conf.max_depth = int(conf.max_depth) 66 | conf.to_archive = arg_to_bool(str(conf.to_archive), default_value=cls.to_archive) 67 | return conf 68 | -------------------------------------------------------------------------------- /pggraph/main.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright Ⓒ 2020 "Sberbank Real Estate Center" Limited Liability Company. Licensed under the MIT license. 3 | Please, see the LICENSE.md file in project's root for full licensing information. 4 | """ 5 | from argparse import ArgumentParser, Namespace 6 | import logging 7 | from logging import handlers 8 | import os 9 | from pprint import pprint 10 | 11 | from pggraph.api import PgGraphApi 12 | from pggraph.utils.action_enum import ActionEnum 13 | 14 | 15 | def main(): 16 | args = parse_args() 17 | setup_logging(args.log_level, args.log_path) 18 | 19 | pg_graph_api = PgGraphApi(config_path=args.config_path) 20 | result = pg_graph_api.run_action(args) 21 | 22 | pprint(result) 23 | 24 | 25 | def setup_logging(log_level: str = 'INFO', log_path: str = None): 26 | log_handlers = [logging.StreamHandler()] 27 | if log_path: 28 | log_path = os.path.join(log_path, "pggraph.log") 29 | log_handlers.append( 30 | logging.handlers.RotatingFileHandler(log_path, maxBytes=1000000, backupCount=3, encoding="UTF-8") 31 | ) 32 | 33 | logging.basicConfig(handlers=log_handlers, 34 | level=log_level or logging.INFO, 35 | format='%(asctime)s %(levelname)s: %(message)s', 36 | datefmt='%Y-%m-%d %H:%M:%S' 37 | ) 38 | 39 | logging.getLogger('psycopg2').setLevel(log_level) 40 | 41 | 42 | def parse_args() -> Namespace: 43 | parser = ArgumentParser() 44 | parser.add_argument( 45 | "action", 46 | type=str, 47 | help=f"required action: {', '.join(ActionEnum.list_values())}", 48 | ) 49 | parser.add_argument( 50 | "--table", 51 | type=str, 52 | default=None, 53 | help="table name", 54 | required=True, 55 | ) 56 | parser.add_argument( 57 | "--ids", 58 | type=str, 59 | default=None, 60 | help="primary key ids, separated by comma, e.g. 1,2,3", 61 | ) 62 | parser.add_argument( 63 | "--config_path", 64 | type=str, 65 | default='config.ini', 66 | help="path to config.ini", 67 | ) 68 | parser.add_argument( 69 | "--log_path", 70 | type=str, 71 | default=None, 72 | help="path to log dir", 73 | ) 74 | parser.add_argument( 75 | "--log_level", 76 | type=str, 77 | default='info', 78 | help="log level (debug, info, error)", 79 | ) 80 | args = parser.parse_args() 81 | 82 | args.action = ActionEnum[args.action] 83 | if args.ids: 84 | args.ids = [int(id_) for id_ in str(args.ids).split(',')] 85 | if args.log_level: 86 | args.log_level = str(args.log_level).upper() 87 | 88 | return args 89 | 90 | 91 | if __name__ == "__main__": 92 | main() 93 | 94 | -------------------------------------------------------------------------------- /pggraph/tests/test_api.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright Ⓒ 2020 "Sberbank Real Estate Center" Limited Liability Company. Licensed under the MIT license. 3 | Please, see the LICENSE.md file in project's root for full licensing information. 4 | """ 5 | from unittest.mock import ANY 6 | 7 | from pggraph.api import PgGraphApi 8 | from pggraph.db.base import get_db_conn 9 | from pggraph.utils.classes.foreign_key import ForeignKey 10 | 11 | 12 | def test_get_table_references(): 13 | api = PgGraphApi(config_path='config.test.ini') 14 | 15 | publisher_refs = api.get_table_references('publisher') 16 | assert publisher_refs == { 17 | 'in_refs': { 18 | 'book': [ForeignKey(pk_main='id', pk_ref='id', fk_ref='publisher_id', fk_name=ANY)] 19 | }, 20 | 'out_refs': {} 21 | } 22 | 23 | author_book_refs = api.get_table_references('author_book') 24 | assert author_book_refs == { 25 | 'in_refs': {}, 26 | 'out_refs': { 27 | 'book': [ForeignKey(pk_main='id', pk_ref='author_id, book_id', fk_ref='book_id', fk_name=ANY)], 28 | 'author': [ForeignKey(pk_main='id', pk_ref='author_id, book_id', fk_ref='author_id', fk_name=ANY)] 29 | } 30 | } 31 | 32 | 33 | def test_get_rows_references(): 34 | api = PgGraphApi(config_path='config.test.ini') 35 | 36 | publisher_refs = api.get_rows_references('publisher', [1, 2]) 37 | assert publisher_refs == { 38 | 1: {'book': {'publisher_id': [{'id': 1, 'publisher_id': 1}, {'id': 2, 'publisher_id': 1}]}}, 39 | 2: {'book': {'publisher_id': [{'id': 3, 'publisher_id': 2}]}} 40 | } 41 | 42 | author_refs = api.get_rows_references('author', [1, 2]) 43 | assert author_refs == { 44 | 1: {'author_book': {'author_id': [{'author_id': 1, 'book_id': 1}]}}, 45 | 2: {'author_book': {'author_id': [{'author_id': 2, 'book_id': 1}]}} 46 | } 47 | 48 | 49 | def test_archive_table(): 50 | api = PgGraphApi(config_path='config.test.ini') 51 | 52 | api.archive_table('publisher', [1, 2]) 53 | conn = get_db_conn(api.config) 54 | with conn.cursor() as cursor: 55 | cursor.execute('SELECT author_id, book_id FROM author_book;') 56 | ab_rows = [dict(row) for row in cursor.fetchall()] 57 | 58 | cursor.execute('SELECT author_id, book_id FROM author_book_archive;') 59 | ab_archive_rows = [dict(row) for row in cursor.fetchall()] 60 | 61 | cursor.execute('SELECT id FROM book;') 62 | book_rows = [dict(row) for row in cursor.fetchall()] 63 | 64 | cursor.execute('SELECT id FROM book_archive;') 65 | book_archive_rows = [dict(row) for row in cursor.fetchall()] 66 | 67 | cursor.execute('SELECT id FROM publisher;') 68 | pub_rows = [dict(row) for row in cursor.fetchall()] 69 | 70 | cursor.execute('SELECT id FROM publisher_archive;') 71 | pub_archive_rows = [dict(row) for row in cursor.fetchall()] 72 | 73 | conn.close() 74 | 75 | assert ab_rows == [{'author_id': 7, 'book_id': 4}, {'author_id': 7, 'book_id': 5}] 76 | assert ab_archive_rows == [ 77 | {'author_id': 1, 'book_id': 1}, {'author_id': 2, 'book_id': 1}, {'author_id': 3, 'book_id': 2}, 78 | {'author_id': 4, 'book_id': 2}, {'author_id': 5, 'book_id': 3}, {'author_id': 6, 'book_id': 3} 79 | ] 80 | 81 | assert book_rows == [{'id': 4}, {'id': 5}] 82 | assert book_archive_rows == [{'id': 1}, {'id': 2}, {'id': 3}] 83 | 84 | assert pub_rows == [{'id': 3}] 85 | assert pub_archive_rows == [{'id': 1}, {'id': 2}] 86 | -------------------------------------------------------------------------------- /conftest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright Ⓒ 2020 "Sberbank Real Estate Center" Limited Liability Company. Licensed under the MIT license. 3 | Please, see the LICENSE.md file in project's root for full licensing information. 4 | """ 5 | import pytest 6 | 7 | from pggraph.config import Config 8 | from pggraph.db.base import get_db_conn 9 | 10 | 11 | @pytest.fixture(scope="session", autouse=True) 12 | def test_db(): 13 | """ 14 | This fixture will be executed once in the entire suite, independently of the filters you use 15 | running pytest. It will create the test_db at the beginning and droping the table at the end 16 | of all tests. 17 | """ 18 | config = Config('config.test.ini') 19 | _create_db(config) 20 | _clear_tables(config) 21 | _fill_db(config) 22 | yield 23 | _drop_db(config) 24 | 25 | 26 | def _create_db(config): 27 | connection = get_db_conn(config, with_db=False) 28 | connection.autocommit = True 29 | try: 30 | with connection.cursor() as cursor: 31 | cursor.execute(f"CREATE DATABASE {config.db_config.dbname};") 32 | except Exception as error: 33 | if not hasattr(error, "pgerror") or "already exists" not in error.pgerror: 34 | raise error 35 | print("Database '%s' already exists.", config.db_config.dbname) 36 | finally: 37 | connection.close() 38 | 39 | 40 | def _kill_connections(config): 41 | connection = get_db_conn(config) 42 | connection.autocommit = True 43 | try: 44 | with connection.cursor() as cursor: 45 | cursor.execute( 46 | "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = %s;", 47 | (config.db_config.dbname, ) 48 | ) 49 | except Exception as err: 50 | print('error while kill conns', err) 51 | 52 | 53 | def _drop_db(config): 54 | connection = get_db_conn(config, with_db=False) 55 | connection.autocommit = True 56 | try: 57 | with connection.cursor() as cursor: 58 | cursor.execute(f"DROP DATABASE {config.db_config.dbname};") 59 | finally: 60 | connection.close() 61 | 62 | 63 | def _clear_tables(config): 64 | connection = get_db_conn(config) 65 | connection.autocommit = True 66 | try: 67 | with connection.cursor() as cursor: 68 | cursor.execute(f""" 69 | DROP TABLE IF EXISTS publisher CASCADE; 70 | DROP TABLE IF EXISTS publisher_archive CASCADE; 71 | DROP TABLE IF EXISTS book CASCADE; 72 | DROP TABLE IF EXISTS book_archive CASCADE; 73 | DROP TABLE IF EXISTS author CASCADE; 74 | DROP TABLE IF EXISTS author_archive CASCADE; 75 | DROP TABLE IF EXISTS author_book CASCADE; 76 | DROP TABLE IF EXISTS author_book_archive CASCADE; 77 | """) 78 | except Exception as error: 79 | if not hasattr(error, "pgerror") or "does not exist" not in error.pgerror: 80 | raise error 81 | print("Database '%s' does not exist.", config.db_config.dbname) 82 | finally: 83 | connection.close() 84 | 85 | 86 | def _fill_db(config): 87 | connection = get_db_conn(config) 88 | connection.autocommit = True 89 | try: 90 | with connection.cursor() as cursor: 91 | cursor.execute(""" 92 | CREATE TABLE IF NOT EXISTS publisher ( 93 | id serial PRIMARY KEY, 94 | name text NOT NULL 95 | ); 96 | 97 | CREATE TABLE IF NOT EXISTS book ( 98 | id serial PRIMARY KEY, 99 | name text NOT NULL, 100 | publisher_id integer REFERENCES publisher (id) 101 | ); 102 | 103 | CREATE TABLE IF NOT EXISTS author ( 104 | id serial PRIMARY KEY, 105 | fio text NOT NULL 106 | ); 107 | 108 | CREATE TABLE IF NOT EXISTS author_book ( 109 | author_id integer REFERENCES author (id), 110 | book_id integer REFERENCES book (id), 111 | PRIMARY KEY (author_id, book_id) 112 | ); 113 | 114 | INSERT INTO publisher (id, name) VALUES (1, 'O Reilly'), (2, 'Packt'), (3, 'Bloomsbury'); 115 | INSERT INTO book (id, name, publisher_id) VALUES 116 | (1, 'High Performance Python', 1), 117 | (2, 'Kubernetes: Up and Running', 1), 118 | (3, 'Python Machine Learning', 2), 119 | (4, 'Harry Potter and the Philosophers Stone', 3), 120 | (5, 'Harry Potter and the Chamber of Secrets', 3); 121 | INSERT INTO author (id, fio) VALUES 122 | (1, 'Ian Ozsvald'), (2, 'Micha Gorelick'), 123 | (3, 'Brendan Burns'), (4, 'Joe Beda'), 124 | (5, 'Sebastian Raschka'), (6, 'Vahid Mirjalili'), 125 | (7, 'J.K. Rowling'); 126 | INSERT INTO author_book (author_id, book_id) VALUES 127 | (1, 1), (2, 1), 128 | (3, 2), (4, 2), 129 | (5, 3), (6, 3), 130 | (7, 4), (7, 5); 131 | """) 132 | finally: 133 | connection.close() 134 | -------------------------------------------------------------------------------- /pggraph/api.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright Ⓒ 2020 "Sberbank Real Estate Center" Limited Liability Company. Licensed under the MIT license. 3 | Please, see the LICENSE.md file in project's root for full licensing information. 4 | """ 5 | import logging 6 | from argparse import Namespace 7 | from typing import List, Dict 8 | 9 | from psycopg2.extras import DictCursor 10 | from psycopg2.sql import SQL 11 | 12 | from pggraph.db import build_references as br 13 | from pggraph.config import Config 14 | from pggraph.db.archiver import Archiver 15 | from pggraph.db.base import get_db_conn 16 | from pggraph.utils.action_enum import ActionEnum 17 | from pggraph.utils.funcs import chunks 18 | 19 | 20 | class PgGraphApi: 21 | config: Config 22 | references: Dict[str, dict] 23 | primary_keys: Dict[str, str] 24 | 25 | def __init__(self, config_path: str = None, config: Config = None): 26 | if config_path: 27 | self.config = Config(config_path) 28 | elif config: 29 | self.config = config 30 | else: 31 | raise ValueError('config or config_path should be set') 32 | 33 | result = br.build_references(config=self.config) 34 | self.references = result['references'] 35 | self.primary_keys = result['primary_keys'] 36 | 37 | def run_action(self, args: Namespace): 38 | if args.action == ActionEnum.archive_table: 39 | return self.archive_table(args.table, ids=args.ids) 40 | elif args.action == ActionEnum.get_rows_references: 41 | return self.get_rows_references(args.table, ids=args.ids) 42 | elif args.action == ActionEnum.get_table_references: 43 | return self.get_table_references(args.table) 44 | else: 45 | raise NotImplementedError(f'Unknown action {args.action}') 46 | 47 | def archive_table(self, table_name, ids: List[int]): 48 | """ 49 | Recursive iterative archiving / deleting rows by %ids% from %table_name% table and related tables. 50 | pk_column - %table_name% primary key 51 | """ 52 | conn = get_db_conn(self.config) 53 | 54 | try: 55 | logging.info(f'{table_name} - START') 56 | 57 | pk_column = self.primary_keys.get(table_name) 58 | if not pk_column: 59 | raise KeyError(f'Primary key for table {table_name} not found') 60 | 61 | archiver = Archiver(conn, self.references, self.config) 62 | rows = [{pk_column: id_} for id_ in ids] 63 | 64 | for rows_chunk in chunks(rows, self.config.archiver_config.chunk_size): 65 | archiver.archive_recursive(table_name, rows_chunk, pk_column) 66 | 67 | logging.info(f'{table_name} - END') 68 | finally: 69 | conn.close() 70 | 71 | def get_table_references(self, table_name: str): 72 | """ 73 | Get table references: 74 | - referencing tables (in_refs) 75 | - tables referenced by current (out_refs) 76 | 77 | Result (table_name = table_a): 78 | { 79 | 'in_refs': { 80 | 'table_b': [ForeignKey(pk_main='id', pk_ref='id', fk_ref='table_a_id')], 81 | 'table_c': [ForeignKey(pk_main='id', pk_ref='id', fk_ref='a_id')] 82 | }, 83 | 'out_refs': { 84 | 'table_d': [ForeignKey(pk_main='id', pk_ref='id', fk_ref='d_id')], 85 | 'table_e': [ForeignKey(pk_main='id', pk_ref='id', fk_ref='table_e_id')], 86 | } 87 | } 88 | """ 89 | if table_name not in self.references: 90 | raise KeyError(f'Table {table_name} not found') 91 | 92 | in_refs = {} 93 | for ref_table_name, ref_table_data in self.references[table_name].items(): 94 | in_refs[ref_table_name] = ref_table_data['references'] 95 | 96 | out_refs = {} 97 | for ref_table_name, table_refs in self.references.items(): 98 | if table_name == ref_table_name: 99 | continue 100 | 101 | ref_to_table = table_refs.get(table_name) 102 | if ref_to_table: 103 | out_refs[ref_table_name] = ref_to_table['references'] 104 | 105 | return {'in_refs': in_refs, 'out_refs': out_refs} 106 | 107 | def get_rows_references(self, table_name: str, ids: List[int]): 108 | """ 109 | Get dictionary of links to %ids% rows in %table_name% table from other tables 110 | 111 | Result (table_name = table_a, ids = [1, 5, 6]): 112 | { 113 | 1: { 114 | 'table_b': {'table_a_id': [1, 4, 6]}, 115 | 'table_c': {'a_id': [29]}, 116 | }, 117 | 5: { 118 | 'table_b': {'table_a_id': []}, 119 | 'table_c': {'a_id': [12, 13]}, 120 | }, 121 | 6: { 122 | 'table_b': {'table_a_id': []}, 123 | 'table_c': {'a_id': []}, 124 | } 125 | } 126 | """ 127 | if table_name not in self.references: 128 | raise KeyError(f'Table {table_name} not found') 129 | 130 | rows_refs = {id_: {} for id_ in ids} 131 | s_in = ', '.join('%s' for _ in ids) 132 | conn = get_db_conn(self.config) 133 | try: 134 | for ref_table_name, ref_table_data in self.references[table_name].items(): 135 | for ref_tables in rows_refs.values(): 136 | ref_tables[ref_table_name] = {fk.fk_ref: [] for fk in ref_table_data['references']} 137 | 138 | for fk in ref_table_data['references']: 139 | query = SQL( 140 | f"SELECT {fk.pk_ref}, {fk.fk_ref} " 141 | f"FROM {self.config.db_config.schema}.{ref_table_name} " 142 | f"WHERE {fk.fk_ref} IN ({s_in})" 143 | ) 144 | with conn.cursor(cursor_factory=DictCursor) as curs: 145 | curs.execute(query, ids) 146 | result = curs.fetchall() 147 | rows = [dict(row) for row in result] 148 | 149 | for row in rows: 150 | tmp = rows_refs[row[fk.fk_ref]][ref_table_name][fk.fk_ref] 151 | tmp.append(row) 152 | finally: 153 | conn.close() 154 | 155 | return rows_refs 156 | -------------------------------------------------------------------------------- /pggraph/db/build_references.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright Ⓒ 2020 "Sberbank Real Estate Center" Limited Liability Company. Licensed under the MIT license. 3 | Please, see the LICENSE.md file in project's root for full licensing information. 4 | """ 5 | import logging 6 | from collections import OrderedDict 7 | from typing import Set, Dict, List 8 | 9 | from psycopg2._psycopg import connection 10 | from psycopg2.extras import DictCursor 11 | 12 | from pggraph.config import Config, DBConfig 13 | from pggraph.utils.classes.foreign_key import ForeignKey 14 | from pggraph.db.base import get_db_conn 15 | 16 | 17 | def build_references(config: Config, conn: connection = None) -> Dict[str, dict]: 18 | """ 19 | Build a tables dependency graph 20 | 21 | Algorithm: 22 | 1) Get all table names 23 | 2) Get all Foreign Keys 24 | 3) Build a tables dependency graph (references dict) 25 | For each table: 26 | For each child table: 27 | build dependency graph recursive 28 | 29 | Result: 30 | { 31 | 'references': { 32 | 'table_a': { 33 | 'table_b': { 34 | 'references': [{'pk_ref': 'id', 'fk_ref': 'table_b_id'}] 35 | 'ref_tables': { 36 | 'table_c': { 37 | 'table_a': {}, 38 | 'table_b': {} 39 | }, 40 | ... 41 | } 42 | }, 43 | 'table_c': {...} 44 | }, 45 | 'table_b': {...} 46 | }, 47 | 'primary_keys': { 48 | 'table_a': 'id', 49 | 'table_b': 'id', 50 | 'table_c': 'id' 51 | } 52 | } 53 | """ 54 | 55 | if not conn: 56 | conn = get_db_conn(config) 57 | 58 | try: 59 | references = {} 60 | tables = get_all_tables(conn, config.db_config) 61 | foreign_keys = get_all_fk(conn, config.db_config) 62 | primary_keys = get_all_pk(conn, config.db_config) 63 | 64 | for table in tables: 65 | references[table['table_name']] = {} 66 | 67 | for fk in foreign_keys: 68 | if fk['main_table'] not in references: 69 | references[fk['main_table']] = {} 70 | 71 | if not fk['ref_table'] in references[fk['main_table']]: 72 | references[fk['main_table']][fk['ref_table']] = { 73 | 'ref_tables': {}, 74 | 'references': [] 75 | } 76 | 77 | table_references = references[fk['main_table']][fk['ref_table']]['references'] 78 | table_references.append(ForeignKey( 79 | pk_main=fk['main_table_column'], 80 | pk_ref=fk['ref_pk_columns'], 81 | fk_ref=fk['ref_fk_column'], 82 | fk_name=fk['constraint_name'], 83 | )) 84 | 85 | if references: 86 | references = OrderedDict(sorted(references.items(), key=lambda row: len(row[1]), reverse=True)) 87 | 88 | for parent, refs in references.items(): 89 | for ref, ref_data in refs.items(): 90 | visited = {parent, ref} 91 | ref_childs = ref_data['ref_tables'] 92 | recursive_build(ref, ref_childs, references, visited) 93 | finally: 94 | conn.close() 95 | 96 | result = { 97 | 'references': references, 98 | 'primary_keys': primary_keys 99 | } 100 | return result 101 | 102 | 103 | def recursive_build(parent_table: str, 104 | parent_childs: dict, 105 | references: Dict[str, dict], 106 | visited: Set[str] = None, 107 | depth: int = 1) -> Dict[str, dict]: 108 | if visited is None: 109 | visited = set() 110 | 111 | visited.add(parent_table) 112 | 113 | tabs = '*' * depth 114 | logging.debug(f'{tabs}{parent_table} start build') 115 | for ref_table in references[parent_table]: 116 | new_visited = visited.copy() 117 | if ref_table in visited: 118 | parent_childs[ref_table] = 'САМ НА СЕБЯ' if ref_table == parent_table else 'РЕКУРСИЯ' 119 | logging.debug(f'{tabs}*{ref_table} {parent_childs[ref_table]} - {visited}') 120 | 121 | continue 122 | 123 | parent_childs[ref_table] = {} 124 | parent_childs[ref_table] = recursive_build( 125 | ref_table, parent_childs[ref_table], references, new_visited, depth + 1 126 | ) 127 | 128 | if parent_childs: 129 | parent_childs = OrderedDict(sorted( 130 | parent_childs.items(), key=lambda ref: len(ref[1]), reverse=True 131 | )) 132 | 133 | return parent_childs 134 | 135 | 136 | def get_all_tables(conn, db_config: DBConfig) -> List[dict]: 137 | query = "SELECT * FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND table_schema = %(schema)s" 138 | 139 | with conn.cursor(cursor_factory=DictCursor) as curs: 140 | curs.execute(query.strip(), {'schema': db_config.schema}) 141 | result = curs.fetchall() 142 | 143 | base_tables = [dict(row) for row in result] 144 | return base_tables 145 | 146 | 147 | def get_all_fk(conn, db_config: DBConfig) -> List[dict]: 148 | query = """ 149 | WITH contraints_columns_table AS ( 150 | SELECT main_table_name, 151 | ref_table_name, 152 | constraint_catalog, 153 | constraint_schema, 154 | constraint_name, 155 | constraint_type, 156 | string_agg(distinct column_name, ', ') as column_name 157 | FROM ( 158 | SELECT ccu_in.table_name as main_table_name, 159 | ccu_in.constraint_catalog, 160 | ccu_in.constraint_schema, 161 | ccu_in.constraint_name, 162 | tc_in.constraint_type, 163 | kcu.table_name as ref_table_name, 164 | kcu.column_name 165 | FROM information_schema.constraint_column_usage ccu_in 166 | INNER JOIN information_schema.table_constraints tc_in 167 | ON ccu_in.constraint_name = tc_in.constraint_name 168 | AND ccu_in.constraint_schema = tc_in.constraint_schema 169 | AND ccu_in.constraint_catalog = tc_in.constraint_catalog 170 | INNER JOIN information_schema.key_column_usage kcu 171 | ON ccu_in.constraint_name = kcu.constraint_name 172 | AND ccu_in.constraint_schema = kcu.constraint_schema 173 | AND ccu_in.constraint_catalog = kcu.constraint_catalog 174 | WHERE ccu_in.constraint_schema = %(schema)s 175 | ORDER BY ccu_in.constraint_catalog, ccu_in.constraint_schema, ccu_in.constraint_name, 176 | kcu.ordinal_position 177 | ) as subq 178 | GROUP BY main_table_name, ref_table_name, constraint_catalog, constraint_schema, constraint_name, constraint_type 179 | ) 180 | SELECT 181 | ccu.main_table_name AS main_table, 182 | ccu.column_name AS main_table_column, 183 | tc.table_name AS ref_table, 184 | pk_table.column_name AS ref_pk_columns, 185 | kcu.column_name AS ref_fk_column, 186 | ccu.constraint_name as constraint_name 187 | 188 | FROM information_schema.table_constraints tc 189 | 190 | LEFT JOIN ( 191 | select ccu_in.constraint_catalog, ccu_in.constraint_schema, ccu_in.constraint_name, 192 | cct.main_table_name, cct.column_name 193 | FROM contraints_columns_table cct 194 | INNER JOIN information_schema.constraint_column_usage ccu_in 195 | ON ccu_in.table_catalog = cct.constraint_catalog 196 | AND ccu_in.table_schema = cct.constraint_schema 197 | AND ccu_in.table_name = cct.main_table_name 198 | WHERE lower(cct.constraint_type) in ('primary key') 199 | ) ccu ON tc.constraint_catalog = ccu.constraint_catalog 200 | AND tc.constraint_schema = ccu.constraint_schema 201 | AND tc.constraint_name = ccu.constraint_name 202 | 203 | LEFT JOIN ( 204 | select * FROM contraints_columns_table cct 205 | WHERE lower(cct.constraint_type) in ('foreign key') 206 | ) kcu ON tc.constraint_catalog = kcu.constraint_catalog 207 | AND tc.constraint_schema = kcu.constraint_schema 208 | AND tc.constraint_name = kcu.constraint_name 209 | AND tc.table_name = kcu.ref_table_name 210 | 211 | LEFT JOIN ( 212 | select * FROM contraints_columns_table cct 213 | WHERE lower(cct.constraint_type) in ('primary key') 214 | ) pk_table ON pk_table.main_table_name = tc.table_name 215 | 216 | WHERE lower(tc.constraint_type) in ('foreign key') 217 | AND tc.constraint_schema = %(schema)s 218 | AND ccu.main_table_name is not null 219 | GROUP BY ccu.main_table_name, ccu.column_name, pk_table.column_name, tc.table_name, kcu.column_name, ccu.constraint_name 220 | ORDER BY ccu.main_table_name, tc.table_name; 221 | """ 222 | 223 | with conn.cursor(cursor_factory=DictCursor) as curs: 224 | curs.execute(query.strip(), {'schema': db_config.schema}) 225 | result = curs.fetchall() 226 | 227 | foreign_keys = [dict(row) for row in result] 228 | return foreign_keys 229 | 230 | 231 | def get_all_pk(conn, db_config: DBConfig) -> Dict[str, str]: 232 | query = """ 233 | select kcu.table_name as table_name, string_agg(distinct kcu.column_name, ', ') as column_names 234 | from information_schema.key_column_usage kcu 235 | INNER JOIN information_schema.table_constraints tc_in 236 | ON tc_in.constraint_name = kcu.constraint_name 237 | AND tc_in.constraint_schema = kcu.constraint_schema 238 | AND tc_in.constraint_catalog = kcu.constraint_catalog 239 | where tc_in.constraint_type = 'PRIMARY KEY' AND tc_in.table_schema = %(schema)s 240 | group by kcu.table_name, kcu.constraint_catalog, kcu.constraint_schema, kcu.constraint_name; 241 | """ 242 | with conn.cursor(cursor_factory=DictCursor) as curs: 243 | curs.execute(query.strip(), {'schema': db_config.schema}) 244 | result = curs.fetchall() 245 | 246 | primary_keys = [dict(row) for row in result] 247 | return {row['table_name']: row['column_names'] for row in primary_keys} 248 | -------------------------------------------------------------------------------- /pggraph/db/archiver.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright Ⓒ 2020 "Sberbank Real Estate Center" Limited Liability Company. Licensed under the MIT license. 3 | Please, see the LICENSE.md file in project's root for full licensing information. 4 | """ 5 | import logging 6 | from typing import List 7 | 8 | from psycopg2._json import Json 9 | from psycopg2._psycopg import connection 10 | from psycopg2.extras import execute_values, DictCursor 11 | from psycopg2.sql import SQL 12 | 13 | from pggraph.config import Config 14 | from pggraph.utils.classes.foreign_key import ForeignKey 15 | 16 | TAB_SYMBOL = '\t' 17 | 18 | 19 | class Archiver: 20 | conn: connection 21 | config: Config 22 | current_depth: int 23 | references: dict 24 | 25 | def __init__(self, conn: connection, references: dict, config: Config): 26 | self.conn = conn 27 | self.config = config 28 | self.current_depth = 0 29 | self.references = references 30 | 31 | def archive_recursive(self, table_name: str, rows: List[dict], pk_cols: str = 'id'): 32 | """ 33 | Recursive archiving/clearing table 34 | Algorithm: 35 | - For each dependency of the table (ref_table) 36 | - For each Foreign Key, referencing to the main table 37 | If dependent table doesn't have its own dependencies 38 | archive the table by Foreign Keys and move on to the next dependency 39 | If dependent table has its own dependencies 40 | archive rows, using archive_recursive 41 | - After archiving all the dependencies, archive the main table 42 | 43 | :param table_name: name of the table to be archived 44 | :param rows: list of archived row IDs 45 | :param pk_cols: Primary Key columns 46 | """ 47 | tabs = TAB_SYMBOL*self.current_depth 48 | 49 | logging.info(f'{tabs}{table_name} - start archive_recursive {len(rows)} rows (depth={self.current_depth})') 50 | 51 | if self.current_depth >= self.config.archiver_config.max_depth: 52 | logging.info(f'{tabs}{table_name} - MAX_DEPTH exceeded (depth={self.current_depth})') 53 | return 54 | 55 | if not rows: 56 | logging.info(f'{tabs}{table_name} - EMPTY rows - return') 57 | return 58 | 59 | tabs = tabs + TAB_SYMBOL 60 | self.current_depth += 1 61 | 62 | logging.info(f'{tabs}START ARCHIVE REFERRING TABLES') 63 | 64 | for ref_table, ref_data in self.references[table_name].items(): 65 | for ref_fk in ref_data['references']: 66 | logging.debug(f'{tabs}{ref_table} - {ref_fk}') 67 | 68 | if self.config.archiver_config.is_debug: 69 | self.archive_recursive(ref_table, rows, ref_fk.pk_ref) 70 | continue 71 | 72 | if not self.references.get(ref_table): 73 | self.archive_by_fk(ref_table, ref_fk, fk_rows=rows) 74 | continue 75 | 76 | with self.conn.cursor(cursor_factory=DictCursor) as cursor: 77 | self.select_rows_by_fk(cursor, table_name=ref_table, fk=ref_fk, rows=rows, tabs=tabs) 78 | ref_rows_chunk = cursor.fetchmany(size=self.config.archiver_config.chunk_size) 79 | while ref_rows_chunk: 80 | self.archive_recursive(ref_table, ref_rows_chunk, ref_fk.pk_ref) 81 | ref_rows_chunk = cursor.fetchmany(size=self.config.archiver_config.chunk_size) 82 | 83 | logging.info(f'{tabs}END ARCHIVE REFERRING TABLES') 84 | 85 | self.current_depth -= 1 86 | self.archive_by_ids(table_name=table_name, pk_columns=pk_cols, row_pks=rows) 87 | 88 | def archive_by_fk(self, table_name: str, fk: ForeignKey, fk_rows: List[dict]): 89 | """ 90 | Archiving a table with the specified foreign keys 91 | 92 | :param table_name: name of the table to be archived 93 | :param fk: ForeignKey object 94 | :param fk_rows: foreign key values to be archived 95 | """ 96 | tabs = TAB_SYMBOL*self.current_depth 97 | logging.info(f'{tabs}{table_name} - archive_by_fk {len(fk_rows)} rows by {fk}') 98 | 99 | if self.config.archiver_config.is_debug: 100 | return 101 | 102 | total_archived_rows = 0 103 | with self.conn: # транзакция 104 | if self.config.archiver_config.to_archive: 105 | archive_table_name = self.create_archive_table(table_name, tabs=tabs) 106 | 107 | with self.conn.cursor(cursor_factory=DictCursor) as cursor: 108 | self.select_rows_by_fk(cursor, table_name, fk=fk, rows=fk_rows, tabs=tabs, for_update=True) 109 | self.delete_rows_by_fk(cursor, table_name, fk=fk, fk_rows=fk_rows, tabs=tabs) 110 | 111 | if self.config.archiver_config.to_archive: 112 | rows_chunk = cursor.fetchmany(size=self.config.archiver_config.chunk_size) 113 | while rows_chunk: 114 | total_archived_rows += len(rows_chunk) 115 | self.insert_rows(archive_table_name=archive_table_name, values=rows_chunk, tabs=tabs) 116 | rows_chunk = cursor.fetchmany(size=self.config.archiver_config.chunk_size) 117 | 118 | return total_archived_rows 119 | 120 | def archive_by_ids(self, table_name: str, pk_columns: str, row_pks: List[dict]): 121 | """ 122 | Archiving a table with the specified primary keys 123 | 124 | :param table_name: name of the table to be archived 125 | :param pk_columns: primary key columns 126 | :param row_pks: primary keys values to be archived 127 | """ 128 | tabs = TAB_SYMBOL*self.current_depth 129 | logging.info(f'{tabs}{table_name} - archive_by_ids {len(row_pks)} rows by {pk_columns}') 130 | 131 | if self.config.archiver_config.is_debug: 132 | return 133 | 134 | total_archived_rows = 0 135 | with self.conn: # транзакция 136 | if self.config.archiver_config.to_archive: 137 | archive_table_name = self.create_archive_table(table_name, tabs=tabs) 138 | 139 | with self.conn.cursor(cursor_factory=DictCursor) as cursor: 140 | self.select_rows_for_update(cursor, table_name, pk_columns=pk_columns, rows=row_pks, tabs=tabs) 141 | self.delete_rows_by_ids(cursor, table_name, pk_columns=pk_columns, rows=row_pks, tabs=tabs) 142 | 143 | if self.config.archiver_config.to_archive: 144 | rows_chunk = cursor.fetchmany(size=self.config.archiver_config.chunk_size) 145 | while rows_chunk: 146 | total_archived_rows += len(rows_chunk) 147 | self.insert_rows(archive_table_name=archive_table_name, values=rows_chunk, tabs=tabs) 148 | rows_chunk = cursor.fetchmany(size=self.config.archiver_config.chunk_size) 149 | 150 | return total_archived_rows 151 | 152 | def create_archive_table(self, table_name: str, tabs: str) -> str: 153 | new_table_name = f"{table_name}_{self.config.archiver_config.archive_suffix}" 154 | query = SQL( 155 | f"CREATE TABLE IF NOT EXISTS {self.config.db_config.schema}.{new_table_name} " 156 | f"(LIKE {self.config.db_config.schema}.{table_name})" 157 | ) 158 | 159 | with self.conn.cursor(cursor_factory=DictCursor) as cur: 160 | cur.execute(query) 161 | 162 | logging.debug(f"{tabs}{query}") 163 | 164 | return new_table_name 165 | 166 | def insert_rows(self, archive_table_name: str, values: List[dict], tabs: str): 167 | column_names = ', '.join(values[0].keys()) 168 | query = SQL(f'INSERT INTO {self.config.db_config.schema}.{archive_table_name} ({column_names}) VALUES %s') 169 | 170 | # Convert dict to json 171 | for row in values: 172 | for col_name, col_val in row.items(): 173 | if isinstance(col_val, dict): 174 | row[col_name] = Json(col_val) 175 | if isinstance(col_val, list) and col_val and isinstance(col_val[0], dict): 176 | row[col_name] = Json(col_val) 177 | 178 | logging.debug(f"{tabs}INSERT INTO {archive_table_name} - {len(values)} rows") 179 | with self.conn.cursor(cursor_factory=DictCursor) as cursor: 180 | execute_values(cursor, query.as_string(cursor), values) 181 | 182 | def delete_rows_by_fk(self, cursor, table_name: str, fk: ForeignKey, fk_rows: List, tabs: str): 183 | pk_cols = fk.pk_main.split(', ') 184 | row_ids = [tuple(row[pk] for pk in pk_cols) for row in fk_rows] 185 | in_s = ', '.join('%s' for _ in range(len(fk_rows))) 186 | 187 | query = SQL( 188 | f"DELETE FROM {self.config.db_config.schema}.{table_name} WHERE ({fk.fk_ref}) IN ({in_s}) RETURNING *" 189 | ) 190 | 191 | logging.debug(f"{tabs}DELETE FROM {table_name} by FK {fk.fk_ref} - {len(fk_rows)} rows") 192 | cursor.execute(query, row_ids) 193 | 194 | def delete_rows_by_ids(self, cursor, table_name: str, pk_columns: str, rows: List[dict], tabs: str): 195 | pk_cols = [pk.strip() for pk in pk_columns.split(',')] 196 | row_ids = [tuple(row[pk] for pk in pk_cols) for row in rows] 197 | in_s = ', '.join('%s' for _ in range(len(rows))) 198 | 199 | query = SQL( 200 | f"DELETE FROM {self.config.db_config.schema}.{table_name} " 201 | f"WHERE ({pk_columns}) IN ({in_s}) RETURNING *" 202 | ) 203 | 204 | logging.debug(f"{tabs}DELETE FROM {table_name} by {pk_columns} - {len(rows)} rows") 205 | cursor.execute(query, row_ids) 206 | 207 | def select_rows_by_fk(self, cursor, table_name: str, fk: ForeignKey, rows: List[dict], tabs: str, for_update: bool = False): 208 | pk_cols = fk.pk_main.split(', ') 209 | row_ids = [tuple(row[pk] for pk in pk_cols) for row in rows] 210 | in_s = ', '.join('%s' for _ in range(len(rows))) 211 | 212 | query = f"SELECT {fk.pk_ref} FROM {self.config.db_config.schema}.{table_name} WHERE ({fk.fk_ref}) IN ({in_s})" 213 | if for_update: 214 | query += f" FOR UPDATE" 215 | query = SQL(query) 216 | 217 | logging.debug(f"{tabs}{query}"[:1000]) 218 | cursor.execute(query, row_ids) 219 | 220 | def select_rows_for_update(self, cursor, table_name: str, pk_columns: str, rows: List[dict], tabs: str): 221 | pk_cols = [pk.strip() for pk in pk_columns.split(',')] 222 | row_ids = [tuple(row[pk] for pk in pk_cols) for row in rows] 223 | in_s = ', '.join('%s' for _ in range(len(rows))) 224 | 225 | query = SQL( 226 | f"SELECT {pk_columns} FROM {self.config.db_config.schema}.{table_name} " 227 | f"WHERE ({pk_columns}) IN ({in_s}) FOR UPDATE" 228 | ) 229 | 230 | logging.debug(f"{tabs}SELECT {pk_columns} FROM {table_name} FOR UPDATE by {pk_columns} - {len(rows)} rows") 231 | cursor.execute(query, row_ids) 232 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PgGraph 2 | 3 | [![PyPI](https://img.shields.io/pypi/v/pggraph.svg)](https://pypi.org/project/pggraph/) ![3.6, 3.7, 3.8](https://img.shields.io/pypi/pyversions/pggraph.svg) [![License: MIT](https://img.shields.io/github/license/domclick/pggraph)](https://github.com/domclick/pggraph/blob/master/LICENSE.md) 4 | 5 | Утилита для работы с зависимостями таблиц в PostgreSQL 6 | 7 | Основной функционал: 8 | - Рекурсивное удаление и архивация строк в таблице с указанными Primary Key
9 | *Под архивацией понимается перенос строк в архивную таблицу (например, из "books" в "books_archive")* 10 | - Поиск зависимостей для указанной таблицы (ссылающиеся таблицы и таблицы на которые ссылается данная) 11 | - Поиск ссылок на строки с указанными Primary Key данной таблицы 12 | 13 | ## Установка 14 | ```$ pip install pggraph``` 15 | 16 | ## Файл конфигурации config.ini 17 | Для работы утилиты нужно создать на локальной машине конфигурационный файл config.ini со следующим содержимым: 18 | ```ini 19 | [db] 20 | host = localhost 21 | port = 5432 22 | user = postgres 23 | password = postgres 24 | dbname = postgres 25 | schema = public ; Необязательный параметр, указано значение по умолчанию 26 | 27 | [archive] ; Данный раздел заполнять необязательно, ниже указаны значения по умолчанию 28 | is_debug = false ; Запуск в режиме debug (удаление из таблицы происходить не будет) 29 | chunk_size = 1000 ; Кол-во строк, которое архивируется за 1 шаг 30 | max_depth = 20 ; Максимальная глубина рекурсии 31 | to_archive = true ; Режим архивации (строки из таблицы "a" переносятся в таблицу "a_%archive_suffix%") 32 | archive_suffix = 'archive' ; Суффикс архивной таблицы 33 | ``` 34 | 35 | ## Структура 36 | - **core** - основной функционал 37 | - **db** - функции и классы для работы с БД 38 | - archiver.py - Archiver - класс с функционалом архивации таблиц 39 | - build_references.py - построение графа зависимостей между таблицами 40 | - **utils** - вспомогательные функции и классы 41 | - api.py - PgGraphApi, основной класс для работы 42 | - config.py - парсинг конфигурации 43 | 44 | 45 | ## Команды для запуска 46 | Утилиту можно запускать из консоли, также с ней можно работать из кода или интерактивных оболочек, вроде IPython/JupyterLab. 47 | 48 | ### Запуск из консоли 49 | 50 | #### Параметры 51 | Позиционные аргументы: 52 | - action - требуемое действие: archive_table, get_table_references или get_rows_references 53 | 54 | Именованные аргументы: 55 | - --config_path - путь к конфиг-файлу 56 | - --table - таблица с которой нужно совершить действие 57 | - --ids - список id через запятую, пример - 1,2,3 (необязательный параметр) 58 | - --log_path - путь к папке для логов (необязательный параметр, по умолчанию - None) 59 | - --log_level - уровень логирования (необязательный параметр, по умолчанию - INFO) 60 | 61 | ```shell script 62 | $ pggraph -h 63 | usage: pggraph action [-h] --table TABLE [--ids IDS] [--config_path CONFIG_PATH] 64 | positional arguments: 65 | action required action: archive_table, get_table_references, get_rows_references 66 | 67 | optional arguments: 68 | -h, --help show this help message and exit 69 | --table TABLE table name 70 | --ids IDS primary key ids, separated by comma, e.g. 1,2,3 71 | --config_path CONFIG_PATH path to config.ini 72 | --log_path LOG_PATH path to log dir 73 | --log_level LOG_LEVEL log level (debug, info, error) 74 | ``` 75 | 76 | #### Примеры команд 77 | 78 | Архивация таблицы 79 | ```shell script 80 | $ pggraph archive_table --config_path config.hw.local.ini --table flights --ids 1,2,3 81 | 2020-06-20 19:27:44 INFO: flights - START 82 | 2020-06-20 19:27:44 INFO: flights - start archive_recursive 3 rows (depth=0) 83 | 2020-06-20 19:27:44 INFO: START ARCHIVE REFERRING TABLES 84 | 2020-06-20 19:27:44 INFO: ticket_flights - start archive_recursive 3 rows (depth=1) 85 | 2020-06-20 19:27:44 INFO: START ARCHIVE REFERRING TABLES 86 | 2020-06-20 19:27:44 INFO: boarding_passes - start archive_recursive 3 rows (depth=2) 87 | 2020-06-20 19:27:44 INFO: START ARCHIVE REFERRING TABLES 88 | 2020-06-20 19:27:44 INFO: END ARCHIVE REFERRING TABLES 89 | 2020-06-20 19:27:44 INFO: boarding_passes - archive_by_ids 3 rows by ticket_no, flight_id 90 | 2020-06-20 19:27:44 INFO: boarding_passes - start archive_recursive 3 rows (depth=2) 91 | 2020-06-20 19:27:44 INFO: START ARCHIVE REFERRING TABLES 92 | 2020-06-20 19:27:44 INFO: END ARCHIVE REFERRING TABLES 93 | 2020-06-20 19:27:44 INFO: boarding_passes - archive_by_ids 3 rows by ticket_no, flight_id 94 | 2020-06-20 19:27:44 INFO: boarding_passes - start archive_recursive 3 rows (depth=2) 95 | 2020-06-20 19:27:44 INFO: START ARCHIVE REFERRING TABLES 96 | 2020-06-20 19:27:44 INFO: END ARCHIVE REFERRING TABLES 97 | 2020-06-20 19:27:44 INFO: boarding_passes - archive_by_ids 3 rows by ticket_no, flight_id 98 | 2020-06-20 19:27:44 INFO: boarding_passes - start archive_recursive 3 rows (depth=2) 99 | 2020-06-20 19:27:44 INFO: START ARCHIVE REFERRING TABLES 100 | 2020-06-20 19:27:44 INFO: END ARCHIVE REFERRING TABLES 101 | 2020-06-20 19:27:44 INFO: boarding_passes - archive_by_ids 3 rows by ticket_no, flight_id 102 | 2020-06-20 19:27:44 INFO: END ARCHIVE REFERRING TABLES 103 | 2020-06-20 19:27:44 INFO: ticket_flights - archive_by_ids 3 rows by ticket_no, flight_id 104 | 2020-06-20 19:27:44 INFO: END ARCHIVE REFERRING TABLES 105 | 2020-06-20 19:27:44 INFO: flights - archive_by_ids 3 rows by id 106 | 2020-06-20 19:27:44 INFO: flights - END 107 | ``` 108 | 109 | Поиск зависимостей для указанной таблицы 110 | ```shell script 111 | $ pggraph get_table_references --config_path config.hw.local.ini --table flights 112 | {'in_refs': {'ticket_flights': [ForeignKey(pk_main='flight_id', pk_ref='ticket_no, flight_id', fk_ref='flight_id')]}, 113 | 'out_refs': {'aircrafts': [ForeignKey(pk_main='aircraft_code', pk_ref='flight_id', fk_ref='aircraft_code')], 114 | 'airports': [ForeignKey(pk_main='airport_code', pk_ref='flight_id', fk_ref='arrival_airport'), 115 | ForeignKey(pk_main='airport_code', pk_ref='flight_id', fk_ref='departure_airport')]}} 116 | ``` 117 | 118 | Поиск ссылок на строки с указанными Primary Key 119 | ```shell script 120 | $ pggraph get_rows_references --config_path config.hw.local.ini --table flights --ids 1,2,3 121 | {1: {'ticket_flights': {'flight_id': [{'flight_id': 1, 122 | 'ticket_no': '0005432816945'}, 123 | {'flight_id': 1, 124 | 'ticket_no': '0005432816941'}]}}, 125 | 2: {'ticket_flights': {'flight_id': [{'flight_id': 2, 126 | 'ticket_no': '0005433101832'}, 127 | {'flight_id': 2, 128 | 'ticket_no': '0005433101864'}, 129 | {'flight_id': 2, 130 | 'ticket_no': '0005432919715'}]}}, 131 | 3: {'ticket_flights': {'flight_id': [{'flight_id': 3, 132 | 'ticket_no': '0005432817560'}, 133 | {'flight_id': 3, 134 | 'ticket_no': '0005432817568'}, 135 | {'flight_id': 3, 136 | 'ticket_no': '0005432817559'}]}}} 137 | ``` 138 | 139 | ### Работа в интерактивной консоли iPython 140 | Архивация таблицы 141 | ```python 142 | >>> from pggraph.main import setup_logging 143 | >>> setup_logging(log_level='DEBUG') 144 | >>> from pggraph.api import PgGraphApi 145 | >>> api = PgGraphApi(config_path='config.hw.local.ini') 146 | >>> api.archive_table('flights', [4,5]) 147 | 2020-06-20 23:12:08 INFO: flights - START 148 | 2020-06-20 23:12:08 INFO: flights - start archive_recursive 2 rows (depth=0) 149 | 2020-06-20 23:12:08 INFO: START ARCHIVE REFERRING TABLES 150 | 2020-06-20 23:12:08 DEBUG: ticket_flights - ForeignKey(pk_main='flight_id', pk_ref='flight_id, ticket_no', fk_ref='flight_id') 151 | 2020-06-20 23:12:08 DEBUG: SQL('SELECT flight_id, ticket_no FROM bookings.ticket_flights WHERE (flight_id) IN (%s, %s)') 152 | 2020-06-20 23:12:08 INFO: ticket_flights - start archive_recursive 30 rows (depth=1) 153 | 2020-06-20 23:12:08 INFO: START ARCHIVE REFERRING TABLES 154 | 2020-06-20 23:12:08 DEBUG: boarding_passes - ForeignKey(pk_main='flight_id, ticket_no', pk_ref='flight_id, ticket_no', fk_ref='flight_id, ticket_no') 155 | 2020-06-20 23:12:08 INFO: boarding_passes - archive_by_fk 30 rows by ForeignKey(pk_main='flight_id, ticket_no', pk_ref='flight_id, ticket_no', fk_ref='flight_id, ticket_no') 156 | 2020-06-20 23:12:08 DEBUG: SQL('CREATE TABLE IF NOT EXISTS bookings.boarding_passes_archive (LIKE bookings.boarding_passes)') 157 | 2020-06-20 23:12:08 DEBUG: DELETE FROM boarding_passes by FK flight_id, ticket_no - 30 rows 158 | 2020-06-20 23:12:08 INFO: END ARCHIVE REFERRING TABLES 159 | 2020-06-20 23:12:08 INFO: ticket_flights - archive_by_ids 30 rows by flight_id, ticket_no 160 | 2020-06-20 23:12:08 DEBUG: SQL('CREATE TABLE IF NOT EXISTS bookings.ticket_flights_archive (LIKE bookings.ticket_flights)') 161 | 2020-06-20 23:12:08 DEBUG: DELETE FROM ticket_flights by flight_id, ticket_no - 30 rows 162 | 2020-06-20 23:12:08 DEBUG: INSERT INTO ticket_flights_archive - 30 rows 163 | 2020-06-20 23:12:08 INFO: ticket_flights - start archive_recursive 30 rows (depth=1) 164 | 2020-06-20 23:12:08 INFO: START ARCHIVE REFERRING TABLES 165 | 2020-06-20 23:12:08 DEBUG: boarding_passes - ForeignKey(pk_main='flight_id, ticket_no', pk_ref='flight_id, ticket_no', fk_ref='flight_id, ticket_no') 166 | 2020-06-20 23:12:08 INFO: boarding_passes - archive_by_fk 30 rows by ForeignKey(pk_main='flight_id, ticket_no', pk_ref='flight_id, ticket_no', fk_ref='flight_id, ticket_no') 167 | 2020-06-20 23:12:08 DEBUG: SQL('CREATE TABLE IF NOT EXISTS bookings.boarding_passes_archive (LIKE bookings.boarding_passes)') 168 | 2020-06-20 23:12:08 DEBUG: DELETE FROM boarding_passes by FK flight_id, ticket_no - 30 rows 169 | 2020-06-20 23:12:08 INFO: END ARCHIVE REFERRING TABLES 170 | 2020-06-20 23:12:08 INFO: ticket_flights - archive_by_ids 30 rows by flight_id, ticket_no 171 | 2020-06-20 23:12:08 DEBUG: SQL('CREATE TABLE IF NOT EXISTS bookings.ticket_flights_archive (LIKE bookings.ticket_flights)') 172 | 2020-06-20 23:12:08 DEBUG: DELETE FROM ticket_flights by flight_id, ticket_no - 30 rows 173 | 2020-06-20 23:12:08 DEBUG: INSERT INTO ticket_flights_archive - 30 rows 174 | 2020-06-20 23:12:08 INFO: ticket_flights - start archive_recursive 30 rows (depth=1) 175 | 2020-06-20 23:12:08 INFO: START ARCHIVE REFERRING TABLES 176 | 2020-06-20 23:12:08 DEBUG: boarding_passes - ForeignKey(pk_main='flight_id, ticket_no', pk_ref='flight_id, ticket_no', fk_ref='flight_id, ticket_no') 177 | 2020-06-20 23:12:08 INFO: boarding_passes - archive_by_fk 30 rows by ForeignKey(pk_main='flight_id, ticket_no', pk_ref='flight_id, ticket_no', fk_ref='flight_id, ticket_no') 178 | 2020-06-20 23:12:08 DEBUG: SQL('CREATE TABLE IF NOT EXISTS bookings.boarding_passes_archive (LIKE bookings.boarding_passes)') 179 | 2020-06-20 23:12:08 DEBUG: DELETE FROM boarding_passes by FK flight_id, ticket_no - 30 rows 180 | 2020-06-20 23:12:08 INFO: END ARCHIVE REFERRING TABLES 181 | 2020-06-20 23:12:08 INFO: ticket_flights - archive_by_ids 30 rows by flight_id, ticket_no 182 | 2020-06-20 23:12:08 DEBUG: SQL('CREATE TABLE IF NOT EXISTS bookings.ticket_flights_archive (LIKE bookings.ticket_flights)') 183 | 2020-06-20 23:12:08 DEBUG: DELETE FROM ticket_flights by flight_id, ticket_no - 30 rows 184 | 2020-06-20 23:12:08 DEBUG: INSERT INTO ticket_flights_archive - 30 rows 185 | 2020-06-20 23:12:08 INFO: ticket_flights - start archive_recursive 3 rows (depth=1) 186 | 2020-06-20 23:12:08 INFO: START ARCHIVE REFERRING TABLES 187 | 2020-06-20 23:12:08 DEBUG: boarding_passes - ForeignKey(pk_main='flight_id, ticket_no', pk_ref='flight_id, ticket_no', fk_ref='flight_id, ticket_no') 188 | 2020-06-20 23:12:08 INFO: boarding_passes - archive_by_fk 3 rows by ForeignKey(pk_main='flight_id, ticket_no', pk_ref='flight_id, ticket_no', fk_ref='flight_id, ticket_no') 189 | 2020-06-20 23:12:08 DEBUG: SQL('CREATE TABLE IF NOT EXISTS bookings.boarding_passes_archive (LIKE bookings.boarding_passes)') 190 | 2020-06-20 23:12:08 DEBUG: DELETE FROM boarding_passes by FK flight_id, ticket_no - 3 rows 191 | 2020-06-20 23:12:08 INFO: END ARCHIVE REFERRING TABLES 192 | 2020-06-20 23:12:08 INFO: ticket_flights - archive_by_ids 3 rows by flight_id, ticket_no 193 | 2020-06-20 23:12:08 DEBUG: SQL('CREATE TABLE IF NOT EXISTS bookings.ticket_flights_archive (LIKE bookings.ticket_flights)') 194 | 2020-06-20 23:12:08 DEBUG: DELETE FROM ticket_flights by flight_id, ticket_no - 3 rows 195 | 2020-06-20 23:12:08 DEBUG: INSERT INTO ticket_flights_archive - 3 rows 196 | 2020-06-20 23:12:08 INFO: END ARCHIVE REFERRING TABLES 197 | 2020-06-20 23:12:08 INFO: flights - archive_by_ids 2 rows by flight_id 198 | 2020-06-20 23:12:09 DEBUG: SQL('CREATE TABLE IF NOT EXISTS bookings.flights_archive (LIKE bookings.flights)') 199 | 2020-06-20 23:12:09 DEBUG: DELETE FROM flights by flight_id - 2 rows 200 | 2020-06-20 23:12:09 DEBUG: INSERT INTO flights_archive - 2 rows 201 | 2020-06-20 23:12:09 INFO: flights - END 202 | ``` 203 | 204 | Поиск зависимостей для указанной таблицы 205 | ```python 206 | >>> from pggraph.api import PgGraphApi 207 | >>> from pprint import pprint 208 | >>> api = PgGraphApi(config_path='config.hw.local.ini') 209 | >>> res = api.get_table_references('flights') 210 | >>> pprint(res) 211 | {'in_refs': {'ticket_flights': [ForeignKey(pk_main='flight_id', pk_ref='flight_id, ticket_no', fk_ref='flight_id')]}, 212 | 'out_refs': {'aircrafts': [ForeignKey(pk_main='aircraft_code', pk_ref='flight_id', fk_ref='aircraft_code')], 213 | 'airports': [ForeignKey(pk_main='airport_code', pk_ref='flight_id', fk_ref='arrival_airport'), 214 | ForeignKey(pk_main='airport_code', pk_ref='flight_id', fk_ref='departure_airport')]}} 215 | ``` 216 | 217 | Поиск ссылок на строки с указанными Primary Key 218 | ```python 219 | >>> from pggraph.api import PgGraphApi 220 | >>> from pprint import pprint 221 | >>> api = PgGraphApi(config_path='config.hw.local.ini') 222 | >>> rows = api.get_rows_references('flights', [1,2,3]) 223 | >>> pprint(rows) 224 | {1: {'ticket_flights': {'flight_id': [{'flight_id': 1, 225 | 'ticket_no': '0005432816945'}, 226 | {'flight_id': 1, 227 | 'ticket_no': '0005432816941'}]}}, 228 | 2: {'ticket_flights': {'flight_id': [{'flight_id': 2, 229 | 'ticket_no': '0005433101832'}, 230 | {'flight_id': 2, 231 | 'ticket_no': '0005433101864'}, 232 | {'flight_id': 2, 233 | 'ticket_no': '0005432919715'}]}}, 234 | 3: {'ticket_flights': {'flight_id': [{'flight_id': 3, 235 | 'ticket_no': '0005432817560'}, 236 | {'flight_id': 3, 237 | 'ticket_no': '0005432817568'}, 238 | {'flight_id': 3, 239 | 'ticket_no': '0005432817559'}]}}} 240 | ``` 241 | 242 | ## Author 243 | - [Borzov Oleg](https://github.com/olegborzov) (Author) 244 | 245 | ## Contributor Notice 246 | 247 | We are always open for contributions. Feel free to submit an issue 248 | or a PR. However, when submitting a PR we will ask you to sign 249 | our [CLA (Contributor License Agreement)][cla-text] to confirm that you 250 | have the rights to submit your contributions and to give us the rights 251 | to actually use them. 252 | 253 | When submitting a PR our special bot will ask you to review and to sign 254 | our [CLA][cla-text]. This will happen only once for all our GitHub repositories. 255 | 256 | ## License 257 | 258 | Copyright Ⓒ 2020 ["Sberbank Real Estate Center" Limited Liability Company](https://domclick.ru/). 259 | 260 | [MIT License](./LICENSE.md) 261 | --------------------------------------------------------------------------------