├── requirements.txt ├── .gitignore ├── requirements-dev.txt ├── docs ├── governator.png ├── erd.plantuml └── erd.svg ├── governator ├── core │ ├── schema.py │ ├── column.py │ ├── action.py │ ├── user.py │ ├── catalog.py │ ├── group.py │ ├── project.py │ ├── role.py │ ├── database.py │ ├── permission.py │ ├── relation.py │ └── base.py ├── utils.py ├── config.py ├── cli.py ├── __init__.py └── database_inferface │ └── base.py ├── examples └── simple.py ├── rename.py ├── pyproject.toml ├── .pre-commit-config.yaml └── README.md /requirements.txt: -------------------------------------------------------------------------------- 1 | sqlalchemy 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .python-version 2 | governator.egg-info 3 | __pycache__ 4 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | pre-commit 3 | black 4 | mypy 5 | ruff 6 | types-PyYAML 7 | -------------------------------------------------------------------------------- /docs/governator.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mistercrunch/governator/HEAD/docs/governator.png -------------------------------------------------------------------------------- /governator/core/schema.py: -------------------------------------------------------------------------------- 1 | class Schema: 2 | def __init__(self, schema): 3 | self.schema = schema 4 | 5 | @property 6 | def key(self): 7 | return f"{self.schema}" 8 | -------------------------------------------------------------------------------- /governator/utils.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import uuid 3 | 4 | 5 | def generate_short_id(length=8): 6 | uuid_obj = uuid.uuid4() 7 | return ( 8 | base64.urlsafe_b64encode(uuid_obj.bytes).rstrip(b"=").decode("utf-8")[:length] 9 | ) 10 | -------------------------------------------------------------------------------- /governator/core/column.py: -------------------------------------------------------------------------------- 1 | class Column: 2 | def __init__(self, key: str, name: str, description: str, data_type: str): 3 | self.key = key 4 | self.name = name 5 | self.description = description 6 | self.data_type = data_type 7 | -------------------------------------------------------------------------------- /governator/core/action.py: -------------------------------------------------------------------------------- 1 | from governator.core.base import Serializable 2 | 3 | 4 | class Action(Serializable): 5 | def __init__(self, action): 6 | self.action = action 7 | 8 | @property 9 | def key(self): 10 | return f"{self.action}" 11 | -------------------------------------------------------------------------------- /governator/core/user.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | from governator.core.base import Serializable 4 | 5 | 6 | @dataclass 7 | class User(Serializable): 8 | username: str 9 | 10 | @property 11 | def key(self): 12 | return self.username 13 | -------------------------------------------------------------------------------- /governator/core/catalog.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | from governator.core.base import Serializable 4 | 5 | 6 | @dataclass 7 | class Catalog(Serializable): 8 | catalog: str 9 | 10 | @property 11 | def key(self): 12 | return f"{self.catalog}" 13 | -------------------------------------------------------------------------------- /governator/core/group.py: -------------------------------------------------------------------------------- 1 | from governator.core.base import Serializable 2 | 3 | 4 | class Group(Serializable): 5 | def __init__(self, group, users=None, description=None): 6 | self.group = group 7 | self.users = users or [] 8 | self.description = description 9 | 10 | @property 11 | def key(self): 12 | return self.group 13 | -------------------------------------------------------------------------------- /governator/config.py: -------------------------------------------------------------------------------- 1 | from os import environ 2 | 3 | config_defaults = { 4 | "GOVERNATOR_FOLDER": "/tmp/allstars", 5 | "GOVERNATOR_SQLA_CONN": "mysql://", 6 | "GOVERNATOR_PROJECT": "jaffleshop", 7 | } 8 | 9 | conf = {} 10 | 11 | for k in config_defaults.keys(): 12 | if k in environ and environ.get(k): 13 | conf[k] = environ.get(k) 14 | else: 15 | conf[k] = config_defaults[k] 16 | -------------------------------------------------------------------------------- /governator/core/project.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Optional 3 | 4 | from governator.config import conf 5 | from governator.core.database import Database 6 | 7 | 8 | @dataclass 9 | class Project: 10 | databases: list[Database] 11 | path: Optional[str] = None 12 | 13 | def __post_init__(self): 14 | self.path = self.path or conf["GOVERNATOR_FOLDER"] 15 | 16 | def load(self, database_schema=None): 17 | pass 18 | 19 | def flush(self): 20 | pass 21 | -------------------------------------------------------------------------------- /governator/cli.py: -------------------------------------------------------------------------------- 1 | import click 2 | 3 | 4 | @click.group() 5 | def cli(): 6 | """Governator: Manage your data access policies as code.""" 7 | pass 8 | 9 | 10 | @click.command() 11 | def sync(): 12 | """Synchronize your data access policies across systems.""" 13 | click.echo("Synchronizing data access policies...") 14 | 15 | 16 | @click.command() 17 | def check(): 18 | """Check the current state of data access policies.""" 19 | click.echo("Checking data access policies...") 20 | 21 | 22 | cli.add_command(sync) 23 | cli.add_command(check) 24 | 25 | if __name__ == "__main__": 26 | cli() 27 | -------------------------------------------------------------------------------- /governator/core/role.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Iterable 2 | from typing import TYPE_CHECKING, Optional 3 | 4 | if TYPE_CHECKING: 5 | import governator 6 | 7 | 8 | class Role: 9 | def __init__( 10 | self, 11 | role: str, 12 | description: "Optional[str]" = None, 13 | users: "Optional[Iterable[governator.User]]" = None, 14 | groups: "Optional[Iterable[governator.Group]]" = None, 15 | permissions: "Optional[Iterable[governator.Permission]]" = None, 16 | ): 17 | self.role = role 18 | self.description = description 19 | self.users = users or set() 20 | self.groups = groups or set() 21 | self.permissions = permissions or set() 22 | 23 | @property 24 | def key(self) -> str: 25 | return self.role 26 | -------------------------------------------------------------------------------- /examples/simple.py: -------------------------------------------------------------------------------- 1 | from governator import Action, Database, Group, Permission, Project, Role, Schema, User 2 | 3 | database1 = Database(key="mydb") 4 | 5 | finance_group = Group( 6 | group="finance", 7 | users=[ 8 | User("finance_user_1"), 9 | User("finance_user_2"), 10 | ], 11 | ) 12 | engineer_group = Group( 13 | group="engineering", 14 | users=[ 15 | User("eng_user_1"), 16 | User("eng_user_2"), 17 | User("eng_user_3"), 18 | ], 19 | ) 20 | 21 | finance_database_schema = Schema("finance_schema") 22 | 23 | finance_role = Role( 24 | role="finance_role", 25 | permissions=[ 26 | Permission( 27 | schemas=[finance_database_schema], 28 | actions=[Action("SELECT")], 29 | ) 30 | ], 31 | groups=[finance_group], 32 | ) 33 | 34 | project = Project(databases=[database1]) 35 | -------------------------------------------------------------------------------- /governator/__init__.py: -------------------------------------------------------------------------------- 1 | from governator import utils 2 | from governator.core.action import Action 3 | from governator.core.base import Serializable 4 | from governator.core.catalog import Catalog 5 | from governator.core.column import Column 6 | from governator.core.database import Database 7 | from governator.core.group import Group 8 | from governator.core.permission import Permission 9 | from governator.core.project import Project 10 | from governator.core.relation import Relation 11 | from governator.core.role import Role 12 | from governator.core.schema import Schema 13 | from governator.core.user import User 14 | 15 | __all__ = [ 16 | "Action", 17 | "Catalog", 18 | "Column", 19 | "Database", 20 | "Group", 21 | "Permission", 22 | "Project", 23 | "Relation", 24 | "Role", 25 | "Schema", 26 | "Serializable", 27 | "User", 28 | "utils", 29 | ] 30 | -------------------------------------------------------------------------------- /rename.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | def replace_strings_in_file(file_path, _from, _to): 5 | with open(file_path) as f: 6 | file_contents = f.read() 7 | 8 | if (modified_contents := file_contents.replace(_from, _to)) != file_contents: 9 | with open(file_path, "w") as f: 10 | f.write(modified_contents) 11 | print(f"Updated {file_path}") 12 | 13 | 14 | def crawl_directory(directory, _from, _to): 15 | for root, _, files in os.walk(directory): 16 | for file_name in files: 17 | if file_name.endswith((".py", ".md")): 18 | file_path = os.path.join(root, file_name) 19 | replace_strings_in_file(file_path, _from, _to) 20 | 21 | 22 | if __name__ == "__main__": 23 | start_directory = "/Users/max/code/governator/" # Replace with your directory path 24 | crawl_directory(start_directory, "GOVERNATOR", "GOVERNATOR") 25 | -------------------------------------------------------------------------------- /governator/core/database.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from governator.core.base import Serializable 4 | 5 | 6 | class Database(Serializable): 7 | def __init__( 8 | self, 9 | key: str, 10 | label: Optional[str] = None, 11 | description: Optional[str] = None, 12 | connection_string: Optional[str] = None, 13 | ): 14 | self.key = key 15 | self.label = label 16 | self.description = description 17 | self.connection_string = connection_string 18 | 19 | def to_dict(self): 20 | return { 21 | "key": self.key, 22 | "label": self.label, 23 | "description": self.description, 24 | "connection_string": self.connection_string, 25 | } 26 | 27 | def get_database_interface(self): 28 | pass 29 | 30 | def crawl(self, fetch_column_metadata: bool = False): 31 | pass 32 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "governator" 3 | version = "0.0.1" 4 | description = "A toolkit for managing data access policies as code" 5 | readme = "README.md" 6 | authors = [{name = "Maxime Beauchemin"}] 7 | license = {text = "apache2"} 8 | classifiers = [ 9 | ] 10 | dependencies = [ 11 | "click", 12 | ] 13 | requires-python = ">3.9" 14 | 15 | [project.entry-points."console_scripts"] 16 | gor = "governator.cli:cli" 17 | governator = "governator.cli:cli" 18 | 19 | 20 | 21 | [project.urls] 22 | homepage = "https://github.com/preset-io/governator" 23 | Changelog = "https://github.com/preset-io/governator/releases" 24 | Issues = "https://github.com/preset-io/governator/issues" 25 | CI = "https://github.com/preset-io/governator/actions" 26 | 27 | [project.optional-dependencies] 28 | test = ["pytest"] 29 | dev = ["Flake8-pyproject"] 30 | 31 | [tool.flake8] 32 | max-line-length = 90 33 | 34 | [tool.mypy] 35 | ignore_missing_imports = true 36 | -------------------------------------------------------------------------------- /governator/core/permission.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Iterable 2 | from typing import TYPE_CHECKING 3 | 4 | from governator.core.base import Serializable 5 | 6 | if TYPE_CHECKING: 7 | from governator import Action 8 | 9 | 10 | class Permission(Serializable): 11 | def __init__( 12 | self, 13 | actions: "Iterable[Action]", 14 | databases=None, 15 | catalogs=None, 16 | schemas=None, 17 | relations=None, 18 | ): 19 | self.actions = actions 20 | self.databases = databases or set() 21 | self.catalogs = catalogs or set() 22 | self.schemas = schemas or set() 23 | self.relations = relations or set() 24 | 25 | @property 26 | def key(self): 27 | return "???" 28 | 29 | __slots__ = ("actions", "databases", "catalogs", "schemas", "relations") 30 | 31 | __annotations__ = { 32 | "actions": "set[Action]", 33 | "databases": "set[Database]", 34 | "catalogs": "set[Catalog]", 35 | "schemas": "set[Schema]", 36 | "relations": "set[Relation]", 37 | } 38 | -------------------------------------------------------------------------------- /governator/core/relation.py: -------------------------------------------------------------------------------- 1 | from typing import Literal 2 | 3 | from governator.core.base import SerializableCollection 4 | 5 | 6 | class Column: 7 | def __init__(self, key: str, name: str, description: str, data_type: str): 8 | self.key = key 9 | self.name = name 10 | self.description = description 11 | self.data_type = data_type 12 | 13 | 14 | class Relation: 15 | # the database schema 16 | database_schema: str 17 | # the view_name or table_name 18 | reference: str 19 | relation_type: Literal["view", "table"] 20 | columns: SerializableCollection 21 | 22 | include_count_metric: bool = True 23 | include_columns_as_dimensions: bool = True 24 | 25 | def __init__( 26 | self, 27 | database_schema: str, 28 | reference: str, 29 | relation_type: Literal["view", "table"], 30 | columns: SerializableCollection, 31 | include_count_metric: bool = True, 32 | include_columns_as_dimensions: bool = True, 33 | ): 34 | self.database_schema = database_schema 35 | self.reference = reference 36 | self.relation_type = relation_type 37 | self.columns = columns 38 | self.include_count_metric = include_count_metric 39 | self.include_columns_as_dimensions = include_columns_as_dimensions 40 | 41 | @property 42 | def key(self): 43 | return f"{self.database_schema}.{self.reference}" 44 | -------------------------------------------------------------------------------- /docs/erd.plantuml: -------------------------------------------------------------------------------- 1 | @startuml entity-relationship-diagram 2 | 3 | title Governator Entity Relationship Diagram 4 | 5 | '!theme blueprint 6 | !theme crt-amber 7 | 8 | ' avoid problems with angled crows feet 9 | 'left to right direction 10 | 11 | !define GroupBackground #black 12 | 13 | skinparam linetype ortho 14 | skinparam classBorderColor #grey 15 | skinparam BackgroundColor #444 16 | 17 | skinparam classBorderColor<> #white 18 | skinparam classBorderThickness<> 1 19 | skinparam classLineStyle<> Dashed 20 | skinparam ClassBackgroundColor<> #204143 21 | 22 | ' Models 23 | 24 | rectangle "Members" GroupBackground { 25 | entity "User" as user { 26 | *id: number <> 27 | -- 28 | uuid: BINARY(16) 29 | } 30 | entity "Group" as group { 31 | *id: number <> 32 | -- 33 | uuid: BINARY(16) 34 | } 35 | } 36 | 37 | rectangle "Core" GroupBackground { 38 | entity "Role" as role { 39 | *id: number <> 40 | -- 41 | uuid: BINARY(16) 42 | } 43 | entity "Permission" as perm { 44 | *id: number <> 45 | -- 46 | uuid: BINARY(16) 47 | } 48 | } 49 | 50 | role }|--|{ user 51 | role }|--|{ group 52 | group }|--|{ user 53 | 54 | rectangle "Objects" GroupBackground { 55 | entity "Database" as database { 56 | *id: number <> 57 | -- 58 | uuid: BINARY(16) 59 | } 60 | entity "Catalog" as catalog { 61 | *id: number <> 62 | -- 63 | uuid: BINARY(16) 64 | } 65 | entity "Schema" as schema { 66 | *id: number <> 67 | -- 68 | uuid: BINARY(16) 69 | } 70 | entity "Relation" as relation { 71 | *id: number <> 72 | -- 73 | uuid: BINARY(16) 74 | } 75 | entity "Column" as column { 76 | *id: number <> 77 | -- 78 | uuid: BINARY(16) 79 | } 80 | entity "Predicate" as predicate { 81 | *id: number <> 82 | -- 83 | uuid: BINARY(16) 84 | } 85 | } 86 | database ||--|{ catalog 87 | catalog ||--|{ schema 88 | database ||--|{ schema 89 | schema ||--|{ relation 90 | relation ||--|{ column 91 | relation ||--|{ predicate 92 | 93 | 94 | role }|--|{ perm 95 | 96 | perm ||--|{ database 97 | perm ||--|{ catalog 98 | perm ||--|{ schema 99 | perm ||--|{ relation 100 | perm ||--|{ predicate 101 | -------------------------------------------------------------------------------- /governator/database_inferface/base.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | from sqlalchemy import MetaData, create_engine 4 | from sqlalchemy.engine.reflection import Inspector 5 | 6 | 7 | class BaseDatabaseInterface: 8 | def __init__(self, connection_string): 9 | self.connection_string = connection_string 10 | self.engine = self.get_sqla_engine(connection_string) 11 | self.metadata = MetaData(bind=self.engine) 12 | self.inspector = self.get_sqla_inspector() 13 | 14 | def get_sqla_engine(self, connection_string): 15 | return create_engine(self.connection_string) 16 | 17 | def get_sqla_inspector(self): 18 | return Inspector.from_engine(self.engine) 19 | 20 | def get_schemas(self): 21 | inspector = Inspector.from_engine(self.engine) 22 | return inspector.get_schema_names() 23 | 24 | def get_tables(self, schema=None): 25 | if schema: 26 | return self.inspector.get_table_names(schema=schema) 27 | else: 28 | tables = [] 29 | for schema in self.get_schemas(): 30 | tables += self.inspector.get_table_names(schema=schema) 31 | return tables 32 | 33 | def get_columns(self, table_name, schema=None): 34 | return self.inspector.get_columns(table_name, schema=schema) 35 | 36 | def crawl(self, fetch_column_metadata=False) -> dict[str, Any]: 37 | inspector = Inspector.from_engine(self.engine) 38 | database_structure: dict[str, Any] = {} 39 | 40 | for schema in inspector.get_schema_names(): 41 | database_structure[schema] = {"tables": {}} 42 | for table_name in inspector.get_table_names(schema=schema): 43 | database_structure[schema]["tables"][table_name] = {"columns": {}} 44 | if fetch_column_metadata: 45 | for column in inspector.get_columns(table_name, schema=schema): 46 | database_structure[schema]["tables"][table_name]["columns"][ 47 | column["name"] 48 | ] = { 49 | "type": str(column["type"]), 50 | "nullable": column["nullable"], 51 | "default": str(column["default"]) 52 | if column["default"] 53 | else None, 54 | } 55 | return database_structure 56 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | repos: 18 | - repo: https://github.com/MarcoGorelli/auto-walrus 19 | rev: v0.2.2 20 | hooks: 21 | - id: auto-walrus 22 | - repo: https://github.com/asottile/pyupgrade 23 | rev: v3.4.0 24 | hooks: 25 | - id: pyupgrade 26 | args: 27 | - --py39-plus 28 | - repo: https://github.com/hadialqattan/pycln 29 | rev: v2.1.2 30 | hooks: 31 | - id: pycln 32 | args: 33 | - --disable-all-dunder-policy 34 | - --exclude=superset/config.py 35 | - --extend-exclude=tests/integration_tests/superset_test_config.*.py 36 | - repo: https://github.com/pre-commit/mirrors-mypy 37 | rev: v1.3.0 38 | hooks: 39 | - id: mypy 40 | args: [--check-untyped-defs] 41 | additional_dependencies: [ 42 | types-simplejson, 43 | types-PyYAML, 44 | types-Markdown, 45 | ] 46 | - repo: https://github.com/pre-commit/pre-commit-hooks 47 | rev: v4.4.0 48 | hooks: 49 | - id: check-docstring-first 50 | - id: check-added-large-files 51 | - id: check-yaml 52 | - id: debug-statements 53 | - id: end-of-file-fixer 54 | - id: trailing-whitespace 55 | - repo: https://github.com/psf/black 56 | rev: 23.1.0 57 | hooks: 58 | - id: black 59 | language_version: python3 60 | - repo: https://github.com/pre-commit/mirrors-prettier 61 | rev: v2.4.1 # Use the sha or tag you want to point at 62 | hooks: 63 | - id: prettier 64 | args: ["--ignore-path=./superset-frontend/.prettierignore"] 65 | files: "superset-frontend" 66 | - repo: https://github.com/astral-sh/ruff-pre-commit 67 | rev: v0.3.7 68 | hooks: 69 | - id: ruff 70 | args: [ --fix ] 71 | - id: ruff-format 72 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Governator 2 | 3 | 4 | 5 | **Governator** is a data governance toolkit that allows for managing 6 | **data access policies** (as in who as access to what) as code. 7 | 8 | With **Governator**, you can diff, pull, push and ultimately synchronize 9 | data access rules across different systems, namely database engines and 10 | business intelligence tools. 11 | 12 | 13 | 14 | ## But why? 15 | 16 | Navigating data access in today's data infrastructure stacks is 17 | tricky. Data is spread across data warehouses, 18 | clouds, various databases, and a multitude of BI tools, 19 | each with its own access rules. 20 | 21 | It's a lot to manage. 22 | 23 | Here's what we're dealing with: 24 | 25 | - Complex data environments that are evolving quickly 26 | - Diverse access rules that are hard to keep straight 27 | - Data privacy and compliance rules that simply can't be overlooked 28 | - Conmplex yet critical auditing requirements: 29 | - who has access to what? 30 | - who **had** access to this asset last year? 31 | - who **gave** access to this asset to this group? 32 | - who should be able to grant access to these resources in the first place? 33 | 34 | ### Syncing BI tools and RDMS 35 | 36 | Historically most BI tools would tend to use service accounts to access 37 | RDBMS, and have their own data access policy model to expose or hide 38 | resources within their system. But in a world with multiple BI tools, and 39 | where some actors may need more direct access to database - for example 40 | a data scientist that wants to connect programmatically to Snowflake, or 41 | from Airflow or `DBT` for instance - we want to make sure that 1) they 42 | have a consistent view and access to resources across tools, and 2) that 43 | their identify gets carried through, regardless of the layer they hit. 44 | 45 | ### A single pane of glass 46 | 47 | When a new data scientist joins the growth team, they'll need access to 48 | a bunch of things, various databases, various BI tools, some s3 buckets, 49 | ... Here we're hoping you can simply add them to a group and/or assign them 50 | some roles, and for the magic to happen automatically. Similarly, when 51 | they change position within the company, or the roles is given a broaded 52 | or more limited access to assets, everyone should get that access. 53 | 54 | ### The power of source control 55 | 56 | When managed as code, data access policies get: 57 | 58 | 1. a review/approval flow, through the proven code review processes, leaving 59 | a trace of who granted what, who approved and merged the change, when 60 | 1. a full trace of what hapenned in time 61 | 1. complex rules and logic can be established, compiled and reviewed 62 | 1. automation: continuous integration-type jobs can be executed to trigger 63 | arbitrary workflow, notifications and alerts 64 | 65 | 66 | ## Information Architecture 67 | 68 | ### Data objects 69 | 70 | * Databases 71 | * Schemas 72 | * Relations (views and tables) 73 | * Row-level predicates 74 | * Column restrictions 75 | 76 | ### Accessors (Users and service accounts) 77 | * Users 78 | * Groups 79 | 80 | ## Integrations 81 | 82 | Governator is extensible, and the goal is to extend it to support a wider 83 | array of systems. It's build with extensibility in mind and it should 84 | be straightforward to add support to new systems. 85 | 86 | * **File system**: as `yaml`, allowing for managing in source control. While 87 | you may define semantics as code, they can be materialize atomically as 88 | yaml in source control. 89 | * RDBMS: 90 | * Snowflake 91 | * BigQuery 92 | * BI tools: 93 | * Apache Superset 94 | -------------------------------------------------------------------------------- /governator/core/base.py: -------------------------------------------------------------------------------- 1 | from dataclasses import fields, is_dataclass 2 | from typing import Any 3 | 4 | import yaml 5 | 6 | 7 | class Serializable: 8 | """Serializable mixin providing serialization to/from dictionary and YAML.""" 9 | 10 | def to_dict(self) -> dict: 11 | """Converts the object to a dictionary, including properties.""" 12 | if is_dataclass(self.__class__): 13 | d = {} 14 | for field in fields(self): # type: ignore 15 | value = getattr(self, field.name) 16 | if hasattr(value, "to_serializable"): 17 | d[field.name] = value.to_serializable() 18 | else: 19 | d[field.name] = value 20 | 21 | props = {name: getattr(self, name) for name in self.properties()} 22 | return {**props, **d} 23 | else: 24 | raise Exception("Nah gotta provide a to_serializable for non-dataclass") 25 | 26 | def to_serializable(self) -> dict: 27 | return self.to_dict() 28 | 29 | @classmethod 30 | def properties(cls) -> set: 31 | """Finds all properties in the class.""" 32 | return { 33 | name for name, value in vars(cls).items() if isinstance(value, property) 34 | } 35 | 36 | @classmethod 37 | def from_dict(cls, d: dict[Any, Any]) -> "Serializable": 38 | """Creates an instance of the class from a dictionary, excluding properties.""" 39 | filtered_dict = { 40 | key: value for key, value in d.items() if key not in cls.properties() 41 | } 42 | return cls(**filtered_dict) 43 | 44 | def to_yaml(self, key=None) -> str: 45 | """Converts the object to a YAML string.""" 46 | obj = self.to_serializable() 47 | if key and key in obj: 48 | obj = obj[key] 49 | return yaml.dump(obj, sort_keys=False) 50 | 51 | def to_yaml_file(self, filename: str, wrap_under: str | None = None) -> None: 52 | """Writes the object to a YAML file.""" 53 | d = self.to_serializable() 54 | if wrap_under is not None: 55 | d = {wrap_under: d} 56 | with open(filename, "w") as file: 57 | yaml.dump(d, file, sort_keys=False) 58 | 59 | @classmethod 60 | def from_yaml(cls, yaml_string: str) -> "Serializable": 61 | """Creates an instance of the class from a YAML string, excluding properties.""" 62 | data = yaml.load(yaml_string, Loader=yaml.FullLoader) 63 | return cls.from_dict(data) 64 | 65 | @classmethod 66 | def from_yaml_file(cls, filename: str, verbose: bool = True) -> "Serializable": 67 | """Creates an instance of the class from a YAML file, excluding properties.""" 68 | with open(filename) as file: 69 | print(f"Loading file {filename}") 70 | data = yaml.load(file, Loader=yaml.FullLoader) 71 | return cls.from_dict(data) 72 | 73 | 74 | class SerializableCollection(dict, Serializable): 75 | def __init__(self, iterable: list | None = None) -> None: 76 | iterable = iterable or [] 77 | for o in iterable: 78 | self[o.key] = o 79 | 80 | def to_serializable(self): 81 | iterable = [] 82 | for o in self: 83 | if hasattr(o, "to_dict"): 84 | o = o.to_dict() 85 | iterable.append(o) 86 | return iterable 87 | 88 | @classmethod 89 | def from_yaml_file( # type: ignore 90 | cls, filename: str, object_class: Serializable, key: str | None = None 91 | ) -> "SerializableCollection": 92 | """Creates an instance of the class from a YAML file, excluding properties.""" 93 | data = None 94 | with open(filename) as file: 95 | data = yaml.load(file, Loader=yaml.FullLoader) 96 | if key and data: 97 | data = data.get(key) 98 | if data: 99 | data = [object_class.from_dict(o) for o in data] 100 | return cls(data) 101 | 102 | def append(self, obj): 103 | self[obj.key] = obj 104 | 105 | def upsert(self, collection): 106 | self.update(collection) 107 | 108 | def __iter__(self): 109 | return iter(self.values()) 110 | -------------------------------------------------------------------------------- /docs/erd.svg: -------------------------------------------------------------------------------- 1 | Governator Entity Relationship DiagramMembersCoreObjectsUserid: number «generated»uuid: BINARY(16)Groupid: number «generated»uuid: BINARY(16)Roleid: number «generated»uuid: BINARY(16)Permissionid: number «generated»uuid: BINARY(16)Databaseid: number «generated»uuid: BINARY(16)Catalogid: number «generated»uuid: BINARY(16)Schemaid: number «generated»uuid: BINARY(16)Relationid: number «generated»uuid: BINARY(16)Columnid: number «generated»uuid: BINARY(16)Predicateid: number «generated»uuid: BINARY(16) 2 | --------------------------------------------------------------------------------