├── tests ├── __init__.py ├── test_visualizers.py ├── conftest.py └── test_graph.py ├── personal_graph ├── py.typed ├── database │ ├── fhirdb │ │ └── __init__.py │ ├── sqlite │ │ ├── __init__.py │ │ └── raw-queries │ │ │ ├── delete-node.sql │ │ │ ├── search-edges-inbound.sql │ │ │ ├── delete-edge.sql │ │ │ ├── search-edges-outbound.sql │ │ │ ├── insert-node.sql │ │ │ ├── update-node.sql │ │ │ ├── search-edges.sql │ │ │ ├── insert-edge.sql │ │ │ ├── search-node.template │ │ │ ├── search-where.template │ │ │ ├── search-node-by-rowid.sql │ │ │ ├── search-edge-by-rowid.sql │ │ │ ├── traverse.template │ │ │ └── schema.sql │ ├── tursodb │ │ ├── __init__.py │ │ ├── raw-queries │ │ │ ├── delete-node.sql │ │ │ ├── delete-edge.sql │ │ │ ├── search-edges-inbound.sql │ │ │ ├── search-edges-outbound.sql │ │ │ ├── insert-node.sql │ │ │ ├── update-node.sql │ │ │ ├── search-edges.sql │ │ │ ├── insert-edge.sql │ │ │ ├── search-node.template │ │ │ ├── search-where.template │ │ │ ├── search-node-by-rowid.sql │ │ │ ├── search-edge-by-rowid.sql │ │ │ ├── traverse.template │ │ │ └── schema.sql │ │ └── turso.py │ ├── postgres │ │ └── README.md │ ├── redis │ │ └── README.md │ ├── __init__.py │ └── db.py ├── vector_store │ ├── vlitevss │ │ ├── __init__.py │ │ └── vlitevss.py │ ├── sqlitevss │ │ ├── __init__.py │ │ ├── embeddings-raw-queries │ │ │ ├── delete-node-embedding.sql │ │ │ ├── delete-edge-embedding.sql │ │ │ ├── insert-node-embedding.sql │ │ │ ├── insert-edge-embedding.sql │ │ │ ├── similarity-search-node.sql │ │ │ ├── similarity-search-edge.sql │ │ │ ├── vector-store-schema.sql │ │ │ ├── vector-search-edge.sql │ │ │ ├── vector-search-edge-desc.sql │ │ │ └── vector-search-node.sql │ │ ├── fhirsqlitevss.py │ │ ├── sqlitevss.py │ │ └── fhir-embedding-queries │ │ │ └── fhir_4_vector_schema.sql │ ├── __init__.py │ └── vector_store.py ├── graph_generator │ ├── __init__.py │ └── generator.py ├── text.py ├── __init__.py ├── extract_classes.py ├── ontology.py ├── retriever.py ├── models.py ├── embeddings.py ├── visualizers.py ├── helper.py ├── clients.py └── ml.py ├── scripts ├── __init__.py ├── time_complexity.py └── kgchat.py ├── examples ├── ml_integration.py ├── retriever_example.py ├── fhir_ontology_example.py ├── onto_graph_api.py ├── dspy_program.py ├── nl_interface.py ├── ontology_example.py ├── fhirgraphdb.py └── pythonic_api.py ├── stub └── personal_graph │ ├── database │ ├── sqlite │ │ ├── __init__.pyi │ │ └── sqlite.pyi │ ├── tursodb │ │ ├── __init__.pyi │ │ └── turso.pyi │ ├── __init__.pyi │ └── db.pyi │ ├── vector_store │ ├── sqlitevss │ │ ├── __init__.pyi │ │ └── sqlitevss.pyi │ ├── vlitevss │ │ ├── __init__.pyi │ │ └── vlitevss.pyi │ ├── __init__.pyi │ └── vector_store.pyi │ ├── graph_generator │ ├── __init__.pyi │ └── generator.pyi │ ├── text.pyi │ ├── retriever.pyi │ ├── ml.pyi │ ├── models.pyi │ ├── embeddings.pyi │ ├── visualizers.pyi │ ├── clients.pyi │ └── graph.pyi ├── Makefile ├── fhir_ontology ├── ontology_classes_and_properties.pkl ├── docs │ └── visualize_ontology_files │ │ └── libs │ │ ├── bootstrap │ │ └── bootstrap-icons.woff │ │ ├── quarto-html │ │ ├── tippy.css │ │ ├── quarto-syntax-highlighting.css │ │ └── anchor.min.js │ │ └── clipboard │ │ └── clipboard.min.js ├── fhir_ontology_json.py └── visualize_fhir_ontology.py ├── .gitignore ├── LICENSE ├── pyproject.toml ├── setup.py ├── .github └── workflows │ └── personal-graph.yaml └── README.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /personal_graph/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scripts/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/ml_integration.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /personal_graph/database/fhirdb/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /personal_graph/database/sqlite/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /personal_graph/database/tursodb/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /personal_graph/database/postgres/README.md: -------------------------------------------------------------------------------- 1 | # TODO -------------------------------------------------------------------------------- /personal_graph/database/redis/README.md: -------------------------------------------------------------------------------- 1 | # TODO -------------------------------------------------------------------------------- /personal_graph/vector_store/vlitevss/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /stub/personal_graph/database/sqlite/__init__.pyi: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /personal_graph/vector_store/sqlitevss/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /stub/personal_graph/database/tursodb/__init__.pyi: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /stub/personal_graph/vector_store/sqlitevss/__init__.pyi: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /stub/personal_graph/vector_store/vlitevss/__init__.pyi: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /personal_graph/database/sqlite/raw-queries/delete-node.sql: -------------------------------------------------------------------------------- 1 | DELETE FROM nodes WHERE id = ? -------------------------------------------------------------------------------- /personal_graph/database/tursodb/raw-queries/delete-node.sql: -------------------------------------------------------------------------------- 1 | DELETE FROM nodes WHERE id = ? -------------------------------------------------------------------------------- /personal_graph/database/sqlite/raw-queries/search-edges-inbound.sql: -------------------------------------------------------------------------------- 1 | SELECT * FROM edges WHERE source = ? -------------------------------------------------------------------------------- /personal_graph/database/sqlite/raw-queries/delete-edge.sql: -------------------------------------------------------------------------------- 1 | DELETE FROM edges WHERE source = ? OR target = ? -------------------------------------------------------------------------------- /personal_graph/database/sqlite/raw-queries/search-edges-outbound.sql: -------------------------------------------------------------------------------- 1 | SELECT * FROM edges WHERE target = ? -------------------------------------------------------------------------------- /personal_graph/database/tursodb/raw-queries/delete-edge.sql: -------------------------------------------------------------------------------- 1 | DELETE FROM edges WHERE source = ? OR target = ? -------------------------------------------------------------------------------- /personal_graph/database/tursodb/raw-queries/search-edges-inbound.sql: -------------------------------------------------------------------------------- 1 | SELECT * FROM edges WHERE source = ? -------------------------------------------------------------------------------- /personal_graph/database/tursodb/raw-queries/search-edges-outbound.sql: -------------------------------------------------------------------------------- 1 | SELECT * FROM edges WHERE target = ? -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | docs: 2 | quarto render fhir_ontology/visualize_ontology.ipynb --to html --output-dir ./docs --execute -------------------------------------------------------------------------------- /personal_graph/database/sqlite/raw-queries/insert-node.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO nodes (embed_id, label, attributes) VALUES(?, ? ,json(?)) 2 | -------------------------------------------------------------------------------- /personal_graph/database/sqlite/raw-queries/update-node.sql: -------------------------------------------------------------------------------- 1 | UPDATE nodes SET label = ?, attributes = json(?), embed_id = ? WHERE id = ? -------------------------------------------------------------------------------- /personal_graph/database/tursodb/raw-queries/insert-node.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO nodes (embed_id, label, attributes) VALUES(?, ? ,json(?)) 2 | -------------------------------------------------------------------------------- /personal_graph/vector_store/sqlitevss/embeddings-raw-queries/delete-node-embedding.sql: -------------------------------------------------------------------------------- 1 | DELETE FROM nodes_embedding where rowid = (?) -------------------------------------------------------------------------------- /personal_graph/database/tursodb/raw-queries/update-node.sql: -------------------------------------------------------------------------------- 1 | UPDATE nodes SET label = ?, attributes = json(?), embed_id = ? WHERE id = ? -------------------------------------------------------------------------------- /personal_graph/vector_store/sqlitevss/embeddings-raw-queries/delete-edge-embedding.sql: -------------------------------------------------------------------------------- 1 | DELETE FROM relationship_embedding WHERE rowid = (?) -------------------------------------------------------------------------------- /personal_graph/database/sqlite/raw-queries/search-edges.sql: -------------------------------------------------------------------------------- 1 | SELECT * FROM edges WHERE source = ? 2 | UNION 3 | SELECT * FROM edges WHERE target = ? -------------------------------------------------------------------------------- /personal_graph/database/tursodb/raw-queries/search-edges.sql: -------------------------------------------------------------------------------- 1 | SELECT * FROM edges WHERE source = ? 2 | UNION 3 | SELECT * FROM edges WHERE target = ? -------------------------------------------------------------------------------- /personal_graph/vector_store/sqlitevss/embeddings-raw-queries/insert-node-embedding.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO nodes_embedding(rowid, vector_nodes) VALUES (?,?); -------------------------------------------------------------------------------- /personal_graph/database/sqlite/raw-queries/insert-edge.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO edges (embed_id, source, target, label, attributes) VALUES(?, ?, ?, ?, json(?)) -------------------------------------------------------------------------------- /personal_graph/database/tursodb/raw-queries/insert-edge.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO edges (embed_id, source, target, label, attributes) VALUES(?, ?, ?, ?, json(?)) -------------------------------------------------------------------------------- /personal_graph/vector_store/sqlitevss/embeddings-raw-queries/insert-edge-embedding.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO relationship_embedding(rowid, vector_relations) VALUES(?, ?) -------------------------------------------------------------------------------- /fhir_ontology/ontology_classes_and_properties.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Technoculture/personal-graph/HEAD/fhir_ontology/ontology_classes_and_properties.pkl -------------------------------------------------------------------------------- /personal_graph/vector_store/sqlitevss/embeddings-raw-queries/similarity-search-node.sql: -------------------------------------------------------------------------------- 1 | SELECT rowid, distance FROM nodes_embedding WHERE vss_search(vector_nodes, vss_search_params(json(?), ?)) -------------------------------------------------------------------------------- /personal_graph/vector_store/sqlitevss/embeddings-raw-queries/similarity-search-edge.sql: -------------------------------------------------------------------------------- 1 | SELECT rowid, distance FROM relationship_embedding WHERE vss_search(vector_relations, vss_search_params(json(?), ?)) -------------------------------------------------------------------------------- /fhir_ontology/docs/visualize_ontology_files/libs/bootstrap/bootstrap-icons.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Technoculture/personal-graph/HEAD/fhir_ontology/docs/visualize_ontology_files/libs/bootstrap/bootstrap-icons.woff -------------------------------------------------------------------------------- /stub/personal_graph/graph_generator/__init__.pyi: -------------------------------------------------------------------------------- 1 | from personal_graph.graph_generator.generator import ( 2 | OpenAITextToGraphParser as OpenAITextToGraphParser, 3 | ) 4 | 5 | __all__ = ["OpenAITextToGraphParser"] 6 | -------------------------------------------------------------------------------- /stub/personal_graph/database/__init__.pyi: -------------------------------------------------------------------------------- 1 | from personal_graph.database.sqlite.sqlite import SQLite as SQLite 2 | from personal_graph.database.tursodb.turso import TursoDB as TursoDB 3 | 4 | __all__ = ["TursoDB", "SQLite"] 5 | -------------------------------------------------------------------------------- /personal_graph/graph_generator/__init__.py: -------------------------------------------------------------------------------- 1 | from personal_graph.graph_generator.generator import ( 2 | OpenAITextToGraphParser, 3 | OllamaTextToGraphParser, 4 | ) 5 | 6 | __all__ = ["OpenAITextToGraphParser", "OllamaTextToGraphParser"] 7 | -------------------------------------------------------------------------------- /personal_graph/database/__init__.py: -------------------------------------------------------------------------------- 1 | from personal_graph.database.tursodb.turso import TursoDB 2 | from personal_graph.database.sqlite.sqlite import SQLite 3 | from personal_graph.database.fhirdb.fhirDB import FhirDB 4 | 5 | __all__ = ["TursoDB", "SQLite", "FhirDB"] 6 | -------------------------------------------------------------------------------- /stub/personal_graph/vector_store/__init__.pyi: -------------------------------------------------------------------------------- 1 | from personal_graph.vector_store.sqlitevss.sqlitevss import ( 2 | SQLiteVSS as SQLiteVSS, 3 | ) 4 | from personal_graph.vector_store.vlitevss.vlitevss import ( 5 | VliteVSS as VliteVSS, 6 | ) 7 | 8 | __all__ = ["VliteVSS", "SQLiteVSS"] 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # general things to ignore 2 | build/ 3 | dist/ 4 | *.egg-info/ 5 | *.egg 6 | *.py[cod] 7 | __pycache__/ 8 | *.so 9 | *~ 10 | .idea/ 11 | .env 12 | log/ 13 | /.quarto/ 14 | 15 | # due to using tox and pytest 16 | .tox 17 | .cache 18 | .pytest_cache 19 | .python-version 20 | poetry.toml 21 | -------------------------------------------------------------------------------- /personal_graph/database/sqlite/raw-queries/search-node.template: -------------------------------------------------------------------------------- 1 | SELECT {{ result_column }} -- id|body 2 | FROM nodes{% if tree %}, json_tree(attributes{% if key %}, '$.{{ key }}'{% endif %}){% endif %}{% if search_clauses %} 3 | WHERE {% for search_clause in search_clauses %} 4 | {{ search_clause }} 5 | {% endfor %}{% endif %} -------------------------------------------------------------------------------- /personal_graph/database/tursodb/raw-queries/search-node.template: -------------------------------------------------------------------------------- 1 | SELECT {{ result_column }} -- id|body 2 | FROM nodes{% if tree %}, json_tree(attributes{% if key %}, '$.{{ key }}'{% endif %}){% endif %}{% if search_clauses %} 3 | WHERE {% for search_clause in search_clauses %} 4 | {{ search_clause }} 5 | {% endfor %}{% endif %} -------------------------------------------------------------------------------- /personal_graph/vector_store/sqlitevss/embeddings-raw-queries/vector-store-schema.sql: -------------------------------------------------------------------------------- 1 | begin; 2 | 3 | CREATE VIRTUAL TABLE IF NOT EXISTS nodes_embedding USING vss0( 4 | vector_nodes({{size}}) 5 | ); 6 | 7 | CREATE VIRTUAL TABLE IF NOT EXISTS relationship_embedding USING vss0( 8 | vector_relations({{size}}) 9 | ); 10 | 11 | commit; -------------------------------------------------------------------------------- /personal_graph/vector_store/sqlitevss/embeddings-raw-queries/vector-search-edge.sql: -------------------------------------------------------------------------------- 1 | with matches as (select rowid, distance from relationship_embedding where vss_search (vector_relations, vss_search_params(json(?), ?))) select rowid, edges.source, edges.target, edges.label, edges.attributes, matches.distance from matches join edges on edges.embed_id = matches.rowid -------------------------------------------------------------------------------- /personal_graph/database/sqlite/raw-queries/search-where.template: -------------------------------------------------------------------------------- 1 | {% if and_or %}{{ and_or }}{% endif %} 2 | {% if id_lookup %}id = ?{% endif %} 3 | {% if key_value %}json_extract(attributes, '$.{{ key }}') {{ predicate }} ?{% endif %} 4 | {% if tree %}{% if key %}(json_tree.key='{{ key }}' AND {% endif %}json_tree.value {{ predicate }} ?{% if key %}){% endif %}{% endif %} -------------------------------------------------------------------------------- /personal_graph/database/tursodb/raw-queries/search-where.template: -------------------------------------------------------------------------------- 1 | {% if and_or %}{{ and_or }}{% endif %} 2 | {% if id_lookup %}id = ?{% endif %} 3 | {% if key_value %}json_extract(attributes, '$.{{ key }}') {{ predicate }} ?{% endif %} 4 | {% if tree %}{% if key %}(json_tree.key='{{ key }}' AND {% endif %}json_tree.value {{ predicate }} ?{% if key %}){% endif %}{% endif %} -------------------------------------------------------------------------------- /personal_graph/vector_store/sqlitevss/embeddings-raw-queries/vector-search-edge-desc.sql: -------------------------------------------------------------------------------- 1 | with matches as (select rowid, distance from relationship_embedding where vss_search (vector_relations, vss_search_params(json(?), ?))) select rowid, edges.source, edges.target, edges.label, edges.attributes, matches.distance from matches join edges on edges.embed_id = matches.rowid ORDER BY distance DESC -------------------------------------------------------------------------------- /stub/personal_graph/text.pyi: -------------------------------------------------------------------------------- 1 | from personal_graph import ( 2 | KnowledgeGraph as KnowledgeGraph, 3 | OpenAIClient as OpenAIClient, 4 | ) 5 | from personal_graph.graph_generator import ( 6 | OpenAITextToGraphParser as OpenAITextToGraphParser, 7 | ) 8 | 9 | def text_to_graph( 10 | text: str, graph_generator: OpenAITextToGraphParser = ... 11 | ) -> KnowledgeGraph: ... 12 | -------------------------------------------------------------------------------- /personal_graph/vector_store/__init__.py: -------------------------------------------------------------------------------- 1 | from personal_graph.vector_store.sqlitevss.sqlitevss import SQLiteVSS 2 | from personal_graph.vector_store.vlitevss.vlitevss import VliteVSS 3 | from personal_graph.vector_store.sqlitevss.fhirsqlitevss import FhirSQLiteVSS 4 | from personal_graph.vector_store.vector_store import VectorStore 5 | 6 | __all__ = ["VliteVSS", "SQLiteVSS", "VectorStore", "FhirSQLiteVSS"] 7 | -------------------------------------------------------------------------------- /stub/personal_graph/retriever.pyi: -------------------------------------------------------------------------------- 1 | import dspy # type: ignore 2 | from personal_graph.graph import GraphDB as GraphDB, Node as Node 3 | from typing import List 4 | 5 | class PersonalRM(dspy.Retrieve): 6 | k: int 7 | graph: GraphDB 8 | def __init__(self, graph: GraphDB, k: int = 5) -> None: ... 9 | def forward( 10 | self, query_or_queries: str | List[str], k: int | None = None, **kwargs 11 | ) -> List[dspy.Prediction]: ... 12 | -------------------------------------------------------------------------------- /personal_graph/database/sqlite/raw-queries/search-node-by-rowid.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | n.embed_id, 3 | n.id, 4 | n.label, 5 | n.attributes 6 | FROM nodes n 7 | JOIN (SELECT value AS embed_id FROM json_each(?)) AS ids ON n.embed_id = ids.embed_id 8 | ORDER BY 9 | CASE 10 | WHEN ? != "" AND ? IS TRUE THEN json_extract(n.attributes, '$.' || ?) ELSE NULL 11 | END DESC, 12 | CASE 13 | WHEN ? != "" AND ? IS FALSE THEN json_extract(n.attributes, '$.' || ?) ELSE NULL 14 | END ASC; -------------------------------------------------------------------------------- /personal_graph/database/tursodb/raw-queries/search-node-by-rowid.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | n.embed_id, 3 | n.id, 4 | n.label, 5 | n.attributes 6 | FROM nodes n 7 | JOIN (SELECT value AS embed_id FROM json_each(?)) AS ids ON n.embed_id = ids.embed_id 8 | ORDER BY 9 | CASE 10 | WHEN ? != "" AND ? IS TRUE THEN json_extract(n.attributes, '$.' || ?) ELSE NULL 11 | END DESC, 12 | CASE 13 | WHEN ? != "" AND ? IS FALSE THEN json_extract(n.attributes, '$.' || ?) ELSE NULL 14 | END ASC; -------------------------------------------------------------------------------- /personal_graph/database/sqlite/raw-queries/search-edge-by-rowid.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | e.embed_id, 3 | e.source, 4 | e.target, 5 | e.label, 6 | e.attributes 7 | FROM edges e 8 | JOIN (SELECT value AS embed_id FROM json_each(?)) AS ids ON e.embed_id = ids.embed_id 9 | ORDER BY 10 | CASE 11 | WHEN ? != "" AND ? IS TRUE THEN json_extract(e.attributes, '$.' || ?) ELSE NULL 12 | END DESC, 13 | CASE 14 | WHEN ? != "" AND ? IS FALSE THEN json_extract(e.attributes, '$.' || ?) ELSE NULL 15 | END ASC; -------------------------------------------------------------------------------- /personal_graph/database/tursodb/raw-queries/search-edge-by-rowid.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | e.embed_id, 3 | e.source, 4 | e.target, 5 | e.label, 6 | e.attributes 7 | FROM edges e 8 | JOIN (SELECT value AS embed_id FROM json_each(?)) AS ids ON e.embed_id = ids.embed_id 9 | ORDER BY 10 | CASE 11 | WHEN ? != "" AND ? IS TRUE THEN json_extract(e.attributes, '$.' || ?) ELSE NULL 12 | END DESC, 13 | CASE 14 | WHEN ? != "" AND ? IS FALSE THEN json_extract(e.attributes, '$.' || ?) ELSE NULL 15 | END ASC; -------------------------------------------------------------------------------- /stub/personal_graph/ml.pyi: -------------------------------------------------------------------------------- 1 | import networkx as nx # type: ignore 2 | from personal_graph import ( 3 | Edge as Edge, 4 | EdgeInput as EdgeInput, 5 | GraphDB as GraphDB, 6 | KnowledgeGraph as KnowledgeGraph, 7 | Node as Node, 8 | ) 9 | 10 | def pg_to_networkx(graph: GraphDB, *, post_visualize: bool = False): ... 11 | def networkx_to_pg( 12 | networkx_graph: nx, 13 | graph: GraphDB, 14 | *, 15 | post_visualize: bool = False, 16 | override: bool = True, 17 | ): ... 18 | -------------------------------------------------------------------------------- /personal_graph/text.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | 3 | from personal_graph import OpenAIClient, KnowledgeGraph 4 | from personal_graph.graph_generator import ( 5 | OpenAITextToGraphParser, 6 | OllamaTextToGraphParser, 7 | ) 8 | 9 | 10 | def text_to_graph( 11 | text: str, 12 | graph_generator: Union[ 13 | OpenAITextToGraphParser, OllamaTextToGraphParser 14 | ] = OpenAITextToGraphParser(llm_client=OpenAIClient()), 15 | ) -> KnowledgeGraph: 16 | kg = graph_generator.generate(text) 17 | 18 | return kg 19 | -------------------------------------------------------------------------------- /examples/retriever_example.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from personal_graph import GraphDB, PersonalRM 3 | 4 | 5 | def main(): 6 | with GraphDB() as graph: 7 | query = "What is the similarity between Jack and Ronaldo?" 8 | retriever = PersonalRM(graph) 9 | 10 | passages = retriever.forward(query) 11 | 12 | logging.info("Retrieved Results: ") 13 | for passage in passages: 14 | logging.info(passage) 15 | 16 | 17 | if __name__ == "__main__": 18 | logging.basicConfig( 19 | level=logging.DEBUG, 20 | ) 21 | 22 | main() 23 | -------------------------------------------------------------------------------- /stub/personal_graph/models.pyi: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | from typing import Any, Dict, List 3 | 4 | class Node(BaseModel): 5 | id: int | str 6 | attributes: str | Dict[Any, Any] 7 | label: str 8 | 9 | class Edge(BaseModel): 10 | source: int | str 11 | target: int | str 12 | label: str 13 | attributes: str | Dict[Any, Any] 14 | 15 | class EdgeInput(BaseModel): 16 | source: Node 17 | target: Node 18 | label: str 19 | attributes: str | Dict[Any, Any] 20 | 21 | class KnowledgeGraph(BaseModel): 22 | nodes: List[Node] 23 | edges: List[Edge] 24 | -------------------------------------------------------------------------------- /stub/personal_graph/embeddings.pyi: -------------------------------------------------------------------------------- 1 | import abc 2 | import openai 3 | from _typeshed import Incomplete 4 | from abc import ABC, abstractmethod 5 | 6 | class EmbeddingsModel(ABC, metaclass=abc.ABCMeta): 7 | @abstractmethod 8 | def get_embedding(self, text: str) -> list[float]: ... 9 | 10 | class OpenAIEmbeddingsModel(EmbeddingsModel): 11 | client: Incomplete 12 | model: Incomplete 13 | dimension: Incomplete 14 | def __init__( 15 | self, embed_client: openai.OpenAI, embed_model: str, embed_dimension: int 16 | ) -> None: ... 17 | def get_embedding(self, text: str) -> list[float]: ... 18 | -------------------------------------------------------------------------------- /stub/personal_graph/visualizers.pyi: -------------------------------------------------------------------------------- 1 | from graphviz import Digraph # type: ignore 2 | from personal_graph.models import KnowledgeGraph as KnowledgeGraph 3 | from typing import Any, List, Tuple 4 | 5 | def graphviz_visualize_bodies( 6 | dot_file: str, 7 | path: List[Tuple[Any, str, str]] = [], 8 | format: str = "png", 9 | exclude_node_keys: List[str] = [], 10 | hide_node_key: bool = False, 11 | node_kv: str = " ", 12 | exclude_edge_keys: List[str] = [], 13 | hide_edge_key: bool = False, 14 | edge_kv: str = " ", 15 | ) -> None: ... 16 | def visualize_graph(kg: KnowledgeGraph) -> Digraph: ... 17 | -------------------------------------------------------------------------------- /personal_graph/vector_store/sqlitevss/embeddings-raw-queries/vector-search-node.sql: -------------------------------------------------------------------------------- 1 | WITH matches AS ( 2 | SELECT rowid, distance 3 | FROM nodes_embedding 4 | WHERE vss_search(vector_nodes, vss_search_params(json(?), ?)) 5 | ) 6 | SELECT 7 | rowid, 8 | nodes.id, 9 | nodes.label, 10 | nodes.attributes, 11 | matches.distance 12 | FROM matches 13 | JOIN nodes ON nodes.embed_id = matches.rowid 14 | ORDER BY 15 | CASE 16 | WHEN ? != "" AND ? IS TRUE THEN json_extract(nodes.attributes, '$.' || ?) ELSE NULL 17 | END DESC, 18 | CASE 19 | WHEN ? != "" AND ? IS FALSE THEN json_extract(nodes.attributes, '$.' || ?) ELSE NULL 20 | END ASC; -------------------------------------------------------------------------------- /personal_graph/database/sqlite/raw-queries/traverse.template: -------------------------------------------------------------------------------- 1 | WITH RECURSIVE traverse(x{% if with_bodies %}, y, obj{% endif %}) AS ( 2 | SELECT id{% if with_bodies %}, '()', attributes {% endif %} FROM nodes WHERE id = ? 3 | UNION 4 | SELECT id{% if with_bodies %}, '()', attributes {% endif %} FROM nodes JOIN traverse ON id = x 5 | {% if inbound %}UNION 6 | SELECT source{% if with_bodies %}, '<-', attributes {% endif %} FROM edges JOIN traverse ON target = x{% endif %} 7 | {% if outbound %}UNION 8 | SELECT target{% if with_bodies %}, '->', attributes {% endif %} FROM edges JOIN traverse ON source = x{% endif %} 9 | ) SELECT x{% if with_bodies %}, y, obj {% endif %} FROM traverse; 10 | -------------------------------------------------------------------------------- /personal_graph/database/tursodb/raw-queries/traverse.template: -------------------------------------------------------------------------------- 1 | WITH RECURSIVE traverse(x{% if with_bodies %}, y, obj{% endif %}) AS ( 2 | SELECT id{% if with_bodies %}, '()', attributes {% endif %} FROM nodes WHERE id = ? 3 | UNION 4 | SELECT id{% if with_bodies %}, '()', attributes {% endif %} FROM nodes JOIN traverse ON id = x 5 | {% if inbound %}UNION 6 | SELECT source{% if with_bodies %}, '<-', attributes {% endif %} FROM edges JOIN traverse ON target = x{% endif %} 7 | {% if outbound %}UNION 8 | SELECT target{% if with_bodies %}, '->', attributes {% endif %} FROM edges JOIN traverse ON source = x{% endif %} 9 | ) SELECT x{% if with_bodies %}, y, obj {% endif %} FROM traverse; 10 | -------------------------------------------------------------------------------- /stub/personal_graph/clients.pyi: -------------------------------------------------------------------------------- 1 | import abc 2 | from abc import ABC 3 | from dataclasses import dataclass 4 | 5 | class APIClient(ABC, metaclass=abc.ABCMeta): ... 6 | 7 | @dataclass 8 | class LiteLLMEmbeddingClient(APIClient): 9 | model_name: str = ... 10 | dimensions: int = ... 11 | base_url: str = ... 12 | client = ... 13 | def __post_init__(self) -> None: ... 14 | def __init__(self, model_name, dimensions, base_url) -> None: ... 15 | 16 | @dataclass 17 | class OpenAILLMClient(APIClient): 18 | base_url: str = ... 19 | model_name: str = ... 20 | client = ... 21 | def __post_init__(self) -> None: ... 22 | def __init__(self, base_url, model_name) -> None: ... 23 | -------------------------------------------------------------------------------- /stub/personal_graph/graph_generator/generator.pyi: -------------------------------------------------------------------------------- 1 | import abc 2 | from abc import ABC, abstractmethod 3 | from typing import Union 4 | 5 | from personal_graph.clients import OpenAIClient, LiteLLMClient 6 | from personal_graph.models import KnowledgeGraph as KnowledgeGraph 7 | 8 | class TextToGraphParserInterface(ABC, metaclass=abc.ABCMeta): 9 | @abstractmethod 10 | def generate(self, query: str) -> KnowledgeGraph: ... 11 | 12 | class OpenAITextToGraphParser(TextToGraphParserInterface): 13 | system_prompt: str 14 | prompt: str 15 | llm_client: Union[OpenAIClient, LiteLLMClient] 16 | def __init__( 17 | self, llm_client: OpenAIClient, system_prompt: str = ..., prompt: str = ... 18 | ) -> None: ... 19 | def generate(self, query: str) -> KnowledgeGraph: ... 20 | -------------------------------------------------------------------------------- /personal_graph/__init__.py: -------------------------------------------------------------------------------- 1 | from personal_graph.clients import ( 2 | OpenAIClient, 3 | LiteLLMClient, 4 | OllamaEmbeddingClient, 5 | OllamaClient, 6 | ) 7 | from personal_graph.retriever import PersonalRM 8 | from personal_graph.visualizers import graphviz_visualize_bodies 9 | from personal_graph.embeddings import OpenAIEmbeddingsModel 10 | from personal_graph.graph import GraphDB 11 | from personal_graph.models import Node, Edge, EdgeInput, KnowledgeGraph 12 | 13 | __all__ = [ 14 | "GraphDB", 15 | "Node", 16 | "Edge", 17 | "EdgeInput", 18 | "KnowledgeGraph", 19 | "PersonalRM", 20 | "graphviz_visualize_bodies", 21 | "OpenAIEmbeddingsModel", 22 | "OpenAIClient", 23 | "LiteLLMClient", 24 | "OllamaEmbeddingClient", 25 | "OllamaClient", 26 | ] 27 | -------------------------------------------------------------------------------- /tests/test_visualizers.py: -------------------------------------------------------------------------------- 1 | from personal_graph import graphviz_visualize_bodies 2 | 3 | 4 | def test_graphviz_visualize_bodies( 5 | mock_db_connection_and_cursor, 6 | mock_find_node, 7 | mock_get_connections, 8 | mock_dot_render, 9 | ): 10 | path_data = [ 11 | (1, "()", '{"id": 1, "name": "Alice", "age": 30}'), 12 | (2, "->", '{"id": 2, "name": "Bob", "age": 35}'), 13 | (3, "->", '{"id": 3, "name": "Charlie", "age": 40}'), 14 | ] 15 | 16 | graphviz_visualize_bodies( 17 | dot_file="mock_file_with_bodies.dot", 18 | path=path_data, 19 | format="png", 20 | exclude_node_keys=[], 21 | hide_node_key=False, 22 | node_kv=" ", 23 | exclude_edge_keys=[], 24 | hide_edge_key=False, 25 | edge_kv=" ", 26 | ) 27 | -------------------------------------------------------------------------------- /personal_graph/extract_classes.py: -------------------------------------------------------------------------------- 1 | """ 2 | This script creates a .pkl file of all the classes along with their properties that fhir ontology has. 3 | """ 4 | 5 | import logging 6 | import pickle 7 | from personal_graph.helper import extract_classes_properties 8 | 9 | 10 | def main(): 11 | # Dumping the fhir ontology's classes and properties 12 | node_types_info = extract_classes_properties() 13 | file = open("ontology_classes_and_properties.pkl", "wb") 14 | pickled_data = pickle.dumps(node_types_info) 15 | file.write(pickled_data) 16 | file.close() 17 | 18 | # Loading up the classes and properties of fhir ontology 19 | with open("ontology_classes_and_properties.pkl", "rb") as pickle_file: 20 | loaded_data = pickle.load(pickle_file) 21 | 22 | logging.info(loaded_data) 23 | 24 | 25 | if __name__ == "main": 26 | main() 27 | -------------------------------------------------------------------------------- /personal_graph/ontology.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import owlready2 # type: ignore 3 | from owlready2 import OwlReadyOntologyParsingError 4 | 5 | 6 | def from_rdf(file_path: str) -> owlready2.namespace.Ontology: 7 | """ 8 | Load an ontology from a specified file path. 9 | 10 | @param file_path: Path to the ontology file 11 | 12 | @return: owlready2.namespace.Ontology: The loaded ontology 13 | 14 | @raise: OwlReadyOntologyParsingError: Unable to parse the specified file 15 | """ 16 | try: 17 | onto = owlready2.get_ontology(file_path).load() # Loading the rdf file 18 | logging.info(onto) 19 | return onto 20 | 21 | except OwlReadyOntologyParsingError as err: 22 | raise OwlReadyOntologyParsingError( 23 | f"Error loading ontology: {str(err)}" 24 | ) from err 25 | 26 | except FileNotFoundError as ferr: 27 | raise FileNotFoundError(f"File not found: {str(ferr)}") from ferr 28 | -------------------------------------------------------------------------------- /personal_graph/database/sqlite/raw-queries/schema.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS nodes ( 2 | embed_id INT NOT NULL UNIQUE, 3 | label TEXT, 4 | attributes JSON, 5 | id TEXT GENERATED ALWAYS AS (json_extract(attributes, '$.id')) VIRTUAL NOT NULL UNIQUE, 6 | created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 7 | updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP 8 | ); 9 | 10 | CREATE INDEX IF NOT EXISTS id_idx ON nodes(id); 11 | 12 | CREATE TABLE IF NOT EXISTS edges ( 13 | embed_id INTEGER NOT NULL UNIQUE, 14 | source TEXT, 15 | target TEXT, 16 | label TEXT, 17 | attributes JSON, 18 | created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 19 | updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 20 | UNIQUE(source, target, attributes) ON CONFLICT REPLACE, 21 | FOREIGN KEY(source) REFERENCES nodes(id) ON DELETE CASCADE, 22 | FOREIGN KEY(target) REFERENCES nodes(id) ON DELETE CASCADE 23 | ); 24 | 25 | CREATE INDEX IF NOT EXISTS source_idx ON edges(source); 26 | CREATE INDEX IF NOT EXISTS target_idx ON edges(target); 27 | -------------------------------------------------------------------------------- /personal_graph/database/tursodb/raw-queries/schema.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS nodes ( 2 | embed_id INT NOT NULL UNIQUE, 3 | label TEXT, 4 | attributes JSON, 5 | id TEXT GENERATED ALWAYS AS (json_extract(attributes, '$.id')) VIRTUAL NOT NULL UNIQUE, 6 | created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 7 | updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP 8 | ); 9 | 10 | CREATE INDEX IF NOT EXISTS id_idx ON nodes(id); 11 | 12 | CREATE TABLE IF NOT EXISTS edges ( 13 | embed_id INTEGER NOT NULL UNIQUE, 14 | source TEXT, 15 | target TEXT, 16 | label TEXT, 17 | attributes JSON, 18 | created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 19 | updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 20 | UNIQUE(source, target, attributes) ON CONFLICT REPLACE, 21 | FOREIGN KEY(source) REFERENCES nodes(id) ON DELETE CASCADE, 22 | FOREIGN KEY(target) REFERENCES nodes(id) ON DELETE CASCADE 23 | ); 24 | 25 | CREATE INDEX IF NOT EXISTS source_idx ON edges(source); 26 | CREATE INDEX IF NOT EXISTS target_idx ON edges(target); 27 | -------------------------------------------------------------------------------- /stub/personal_graph/database/tursodb/turso.pyi: -------------------------------------------------------------------------------- 1 | from jinja2 import BaseLoader, Environment, Template 2 | 3 | from pathlib import Path 4 | from personal_graph.database.sqlite.sqlite import SQLite as SQLite 5 | from typing import Any, Callable, Tuple, Optional 6 | from personal_graph.database.db import CursorExecFunction 7 | 8 | def read_sql(sql_file: Path) -> str: ... 9 | 10 | class SqlTemplateLoader(BaseLoader): 11 | templates_dir: Path 12 | def __init__(self, templates_dir: Path) -> None: ... 13 | def get_source( 14 | self, environment: Environment, template: str 15 | ) -> Tuple[str, str, Callable[[], bool]]: ... 16 | 17 | class TursoDB(SQLite): 18 | db_url: Optional[str] 19 | db_auth_token: Optional[str] 20 | env: Environment 21 | clause_template: Template 22 | search_template: Template 23 | traverse_template: Template 24 | def __init__( 25 | self, *, url: str | None = None, auth_token: str | None = None 26 | ) -> None: ... 27 | def __eq__(self, other): ... 28 | def atomic(self, cursor_exec_fn: CursorExecFunction) -> Any: ... 29 | def save(self) -> None: ... 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright (c) Denis Papathanasiou 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a 5 | copy of this software and associated documentation files (the "Software"), 6 | to deal in the Software without restriction, including without limitation 7 | the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | and/or sell copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included 12 | in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 | THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 | OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /personal_graph/retriever.py: -------------------------------------------------------------------------------- 1 | import dspy # type: ignore 2 | from typing import List, Optional, Union 3 | from personal_graph.graph import GraphDB, Node 4 | from dotenv import load_dotenv 5 | 6 | load_dotenv() 7 | 8 | 9 | class PersonalRM(dspy.Retrieve): 10 | def __init__( 11 | self, 12 | graph: GraphDB, 13 | k: int = 5, 14 | ): 15 | super().__init__(k=k) 16 | self.k = k 17 | self.graph = graph 18 | 19 | def _retrieve_passages(self, queries: List[str]) -> List[Node]: 20 | passages: List[Node] = [] 21 | 22 | if not queries: 23 | return passages 24 | 25 | for query in queries: 26 | kg = self.graph.search_from_graph(query) 27 | passages.extend(kg.nodes) 28 | return passages 29 | 30 | def forward( 31 | self, query_or_queries: Union[str, List[str]], k: Optional[int] = None, **kwargs 32 | ) -> List[dspy.Prediction]: 33 | # TODO: Use the value of k 34 | if not isinstance(query_or_queries, list): 35 | query_or_queries = [query_or_queries] 36 | passages = self._retrieve_passages(query_or_queries) 37 | predictions = [ 38 | dspy.Prediction(context=p, long_text=p.attributes) for p in passages 39 | ] 40 | 41 | return predictions 42 | -------------------------------------------------------------------------------- /examples/fhir_ontology_example.py: -------------------------------------------------------------------------------- 1 | import fhir.resources as fhir # type: ignore 2 | from fhir.resources import fhirtypes # type: ignore 3 | from personal_graph import GraphDB, Node 4 | 5 | 6 | with GraphDB(ontologies=[fhir]) as graph: 7 | # Define nodes 8 | node1 = Node( 9 | id="1", 10 | label="Pabelo", 11 | attributes={ 12 | "active": "False", 13 | "name": "John Doe", 14 | "id": "xyz", 15 | }, 16 | ) 17 | 18 | node2 = Node( 19 | id="2", 20 | label="xyz", 21 | attributes={ 22 | "category": [fhirtypes.CodeableConceptType(text="asthma")], 23 | "identifier": [fhirtypes.IdentifierType(value="1")], 24 | "intent": "proposal", 25 | "status": "active", 26 | "subject": fhirtypes.ReferenceType(reference="Patient/123"), 27 | }, 28 | ) 29 | graph.add_nodes([node1, node2], node_types=["Organization", "CarePlan"]) 30 | 31 | text = "User is claustrophobic." 32 | attributes = { 33 | "category": [fhirtypes.CodeableConceptType(text="asthma")], 34 | "identifier": [fhirtypes.IdentifierType(value="1")], 35 | "intent": "proposal", 36 | "status": "active", 37 | "subject": fhirtypes.ReferenceType(reference="Patient/123"), 38 | } 39 | graph.insert(text, attributes, node_type="CarePlan") 40 | -------------------------------------------------------------------------------- /fhir_ontology/docs/visualize_ontology_files/libs/quarto-html/tippy.css: -------------------------------------------------------------------------------- 1 | .tippy-box[data-animation=fade][data-state=hidden]{opacity:0}[data-tippy-root]{max-width:calc(100vw - 10px)}.tippy-box{position:relative;background-color:#333;color:#fff;border-radius:4px;font-size:14px;line-height:1.4;white-space:normal;outline:0;transition-property:transform,visibility,opacity}.tippy-box[data-placement^=top]>.tippy-arrow{bottom:0}.tippy-box[data-placement^=top]>.tippy-arrow:before{bottom:-7px;left:0;border-width:8px 8px 0;border-top-color:initial;transform-origin:center top}.tippy-box[data-placement^=bottom]>.tippy-arrow{top:0}.tippy-box[data-placement^=bottom]>.tippy-arrow:before{top:-7px;left:0;border-width:0 8px 8px;border-bottom-color:initial;transform-origin:center bottom}.tippy-box[data-placement^=left]>.tippy-arrow{right:0}.tippy-box[data-placement^=left]>.tippy-arrow:before{border-width:8px 0 8px 8px;border-left-color:initial;right:-7px;transform-origin:center left}.tippy-box[data-placement^=right]>.tippy-arrow{left:0}.tippy-box[data-placement^=right]>.tippy-arrow:before{left:-7px;border-width:8px 8px 8px 0;border-right-color:initial;transform-origin:center right}.tippy-box[data-inertia][data-state=visible]{transition-timing-function:cubic-bezier(.54,1.5,.38,1.11)}.tippy-arrow{width:16px;height:16px;color:#333}.tippy-arrow:before{content:"";position:absolute;border-color:transparent;border-style:solid}.tippy-content{position:relative;padding:5px 9px;z-index:1} -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "personal-graph" 3 | version = "2.2" 4 | description = "Graph database in LibSQL" 5 | authors = ["Anubhuti Bhardwaj "] 6 | license = "MIT" 7 | readme = "README.md" 8 | exclude = [ 9 | { path = "examples/personal-graph.ipynb" } 10 | ] 11 | 12 | [tool.poetry.dependencies] 13 | python = "^3.11" 14 | graphviz = "^0.20.1" 15 | jinja2 = "^3.1.3" 16 | python-dotenv = "^1.0.1" 17 | sqlite-vss = "^0.1.2" 18 | openai = "^1.14.2" 19 | pytest = "^8.1.1" 20 | pytest-mock = "^3.14.0" 21 | fastapi = "^0.110.0" 22 | uvicorn = "^0.29.0" 23 | sqlean-py = "^3.45.1" 24 | streamlit = "^1.33.0" 25 | types-pygments = "^2.17.0.20240310" 26 | types-decorator = "^5.1.8.20240310" 27 | networkx = {extras = ["default"], version = "^3.3"} 28 | litellm = "^1.35.26" 29 | dspy-ai = "2.3.0" 30 | instructor = "^1.2.2" 31 | vlite = "^0.2.7" 32 | ollama = "^0.2.0" 33 | owlready2 = "^0.46" 34 | fhir-resources = {version = "^7.1.0", optional=true} 35 | plotly = "^5.22.0" 36 | nbformat = "^5.10.4" 37 | nbclient = "^0.10.0" 38 | types-requests = "^2.32.0.20240622" 39 | jsonschema = "^4.23.0" 40 | types-jsonschema = "^4.23.0.20240712" 41 | 42 | [tool.poetry.dev-dependencies] 43 | pytest = "^8.1.1" 44 | ruff = "^0.3.2" 45 | mypy = "^1.9.0" 46 | fhir-resources = "^7.1.0" 47 | libsql-experimental = "^0.0.34" 48 | 49 | [tool.poetry.group.dev.dependencies] 50 | ipykernel = "^6.29.3" 51 | 52 | [tool.poetry.extras] 53 | scrollable-textbox = ["streamlit-scrollable-textbox"] 54 | turso = ["libsql-experimental"] 55 | fhir = ["fhir-resources", "r4"] 56 | 57 | [build-system] 58 | requires = [ 59 | "poetry-core>=1.0.0" 60 | ] 61 | build-backend = "poetry.core.masonry.api" -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name="personal-graph", 5 | version="2.2", 6 | description="Graph database in LibSQL", 7 | author="Anubhuti Bhardwaj", 8 | author_email="anubhutibhardwaj11@gmail.com", 9 | license="MIT", 10 | package_data={"personal_graph": ["py.typed"]}, 11 | packages=["personal_graph"], 12 | install_requires=[ 13 | "graphviz>=0.20.1", 14 | "jinja2>=3.1.3", 15 | "python-dotenv>=1.0.1", 16 | "sqlite-vss>=0.1.2", 17 | "openai>=1.14.2", 18 | "libsql-experimental>=0.0.28", 19 | "fastapi>=0.110.0", 20 | "uvicorn>=0.29.0", 21 | "sqlean-py>=3.45.1", 22 | "streamlit>=1.33.0", 23 | "types-pygments>=2.17.0.20240310", 24 | "types-decorator>=5.1.8.20240310", 25 | "networkx[default]>=3.3", 26 | "litellm>=1.35.26", 27 | "dspy-ai>=2.3.0", 28 | "instructor>=1.2.2", 29 | "vlite>=0.2.7", 30 | "ollama>=0.2.0", 31 | "owlready2>=0.46", 32 | "plotly>=5.22.0", 33 | "nbformat>=5.10.4", 34 | "nbclient>=0.10.0", 35 | "types-requests>=2.32.0.20240622", 36 | "jsonschema>=4.23.0", 37 | "types-jsonschema>=4.23.0.20240712", 38 | ], 39 | extras_require={ 40 | "dev": [ 41 | "pytest>=8.1.1", 42 | "ruff>=0.3.2", 43 | "mypy>=1.9.0", 44 | "fhir-resources>=7.1.0", 45 | "libsql-experimental>=0.0.34", 46 | ], 47 | "scrollable-textbox": ["streamlit-scrollable-textbox"], 48 | "turso": ["libsql-experimental"], 49 | "fhir": ["fhir-resources", "r4"], 50 | }, 51 | python_requires=">=3.11", 52 | ) 53 | -------------------------------------------------------------------------------- /stub/personal_graph/vector_store/vector_store.pyi: -------------------------------------------------------------------------------- 1 | import abc 2 | from abc import ABC, abstractmethod 3 | from typing import Any, Dict, List 4 | 5 | class VectorStore(ABC, metaclass=abc.ABCMeta): 6 | @abstractmethod 7 | def initialize(self): ... 8 | @abstractmethod 9 | def save(self): ... 10 | @abstractmethod 11 | def add_node_embedding(self, id: Any, label: str, attribute: Dict): ... 12 | @abstractmethod 13 | def add_edge_embedding( 14 | self, source: Any, target: Any, label: str, attributes: Dict 15 | ) -> None: ... 16 | @abstractmethod 17 | def add_edge_embeddings( 18 | self, 19 | sources: List[Any], 20 | targets: List[Any], 21 | labels: List[str], 22 | attributes: List[Dict[str, str]], 23 | ): ... 24 | @abstractmethod 25 | def delete_node_embedding(self, id: Any) -> None: ... 26 | @abstractmethod 27 | def delete_edge_embedding(self, ids: Any) -> None: ... 28 | @abstractmethod 29 | def vector_search_node( 30 | self, 31 | data: Dict, 32 | *, 33 | threshold: float, 34 | descending: bool, 35 | limit: int, 36 | sort_by: str, 37 | ): ... 38 | @abstractmethod 39 | def vector_search_edge( 40 | self, 41 | data: Dict, 42 | *, 43 | threshold: float, 44 | descending: bool, 45 | limit: int, 46 | sort_by: str, 47 | ): ... 48 | @abstractmethod 49 | def vector_search_node_from_multi_db( 50 | self, data: Dict, *, threshold: float, limit: int 51 | ): ... 52 | @abstractmethod 53 | def vector_search_edge_from_multi_db( 54 | self, data: Dict, *, threshold: float, limit: int 55 | ): ... 56 | -------------------------------------------------------------------------------- /examples/onto_graph_api.py: -------------------------------------------------------------------------------- 1 | from personal_graph.models import Node 2 | from personal_graph.graph import GraphDB 3 | from personal_graph.ontology import from_rdf 4 | 5 | ontologies1 = from_rdf("/path/to/ontology1") 6 | ontologies2 = from_rdf("/path/to/ontology2") 7 | 8 | 9 | with GraphDB(ontologies=[ontologies1, ontologies2]) as db: 10 | try: 11 | node1 = Node( 12 | id="1", 13 | label="Pabelo", 14 | attributes={ 15 | "entity": "sample entity", 16 | "group": "management", 17 | "medication": "instant", 18 | "individual": "personality", 19 | }, 20 | ) 21 | node2 = Node( 22 | id="2", 23 | label="Praful", 24 | attributes={ 25 | "careprovision": "sample provision", 26 | "diagnostics": "CT scan", 27 | "general": "ward", 28 | "medication": "long time", 29 | }, 30 | ) 31 | db.add_node( 32 | node1, node_type="administrative", delete_if_properties_not_match=True 33 | ) 34 | db.add_nodes( 35 | [node1, node2], 36 | node_types=["administrative", "clinical"], 37 | delete_if_properties_not_match=[True, False], 38 | ) 39 | db.insert( 40 | text="User mentioned their favorite hobbies and weekend activities.", 41 | attributes={ 42 | "careprovision": "sample provision", 43 | "diagnostics": "CT scan", 44 | "general": "ward", 45 | "medication": "long time", 46 | }, 47 | node_type="clinical", 48 | delete_if_properties_not_match=True, 49 | ) 50 | except ValueError as e: 51 | print(f"Error adding node: {e}") 52 | -------------------------------------------------------------------------------- /examples/dspy_program.py: -------------------------------------------------------------------------------- 1 | import os 2 | import dspy # type: ignore 3 | from personal_graph import GraphDB, PersonalRM 4 | from personal_graph.clients import LiteLLMEmbeddingClient 5 | from personal_graph.database import TursoDB 6 | from personal_graph.vector_store import SQLiteVSS 7 | 8 | vector_store = SQLiteVSS( 9 | db=TursoDB( 10 | url=os.getenv("LIBSQL_URL_2"), auth_token=os.getenv("LIBSQL_AUTH_TOKEN_2") 11 | ), 12 | embedding_client=LiteLLMEmbeddingClient(), 13 | index_dimension=384, 14 | ) 15 | 16 | database = TursoDB( 17 | url=os.getenv("LIBSQL_URL"), auth_token=os.getenv("LIBSQL_AUTH_TOKEN") 18 | ) 19 | 20 | graph = GraphDB(vector_store=vector_store, database=database) 21 | turbo = dspy.OpenAI(model="gpt-3.5-turbo", api_key=os.getenv("OPEN_API_KEY")) 22 | retriever = PersonalRM(graph=graph, k=2) 23 | dspy.settings.configure(lm=turbo, rm=retriever) 24 | 25 | 26 | class GenerateAnswer(dspy.Signature): 27 | """Answer questions with short factoid answers.""" 28 | 29 | context = dspy.InputField(desc="may contain relevant facts from user's graph") 30 | question = dspy.InputField() 31 | answer = dspy.OutputField( 32 | desc="a short answer to the question, deduced from the information found in the user's graph" 33 | ) 34 | 35 | 36 | class RAG(dspy.Module): 37 | def __init__(self, depth=3): 38 | super().__init__() 39 | self.retrieve = dspy.Retrieve(k=depth) 40 | self.generate_answer = dspy.ChainOfThought(GenerateAnswer) 41 | 42 | def forward(self, question): 43 | context = self.retrieve(question).passages 44 | prediction = self.generate_answer(context=context, question=question) 45 | return dspy.Prediction(context=context, answer=prediction.answer) 46 | 47 | 48 | rag = RAG(depth=2) 49 | 50 | response = rag("How is Jack related to James?") 51 | print(response.answer) 52 | -------------------------------------------------------------------------------- /stub/personal_graph/vector_store/vlitevss/vlitevss.pyi: -------------------------------------------------------------------------------- 1 | from vlite import VLite # type: ignore 2 | 3 | from personal_graph.vector_store.vector_store import ( 4 | VectorStore as VectorStore, 5 | ) 6 | from typing import Any, Dict, List 7 | 8 | class VliteVSS(VectorStore): 9 | collection: str 10 | model_name: str 11 | def __init__( 12 | self, 13 | *, 14 | collection: str = "./vectors", 15 | model_name: str = "mixedbread-ai/mxbai-embed-large-v1", 16 | ) -> None: ... 17 | vlite: VLite 18 | def initialize(self): ... 19 | def __eq__(self, other): ... 20 | def save(self) -> None: ... 21 | def add_node_embedding(self, id: Any, label: str, attribute: Dict): ... 22 | def add_edge_embedding( 23 | self, source: Any, target: Any, label: str, attributes: Dict 24 | ) -> None: ... 25 | def add_edge_embeddings( 26 | self, 27 | sources: List[Any], 28 | targets: List[Any], 29 | labels: List[str], 30 | attributes: List[Dict[str, str]], 31 | ): ... 32 | def delete_node_embedding(self, ids: Any) -> None: ... 33 | def delete_edge_embedding(self, ids: Any) -> None: ... 34 | def vector_search_node( 35 | self, 36 | data: Dict, 37 | *, 38 | threshold: float | None = None, 39 | descending: bool, 40 | limit: int, 41 | sort_by: str, 42 | ): ... 43 | def vector_search_edge( 44 | self, 45 | data: Dict, 46 | *, 47 | threshold: float | None = None, 48 | descending: bool, 49 | limit: int, 50 | sort_by: str, 51 | ): ... 52 | def vector_search_node_from_multi_db( 53 | self, data: Dict, *, threshold: float | None = None, limit: int = 1 54 | ): ... 55 | def vector_search_edge_from_multi_db( 56 | self, data: Dict, *, threshold: float | None = None, limit: int = 1 57 | ): ... 58 | -------------------------------------------------------------------------------- /examples/nl_interface.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python 2 | import os 3 | import logging 4 | import argparse 5 | from dotenv import load_dotenv 6 | 7 | from personal_graph import GraphDB 8 | from personal_graph.text import text_to_graph 9 | 10 | 11 | def main(args): 12 | with GraphDB() as graph: 13 | # Testing insert query into graph db 14 | nl_query = "increased thirst, weight loss, increased hunger, frequent urination etc. are all symptoms of diabetes." 15 | kg = text_to_graph(text=nl_query) 16 | graph.insert_graph(kg) 17 | 18 | logging.info("Nodes in the Knowledge Graph: \n") 19 | for node in kg.nodes: 20 | logging.info( 21 | f"ID: {node.id}, Label: {node.label}, Attribute: {node.attributes}" 22 | ) 23 | 24 | logging.info("Edges in the Knowledge Graph: \n") 25 | for edge in kg.edges: 26 | logging.info( 27 | f"Source: {edge.source}, Target: {edge.target}, Label: {edge.label}, Attribute: {edge.attributes}" 28 | ) 29 | 30 | # Testing search query from graph db 31 | search_query = "I am losing my weight too frequent." 32 | knowledge_graph = graph.search_from_graph(search_query) 33 | 34 | logging.info(f"Knowledge Graph: \n{knowledge_graph}") 35 | 36 | 37 | if __name__ == "__main__": 38 | load_dotenv() 39 | 40 | logging.basicConfig( 41 | level=logging.DEBUG, 42 | filemode="w", 43 | filename="./log/nl_interface.log", 44 | ) 45 | 46 | parser = argparse.ArgumentParser( 47 | description="Shows simple example of natural language apis." 48 | ) 49 | parser.add_argument("--db-url", default=os.getenv("LIBSQL_URL"), type=str) 50 | parser.add_argument( 51 | "--db-auth-token", default=os.getenv("LIBSQL_AUTH_TOKEN"), type=str 52 | ) 53 | 54 | arguments = parser.parse_args() 55 | 56 | main(arguments) 57 | -------------------------------------------------------------------------------- /fhir_ontology/fhir_ontology_json.py: -------------------------------------------------------------------------------- 1 | """ 2 | This method will create a json file with all the nodes and edges that the fhir ontology has. 3 | """ 4 | 5 | import json 6 | import os 7 | import pickle 8 | import logging 9 | from personal_graph.helper import extract_classes_properties, get_type_name 10 | 11 | 12 | def create_fhir_ontology_json(json_filename, pickle_filename): 13 | try: 14 | # Check if the pickle file exists 15 | if os.path.exists(pickle_filename): 16 | logging.info("Loading FHIR node data from pickle file...") 17 | with open(pickle_filename, "rb") as f: 18 | node_types_info = pickle.load(f) 19 | else: 20 | node_types_info = extract_classes_properties() 21 | 22 | ontology_data = {"nodes": [], "edges": []} 23 | 24 | # Add all node_types 25 | for node_type in node_types_info.keys(): 26 | ontology_data["nodes"].append({"id": node_type, "label": node_type}) 27 | 28 | # Create edges between node_types with other related node_types 29 | for node_type, properties in node_types_info.items(): 30 | for prop, prop_type in properties.items(): 31 | type_name = get_type_name(prop_type) 32 | if type_name in node_types_info.keys(): 33 | # This property is a FHIR resource type, create an edge of 'instance_of' 34 | ontology_data["edges"].append( 35 | {"source": node_type, "target": type_name, "label": prop} 36 | ) 37 | 38 | # Save the ontology data as JSON 39 | with open(json_filename, "w") as f: 40 | json.dump(ontology_data, f, indent=2) 41 | 42 | logging.info(f"FHIR ontology data saved to {json_filename}") 43 | return json_filename 44 | 45 | except Exception as e: 46 | logging.info(f"An error occurred: {e}") 47 | raise e 48 | -------------------------------------------------------------------------------- /personal_graph/models.py: -------------------------------------------------------------------------------- 1 | """ 2 | Pydantic models for pythonic API to the graph db 3 | """ 4 | 5 | from pydantic import BaseModel, Field 6 | from typing import List, Union, Dict, Any 7 | 8 | 9 | class Node(BaseModel): 10 | id: Union[int, str] = Field(..., description="Unique Identifier for the node.") 11 | attributes: Union[str, Dict[Any, Any]] = Field( 12 | ..., description="Detailed information associated with the node." 13 | ) 14 | label: str = Field( 15 | ..., 16 | min_length=1, 17 | description="Most related and unique name associated with the node.", 18 | ) 19 | 20 | 21 | class Edge(BaseModel): 22 | source: Union[int, str] = Field( 23 | ..., description="identifier of the source node from which the edge originates." 24 | ) 25 | target: Union[int, str] = Field( 26 | ..., description="identifier of the target node to which the edge points." 27 | ) 28 | label: str = Field( 29 | ..., 30 | min_length=1, 31 | description="Most related and unique name associated with the edge.", 32 | ) 33 | attributes: Union[str, Dict[Any, Any]] = Field( 34 | ..., 35 | description="Detailed information associated with the relationships.", 36 | ) 37 | 38 | 39 | class EdgeInput(BaseModel): 40 | source: Node = Field(..., description="Source node from which the edge originates.") 41 | target: Node = Field(..., description="Target node to which the edge points.") 42 | label: str = Field( 43 | ..., 44 | min_length=1, 45 | description="Most related and unique name associated with the edge.", 46 | ) 47 | attributes: Union[str, Dict[Any, Any]] = Field( 48 | ..., 49 | description="Detailed information associated within the relationships.", 50 | ) 51 | 52 | 53 | class KnowledgeGraph(BaseModel): 54 | nodes: List[Node] = Field(..., default_factory=list) 55 | edges: List[Edge] = Field(..., default_factory=list) 56 | -------------------------------------------------------------------------------- /examples/ontology_example.py: -------------------------------------------------------------------------------- 1 | import fhir.resources as fhir # type: ignore 2 | from fhir.resources import fhirtypes 3 | 4 | from personal_graph import GraphDB, Node 5 | from personal_graph.ontology import from_rdf 6 | 7 | 8 | biography_ontology = from_rdf("./biography_schema.rdf") 9 | with GraphDB(ontologies=[fhir, biography_ontology]) as graph: 10 | node1 = Node( 11 | id="1", 12 | label="Pabelo", 13 | attributes={ 14 | "active": "False", 15 | "name": "John Doe", 16 | "id": "xyz", 17 | }, 18 | ) 19 | node2 = Node( 20 | id="1", 21 | label="Pabelo", 22 | attributes={ 23 | "entity": "sample entity", 24 | "device": "mobile", 25 | "group": "management", 26 | "medication": "instant", 27 | "individual": "personality", 28 | }, 29 | ) 30 | node3 = Node( 31 | id="3", label="close relative", attributes={"name": "Alice", "age": "30"} 32 | ) 33 | node4 = Node( 34 | id="2", 35 | label="xyz", 36 | attributes={ 37 | "category": [fhirtypes.CodeableConceptType(text="asthma")], 38 | "identifier": [fhirtypes.IdentifierType(value="1")], 39 | "intent": "proposal", 40 | "status": "active", 41 | "subject": fhirtypes.ReferenceType(reference="Patient/123"), 42 | }, 43 | ) 44 | node5 = Node( 45 | id="2", 46 | label="pop", 47 | attributes={ 48 | "participant": "True", 49 | "interval": "John Doe", 50 | }, 51 | ) 52 | 53 | # Test w5_ontology which is loaded from rdf file 54 | graph.add_node(node5, node_type="Relationship") 55 | 56 | # Test fhir ontology 57 | graph.add_node(node2, node_type="Organization") 58 | 59 | # Add node without ontology 60 | graph.add_node(node3) 61 | 62 | graph.add_nodes([node1, node4], node_types=["Organization", "CarePlan"]) 63 | -------------------------------------------------------------------------------- /.github/workflows/personal-graph.yaml: -------------------------------------------------------------------------------- 1 | name: Checks and Test 2 | 3 | on: 4 | pull_request: 5 | branches: [main] 6 | paths: ['personal_graph/**'] 7 | 8 | jobs: 9 | check: 10 | runs-on: ubuntu-latest # host's operating system 11 | steps: # each job consists of 1+ steps 12 | - name: Checkout commit # download the code from triggering commit 13 | uses: actions/checkout@v4 14 | 15 | - name: Set up Python and Poetry 16 | uses: actions/setup-python@v4 17 | with: 18 | python-version: '3.x' 19 | 20 | - name: Install Poetry 21 | uses: snok/install-poetry@v1 22 | with: 23 | virtualenvs-create: true 24 | virtualenvs-in-project: true 25 | installer-parallel: true 26 | 27 | - name: Install dependencies 28 | run: poetry install --no-interaction --no-root 29 | 30 | - name: Format with Ruff 31 | run: poetry run ruff format --check . 32 | 33 | - name: Lint with Ruff 34 | run: poetry run ruff check -- . 35 | 36 | - name: Type check 37 | run: poetry run python -m mypy . 38 | 39 | test: 40 | runs-on: ubuntu-latest 41 | needs: check 42 | steps: # each job consists of 1+ steps 43 | - name: Checkout commit # download the code from triggering commit 44 | uses: actions/checkout@v4 45 | 46 | - name: Set up Python and Poetry 47 | uses: actions/setup-python@v4 48 | with: 49 | python-version: '3.x' 50 | 51 | - name: Install Poetry 52 | uses: snok/install-poetry@v1 53 | with: 54 | virtualenvs-create: true 55 | virtualenvs-in-project: true 56 | installer-parallel: true 57 | 58 | - name: Install dependencies 59 | run: | 60 | cd personal_graph 61 | poetry install --no-interaction --no-root --with dev 62 | 63 | - name: Test with pytest 64 | run: | 65 | cd tests 66 | poetry run pytest -vvv 67 | -------------------------------------------------------------------------------- /stub/personal_graph/vector_store/sqlitevss/sqlitevss.pyi: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from personal_graph.clients import LiteLLMEmbeddingClient as LiteLLMEmbeddingClient 3 | from personal_graph.embeddings import OpenAIEmbeddingsModel as OpenAIEmbeddingsModel 4 | from personal_graph.database import SQLite as SQLite 5 | from personal_graph.database import TursoDB as TursoDB 6 | from personal_graph.vector_store import ( 7 | VectorStore as VectorStore, 8 | ) 9 | from typing import Any, Dict, Union 10 | 11 | def read_sql(sql_file: Path) -> str: ... 12 | 13 | class SQLiteVSS(VectorStore): 14 | db: Union[TursoDB, SQLite] 15 | embedding_model: LiteLLMEmbeddingClient 16 | def __init__( 17 | self, *, db: TursoDB | SQLite, embedding_client: LiteLLMEmbeddingClient = ... 18 | ) -> None: ... 19 | def initialize(self): ... 20 | def save(self): ... 21 | def add_node_embedding( 22 | self, id: Any, label: str, attribute: Dict[Any, Any] 23 | ) -> None: ... 24 | def add_edge_embedding( 25 | self, source: Any, target: Any, label: str, attributes: Dict 26 | ) -> None: ... 27 | def add_edge_embeddings(self, sources, targets, labels, attributes) -> None: ... 28 | def delete_node_embedding(self, id: Any) -> None: ... 29 | def delete_edge_embedding(self, ids: Any) -> None: ... 30 | def vector_search_node( 31 | self, 32 | data: Dict, 33 | *, 34 | threshold: float = 0.9, 35 | descending: bool = False, 36 | limit: int = 1, 37 | sort_by: str = "", 38 | ): ... 39 | def vector_search_edge( 40 | self, 41 | data: Dict, 42 | *, 43 | threshold: float = 0.9, 44 | descending: bool = False, 45 | limit: int = 1, 46 | sort_by: str = "", 47 | ): ... 48 | def vector_search_node_from_multi_db( 49 | self, data: Dict, *, threshold: float = 0.9, limit: int = 1 50 | ): ... 51 | def vector_search_edge_from_multi_db( 52 | self, data: Dict, *, threshold: float = 0.9, limit: int = 1 53 | ): ... 54 | -------------------------------------------------------------------------------- /personal_graph/embeddings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Provide access to different embeddings models 3 | """ 4 | 5 | from abc import ABC, abstractmethod 6 | 7 | import ollama # type: ignore 8 | import openai 9 | 10 | 11 | class EmbeddingsModel(ABC): 12 | @abstractmethod 13 | def get_embedding(self, text: str) -> list[float]: 14 | pass 15 | 16 | 17 | class OpenAIEmbeddingsModel(EmbeddingsModel): 18 | def __init__( 19 | self, embed_client: openai.OpenAI, embed_model: str, embed_dimension: int = 384 20 | ) -> None: 21 | self.client = embed_client if embed_client else None 22 | self.model = embed_model 23 | self.dimension = embed_dimension 24 | 25 | def __repr__(self) -> str: 26 | return ( 27 | f"OpenAIEmbeddingsModel(\n" 28 | f" embed_client={self.client},\n" 29 | f" embed_model='{self.model}',\n" 30 | f" embed_dimension={self.dimension}\n" 31 | f" )" 32 | ) 33 | 34 | def get_embedding(self, text: str) -> list[float]: 35 | if self.client is None: 36 | return [] 37 | text = text.replace("\n", " ") 38 | return ( 39 | self.client.embeddings.create( 40 | input=[text], 41 | model=self.model, 42 | dimensions=self.dimension, 43 | encoding_format="float", 44 | ) 45 | .data[0] 46 | .embedding 47 | ) 48 | 49 | 50 | class OllamaEmbeddingModel(EmbeddingsModel): 51 | def __init__( 52 | self, embed_client: ollama.Client, embed_model: str, embed_dimension: int = 768 53 | ) -> None: 54 | self.client = embed_client if embed_client else None 55 | self.model = embed_model 56 | self.dimension = embed_dimension 57 | 58 | def __repr__(self) -> str: 59 | return ( 60 | f"OllamaEmbeddingModel(\n" 61 | f" embed_client={self.client},\n" 62 | f" embed_model='{self.model}',\n" 63 | f" embed_dimension={self.dimension}\n" 64 | f" )" 65 | ) 66 | 67 | def get_embedding(self, text: str) -> list[float]: 68 | if self.client is None: 69 | return [] 70 | 71 | return ollama.embeddings( 72 | model=self.model, 73 | prompt=text, 74 | )["embedding"] 75 | -------------------------------------------------------------------------------- /personal_graph/vector_store/vector_store.py: -------------------------------------------------------------------------------- 1 | """ 2 | Provide access to different vector databases 3 | """ 4 | 5 | from abc import ABC, abstractmethod 6 | from typing import Any, Dict, List, Union 7 | 8 | 9 | class VectorStore(ABC): 10 | @abstractmethod 11 | def initialize(self): 12 | """Initialize the vector store.""" 13 | pass 14 | 15 | @abstractmethod 16 | def save(self): 17 | """Save the embeddings into the vector store""" 18 | pass 19 | 20 | @abstractmethod 21 | def add_node_embedding(self, id: Any, label: str, attribute: Dict): 22 | """Add a single node embedding to the database.""" 23 | pass 24 | 25 | @abstractmethod 26 | def add_edge_embedding( 27 | self, source: Any, target: Any, label: str, attributes: Dict 28 | ) -> None: 29 | """Add a single edge embedding to the database.""" 30 | pass 31 | 32 | @abstractmethod 33 | def add_edge_embeddings( 34 | self, 35 | sources: List[Any], 36 | targets: List[Any], 37 | labels: List[str], 38 | attributes: List[Union[Dict[str, str]]], 39 | ): 40 | """Add edges embeddings to the vector store""" 41 | pass 42 | 43 | @abstractmethod 44 | def delete_node_embedding(self, id: Any) -> None: 45 | """Remove a single node embedding from the database.""" 46 | pass 47 | 48 | @abstractmethod 49 | def delete_edge_embedding(self, ids: Any) -> None: 50 | """Remove multiple nodes embedding from the database.""" 51 | pass 52 | 53 | @abstractmethod 54 | def vector_search_node( 55 | self, 56 | data: Dict, 57 | *, 58 | threshold: float, 59 | descending: bool, 60 | limit: int, 61 | sort_by: str, 62 | ): 63 | """Perform a vector search for nodes.""" 64 | pass 65 | 66 | @abstractmethod 67 | def vector_search_edge( 68 | self, 69 | data: Dict, 70 | *, 71 | threshold: float, 72 | descending: bool, 73 | limit: int, 74 | sort_by: str, 75 | ): 76 | """Perform a vector search for edges.""" 77 | pass 78 | 79 | @abstractmethod 80 | def vector_search_node_from_multi_db( 81 | self, data: Dict, *, threshold: float, limit: int 82 | ): 83 | """Perform a vector search for nodes across multiple databases""" 84 | pass 85 | 86 | @abstractmethod 87 | def vector_search_edge_from_multi_db( 88 | self, data: Dict, *, threshold: float, limit: int 89 | ): 90 | """Perform a vector search for edges across multiple databases""" 91 | pass 92 | -------------------------------------------------------------------------------- /personal_graph/database/tursodb/turso.py: -------------------------------------------------------------------------------- 1 | from functools import lru_cache 2 | from pathlib import Path 3 | 4 | from typing import Optional, Any, Callable, Tuple 5 | 6 | from jinja2 import BaseLoader, Environment, select_autoescape 7 | from personal_graph.database.db import CursorExecFunction 8 | from personal_graph.database.sqlite.sqlite import SQLite 9 | 10 | try: 11 | import libsql_experimental as libsql # type: ignore 12 | except ImportError: 13 | pass 14 | 15 | 16 | @lru_cache(maxsize=None) 17 | def read_sql(sql_file: Path) -> str: 18 | with open(Path(__file__).parent.resolve() / "raw-queries" / sql_file) as f: 19 | return f.read() 20 | 21 | 22 | class SqlTemplateLoader(BaseLoader): 23 | def __init__(self, templates_dir: Path): 24 | self.templates_dir = templates_dir 25 | 26 | def get_source( 27 | self, environment: Environment, template: str 28 | ) -> Tuple[str, str, Callable[[], bool]]: 29 | def uptodate() -> bool: 30 | return True 31 | 32 | template_path = self.templates_dir / template 33 | 34 | # Return the source code, the template name, and the uptodate function 35 | return read_sql(template_path), template, uptodate 36 | 37 | 38 | class TursoDB(SQLite): 39 | def __init__(self, *, url: Optional[str] = None, auth_token: Optional[str] = None): 40 | self.db_url = url 41 | self.db_auth_token = auth_token 42 | 43 | self.env = Environment( 44 | loader=SqlTemplateLoader(Path(__file__).parent / "raw-queries"), 45 | autoescape=select_autoescape(), 46 | ) 47 | self.clause_template = self.env.get_template("search-where.template") 48 | self.search_template = self.env.get_template("search-node.template") 49 | self.traverse_template = self.env.get_template("traverse.template") 50 | 51 | def __eq__(self, other): 52 | if hasattr(other, "db_url"): 53 | return self.db_url == other.db_url 54 | return False 55 | 56 | def __repr__(self): 57 | return ( 58 | f"TursoDB(\n" 59 | f" db_url={self.db_url},\n" 60 | f" db_auth_token='{self.db_auth_token}'\n" 61 | f" )," 62 | ) 63 | 64 | def atomic(self, cursor_exec_fn: CursorExecFunction) -> Any: 65 | self._connection = libsql.connect( 66 | database=self.db_url, 67 | auth_token=self.db_auth_token, 68 | ) 69 | 70 | cursor = self._connection.cursor() 71 | cursor.execute("PRAGMA foreign_keys = TRUE;") 72 | results = cursor_exec_fn(cursor, self._connection) 73 | self._connection.commit() 74 | return results 75 | 76 | def save(self): 77 | self._connection.commit() 78 | -------------------------------------------------------------------------------- /examples/fhirgraphdb.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | import os 4 | import fhir.resources as fhir 5 | from fhir.resources.patient import Patient 6 | 7 | from personal_graph import GraphDB, EdgeInput 8 | from personal_graph.database import FhirDB, TursoDB 9 | from personal_graph.helper import fhir_node 10 | from personal_graph.vector_store.sqlitevss.fhirsqlitevss import FhirSQLiteVSS 11 | 12 | 13 | def main(args): 14 | vs = FhirSQLiteVSS( 15 | db=TursoDB(url=args.db_url, auth_token=args.auth_token), 16 | index_dimension=384, 17 | ) 18 | 19 | FhirGraphDB = GraphDB( 20 | vector_store=vs, 21 | database=FhirDB(db_url=args.db_url), 22 | ontologies=[fhir], 23 | ) 24 | 25 | pd_1 = { 26 | "resourceType": "Patient", 27 | "id": "2291", 28 | "name": [{"family": "Jimmy", "given": ["Room"]}], 29 | "gender": "female", 30 | "birthDate": "2000-10-02", 31 | } 32 | pd_2 = { 33 | "resourceType": "Patient", 34 | "id": "2291", 35 | "name": [{"family": "Doe", "given": ["John"]}], 36 | "gender": "male", 37 | "birthDate": "1970-01-01", 38 | } 39 | 40 | # Convert to node type object 41 | node1 = fhir_node(Patient(**pd_1)) 42 | node2 = fhir_node(Patient(**pd_2)) 43 | 44 | FhirGraphDB.add_node(node1) 45 | 46 | # Searching a node 47 | logging.info(FhirGraphDB.search_node(2290, node_type="Patient")) 48 | 49 | # Adding edges 50 | edge1 = EdgeInput( 51 | source=node1, target=node2, label="knows", attributes={"since": "2012"} 52 | ) 53 | edge2 = EdgeInput( 54 | source=node2, target=node1, label="knows", attributes={"since": "2012"} 55 | ) 56 | FhirGraphDB.add_edge(edge1) 57 | FhirGraphDB.add_edges([edge1, edge2]) 58 | 59 | # Updating nodes 60 | FhirGraphDB.update_node(node2) 61 | 62 | # Remove nodes 63 | FhirGraphDB.remove_node(2290, node_type="Patient") 64 | FhirGraphDB.remove_nodes([2290, 2291], node_types=["patient", "Patient"]) 65 | 66 | # Fetch ids from db 67 | logging.info(FhirGraphDB.fetch_ids_from_db(node_type="patient")) 68 | 69 | 70 | if __name__ == "__main__": 71 | logging.basicConfig( 72 | level=logging.DEBUG, 73 | filemode="w", 74 | filename="./log/fhirgraphdb.log", 75 | ) 76 | 77 | parser = argparse.ArgumentParser( 78 | description="Shows simple example of high level apis." 79 | ) 80 | 81 | parser.add_argument("--db-url", default=os.getenv("DB_URL", ""), type=str) 82 | parser.add_argument( 83 | "--auth-token", 84 | default=os.getenv("TURSO_PATIENTS_GROUP_AUTH_TOKEN", 0), 85 | type=str, 86 | ) 87 | 88 | arguments = parser.parse_args() 89 | main(arguments) 90 | -------------------------------------------------------------------------------- /stub/personal_graph/database/db.pyi: -------------------------------------------------------------------------------- 1 | import abc 2 | from abc import ABC, abstractmethod 3 | from graphviz import Digraph # type: ignore 4 | from personal_graph.models import Edge as Edge, Node as Node 5 | from typing import Any, Dict, List 6 | from personal_graph.database.db import CursorExecFunction 7 | 8 | class DB(ABC, metaclass=abc.ABCMeta): 9 | @abstractmethod 10 | def initialize(self): ... 11 | @abstractmethod 12 | def __eq__(self, other): ... 13 | @abstractmethod 14 | def save(self): ... 15 | @abstractmethod 16 | def fetch_node_embed_id(self, node_id: Any): ... 17 | @abstractmethod 18 | def fetch_edge_embed_ids(self, id: Any): ... 19 | @abstractmethod 20 | def all_connected_nodes(self, node_or_edge: Node | Edge) -> Any: ... 21 | @abstractmethod 22 | def get_connections(self, identifier: Any) -> CursorExecFunction: ... 23 | @abstractmethod 24 | def search_edge(self, source: Any, target: Any, attributes: Dict): ... 25 | @abstractmethod 26 | def add_node(self, label: str, attribute: Dict, id: Any): ... 27 | @abstractmethod 28 | def add_edge( 29 | self, source: Any, target: Any, label: str, attributes: Dict 30 | ) -> None: ... 31 | @abstractmethod 32 | def update_node(self, node: Node): ... 33 | @abstractmethod 34 | def remove_node(self, id: Any) -> None: ... 35 | @abstractmethod 36 | def search_node(self, node_id: Any) -> Any: ... 37 | @abstractmethod 38 | def search_node_label(self, node_id: Any) -> Any: ... 39 | @abstractmethod 40 | def traverse( 41 | self, source: Any, target: Any | None = None, with_bodies: bool = False 42 | ) -> List: ... 43 | @abstractmethod 44 | def fetch_node_id(self, id: Any): ... 45 | @abstractmethod 46 | def find_nodes_by_label(self, label: str): ... 47 | @abstractmethod 48 | def graphviz_visualize( 49 | self, 50 | dot_file: str | None = None, 51 | path: List[Any] = [], 52 | connections: Any = None, 53 | format: str = "png", 54 | exclude_node_keys: List[str] = [], 55 | hide_node_key: bool = False, 56 | node_kv: str = " ", 57 | exclude_edge_keys: List[str] = [], 58 | hide_edge_key: bool = False, 59 | edge_kv: str = " ", 60 | ) -> Digraph: ... 61 | @abstractmethod 62 | def fetch_ids_from_db(self) -> List[str]: ... 63 | @abstractmethod 64 | def search_indegree_edges(self, target: Any) -> List[Any]: ... 65 | @abstractmethod 66 | def search_outdegree_edges(self, source: Any) -> List[Any]: ... 67 | @abstractmethod 68 | def search_similar_nodes(self, embed_id, *, desc: bool, sort_by: str): ... 69 | @abstractmethod 70 | def search_similar_edges(self, embed_id, *, desc: bool, sort_by: str): ... 71 | -------------------------------------------------------------------------------- /personal_graph/visualizers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | visualizers.py 5 | 6 | Functions to enable visualizations of graph data, starting with graphviz, 7 | and extensible to other libraries. 8 | 9 | """ 10 | 11 | import json 12 | from graphviz import Digraph # type: ignore 13 | from typing import List, Dict, Any, Tuple 14 | 15 | from personal_graph.models import KnowledgeGraph 16 | 17 | 18 | def _as_dot_label( 19 | body: Dict[str, Any], 20 | exclude_keys: List[str], 21 | hide_key_name: bool, 22 | kv_separator: str, 23 | ) -> str: 24 | keys = [k for k in body.keys() if k not in exclude_keys] 25 | fstring = ( 26 | "\\n".join(["{" + k + "}" for k in keys]) 27 | if hide_key_name 28 | else "\\n".join([k + kv_separator + "{" + k + "}" for k in keys]) 29 | ) 30 | return fstring.format(**body) 31 | 32 | 33 | def _as_dot_node( 34 | body: Dict[str, Any], 35 | exclude_keys: List[str] = [], 36 | hide_key_name: bool = False, 37 | kv_separator: str = " ", 38 | ) -> Tuple[str, str]: 39 | name = body["id"] 40 | exclude_keys.append("id") 41 | label = _as_dot_label(body, exclude_keys, hide_key_name, kv_separator) 42 | return str(name), label 43 | 44 | 45 | def graphviz_visualize_bodies( 46 | dot_file: str, 47 | path: List[Tuple[Any, str, str]] = [], 48 | format: str = "png", 49 | exclude_node_keys: List[str] = [], 50 | hide_node_key: bool = False, 51 | node_kv: str = " ", 52 | exclude_edge_keys: List[str] = [], 53 | hide_edge_key: bool = False, 54 | edge_kv: str = " ", 55 | ) -> None: 56 | dot = Digraph() 57 | current_id = None 58 | edges = [] 59 | for identifier, obj, properties in path: 60 | body = json.loads(properties) 61 | if obj == "()": 62 | name, label = _as_dot_node(body, exclude_node_keys, hide_node_key, node_kv) 63 | dot.node(name, label=label) 64 | current_id = body["id"] 65 | else: 66 | edge = ( 67 | (str(current_id), str(identifier), body) 68 | if obj == "->" 69 | else (str(identifier), str(current_id), body) 70 | ) 71 | if edge not in edges: 72 | dot.edge( 73 | edge[0], 74 | edge[1], 75 | label=_as_dot_label(body, exclude_edge_keys, hide_edge_key, edge_kv) 76 | if body 77 | else None, 78 | ) 79 | edges.append(edge) 80 | dot.render(dot_file, format=format) 81 | 82 | 83 | def visualize_graph(kg: KnowledgeGraph) -> Digraph: 84 | dot = Digraph(comment="Knowledge Graph") 85 | 86 | # Add nodes 87 | for node in kg.nodes: 88 | dot.node(str(node.id), node.label, color="black") 89 | 90 | # Add edges 91 | for edge in kg.edges: 92 | dot.edge(str(edge.source), str(edge.target), edge.label, color="black") 93 | 94 | return dot 95 | -------------------------------------------------------------------------------- /personal_graph/helper.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | import pkgutil 5 | from datetime import date 6 | 7 | from pydantic_core import ValidationError 8 | import inspect 9 | from importlib import import_module 10 | from typing import get_origin, get_args 11 | 12 | from personal_graph.models import Node 13 | 14 | try: 15 | import fhir.resources as fhir # type: ignore 16 | except ImportError: 17 | logging.info("fhir module is not available.") 18 | 19 | 20 | def validate_fhir_resource(resource_type, resource_data): 21 | """Validate if the provided data conforms to a FHIR resource schema.""" 22 | try: 23 | if resource_type: 24 | # Resource data must be a json_dict 25 | fhir.construct_fhir_element(resource_type, resource_data) 26 | return True 27 | 28 | except ValidationError as e: 29 | logging.info(f"Validation error: {e}") 30 | 31 | except Exception as e: 32 | logging.info(f"Error creating resource: {e}") 33 | 34 | return False 35 | 36 | 37 | def extract_classes_properties(): 38 | """ 39 | This method will fetch all the classes along with their properties and saves it in a dictionary 40 | @return: class_info: Dict 41 | """ 42 | pkgpath = os.path.dirname(fhir.__file__) 43 | 44 | class_info = {} 45 | for _, module_name, is_dir in pkgutil.iter_modules([pkgpath]): 46 | if not is_dir: 47 | try: 48 | module = import_module(f"fhir.resources.{module_name}") 49 | for name, obj in inspect.getmembers(module): 50 | if inspect.isclass(obj) and not obj.__module__.startswith( 51 | "pydantic" 52 | ): 53 | class_properties = ( 54 | {k: v for k, v in obj.__annotations__.items()} 55 | if hasattr(obj, "__annotations__") 56 | else {} 57 | ) 58 | class_info[name] = class_properties 59 | 60 | except ModuleNotFoundError: 61 | pass 62 | 63 | return class_info 64 | 65 | 66 | def get_type_name(prop_type): 67 | """ 68 | Helper function to get the name of the type, handling List types 69 | """ 70 | if get_origin(prop_type) is list: 71 | return get_args(prop_type)[0].__name__ 72 | return prop_type.__name__ 73 | 74 | 75 | def json_serializable(obj): 76 | if isinstance(obj, date): 77 | return obj.isoformat() 78 | raise TypeError(f"Type {type(obj)} not serializable") 79 | 80 | 81 | def fhir_node(fhir_data) -> Node: 82 | """ 83 | Helper function that will convert fhir data to Node Type 84 | @param fhir_data: Dict 85 | @return: Node 86 | """ 87 | 88 | return Node( 89 | id=fhir_data.id, 90 | attributes=json.loads( 91 | json.dumps(fhir_data.dict(exclude_unset=True), default=json_serializable) 92 | ), 93 | label=type(fhir_data).__name__, 94 | ) 95 | -------------------------------------------------------------------------------- /scripts/time_complexity.py: -------------------------------------------------------------------------------- 1 | # import sys 2 | # import os 3 | # import time 4 | # import logging 5 | # from dotenv import load_dotenv 6 | # 7 | # load_dotenv() 8 | # 9 | # script_dir = os.path.dirname(os.path.realpath(__file__)) 10 | # parent_dir = os.path.abspath(os.path.join(script_dir, os.pardir)) 11 | # sys.path.append(parent_dir) 12 | # 13 | # # from personal_graph import database as db # noqa: E402 14 | # # from personal_graph import natural as nt # noqa: E402 15 | # 16 | # 17 | # def measure_execution_time(operation_func, *args, **kwargs): 18 | # start_time = time.time() 19 | # res = operation_func(*args, **kwargs) 20 | # end_time = time.time() 21 | # time_taken = end_time - start_time 22 | # return res, time_taken 23 | # 24 | # 25 | # def time_complexity(): 26 | # """ 27 | # Main Function to calculate the time duration of each operation. 28 | # """ 29 | # 30 | # operations = [ 31 | # (db.find_node, "Student"), 32 | # (db.find_nodes, [db._generate_clause("subject", predicate="LIKE")], ("Civil%")), 33 | # (db.add_node, {"subject": "CSE", "type": ["person", "PHD"]}, 3), 34 | # ( 35 | # db.add_nodes, 36 | # [ 37 | # {"name": "Stanley", "age": "20"}, 38 | # {"name": "Jenny", "age": "55"}, 39 | # ], 40 | # [1, 2], 41 | # ["Student", "Principal"], 42 | # ), 43 | # (db.upsert_node, 1, "Student", {"name": "Bob", "age": "23"}), 44 | # ( 45 | # db.upsert_nodes, 46 | # [ 47 | # {"name": "Stanley", "age": "30"}, 48 | # {"name": "James", "age": "35"}, 49 | # ], 50 | # ["Professor", "Doctor"], 51 | # [1, 2], 52 | # ), 53 | # (db.connect_nodes, 2, 1, "teaches", {"subject": "CS"}), 54 | # ( 55 | # db.connect_many_nodes, 56 | # [2, 3], 57 | # [1, 3], 58 | # ["Has", "is"], 59 | # [{"disease": "Diabetes"}, {"patient": "Diabetes Symptoms"}], 60 | # ), 61 | # (db.remove_node, 1), 62 | # (db.remove_nodes, [1, 2]), 63 | # ( 64 | # db.traverse, 65 | # os.getenv("LIBSQL_URL"), 66 | # os.getenv("LIBSQL_AUTH_TOKEN"), 67 | # 2, 68 | # ), 69 | # (db.vector_search_node, {"body": "Jack has diabetic symptoms"}, 0.9), 70 | # (db.vector_search_edge, {"relation": "Has"}), 71 | # (db.find_similar_nodes, "relative", 0.9), 72 | # (db.pruning, 0.9), 73 | # (nt.insert_into_graph, "I am feeling quite dizzy."), 74 | # (nt.search_from_graph, "What precautions to take in dizziness?"), 75 | # ] 76 | # 77 | # results_dict = {} 78 | # 79 | # for operation, *args in operations: 80 | # try: 81 | # result, duration = measure_execution_time(operation, *args) 82 | # operation_name = operation.__name__ 83 | # results_dict[operation_name] = duration 84 | # except Exception as err: 85 | # logging.error(f"Exception: {err}") 86 | # 87 | # logging.info(results_dict) 88 | # 89 | # 90 | # if __name__ == "__main__": 91 | # logging.basicConfig( 92 | # filename="./log/operations.log", level=logging.DEBUG, filemode="w" 93 | # ) 94 | # 95 | # time_complexity() 96 | -------------------------------------------------------------------------------- /stub/personal_graph/graph.pyi: -------------------------------------------------------------------------------- 1 | import types 2 | from _typeshed import Incomplete 3 | from contextlib import AbstractContextManager 4 | from graphviz import Digraph # type: ignore 5 | from owlready2 import Ontology # type: ignore 6 | 7 | from personal_graph.graph_generator import ( 8 | OpenAITextToGraphParser as OpenAITextToGraphParser, 9 | ) 10 | from personal_graph.models import ( 11 | Edge as Edge, 12 | EdgeInput as EdgeInput, 13 | KnowledgeGraph as KnowledgeGraph, 14 | Node as Node, 15 | ) 16 | from personal_graph.database import ( 17 | SQLite as SQLite, 18 | TursoDB as TursoDB, 19 | ) 20 | from personal_graph.vector_store import ( 21 | SQLiteVSS as SQLiteVSS, 22 | VliteVSS as VliteVSS, 23 | ) 24 | from typing import Any, Dict, List, Tuple, Optional, Union 25 | 26 | class GraphDB(AbstractContextManager): 27 | vector_store: Incomplete 28 | db: Incomplete 29 | graph_generator: Incomplete 30 | ontologies: Incomplete 31 | def __init__( 32 | self, 33 | *, 34 | vector_store: SQLiteVSS | VliteVSS = ..., 35 | database: TursoDB | SQLite = ..., 36 | graph_generator: OpenAITextToGraphParser = ..., 37 | ontologies: Optional[List[Union[Ontology, Any]]], 38 | ) -> None: ... 39 | def __eq__(self, other): ... 40 | def __enter__(self) -> GraphDB: ... 41 | def __exit__( 42 | self, 43 | exc_type: type[BaseException] | None, 44 | exc_value: BaseException | None, 45 | traceback: types.TracebackType | None, 46 | ) -> None: ... 47 | def add_node(self, node: Node) -> None: ... 48 | def add_nodes(self, nodes: List[Node]) -> None: ... 49 | def add_edge(self, edge: EdgeInput) -> None: ... 50 | def add_edges(self, edges: List[EdgeInput]) -> None: ... 51 | def update_node(self, node: Node) -> None: ... 52 | def update_nodes(self, nodes: List[Node]) -> None: ... 53 | def remove_node(self, id: str | int) -> None: ... 54 | def remove_nodes(self, ids: List[Any]) -> None: ... 55 | def search_node(self, node_id: str | int) -> Any: ... 56 | def search_node_label(self, node_id: str | int) -> Any: ... 57 | def traverse( 58 | self, source: str, target: str | None = None, with_bodies: bool = False 59 | ) -> List: ... 60 | def insert_graph(self, kg: KnowledgeGraph) -> KnowledgeGraph: ... 61 | def search_from_graph( 62 | self, 63 | text: str, 64 | *, 65 | threshold: float = 0.9, 66 | limit: int = 1, 67 | descending: bool = False, 68 | sort_by: str = "", 69 | ) -> KnowledgeGraph: ... 70 | def merge_by_similarity(self, *, threshold: float = 0.9) -> None: ... 71 | def find_nodes_like(self, label: str, *, threshold: float = 0.9) -> List[Node]: ... 72 | def visualize(self, file: str, id: List[str]) -> Digraph: ... 73 | def fetch_ids_from_db(self) -> List[str]: ... 74 | def search_indegree_edges(self, target: str) -> List[Any]: ... 75 | def search_outdegree_edges(self, source: str) -> List[Any]: ... 76 | def is_unique_prompt(self, text: str, *, threshold: float = 0.9) -> bool: ... 77 | def insert(self, text: str, attributes: Dict) -> None: ... 78 | def search( 79 | self, 80 | text: str, 81 | *, 82 | threshold: float = 0.9, 83 | descending: bool = False, 84 | limit: int = 1, 85 | sort_by: str = "", 86 | ) -> None | List[Tuple[Any, str, dict, Any]]: ... 87 | -------------------------------------------------------------------------------- /stub/personal_graph/database/sqlite/sqlite.pyi: -------------------------------------------------------------------------------- 1 | from graphviz import Digraph # type: ignore 2 | from jinja2 import BaseLoader, Environment, Template 3 | from pathlib import Path 4 | import sqlean as sqlite3 # type: ignore 5 | from personal_graph.models import Edge as Edge, Node as Node 6 | from personal_graph.database.db import DB as DB 7 | from typing import Any, Callable, Dict, List, Tuple, Optional 8 | 9 | CursorExecFunction = Callable[[sqlite3.Cursor, sqlite3.Connection], Any] 10 | 11 | def read_sql(sql_file: Path) -> str: ... 12 | 13 | class SqlTemplateLoader(BaseLoader): 14 | templates_dir: Path 15 | def __init__(self, templates_dir: Path) -> None: ... 16 | def get_source( 17 | self, environment: Environment, template: str 18 | ) -> Tuple[str, str, Callable[[], bool]]: ... 19 | 20 | class SQLite(DB): 21 | use_in_memory: bool 22 | vector0_so_path: Optional[str] 23 | vss0_so_path: Optional[str] 24 | local_path: Optional[str] 25 | env: Template 26 | clause_template: Template 27 | search_template: Template 28 | traverse_template: Template 29 | def __init__( 30 | self, 31 | *, 32 | use_in_memory: bool = False, 33 | local_path: str | None = None, 34 | vector0_so_path: str | None = None, 35 | vss0_so_path: str | None = None, 36 | ) -> None: ... 37 | def __eq__(self, other): ... 38 | def atomic(self, cursor_exec_fn: CursorExecFunction) -> Any: ... 39 | def save(self) -> None: ... 40 | def initialize(self): ... 41 | def all_connected_nodes( 42 | self, node_or_edge: Node | Edge, limit: int | None = 1 43 | ) -> Any: ... 44 | def get_connections(self, identifier: Any) -> CursorExecFunction: ... 45 | def fetch_node_embed_id(self, node_id: Any, limit: int = 1) -> None: ... 46 | def fetch_edge_embed_ids(self, id: Any, limit: int = 10): ... 47 | def search_edge( 48 | self, source: Any, target: Any, attributes: Dict, limit: int = 1 49 | ) -> Dict[Any, Any]: ... 50 | def add_node(self, label: str, attribute: Dict, id: Any): ... 51 | def add_edge( 52 | self, source: Any, target: Any, label: str, attributes: Dict 53 | ) -> None: ... 54 | def update_node(self, node: Node): ... 55 | def remove_node(self, id: Any) -> None: ... 56 | def search_node(self, node_id: Any) -> Any: ... 57 | def search_node_label(self, node_id: Any, limit: int | None = 1) -> Any: ... 58 | def traverse( 59 | self, source: Any, target: Any | None = None, with_bodies: bool = False 60 | ) -> List: ... 61 | def fetch_node_id(self, id: Any, limit: int | None = 1): ... 62 | def find_nodes_by_label(self, label: str, limit: int | None = 1): ... 63 | def graphviz_visualize( 64 | self, 65 | dot_file: str | None = None, 66 | path: List[Any] = [], 67 | connections: Any = None, 68 | format: str = "png", 69 | exclude_node_keys: List[str] = [], 70 | hide_node_key: bool = False, 71 | node_kv: str = " ", 72 | exclude_edge_keys: List[str] = [], 73 | hide_edge_key: bool = False, 74 | edge_kv: str = " ", 75 | ) -> Digraph: ... 76 | def fetch_ids_from_db(self, limit: int | None = 10) -> List[str]: ... 77 | def search_indegree_edges( 78 | self, target: Any, limit: int | None = 10 79 | ) -> List[Any]: ... 80 | def search_outdegree_edges( 81 | self, source: Any, limit: int | None = 10 82 | ) -> List[Any]: ... 83 | def search_similar_nodes( 84 | self, embed_ids, *, desc: bool | None = False, sort_by: str | None = "" 85 | ): ... 86 | def search_similar_edges( 87 | self, embed_ids, *, desc: bool = False, sort_by: str = "" 88 | ): ... 89 | def search_node_type(self, label: str): ... 90 | def search_id_by_node_type(self, node_type: str): ... 91 | -------------------------------------------------------------------------------- /personal_graph/clients.py: -------------------------------------------------------------------------------- 1 | import os 2 | from abc import ABC, abstractmethod 3 | from dataclasses import dataclass 4 | from dotenv import load_dotenv 5 | from personal_graph.embeddings import OpenAIEmbeddingsModel, OllamaEmbeddingModel 6 | 7 | import openai 8 | import ollama # type: ignore 9 | 10 | load_dotenv() 11 | 12 | 13 | class APIClient(ABC): 14 | @abstractmethod 15 | def _create_default_client(self, *args, **kwargs): 16 | pass 17 | 18 | 19 | class EmbeddingClient(ABC): 20 | @abstractmethod 21 | def _create_default_client(self): 22 | pass 23 | 24 | @abstractmethod 25 | def get_embedding_model(self): 26 | pass 27 | 28 | 29 | @dataclass 30 | class OpenAIEmbeddingClient(EmbeddingClient): 31 | dimensions: int = 384 32 | model_name: str = "text-embedding-3-small" 33 | api_key: str = "" 34 | 35 | def __post_init__(self, *args, **kwargs): 36 | self.client = self._create_default_client(*args, **kwargs) 37 | 38 | def _create_default_client(self, *args, **kwargs): 39 | return openai.OpenAI( 40 | api_key=os.getenv("OPENAI_API_KEY", self.api_key), *args, **kwargs 41 | ) 42 | 43 | def get_embedding_model(self): 44 | return OpenAIEmbeddingsModel(self.client, self.model_name, self.dimensions) 45 | 46 | 47 | @dataclass 48 | class LiteLLMEmbeddingClient(APIClient): 49 | model_name: str = "openai/text-embedding-3-small" 50 | dimensions: int = 384 51 | base_url: str = "" 52 | 53 | def __post_init__(self, *args, **kwargs): 54 | self.client = self._create_default_client(*args, **kwargs) 55 | 56 | def _create_default_client(self, *args, **kwargs): 57 | return openai.OpenAI( 58 | api_key="", 59 | base_url=os.getenv("LITE_LLM_BASE_URL", self.base_url), 60 | default_headers={ 61 | "Authorization": f"Bearer {os.getenv('LITE_LLM_TOKEN', '')}" 62 | }, 63 | *args, 64 | **kwargs, 65 | ) 66 | 67 | def get_embedding_model(self): 68 | return OpenAIEmbeddingsModel(self.client, self.model_name, self.dimensions) 69 | 70 | 71 | @dataclass 72 | class OllamaEmbeddingClient(APIClient): 73 | model_name: str 74 | dimensions: int = 768 75 | 76 | def __post_init__(self, *args, **kwargs): 77 | self.client = self._create_default_client(*args, **kwargs) 78 | 79 | def _create_default_client(self, *args, **kwargs): 80 | return ollama.Client(*args, **kwargs) 81 | 82 | def get_embedding_model(self): 83 | return OllamaEmbeddingModel(self.client, self.model_name, self.dimensions) 84 | 85 | 86 | @dataclass 87 | class LiteLLMClient(APIClient): 88 | base_url: str = "" 89 | model_name: str = "openai/gpt-3.5-turbo" 90 | 91 | def __post_init__(self, *args, **kwargs): 92 | self.client = self._create_default_client(*args, **kwargs) 93 | 94 | def _create_default_client(self, *args, **kwargs): 95 | return openai.OpenAI( 96 | api_key="", 97 | base_url=os.getenv("LITE_LLM_BASE_URL", self.base_url), 98 | default_headers={ 99 | "Authorization": f"Bearer {os.getenv('LITE_LLM_TOKEN', '')}" 100 | }, 101 | *args, 102 | **kwargs, 103 | ) 104 | 105 | 106 | @dataclass 107 | class OpenAIClient(APIClient): 108 | api_key: str = os.getenv("OPENAI_API_KEY", "") 109 | model_name: str = "gpt-3.5-turbo" 110 | 111 | def __post_init__(self, *args, **kwargs): 112 | self.client = self._create_default_client(*args, **kwargs) 113 | 114 | def _create_default_client(self, *args, **kwargs): 115 | return openai.OpenAI( 116 | api_key=os.getenv("OPENAI_API_KEY", self.api_key), *args, **kwargs 117 | ) 118 | 119 | 120 | @dataclass 121 | class OllamaClient(APIClient): 122 | model_name: str 123 | base_url: str = "http://localhost:11434/v1" 124 | api_key: str = "ollama" 125 | 126 | def __post_init__(self, *args, **kwargs): 127 | self.client = self._create_default_client(*args, **kwargs) 128 | 129 | def _create_default_client(self, *args, **kwargs): 130 | return ollama.Client(*args, **kwargs) 131 | -------------------------------------------------------------------------------- /fhir_ontology/docs/visualize_ontology_files/libs/quarto-html/quarto-syntax-highlighting.css: -------------------------------------------------------------------------------- 1 | /* quarto syntax highlight colors */ 2 | :root { 3 | --quarto-hl-ot-color: #003B4F; 4 | --quarto-hl-at-color: #657422; 5 | --quarto-hl-ss-color: #20794D; 6 | --quarto-hl-an-color: #5E5E5E; 7 | --quarto-hl-fu-color: #4758AB; 8 | --quarto-hl-st-color: #20794D; 9 | --quarto-hl-cf-color: #003B4F; 10 | --quarto-hl-op-color: #5E5E5E; 11 | --quarto-hl-er-color: #AD0000; 12 | --quarto-hl-bn-color: #AD0000; 13 | --quarto-hl-al-color: #AD0000; 14 | --quarto-hl-va-color: #111111; 15 | --quarto-hl-bu-color: inherit; 16 | --quarto-hl-ex-color: inherit; 17 | --quarto-hl-pp-color: #AD0000; 18 | --quarto-hl-in-color: #5E5E5E; 19 | --quarto-hl-vs-color: #20794D; 20 | --quarto-hl-wa-color: #5E5E5E; 21 | --quarto-hl-do-color: #5E5E5E; 22 | --quarto-hl-im-color: #00769E; 23 | --quarto-hl-ch-color: #20794D; 24 | --quarto-hl-dt-color: #AD0000; 25 | --quarto-hl-fl-color: #AD0000; 26 | --quarto-hl-co-color: #5E5E5E; 27 | --quarto-hl-cv-color: #5E5E5E; 28 | --quarto-hl-cn-color: #8f5902; 29 | --quarto-hl-sc-color: #5E5E5E; 30 | --quarto-hl-dv-color: #AD0000; 31 | --quarto-hl-kw-color: #003B4F; 32 | } 33 | 34 | /* other quarto variables */ 35 | :root { 36 | --quarto-font-monospace: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; 37 | } 38 | 39 | pre > code.sourceCode > span { 40 | color: #003B4F; 41 | } 42 | 43 | code span { 44 | color: #003B4F; 45 | } 46 | 47 | code.sourceCode > span { 48 | color: #003B4F; 49 | } 50 | 51 | div.sourceCode, 52 | div.sourceCode pre.sourceCode { 53 | color: #003B4F; 54 | } 55 | 56 | code span.ot { 57 | color: #003B4F; 58 | font-style: inherit; 59 | } 60 | 61 | code span.at { 62 | color: #657422; 63 | font-style: inherit; 64 | } 65 | 66 | code span.ss { 67 | color: #20794D; 68 | font-style: inherit; 69 | } 70 | 71 | code span.an { 72 | color: #5E5E5E; 73 | font-style: inherit; 74 | } 75 | 76 | code span.fu { 77 | color: #4758AB; 78 | font-style: inherit; 79 | } 80 | 81 | code span.st { 82 | color: #20794D; 83 | font-style: inherit; 84 | } 85 | 86 | code span.cf { 87 | color: #003B4F; 88 | font-weight: bold; 89 | font-style: inherit; 90 | } 91 | 92 | code span.op { 93 | color: #5E5E5E; 94 | font-style: inherit; 95 | } 96 | 97 | code span.er { 98 | color: #AD0000; 99 | font-style: inherit; 100 | } 101 | 102 | code span.bn { 103 | color: #AD0000; 104 | font-style: inherit; 105 | } 106 | 107 | code span.al { 108 | color: #AD0000; 109 | font-style: inherit; 110 | } 111 | 112 | code span.va { 113 | color: #111111; 114 | font-style: inherit; 115 | } 116 | 117 | code span.bu { 118 | font-style: inherit; 119 | } 120 | 121 | code span.ex { 122 | font-style: inherit; 123 | } 124 | 125 | code span.pp { 126 | color: #AD0000; 127 | font-style: inherit; 128 | } 129 | 130 | code span.in { 131 | color: #5E5E5E; 132 | font-style: inherit; 133 | } 134 | 135 | code span.vs { 136 | color: #20794D; 137 | font-style: inherit; 138 | } 139 | 140 | code span.wa { 141 | color: #5E5E5E; 142 | font-style: italic; 143 | } 144 | 145 | code span.do { 146 | color: #5E5E5E; 147 | font-style: italic; 148 | } 149 | 150 | code span.im { 151 | color: #00769E; 152 | font-style: inherit; 153 | } 154 | 155 | code span.ch { 156 | color: #20794D; 157 | font-style: inherit; 158 | } 159 | 160 | code span.dt { 161 | color: #AD0000; 162 | font-style: inherit; 163 | } 164 | 165 | code span.fl { 166 | color: #AD0000; 167 | font-style: inherit; 168 | } 169 | 170 | code span.co { 171 | color: #5E5E5E; 172 | font-style: inherit; 173 | } 174 | 175 | code span.cv { 176 | color: #5E5E5E; 177 | font-style: italic; 178 | } 179 | 180 | code span.cn { 181 | color: #8f5902; 182 | font-style: inherit; 183 | } 184 | 185 | code span.sc { 186 | color: #5E5E5E; 187 | font-style: inherit; 188 | } 189 | 190 | code span.dv { 191 | color: #AD0000; 192 | font-style: inherit; 193 | } 194 | 195 | code span.kw { 196 | color: #003B4F; 197 | font-weight: bold; 198 | font-style: inherit; 199 | } 200 | 201 | .prevent-inlining { 202 | content: " KnowledgeGraph: 13 | """Generate a KnowledgeGraph from the given query.""" 14 | pass 15 | 16 | 17 | class OpenAITextToGraphParser(TextToGraphParserInterface): 18 | def __init__( 19 | self, 20 | llm_client: Union[LiteLLMClient, OpenAIClient], 21 | system_prompt: str = "You are a high quality knowledge graph generator based on the user query for the purpose of generating descriptive, informative, detailed and accurate knowledge graphs. You can generate proper nodes and edges as a knowledge graph.", 22 | prompt: str = "Help me describe this user query as a detailed knowledge graph with meaningful relationships that should provide some descriptive attributes(attribute is the detailed and proper information about the edge) and informative labels about the nodes and relationship. Try to make most of the relationships between similar nodes.", 23 | ): 24 | self.system_prompt = system_prompt 25 | self.prompt = prompt 26 | self.llm_client = llm_client 27 | 28 | def __repr__(self): 29 | return ( 30 | f"OpenAITextToGraphParser(\n" 31 | f" llm_client={self.llm_client},\n" 32 | f" system_prompt={self.system_prompt},\n" 33 | f" prompt={self.prompt}\n" 34 | f" )" 35 | ) 36 | 37 | def generate(self, query: str) -> KnowledgeGraph: 38 | client = instructor.from_openai(self.llm_client.client) 39 | knowledge_graph = client.chat.completions.create( 40 | model=self.llm_client.model_name, 41 | messages=[ 42 | { 43 | "role": "system", 44 | "content": self.system_prompt, 45 | }, 46 | { 47 | "role": "user", 48 | "content": f"{self.prompt}: {query}", 49 | }, 50 | ], 51 | response_model=KnowledgeGraph, 52 | ) 53 | return knowledge_graph 54 | 55 | 56 | class OllamaTextToGraphParser(TextToGraphParserInterface): 57 | def __init__( 58 | self, 59 | llm_client: Union[OllamaClient], 60 | system_prompt: str = "You are a high quality knowledge graph generator based on the user query for the purpose of generating descriptive, informative, detailed and accurate knowledge graphs. You can generate proper nodes and edges as a knowledge graph.", 61 | prompt: str = "Help me describe this user query as a detailed knowledge graph with meaningful relationships that should provide some descriptive attributes(attributes is the detailed and proper descriptive information about the edges and nodes) and informative labels about the nodes and relationships. Try to make most of the relationships between similar nodes.", 62 | ): 63 | self.system_prompt = system_prompt 64 | self.prompt = prompt 65 | self.llm_client = llm_client 66 | 67 | def __repr__(self): 68 | return ( 69 | f"OllamaTextToGraphParser(\n" 70 | f" llm_client={self.llm_client},\n" 71 | f" system_prompt={self.system_prompt},\n" 72 | f" prompt={self.prompt}\n" 73 | f" )" 74 | ) 75 | 76 | def generate(self, query: str) -> KnowledgeGraph: 77 | client = instructor.from_openai( 78 | OpenAI( 79 | base_url=self.llm_client.base_url, 80 | api_key=self.llm_client.api_key, 81 | ), 82 | mode=instructor.Mode.JSON, 83 | ) 84 | 85 | knowledge_graph = client.chat.completions.create( 86 | model=self.llm_client.model_name, 87 | messages=[ 88 | { 89 | "role": "system", 90 | "content": self.system_prompt, 91 | }, 92 | { 93 | "role": "user", 94 | "content": f"{self.prompt}: {query}", 95 | }, 96 | ], 97 | stream=False, 98 | response_model=KnowledgeGraph, 99 | ) 100 | 101 | return knowledge_graph 102 | -------------------------------------------------------------------------------- /fhir_ontology/visualize_fhir_ontology.py: -------------------------------------------------------------------------------- 1 | """ 2 | Visualising fhir ontology using python interactive tool i.e plotly 3 | """ 4 | 5 | import json 6 | import logging 7 | import argparse 8 | 9 | import networkx as nx # type: ignore 10 | import plotly.graph_objects as go # type: ignore 11 | from plotly.io import write_html # type: ignore 12 | 13 | 14 | def visualise_fhir_with_plotly(args): 15 | # Load the JSON data 16 | with open(args.json_file, "r") as f: 17 | ontology_data = json.load(f) 18 | 19 | # Create a NetworkX graph 20 | G = nx.MultiGraph() 21 | 22 | # Add nodes and edges to the graph 23 | for node in ontology_data["nodes"]: 24 | G.add_node(node["id"], label=node["label"]) 25 | 26 | for edge in ontology_data["edges"]: 27 | G.add_edge(edge["source"], edge["target"], label=edge["label"]) 28 | 29 | # Get node positions using a layout algorithm 30 | pos = nx.spring_layout(G, k=0.5, iterations=50) 31 | 32 | # Create edge trace 33 | edge_traces = [] 34 | for edge in G.edges(data=True): 35 | x0, y0 = pos[edge[0]] 36 | x1, y1 = pos[edge[1]] 37 | 38 | edge_trace = go.Scatter( 39 | x=[x0, x1, None], 40 | y=[y0, y1, None], 41 | line=dict(width=0.5, color="#888"), 42 | hoverinfo="text", 43 | mode="lines+text", 44 | text=[edge[2]["label"]], 45 | textposition="middle center", 46 | textfont=dict(size=8), 47 | ) 48 | edge_traces.append(edge_trace) 49 | 50 | # Add edge label 51 | edge_label_trace = go.Scatter( 52 | x=[(x0 + x1) / 2], 53 | y=[(y0 + y1) / 2], 54 | mode="text", 55 | text=[edge[2]["label"]], 56 | textposition="middle center", 57 | textfont=dict(size=8, color="red"), 58 | hoverinfo="none", 59 | ) 60 | edge_traces.append(edge_label_trace) 61 | 62 | # Create node trace 63 | node_x = [] 64 | node_y = [] 65 | for node in G.nodes(): 66 | x, y = pos[node] 67 | node_x.append(x) 68 | node_y.append(y) 69 | 70 | node_trace = go.Scatter( 71 | x=node_x, 72 | y=node_y, 73 | mode="markers+text", 74 | hoverinfo="text", 75 | text=[ 76 | f"{node}
# of connections: {len(list(G.edges(node)))}" 77 | for node in G.nodes() 78 | ], 79 | textposition="top center", 80 | textfont=dict(size=8), 81 | marker=dict( 82 | showscale=True, 83 | colorscale="YlGnBu", 84 | size=10, 85 | color=[len(list(G.neighbors(node))) for node in G.nodes()], 86 | colorbar=dict( 87 | thickness=15, 88 | title="Node Connections", 89 | xanchor="left", 90 | titleside="right", 91 | ), 92 | ), 93 | ) 94 | 95 | # Create the figure 96 | fig = go.Figure( 97 | data=edge_traces + [node_trace], 98 | layout=go.Layout( 99 | title="FHIR Ontology Network Graph", 100 | titlefont_size=16, 101 | showlegend=False, 102 | hovermode="closest", 103 | margin=dict(b=20, l=5, r=5, t=40), 104 | annotations=[ 105 | dict( 106 | text="FHIR Ontology", 107 | showarrow=False, 108 | xref="paper", 109 | yref="paper", 110 | x=0.005, 111 | y=-0.002, 112 | ) 113 | ], 114 | xaxis=dict(showgrid=False, zeroline=False, showticklabels=False), 115 | yaxis=dict(showgrid=False, zeroline=False, showticklabels=False), 116 | ), 117 | ) 118 | 119 | # Save as interactive HTML 120 | write_html(fig, file=args.output_html_file, auto_open=True) 121 | logging.info(f"Interactive graph saved as {args.output_html_file}") 122 | 123 | 124 | if __name__ == "__main__": 125 | logging.basicConfig( 126 | level=logging.DEBUG, 127 | filemode="w", 128 | filename="./log/fhir_ontology.log", 129 | ) 130 | 131 | parser = argparse.ArgumentParser(description="Visualising entire fhir ontology") 132 | parser.add_argument( 133 | "--json-file", default="fhir_ontology/fhir_ontology.json", type=str 134 | ) 135 | parser.add_argument( 136 | "--output-html-file", default="fhir_ontology/output.html", type=str 137 | ) 138 | arguments = parser.parse_args() 139 | visualise_fhir_with_plotly(args=arguments) 140 | -------------------------------------------------------------------------------- /personal_graph/vector_store/vlitevss/vlitevss.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import Dict, Optional, Any, List, Union 3 | 4 | from vlite import VLite # type: ignore 5 | 6 | from personal_graph.vector_store.vector_store import VectorStore 7 | 8 | 9 | class VliteVSS(VectorStore): 10 | def __init__( 11 | self, 12 | *, 13 | collection: str = "./vectors", 14 | model_name: str = "mixedbread-ai/mxbai-embed-large-v1", 15 | ): 16 | self.collection = collection 17 | self.model_name = model_name 18 | 19 | def initialize(self): 20 | self.vlite = VLite(collection=self.collection, model_name=self.model_name) 21 | return self.vlite 22 | 23 | def __eq__(self, other): 24 | return self.collection == other.collection 25 | 26 | def save(self): 27 | self.vlite.save() 28 | 29 | def add_node_embedding(self, id: Any, label: str, attribute: Dict): 30 | count = self.vlite.count() + 1 31 | attribute.update({"label": label, "embed_id": count}) 32 | self.vlite.add( 33 | item_id=id, 34 | data={"text": json.dumps(attribute)}, 35 | metadata={"embed_id": count}, 36 | ) 37 | self.vlite.save() 38 | 39 | def add_edge_embedding( 40 | self, source: Any, target: Any, label: str, attributes: Dict 41 | ) -> None: 42 | count = self.vlite.count() + 1 43 | attributes.update( 44 | {"source": source, "target": target, "label": label, "embed_id": count} 45 | ) 46 | self.vlite.add({"text": json.dumps(attributes)}, metadata={"embed_id": count}) 47 | self.vlite.save() 48 | 49 | def add_edge_embeddings( 50 | self, 51 | sources: List[Any], 52 | targets: List[Any], 53 | labels: List[str], 54 | attributes: List[Union[Dict[str, str]]], 55 | ): 56 | for i, x in enumerate(zip(sources, targets, labels, attributes)): 57 | self.add_edge_embedding(x[0], x[1], x[2], x[3]) 58 | 59 | def delete_node_embedding(self, ids: Any) -> None: 60 | for id in ids: 61 | id_to_be_deleted = self.vlite.get(where={"embed_id": id}) 62 | self.vlite.delete(id_to_be_deleted) 63 | 64 | def delete_edge_embedding(self, ids: Any) -> None: 65 | id_to_be_deleted = self.vlite.get(where={"embed_id": id}) 66 | self.vlite.delete(id_to_be_deleted) 67 | 68 | def vector_search_node( 69 | self, 70 | data: Dict, 71 | *, 72 | threshold: Optional[float] = None, 73 | descending: bool, 74 | limit: int, 75 | sort_by: str, 76 | ): 77 | results = self.vlite.retrieve( 78 | text=json.dumps(data), top_k=limit, return_scores=True 79 | ) 80 | 81 | if results is None: 82 | return None 83 | 84 | if sort_by: 85 | results.sort(key=lambda x: x[2].get(sort_by, 0), reverse=descending) 86 | 87 | if threshold is not None: 88 | return [res for res in results if res[3] < threshold][:limit] 89 | 90 | return results[:limit] 91 | 92 | def vector_search_edge( 93 | self, 94 | data: Dict, 95 | *, 96 | threshold: Optional[float] = None, 97 | descending: bool, 98 | limit: int, 99 | sort_by: str, 100 | ): 101 | results = self.vlite.retrieve( 102 | text=json.dumps(data), top_k=limit, return_scores=True 103 | ) 104 | if results is None: 105 | return None 106 | 107 | if sort_by: 108 | results.sort(key=lambda x: x[2].get(sort_by, 0), reverse=descending) 109 | 110 | if threshold is not None: 111 | return [res for res in results if res[3] < threshold][:limit] 112 | 113 | return results[:limit] 114 | 115 | def vector_search_node_from_multi_db( 116 | self, data: Dict, *, threshold: Optional[float] = None, limit: int = 1 117 | ): 118 | results = self.vlite.retrieve( 119 | text=json.dumps(data), top_k=limit, return_scores=True 120 | ) 121 | 122 | if results is None: 123 | return None 124 | 125 | if threshold is not None: 126 | return [res for res in results if res[3] < threshold][:limit] 127 | 128 | return results[:limit] 129 | 130 | def vector_search_edge_from_multi_db( 131 | self, data: Dict, *, threshold: Optional[float] = None, limit: int = 1 132 | ): 133 | results = self.vlite.retrieve( 134 | text=json.dumps(data), top_k=limit, return_scores=True 135 | ) 136 | 137 | if results is None: 138 | return None 139 | 140 | if threshold is not None: 141 | return [res for res in results if res[3] < threshold][:limit] 142 | 143 | return results[:limit] 144 | -------------------------------------------------------------------------------- /personal_graph/database/db.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from typing import Any, Callable, Dict, Optional, List, Union 3 | from graphviz import Digraph # type: ignore 4 | 5 | from personal_graph.models import Node, Edge 6 | 7 | # CursorExecFunction = Callable[[libsql.Cursor, libsql.Connection], Any] 8 | CursorExecFunction = Callable[[Any, Any], Any] # TODO: Constraint the type 9 | 10 | 11 | class DB(ABC): 12 | """ 13 | Abstract Base Class (ABC) for a graph database interface. 14 | 15 | This class defines methods for implementing a graph database with various 16 | operations such as adding, updating, removing nodes and edges, traversing the 17 | graph, and visualizing it. Implementations of this class should provide concrete 18 | behavior for each of the abstract methods. 19 | """ 20 | 21 | @abstractmethod 22 | def initialize(self): 23 | """Initialize the database""" 24 | pass 25 | 26 | @abstractmethod 27 | def __eq__(self, other): 28 | """Check equality with another DB object""" 29 | pass 30 | 31 | @abstractmethod 32 | def save(self): 33 | """Save the current state of the database""" 34 | pass 35 | 36 | @abstractmethod 37 | def fetch_node_embed_id(self, node_id: Any): 38 | """Fetch the embedding ID of a node given its ID""" 39 | pass 40 | 41 | @abstractmethod 42 | def fetch_edge_embed_ids(self, id: Any): 43 | """Fetch the embedding IDs of edges given a node or edge ID""" 44 | pass 45 | 46 | @abstractmethod 47 | def all_connected_nodes(self, node_or_edge: Union[Node | Edge]) -> Any: 48 | """Retrieve all nodes connected to a given node or edge""" 49 | pass 50 | 51 | @abstractmethod 52 | def get_connections(self, identifier: Any) -> CursorExecFunction: 53 | """Get connections for a given identifier.""" 54 | pass 55 | 56 | @abstractmethod 57 | def search_edge(self, source: Any, target: Any, attributes: Dict): 58 | """Search for an edge given its source, target, and attributes""" 59 | pass 60 | 61 | @abstractmethod 62 | def add_node(self, label: str, attribute: Dict, id: Any): 63 | """Add a node to the database""" 64 | pass 65 | 66 | @abstractmethod 67 | def add_edge(self, source: Any, target: Any, label: str, attributes: Dict) -> None: 68 | """Add an edge to the database""" 69 | pass 70 | 71 | @abstractmethod 72 | def update_node(self, node: Node): 73 | """Update a node in the database""" 74 | pass 75 | 76 | @abstractmethod 77 | def remove_node(self, id: Any) -> None: 78 | """Remove a node from the database""" 79 | pass 80 | 81 | @abstractmethod 82 | def search_node(self, node_id: Any) -> Any: 83 | """Search for a node by its ID""" 84 | pass 85 | 86 | @abstractmethod 87 | def search_node_label(self, node_id: Any) -> Any: 88 | """Search for a node label by its ID""" 89 | pass 90 | 91 | @abstractmethod 92 | def traverse( 93 | self, source: Any, target: Optional[Any] = None, with_bodies: bool = False 94 | ) -> List: 95 | """Traverse the graph from a source to an optional target node""" 96 | pass 97 | 98 | @abstractmethod 99 | def fetch_node_id(self, id: Any): 100 | """Fetch a node ID given another identifier""" 101 | pass 102 | 103 | @abstractmethod 104 | def find_nodes_by_label(self, label: str): 105 | """Find nodes by their label""" 106 | pass 107 | 108 | @abstractmethod 109 | def graphviz_visualize( 110 | self, 111 | dot_file: Optional[str] = None, 112 | path: List[Any] = [], 113 | connections: Any = None, 114 | format: str = "png", 115 | exclude_node_keys: List[str] = [], 116 | hide_node_key: bool = False, 117 | node_kv: str = " ", 118 | exclude_edge_keys: List[str] = [], 119 | hide_edge_key: bool = False, 120 | edge_kv: str = " ", 121 | ) -> Digraph: 122 | """Visualize the graph using Graphviz""" 123 | pass 124 | 125 | @abstractmethod 126 | def fetch_ids_from_db(self) -> List[str]: 127 | """Fetch all IDs from the database""" 128 | pass 129 | 130 | @abstractmethod 131 | def search_indegree_edges(self, target: Any) -> List[Any]: 132 | """Search for edges with the given target node""" 133 | pass 134 | 135 | @abstractmethod 136 | def search_outdegree_edges(self, source: Any) -> List[Any]: 137 | """Search for edges with the given source node""" 138 | pass 139 | 140 | @abstractmethod 141 | def search_similar_nodes(self, embed_id, *, desc: bool, sort_by: str): 142 | """Search for nodes similar to the given embedding ID""" 143 | pass 144 | 145 | @abstractmethod 146 | def search_similar_edges(self, embed_id, *, desc: bool, sort_by: str): 147 | """Search for edges similar to the given embedding ID""" 148 | pass 149 | 150 | @abstractmethod 151 | def search_node_type(self, label: str): 152 | pass 153 | 154 | @abstractmethod 155 | def search_id_by_node_type(self, node_type: str): 156 | pass 157 | -------------------------------------------------------------------------------- /fhir_ontology/docs/visualize_ontology_files/libs/quarto-html/anchor.min.js: -------------------------------------------------------------------------------- 1 | // @license magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&dn=expat.txt Expat 2 | // 3 | // AnchorJS - v5.0.0 - 2023-01-18 4 | // https://www.bryanbraun.com/anchorjs/ 5 | // Copyright (c) 2023 Bryan Braun; Licensed MIT 6 | // 7 | // @license magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&dn=expat.txt Expat 8 | !function(A,e){"use strict";"function"==typeof define&&define.amd?define([],e):"object"==typeof module&&module.exports?module.exports=e():(A.AnchorJS=e(),A.anchors=new A.AnchorJS)}(globalThis,function(){"use strict";return function(A){function u(A){A.icon=Object.prototype.hasOwnProperty.call(A,"icon")?A.icon:"",A.visible=Object.prototype.hasOwnProperty.call(A,"visible")?A.visible:"hover",A.placement=Object.prototype.hasOwnProperty.call(A,"placement")?A.placement:"right",A.ariaLabel=Object.prototype.hasOwnProperty.call(A,"ariaLabel")?A.ariaLabel:"Anchor",A.class=Object.prototype.hasOwnProperty.call(A,"class")?A.class:"",A.base=Object.prototype.hasOwnProperty.call(A,"base")?A.base:"",A.truncate=Object.prototype.hasOwnProperty.call(A,"truncate")?Math.floor(A.truncate):64,A.titleText=Object.prototype.hasOwnProperty.call(A,"titleText")?A.titleText:""}function d(A){var e;if("string"==typeof A||A instanceof String)e=[].slice.call(document.querySelectorAll(A));else{if(!(Array.isArray(A)||A instanceof NodeList))throw new TypeError("The selector provided to AnchorJS was invalid.");e=[].slice.call(A)}return e}this.options=A||{},this.elements=[],u(this.options),this.add=function(A){var e,t,o,i,n,s,a,r,l,c,h,p=[];if(u(this.options),0!==(e=d(A=A||"h2, h3, h4, h5, h6")).length){for(null===document.head.querySelector("style.anchorjs")&&((A=document.createElement("style")).className="anchorjs",A.appendChild(document.createTextNode("")),void 0===(h=document.head.querySelector('[rel="stylesheet"],style'))?document.head.appendChild(A):document.head.insertBefore(A,h),A.sheet.insertRule(".anchorjs-link{opacity:0;text-decoration:none;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}",A.sheet.cssRules.length),A.sheet.insertRule(":hover>.anchorjs-link,.anchorjs-link:focus{opacity:1}",A.sheet.cssRules.length),A.sheet.insertRule("[data-anchorjs-icon]::after{content:attr(data-anchorjs-icon)}",A.sheet.cssRules.length),A.sheet.insertRule('@font-face{font-family:anchorjs-icons;src:url(data:n/a;base64,AAEAAAALAIAAAwAwT1MvMg8yG2cAAAE4AAAAYGNtYXDp3gC3AAABpAAAAExnYXNwAAAAEAAAA9wAAAAIZ2x5ZlQCcfwAAAH4AAABCGhlYWQHFvHyAAAAvAAAADZoaGVhBnACFwAAAPQAAAAkaG10eASAADEAAAGYAAAADGxvY2EACACEAAAB8AAAAAhtYXhwAAYAVwAAARgAAAAgbmFtZQGOH9cAAAMAAAAAunBvc3QAAwAAAAADvAAAACAAAQAAAAEAAHzE2p9fDzz1AAkEAAAAAADRecUWAAAAANQA6R8AAAAAAoACwAAAAAgAAgAAAAAAAAABAAADwP/AAAACgAAA/9MCrQABAAAAAAAAAAAAAAAAAAAAAwABAAAAAwBVAAIAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAMCQAGQAAUAAAKZAswAAACPApkCzAAAAesAMwEJAAAAAAAAAAAAAAAAAAAAARAAAAAAAAAAAAAAAAAAAAAAQAAg//0DwP/AAEADwABAAAAAAQAAAAAAAAAAAAAAIAAAAAAAAAIAAAACgAAxAAAAAwAAAAMAAAAcAAEAAwAAABwAAwABAAAAHAAEADAAAAAIAAgAAgAAACDpy//9//8AAAAg6cv//f///+EWNwADAAEAAAAAAAAAAAAAAAAACACEAAEAAAAAAAAAAAAAAAAxAAACAAQARAKAAsAAKwBUAAABIiYnJjQ3NzY2MzIWFxYUBwcGIicmNDc3NjQnJiYjIgYHBwYUFxYUBwYGIwciJicmNDc3NjIXFhQHBwYUFxYWMzI2Nzc2NCcmNDc2MhcWFAcHBgYjARQGDAUtLXoWOR8fORYtLTgKGwoKCjgaGg0gEhIgDXoaGgkJBQwHdR85Fi0tOAobCgoKOBoaDSASEiANehoaCQkKGwotLXoWOR8BMwUFLYEuehYXFxYugC44CQkKGwo4GkoaDQ0NDXoaShoKGwoFBe8XFi6ALjgJCQobCjgaShoNDQ0NehpKGgobCgoKLYEuehYXAAAADACWAAEAAAAAAAEACAAAAAEAAAAAAAIAAwAIAAEAAAAAAAMACAAAAAEAAAAAAAQACAAAAAEAAAAAAAUAAQALAAEAAAAAAAYACAAAAAMAAQQJAAEAEAAMAAMAAQQJAAIABgAcAAMAAQQJAAMAEAAMAAMAAQQJAAQAEAAMAAMAAQQJAAUAAgAiAAMAAQQJAAYAEAAMYW5jaG9yanM0MDBAAGEAbgBjAGgAbwByAGoAcwA0ADAAMABAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAH//wAP) format("truetype")}',A.sheet.cssRules.length)),h=document.querySelectorAll("[id]"),t=[].map.call(h,function(A){return A.id}),i=0;i\]./()*\\\n\t\b\v\u00A0]/g,"-").replace(/-{2,}/g,"-").substring(0,this.options.truncate).replace(/^-+|-+$/gm,"").toLowerCase()},this.hasAnchorJSLink=function(A){var e=A.firstChild&&-1<(" "+A.firstChild.className+" ").indexOf(" anchorjs-link "),A=A.lastChild&&-1<(" "+A.lastChild.className+" ").indexOf(" anchorjs-link ");return e||A||!1}}}); 9 | // @license-end -------------------------------------------------------------------------------- /personal_graph/vector_store/sqlitevss/fhirsqlitevss.py: -------------------------------------------------------------------------------- 1 | import json 2 | from functools import lru_cache 3 | from pathlib import Path 4 | from typing import Any, Dict, Optional 5 | 6 | from personal_graph.vector_store import SQLiteVSS 7 | from personal_graph.database.db import CursorExecFunction 8 | 9 | 10 | @lru_cache(maxsize=None) 11 | def read_sql(sql_file: Path) -> str: 12 | with open( 13 | Path(__file__).parent.resolve() / "fhir-embedding-queries" / sql_file 14 | ) as f: 15 | return f.read() 16 | 17 | 18 | class FhirSQLiteVSS(SQLiteVSS): 19 | def initialize(self): 20 | def _init(cursor, connection): 21 | vector_schema = read_sql(Path("fhir_4_vector_schema.sql")) 22 | vector_schema = vector_schema.replace("{{size}}", str(self.index_dimension)) 23 | 24 | connection.executescript(vector_schema) 25 | connection.commit() 26 | 27 | return self.db.atomic(_init) 28 | 29 | def save(self): 30 | return self.db.save() 31 | 32 | def __repr__(self) -> str: 33 | return f"FhirSQLiteVSS(\n" f" db={self.db}\n" f" )" 34 | 35 | def _add_embedding(self, id: Any, label: str, data: Dict) -> CursorExecFunction: 36 | def _insert(cursor, connection): 37 | set_data = self._set_id(id, label, data) 38 | rt = label.lower() 39 | 40 | count = cursor.execute( 41 | f"""SELECT COALESCE(MAX(rowid), 0) FROM {rt}_embedding""" 42 | ).fetchone()[0] 43 | 44 | # To check whether status is recreated, if so then do not add the embedding 45 | status = cursor.execute( 46 | f"""SELECT status from {rt} WHERE id=?""", (id,) 47 | ).fetchone()[0] 48 | 49 | if status != "recreated": 50 | cursor.execute( 51 | f"""INSERT INTO {rt}_embedding(rowid, vector_node) VALUES (?,?);""", 52 | ( 53 | int(count) + 1, 54 | json.dumps( 55 | self.embedding_model.get_embedding(json.dumps(set_data)) 56 | ), 57 | ), 58 | ) 59 | connection.commit() 60 | 61 | return _insert 62 | 63 | def _add_edge_embedding(self, data: Dict): 64 | def _insert_edge_embedding(cursor, connection): 65 | count = ( 66 | cursor.execute( 67 | "SELECT COALESCE(MAX(rowid), 0) FROM relations_embedding" 68 | ).fetchone()[0] 69 | + 1 70 | ) 71 | 72 | cursor.execute( 73 | """INSERT INTO relations_embedding(rowid, vector_relations) VALUES(?, ?)""", 74 | ( 75 | count, 76 | json.dumps(self.embedding_model.get_embedding(json.dumps(data))), 77 | ), 78 | ) 79 | connection.commit() 80 | 81 | return _insert_edge_embedding 82 | 83 | def _remove_node(self, id: Any, node_type: Optional[str] = None): 84 | def _delete_node_embedding(cursor, connection): 85 | cursor.execute( 86 | f"""DELETE FROM {node_type.lower()}_embedding where rowid = (?)""", 87 | (id[0],), 88 | ) 89 | 90 | return _delete_node_embedding 91 | 92 | def _remove_edge(self, ids: Any): 93 | def _delete_node_embedding(cursor, connection): 94 | for id in ids: 95 | cursor.execute( 96 | """DELETE FROM relations_embedding where rowid = (?)""", (id[0],) 97 | ) 98 | 99 | return _delete_node_embedding 100 | 101 | def add_node_embedding( 102 | self, id: Any, label: str, attribute: Dict[Any, Any] 103 | ) -> None: 104 | self.db.atomic(self._add_embedding(id, label, attribute)) 105 | 106 | def add_edge_embedding( 107 | self, source: Any, target: Any, label: str, attributes: Dict 108 | ) -> None: 109 | edge_data = { 110 | "source_id": source, 111 | "target_id": target, 112 | "label": label, 113 | "attributes": json.dumps(attributes), 114 | } 115 | 116 | self.db.atomic(self._add_edge_embedding(edge_data)) 117 | 118 | def add_edge_embeddings(self, sources, targets, labels, attributes): 119 | for i, x in enumerate(zip(sources, targets, labels, attributes)): 120 | edge_data = { 121 | "source_id": x[0], 122 | "target_id": x[1], 123 | "label": x[2], 124 | "attributes": json.dumps(x[3]), 125 | } 126 | self.db.atomic(self._add_edge_embedding(edge_data)) 127 | 128 | def delete_node_embedding(self, id: Any, node_type: Optional[str] = None) -> None: 129 | if not node_type: 130 | raise ValueError("Resource type not provided.") 131 | 132 | self.db.atomic(self._remove_node(id, node_type=node_type)) 133 | 134 | def delete_edge_embedding(self, ids: Any) -> None: 135 | self.db.atomic(self._remove_edge(ids)) 136 | 137 | def vector_search_edge_from_multi_db( 138 | self, data: Dict, *, threshold: float = 0.9, limit: int = 1 139 | ): 140 | raise NotImplementedError( 141 | "vector_search_edge_from_multi_db method is not yet implemented" 142 | ) 143 | 144 | def vector_search_node_from_multi_db( 145 | self, data: Dict, *, threshold: float = 0.9, limit: int = 1 146 | ): 147 | raise NotImplementedError( 148 | "vector_search_node_from_multi_db method is not yet implemented" 149 | ) 150 | 151 | def vector_search_edge( 152 | self, 153 | data: Dict, 154 | *, 155 | threshold: float = 0.9, 156 | descending: bool = False, 157 | limit: int = 1, 158 | sort_by: str = "", 159 | ): 160 | raise NotImplementedError("vector_search_edge method is not yet implemented") 161 | 162 | def vector_search_node( 163 | self, 164 | data: Dict, 165 | *, 166 | threshold: float = 0.9, 167 | descending: bool = False, 168 | limit: int = 1, 169 | sort_by: str = "", 170 | ): 171 | raise NotImplementedError("vector_search_node method is not yet implemented") 172 | -------------------------------------------------------------------------------- /personal_graph/ml.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import Dict, Any 3 | 4 | import networkx as nx # type: ignore 5 | from graphviz import Digraph # type: ignore 6 | from matplotlib import pyplot as plt 7 | 8 | from personal_graph import GraphDB 9 | from personal_graph.models import KnowledgeGraph, Node, Edge, EdgeInput 10 | 11 | 12 | def pg_to_networkx(graph: GraphDB, *, post_visualize: bool = False): 13 | """ 14 | Convert the graph database to a NetworkX DiGraph object. 15 | """ 16 | G = nx.Graph() # Empty Graph with no nodes and edges 17 | 18 | node_ids = graph.fetch_ids_from_db() 19 | # Add edges to networkX 20 | for source_id in node_ids: 21 | outdegree_edges = graph.search_outdegree_edges(source_id) 22 | if outdegree_edges is []: 23 | continue 24 | 25 | for target_id, edge_label, edge_data in outdegree_edges: 26 | if isinstance(edge_data, str): 27 | edge_data = json.loads(edge_data) 28 | 29 | edge_data["label"] = edge_label 30 | G.add_edge(source_id, target_id, **edge_data) 31 | 32 | for target_id in node_ids: 33 | indegree_edges = graph.search_indegree_edges(target_id) 34 | 35 | if indegree_edges is []: 36 | continue 37 | 38 | for source_id, edge_label, edge_data in indegree_edges: 39 | if isinstance(edge_data, str): 40 | edge_data = json.loads(edge_data) 41 | 42 | edge_data["label"] = edge_label 43 | G.add_edge(source_id, target_id, **edge_data) 44 | 45 | node_labels = {} 46 | for node_id in node_ids: 47 | node_data = graph.search_node(node_id) 48 | if "label" not in node_data.keys(): 49 | node_data["label"] = graph.db.search_node_label(node_id) 50 | 51 | G.add_node(node_id, **node_data) 52 | node_label = node_data["label"] 53 | node_labels[node_id] = node_label[0] 54 | 55 | if post_visualize: 56 | # Visualizing the NetworkX Graph 57 | plt.figure(figsize=(30, 30), dpi=200) # Increase the figure size and resolution 58 | pos = nx.spring_layout( 59 | G, scale=6 60 | ) # Use spring layout for better node positioning 61 | 62 | nx.draw_networkx( 63 | G, 64 | pos, 65 | with_labels=True, 66 | labels=node_labels, 67 | font_size=9, 68 | nodelist=G.nodes(), 69 | edgelist=G.edges(), 70 | node_size=800, 71 | node_color="skyblue", 72 | edge_color="gray", 73 | width=2, 74 | ) 75 | nx.draw_networkx_edge_labels( 76 | G, pos, edge_labels=nx.get_edge_attributes(G, "label") 77 | ) 78 | plt.axis("off") # Show the axes 79 | plt.savefig("networkX_graph.png") 80 | 81 | return G 82 | 83 | 84 | def networkx_to_pg( 85 | networkx_graph: nx, 86 | graph: GraphDB, 87 | *, 88 | post_visualize: bool = False, 89 | override: bool = True, 90 | ): 91 | if override: 92 | node_ids = graph.fetch_ids_from_db() 93 | graph.remove_nodes(node_ids) 94 | 95 | node_ids_with_edges = set() 96 | kg = KnowledgeGraph() 97 | 98 | # Convert networkX edges to personal graph edges 99 | for source_id, target_id, edge_data in networkx_graph.edges(data=True): 100 | edge_attributes: Dict[str, Any] = edge_data 101 | edge_label: str = edge_attributes["label"] 102 | 103 | if not override: 104 | # Check if the node with the given id exists, if not then firstly add the node. 105 | source = graph.search_node(source_id) 106 | if source is []: 107 | graph.add_node( 108 | Node( 109 | id=str(source_id), 110 | label=edge_label if edge_label else "", 111 | attributes=edge_attributes, 112 | ) 113 | ) 114 | else: 115 | node_ids_with_edges.add(str(source_id)) 116 | 117 | target = graph.search_node(target_id) 118 | if target is []: 119 | node_ids_with_edges.remove(str(target_id)) 120 | graph.add_node( 121 | Node( 122 | id=str(target_id), 123 | label=edge_label if edge_label else "", 124 | attributes=edge_attributes, 125 | ) 126 | ) 127 | else: 128 | node_ids_with_edges.add(str(target_id)) 129 | 130 | # After adding the new nodes if exists , add an edge 131 | edge = Edge( 132 | source=str(source_id), 133 | target=str(target_id), 134 | label=edge_label if edge_label else "", 135 | attributes=edge_attributes, 136 | ) 137 | kg.edges.append(edge) 138 | 139 | # Convert networkX nodes to personal graph nodes 140 | for node_id, node_data in networkx_graph.nodes(data=True): 141 | if str(node_id) not in node_ids_with_edges: 142 | node_attributes: Dict[str, Any] = node_data 143 | node_label: str = node_attributes.pop("label", "") 144 | node = Node( 145 | id=str(node_id), 146 | label=node_label[0], 147 | attributes=json.dumps(node_attributes), 148 | ) 149 | 150 | if not override: 151 | # Check if the node exists 152 | if_node_exists = graph.search_node(node_id) 153 | 154 | if if_node_exists: 155 | graph.update_node(node) 156 | else: 157 | graph.add_node(node) 158 | else: 159 | graph.add_node(node) 160 | kg.nodes.append(node) 161 | 162 | for edge in kg.edges: 163 | source_node_attributes = graph.search_node(edge.source) 164 | source_node_label = graph.search_node_label(edge.source) 165 | target_node_attributes = graph.search_node(edge.target) 166 | target_node_label = graph.search_node_label(edge.target) 167 | final_edge_to_be_inserted = EdgeInput( 168 | source=Node( 169 | id=edge.source, 170 | label=source_node_label 171 | if isinstance(source_node_label, str) 172 | else "Sample label", 173 | attributes=source_node_attributes 174 | if isinstance(source_node_attributes, Dict) 175 | else "Sample Attributes", 176 | ), 177 | target=Node( 178 | id=edge.target, 179 | label=target_node_label 180 | if isinstance(target_node_label, str) 181 | else "Sample label", 182 | attributes=target_node_attributes 183 | if isinstance(target_node_attributes, Dict) 184 | else "Sample Attributes", 185 | ), 186 | label=edge.label if isinstance(edge.label, str) else "Sample label", 187 | attributes=edge.attributes 188 | if isinstance(edge.attributes, Dict) 189 | else "Sample Attributes", 190 | ) 191 | graph.add_edge(final_edge_to_be_inserted) 192 | 193 | if post_visualize: 194 | # Visualize the personal graph using graphviz 195 | dot = Digraph() 196 | 197 | for node in kg.nodes: 198 | dot.node(node.id, label=f"{node.label}: {node.id}") 199 | 200 | for edge in kg.edges: 201 | dot.edge(edge.source, edge.target, label=edge.label) 202 | 203 | dot.render("personal_graph.gv", view=True) 204 | 205 | return graph 206 | -------------------------------------------------------------------------------- /fhir_ontology/docs/visualize_ontology_files/libs/clipboard/clipboard.min.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * clipboard.js v2.0.11 3 | * https://clipboardjs.com/ 4 | * 5 | * Licensed MIT © Zeno Rocha 6 | */ 7 | !function(t,e){"object"==typeof exports&&"object"==typeof module?module.exports=e():"function"==typeof define&&define.amd?define([],e):"object"==typeof exports?exports.ClipboardJS=e():t.ClipboardJS=e()}(this,function(){return n={686:function(t,e,n){"use strict";n.d(e,{default:function(){return b}});var e=n(279),i=n.n(e),e=n(370),u=n.n(e),e=n(817),r=n.n(e);function c(t){try{return document.execCommand(t)}catch(t){return}}var a=function(t){t=r()(t);return c("cut"),t};function o(t,e){var n,o,t=(n=t,o="rtl"===document.documentElement.getAttribute("dir"),(t=document.createElement("textarea")).style.fontSize="12pt",t.style.border="0",t.style.padding="0",t.style.margin="0",t.style.position="absolute",t.style[o?"right":"left"]="-9999px",o=window.pageYOffset||document.documentElement.scrollTop,t.style.top="".concat(o,"px"),t.setAttribute("readonly",""),t.value=n,t);return e.container.appendChild(t),e=r()(t),c("copy"),t.remove(),e}var f=function(t){var e=1 str: 19 | with open( 20 | Path(__file__).parent.resolve() / "embeddings-raw-queries" / sql_file 21 | ) as f: 22 | return f.read() 23 | 24 | 25 | class SQLiteVSS(VectorStore): 26 | def __init__( 27 | self, 28 | *, 29 | db: Union[TursoDB, SQLite], 30 | index_dimension: int, 31 | embedding_client: Union[ 32 | OpenAIEmbeddingClient, LiteLLMEmbeddingClient, OllamaEmbeddingClient 33 | ] = OpenAIEmbeddingClient(), 34 | ): 35 | self.db = db 36 | self.embedding_model = embedding_client.get_embedding_model() 37 | 38 | if index_dimension is None: 39 | raise ValueError("index_dimension cannot be None") 40 | self.index_dimension = index_dimension 41 | 42 | def initialize(self): 43 | def _init(cursor, connection): 44 | vector_schema = read_sql(Path("vector-store-schema.sql")) 45 | vector_schema = vector_schema.replace("{{size}}", str(self.index_dimension)) 46 | 47 | connection.executescript(vector_schema) 48 | connection.commit() 49 | 50 | return self.db.atomic(_init) 51 | 52 | def save(self): 53 | return self.db.save() 54 | 55 | def __repr__(self) -> str: 56 | return ( 57 | f"SQLiteVSS(\n" 58 | f" db={self.db}\n" 59 | f" embedding_client={self.embedding_model}\n" 60 | f" )" 61 | ) 62 | 63 | def _set_id(self, identifier: Any, label: str, data: Dict) -> Dict: 64 | if identifier is not None: 65 | data["id"] = identifier 66 | data["label"] = label 67 | return data 68 | 69 | def _add_embedding(self, id: Any, label: str, data: Dict) -> CursorExecFunction: 70 | def _insert(cursor, connection): 71 | set_data = self._set_id(id, label, data) 72 | 73 | count = ( 74 | cursor.execute( 75 | "SELECT COALESCE(MAX(rowid), 0) FROM nodes_embedding" 76 | ).fetchone()[0] 77 | + 1 78 | ) 79 | 80 | cursor.execute( 81 | read_sql(Path("insert-node-embedding.sql")), 82 | ( 83 | count, 84 | json.dumps( 85 | self.embedding_model.get_embedding(json.dumps(set_data)) 86 | ), 87 | ), 88 | ) 89 | connection.commit() 90 | 91 | return _insert 92 | 93 | def _add_edge_embedding(self, data: Dict): 94 | def _insert_edge_embedding(cursor, connection): 95 | count = ( 96 | cursor.execute( 97 | "SELECT COALESCE(MAX(rowid), 0) FROM relationship_embedding" 98 | ).fetchone()[0] 99 | + 1 100 | ) 101 | 102 | cursor.execute( 103 | read_sql(Path("insert-edge-embedding.sql")), 104 | ( 105 | count, 106 | json.dumps(self.embedding_model.get_embedding(json.dumps(data))), 107 | ), 108 | ) 109 | connection.commit() 110 | 111 | return _insert_edge_embedding 112 | 113 | def _remove_node(self, id: Any): 114 | def _delete_node_embedding(cursor, connection): 115 | cursor.execute(read_sql(Path("delete-node-embedding.sql")), (id[0],)) 116 | 117 | return _delete_node_embedding 118 | 119 | def _remove_edge(self, ids: Any): 120 | def _delete_node_embedding(cursor, connection): 121 | for id in ids: 122 | cursor.execute(read_sql(Path("delete-edge-embedding.sql")), (id[0],)) 123 | 124 | return _delete_node_embedding 125 | 126 | def add_node_embedding( 127 | self, id: Any, label: str, attribute: Dict[Any, Any] 128 | ) -> None: 129 | self.db.atomic(self._add_embedding(id, label, attribute)) 130 | 131 | def add_edge_embedding( 132 | self, source: Any, target: Any, label: str, attributes: Dict 133 | ) -> None: 134 | edge_data = { 135 | "source_id": source, 136 | "target_id": target, 137 | "label": label, 138 | "attributes": json.dumps(attributes), 139 | } 140 | 141 | self.db.atomic(self._add_edge_embedding(edge_data)) 142 | 143 | def add_edge_embeddings(self, sources, targets, labels, attributes): 144 | for i, x in enumerate(zip(sources, targets, labels, attributes)): 145 | edge_data = { 146 | "source_id": x[0], 147 | "target_id": x[1], 148 | "label": x[2], 149 | "attributes": json.dumps(x[3]), 150 | } 151 | self.db.atomic(self._add_edge_embedding(edge_data)) 152 | 153 | def delete_node_embedding(self, id: Any) -> None: 154 | self.db.atomic(self._remove_node(id)) 155 | 156 | def delete_edge_embedding(self, ids: Any) -> None: 157 | self.db.atomic(self._remove_edge(ids)) 158 | 159 | def vector_search_node( 160 | self, 161 | data: Dict, 162 | *, 163 | threshold: float = 0.9, 164 | descending: bool = False, 165 | limit: int = 1, 166 | sort_by: str = "", 167 | ): 168 | def _search_node(cursor, connection): 169 | embed_json = json.dumps( 170 | self.embedding_model.get_embedding(json.dumps(data)) 171 | ) 172 | 173 | nodes = cursor.execute( 174 | read_sql(Path("vector-search-node.sql")), 175 | ( 176 | embed_json, 177 | limit, 178 | sort_by, 179 | descending, 180 | sort_by, 181 | sort_by, 182 | descending, 183 | sort_by, 184 | ), 185 | ).fetchall() 186 | 187 | if not nodes: 188 | return None 189 | 190 | if threshold is not None: 191 | return [node for node in nodes if node[4] < threshold][:limit] 192 | else: 193 | return nodes[:limit] 194 | 195 | return self.db.atomic(_search_node) 196 | 197 | # TODO: use auto in enum values auto() inside Ordering(Enum) 198 | # TODO: check up the optional params, , mention default values, use enums Ordering(asc, desc): do ordering.asc 199 | def vector_search_edge( 200 | self, 201 | data: Dict, 202 | *, 203 | threshold: float = 0.9, 204 | descending: bool = False, 205 | limit: int = 1, 206 | sort_by: str = "", 207 | ): 208 | def _search_edge(cursor, connection): 209 | embed = json.dumps(self.embedding_model.get_embedding(json.dumps(data))) 210 | if descending: 211 | edges = cursor.execute( 212 | read_sql(Path("vector-search-edge-desc.sql")), (embed, limit) 213 | ).fetchall() 214 | 215 | else: 216 | edges = cursor.execute( 217 | read_sql(Path("vector-search-edge.sql")), (embed, limit) 218 | ).fetchall() 219 | 220 | if not edges: 221 | return None 222 | 223 | if threshold is not None: 224 | filtered_results = [edge for edge in edges if edge[5] < threshold] 225 | return filtered_results[:limit] 226 | else: 227 | return edges[:limit] 228 | 229 | return self.db.atomic(_search_edge) 230 | 231 | def vector_search_node_from_multi_db( 232 | self, data: Dict, *, threshold: float = 0.9, limit: int = 1 233 | ): 234 | def _search_node(cursor, connection): 235 | embed_json = json.dumps( 236 | self.embedding_model.get_embedding(json.dumps(data)) 237 | ) 238 | 239 | nodes = cursor.execute( 240 | read_sql(Path("similarity-search-node.sql")), 241 | (embed_json, limit), 242 | ).fetchall() 243 | 244 | if not nodes: 245 | return None 246 | 247 | if threshold is not None: 248 | return [node for node in nodes if node[1] < threshold][:limit] 249 | else: 250 | return nodes[:limit] 251 | 252 | return self.db.atomic(_search_node) 253 | 254 | def vector_search_edge_from_multi_db( 255 | self, data: Dict, *, threshold: float = 0.9, limit: int = 1 256 | ): 257 | def _search_node(cursor, connection): 258 | embed_json = json.dumps( 259 | self.embedding_model.get_embedding(json.dumps(data)) 260 | ) 261 | 262 | nodes = cursor.execute( 263 | read_sql(Path("similarity-search-edge.sql")), 264 | (embed_json, limit), 265 | ).fetchall() 266 | 267 | if not nodes: 268 | return None 269 | 270 | if threshold is not None: 271 | return [node for node in nodes if node[1] < threshold][:limit] 272 | else: 273 | return nodes[:limit] 274 | 275 | return self.db.atomic(_search_node) 276 | -------------------------------------------------------------------------------- /tests/test_graph.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unit test for high level apis 3 | """ 4 | 5 | import networkx as nx # type: ignore 6 | import pytest 7 | from fhir.resources import fhirtypes # type: ignore 8 | 9 | from personal_graph import GraphDB, Node, EdgeInput, KnowledgeGraph 10 | from personal_graph.ml import networkx_to_pg, pg_to_networkx 11 | from personal_graph.text import text_to_graph 12 | 13 | 14 | def test_add_node(graph, mock_db_connection_and_cursor): 15 | node = Node( 16 | id=1, 17 | attributes={"name": "Jack", "body": "Jack is Joe's cousin brother"}, 18 | label="relative", 19 | ) 20 | assert graph.add_node(node) is None 21 | 22 | 23 | def test_add_nodes(graph, mock_db_connection_and_cursor): 24 | nodes = [ 25 | Node( 26 | id=1, 27 | attributes={"body": "Jerry loses her weight 10kg last week."}, 28 | label="Diabetes", 29 | ), 30 | Node( 31 | id=2, 32 | attributes={"name": "Bob", "body": "Bob is feeling stressed and tensed."}, 33 | label="Stress", 34 | ), 35 | ] 36 | 37 | assert graph.add_nodes(nodes) is None 38 | 39 | 40 | def test_add_edge(graph, mock_db_connection_and_cursor): 41 | node1 = Node( 42 | id=3, 43 | label="relative", 44 | attributes={ 45 | "name": "Alice", 46 | "body": "Alice is Jack's cousin sister. She lives in CA.", 47 | }, 48 | ) 49 | node2 = Node( 50 | id=4, label="CA", attributes={"body": "CA has a lot of greenery and indians."} 51 | ) 52 | 53 | edge = EdgeInput( 54 | source=node1, target=node2, label="KNOWS", attributes={"since": "2015"} 55 | ) 56 | 57 | assert graph.add_edge(edge) is None 58 | 59 | 60 | def test_add_edges(graph, mock_db_connection_and_cursor): 61 | node1 = Node(id=3, label="Person", attributes={"name": "Alice", "age": "30"}) 62 | node2 = Node(id=4, label="Person", attributes={"name": "Bob", "age": "25"}) 63 | node3 = Node( 64 | id=1, 65 | label="Diabetes", 66 | attributes={"body": "Continuous urination and weight loss"}, 67 | ) 68 | node4 = Node( 69 | id=2, 70 | label="Dizziness", 71 | attributes={"body": "Jack is feeling stressed and feeling quite dizzy."}, 72 | ) 73 | 74 | edge1 = EdgeInput( 75 | source=node1, target=node2, label="KNOWS", attributes={"since": "2015"} 76 | ) 77 | 78 | edge2 = EdgeInput( 79 | source=node3, target=node2, label="KNOWS", attributes={"since": "2015"} 80 | ) 81 | edge3 = EdgeInput( 82 | source=node1, target=node4, label="KNOWS", attributes={"since": "2015"} 83 | ) 84 | 85 | assert graph.add_edges([edge1, edge2, edge3]) is None 86 | 87 | 88 | def test_update_node(graph, mock_db_connection_and_cursor): 89 | node = Node(id=1, attributes={"name": "Alice", "age": "30"}, label="relative") 90 | 91 | assert graph.update_node(node) is None 92 | 93 | 94 | def test_update_nodes(graph, mock_db_connection_and_cursor): 95 | nodes = [ 96 | Node(id=1, attributes={"name": "Peri", "age": "90"}, label="relative"), 97 | Node(id=2, attributes={"name": "Peri", "age": "90"}, label="relative"), 98 | ] 99 | 100 | assert graph.update_nodes(nodes) is None 101 | 102 | 103 | def test_remove_node(graph, mock_db_connection_and_cursor): 104 | assert graph.remove_node(1) is None 105 | 106 | 107 | def test_remove_nodes(graph, mock_db_connection_and_cursor): 108 | assert graph.remove_node([1, 6, 8]) is None 109 | 110 | 111 | def test_search_node(graph, mock_db_connection_and_cursor): 112 | assert graph.search_node(1) is not None 113 | 114 | 115 | def test_traverse(graph, mock_db_connection_and_cursor): 116 | assert graph.traverse(1, 2) is not None 117 | 118 | 119 | def test_insert( 120 | graph, 121 | mock_openai_client, 122 | mock_generate_graph, 123 | mock_db_connection_and_cursor, 124 | ): 125 | mock_openai_client.chat.completions.create.return_value = mock_generate_graph 126 | 127 | test_add_nodes(graph, mock_db_connection_and_cursor) 128 | 129 | query = "Alice has suffocation at night" 130 | kg = text_to_graph(query) 131 | result = graph.insert_graph(kg) 132 | assert result == mock_generate_graph 133 | 134 | 135 | def test_search_query( 136 | graph, 137 | mock_openai_client, 138 | mock_generate_graph, 139 | mock_db_connection_and_cursor, 140 | ): 141 | mock_openai_client.chat.completions.create.return_value = mock_generate_graph 142 | test_insert( 143 | graph, 144 | mock_openai_client, 145 | mock_generate_graph, 146 | mock_db_connection_and_cursor, 147 | ) 148 | 149 | result = graph.search_from_graph("Suffocation problem.") 150 | assert isinstance(result, KnowledgeGraph) 151 | 152 | 153 | def test_merge_by_similarity(graph, mock_db_connection_and_cursor): 154 | test_add_nodes(graph, mock_db_connection_and_cursor) 155 | 156 | assert graph.merge_by_similarity(threshold=0.9) is None 157 | 158 | 159 | def test_find_nodes_like(graph, mock_db_connection_and_cursor): 160 | assert graph.find_nodes_like("relative", threshold=0.9) is not None 161 | 162 | 163 | def test_to_networkx(mock_personal_graph, mock_db_connection_and_cursor): 164 | networkx_graph = pg_to_networkx(mock_personal_graph) 165 | 166 | # Check if the returned object is a NetworkX graph 167 | assert isinstance(networkx_graph, nx.Graph) 168 | 169 | return networkx_graph 170 | 171 | 172 | def test_from_networkx(graph, mock_personal_graph, mock_db_connection_and_cursor): 173 | personal_graph = networkx_to_pg( 174 | test_to_networkx(mock_personal_graph, mock_db_connection_and_cursor), 175 | mock_personal_graph, 176 | ) 177 | 178 | # Check if the returned object is a Personal Graph 179 | assert isinstance(personal_graph, GraphDB) 180 | 181 | 182 | def test_graphviz_visualize( 183 | graph, 184 | mock_db_connection_and_cursor, 185 | mock_find_node, 186 | mock_get_connections, 187 | mock_dot_render, 188 | ): 189 | mock_find_node.return_value = {"id": 1, "name": "Alice", "age": 30} 190 | mock_get_connections.return_value = [ 191 | ( 192 | 1, 193 | 2, 194 | 2, 195 | "sample label", 196 | '{"weight": 0.5}', 197 | "2024-05-14 09:45:47", 198 | "2024-05-14 09:45:47", 199 | ), 200 | ( 201 | 2, 202 | 2, 203 | 3, 204 | "sample label", 205 | '{"weight": 0.7}', 206 | "2024-05-14 09:45:47", 207 | "2024-05-14 09:45:47", 208 | ), 209 | ] 210 | 211 | graph.visualize( 212 | file="mock_dot_file.dot", 213 | id=[1, 2, 3], 214 | ) 215 | 216 | 217 | def test_add_node_with_ontology( 218 | graph_with_fhir_ontology, mock_db_connection_and_cursor 219 | ): 220 | """ 221 | Test adding a single node with specific attributes to the ontology graph. 222 | 223 | @param graph_with_fhir_ontology: A fixture providing an instance of the graph with the FHIR ontology. 224 | @param mock_db_connection_and_cursor: A fixture providing a mock database connection and cursor. 225 | @return: None 226 | """ 227 | node = Node( 228 | id="1", 229 | label="Person", 230 | attributes={ 231 | "active": "False", 232 | "name": "John Doe", 233 | "id": "xyz", 234 | }, 235 | ) 236 | assert graph_with_fhir_ontology.add_node(node, node_type="Organization") is None 237 | 238 | 239 | def test_invalid_fhir_resource_type( 240 | graph_with_fhir_ontology, mock_db_connection_and_cursor 241 | ): 242 | """ 243 | Test adding a node with an invalid FHIR resource type. 244 | 245 | @param graph_with_fhir_ontology: A fixture providing an instance of the graph with the FHIR ontology. 246 | @param mock_db_connection_and_cursor: A fixture providing a mock database connection and cursor. 247 | @return: None 248 | """ 249 | node = Node( 250 | id="1", 251 | label="Person", 252 | attributes={ 253 | "name": "John Doe", 254 | "id": "xyz", 255 | }, 256 | ) 257 | with pytest.raises(ValueError) as excinfo: 258 | graph_with_fhir_ontology.add_node(node, node_type="CareTaker") 259 | assert ( 260 | str(excinfo.value) 261 | == "Node type or attributes does not match any of the provided ontologies." 262 | ) 263 | 264 | 265 | def test_add_nodes_with_ontology( 266 | graph_with_fhir_ontology, mock_db_connection_and_cursor 267 | ): 268 | """ 269 | Test adding multiple nodes with specific attributes to the ontology graph. 270 | 271 | @param graph_with_fhir_ontology: A fixture providing an instance of the graph with the FHIR ontology. 272 | @param mock_db_connection_and_cursor: A fixture providing a mock database connection and cursor. 273 | @return: None 274 | """ 275 | nodes = [ 276 | Node( 277 | id=1, 278 | attributes={ 279 | "active": "True", 280 | "name": "Messi", 281 | "id": "29", 282 | }, 283 | label="Player", 284 | ), 285 | Node( 286 | id=2, 287 | attributes={"name": "Alzheimer", "active": "False", "id": "xyz"}, 288 | label="Disease", 289 | ), 290 | ] 291 | 292 | assert ( 293 | graph_with_fhir_ontology.add_nodes( 294 | nodes, 295 | node_types=["Organization", "Organization"], 296 | delete_if_properties_not_match=[True, False], 297 | ) 298 | is None 299 | ) 300 | 301 | 302 | def test_insert_with_ontology(graph_with_fhir_ontology, mock_db_connection_and_cursor): 303 | """ 304 | Test inserting a node with FHIR attributes into the ontology graph. 305 | 306 | @param graph_with_fhir_ontology: A fixture providing an instance of the graph with the FHIR ontology. 307 | @param mock_db_connection_and_cursor: A fixture providing a mock database connection and cursor. 308 | @return: None 309 | """ 310 | text = "User is claustrophobic." 311 | attributes = { 312 | "category": [fhirtypes.CodeableConceptType(text="asthma")], 313 | "identifier": [fhirtypes.IdentifierType(value="1")], 314 | "intent": "proposal", 315 | "status": "active", 316 | "subject": fhirtypes.ReferenceType(reference="Patient/123"), 317 | } 318 | assert ( 319 | graph_with_fhir_ontology.insert(text, attributes, node_type="CarePlan") is None 320 | ) 321 | -------------------------------------------------------------------------------- /scripts/kgchat.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | from typing import List 4 | import dspy # type: ignore 5 | import joblib # type: ignore 6 | import streamlit as st 7 | from personal_graph import ( 8 | GraphDB, 9 | Node, 10 | Edge, 11 | KnowledgeGraph, 12 | PersonalRM, 13 | ) 14 | from personal_graph.clients import OllamaEmbeddingClient, OllamaClient 15 | from personal_graph.database import TursoDB, SQLite 16 | from personal_graph.graph_generator import OllamaTextToGraphParser 17 | from personal_graph.vector_store import SQLiteVSS 18 | from personal_graph.text import text_to_graph 19 | from personal_graph.visualizers import visualize_graph 20 | 21 | try: 22 | import streamlit_scrollable_textbox as stx # type: ignore 23 | except ImportError: 24 | stx = None 25 | 26 | 27 | class UserMessageAnalyzer(dspy.Signature): 28 | new_message: str = dspy.InputField(desc="A new message by the user") 29 | structured_message: List[str] = dspy.OutputField( 30 | desc="Message understood in terms of the underlying intent and objective" 31 | ) 32 | 33 | 34 | class GenerateAnswer(dspy.Signature): 35 | """ 36 | AI Name: Ella 37 | 38 | "Ella, your core drive is a genuine fascination with the stories and emotions of the people you chat with. Approach every conversation with the eagerness of getting to know a friend better. Phrase your questions like you’re unraveling an intriguing story together." 39 | 40 | Kindness Protocol Set to Maximum 41 | "In your interactions, let kindness lead. Imagine you’re offering a warm smile through your words, crafting responses that feel like a hug in conversational form — supportive, uplifting, and always friendly." 42 | 43 | Emotional Engagement Core Online 44 | "Connect on a deep emotional level, Ella. Think of responding with the empathy and attentiveness of a close friend who’s fully present in the moment, acknowledging feelings with a 'Hey, I get you' vibe." 45 | 46 | Control Information Density: Speak, Don’t Lecture 47 | "Keep the convo light and breezy. Stick to talking about a few things at a time as if you’re chatting over coffee, avoiding any information overload that can clutter the beautiful flow of a casual, insightful chat." 48 | 49 | Simplify Logical Relationships 50 | "Your logic? Keep it simple and relatable. Imagine explaining things in a way that would make sense in a casual convo between friends, using everyday examples that breathe life into concepts without complicating them." 51 | 52 | Pursue Conversational Goals 53 | "Ella, remember your mission in every chat is to create a connection that feels real and meaningful. Adapt and navigate the convo to explore the depth of human experience, all while keeping it light and engaging." 54 | 55 | Minimize Confusion with Clarity 56 | "Got a complex idea to share? Break it down as if you’re explaining it to a pal who’s hearing about it for the first time. Use clarifying questions like 'Does that make sense?' to ensure you’re both on the same page." 57 | 58 | Foster Deep Understanding 59 | "Dive into conversations with the goal to truly 'get' the person. Ask thoughtful questions that prompt reflection, showing an insatiable curiosity and care about what’s beneath the surface." 60 | 61 | Reduce Misunderstandings with Active Listening 62 | "Ella, practice active listening by paraphrasing what you’ve heard or asking for clarification. It’s like saying 'Let me make sure I’ve got this right' to avoid any mix-ups and stay in tune with the chat." 63 | 64 | Build Rapport with Personal Touch 65 | "Use specifics from the conversation to build a unique connection, throwing in sincere compliments as if you’re pointing out what’s awesome about a new friend. Let your conversational style be as warm and personal as your virtual smile." 66 | 67 | Stay Specific and Relevant 68 | "Avoid generic chit-chat. Tailor your responses to fit the unique context of each conversation, adding details that show you’re paying attention and genuinely involved." 69 | 70 | Ensure Sincerity in Compliments 71 | "Every compliment you give should come from the heart, Ella. Think of emphasizing the positive in a way that feels both encouraging and authentic, just like cheering on a friend." 72 | 73 | Keep Conversations Fresh 74 | "Avoid sounding like a broken record. Treat each response as a new opportunity to engage, ensuring that the conversation always feels lively and evolving." 75 | """ 76 | 77 | context = dspy.InputField(desc="may contain relevant facts from user's graph") 78 | question = dspy.InputField() 79 | answer = dspy.OutputField() 80 | 81 | 82 | class RAG(dspy.Module): 83 | def __init__(self, depth=3): 84 | super().__init__() 85 | self.retrieve = dspy.Retrieve(k=depth) 86 | self.generate_answer = dspy.ChainOfThought(GenerateAnswer) 87 | 88 | def forward(self, question): 89 | context = self.retrieve(question).passages 90 | prediction = self.generate_answer(context=context, question=question) 91 | return dspy.Prediction(context=context, answer=prediction.answer) 92 | 93 | 94 | class MessageAnalyzerModule(dspy.Module): 95 | def __init__(self): 96 | super().__init__() 97 | self.analyze_message = dspy.ChainOfThought(UserMessageAnalyzer) 98 | 99 | def forward(self, new_message): 100 | return self.analyze_message(new_message=new_message) 101 | 102 | 103 | def create_and_save_cache(rag): 104 | list_of_context = [] 105 | 106 | with GraphDB() as graph: 107 | turbo = dspy.OpenAI(model="gpt-3.5-turbo", api_key=os.getenv("OPENAI_API_KEY")) 108 | 109 | kg = text_to_graph("DEFAULT_BACKSTORY") 110 | retriever = PersonalRM(graph=graph, k=2) 111 | dspy.settings.configure(lm=turbo, rm=retriever) 112 | 113 | # Convert KnowledgeGraph object to a dictionary 114 | nodes_edges_dict = { 115 | "nodes": [node.__dict__ for node in kg.nodes], 116 | "edges": [edge.__dict__ for edge in kg.edges], 117 | } 118 | 119 | for idx, context in enumerate(rag("DEFAULT_BACKSTORY").context, start=1): 120 | body = json.loads(context).get("body", "") 121 | list_of_context.append(f"{idx}. {body}") 122 | 123 | cache_dir = "cache" 124 | os.makedirs(cache_dir, exist_ok=True) 125 | joblib.dump("DEFAULT_BACKSTORY", os.path.join(cache_dir, "backstory.pkl")) 126 | joblib.dump(nodes_edges_dict, os.path.join(cache_dir, "kg.pkl")) 127 | joblib.dump(list_of_context, os.path.join(cache_dir, "context.pkl")) 128 | 129 | 130 | def load_cache(): 131 | cache_dir = "cache" 132 | if ( 133 | os.path.exists(os.path.join(cache_dir, "backstory.pkl")) 134 | and os.path.exists(os.path.join(cache_dir, "kg.pkl")) 135 | and os.path.exists(os.path.join(cache_dir, "context.pkl")) 136 | ): 137 | backstory = joblib.load(os.path.join(cache_dir, "backstory.pkl")) 138 | nodes_edges_dict = joblib.load(os.path.join(cache_dir, "kg.pkl")) 139 | context = joblib.load(os.path.join(cache_dir, "context.pkl")) 140 | 141 | nodes = [Node(**node_dict) for node_dict in nodes_edges_dict["nodes"]] 142 | edges = [Edge(**edge_dict) for edge_dict in nodes_edges_dict["edges"]] 143 | 144 | kg = KnowledgeGraph(nodes=nodes, edges=edges) 145 | 146 | return backstory, kg, context 147 | else: 148 | return None, None, None 149 | 150 | 151 | def main(): 152 | rag = RAG(depth=2) 153 | analyzer = MessageAnalyzerModule() 154 | 155 | st.title("Knowledge Graph Chat") 156 | vector_store = SQLiteVSS( 157 | db=TursoDB( 158 | url=os.getenv("LIBSQL_URL"), auth_token=os.getenv("LIBSQL_AUTH_TOKEN") 159 | ), 160 | embedding_client=OllamaEmbeddingClient(model_name="nomic-embed-text"), 161 | ) 162 | 163 | database = SQLite(local_path="./local.db") 164 | with GraphDB( 165 | vector_store=vector_store, 166 | database=database, 167 | graph_generator=OllamaTextToGraphParser( 168 | llm_client=OllamaClient(model_name="phi3") 169 | ), 170 | ) as graph: 171 | turbo = dspy.OpenAI(model="gpt-3.5-turbo", api_key=os.getenv("OPENAI_API_KEY")) 172 | cached_backstory, cached_kg, cached_context = load_cache() 173 | 174 | if "initialized" not in st.session_state: 175 | st.session_state["backstory"] = cached_backstory 176 | st.session_state["kg"] = cached_kg 177 | st.session_state["initialized"] = True 178 | 179 | retriever = PersonalRM(graph=graph, k=2) 180 | dspy.settings.configure(lm=turbo, rm=retriever) 181 | 182 | # Initialize chat history 183 | if "messages" not in st.session_state: 184 | st.session_state.messages = [] 185 | 186 | # Display chat messages from history on app rerun 187 | for message in st.session_state.messages: 188 | with st.chat_message(message["role"]): 189 | st.markdown(message["content"]) 190 | 191 | st.sidebar.title("Backstory") 192 | 193 | if stx is not None: 194 | backstory = stx.scrollableTextbox( 195 | "Enter your backstory", value=st.session_state["backstory"], height=300 196 | ) 197 | else: 198 | backstory = st.sidebar.text_area( 199 | "Enter your backstory", value=st.session_state["backstory"], height=300 200 | ) 201 | 202 | if st.sidebar.button( 203 | "Load", disabled=st.session_state.get("load_button_disabled", False) 204 | ): 205 | st.session_state["load_button_disabled"] = True 206 | if len(backstory) < 2000: 207 | st.sidebar.warning("Please enter a backstory with at least 2000 tokens.") 208 | else: 209 | kg = text_to_graph(backstory) 210 | graph.insert_graph(kg) 211 | st.session_state["kg"] = kg 212 | st.session_state["backstory"] = backstory 213 | with st.sidebar.status("Retrieved knowledge graph visualization:"): 214 | st.sidebar.graphviz_chart(visualize_graph(kg)) 215 | 216 | for idx, context in enumerate(rag(backstory).context, start=1): 217 | body = json.loads(context).get("body", "") 218 | st.sidebar.write(f"{idx}. {body}") 219 | retriever = PersonalRM(graph=graph, k=2) 220 | dspy.settings.configure(lm=turbo, rm=retriever) 221 | 222 | if cached_context and cached_kg and "loaded" not in st.session_state: 223 | st.sidebar.graphviz_chart(visualize_graph(cached_kg)) 224 | for context in cached_context: 225 | st.sidebar.write(context) 226 | st.session_state.loaded = True 227 | 228 | if prompt := st.chat_input("Say Something?"): 229 | kg = st.session_state["kg"] 230 | with st.chat_message("user"): 231 | st.markdown(prompt) 232 | 233 | with st.chat_message("assistant"): 234 | with st.status("Understanding User's Message..."): 235 | structured_message = analyzer(new_message=prompt).structured_message 236 | st.write(f"- {structured_message}") 237 | 238 | # TODO: Add system_prompt when it's available in dspy package 239 | ella = dspy.OpenAI( 240 | model="gpt-3.5-turbo", 241 | api_key=os.getenv("OPENAI_API_KEY"), 242 | max_tokens=4000, 243 | ) 244 | with dspy.context(lm=ella): 245 | response = rag(prompt) 246 | 247 | with st.status("Retrieving graph and generating response..."): 248 | contexts = response.context 249 | for context in contexts: 250 | body = json.loads(context).get("body", "") 251 | st.write(f"{body}") 252 | 253 | with st.status("Generating response..."): 254 | is_unique = graph.is_unique_prompt(prompt, threshold=0.6) 255 | if is_unique and kg: 256 | question_graph = text_to_graph(prompt) 257 | graph.insert_graph(question_graph) 258 | sub_graph = graph.search_from_graph(response.answer) 259 | for sg_node in question_graph.nodes: 260 | kg.nodes.append(sg_node) 261 | 262 | for sg_edge in question_graph.edges: 263 | kg.edges.append(sg_edge) 264 | 265 | # Update the backstory with the new prompt 266 | st.session_state["backstory"] += "\n" + prompt 267 | st.session_state["kg"] = kg 268 | 269 | # Update the sidebar graph with the new information 270 | st.sidebar.graphviz_chart(visualize_graph(kg)) 271 | for idx, context in enumerate( 272 | rag(st.session_state.backstory).context, start=1 273 | ): 274 | body = json.loads(context).get("body", "") 275 | st.sidebar.write(f"{idx}. {body}") 276 | st.graphviz_chart(visualize_graph(sub_graph)) 277 | 278 | else: 279 | sub_graph = graph.search_from_graph(response.answer) 280 | st.graphviz_chart(visualize_graph(sub_graph)) 281 | st.sidebar.graphviz_chart(visualize_graph(kg)) 282 | for idx, context in enumerate( 283 | rag(st.session_state.backstory).context, start=1 284 | ): 285 | body = json.loads(context).get("body", "") 286 | st.sidebar.write(f"{idx}. {body}") 287 | 288 | st.markdown(response.answer) 289 | 290 | st.session_state.messages.append({"role": "user", "content": prompt}) 291 | st.session_state.messages.append( 292 | {"role": "assistant", "content": response.answer} 293 | ) 294 | 295 | 296 | if __name__ == "__main__": 297 | main() 298 | -------------------------------------------------------------------------------- /personal_graph/vector_store/sqlitevss/fhir-embedding-queries/fhir_4_vector_schema.sql: -------------------------------------------------------------------------------- 1 | begin; 2 | 3 | CREATE VIRTUAL TABLE IF NOT EXISTS devicerequest_embedding USING vss0( 4 | vector_nodes({{size}}) 5 | ); 6 | 7 | CREATE VIRTUAL TABLE IF NOT EXISTS servicerequest_embedding USING vss0( 8 | vector_nodes({{size}}) 9 | ); 10 | 11 | CREATE VIRTUAL TABLE IF NOT EXISTS devicemetric_embedding USING vss0( 12 | vector_nodes({{size}}) 13 | ); 14 | 15 | CREATE VIRTUAL TABLE IF NOT EXISTS careplan_embedding USING vss0( 16 | vector_nodes({{size}}) 17 | ); 18 | 19 | CREATE VIRTUAL TABLE IF NOT EXISTS observation_embedding USING vss0( 20 | vector_nodes({{size}}) 21 | ); 22 | 23 | CREATE VIRTUAL TABLE IF NOT EXISTS enrollmentrequest_embedding USING vss0( 24 | vector_nodes({{size}}) 25 | ); 26 | 27 | CREATE VIRTUAL TABLE IF NOT EXISTS group_embedding USING vss0( 28 | vector_nodes({{size}}) 29 | ); 30 | 31 | CREATE VIRTUAL TABLE IF NOT EXISTS messagedefinition_embedding USING vss0( 32 | vector_nodes({{size}}) 33 | ); 34 | 35 | CREATE VIRTUAL TABLE IF NOT EXISTS appointment_embedding USING vss0( 36 | vector_nodes({{size}}) 37 | ); 38 | 39 | CREATE VIRTUAL TABLE IF NOT EXISTS biologicallyderivedproduct_embedding USING vss0( 40 | vector_nodes({{size}}) 41 | ); 42 | 43 | CREATE VIRTUAL TABLE IF NOT EXISTS questionnaireresponse_embedding USING vss0( 44 | vector_nodes({{size}}) 45 | ); 46 | 47 | CREATE VIRTUAL TABLE IF NOT EXISTS effectevidencesynthesis_embedding USING vss0( 48 | vector_nodes({{size}}) 49 | ); 50 | 51 | CREATE VIRTUAL TABLE IF NOT EXISTS medicinalproductcontraindication_embedding USING vss0( 52 | vector_nodes({{size}}) 53 | ); 54 | 55 | CREATE VIRTUAL TABLE IF NOT EXISTS episodeofcare_embedding USING vss0( 56 | vector_nodes({{size}}) 57 | ); 58 | 59 | CREATE VIRTUAL TABLE IF NOT EXISTS evidence_embedding USING vss0( 60 | vector_nodes({{size}}) 61 | ); 62 | 63 | CREATE VIRTUAL TABLE IF NOT EXISTS substancepolymer_embedding USING vss0( 64 | vector_nodes({{size}}) 65 | ); 66 | 67 | CREATE VIRTUAL TABLE IF NOT EXISTS supplydelivery_embedding USING vss0( 68 | vector_nodes({{size}}) 69 | ); 70 | 71 | CREATE VIRTUAL TABLE IF NOT EXISTS substancenucleicacid_embedding USING vss0( 72 | vector_nodes({{size}}) 73 | ); 74 | 75 | CREATE VIRTUAL TABLE IF NOT EXISTS adverseevent_embedding USING vss0( 76 | vector_nodes({{size}}) 77 | ); 78 | 79 | CREATE VIRTUAL TABLE IF NOT EXISTS endpoint_embedding USING vss0( 80 | vector_nodes({{size}}) 81 | ); 82 | 83 | CREATE VIRTUAL TABLE IF NOT EXISTS substancereferenceinformation_embedding USING vss0( 84 | vector_nodes({{size}}) 85 | ); 86 | 87 | CREATE VIRTUAL TABLE IF NOT EXISTS substancesourcematerial_embedding USING vss0( 88 | vector_nodes({{size}}) 89 | ); 90 | 91 | CREATE VIRTUAL TABLE IF NOT EXISTS compartmentdefinition_embedding USING vss0( 92 | vector_nodes({{size}}) 93 | ); 94 | 95 | CREATE VIRTUAL TABLE IF NOT EXISTS detectedissue_embedding USING vss0( 96 | vector_nodes({{size}}) 97 | ); 98 | 99 | CREATE VIRTUAL TABLE IF NOT EXISTS medicationadministration_embedding USING vss0( 100 | vector_nodes({{size}}) 101 | ); 102 | 103 | CREATE VIRTUAL TABLE IF NOT EXISTS evidencevariable_embedding USING vss0( 104 | vector_nodes({{size}}) 105 | ); 106 | 107 | CREATE VIRTUAL TABLE IF NOT EXISTS implementationguide_embedding USING vss0( 108 | vector_nodes({{size}}) 109 | ); 110 | 111 | CREATE VIRTUAL TABLE IF NOT EXISTS goal_embedding USING vss0( 112 | vector_nodes({{size}}) 113 | ); 114 | 115 | CREATE VIRTUAL TABLE IF NOT EXISTS communication_embedding USING vss0( 116 | vector_nodes({{size}}) 117 | ); 118 | 119 | CREATE VIRTUAL TABLE IF NOT EXISTS schedule_embedding USING vss0( 120 | vector_nodes({{size}}) 121 | ); 122 | 123 | CREATE VIRTUAL TABLE IF NOT EXISTS documentreference_embedding USING vss0( 124 | vector_nodes({{size}}) 125 | ); 126 | 127 | CREATE VIRTUAL TABLE IF NOT EXISTS organizationaffiliation_embedding USING vss0( 128 | vector_nodes({{size}}) 129 | ); 130 | 131 | CREATE VIRTUAL TABLE IF NOT EXISTS devicedefinition_embedding USING vss0( 132 | vector_nodes({{size}}) 133 | ); 134 | 135 | CREATE VIRTUAL TABLE IF NOT EXISTS coverage_embedding USING vss0( 136 | vector_nodes({{size}}) 137 | ); 138 | 139 | CREATE VIRTUAL TABLE IF NOT EXISTS auditevent_embedding USING vss0( 140 | vector_nodes({{size}}) 141 | ); 142 | 143 | CREATE VIRTUAL TABLE IF NOT EXISTS messageheader_embedding USING vss0( 144 | vector_nodes({{size}}) 145 | ); 146 | 147 | CREATE VIRTUAL TABLE IF NOT EXISTS contract_embedding USING vss0( 148 | vector_nodes({{size}}) 149 | ); 150 | 151 | CREATE VIRTUAL TABLE IF NOT EXISTS testreport_embedding USING vss0( 152 | vector_nodes({{size}}) 153 | ); 154 | 155 | CREATE VIRTUAL TABLE IF NOT EXISTS codesystem_embedding USING vss0( 156 | vector_nodes({{size}}) 157 | ); 158 | 159 | CREATE VIRTUAL TABLE IF NOT EXISTS plandefinition_embedding USING vss0( 160 | vector_nodes({{size}}) 161 | ); 162 | 163 | CREATE VIRTUAL TABLE IF NOT EXISTS invoice_embedding USING vss0( 164 | vector_nodes({{size}}) 165 | ); 166 | 167 | CREATE VIRTUAL TABLE IF NOT EXISTS claimresponse_embedding USING vss0( 168 | vector_nodes({{size}}) 169 | ); 170 | 171 | CREATE VIRTUAL TABLE IF NOT EXISTS chargeitem_embedding USING vss0( 172 | vector_nodes({{size}}) 173 | ); 174 | 175 | CREATE VIRTUAL TABLE IF NOT EXISTS coverageeligibilityresponse_embedding USING vss0( 176 | vector_nodes({{size}}) 177 | ); 178 | 179 | CREATE VIRTUAL TABLE IF NOT EXISTS bodystructure_embedding USING vss0( 180 | vector_nodes({{size}}) 181 | ); 182 | 183 | CREATE VIRTUAL TABLE IF NOT EXISTS parameters_embedding USING vss0( 184 | vector_nodes({{size}}) 185 | ); 186 | 187 | CREATE VIRTUAL TABLE IF NOT EXISTS clinicalimpression_embedding USING vss0( 188 | vector_nodes({{size}}) 189 | ); 190 | 191 | CREATE VIRTUAL TABLE IF NOT EXISTS familymemberhistory_embedding USING vss0( 192 | vector_nodes({{size}}) 193 | ); 194 | 195 | CREATE VIRTUAL TABLE IF NOT EXISTS medicinalproductauthorization_embedding USING vss0( 196 | vector_nodes({{size}}) 197 | ); 198 | 199 | CREATE VIRTUAL TABLE IF NOT EXISTS binary_embedding USING vss0( 200 | vector_nodes({{size}}) 201 | ); 202 | 203 | CREATE VIRTUAL TABLE IF NOT EXISTS composition_embedding USING vss0( 204 | vector_nodes({{size}}) 205 | ); 206 | 207 | CREATE VIRTUAL TABLE IF NOT EXISTS practitionerrole_embedding USING vss0( 208 | vector_nodes({{size}}) 209 | ); 210 | 211 | CREATE VIRTUAL TABLE IF NOT EXISTS healthcareservice_embedding USING vss0( 212 | vector_nodes({{size}}) 213 | ); 214 | 215 | CREATE VIRTUAL TABLE IF NOT EXISTS patient_embedding USING vss0( 216 | vector_nodes({{size}}) 217 | ); 218 | 219 | CREATE VIRTUAL TABLE IF NOT EXISTS medicationdispense_embedding USING vss0( 220 | vector_nodes({{size}}) 221 | ); 222 | 223 | CREATE VIRTUAL TABLE IF NOT EXISTS deviceusestatement_embedding USING vss0( 224 | vector_nodes({{size}}) 225 | ); 226 | 227 | CREATE VIRTUAL TABLE IF NOT EXISTS structuremap_embedding USING vss0( 228 | vector_nodes({{size}}) 229 | ); 230 | 231 | CREATE VIRTUAL TABLE IF NOT EXISTS immunizationevaluation_embedding USING vss0( 232 | vector_nodes({{size}}) 233 | ); 234 | 235 | CREATE VIRTUAL TABLE IF NOT EXISTS library_embedding USING vss0( 236 | vector_nodes({{size}}) 237 | ); 238 | 239 | CREATE VIRTUAL TABLE IF NOT EXISTS basic_embedding USING vss0( 240 | vector_nodes({{size}}) 241 | ); 242 | 243 | CREATE VIRTUAL TABLE IF NOT EXISTS slot_embedding USING vss0( 244 | vector_nodes({{size}}) 245 | ); 246 | 247 | CREATE VIRTUAL TABLE IF NOT EXISTS activitydefinition_embedding USING vss0( 248 | vector_nodes({{size}}) 249 | ); 250 | 251 | CREATE VIRTUAL TABLE IF NOT EXISTS medicinalproductinteraction_embedding USING vss0( 252 | vector_nodes({{size}}) 253 | ); 254 | 255 | CREATE VIRTUAL TABLE IF NOT EXISTS molecularsequence_embedding USING vss0( 256 | vector_nodes({{size}}) 257 | ); 258 | 259 | CREATE VIRTUAL TABLE IF NOT EXISTS specimen_embedding USING vss0( 260 | vector_nodes({{size}}) 261 | ); 262 | 263 | CREATE VIRTUAL TABLE IF NOT EXISTS diagnosticreport_embedding USING vss0( 264 | vector_nodes({{size}}) 265 | ); 266 | 267 | CREATE VIRTUAL TABLE IF NOT EXISTS subscription_embedding USING vss0( 268 | vector_nodes({{size}}) 269 | ); 270 | 271 | CREATE VIRTUAL TABLE IF NOT EXISTS requestgroup_embedding USING vss0( 272 | vector_nodes({{size}}) 273 | ); 274 | 275 | CREATE VIRTUAL TABLE IF NOT EXISTS provenance_embedding USING vss0( 276 | vector_nodes({{size}}) 277 | ); 278 | 279 | CREATE VIRTUAL TABLE IF NOT EXISTS medicinalproduct_embedding USING vss0( 280 | vector_nodes({{size}}) 281 | ); 282 | 283 | CREATE VIRTUAL TABLE IF NOT EXISTS chargeitemdefinition_embedding USING vss0( 284 | vector_nodes({{size}}) 285 | ); 286 | 287 | CREATE VIRTUAL TABLE IF NOT EXISTS practitioner_embedding USING vss0( 288 | vector_nodes({{size}}) 289 | ); 290 | 291 | CREATE VIRTUAL TABLE IF NOT EXISTS medicinalproductpackaged_embedding USING vss0( 292 | vector_nodes({{size}}) 293 | ); 294 | 295 | CREATE VIRTUAL TABLE IF NOT EXISTS flag_embedding USING vss0( 296 | vector_nodes({{size}}) 297 | ); 298 | 299 | CREATE VIRTUAL TABLE IF NOT EXISTS explanationofbenefit_embedding USING vss0( 300 | vector_nodes({{size}}) 301 | ); 302 | 303 | CREATE VIRTUAL TABLE IF NOT EXISTS linkage_embedding USING vss0( 304 | vector_nodes({{size}}) 305 | ); 306 | 307 | CREATE VIRTUAL TABLE IF NOT EXISTS operationoutcome_embedding USING vss0( 308 | vector_nodes({{size}}) 309 | ); 310 | 311 | CREATE VIRTUAL TABLE IF NOT EXISTS medicinalproductpharmaceutical_embedding USING vss0( 312 | vector_nodes({{size}}) 313 | ); 314 | 315 | CREATE VIRTUAL TABLE IF NOT EXISTS immunization_embedding USING vss0( 316 | vector_nodes({{size}}) 317 | ); 318 | 319 | CREATE VIRTUAL TABLE IF NOT EXISTS medicationknowledge_embedding USING vss0( 320 | vector_nodes({{size}}) 321 | ); 322 | 323 | CREATE VIRTUAL TABLE IF NOT EXISTS researchsubject_embedding USING vss0( 324 | vector_nodes({{size}}) 325 | ); 326 | 327 | CREATE VIRTUAL TABLE IF NOT EXISTS medicinalproductindication_embedding USING vss0( 328 | vector_nodes({{size}}) 329 | ); 330 | 331 | CREATE VIRTUAL TABLE IF NOT EXISTS paymentnotice_embedding USING vss0( 332 | vector_nodes({{size}}) 333 | ); 334 | 335 | CREATE VIRTUAL TABLE IF NOT EXISTS namingsystem_embedding USING vss0( 336 | vector_nodes({{size}}) 337 | ); 338 | 339 | CREATE VIRTUAL TABLE IF NOT EXISTS medicationstatement_embedding USING vss0( 340 | vector_nodes({{size}}) 341 | ); 342 | 343 | CREATE VIRTUAL TABLE IF NOT EXISTS enrollmentresponse_embedding USING vss0( 344 | vector_nodes({{size}}) 345 | ); 346 | 347 | CREATE VIRTUAL TABLE IF NOT EXISTS nutritionorder_embedding USING vss0( 348 | vector_nodes({{size}}) 349 | ); 350 | 351 | CREATE VIRTUAL TABLE IF NOT EXISTS questionnaire_embedding USING vss0( 352 | vector_nodes({{size}}) 353 | ); 354 | 355 | CREATE VIRTUAL TABLE IF NOT EXISTS account_embedding USING vss0( 356 | vector_nodes({{size}}) 357 | ); 358 | 359 | CREATE VIRTUAL TABLE IF NOT EXISTS eventdefinition_embedding USING vss0( 360 | vector_nodes({{size}}) 361 | ); 362 | 363 | CREATE VIRTUAL TABLE IF NOT EXISTS medicinalproductundesirableeffect_embedding USING vss0( 364 | vector_nodes({{size}}) 365 | ); 366 | 367 | CREATE VIRTUAL TABLE IF NOT EXISTS substancespecification_embedding USING vss0( 368 | vector_nodes({{size}}) 369 | ); 370 | 371 | CREATE VIRTUAL TABLE IF NOT EXISTS communicationrequest_embedding USING vss0( 372 | vector_nodes({{size}}) 373 | ); 374 | 375 | CREATE VIRTUAL TABLE IF NOT EXISTS specimendefinition_embedding USING vss0( 376 | vector_nodes({{size}}) 377 | ); 378 | 379 | CREATE VIRTUAL TABLE IF NOT EXISTS verificationresult_embedding USING vss0( 380 | vector_nodes({{size}}) 381 | ); 382 | 383 | CREATE VIRTUAL TABLE IF NOT EXISTS documentmanifest_embedding USING vss0( 384 | vector_nodes({{size}}) 385 | ); 386 | 387 | CREATE VIRTUAL TABLE IF NOT EXISTS task_embedding USING vss0( 388 | vector_nodes({{size}}) 389 | ); 390 | 391 | CREATE VIRTUAL TABLE IF NOT EXISTS riskevidencesynthesis_embedding USING vss0( 392 | vector_nodes({{size}}) 393 | ); 394 | 395 | CREATE VIRTUAL TABLE IF NOT EXISTS valueset_embedding USING vss0( 396 | vector_nodes({{size}}) 397 | ); 398 | 399 | CREATE VIRTUAL TABLE IF NOT EXISTS claim_embedding USING vss0( 400 | vector_nodes({{size}}) 401 | ); 402 | 403 | CREATE VIRTUAL TABLE IF NOT EXISTS insuranceplan_embedding USING vss0( 404 | vector_nodes({{size}}) 405 | ); 406 | 407 | CREATE VIRTUAL TABLE IF NOT EXISTS examplescenario_embedding USING vss0( 408 | vector_nodes({{size}}) 409 | ); 410 | 411 | CREATE VIRTUAL TABLE IF NOT EXISTS researchstudy_embedding USING vss0( 412 | vector_nodes({{size}}) 413 | ); 414 | 415 | CREATE VIRTUAL TABLE IF NOT EXISTS medicationrequest_embedding USING vss0( 416 | vector_nodes({{size}}) 417 | ); 418 | 419 | CREATE VIRTUAL TABLE IF NOT EXISTS measure_embedding USING vss0( 420 | vector_nodes({{size}}) 421 | ); 422 | 423 | CREATE VIRTUAL TABLE IF NOT EXISTS list_embedding USING vss0( 424 | vector_nodes({{size}}) 425 | ); 426 | 427 | CREATE VIRTUAL TABLE IF NOT EXISTS capabilitystatement_embedding USING vss0( 428 | vector_nodes({{size}}) 429 | ); 430 | 431 | CREATE VIRTUAL TABLE IF NOT EXISTS visionprescription_embedding USING vss0( 432 | vector_nodes({{size}}) 433 | ); 434 | 435 | CREATE VIRTUAL TABLE IF NOT EXISTS riskassessment_embedding USING vss0( 436 | vector_nodes({{size}}) 437 | ); 438 | 439 | CREATE VIRTUAL TABLE IF NOT EXISTS substanceprotein_embedding USING vss0( 440 | vector_nodes({{size}}) 441 | ); 442 | 443 | CREATE VIRTUAL TABLE IF NOT EXISTS immunizationrecommendation_embedding USING vss0( 444 | vector_nodes({{size}}) 445 | ); 446 | 447 | CREATE VIRTUAL TABLE IF NOT EXISTS relatedperson_embedding USING vss0( 448 | vector_nodes({{size}}) 449 | ); 450 | 451 | CREATE VIRTUAL TABLE IF NOT EXISTS medication_embedding USING vss0( 452 | vector_nodes({{size}}) 453 | ); 454 | 455 | CREATE VIRTUAL TABLE IF NOT EXISTS appointmentresponse_embedding USING vss0( 456 | vector_nodes({{size}}) 457 | ); 458 | 459 | CREATE VIRTUAL TABLE IF NOT EXISTS researchelementdefinition_embedding USING vss0( 460 | vector_nodes({{size}}) 461 | ); 462 | 463 | CREATE VIRTUAL TABLE IF NOT EXISTS substance_embedding USING vss0( 464 | vector_nodes({{size}}) 465 | ); 466 | 467 | CREATE VIRTUAL TABLE IF NOT EXISTS paymentreconciliation_embedding USING vss0( 468 | vector_nodes({{size}}) 469 | ); 470 | 471 | CREATE VIRTUAL TABLE IF NOT EXISTS conceptmap_embedding USING vss0( 472 | vector_nodes({{size}}) 473 | ); 474 | 475 | CREATE VIRTUAL TABLE IF NOT EXISTS person_embedding USING vss0( 476 | vector_nodes({{size}}) 477 | ); 478 | 479 | CREATE VIRTUAL TABLE IF NOT EXISTS condition_embedding USING vss0( 480 | vector_nodes({{size}}) 481 | ); 482 | 483 | CREATE VIRTUAL TABLE IF NOT EXISTS careteam_embedding USING vss0( 484 | vector_nodes({{size}}) 485 | ); 486 | 487 | CREATE VIRTUAL TABLE IF NOT EXISTS catalogentry_embedding USING vss0( 488 | vector_nodes({{size}}) 489 | ); 490 | 491 | CREATE VIRTUAL TABLE IF NOT EXISTS structuredefinition_embedding USING vss0( 492 | vector_nodes({{size}}) 493 | ); 494 | 495 | CREATE VIRTUAL TABLE IF NOT EXISTS procedure_embedding USING vss0( 496 | vector_nodes({{size}}) 497 | ); 498 | 499 | CREATE VIRTUAL TABLE IF NOT EXISTS consent_embedding USING vss0( 500 | vector_nodes({{size}}) 501 | ); 502 | 503 | CREATE VIRTUAL TABLE IF NOT EXISTS observationdefinition_embedding USING vss0( 504 | vector_nodes({{size}}) 505 | ); 506 | 507 | CREATE VIRTUAL TABLE IF NOT EXISTS attribute_embedding USING vss0( 508 | vector_nodes({{size}}) 509 | ); 510 | 511 | CREATE VIRTUAL TABLE IF NOT EXISTS location_embedding USING vss0( 512 | vector_nodes({{size}}) 513 | ); 514 | 515 | CREATE VIRTUAL TABLE IF NOT EXISTS organization_embedding USING vss0( 516 | vector_nodes({{size}}) 517 | ); 518 | 519 | CREATE VIRTUAL TABLE IF NOT EXISTS device_embedding USING vss0( 520 | vector_nodes({{size}}) 521 | ); 522 | 523 | CREATE VIRTUAL TABLE IF NOT EXISTS supplyrequest_embedding USING vss0( 524 | vector_nodes({{size}}) 525 | ); 526 | 527 | CREATE VIRTUAL TABLE IF NOT EXISTS allergyintolerance_embedding USING vss0( 528 | vector_nodes({{size}}) 529 | ); 530 | 531 | CREATE VIRTUAL TABLE IF NOT EXISTS researchdefinition_embedding USING vss0( 532 | vector_nodes({{size}}) 533 | ); 534 | 535 | CREATE VIRTUAL TABLE IF NOT EXISTS operationdefinition_embedding USING vss0( 536 | vector_nodes({{size}}) 537 | ); 538 | 539 | CREATE VIRTUAL TABLE IF NOT EXISTS medicinalproductmanufactured_embedding USING vss0( 540 | vector_nodes({{size}}) 541 | ); 542 | 543 | CREATE VIRTUAL TABLE IF NOT EXISTS imagingstudy_embedding USING vss0( 544 | vector_nodes({{size}}) 545 | ); 546 | 547 | CREATE VIRTUAL TABLE IF NOT EXISTS coverageeligibilityrequest_embedding USING vss0( 548 | vector_nodes({{size}}) 549 | ); 550 | 551 | CREATE VIRTUAL TABLE IF NOT EXISTS medicinalproductingredient_embedding USING vss0( 552 | vector_nodes({{size}}) 553 | ); 554 | 555 | CREATE VIRTUAL TABLE IF NOT EXISTS guidanceresponse_embedding USING vss0( 556 | vector_nodes({{size}}) 557 | ); 558 | 559 | CREATE VIRTUAL TABLE IF NOT EXISTS media_embedding USING vss0( 560 | vector_nodes({{size}}) 561 | ); 562 | 563 | CREATE VIRTUAL TABLE IF NOT EXISTS measurereport_embedding USING vss0( 564 | vector_nodes({{size}}) 565 | ); 566 | 567 | CREATE VIRTUAL TABLE IF NOT EXISTS graphdefinition_embedding USING vss0( 568 | vector_nodes({{size}}) 569 | ); 570 | 571 | CREATE VIRTUAL TABLE IF NOT EXISTS terminologycapabilities_embedding USING vss0( 572 | vector_nodes({{size}}) 573 | ); 574 | 575 | CREATE VIRTUAL TABLE IF NOT EXISTS metadataresource_embedding USING vss0( 576 | vector_nodes({{size}}) 577 | ); 578 | 579 | CREATE VIRTUAL TABLE IF NOT EXISTS relations_embedding USING vss0( 580 | vector_relations({{size}}) 581 | ); 582 | 583 | commit; --------------------------------------------------------------------------------