├── .github └── workflows │ ├── pull-request.yaml │ └── release.yaml ├── .gitignore ├── Containerfile ├── DEVELOPER_GUIDE.md ├── LICENSE ├── Makefile ├── README.md ├── TG-future-agents.svg ├── TG-future-horizon.svg ├── TG-horizon-repo.svg ├── TG-layer-diagram.svg ├── TG-ship.jpg ├── TG_layer_diagram.svg ├── containers ├── Containerfile.base ├── Containerfile.bedrock ├── Containerfile.flow ├── Containerfile.hf ├── Containerfile.ocr └── Containerfile.vertexai ├── docs ├── README.agent-demo ├── README.cats ├── README.challenger ├── README.development.md ├── README.quickstart-docker-compose.md └── apis │ ├── README.md │ ├── api-agent.md │ ├── api-document-load.md │ ├── api-embeddings.md │ ├── api-graph-embeddings.md │ ├── api-graph-rag.md │ ├── api-prompt.md │ ├── api-text-completion.md │ ├── api-triples-query.md │ ├── pulsar.md │ └── websocket.md ├── grafana ├── dashboards │ └── dashboard.json └── provisioning │ ├── dashboard.yml │ └── datasource.yml ├── prometheus └── prometheus.yml ├── requirements.txt ├── schema.ttl ├── test-api ├── test-agent-api ├── test-agent2-api ├── test-config-api ├── test-dbpedia ├── test-embeddings-api ├── test-encyclopedia ├── test-graph-rag-api ├── test-internet-search ├── test-knowledge-delete ├── test-knowledge-fetch ├── test-knowledge-fetch2 ├── test-knowledge-list ├── test-library-add-doc ├── test-library-add-doc2 ├── test-library-add-processing ├── test-library-add-processing2 ├── test-library-get-document-content ├── test-library-get-document-metadata ├── test-library-list ├── test-library-list-documents ├── test-library-list-processing ├── test-library-remove-document ├── test-library-remove-document2 ├── test-library-remove-processing ├── test-library-update-doc ├── test-llm-api ├── test-llm2-api ├── test-load-document ├── test-load-text ├── test-prompt-api ├── test-prompt2-api └── test-triples-query-api ├── tests ├── README.prompts ├── query ├── report-chunk-sizes ├── test-agent ├── test-config ├── test-doc-embeddings ├── test-doc-prompt ├── test-doc-rag ├── test-embeddings ├── test-flow ├── test-flow-get-class ├── test-flow-put-class ├── test-flow-start-flow ├── test-flow-stop-flow ├── test-get-config ├── test-graph-embeddings ├── test-graph-rag ├── test-graph-rag2 ├── test-lang-definition ├── test-lang-kg-prompt ├── test-lang-relationships ├── test-lang-topics ├── test-llm ├── test-llm2 ├── test-llm3 ├── test-load-pdf ├── test-load-text ├── test-milvus ├── test-prompt-analyze ├── test-prompt-extraction ├── test-prompt-french-question ├── test-prompt-knowledge ├── test-prompt-question ├── test-prompt-spanish-question ├── test-rows-prompt ├── test-run-extract-row └── test-triples ├── tg-arch-diagram.svg ├── trustgraph-base ├── README.md ├── setup.py └── trustgraph │ ├── api │ ├── __init__.py │ ├── api.py │ ├── config.py │ ├── exceptions.py │ ├── flow.py │ ├── knowledge.py │ ├── library.py │ └── types.py │ ├── base │ ├── __init__.py │ ├── agent_client.py │ ├── agent_service.py │ ├── async_processor.py │ ├── consumer.py │ ├── consumer_spec.py │ ├── document_embeddings_client.py │ ├── document_embeddings_query_service.py │ ├── document_embeddings_store_service.py │ ├── embeddings_client.py │ ├── embeddings_service.py │ ├── flow.py │ ├── flow_processor.py │ ├── graph_embeddings_client.py │ ├── graph_embeddings_query_service.py │ ├── graph_embeddings_store_service.py │ ├── graph_rag_client.py │ ├── llm_service.py │ ├── metrics.py │ ├── producer.py │ ├── producer_spec.py │ ├── prompt_client.py │ ├── publisher.py │ ├── pubsub.py │ ├── request_response_spec.py │ ├── setting_spec.py │ ├── spec.py │ ├── subscriber.py │ ├── subscriber_spec.py │ ├── text_completion_client.py │ ├── triples_client.py │ ├── triples_query_service.py │ └── triples_store_service.py │ ├── clients │ ├── __init__.py │ ├── agent_client.py │ ├── base.py │ ├── config_client.py │ ├── document_embeddings_client.py │ ├── document_rag_client.py │ ├── embeddings_client.py │ ├── graph_embeddings_client.py │ ├── graph_rag_client.py │ ├── llm_client.py │ ├── prompt_client.py │ └── triples_query_client.py │ ├── exceptions.py │ ├── knowledge │ ├── __init__.py │ ├── defs.py │ ├── document.py │ ├── identifier.py │ ├── organization.py │ └── publication.py │ ├── log_level.py │ ├── objects │ ├── __init__.py │ ├── field.py │ └── object.py │ ├── rdf.py │ └── schema │ ├── __init__.py │ ├── agent.py │ ├── config.py │ ├── documents.py │ ├── flows.py │ ├── graph.py │ ├── knowledge.py │ ├── library.py │ ├── lookup.py │ ├── metadata.py │ ├── models.py │ ├── object.py │ ├── prompt.py │ ├── retrieval.py │ ├── topic.py │ └── types.py ├── trustgraph-bedrock ├── README.md ├── scripts │ └── text-completion-bedrock ├── setup.py └── trustgraph │ └── model │ └── text_completion │ └── bedrock │ ├── __init__.py │ ├── __main__.py │ └── llm.py ├── trustgraph-cli ├── README.md ├── scripts │ ├── tg-add-library-document │ ├── tg-delete-flow-class │ ├── tg-delete-kg-core │ ├── tg-dump-msgpack │ ├── tg-get-flow-class │ ├── tg-get-kg-core │ ├── tg-graph-to-turtle │ ├── tg-init-pulsar-manager │ ├── tg-init-trustgraph │ ├── tg-invoke-agent │ ├── tg-invoke-document-rag │ ├── tg-invoke-graph-rag │ ├── tg-invoke-llm │ ├── tg-invoke-prompt │ ├── tg-load-doc-embeds │ ├── tg-load-kg-core │ ├── tg-load-pdf │ ├── tg-load-sample-documents │ ├── tg-load-text │ ├── tg-load-turtle │ ├── tg-put-flow-class │ ├── tg-put-kg-core │ ├── tg-remove-library-document │ ├── tg-save-doc-embeds │ ├── tg-set-prompt │ ├── tg-set-token-costs │ ├── tg-show-config │ ├── tg-show-flow-classes │ ├── tg-show-flow-state │ ├── tg-show-flows │ ├── tg-show-graph │ ├── tg-show-kg-cores │ ├── tg-show-library-documents │ ├── tg-show-library-processing │ ├── tg-show-processor-state │ ├── tg-show-prompts │ ├── tg-show-token-costs │ ├── tg-show-tools │ ├── tg-start-flow │ ├── tg-start-library-processing │ ├── tg-stop-flow │ ├── tg-stop-library-processing │ └── tg-unload-kg-core └── setup.py ├── trustgraph-embeddings-hf ├── README.md ├── scripts │ └── embeddings-hf ├── setup.py └── trustgraph │ ├── __init__.py │ └── embeddings │ ├── __init__.py │ └── hf │ ├── __init__.py │ ├── __main__.py │ └── hf.py ├── trustgraph-flow ├── README.md ├── scripts │ ├── agent-manager-react │ ├── api-gateway │ ├── chunker-recursive │ ├── chunker-token │ ├── config-svc │ ├── de-query-milvus │ ├── de-query-pinecone │ ├── de-query-qdrant │ ├── de-write-milvus │ ├── de-write-pinecone │ ├── de-write-qdrant │ ├── document-embeddings │ ├── document-rag │ ├── embeddings-fastembed │ ├── embeddings-ollama │ ├── ge-query-milvus │ ├── ge-query-pinecone │ ├── ge-query-qdrant │ ├── ge-write-milvus │ ├── ge-write-pinecone │ ├── ge-write-qdrant │ ├── graph-embeddings │ ├── graph-rag │ ├── kg-extract-definitions │ ├── kg-extract-relationships │ ├── kg-extract-topics │ ├── kg-manager │ ├── kg-store │ ├── librarian │ ├── metering │ ├── object-extract-row │ ├── oe-write-milvus │ ├── pdf-decoder │ ├── pdf-ocr-mistral │ ├── prompt-generic │ ├── prompt-template │ ├── rows-write-cassandra │ ├── run-processing │ ├── text-completion-azure │ ├── text-completion-azure-openai │ ├── text-completion-claude │ ├── text-completion-cohere │ ├── text-completion-googleaistudio │ ├── text-completion-llamafile │ ├── text-completion-lmstudio │ ├── text-completion-mistral │ ├── text-completion-ollama │ ├── text-completion-openai │ ├── text-completion-tgi │ ├── triples-query-cassandra │ ├── triples-query-falkordb │ ├── triples-query-memgraph │ ├── triples-query-neo4j │ ├── triples-write-cassandra │ ├── triples-write-falkordb │ ├── triples-write-memgraph │ ├── triples-write-neo4j │ └── wikipedia-lookup ├── setup.py └── trustgraph │ ├── __init__.py │ ├── agent │ ├── __init__.py │ └── react │ │ ├── README.md │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── agent_manager.py │ │ ├── service.py │ │ ├── tools.py │ │ └── types.py │ ├── chunking │ ├── __init__.py │ ├── recursive │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── chunker.py │ └── token │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── chunker.py │ ├── config │ └── service │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── config.py │ │ ├── flow.py │ │ └── service.py │ ├── cores │ ├── __init__.py │ ├── __main__.py │ ├── knowledge.py │ └── service.py │ ├── decoding │ ├── __init__.py │ ├── mistral_ocr │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── processor.py │ └── pdf │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── pdf_decoder.py │ ├── direct │ ├── __init__.py │ ├── cassandra.py │ ├── milvus_doc_embeddings.py │ ├── milvus_graph_embeddings.py │ └── milvus_object_embeddings.py │ ├── embeddings │ ├── __init__.py │ ├── document_embeddings │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── embeddings.py │ ├── fastembed │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── processor.py │ ├── graph_embeddings │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── embeddings.py │ └── ollama │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── processor.py │ ├── external │ ├── __init__.py │ └── wikipedia │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── service.py │ ├── extract │ ├── __init__.py │ ├── kg │ │ ├── __init__.py │ │ ├── definitions │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ └── extract.py │ │ ├── relationships │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ └── extract.py │ │ └── topics │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ └── extract.py │ └── object │ │ ├── __init__.py │ │ └── row │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── extract.py │ ├── gateway │ ├── __init__.py │ ├── __main__.py │ ├── auth.py │ ├── config │ │ └── receiver.py │ ├── dispatch │ │ ├── agent.py │ │ ├── config.py │ │ ├── document_embeddings_export.py │ │ ├── document_embeddings_import.py │ │ ├── document_load.py │ │ ├── document_rag.py │ │ ├── embeddings.py │ │ ├── entity_contexts_export.py │ │ ├── entity_contexts_import.py │ │ ├── flow.py │ │ ├── graph_embeddings_export.py │ │ ├── graph_embeddings_import.py │ │ ├── graph_embeddings_query.py │ │ ├── graph_rag.py │ │ ├── knowledge.py │ │ ├── librarian.py │ │ ├── manager.py │ │ ├── mux.py │ │ ├── prompt.py │ │ ├── requestor.py │ │ ├── sender.py │ │ ├── serialize.py │ │ ├── streamer.py │ │ ├── text_completion.py │ │ ├── text_load.py │ │ ├── triples_export.py │ │ ├── triples_import.py │ │ └── triples_query.py │ ├── endpoint │ │ ├── constant_endpoint.py │ │ ├── manager.py │ │ ├── metrics.py │ │ ├── socket.py │ │ └── variable_endpoint.py │ ├── running.py │ ├── service.py │ └── unused │ │ ├── dbpedia.py │ │ ├── encyclopedia.py │ │ └── internet_search.py │ ├── librarian │ ├── __init__.py │ ├── __main__.py │ ├── blob_store.py │ ├── librarian.py │ └── service.py │ ├── metering │ ├── __init__.py │ ├── __main__.py │ └── counter.py │ ├── model │ ├── __init__.py │ ├── prompt │ │ ├── __init__.py │ │ ├── generic │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ ├── prompts.py │ │ │ └── service.py │ │ └── template │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ ├── prompt_manager.py │ │ │ └── service.py │ └── text_completion │ │ ├── __init__.py │ │ ├── azure │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── llm.py │ │ ├── azure_openai │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── llm.py │ │ ├── claude │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── llm.py │ │ ├── cohere │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── llm.py │ │ ├── googleaistudio │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── llm.py │ │ ├── llamafile │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── llm.py │ │ ├── lmstudio │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── llm.py │ │ ├── mistral │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── llm.py │ │ ├── ollama │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── llm.py │ │ ├── openai │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── llm.py │ │ └── tgi │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── llm.py │ ├── processing │ ├── __init__.py │ ├── __main__.py │ └── processing.py │ ├── query │ ├── __init__.py │ ├── doc_embeddings │ │ ├── __init__.py │ │ ├── milvus │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ └── service.py │ │ ├── pinecone │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ └── service.py │ │ └── qdrant │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ └── service.py │ ├── graph_embeddings │ │ ├── __init__.py │ │ ├── milvus │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ └── service.py │ │ ├── pinecone │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ └── service.py │ │ └── qdrant │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ └── service.py │ └── triples │ │ ├── __init__.py │ │ ├── cassandra │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── service.py │ │ ├── falkordb │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── service.py │ │ ├── memgraph │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── service.py │ │ └── neo4j │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── service.py │ ├── retrieval │ ├── __init__.py │ ├── document_rag │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── document_rag.py │ │ └── rag.py │ └── graph_rag │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── graph_rag.py │ │ └── rag.py │ ├── storage │ ├── __init__.py │ ├── doc_embeddings │ │ ├── __init__.py │ │ ├── milvus │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ └── write.py │ │ ├── pinecone │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ └── write.py │ │ └── qdrant │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ └── write.py │ ├── graph_embeddings │ │ ├── __init__.py │ │ ├── milvus │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ └── write.py │ │ ├── pinecone │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ └── write.py │ │ └── qdrant │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ └── write.py │ ├── knowledge │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── store.py │ ├── object_embeddings │ │ ├── __init__.py │ │ └── milvus │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ └── write.py │ ├── rows │ │ ├── __init__.py │ │ └── cassandra │ │ │ ├── __init__.py │ │ │ ├── __main__.py │ │ │ └── write.py │ └── triples │ │ ├── __init__.py │ │ ├── cassandra │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── write.py │ │ ├── falkordb │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── write.py │ │ ├── memgraph │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── write.py │ │ └── neo4j │ │ ├── __init__.py │ │ ├── __main__.py │ │ └── write.py │ └── tables │ ├── __init__.py │ ├── config.py │ ├── knowledge.py │ └── library.py ├── trustgraph-ocr ├── README.md ├── scripts │ └── pdf-ocr ├── setup.py └── trustgraph │ └── decoding │ └── ocr │ ├── __init__.py │ ├── __main__.py │ └── pdf_decoder.py ├── trustgraph-vertexai ├── README.md ├── scripts │ └── text-completion-vertexai ├── setup.py └── trustgraph │ └── model │ └── text_completion │ └── vertexai │ ├── __init__.py │ ├── __main__.py │ └── llm.py └── trustgraph ├── README.md └── setup.py /.github/workflows/pull-request.yaml: -------------------------------------------------------------------------------- 1 | 2 | name: Test pull request 3 | 4 | on: 5 | pull_request: 6 | 7 | permissions: 8 | contents: read 9 | 10 | jobs: 11 | 12 | container-push: 13 | 14 | name: Do nothing 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - name: Checkout 19 | uses: actions/checkout@v3 20 | 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | __pycache__/ 3 | env/ 4 | */build/ 5 | *.egg-info/ 6 | *.parquet 7 | templates/values/version.jsonnet 8 | trustgraph-base/trustgraph/base_version.py 9 | trustgraph-bedrock/trustgraph/bedrock_version.py 10 | trustgraph-embeddings-hf/trustgraph/embeddings_hf_version.py 11 | trustgraph-flow/trustgraph/flow_version.py 12 | trustgraph-ocr/trustgraph/ocr_version.py 13 | trustgraph-parquet/trustgraph/parquet_version.py 14 | trustgraph-vertexai/trustgraph/vertexai_version.py 15 | trustgraph-cli/trustgraph/ 16 | vertexai/ -------------------------------------------------------------------------------- /TG-ship.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/TG-ship.jpg -------------------------------------------------------------------------------- /docs/README.agent-demo: -------------------------------------------------------------------------------- 1 | podman-compose -f docker-compose.yaml up -d 2 | 3 | 4 | tg-processor-state 5 | 6 | tg-load-text --keyword cats animals home-life --name "Mark's cats" --description "This document describes Mark's cats" --copyright-notice 'Public domain' --publication-organization 'trustgraph.ai' --publication-date 2024-10-23 --copyright-holder 'trustgraph.ai' --copyright-year 2024 --publication-description 'Uploading to Github' --url https://example.com --id TG-000001 ../trustgraph/README.cats 7 | 8 | tg-load-text --keyword nasa challenger space-shuttle shuttle orbiter --name 'Challenger Report Volume 1' --description 'The findings of the Presidential Commission regarding the circumstances surrounding the Challenger accident are reported and recommendations for corrective action are outlined' --copyright-notice 'Work of the US Gov. Public Use Permitted' --publication-organization 'NASA' --publication-date 1986-06-06 --copyright-holder 'US Government' --copyright-year 1986 --publication-description 'The findings of the Commission regarding the circumstances surrounding the Challenger accident are reported' --url https://ntrs.nasa.gov/citations/19860015255 --id AD-A171402 ../trustgraph/README.challenger 9 | 10 | 11 | tg-graph-show 12 | 13 | tg-query-graph-rag -q 'Tell me cat facts' 14 | 15 | 16 | tg-invoke-agent -v -q "How many cats does Mark have? Calculate that number raised to 0.4 power. Is that number lower than the numeric part of the mission identifier of the Space Shuttle Challenger on its last mission? If so, give me an apple pie recipe, otherwise return a poem about cheese." 17 | 18 | 19 | -------------------------------------------------------------------------------- /docs/README.cats: -------------------------------------------------------------------------------- 1 | 2 | My name is Mark. 3 | 4 | I have 2 cats: 5 | - Fred is a big, fat, orange, stripy cat. He is 12 years old and has 4 legs. 6 | - Hope is a small, black cat. She is 7 years old and also has 4 legs. 7 | 8 | Fred has 4 legs. 9 | 10 | Hope has 4 legs. 11 | 12 | Fred and Hope are nice animals, but occasionally they fight. 13 | 14 | Fred is lazy and sleeps a lot. Hope is energetic, runs around a lot and 15 | climbs trees. 16 | 17 | Both cats have tails and whiskers like all cats do. 18 | 19 | Cats have the species name Felis catus. 20 | 21 | The cat (Felis catus), also referred to as domestic cat or house cat, is a 22 | small domesticated carnivorous mammal. It is the only domesticated species of 23 | the family Felidae. Advances in archaeology and genetics have shown that the 24 | domestication of the cat occurred in the Near East around 7500 BC. It is 25 | commonly kept as a pet and farm cat, but also ranges freely as a feral cat 26 | avoiding human contact. Valued by humans for companionship and its ability to 27 | kill vermin, the cat's retractable claws are adapted to killing small prey 28 | like mice and rats. It has a strong, flexible body, quick reflexes, and sharp 29 | teeth, and its night vision and sense of smell are well developed. It is a 30 | social species, but a solitary hunter and a crepuscular predator. Cat 31 | communication includes vocalizations—including meowing, purring, trilling, 32 | hissing, growling, and grunting–as well as body language. It can hear sounds 33 | too faint or too high in frequency for human ears, such as those made by small 34 | mammals. It secretes and perceives pheromones. 35 | 36 | -------------------------------------------------------------------------------- /docs/apis/api-document-load.md: -------------------------------------------------------------------------------- 1 | 2 | Coming soon 3 | 4 | -------------------------------------------------------------------------------- /docs/apis/pulsar.md: -------------------------------------------------------------------------------- 1 | 2 | Coming soon 3 | 4 | -------------------------------------------------------------------------------- /grafana/provisioning/dashboard.yml: -------------------------------------------------------------------------------- 1 | 2 | apiVersion: 1 3 | 4 | providers: 5 | 6 | - name: 'trustgraph.ai' 7 | orgId: 1 8 | folder: 'TrustGraph' 9 | folderUid: 'b6c5be90-d432-4df8-aeab-737c7b151228' 10 | type: file 11 | disableDeletion: false 12 | updateIntervalSeconds: 30 13 | allowUiUpdates: true 14 | options: 15 | path: /var/lib/grafana/dashboards 16 | foldersFromFilesStructure: false 17 | 18 | -------------------------------------------------------------------------------- /grafana/provisioning/datasource.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | prune: true 4 | 5 | datasources: 6 | - name: Prometheus 7 | type: prometheus 8 | access: proxy 9 | orgId: 1 10 | # Sets a custom UID to reference this 11 | # data source in other parts of the configuration. 12 | # If not specified, Grafana generates one. 13 | uid: 'f6b18033-5918-4e05-a1ca-4cb30343b129' 14 | 15 | url: http://prometheus:9090 16 | 17 | basicAuth: false 18 | withCredentials: false 19 | isDefault: true 20 | editable: true 21 | 22 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | torch 2 | urllib3 3 | transformers 4 | sentence-transformers 5 | rdflib 6 | pymilvus 7 | langchain 8 | langchain-core 9 | langchain-huggingface 10 | langchain-text-splitters 11 | langchain-community 12 | huggingface-hub 13 | requests 14 | cassandra-driver 15 | pulsar-client 16 | pypdf 17 | anthropic 18 | google-cloud-aiplatform 19 | pyyaml 20 | prometheus-client 21 | pyarrow 22 | boto3 23 | ollama 24 | -------------------------------------------------------------------------------- /schema.ttl: -------------------------------------------------------------------------------- 1 | @prefix ns1: . 2 | @prefix rdf: . 3 | @prefix rdfs: . 4 | @prefix schema: . 5 | @prefix skos: . 6 | 7 | schema:subjectOf rdfs:label "subject of" . 8 | skos:definition rdfs:label "definition" . 9 | 10 | rdf:type rdfs:label "type" . 11 | 12 | schema:DigitalDocument rdfs:label "digital document" . 13 | schema:Organization rdfs:label "organization" . 14 | schema:PublicationEvent rdfs:label "publication event" . 15 | 16 | schema:copyrightNotice rdfs:label "copyright notice" . 17 | schema:copyrightHolder rdfs:label "copyright holder" . 18 | schema:copyrightYear rdfs:label "copyright year" . 19 | schema:license rdfs:label "license" . 20 | schema:publication rdfs:label "publication" . 21 | schema:startDate rdfs:label "start date" . 22 | schema:endDate rdfs:label "end date" . 23 | schema:publishedBy rdfs:label "published by" . 24 | schema:datePublished rdfs:label "date published" . 25 | schema:publication rdfs:label "publication" . 26 | schema:datePublished rdfs:label "date published" . 27 | schema:url rdfs:label "url" . 28 | schema:identifier rdfs:label "identifier" . 29 | schema:keywords rdfs:label "keyword" . 30 | 31 | skos:definition rdfs:label "definition" . 32 | 33 | -------------------------------------------------------------------------------- /test-api/test-agent-api: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | 7 | url = "http://localhost:8088/api/v1/flow/0000/agent" 8 | 9 | ############################################################################ 10 | 11 | input = { 12 | "question": "What is the highest risk aspect of running a space shuttle program? Provide 5 detailed reasons to justify our answer.", 13 | } 14 | 15 | resp = requests.post( 16 | url, 17 | json=input, 18 | ) 19 | 20 | print(resp.text) 21 | resp = resp.json() 22 | 23 | if "error" in resp: 24 | print(f"Error: {resp['error']}") 25 | sys.exit(1) 26 | 27 | print(resp["answer"]) 28 | 29 | -------------------------------------------------------------------------------- /test-api/test-agent2-api: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | 7 | url = "http://localhost:8088/api/v1/" 8 | 9 | ############################################################################ 10 | 11 | input = { 12 | "question": "What is 14 plus 12. Justify your answer.", 13 | } 14 | 15 | resp = requests.post( 16 | f"{url}agent", 17 | json=input, 18 | ) 19 | 20 | resp = resp.json() 21 | 22 | if "error" in resp: 23 | print(f"Error: {resp['error']}") 24 | sys.exit(1) 25 | 26 | print(resp["answer"]) 27 | 28 | 29 | -------------------------------------------------------------------------------- /test-api/test-dbpedia: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | 7 | url = "http://localhost:8088/api/v1/" 8 | 9 | ############################################################################ 10 | 11 | input = { 12 | "term": "Cornwall", 13 | } 14 | 15 | resp = requests.post( 16 | f"{url}dbpedia", 17 | json=input, 18 | ) 19 | 20 | resp = resp.json() 21 | 22 | if "error" in resp: 23 | print(f"Error: {resp['error']}") 24 | sys.exit(1) 25 | 26 | print(resp["text"]) 27 | 28 | sys.exit(0) 29 | ############################################################################ 30 | 31 | -------------------------------------------------------------------------------- /test-api/test-embeddings-api: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | 7 | url = "http://localhost:8088/api/v1/flow/0000/embeddings" 8 | 9 | ############################################################################ 10 | 11 | input = { 12 | "text": "What is the highest risk aspect of running a space shuttle program? Provide 5 detailed reasons to justify our answer.", 13 | } 14 | 15 | resp = requests.post( 16 | url, 17 | json=input, 18 | ) 19 | 20 | resp = resp.json() 21 | 22 | if "error" in resp: 23 | print(f"Error: {resp['error']}") 24 | sys.exit(1) 25 | 26 | print(resp["vectors"]) 27 | 28 | -------------------------------------------------------------------------------- /test-api/test-encyclopedia: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | 7 | url = "http://localhost:8088/api/v1/" 8 | 9 | ############################################################################ 10 | 11 | input = { 12 | "term": "Cornwall", 13 | } 14 | 15 | resp = requests.post( 16 | f"{url}encyclopedia", 17 | json=input, 18 | ) 19 | 20 | resp = resp.json() 21 | 22 | if "error" in resp: 23 | print(f"Error: {resp['error']}") 24 | sys.exit(1) 25 | 26 | print(resp["text"]) 27 | 28 | sys.exit(0) 29 | ############################################################################ 30 | 31 | -------------------------------------------------------------------------------- /test-api/test-graph-rag-api: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | 7 | url = "http://localhost:8088/api/v1/flow/0000/graph-rag" 8 | 9 | ############################################################################ 10 | 11 | input = { 12 | "query": "Give me 10 facts", 13 | } 14 | 15 | resp = requests.post( 16 | url, 17 | json=input, 18 | ) 19 | 20 | resp = resp.json() 21 | 22 | print(resp) 23 | if "error" in resp: 24 | print(f"Error: {resp['error']}") 25 | sys.exit(1) 26 | 27 | print(resp["response"]) 28 | 29 | sys.exit(0) 30 | ############################################################################ 31 | 32 | -------------------------------------------------------------------------------- /test-api/test-internet-search: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | 7 | url = "http://localhost:8088/api/v1/" 8 | 9 | ############################################################################ 10 | 11 | input = { 12 | "term": "Cornwall", 13 | } 14 | 15 | resp = requests.post( 16 | f"{url}internet-search", 17 | json=input, 18 | ) 19 | 20 | resp = resp.json() 21 | 22 | if "error" in resp: 23 | print(f"Error: {resp['error']}") 24 | sys.exit(1) 25 | 26 | print(resp["text"]) 27 | 28 | sys.exit(0) 29 | ############################################################################ 30 | 31 | -------------------------------------------------------------------------------- /test-api/test-knowledge-delete: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | import base64 7 | import time 8 | 9 | url = "http://localhost:8088/api/v1/" 10 | 11 | ############################################################################ 12 | 13 | input = { 14 | "operation": "delete-kg-core", 15 | "id": "https://trustgraph.ai/doc/intelligence-and-state", 16 | "user": "trustgraph", 17 | } 18 | 19 | resp = requests.post( 20 | f"{url}knowledge", 21 | json=input, 22 | ) 23 | 24 | print(resp.text) 25 | resp = resp.json() 26 | 27 | print(resp) 28 | 29 | if "error" in resp: 30 | print(f"Error: {resp['error']}") 31 | sys.exit(1) 32 | 33 | # print(resp["response"]) 34 | print(resp) 35 | 36 | sys.exit(0) 37 | 38 | ############################################################################ 39 | 40 | -------------------------------------------------------------------------------- /test-api/test-knowledge-fetch: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | import base64 7 | import time 8 | 9 | url = "http://localhost:8088/api/v1/" 10 | 11 | ############################################################################ 12 | 13 | input = { 14 | "operation": "fetch-kg-core", 15 | "id": "https://trustgraph.ai/doc/intelligence-and-state", 16 | "user": "trustgraph", 17 | } 18 | 19 | resp = requests.post( 20 | f"{url}knowledge", 21 | json=input, 22 | ) 23 | 24 | print(resp.text) 25 | resp = resp.json() 26 | 27 | print(resp) 28 | 29 | if "error" in resp: 30 | print(f"Error: {resp['error']}") 31 | sys.exit(1) 32 | 33 | # print(resp["response"]) 34 | print(resp) 35 | 36 | sys.exit(0) 37 | 38 | ############################################################################ 39 | 40 | -------------------------------------------------------------------------------- /test-api/test-knowledge-fetch2: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import asyncio 5 | import json 6 | import sys 7 | import base64 8 | import time 9 | from websockets.asyncio.client import connect 10 | 11 | url = "ws://localhost:8088/api/v1/socket" 12 | 13 | ############################################################################ 14 | 15 | async def run(): 16 | 17 | async with connect(url) as ws: 18 | 19 | req = { 20 | "id": "aa11", 21 | "service": "knowledge", 22 | "request": { 23 | "operation": "fetch-kg-core", 24 | "user": "trustgraph", 25 | "id": "https://trustgraph.ai/doc/intelligence-and-state" 26 | } 27 | } 28 | 29 | await ws.send(json.dumps(req)) 30 | 31 | while True: 32 | 33 | msg = await ws.recv() 34 | obj = json.loads(msg) 35 | 36 | print(obj) 37 | 38 | if "error" in obj: 39 | print(f"Error: {obj['error']}") 40 | break 41 | 42 | if "response" not in obj: continue 43 | 44 | if "eos" in obj["response"]: 45 | if obj["response"]["eos"]: 46 | break 47 | 48 | ############################################################################ 49 | 50 | asyncio.run(run()) 51 | -------------------------------------------------------------------------------- /test-api/test-knowledge-list: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | import base64 7 | import time 8 | 9 | url = "http://localhost:8088/api/v1/" 10 | 11 | ############################################################################ 12 | 13 | input = { 14 | "operation": "list-kg-cores", 15 | "user": "trustgraph", 16 | } 17 | 18 | resp = requests.post( 19 | f"{url}knowledge", 20 | json=input, 21 | ) 22 | 23 | print(resp.text) 24 | resp = resp.json() 25 | 26 | print(resp) 27 | 28 | if "error" in resp: 29 | print(f"Error: {resp['error']}") 30 | sys.exit(1) 31 | 32 | # print(resp["response"]) 33 | print(resp) 34 | 35 | sys.exit(0) 36 | 37 | ############################################################################ 38 | 39 | -------------------------------------------------------------------------------- /test-api/test-library-add-processing: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | import base64 7 | import time 8 | 9 | url = "http://localhost:8088/api/v1/" 10 | 11 | ############################################################################ 12 | 13 | doc_id = "http://trustgraph.ai/doc/9fdee98b-b259-40ac-bcb9-8e82ccedeb04" 14 | 15 | proc_id = "2714fc72-44ab-45f2-94dd-6773fc336535" 16 | 17 | input = { 18 | "operation": "add-processing", 19 | "processing-metadata": { 20 | "id": proc_id, 21 | "document-id": doc_id, 22 | "time": int(time.time()), 23 | "flow": "0000", 24 | "user": "trustgraph", 25 | "collection": "default", 26 | "tags": ["test"], 27 | } 28 | } 29 | 30 | resp = requests.post( 31 | f"{url}librarian", 32 | json=input, 33 | ) 34 | 35 | print(resp.text) 36 | resp = resp.json() 37 | 38 | print(resp) 39 | 40 | if "error" in resp: 41 | print(f"Error: {resp['error']}") 42 | sys.exit(1) 43 | 44 | # print(resp["response"]) 45 | print(resp) 46 | 47 | sys.exit(0) 48 | 49 | ############################################################################ 50 | 51 | -------------------------------------------------------------------------------- /test-api/test-library-add-processing2: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | import base64 7 | import time 8 | 9 | url = "http://localhost:8088/api/v1/" 10 | 11 | ############################################################################ 12 | 13 | doc_id = "http://trustgraph.ai/doc/6d034da9-2759-45c2-af24-14db7f4c44c2" 14 | 15 | proc_id = "72be9c56-a63a-4dde-8f3c-9b35f2598b83" 16 | 17 | input = { 18 | "operation": "add-processing", 19 | "processing-metadata": { 20 | "id": proc_id, 21 | "document-id": doc_id, 22 | "time": int(time.time()), 23 | "flow": "0000", 24 | "user": "trustgraph", 25 | "collection": "default", 26 | "tags": ["test"], 27 | } 28 | } 29 | 30 | resp = requests.post( 31 | f"{url}librarian", 32 | json=input, 33 | ) 34 | 35 | print(resp.text) 36 | resp = resp.json() 37 | 38 | print(resp) 39 | 40 | if "error" in resp: 41 | print(f"Error: {resp['error']}") 42 | sys.exit(1) 43 | 44 | # print(resp["response"]) 45 | print(resp) 46 | 47 | sys.exit(0) 48 | 49 | ############################################################################ 50 | 51 | -------------------------------------------------------------------------------- /test-api/test-library-get-document-content: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | import base64 7 | 8 | url = "http://localhost:8088/api/v1/" 9 | 10 | ############################################################################ 11 | 12 | id = "http://trustgraph.ai/doc/9fdee98b-b259-40ac-bcb9-8e82ccedeb04" 13 | 14 | user = "trustgraph" 15 | 16 | input = { 17 | "operation": "get-document-content", 18 | "user": user, 19 | "document-id": id, 20 | } 21 | 22 | resp = requests.post( 23 | f"{url}librarian", 24 | json=input, 25 | ) 26 | 27 | resp = resp.json() 28 | 29 | if "error" in resp: 30 | print(f"Error: {resp['error']}") 31 | sys.exit(1) 32 | 33 | 34 | content = base64.b64decode(resp["content"]).decode("utf-8") 35 | 36 | print(content) 37 | 38 | sys.exit(0) 39 | 40 | ############################################################################ 41 | 42 | -------------------------------------------------------------------------------- /test-api/test-library-get-document-metadata: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | import base64 7 | 8 | url = "http://localhost:8088/api/v1/" 9 | 10 | ############################################################################ 11 | 12 | id = "http://trustgraph.ai/doc/9fdee98b-b259-40ac-bcb9-8e82ccedeb04" 13 | 14 | user = "trustgraph" 15 | 16 | input = { 17 | "operation": "get-document-metadata", 18 | "user": user, 19 | "document-id": id, 20 | } 21 | 22 | resp = requests.post( 23 | f"{url}librarian", 24 | json=input, 25 | ) 26 | 27 | print(resp.text) 28 | resp = resp.json() 29 | 30 | print(resp) 31 | 32 | if "error" in resp: 33 | print(f"Error: {resp['error']}") 34 | sys.exit(1) 35 | 36 | # print(resp["response"]) 37 | print(resp) 38 | 39 | sys.exit(0) 40 | 41 | ############################################################################ 42 | 43 | -------------------------------------------------------------------------------- /test-api/test-library-list: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | import base64 7 | 8 | url = "http://localhost:8088/api/v1/" 9 | 10 | ############################################################################ 11 | 12 | user = "trustgraph" 13 | 14 | input = { 15 | "operation": "list-documents", 16 | "user": user, 17 | } 18 | 19 | resp = requests.post( 20 | f"{url}librarian", 21 | json=input, 22 | ) 23 | 24 | print(resp.text) 25 | resp = resp.json() 26 | 27 | print(resp) 28 | 29 | if "error" in resp: 30 | print(f"Error: {resp['error']}") 31 | sys.exit(1) 32 | 33 | # print(resp["response"]) 34 | print(resp) 35 | 36 | sys.exit(0) 37 | 38 | ############################################################################ 39 | 40 | -------------------------------------------------------------------------------- /test-api/test-library-list-documents: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | import base64 7 | import time 8 | 9 | url = "http://localhost:8088/api/v1/" 10 | 11 | ############################################################################ 12 | 13 | input = { 14 | "operation": "list-documents", 15 | "user": "trustgraph", 16 | } 17 | 18 | resp = requests.post( 19 | f"{url}librarian", 20 | json=input, 21 | ) 22 | 23 | print(resp.text) 24 | resp = resp.json() 25 | 26 | print(resp) 27 | 28 | if "error" in resp: 29 | print(f"Error: {resp['error']}") 30 | sys.exit(1) 31 | 32 | # print(resp["response"]) 33 | print(resp) 34 | 35 | sys.exit(0) 36 | 37 | ############################################################################ 38 | 39 | -------------------------------------------------------------------------------- /test-api/test-library-list-processing: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | import base64 7 | import time 8 | 9 | url = "http://localhost:8088/api/v1/" 10 | 11 | ############################################################################ 12 | 13 | input = { 14 | "operation": "list-processing", 15 | "user": "trustgraph", 16 | } 17 | 18 | resp = requests.post( 19 | f"{url}librarian", 20 | json=input, 21 | ) 22 | 23 | print(resp.text) 24 | resp = resp.json() 25 | 26 | print(resp) 27 | 28 | if "error" in resp: 29 | print(f"Error: {resp['error']}") 30 | sys.exit(1) 31 | 32 | # print(resp["response"]) 33 | print(resp) 34 | 35 | sys.exit(0) 36 | 37 | ############################################################################ 38 | 39 | -------------------------------------------------------------------------------- /test-api/test-library-remove-document: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | import base64 7 | import time 8 | 9 | url = "http://localhost:8088/api/v1/" 10 | 11 | ############################################################################ 12 | 13 | id = "http://trustgraph.ai/doc/9fdee98b-b259-40ac-bcb9-8e82ccedeb04" 14 | 15 | input = { 16 | "operation": "remove-document", 17 | "user": "trustgraph", 18 | "document-id": id 19 | } 20 | 21 | resp = requests.post( 22 | f"{url}librarian", 23 | json=input, 24 | ) 25 | 26 | print(resp.text) 27 | resp = resp.json() 28 | 29 | print(resp) 30 | 31 | if "error" in resp: 32 | print(f"Error: {resp['error']}") 33 | sys.exit(1) 34 | 35 | # print(resp["response"]) 36 | print(resp) 37 | 38 | sys.exit(0) 39 | 40 | ############################################################################ 41 | 42 | -------------------------------------------------------------------------------- /test-api/test-library-remove-document2: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | import base64 7 | import time 8 | 9 | url = "http://localhost:8088/api/v1/" 10 | 11 | ############################################################################ 12 | 13 | id = "http://trustgraph.ai/doc/6d034da9-2759-45c2-af24-14db7f4c44c2" 14 | 15 | input = { 16 | "operation": "remove-document", 17 | "user": "trustgraph", 18 | "document-id": id 19 | } 20 | 21 | resp = requests.post( 22 | f"{url}librarian", 23 | json=input, 24 | ) 25 | 26 | print(resp.text) 27 | resp = resp.json() 28 | 29 | print(resp) 30 | 31 | if "error" in resp: 32 | print(f"Error: {resp['error']}") 33 | sys.exit(1) 34 | 35 | # print(resp["response"]) 36 | print(resp) 37 | 38 | sys.exit(0) 39 | 40 | ############################################################################ 41 | 42 | -------------------------------------------------------------------------------- /test-api/test-library-remove-processing: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | import base64 7 | import time 8 | 9 | url = "http://localhost:8088/api/v1/" 10 | 11 | ############################################################################ 12 | 13 | proc_id = "2714fc72-44ab-45f2-94dd-6773fc336535" 14 | 15 | input = { 16 | "operation": "remove-processing", 17 | "user": "trustgraph", 18 | "processing-id": proc_id, 19 | } 20 | 21 | resp = requests.post( 22 | f"{url}librarian", 23 | json=input, 24 | ) 25 | 26 | print(resp.text) 27 | resp = resp.json() 28 | 29 | print(resp) 30 | 31 | if "error" in resp: 32 | print(f"Error: {resp['error']}") 33 | sys.exit(1) 34 | 35 | # print(resp["response"]) 36 | print(resp) 37 | 38 | sys.exit(0) 39 | 40 | ############################################################################ 41 | 42 | -------------------------------------------------------------------------------- /test-api/test-llm-api: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | 7 | url = "http://localhost:8088/api/v1/flow/0000/text-completion" 8 | 9 | ############################################################################ 10 | 11 | input = { 12 | "system": "Respond in French. Use long word, form of numbers, no digits", 13 | # "prompt": "Add 2 and 12" 14 | "prompt": "Add 12 and 14, and then make a poem about llamas which incorporates that number. Then write a joke about llamas" 15 | } 16 | 17 | resp = requests.post( 18 | url, 19 | json=input, 20 | ) 21 | 22 | if resp.status_code != 200: 23 | raise RuntimeError(f"Status code: {resp.status_code}") 24 | 25 | resp = resp.json() 26 | 27 | if "error" in resp: 28 | print(f"Error: {resp['error']}") 29 | sys.exit(1) 30 | 31 | print(resp["response"]) 32 | 33 | ############################################################################ 34 | 35 | -------------------------------------------------------------------------------- /test-api/test-llm2-api: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | 7 | url = "http://localhost:8088/api/v1/" 8 | 9 | ############################################################################ 10 | 11 | input = { 12 | "system": "", 13 | "prompt": "Add 2 and 3" 14 | } 15 | 16 | resp = requests.post( 17 | f"{url}text-completion", 18 | json=input, 19 | ) 20 | 21 | if resp.status_code != 200: 22 | raise RuntimeError(f"Status code: {resp.status_code}") 23 | 24 | resp = resp.json() 25 | 26 | if "error" in resp: 27 | print(f"Error: {resp['error']}") 28 | sys.exit(1) 29 | 30 | print(resp["response"]) 31 | 32 | ############################################################################ 33 | 34 | -------------------------------------------------------------------------------- /test-api/test-prompt-api: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | 7 | url = "http://localhost:8088/api/v1/flow/0000/prompt" 8 | 9 | ############################################################################ 10 | 11 | input = { 12 | "id": "question", 13 | "variables": { 14 | "question": "Write a joke about llamas." 15 | } 16 | } 17 | 18 | resp = requests.post( 19 | url, 20 | json=input, 21 | ) 22 | 23 | resp = resp.json() 24 | 25 | if "error" in resp: 26 | print(f"Error: {resp['error']}") 27 | sys.exit(1) 28 | 29 | if "object" in resp: 30 | print(f"Object: {resp['object']}") 31 | sys.exit(1) 32 | 33 | print(resp["text"]) 34 | 35 | sys.exit(0) 36 | ############################################################################ 37 | 38 | -------------------------------------------------------------------------------- /test-api/test-prompt2-api: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | 7 | url = "http://localhost:8088/api/v1/flow/0000/prompt" 8 | 9 | ############################################################################ 10 | 11 | input = { 12 | "id": "extract-definitions", 13 | "variables": { 14 | "text": "A cat is a large mammal." 15 | } 16 | } 17 | 18 | resp = requests.post( 19 | url, 20 | json=input, 21 | ) 22 | 23 | resp = resp.json() 24 | 25 | if "error" in resp: 26 | print(f"Error: {resp['error']}") 27 | sys.exit(1) 28 | 29 | if "object" in resp: 30 | object = json.loads(resp["object"]) 31 | print(json.dumps(object, indent=4)) 32 | sys.exit(1) 33 | 34 | print(resp["text"]) 35 | 36 | sys.exit(0) 37 | ############################################################################ 38 | 39 | -------------------------------------------------------------------------------- /test-api/test-triples-query-api: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | import sys 6 | 7 | url = "http://localhost:8088/api/v1/" 8 | 9 | ############################################################################ 10 | 11 | input = { 12 | "p": { 13 | "v": "http://www.w3.org/2000/01/rdf-schema#label", 14 | "e": True, 15 | }, 16 | "limit": 10 17 | } 18 | 19 | resp = requests.post( 20 | f"{url}triples-query", 21 | json=input, 22 | ) 23 | 24 | print(resp.text) 25 | resp = resp.json() 26 | 27 | 28 | print(resp) 29 | if "error" in resp: 30 | print(f"Error: {resp['error']}") 31 | sys.exit(1) 32 | 33 | print(resp["response"]) 34 | 35 | sys.exit(0) 36 | 37 | ############################################################################ 38 | 39 | -------------------------------------------------------------------------------- /tests/query: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.graph_rag import GraphRag 4 | import sys 5 | 6 | query = " ".join(sys.argv[1:]) 7 | 8 | gr = GraphRag( 9 | verbose=True, 10 | pulsar_host="pulsar://localhost:6650", 11 | pr_request_queue="non-persistent://tg/request/prompt", 12 | pr_response_queue="non-persistent://tg/response/prompt-response", 13 | ) 14 | 15 | if query == "": 16 | query="""This knowledge graph describes the Space Shuttle disaster. 17 | Present 20 facts which are present in the knowledge graph.""" 18 | 19 | resp = gr.query(query) 20 | print(resp) 21 | 22 | -------------------------------------------------------------------------------- /tests/test-agent: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import textwrap 5 | from trustgraph.clients.agent_client import AgentClient 6 | 7 | def wrap(text, width=75): 8 | 9 | if text is None: text = "n/a" 10 | 11 | out = textwrap.wrap( 12 | text, width=width 13 | ) 14 | return "\n".join(out) 15 | 16 | def output(text, prefix="> ", width=78): 17 | 18 | out = textwrap.indent( 19 | text, prefix=prefix 20 | ) 21 | print(out) 22 | 23 | p = AgentClient( 24 | pulsar_host="pulsar://pulsar:6650", 25 | input_queue = "non-persistent://tg/request/agent:0000", 26 | output_queue = "non-persistent://tg/response/agent:0000", 27 | ) 28 | 29 | q = "How many cats does Mark have? Calculate that number raised to 0.4 power. Is that number lower than the numeric part of the mission identifier of the Space Shuttle Challenger on its last mission? If so, give me an apple pie recipe, otherwise return a poem about cheese." 30 | 31 | output(wrap(q), "\U00002753 ") 32 | print() 33 | 34 | def think(x): 35 | output(wrap(x), "\U0001f914 ") 36 | print() 37 | 38 | def observe(x): 39 | output(wrap(x), "\U0001f4a1 ") 40 | print() 41 | 42 | resp = p.request( 43 | question=q, think=think, observe=observe, 44 | ) 45 | 46 | output(resp, "\U0001f4ac ") 47 | print() 48 | 49 | -------------------------------------------------------------------------------- /tests/test-config: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | -------------------------------------------------------------------------------- /tests/test-doc-embeddings: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pulsar 4 | from trustgraph.clients.document_embeddings_client import DocumentEmbeddingsClient 5 | from trustgraph.clients.embeddings_client import EmbeddingsClient 6 | 7 | ec = EmbeddingsClient(pulsar_host="pulsar://localhost:6650") 8 | 9 | vectors = ec.request("What caused the space shuttle to explode?") 10 | 11 | print(vectors) 12 | 13 | llm = DocumentEmbeddingsClient(pulsar_host="pulsar://localhost:6650") 14 | 15 | limit=10 16 | 17 | resp = llm.request(vectors, limit) 18 | 19 | print("Response...") 20 | for val in resp: 21 | print(val) 22 | 23 | -------------------------------------------------------------------------------- /tests/test-doc-prompt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pulsar 4 | from trustgraph.clients.prompt_client import PromptClient 5 | 6 | p = PromptClient(pulsar_host="pulsar://localhost:6650") 7 | 8 | docs = [ 9 | "In our house there is a big cat and a small cat.", 10 | "The small cat is black.", 11 | "The big cat is called Fred.", 12 | "The orange stripey cat is big.", 13 | "The black cat pounces on the big cat.", 14 | "The black cat is called Hope." 15 | ] 16 | 17 | query="What is the name of the cat who pounces on Fred? Provide a full explanation." 18 | 19 | resp = p.request_document_prompt( 20 | query=query, 21 | documents=docs, 22 | ) 23 | 24 | print(resp) 25 | 26 | -------------------------------------------------------------------------------- /tests/test-doc-rag: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pulsar 4 | from trustgraph.clients.document_rag_client import DocumentRagClient 5 | 6 | rag = DocumentRagClient( 7 | pulsar_host="pulsar://localhost:6650", 8 | subscriber="test1", 9 | input_queue = "non-persistent://tg/request/document-rag:default", 10 | output_queue = "non-persistent://tg/response/document-rag:default", 11 | ) 12 | 13 | query=""" 14 | What was the cause of the space shuttle disaster?""" 15 | 16 | resp = rag.request(query) 17 | 18 | print(resp) 19 | 20 | -------------------------------------------------------------------------------- /tests/test-embeddings: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pulsar 4 | from trustgraph.clients.embeddings_client import EmbeddingsClient 5 | 6 | embed = EmbeddingsClient( 7 | pulsar_host="pulsar://pulsar:6650", 8 | input_queue="non-persistent://tg/request/embeddings:default", 9 | output_queue="non-persistent://tg/response/embeddings:default", 10 | subscriber="test1", 11 | ) 12 | 13 | prompt="Write a funny limerick about a llama" 14 | 15 | resp = embed.request(prompt) 16 | 17 | print(resp) 18 | 19 | -------------------------------------------------------------------------------- /tests/test-flow: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | 5 | url = "http://localhost:8088/" 6 | 7 | resp = requests.post( 8 | f"{url}/api/v1/flow", 9 | json={ 10 | "operation": "list-classes", 11 | } 12 | ) 13 | 14 | print(resp) 15 | print(resp.text) 16 | 17 | resp = requests.post( 18 | f"{url}/api/v1/flow", 19 | json={ 20 | "operation": "get-class", 21 | "class-name": "default", 22 | } 23 | ) 24 | 25 | print(resp) 26 | print(resp.text) 27 | 28 | resp = requests.post( 29 | f"{url}/api/v1/flow", 30 | json={ 31 | "operation": "put-class", 32 | "class-name": "bunch", 33 | "class-definition": "{}", 34 | } 35 | ) 36 | 37 | print(resp) 38 | print(resp.text) 39 | 40 | resp = requests.post( 41 | f"{url}/api/v1/flow", 42 | json={ 43 | "operation": "get-class", 44 | "class-name": "bunch", 45 | } 46 | ) 47 | 48 | print(resp) 49 | print(resp.text) 50 | 51 | resp = requests.post( 52 | f"{url}/api/v1/flow", 53 | json={ 54 | "operation": "list-classes", 55 | } 56 | ) 57 | 58 | print(resp) 59 | print(resp.text) 60 | 61 | 62 | resp = requests.post( 63 | f"{url}/api/v1/flow", 64 | json={ 65 | "operation": "delete-class", 66 | "class-name": "bunch", 67 | } 68 | ) 69 | 70 | print(resp) 71 | print(resp.text) 72 | 73 | 74 | resp = requests.post( 75 | f"{url}/api/v1/flow", 76 | json={ 77 | "operation": "list-classes", 78 | } 79 | ) 80 | 81 | print(resp) 82 | print(resp.text) 83 | 84 | resp = requests.post( 85 | f"{url}/api/v1/flow", 86 | json={ 87 | "operation": "list-flows", 88 | } 89 | ) 90 | 91 | print(resp) 92 | print(resp.text) 93 | -------------------------------------------------------------------------------- /tests/test-flow-get-class: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | 5 | url = "http://localhost:8088/" 6 | 7 | resp = requests.post( 8 | f"{url}/api/v1/flow", 9 | json={ 10 | "operation": "get-class", 11 | "class-name": "default", 12 | } 13 | ) 14 | 15 | resp = resp.json() 16 | 17 | print(resp["class-definition"]) 18 | 19 | 20 | -------------------------------------------------------------------------------- /tests/test-flow-start-flow: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | 6 | url = "http://localhost:8088/" 7 | 8 | resp = requests.post( 9 | f"{url}/api/v1/flow", 10 | json={ 11 | "operation": "start-flow", 12 | "flow-id": "0003", 13 | "class-name": "default", 14 | } 15 | ) 16 | 17 | print(resp) 18 | print(resp.text) 19 | resp = resp.json() 20 | 21 | 22 | print(resp) 23 | 24 | -------------------------------------------------------------------------------- /tests/test-flow-stop-flow: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import requests 4 | import json 5 | 6 | url = "http://localhost:8088/" 7 | 8 | resp = requests.post( 9 | f"{url}/api/v1/flow", 10 | json={ 11 | "operation": "stop-flow", 12 | "flow-id": "0003", 13 | } 14 | ) 15 | 16 | print(resp) 17 | print(resp.text) 18 | resp = resp.json() 19 | 20 | 21 | print(resp) 22 | 23 | -------------------------------------------------------------------------------- /tests/test-get-config: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pulsar 4 | from trustgraph.clients.config_client import ConfigClient 5 | 6 | cli = ConfigClient(pulsar_host="pulsar://localhost:6650") 7 | 8 | resp = cli.request_config() 9 | 10 | print(resp) 11 | 12 | -------------------------------------------------------------------------------- /tests/test-graph-embeddings: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pulsar 4 | from trustgraph.clients.graph_embeddings_client import GraphEmbeddingsClient 5 | from trustgraph.clients.embeddings_client import EmbeddingsClient 6 | 7 | ec = EmbeddingsClient(pulsar_host="pulsar://localhost:6650") 8 | 9 | vectors = ec.request("What caused the space shuttle to explode?") 10 | 11 | print(vectors) 12 | 13 | llm = GraphEmbeddingsClient(pulsar_host="pulsar://localhost:6650") 14 | 15 | limit=10 16 | 17 | resp = llm.request(vectors, limit) 18 | 19 | print("Response...") 20 | for val in resp: 21 | print(val.value) 22 | 23 | -------------------------------------------------------------------------------- /tests/test-graph-rag: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pulsar 4 | from trustgraph.clients.graph_rag_client import GraphRagClient 5 | 6 | rag = GraphRagClient( 7 | pulsar_host="pulsar://localhost:6650", 8 | subscriber="test1", 9 | input_queue = "non-persistent://tg/request/graph-rag:default", 10 | output_queue = "non-persistent://tg/response/graph-rag:default", 11 | ) 12 | 13 | #query=""" 14 | #This knowledge graph describes the Space Shuttle disaster. 15 | #Present 20 facts which are present in the knowledge graph.""" 16 | 17 | query = "How many cats does Mark have?" 18 | 19 | resp = rag.request(query) 20 | 21 | print(resp) 22 | 23 | -------------------------------------------------------------------------------- /tests/test-graph-rag2: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pulsar 4 | from trustgraph.clients.graph_rag_client import GraphRagClient 5 | 6 | rag = GraphRagClient(pulsar_host="pulsar://localhost:6650") 7 | 8 | query="""List 20 key points to describe the research that led to the discovery of Leo VI. 9 | """ 10 | 11 | resp = rag.request(query) 12 | 13 | print(resp) 14 | 15 | -------------------------------------------------------------------------------- /tests/test-lang-definition: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pulsar 4 | from trustgraph.clients.prompt_client import PromptClient 5 | 6 | p = PromptClient(pulsar_host="pulsar://localhost:6650") 7 | 8 | chunk = """I noticed a cat in my garden. It is a four-legged animal 9 | which is a mammal and can be tame or wild. I wonder if it will be friends 10 | with me. I think the cat's name is Fred and it has 4 legs. 11 | 12 | A cat is a small mammal. 13 | 14 | A grapefruit is a citrus fruit. 15 | 16 | """ 17 | 18 | resp = p.request_definitions( 19 | chunk=chunk, 20 | ) 21 | 22 | for d in resp: 23 | print(d.name, ":", d.definition) 24 | 25 | -------------------------------------------------------------------------------- /tests/test-lang-relationships: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pulsar 4 | from trustgraph.clients.prompt_client import PromptClient 5 | 6 | p = PromptClient(pulsar_host="pulsar://localhost:6650") 7 | 8 | chunk = """I noticed a cat in my garden. It is a four-legged animal 9 | which is a mammal and can be tame or wild. I wonder if it will be friends 10 | with me. I think the cat's name is Fred and it has 4 legs""" 11 | 12 | resp = p.request_relationships( 13 | chunk=chunk, 14 | ) 15 | 16 | for d in resp: 17 | print(d.s) 18 | print(" ", d.p) 19 | print(" ", d.o) 20 | print(" ", d.o_entity) 21 | 22 | -------------------------------------------------------------------------------- /tests/test-lang-topics: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pulsar 4 | from trustgraph.clients.prompt_client import PromptClient 5 | 6 | p = PromptClient(pulsar_host="pulsar://localhost:6650") 7 | 8 | chunk = """I noticed a cat in my garden. It is a four-legged animal 9 | which is a mammal and can be tame or wild. I wonder if it will be friends 10 | with me. I think the cat's name is Fred and it has 4 legs""" 11 | 12 | resp = p.request_topics( 13 | chunk=chunk, 14 | ) 15 | 16 | for d in resp: 17 | print(d.topic) 18 | print(" ", d.definition) 19 | 20 | -------------------------------------------------------------------------------- /tests/test-llm: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pulsar 4 | from trustgraph.clients.llm_client import LlmClient 5 | 6 | llm = LlmClient( 7 | pulsar_host="pulsar://pulsar:6650", 8 | input_queue="non-persistent://tg/request/text-completion:default", 9 | output_queue="non-persistent://tg/response/text-completion:default", 10 | subscriber="test1", 11 | ) 12 | 13 | system = "You are a lovely assistant." 14 | prompt="what is 2 + 2 == 5" 15 | 16 | resp = llm.request(system, prompt) 17 | 18 | print(resp) 19 | 20 | -------------------------------------------------------------------------------- /tests/test-llm2: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pulsar 4 | from trustgraph.clients.llm_client import LlmClient 5 | 6 | llm = LlmClient(pulsar_host="pulsar://localhost:6650") 7 | 8 | prompt="What is 2 + 12?" 9 | 10 | try: 11 | resp = llm.request(prompt) 12 | print(resp) 13 | except Exception as e: 14 | print(f"{e.__class__.__name__}: {e}") 15 | 16 | -------------------------------------------------------------------------------- /tests/test-llm3: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pulsar 4 | from trustgraph.clients.llm_client import LlmClient 5 | 6 | llm = LlmClient(pulsar_host="pulsar://localhost:6650") 7 | 8 | prompt="What is 2 + 12?" 9 | 10 | try: 11 | resp = llm.request(prompt) 12 | print(resp) 13 | except Exception as e: 14 | print(f"{e.__class__.__name__}: {e}") 15 | 16 | -------------------------------------------------------------------------------- /tests/test-load-pdf: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pulsar 4 | from pulsar.schema import JsonSchema 5 | import base64 6 | 7 | from trustgraph.schema import Document, Metadata 8 | 9 | client = pulsar.Client("pulsar://localhost:6650", listener_name="localhost") 10 | 11 | prod = client.create_producer( 12 | topic="persistent://tg/flow/document-load:0000", 13 | schema=JsonSchema(Document), 14 | chunking_enabled=True, 15 | ) 16 | 17 | path = "../sources/Challenger-Report-Vol1.pdf" 18 | 19 | with open(path, "rb") as f: 20 | blob = base64.b64encode(f.read()).decode("utf-8") 21 | 22 | message = Document( 23 | metadata = Metadata( 24 | id = "00001", 25 | metadata = [], 26 | user="trustgraph", 27 | collection="default", 28 | ), 29 | data=blob 30 | ) 31 | 32 | prod.send(message) 33 | 34 | prod.close() 35 | client.close() 36 | 37 | -------------------------------------------------------------------------------- /tests/test-load-text: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pulsar 4 | from pulsar.schema import JsonSchema 5 | import base64 6 | 7 | from trustgraph.schema import TextDocument, Metadata 8 | 9 | client = pulsar.Client("pulsar://localhost:6650", listener_name="localhost") 10 | 11 | prod = client.create_producer( 12 | topic="persistent://tg/flow/text-document-load:0000", 13 | schema=JsonSchema(TextDocument), 14 | chunking_enabled=True, 15 | ) 16 | 17 | path = "../trustgraph/docs/README.cats" 18 | 19 | with open(path, "r") as f: 20 | # blob = base64.b64encode(f.read()).decode("utf-8") 21 | blob = f.read() 22 | 23 | message = TextDocument( 24 | metadata = Metadata( 25 | id = "00001", 26 | metadata = [], 27 | user="trustgraph", 28 | collection="default", 29 | ), 30 | text=blob 31 | ) 32 | 33 | prod.send(message) 34 | 35 | prod.close() 36 | client.close() 37 | 38 | -------------------------------------------------------------------------------- /tests/test-milvus: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from langchain_huggingface import HuggingFaceEmbeddings 4 | 5 | from trustgraph.direct.milvus import TripleVectors 6 | 7 | client = TripleVectors() 8 | 9 | embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2") 10 | 11 | text="""A cat is a small animal. A dog is a large animal. 12 | Cats say miaow. Dogs go woof. 13 | """ 14 | 15 | embeds = embeddings.embed_documents([text])[0] 16 | 17 | text2="""If you couldn't download the model due to network issues, as a walkaround, you can use random vectors to represent the text and still finish the example. Just note that the search result won't reflect semantic similarity as the vectors are fake ones. 18 | """ 19 | 20 | embeds2 = embeddings.embed_documents([text2])[0] 21 | 22 | client.insert(embeds, "animals") 23 | client.insert(embeds, "vectors") 24 | 25 | query="""What noise does a cat make?""" 26 | 27 | qembeds = embeddings.embed_documents([query])[0] 28 | 29 | res = client.search( 30 | qembeds, 31 | limit=2 32 | ) 33 | 34 | print(res) 35 | 36 | -------------------------------------------------------------------------------- /tests/test-prompt-analyze: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | from trustgraph.clients.prompt_client import PromptClient 5 | 6 | p = PromptClient(pulsar_host="pulsar://localhost:6650") 7 | 8 | description = """Fred is a 4-legged cat who is 12 years old""" 9 | 10 | resp = p.request( 11 | id="analyze", 12 | terms = { 13 | "description": description, 14 | } 15 | ) 16 | 17 | print(json.dumps(resp, indent=4)) 18 | 19 | -------------------------------------------------------------------------------- /tests/test-prompt-french-question: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pulsar 4 | from trustgraph.clients.prompt_client import PromptClient 5 | 6 | p = PromptClient(pulsar_host="pulsar://localhost:6650") 7 | 8 | question = """What is the square root of 16?""" 9 | 10 | resp = p.request( 11 | id="french-question", 12 | terms = { 13 | "question": question 14 | } 15 | ) 16 | 17 | print(resp) 18 | 19 | -------------------------------------------------------------------------------- /tests/test-prompt-question: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pulsar 4 | from trustgraph.clients.prompt_client import PromptClient 5 | 6 | p = PromptClient( 7 | pulsar_host="pulsar://localhost:6650", 8 | input_queue="non-persistent://tg/request/prompt:default", 9 | output_queue="non-persistent://tg/response/prompt:default", 10 | subscriber="test1", 11 | ) 12 | 13 | question = """What is the square root of 16?""" 14 | 15 | resp = p.request( 16 | id="question", 17 | variables = { 18 | "question": question 19 | } 20 | ) 21 | 22 | print(resp) 23 | 24 | -------------------------------------------------------------------------------- /tests/test-prompt-spanish-question: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pulsar 4 | from trustgraph.clients.prompt_client import PromptClient 5 | 6 | p = PromptClient(pulsar_host="pulsar://localhost:6650") 7 | 8 | question = """What is the square root of 16?""" 9 | 10 | resp = p.request( 11 | id="question", 12 | terms = { 13 | "question": question, 14 | "attitude": "Spanish-speaking bot" 15 | } 16 | ) 17 | 18 | print(resp) 19 | 20 | -------------------------------------------------------------------------------- /tests/test-rows-prompt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import pulsar 4 | from trustgraph.clients.prompt_client import PromptClient 5 | from trustgraph.objects.object import Schema 6 | from trustgraph.objects.field import Field, FieldType 7 | 8 | schema = Schema( 9 | name="actors", 10 | description="actors in this story", 11 | fields=[ 12 | Field( 13 | name="name", type=FieldType.STRING, 14 | description="Name of the animal or person in the story" 15 | ), 16 | Field( 17 | name="legs", type=FieldType.INT, 18 | description="Number of legs of the animal or person" 19 | ), 20 | Field( 21 | name="notes", type=FieldType.STRING, 22 | description="Additional notes or observations about this animal or person" 23 | ), 24 | ] 25 | ) 26 | 27 | chunk = """I noticed a cat in my garden. It is a four-legged animal 28 | which is a mammal and can be tame or wild. I wonder if it will be friends 29 | with me? I think the cat's name is Fred and it has 4 legs. 30 | There is also a dog barking outside. The dog has 4 legs also. 31 | The dog comes to my call when I shout "Come here, Bernard". 32 | 33 | I am also standing in the garden, my name is Steve and I have 2 legs. 34 | 35 | My friend Clifford is coming to visit shortly, he has 3 legs due to 36 | a freak accident at birth. 37 | """ 38 | 39 | p = PromptClient(pulsar_host="pulsar://localhost:6650") 40 | 41 | resp = p.request_rows( 42 | schema=schema, 43 | chunk=chunk, 44 | ) 45 | 46 | for d in resp: 47 | print(f"Name: {d['name']}") 48 | print(f" No. of legs: {d['legs']}") 49 | print(f" Notes: {d['notes']}") 50 | print() 51 | 52 | -------------------------------------------------------------------------------- /tests/test-run-extract-row: -------------------------------------------------------------------------------- 1 | 2 | scripts/object-extract-row \ 3 | -p pulsar://localhost:6650 \ 4 | --field 'name:string:100:pri:Name of the person in the story' \ 5 | --field 'job:string:100::Job title or role' \ 6 | --field 'date:string:20::Date entered into role if known' \ 7 | --field 'supervisor:string:100::Supervisor or manager of this person, if known' \ 8 | --field 'location:string:100::Main base or location of work, if known' \ 9 | --field 'notes:string:1000::Additional notes or observations about this animal or person' \ 10 | --no-metrics \ 11 | --name actors \ 12 | --description 'Relevant people' 13 | 14 | -------------------------------------------------------------------------------- /trustgraph-base/README.md: -------------------------------------------------------------------------------- 1 | See https://trustgraph.ai/ 2 | -------------------------------------------------------------------------------- /trustgraph-base/setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | import os 3 | import importlib 4 | 5 | with open("README.md", "r") as fh: 6 | long_description = fh.read() 7 | 8 | # Load a version number module 9 | spec = importlib.util.spec_from_file_location( 10 | 'version', 'trustgraph/base_version.py' 11 | ) 12 | version_module = importlib.util.module_from_spec(spec) 13 | spec.loader.exec_module(version_module) 14 | 15 | version = version_module.__version__ 16 | 17 | setuptools.setup( 18 | name="trustgraph-base", 19 | version=version, 20 | author="trustgraph.ai", 21 | author_email="security@trustgraph.ai", 22 | description="TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.", 23 | long_description=long_description, 24 | long_description_content_type="text/markdown", 25 | url="https://github.com/trustgraph-ai/trustgraph", 26 | packages=setuptools.find_namespace_packages( 27 | where='./', 28 | ), 29 | classifiers=[ 30 | "Programming Language :: Python :: 3", 31 | "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", 32 | "Operating System :: OS Independent", 33 | ], 34 | python_requires='>=3.8', 35 | download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz", 36 | install_requires=[ 37 | "pulsar-client", 38 | "prometheus-client", 39 | ], 40 | scripts=[ 41 | ] 42 | ) 43 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/api/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . api import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/api/exceptions.py: -------------------------------------------------------------------------------- 1 | 2 | class ProtocolException(Exception): 3 | pass 4 | 5 | class ApplicationException(Exception): 6 | pass 7 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/api/types.py: -------------------------------------------------------------------------------- 1 | 2 | import dataclasses 3 | import datetime 4 | from typing import List 5 | from .. knowledge import hash, Uri, Literal 6 | 7 | @dataclasses.dataclass 8 | class Triple: 9 | s : str 10 | p : str 11 | o : str 12 | 13 | @dataclasses.dataclass 14 | class ConfigKey: 15 | type : str 16 | key : str 17 | 18 | @dataclasses.dataclass 19 | class ConfigValue: 20 | type : str 21 | key : str 22 | value : str 23 | 24 | @dataclasses.dataclass 25 | class DocumentMetadata: 26 | id : str 27 | time : datetime.datetime 28 | kind : str 29 | title : str 30 | comments : str 31 | metadata : List[Triple] 32 | user : str 33 | tags : List[str] 34 | 35 | @dataclasses.dataclass 36 | class ProcessingMetadata: 37 | id : str 38 | document_id : str 39 | time : datetime.datetime 40 | flow : str 41 | user : str 42 | collection : str 43 | tags : List[str] 44 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/base/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . pubsub import PulsarClient 3 | from . async_processor import AsyncProcessor 4 | from . consumer import Consumer 5 | from . producer import Producer 6 | from . publisher import Publisher 7 | from . subscriber import Subscriber 8 | from . metrics import ProcessorMetrics, ConsumerMetrics, ProducerMetrics 9 | from . flow_processor import FlowProcessor 10 | from . consumer_spec import ConsumerSpec 11 | from . setting_spec import SettingSpec 12 | from . producer_spec import ProducerSpec 13 | from . subscriber_spec import SubscriberSpec 14 | from . request_response_spec import RequestResponseSpec 15 | from . llm_service import LlmService, LlmResult 16 | from . embeddings_service import EmbeddingsService 17 | from . embeddings_client import EmbeddingsClientSpec 18 | from . text_completion_client import TextCompletionClientSpec 19 | from . prompt_client import PromptClientSpec 20 | from . triples_store_service import TriplesStoreService 21 | from . graph_embeddings_store_service import GraphEmbeddingsStoreService 22 | from . document_embeddings_store_service import DocumentEmbeddingsStoreService 23 | from . triples_query_service import TriplesQueryService 24 | from . graph_embeddings_query_service import GraphEmbeddingsQueryService 25 | from . document_embeddings_query_service import DocumentEmbeddingsQueryService 26 | from . graph_embeddings_client import GraphEmbeddingsClientSpec 27 | from . triples_client import TriplesClientSpec 28 | from . document_embeddings_client import DocumentEmbeddingsClientSpec 29 | from . agent_service import AgentService 30 | from . graph_rag_client import GraphRagClientSpec 31 | 32 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/base/agent_client.py: -------------------------------------------------------------------------------- 1 | 2 | from . request_response_spec import RequestResponse, RequestResponseSpec 3 | from .. schema import AgentRequest, AgentResponse 4 | from .. knowledge import Uri, Literal 5 | 6 | class AgentClient(RequestResponse): 7 | async def request(self, recipient, question, plan=None, state=None, 8 | history=[], timeout=300): 9 | 10 | resp = await self.request( 11 | AgentRequest( 12 | question = question, 13 | plan = plan, 14 | state = state, 15 | history = history, 16 | ), 17 | recipient=recipient, 18 | timeout=timeout, 19 | ) 20 | 21 | print(resp, flush=True) 22 | 23 | if resp.error: 24 | raise RuntimeError(resp.error.message) 25 | 26 | return resp 27 | 28 | class GraphEmbeddingsClientSpec(RequestResponseSpec): 29 | def __init__( 30 | self, request_name, response_name, 31 | ): 32 | super(GraphEmbeddingsClientSpec, self).__init__( 33 | request_name = request_name, 34 | request_schema = GraphEmbeddingsRequest, 35 | response_name = response_name, 36 | response_schema = GraphEmbeddingsResponse, 37 | impl = GraphEmbeddingsClient, 38 | ) 39 | 40 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/base/consumer_spec.py: -------------------------------------------------------------------------------- 1 | 2 | from . metrics import ConsumerMetrics 3 | from . consumer import Consumer 4 | from . spec import Spec 5 | 6 | class ConsumerSpec(Spec): 7 | def __init__(self, name, schema, handler): 8 | self.name = name 9 | self.schema = schema 10 | self.handler = handler 11 | 12 | def add(self, flow, processor, definition): 13 | 14 | consumer_metrics = ConsumerMetrics( 15 | processor = flow.id, flow = flow.name, name = self.name, 16 | ) 17 | 18 | consumer = Consumer( 19 | taskgroup = processor.taskgroup, 20 | flow = flow, 21 | client = processor.pulsar_client, 22 | topic = definition[self.name], 23 | subscriber = processor.id + "--" + flow.name + "--" + self.name, 24 | schema = self.schema, 25 | handler = self.handler, 26 | metrics = consumer_metrics, 27 | ) 28 | 29 | # Consumer handle gets access to producers and other 30 | # metadata 31 | consumer.id = flow.id 32 | consumer.name = self.name 33 | consumer.flow = flow 34 | 35 | flow.consumer[self.name] = consumer 36 | 37 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/base/document_embeddings_client.py: -------------------------------------------------------------------------------- 1 | 2 | from . request_response_spec import RequestResponse, RequestResponseSpec 3 | from .. schema import DocumentEmbeddingsRequest, DocumentEmbeddingsResponse 4 | from .. knowledge import Uri, Literal 5 | 6 | class DocumentEmbeddingsClient(RequestResponse): 7 | async def query(self, vectors, limit=20, user="trustgraph", 8 | collection="default", timeout=30): 9 | 10 | resp = await self.request( 11 | DocumentEmbeddingsRequest( 12 | vectors = vectors, 13 | limit = limit, 14 | user = user, 15 | collection = collection 16 | ), 17 | timeout=timeout 18 | ) 19 | 20 | print(resp, flush=True) 21 | 22 | if resp.error: 23 | raise RuntimeError(resp.error.message) 24 | 25 | return resp.documents 26 | 27 | class DocumentEmbeddingsClientSpec(RequestResponseSpec): 28 | def __init__( 29 | self, request_name, response_name, 30 | ): 31 | super(DocumentEmbeddingsClientSpec, self).__init__( 32 | request_name = request_name, 33 | request_schema = DocumentEmbeddingsRequest, 34 | response_name = response_name, 35 | response_schema = DocumentEmbeddingsResponse, 36 | impl = DocumentEmbeddingsClient, 37 | ) 38 | 39 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/base/document_embeddings_store_service.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | Document embeddings store base class 4 | """ 5 | 6 | from .. schema import DocumentEmbeddings 7 | from .. base import FlowProcessor, ConsumerSpec 8 | from .. exceptions import TooManyRequests 9 | 10 | default_ident = "document-embeddings-write" 11 | 12 | class DocumentEmbeddingsStoreService(FlowProcessor): 13 | 14 | def __init__(self, **params): 15 | 16 | id = params.get("id") 17 | 18 | super(DocumentEmbeddingsStoreService, self).__init__( 19 | **params | { "id": id } 20 | ) 21 | 22 | self.register_specification( 23 | ConsumerSpec( 24 | name = "input", 25 | schema = DocumentEmbeddings, 26 | handler = self.on_message 27 | ) 28 | ) 29 | 30 | async def on_message(self, msg, consumer, flow): 31 | 32 | try: 33 | 34 | request = msg.value() 35 | 36 | await self.store_document_embeddings(request) 37 | 38 | except TooManyRequests as e: 39 | raise e 40 | 41 | except Exception as e: 42 | 43 | print(f"Exception: {e}") 44 | raise e 45 | 46 | @staticmethod 47 | def add_args(parser): 48 | 49 | FlowProcessor.add_args(parser) 50 | 51 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/base/embeddings_client.py: -------------------------------------------------------------------------------- 1 | 2 | from . request_response_spec import RequestResponse, RequestResponseSpec 3 | from .. schema import EmbeddingsRequest, EmbeddingsResponse 4 | 5 | class EmbeddingsClient(RequestResponse): 6 | async def embed(self, text, timeout=30): 7 | 8 | resp = await self.request( 9 | EmbeddingsRequest( 10 | text = text 11 | ), 12 | timeout=timeout 13 | ) 14 | 15 | if resp.error: 16 | raise RuntimeError(resp.error.message) 17 | 18 | return resp.vectors 19 | 20 | class EmbeddingsClientSpec(RequestResponseSpec): 21 | def __init__( 22 | self, request_name, response_name, 23 | ): 24 | super(EmbeddingsClientSpec, self).__init__( 25 | request_name = request_name, 26 | request_schema = EmbeddingsRequest, 27 | response_name = response_name, 28 | response_schema = EmbeddingsResponse, 29 | impl = EmbeddingsClient, 30 | ) 31 | 32 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/base/flow.py: -------------------------------------------------------------------------------- 1 | 2 | import asyncio 3 | 4 | class Flow: 5 | def __init__(self, id, flow, processor, defn): 6 | 7 | self.id = id 8 | self.name = flow 9 | 10 | self.producer = {} 11 | 12 | # Consumers and publishers. Is this a bit untidy? 13 | self.consumer = {} 14 | 15 | self.setting = {} 16 | 17 | for spec in processor.specifications: 18 | spec.add(self, processor, defn) 19 | 20 | async def start(self): 21 | for c in self.consumer.values(): 22 | await c.start() 23 | 24 | async def stop(self): 25 | for c in self.consumer.values(): 26 | await c.stop() 27 | 28 | def __call__(self, key): 29 | if key in self.producer: return self.producer[key] 30 | if key in self.consumer: return self.consumer[key] 31 | if key in self.setting: return self.setting[key].value 32 | return None 33 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/base/graph_embeddings_client.py: -------------------------------------------------------------------------------- 1 | 2 | from . request_response_spec import RequestResponse, RequestResponseSpec 3 | from .. schema import GraphEmbeddingsRequest, GraphEmbeddingsResponse 4 | from .. knowledge import Uri, Literal 5 | 6 | def to_value(x): 7 | if x.is_uri: return Uri(x.value) 8 | return Literal(x.value) 9 | 10 | class GraphEmbeddingsClient(RequestResponse): 11 | async def query(self, vectors, limit=20, user="trustgraph", 12 | collection="default", timeout=30): 13 | 14 | resp = await self.request( 15 | GraphEmbeddingsRequest( 16 | vectors = vectors, 17 | limit = limit, 18 | user = user, 19 | collection = collection 20 | ), 21 | timeout=timeout 22 | ) 23 | 24 | print(resp, flush=True) 25 | 26 | if resp.error: 27 | raise RuntimeError(resp.error.message) 28 | 29 | return [ 30 | to_value(v) 31 | for v in resp.entities 32 | ] 33 | 34 | class GraphEmbeddingsClientSpec(RequestResponseSpec): 35 | def __init__( 36 | self, request_name, response_name, 37 | ): 38 | super(GraphEmbeddingsClientSpec, self).__init__( 39 | request_name = request_name, 40 | request_schema = GraphEmbeddingsRequest, 41 | response_name = response_name, 42 | response_schema = GraphEmbeddingsResponse, 43 | impl = GraphEmbeddingsClient, 44 | ) 45 | 46 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/base/graph_embeddings_store_service.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | Graph embeddings store base class 4 | """ 5 | 6 | from .. schema import GraphEmbeddings 7 | from .. base import FlowProcessor, ConsumerSpec 8 | from .. exceptions import TooManyRequests 9 | 10 | default_ident = "graph-embeddings-write" 11 | 12 | class GraphEmbeddingsStoreService(FlowProcessor): 13 | 14 | def __init__(self, **params): 15 | 16 | id = params.get("id") 17 | 18 | super(GraphEmbeddingsStoreService, self).__init__( 19 | **params | { "id": id } 20 | ) 21 | 22 | self.register_specification( 23 | ConsumerSpec( 24 | name = "input", 25 | schema = GraphEmbeddings, 26 | handler = self.on_message 27 | ) 28 | ) 29 | 30 | async def on_message(self, msg, consumer, flow): 31 | 32 | try: 33 | 34 | request = msg.value() 35 | 36 | await self.store_graph_embeddings(request) 37 | 38 | except TooManyRequests as e: 39 | raise e 40 | 41 | except Exception as e: 42 | 43 | print(f"Exception: {e}") 44 | raise e 45 | 46 | @staticmethod 47 | def add_args(parser): 48 | 49 | FlowProcessor.add_args(parser) 50 | 51 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/base/graph_rag_client.py: -------------------------------------------------------------------------------- 1 | 2 | from . request_response_spec import RequestResponse, RequestResponseSpec 3 | from .. schema import GraphRagQuery, GraphRagResponse 4 | 5 | class GraphRagClient(RequestResponse): 6 | async def rag(self, query, user="trustgraph", collection="default", 7 | timeout=600): 8 | resp = await self.request( 9 | GraphRagQuery( 10 | query = query, 11 | user = user, 12 | collection = collection, 13 | ), 14 | timeout=timeout 15 | ) 16 | 17 | if resp.error: 18 | raise RuntimeError(resp.error.message) 19 | 20 | return resp.response 21 | 22 | class GraphRagClientSpec(RequestResponseSpec): 23 | def __init__( 24 | self, request_name, response_name, 25 | ): 26 | super(GraphRagClientSpec, self).__init__( 27 | request_name = request_name, 28 | request_schema = GraphRagQuery, 29 | response_name = response_name, 30 | response_schema = GraphRagResponse, 31 | impl = GraphRagClient, 32 | ) 33 | 34 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/base/producer_spec.py: -------------------------------------------------------------------------------- 1 | 2 | from . producer import Producer 3 | from . metrics import ProducerMetrics 4 | from . spec import Spec 5 | 6 | class ProducerSpec(Spec): 7 | def __init__(self, name, schema): 8 | self.name = name 9 | self.schema = schema 10 | 11 | def add(self, flow, processor, definition): 12 | 13 | producer_metrics = ProducerMetrics( 14 | processor = flow.id, flow = flow.name, name = self.name 15 | ) 16 | 17 | producer = Producer( 18 | client = processor.pulsar_client, 19 | topic = definition[self.name], 20 | schema = self.schema, 21 | metrics = producer_metrics, 22 | ) 23 | 24 | flow.producer[self.name] = producer 25 | 26 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/base/setting_spec.py: -------------------------------------------------------------------------------- 1 | 2 | from . spec import Spec 3 | 4 | class Setting: 5 | def __init__(self, value): 6 | self.value = value 7 | async def start(): 8 | pass 9 | async def stop(): 10 | pass 11 | 12 | class SettingSpec(Spec): 13 | def __init__(self, name): 14 | self.name = name 15 | 16 | def add(self, flow, processor, definition): 17 | 18 | flow.config[self.name] = Setting(definition[self.name]) 19 | 20 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/base/spec.py: -------------------------------------------------------------------------------- 1 | 2 | class Spec: 3 | pass 4 | 5 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/base/subscriber_spec.py: -------------------------------------------------------------------------------- 1 | 2 | from . metrics import SubscriberMetrics 3 | from . subscriber import Subscriber 4 | from . spec import Spec 5 | 6 | class SubscriberSpec(Spec): 7 | 8 | def __init__(self, name, schema): 9 | self.name = name 10 | self.schema = schema 11 | 12 | def add(self, flow, processor, definition): 13 | 14 | subscriber_metrics = SubscriberMetrics( 15 | processor = flow.id, flow = flow.name, name = self.name 16 | ) 17 | 18 | subscriber = Subscriber( 19 | client = processor.pulsar_client, 20 | topic = definition[self.name], 21 | subscription = flow.id, 22 | consumer_name = flow.id, 23 | schema = self.schema, 24 | metrics = subscriber_metrics, 25 | ) 26 | 27 | # Put it in the consumer map, does that work? 28 | # It means it gets start/stop call. 29 | flow.consumer[self.name] = subscriber 30 | 31 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/base/text_completion_client.py: -------------------------------------------------------------------------------- 1 | 2 | from . request_response_spec import RequestResponse, RequestResponseSpec 3 | from .. schema import TextCompletionRequest, TextCompletionResponse 4 | 5 | class TextCompletionClient(RequestResponse): 6 | async def text_completion(self, system, prompt, timeout=600): 7 | resp = await self.request( 8 | TextCompletionRequest( 9 | system = system, prompt = prompt 10 | ), 11 | timeout=timeout 12 | ) 13 | 14 | if resp.error: 15 | raise RuntimeError(resp.error.message) 16 | 17 | return resp.response 18 | 19 | class TextCompletionClientSpec(RequestResponseSpec): 20 | def __init__( 21 | self, request_name, response_name, 22 | ): 23 | super(TextCompletionClientSpec, self).__init__( 24 | request_name = request_name, 25 | request_schema = TextCompletionRequest, 26 | response_name = response_name, 27 | response_schema = TextCompletionResponse, 28 | impl = TextCompletionClient, 29 | ) 30 | 31 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/base/triples_store_service.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | Triples store base class 4 | """ 5 | 6 | from .. schema import Triples 7 | from .. base import FlowProcessor, ConsumerSpec 8 | 9 | default_ident = "triples-write" 10 | 11 | class TriplesStoreService(FlowProcessor): 12 | 13 | def __init__(self, **params): 14 | 15 | id = params.get("id") 16 | 17 | super(TriplesStoreService, self).__init__(**params | { "id": id }) 18 | 19 | self.register_specification( 20 | ConsumerSpec( 21 | name = "input", 22 | schema = Triples, 23 | handler = self.on_message 24 | ) 25 | ) 26 | 27 | async def on_message(self, msg, consumer, flow): 28 | 29 | try: 30 | 31 | request = msg.value() 32 | 33 | await self.store_triples(request) 34 | 35 | except TooManyRequests as e: 36 | raise e 37 | 38 | except Exception as e: 39 | 40 | print(f"Exception: {e}") 41 | raise e 42 | 43 | @staticmethod 44 | def add_args(parser): 45 | 46 | FlowProcessor.add_args(parser) 47 | 48 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/clients/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-base/trustgraph/clients/__init__.py -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/clients/document_rag_client.py: -------------------------------------------------------------------------------- 1 | 2 | import _pulsar 3 | 4 | from .. schema import DocumentRagQuery, DocumentRagResponse 5 | from .. schema import document_rag_request_queue, document_rag_response_queue 6 | from . base import BaseClient 7 | 8 | # Ugly 9 | ERROR=_pulsar.LoggerLevel.Error 10 | WARN=_pulsar.LoggerLevel.Warn 11 | INFO=_pulsar.LoggerLevel.Info 12 | DEBUG=_pulsar.LoggerLevel.Debug 13 | 14 | class DocumentRagClient(BaseClient): 15 | 16 | def __init__( 17 | self, 18 | log_level=ERROR, 19 | subscriber=None, 20 | input_queue=None, 21 | output_queue=None, 22 | pulsar_host="pulsar://pulsar:6650", 23 | pulsar_api_key=None, 24 | ): 25 | 26 | if input_queue == None: 27 | input_queue = document_rag_request_queue 28 | 29 | if output_queue == None: 30 | output_queue = document_rag_response_queue 31 | 32 | super(DocumentRagClient, self).__init__( 33 | log_level=log_level, 34 | subscriber=subscriber, 35 | input_queue=input_queue, 36 | output_queue=output_queue, 37 | pulsar_host=pulsar_host, 38 | pulsar_api_key=pulsar_api_key, 39 | input_schema=DocumentRagQuery, 40 | output_schema=DocumentRagResponse, 41 | ) 42 | 43 | def request(self, query, timeout=300): 44 | 45 | return self.call( 46 | query=query, timeout=timeout 47 | ).response 48 | 49 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/clients/embeddings_client.py: -------------------------------------------------------------------------------- 1 | 2 | from pulsar.schema import JsonSchema 3 | from .. schema import EmbeddingsRequest, EmbeddingsResponse 4 | from . base import BaseClient 5 | 6 | import _pulsar 7 | 8 | # Ugly 9 | ERROR=_pulsar.LoggerLevel.Error 10 | WARN=_pulsar.LoggerLevel.Warn 11 | INFO=_pulsar.LoggerLevel.Info 12 | DEBUG=_pulsar.LoggerLevel.Debug 13 | 14 | class EmbeddingsClient(BaseClient): 15 | 16 | def __init__( 17 | self, log_level=ERROR, 18 | input_queue=None, 19 | output_queue=None, 20 | subscriber=None, 21 | pulsar_host="pulsar://pulsar:6650", 22 | pulsar_api_key=None, 23 | ): 24 | 25 | super(EmbeddingsClient, self).__init__( 26 | log_level=log_level, 27 | subscriber=subscriber, 28 | input_queue=input_queue, 29 | output_queue=output_queue, 30 | pulsar_host=pulsar_host, 31 | pulsar_api_key=pulsar_api_key, 32 | input_schema=EmbeddingsRequest, 33 | output_schema=EmbeddingsResponse, 34 | ) 35 | 36 | def request(self, text, timeout=300): 37 | return self.call(text=text, timeout=timeout).vectors 38 | 39 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/clients/graph_rag_client.py: -------------------------------------------------------------------------------- 1 | 2 | import _pulsar 3 | 4 | from .. schema import GraphRagQuery, GraphRagResponse 5 | from .. schema import graph_rag_request_queue, graph_rag_response_queue 6 | from . base import BaseClient 7 | 8 | # Ugly 9 | ERROR=_pulsar.LoggerLevel.Error 10 | WARN=_pulsar.LoggerLevel.Warn 11 | INFO=_pulsar.LoggerLevel.Info 12 | DEBUG=_pulsar.LoggerLevel.Debug 13 | 14 | class GraphRagClient(BaseClient): 15 | 16 | def __init__( 17 | self, 18 | log_level=ERROR, 19 | subscriber=None, 20 | input_queue=None, 21 | output_queue=None, 22 | pulsar_host="pulsar://pulsar:6650", 23 | pulsar_api_key=None, 24 | ): 25 | 26 | if input_queue == None: 27 | input_queue = graph_rag_request_queue 28 | 29 | if output_queue == None: 30 | output_queue = graph_rag_response_queue 31 | 32 | super(GraphRagClient, self).__init__( 33 | log_level=log_level, 34 | subscriber=subscriber, 35 | input_queue=input_queue, 36 | output_queue=output_queue, 37 | pulsar_host=pulsar_host, 38 | pulsar_api_key=pulsar_api_key, 39 | input_schema=GraphRagQuery, 40 | output_schema=GraphRagResponse, 41 | ) 42 | 43 | def request( 44 | self, query, user="trustgraph", collection="default", 45 | timeout=500 46 | ): 47 | 48 | return self.call( 49 | user=user, collection=collection, query=query, timeout=timeout 50 | ).response 51 | 52 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/clients/llm_client.py: -------------------------------------------------------------------------------- 1 | 2 | import _pulsar 3 | 4 | from .. schema import TextCompletionRequest, TextCompletionResponse 5 | from .. schema import text_completion_request_queue 6 | from .. schema import text_completion_response_queue 7 | from . base import BaseClient 8 | 9 | # Ugly 10 | ERROR=_pulsar.LoggerLevel.Error 11 | WARN=_pulsar.LoggerLevel.Warn 12 | INFO=_pulsar.LoggerLevel.Info 13 | DEBUG=_pulsar.LoggerLevel.Debug 14 | 15 | class LlmClient(BaseClient): 16 | 17 | def __init__( 18 | self, log_level=ERROR, 19 | subscriber=None, 20 | input_queue=None, 21 | output_queue=None, 22 | pulsar_host="pulsar://pulsar:6650", 23 | pulsar_api_key=None, 24 | ): 25 | 26 | if input_queue is None: input_queue = text_completion_request_queue 27 | if output_queue is None: output_queue = text_completion_response_queue 28 | 29 | super(LlmClient, self).__init__( 30 | log_level=log_level, 31 | subscriber=subscriber, 32 | input_queue=input_queue, 33 | output_queue=output_queue, 34 | pulsar_host=pulsar_host, 35 | pulsar_api_key=pulsar_api_key, 36 | input_schema=TextCompletionRequest, 37 | output_schema=TextCompletionResponse, 38 | ) 39 | 40 | def request(self, system, prompt, timeout=300): 41 | return self.call( 42 | system=system, prompt=prompt, timeout=timeout 43 | ).response 44 | 45 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/exceptions.py: -------------------------------------------------------------------------------- 1 | 2 | class TooManyRequests(Exception): 3 | pass 4 | 5 | class LlmError(Exception): 6 | pass 7 | 8 | class ParseError(Exception): 9 | pass 10 | 11 | class RequestError(Exception): 12 | pass 13 | 14 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/knowledge/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . defs import * 3 | from . identifier import * 4 | from . publication import * 5 | from . document import * 6 | from . organization import * 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/knowledge/defs.py: -------------------------------------------------------------------------------- 1 | 2 | IS_A = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 3 | LABEL = 'http://www.w3.org/2000/01/rdf-schema#label' 4 | 5 | DIGITAL_DOCUMENT = 'https://schema.org/DigitalDocument' 6 | PUBLICATION_EVENT = 'https://schema.org/PublicationEvent' 7 | ORGANIZATION = 'https://schema.org/Organization' 8 | 9 | NAME = 'https://schema.org/name' 10 | DESCRIPTION = 'https://schema.org/description' 11 | COPYRIGHT_NOTICE = 'https://schema.org/copyrightNotice' 12 | COPYRIGHT_HOLDER = 'https://schema.org/copyrightHolder' 13 | COPYRIGHT_YEAR = 'https://schema.org/copyrightYear' 14 | LICENSE = 'https://schema.org/license' 15 | PUBLICATION = 'https://schema.org/publication' 16 | START_DATE = 'https://schema.org/startDate' 17 | END_DATE = 'https://schema.org/endDate' 18 | PUBLISHED_BY = 'https://schema.org/publishedBy' 19 | DATE_PUBLISHED = 'https://schema.org/datePublished' 20 | PUBLICATION = 'https://schema.org/publication' 21 | DATE_PUBLISHED = 'https://schema.org/datePublished' 22 | URL = 'https://schema.org/url' 23 | IDENTIFIER = 'https://schema.org/identifier' 24 | KEYWORD = 'https://schema.org/keywords' 25 | 26 | class Uri(str): 27 | def is_uri(self): return True 28 | def is_literal(self): return False 29 | 30 | class Literal(str): 31 | def is_uri(self): return False 32 | def is_literal(self): return True 33 | 34 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/knowledge/identifier.py: -------------------------------------------------------------------------------- 1 | 2 | import uuid 3 | import hashlib 4 | 5 | def hash(data): 6 | 7 | if isinstance(data, str): 8 | data = data.encode("utf-8") 9 | 10 | # Create a SHA256 hash from the data 11 | id = hashlib.sha256(data).hexdigest() 12 | 13 | # Convert into a UUID, 64-byte hash becomes 32-byte UUID 14 | id = str(uuid.UUID(id[::2])) 15 | 16 | return id 17 | 18 | def to_uri(pref, id): 19 | return f"https://trustgraph.ai/{pref}/{id}" 20 | 21 | PREF_PUBEV = "pubev" 22 | PREF_ORG = "org" 23 | PREF_DOC = "doc" 24 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/knowledge/organization.py: -------------------------------------------------------------------------------- 1 | 2 | from . defs import * 3 | 4 | def Value(value, is_uri): 5 | if is_uri: 6 | return Uri(value) 7 | else: 8 | return Literal(value) 9 | 10 | def Triple(s, p, o): 11 | return { 12 | "s": s, "p": p, "o": o, 13 | } 14 | 15 | class Organization: 16 | def __init__(self, id, name=None, description=None): 17 | self.id = id 18 | self.name = name 19 | self.description = description 20 | 21 | def emit(self, emit): 22 | 23 | emit(Triple( 24 | s=Value(value=self.id, is_uri=True), 25 | p=Value(value=IS_A, is_uri=True), 26 | o=Value(value=ORGANIZATION, is_uri=True) 27 | )) 28 | 29 | if self.name: 30 | 31 | emit(Triple( 32 | s=Value(value=self.id, is_uri=True), 33 | p=Value(value=LABEL, is_uri=True), 34 | o=Value(value=self.name, is_uri=False) 35 | )) 36 | 37 | emit(Triple( 38 | s=Value(value=self.id, is_uri=True), 39 | p=Value(value=NAME, is_uri=True), 40 | o=Value(value=self.name, is_uri=False) 41 | )) 42 | 43 | if self.description: 44 | 45 | emit(Triple( 46 | s=Value(value=self.id, is_uri=True), 47 | p=Value(value=DESCRIPTION, is_uri=True), 48 | o=Value(value=self.description, is_uri=False) 49 | )) 50 | 51 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/log_level.py: -------------------------------------------------------------------------------- 1 | 2 | from enum import Enum 3 | import _pulsar 4 | 5 | class LogLevel(Enum): 6 | DEBUG = 'debug' 7 | INFO = 'info' 8 | WARN = 'warn' 9 | ERROR = 'error' 10 | 11 | def __str__(self): 12 | return self.value 13 | 14 | def to_pulsar(self): 15 | if self == LogLevel.DEBUG: return _pulsar.LoggerLevel.Debug 16 | if self == LogLevel.INFO: return _pulsar.LoggerLevel.Info 17 | if self == LogLevel.WARN: return _pulsar.LoggerLevel.Warn 18 | if self == LogLevel.ERROR: return _pulsar.LoggerLevel.Error 19 | raise RuntimeError("Log level mismatch") 20 | 21 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/objects/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-base/trustgraph/objects/__init__.py -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/objects/object.py: -------------------------------------------------------------------------------- 1 | 2 | class Schema: 3 | def __init__(self, name, description, fields): 4 | self.name = name 5 | self.description = description 6 | self.fields = fields 7 | 8 | 9 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/rdf.py: -------------------------------------------------------------------------------- 1 | 2 | RDF_LABEL = "http://www.w3.org/2000/01/rdf-schema#label" 3 | DEFINITION = "http://www.w3.org/2004/02/skos/core#definition" 4 | SUBJECT_OF = "https://schema.org/subjectOf" 5 | 6 | TRUSTGRAPH_ENTITIES = "http://trustgraph.ai/e/" 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/schema/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . types import * 3 | from . prompt import * 4 | from . documents import * 5 | from . models import * 6 | from . object import * 7 | from . topic import * 8 | from . graph import * 9 | from . retrieval import * 10 | from . metadata import * 11 | from . agent import * 12 | from . lookup import * 13 | from . library import * 14 | from . config import * 15 | from . flows import * 16 | from . knowledge import * 17 | 18 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/schema/agent.py: -------------------------------------------------------------------------------- 1 | 2 | from pulsar.schema import Record, String, Array, Map 3 | 4 | from . topic import topic 5 | from . types import Error 6 | 7 | ############################################################################ 8 | 9 | # Prompt services, abstract the prompt generation 10 | 11 | class AgentStep(Record): 12 | thought = String() 13 | action = String() 14 | arguments = Map(String()) 15 | observation = String() 16 | 17 | class AgentRequest(Record): 18 | question = String() 19 | plan = String() 20 | state = String() 21 | history = Array(AgentStep()) 22 | 23 | class AgentResponse(Record): 24 | answer = String() 25 | error = Error() 26 | thought = String() 27 | observation = String() 28 | 29 | ############################################################################ 30 | 31 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/schema/documents.py: -------------------------------------------------------------------------------- 1 | 2 | from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array, Double 3 | from . topic import topic 4 | from . types import Error 5 | from . metadata import Metadata 6 | 7 | ############################################################################ 8 | 9 | # PDF docs etc. 10 | class Document(Record): 11 | metadata = Metadata() 12 | data = Bytes() 13 | 14 | ############################################################################ 15 | 16 | # Text documents / text from PDF 17 | 18 | class TextDocument(Record): 19 | metadata = Metadata() 20 | text = Bytes() 21 | 22 | ############################################################################ 23 | 24 | # Chunks of text 25 | 26 | class Chunk(Record): 27 | metadata = Metadata() 28 | chunk = Bytes() 29 | 30 | ############################################################################ 31 | 32 | # Document embeddings are embeddings associated with a chunk 33 | 34 | class ChunkEmbeddings(Record): 35 | chunk = Bytes() 36 | vectors = Array(Array(Double())) 37 | 38 | # This is a 'batching' mechanism for the above data 39 | class DocumentEmbeddings(Record): 40 | metadata = Metadata() 41 | chunks = Array(ChunkEmbeddings()) 42 | 43 | ############################################################################ 44 | 45 | # Doc embeddings query 46 | 47 | class DocumentEmbeddingsRequest(Record): 48 | vectors = Array(Array(Double())) 49 | limit = Integer() 50 | user = String() 51 | collection = String() 52 | 53 | class DocumentEmbeddingsResponse(Record): 54 | error = Error() 55 | documents = Array(Bytes()) 56 | 57 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/schema/knowledge.py: -------------------------------------------------------------------------------- 1 | 2 | from pulsar.schema import Record, Bytes, String, Array, Long, Boolean 3 | from . types import Triple 4 | from . topic import topic 5 | from . types import Error 6 | from . metadata import Metadata 7 | from . documents import Document, TextDocument 8 | from . graph import Triples, GraphEmbeddings 9 | 10 | # get-kg-core 11 | # -> (???) 12 | # <- () 13 | # <- (error) 14 | 15 | # delete-kg-core 16 | # -> (???) 17 | # <- () 18 | # <- (error) 19 | 20 | # list-kg-cores 21 | # -> (user) 22 | # <- () 23 | # <- (error) 24 | 25 | class KnowledgeRequest(Record): 26 | 27 | # get-kg-core, delete-kg-core, list-kg-cores, put-kg-core 28 | # load-kg-core, unload-kg-core 29 | operation = String() 30 | 31 | # list-kg-cores, delete-kg-core, put-kg-core 32 | user = String() 33 | 34 | # get-kg-core, list-kg-cores, delete-kg-core, put-kg-core, 35 | # load-kg-core, unload-kg-core 36 | id = String() 37 | 38 | # load-kg-core 39 | flow = String() 40 | 41 | # load-kg-core 42 | collection = String() 43 | 44 | # put-kg-core 45 | triples = Triples() 46 | graph_embeddings = GraphEmbeddings() 47 | 48 | class KnowledgeResponse(Record): 49 | error = Error() 50 | ids = Array(String()) 51 | eos = Boolean() # Indicates end of knowledge core stream 52 | triples = Triples() 53 | graph_embeddings = GraphEmbeddings() 54 | 55 | knowledge_request_queue = topic( 56 | 'knowledge', kind='non-persistent', namespace='request' 57 | ) 58 | knowledge_response_queue = topic( 59 | 'knowledge', kind='non-persistent', namespace='response', 60 | ) 61 | 62 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/schema/lookup.py: -------------------------------------------------------------------------------- 1 | 2 | from pulsar.schema import Record, String 3 | 4 | from . types import Error, Value, Triple 5 | from . topic import topic 6 | from . metadata import Metadata 7 | 8 | ############################################################################ 9 | 10 | # Lookups 11 | 12 | class LookupRequest(Record): 13 | kind = String() 14 | term = String() 15 | 16 | class LookupResponse(Record): 17 | text = String() 18 | error = Error() 19 | 20 | ############################################################################ 21 | 22 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/schema/metadata.py: -------------------------------------------------------------------------------- 1 | 2 | from pulsar.schema import Record, String, Array 3 | from . types import Triple 4 | 5 | class Metadata(Record): 6 | 7 | # Source identifier 8 | id = String() 9 | 10 | # Subgraph 11 | metadata = Array(Triple()) 12 | 13 | # Collection management 14 | user = String() 15 | collection = String() 16 | 17 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/schema/models.py: -------------------------------------------------------------------------------- 1 | 2 | from pulsar.schema import Record, String, Array, Double, Integer 3 | 4 | from . topic import topic 5 | from . types import Error 6 | 7 | ############################################################################ 8 | 9 | # LLM text completion 10 | 11 | class TextCompletionRequest(Record): 12 | system = String() 13 | prompt = String() 14 | 15 | class TextCompletionResponse(Record): 16 | error = Error() 17 | response = String() 18 | in_token = Integer() 19 | out_token = Integer() 20 | model = String() 21 | 22 | ############################################################################ 23 | 24 | # Embeddings 25 | 26 | class EmbeddingsRequest(Record): 27 | text = String() 28 | 29 | class EmbeddingsResponse(Record): 30 | error = Error() 31 | vectors = Array(Array(Double())) 32 | 33 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/schema/object.py: -------------------------------------------------------------------------------- 1 | 2 | from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array 3 | from pulsar.schema import Double, Map 4 | 5 | from . metadata import Metadata 6 | from . types import Value, RowSchema 7 | from . topic import topic 8 | 9 | ############################################################################ 10 | 11 | # Object embeddings are embeddings associated with the primary key of an 12 | # object 13 | 14 | class ObjectEmbeddings(Record): 15 | metadata = Metadata() 16 | vectors = Array(Array(Double())) 17 | name = String() 18 | key_name = String() 19 | id = String() 20 | 21 | ############################################################################ 22 | 23 | # Stores rows of information 24 | 25 | class Rows(Record): 26 | metadata = Metadata() 27 | row_schema = RowSchema() 28 | rows = Array(Map(String())) 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/schema/prompt.py: -------------------------------------------------------------------------------- 1 | 2 | from pulsar.schema import Record, Bytes, String, Boolean, Array, Map, Integer 3 | 4 | from . topic import topic 5 | from . types import Error, RowSchema 6 | 7 | ############################################################################ 8 | 9 | # Prompt services, abstract the prompt generation 10 | 11 | class Definition(Record): 12 | name = String() 13 | definition = String() 14 | 15 | class Topic(Record): 16 | name = String() 17 | definition = String() 18 | 19 | class Relationship(Record): 20 | s = String() 21 | p = String() 22 | o = String() 23 | o_entity = Boolean() 24 | 25 | class Fact(Record): 26 | s = String() 27 | p = String() 28 | o = String() 29 | 30 | # extract-definitions: 31 | # chunk -> definitions 32 | # extract-relationships: 33 | # chunk -> relationships 34 | # kg-prompt: 35 | # query, triples -> answer 36 | # document-prompt: 37 | # query, documents -> answer 38 | # extract-rows 39 | # schema, chunk -> rows 40 | 41 | class PromptRequest(Record): 42 | id = String() 43 | 44 | # JSON encoded values 45 | terms = Map(String()) 46 | 47 | class PromptResponse(Record): 48 | 49 | # Error case 50 | error = Error() 51 | 52 | # Just plain text 53 | text = String() 54 | 55 | # JSON encoded 56 | object = String() 57 | 58 | ############################################################################ 59 | 60 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/schema/retrieval.py: -------------------------------------------------------------------------------- 1 | 2 | from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array, Double 3 | from . topic import topic 4 | from . types import Error, Value 5 | 6 | ############################################################################ 7 | 8 | # Graph RAG text retrieval 9 | 10 | class GraphRagQuery(Record): 11 | query = String() 12 | user = String() 13 | collection = String() 14 | entity_limit = Integer() 15 | triple_limit = Integer() 16 | max_subgraph_size = Integer() 17 | max_path_length = Integer() 18 | 19 | class GraphRagResponse(Record): 20 | error = Error() 21 | response = String() 22 | 23 | ############################################################################ 24 | 25 | # Document RAG text retrieval 26 | 27 | class DocumentRagQuery(Record): 28 | query = String() 29 | user = String() 30 | collection = String() 31 | doc_limit = Integer() 32 | 33 | class DocumentRagResponse(Record): 34 | error = Error() 35 | response = String() 36 | 37 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/schema/topic.py: -------------------------------------------------------------------------------- 1 | 2 | def topic(topic, kind='persistent', tenant='tg', namespace='flow'): 3 | return f"{kind}://{tenant}/{namespace}/{topic}" 4 | 5 | -------------------------------------------------------------------------------- /trustgraph-base/trustgraph/schema/types.py: -------------------------------------------------------------------------------- 1 | 2 | from pulsar.schema import Record, String, Boolean, Array, Integer 3 | 4 | class Error(Record): 5 | type = String() 6 | message = String() 7 | 8 | class Value(Record): 9 | value = String() 10 | is_uri = Boolean() 11 | type = String() 12 | 13 | class Triple(Record): 14 | s = Value() 15 | p = Value() 16 | o = Value() 17 | 18 | class Field(Record): 19 | name = String() 20 | # int, string, long, bool, float, double 21 | type = String() 22 | size = Integer() 23 | primary = Boolean() 24 | description = String() 25 | 26 | class RowSchema(Record): 27 | name = String() 28 | description = String() 29 | fields = Array(Field()) 30 | 31 | -------------------------------------------------------------------------------- /trustgraph-bedrock/README.md: -------------------------------------------------------------------------------- 1 | See https://trustgraph.ai/ 2 | -------------------------------------------------------------------------------- /trustgraph-bedrock/scripts/text-completion-bedrock: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.model.text_completion.bedrock import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-bedrock/setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | import os 3 | import importlib 4 | 5 | with open("README.md", "r") as fh: 6 | long_description = fh.read() 7 | 8 | # Load a version number module 9 | spec = importlib.util.spec_from_file_location( 10 | 'version', 'trustgraph/bedrock_version.py' 11 | ) 12 | version_module = importlib.util.module_from_spec(spec) 13 | spec.loader.exec_module(version_module) 14 | 15 | version = version_module.__version__ 16 | 17 | setuptools.setup( 18 | name="trustgraph-bedrock", 19 | version=version, 20 | author="trustgraph.ai", 21 | author_email="security@trustgraph.ai", 22 | description="TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.", 23 | long_description=long_description, 24 | long_description_content_type="text/markdown", 25 | url="https://github.com/trustgraph-ai/trustgraph", 26 | packages=setuptools.find_namespace_packages( 27 | where='./', 28 | ), 29 | classifiers=[ 30 | "Programming Language :: Python :: 3", 31 | "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", 32 | "Operating System :: OS Independent", 33 | ], 34 | python_requires='>=3.8', 35 | download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz", 36 | install_requires=[ 37 | "trustgraph-base>=1.0,<1.1", 38 | "pulsar-client", 39 | "prometheus-client", 40 | "boto3", 41 | ], 42 | scripts=[ 43 | "scripts/text-completion-bedrock", 44 | ] 45 | ) 46 | -------------------------------------------------------------------------------- /trustgraph-bedrock/trustgraph/model/text_completion/bedrock/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . llm import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-bedrock/trustgraph/model/text_completion/bedrock/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . llm import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-cli/README.md: -------------------------------------------------------------------------------- 1 | See https://trustgraph.ai/ 2 | -------------------------------------------------------------------------------- /trustgraph-cli/scripts/tg-delete-flow-class: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Deletes a flow class 5 | """ 6 | 7 | import argparse 8 | import os 9 | import tabulate 10 | from trustgraph.api import Api 11 | import json 12 | 13 | default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') 14 | 15 | def delete_flow_class(url, class_name): 16 | 17 | api = Api(url).flow() 18 | 19 | class_names = api.delete_class(class_name) 20 | 21 | def main(): 22 | 23 | parser = argparse.ArgumentParser( 24 | prog='tg-delete-flow-class', 25 | description=__doc__, 26 | ) 27 | 28 | parser.add_argument( 29 | '-u', '--api-url', 30 | default=default_url, 31 | help=f'API URL (default: {default_url})', 32 | ) 33 | 34 | parser.add_argument( 35 | '-n', '--class-name', 36 | help=f'Flow class name', 37 | ) 38 | 39 | args = parser.parse_args() 40 | 41 | try: 42 | 43 | delete_flow_class( 44 | url=args.api_url, 45 | class_name=args.class_name, 46 | ) 47 | 48 | except Exception as e: 49 | 50 | print("Exception:", e, flush=True) 51 | 52 | main() 53 | 54 | -------------------------------------------------------------------------------- /trustgraph-cli/scripts/tg-delete-kg-core: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Deletes a flow class 5 | """ 6 | 7 | import argparse 8 | import os 9 | import tabulate 10 | from trustgraph.api import Api 11 | import json 12 | 13 | default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') 14 | 15 | def delete_kg_core(url, user, id): 16 | 17 | api = Api(url).knowledge() 18 | 19 | class_names = api.delete_kg_core(user = user, id = id) 20 | 21 | def main(): 22 | 23 | parser = argparse.ArgumentParser( 24 | prog='tg-delete-flow-class', 25 | description=__doc__, 26 | ) 27 | 28 | parser.add_argument( 29 | '-u', '--api-url', 30 | default=default_url, 31 | help=f'API URL (default: {default_url})', 32 | ) 33 | 34 | parser.add_argument( 35 | '-U', '--user', 36 | default="trustgraph", 37 | help='API URL (default: trustgraph)', 38 | ) 39 | 40 | parser.add_argument( 41 | '--id', '--identifier', 42 | required=True, 43 | help=f'Knowledge core ID', 44 | ) 45 | 46 | args = parser.parse_args() 47 | 48 | try: 49 | 50 | delete_kg_core( 51 | url=args.api_url, 52 | user=args.user, 53 | id=args.id, 54 | ) 55 | 56 | except Exception as e: 57 | 58 | print("Exception:", e, flush=True) 59 | 60 | main() 61 | 62 | -------------------------------------------------------------------------------- /trustgraph-cli/scripts/tg-get-flow-class: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Outputs a flow class definition in JSON format. 5 | """ 6 | 7 | import argparse 8 | import os 9 | import tabulate 10 | from trustgraph.api import Api 11 | import json 12 | 13 | default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') 14 | 15 | def get_flow_class(url, class_name): 16 | 17 | api = Api(url).flow() 18 | 19 | cls = api.get_class(class_name) 20 | 21 | print(json.dumps(cls, indent=4)) 22 | 23 | def main(): 24 | 25 | parser = argparse.ArgumentParser( 26 | prog='tg-get-flow-class', 27 | description=__doc__, 28 | ) 29 | 30 | parser.add_argument( 31 | '-u', '--api-url', 32 | default=default_url, 33 | help=f'API URL (default: {default_url})', 34 | ) 35 | 36 | parser.add_argument( 37 | '-n', '--class-name', 38 | required=True, 39 | help=f'Flow class name', 40 | ) 41 | 42 | args = parser.parse_args() 43 | 44 | try: 45 | 46 | get_flow_class( 47 | url=args.api_url, 48 | class_name=args.class_name, 49 | ) 50 | 51 | except Exception as e: 52 | 53 | print("Exception:", e, flush=True) 54 | 55 | main() 56 | 57 | -------------------------------------------------------------------------------- /trustgraph-cli/scripts/tg-init-pulsar-manager: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | CSRF_TOKEN=$(curl http://localhost:7750/pulsar-manager/csrf-token) 4 | 5 | curl \ 6 | -H "X-XSRF-TOKEN: $CSRF_TOKEN" \ 7 | -H "Cookie: XSRF-TOKEN=$CSRF_TOKEN;" \ 8 | -H 'Content-Type: application/json' \ 9 | -X PUT \ 10 | http://localhost:7750/pulsar-manager/users/superuser \ 11 | -d '{"name": "admin", "password": "apachepulsar", "description": "test", "email": "username@test.org"}' 12 | -------------------------------------------------------------------------------- /trustgraph-cli/scripts/tg-invoke-llm: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Invokes the text completion service by specifying an LLM system prompt 5 | and user prompt. Both arguments are required. 6 | """ 7 | 8 | import argparse 9 | import os 10 | import json 11 | from trustgraph.api import Api 12 | 13 | default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') 14 | 15 | def query(url, flow_id, system, prompt): 16 | 17 | api = Api(url).flow().id(flow_id) 18 | 19 | resp = api.text_completion(system=system, prompt=prompt) 20 | 21 | print(resp) 22 | 23 | def main(): 24 | 25 | parser = argparse.ArgumentParser( 26 | prog='tg-invoke-llm', 27 | description=__doc__, 28 | ) 29 | 30 | parser.add_argument( 31 | '-u', '--url', 32 | default=default_url, 33 | help=f'API URL (default: {default_url})', 34 | ) 35 | 36 | parser.add_argument( 37 | 'system', 38 | nargs=1, 39 | help='LLM system prompt e.g. You are a helpful assistant', 40 | ) 41 | 42 | parser.add_argument( 43 | 'prompt', 44 | nargs=1, 45 | help='LLM prompt e.g. What is 2 + 2?', 46 | ) 47 | 48 | parser.add_argument( 49 | '-f', '--flow-id', 50 | default="default", 51 | help=f'Flow ID (default: default)' 52 | ) 53 | 54 | args = parser.parse_args() 55 | 56 | try: 57 | 58 | query( 59 | url=args.url, 60 | flow_id = args.flow_id, 61 | system=args.system[0], 62 | prompt=args.prompt[0], 63 | ) 64 | 65 | except Exception as e: 66 | 67 | print("Exception:", e, flush=True) 68 | 69 | main() 70 | 71 | -------------------------------------------------------------------------------- /trustgraph-cli/scripts/tg-put-flow-class: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Uploads a flow class definition. You can take the output of 5 | tg-get-flow-class and load it back in using this utility. 6 | """ 7 | 8 | import argparse 9 | import os 10 | from trustgraph.api import Api 11 | import json 12 | 13 | default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') 14 | 15 | def put_flow_class(url, class_name, config): 16 | 17 | api = Api(url) 18 | 19 | class_names = api.flow().put_class(class_name, config) 20 | 21 | def main(): 22 | 23 | parser = argparse.ArgumentParser( 24 | prog='tg-put-flow-class', 25 | description=__doc__, 26 | ) 27 | 28 | parser.add_argument( 29 | '-u', '--api-url', 30 | default=default_url, 31 | help=f'API URL (default: {default_url})', 32 | ) 33 | 34 | parser.add_argument( 35 | '-n', '--class-name', 36 | help=f'Flow class name', 37 | ) 38 | 39 | parser.add_argument( 40 | '-c', '--config', 41 | help=f'Initial configuration to load, should be raw JSON', 42 | ) 43 | 44 | args = parser.parse_args() 45 | 46 | try: 47 | 48 | put_flow_class( 49 | url=args.api_url, 50 | class_name=args.class_name, 51 | config=json.loads(args.config), 52 | ) 53 | 54 | except Exception as e: 55 | 56 | print("Exception:", e, flush=True) 57 | 58 | main() 59 | 60 | -------------------------------------------------------------------------------- /trustgraph-cli/scripts/tg-remove-library-document: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Remove a document from the library 5 | """ 6 | 7 | import argparse 8 | import os 9 | import uuid 10 | 11 | from trustgraph.api import Api 12 | 13 | default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') 14 | default_user = 'trustgraph' 15 | 16 | 17 | def remove_doc(url, user, id): 18 | 19 | api = Api(url).library() 20 | 21 | api.remove_document(user=user, id=id) 22 | 23 | def main(): 24 | 25 | parser = argparse.ArgumentParser( 26 | prog='tg-remove-library-document', 27 | description=__doc__, 28 | ) 29 | 30 | parser.add_argument( 31 | '-u', '--url', 32 | default=default_url, 33 | help=f'API URL (default: {default_url})', 34 | ) 35 | 36 | parser.add_argument( 37 | '-U', '--user', 38 | default=default_user, 39 | help=f'User ID (default: {default_user})' 40 | ) 41 | 42 | parser.add_argument( 43 | '--identifier', '--id', 44 | required=True, 45 | help=f'Document ID' 46 | ) 47 | 48 | args = parser.parse_args() 49 | 50 | try: 51 | 52 | remove_doc(args.url, args.user, args.identifier) 53 | 54 | except Exception as e: 55 | 56 | print("Exception:", e, flush=True) 57 | 58 | main() 59 | 60 | -------------------------------------------------------------------------------- /trustgraph-cli/scripts/tg-show-config: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Dumps out the current configuration 5 | """ 6 | 7 | import argparse 8 | import os 9 | from trustgraph.api import Api 10 | import json 11 | 12 | default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') 13 | 14 | def show_config(url): 15 | 16 | api = Api(url).config() 17 | 18 | config, version = api.all() 19 | 20 | print("Version:", version) 21 | print(json.dumps(config, indent=4)) 22 | 23 | def main(): 24 | 25 | parser = argparse.ArgumentParser( 26 | prog='tg-show-config', 27 | description=__doc__, 28 | ) 29 | 30 | parser.add_argument( 31 | '-u', '--api-url', 32 | default=default_url, 33 | help=f'API URL (default: {default_url})', 34 | ) 35 | 36 | args = parser.parse_args() 37 | 38 | try: 39 | 40 | show_config( 41 | url=args.api_url, 42 | ) 43 | 44 | except Exception as e: 45 | 46 | print("Exception:", e, flush=True) 47 | 48 | main() 49 | 50 | -------------------------------------------------------------------------------- /trustgraph-cli/scripts/tg-show-flow-classes: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Shows all defined flow classes. 5 | """ 6 | 7 | import argparse 8 | import os 9 | import tabulate 10 | from trustgraph.api import Api 11 | import json 12 | 13 | default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') 14 | 15 | def show_flow_classes(url): 16 | 17 | api = Api(url).flow() 18 | 19 | class_names = api.list_classes() 20 | 21 | if len(class_names) == 0: 22 | print("No flows.") 23 | return 24 | 25 | classes = [] 26 | 27 | for class_name in class_names: 28 | cls = api.get_class(class_name) 29 | classes.append(( 30 | class_name, 31 | cls.get("description", ""), 32 | ", ".join(cls.get("tags", [])), 33 | )) 34 | 35 | print(tabulate.tabulate( 36 | classes, 37 | tablefmt="pretty", 38 | maxcolwidths=[None, 40, 20], 39 | stralign="left", 40 | headers = ["flow class", "description", "tags"], 41 | )) 42 | 43 | def main(): 44 | 45 | parser = argparse.ArgumentParser( 46 | prog='tg-show-flow-classes', 47 | description=__doc__, 48 | ) 49 | 50 | parser.add_argument( 51 | '-u', '--api-url', 52 | default=default_url, 53 | help=f'API URL (default: {default_url})', 54 | ) 55 | 56 | args = parser.parse_args() 57 | 58 | try: 59 | 60 | show_flow_classes( 61 | url=args.api_url, 62 | ) 63 | 64 | except Exception as e: 65 | 66 | print("Exception:", e, flush=True) 67 | 68 | main() 69 | 70 | -------------------------------------------------------------------------------- /trustgraph-cli/scripts/tg-show-kg-cores: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Shows knowledge cores 5 | """ 6 | 7 | import argparse 8 | import os 9 | import tabulate 10 | from trustgraph.api import Api, ConfigKey 11 | import json 12 | 13 | default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') 14 | 15 | def show_cores(url, user): 16 | 17 | api = Api(url).knowledge() 18 | 19 | ids = api.list_kg_cores() 20 | 21 | if len(ids) == 0: 22 | print("No knowledge cores.") 23 | 24 | for id in ids: 25 | print(id) 26 | 27 | def main(): 28 | 29 | parser = argparse.ArgumentParser( 30 | prog='tg-show-flows', 31 | description=__doc__, 32 | ) 33 | 34 | parser.add_argument( 35 | '-u', '--api-url', 36 | default=default_url, 37 | help=f'API URL (default: {default_url})', 38 | ) 39 | 40 | parser.add_argument( 41 | '-U', '--user', 42 | default="trustgraph", 43 | help='API URL (default: trustgraph)', 44 | ) 45 | 46 | args = parser.parse_args() 47 | 48 | try: 49 | 50 | show_cores( 51 | url=args.api_url, user=args.user 52 | ) 53 | 54 | except Exception as e: 55 | 56 | print("Exception:", e, flush=True) 57 | 58 | main() 59 | 60 | -------------------------------------------------------------------------------- /trustgraph-cli/scripts/tg-show-processor-state: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Dump out TrustGraph processor states. 5 | """ 6 | 7 | import requests 8 | import argparse 9 | 10 | default_metrics_url = "http://localhost:8088/api/metrics" 11 | 12 | def dump_status(url): 13 | 14 | url = f"{url}/query?query=processor_info" 15 | 16 | resp = requests.get(url) 17 | 18 | obj = resp.json() 19 | 20 | tbl = [ 21 | [ 22 | m["metric"]["job"], 23 | "\U0001f49a" 24 | ] 25 | for m in obj["data"]["result"] 26 | ] 27 | 28 | for row in tbl: 29 | print(f" {row[0]:30} {row[1]}") 30 | 31 | def main(): 32 | 33 | parser = argparse.ArgumentParser( 34 | prog='tg-show-processor-state', 35 | description=__doc__, 36 | ) 37 | 38 | parser.add_argument( 39 | '-m', '--metrics-url', 40 | default=default_metrics_url, 41 | help=f'Metrics URL (default: {default_metrics_url})', 42 | ) 43 | 44 | args = parser.parse_args() 45 | 46 | try: 47 | 48 | dump_status(args.metrics_url) 49 | 50 | except Exception as e: 51 | 52 | print("Exception:", e, flush=True) 53 | 54 | main() 55 | 56 | -------------------------------------------------------------------------------- /trustgraph-cli/scripts/tg-start-flow: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Starts a processing flow using a defined flow class 5 | """ 6 | 7 | import argparse 8 | import os 9 | import tabulate 10 | from trustgraph.api import Api 11 | import json 12 | 13 | default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') 14 | 15 | def start_flow(url, class_name, flow_id, description): 16 | 17 | api = Api(url).flow() 18 | 19 | api.start( 20 | class_name = class_name, 21 | id = flow_id, 22 | description = description, 23 | ) 24 | 25 | def main(): 26 | 27 | parser = argparse.ArgumentParser( 28 | prog='tg-start-flow', 29 | description=__doc__, 30 | ) 31 | 32 | parser.add_argument( 33 | '-u', '--api-url', 34 | default=default_url, 35 | help=f'API URL (default: {default_url})', 36 | ) 37 | 38 | parser.add_argument( 39 | '-n', '--class-name', 40 | required=True, 41 | help=f'Flow class name', 42 | ) 43 | 44 | parser.add_argument( 45 | '-i', '--flow-id', 46 | required=True, 47 | help=f'Flow ID', 48 | ) 49 | 50 | parser.add_argument( 51 | '-d', '--description', 52 | required=True, 53 | help=f'Flow description', 54 | ) 55 | 56 | args = parser.parse_args() 57 | 58 | try: 59 | 60 | start_flow( 61 | url = args.api_url, 62 | class_name = args.class_name, 63 | flow_id = args.flow_id, 64 | description = args.description, 65 | ) 66 | 67 | except Exception as e: 68 | 69 | print("Exception:", e, flush=True) 70 | 71 | main() 72 | 73 | -------------------------------------------------------------------------------- /trustgraph-cli/scripts/tg-stop-flow: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Stops a processing flow. 5 | """ 6 | 7 | import argparse 8 | import os 9 | import tabulate 10 | from trustgraph.api import Api 11 | import json 12 | 13 | default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') 14 | 15 | def stop_flow(url, flow_id): 16 | 17 | api = Api(url).flow() 18 | 19 | api.stop(id = flow_id) 20 | 21 | def main(): 22 | 23 | parser = argparse.ArgumentParser( 24 | prog='tg-stop-flow', 25 | description=__doc__, 26 | ) 27 | 28 | parser.add_argument( 29 | '-u', '--api-url', 30 | default=default_url, 31 | help=f'API URL (default: {default_url})', 32 | ) 33 | 34 | parser.add_argument( 35 | '-i', '--flow-id', 36 | required=True, 37 | help=f'Flow ID', 38 | ) 39 | 40 | args = parser.parse_args() 41 | 42 | try: 43 | 44 | stop_flow( 45 | url=args.api_url, 46 | flow_id=args.flow_id, 47 | ) 48 | 49 | except Exception as e: 50 | 51 | print("Exception:", e, flush=True) 52 | 53 | main() 54 | 55 | -------------------------------------------------------------------------------- /trustgraph-cli/scripts/tg-stop-library-processing: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | Removes a library document processing record. This is just a record of 5 | procesing, it doesn't stop in-flight processing at the moment. 6 | """ 7 | 8 | import argparse 9 | import os 10 | import tabulate 11 | from trustgraph.api import Api, ConfigKey 12 | import json 13 | 14 | default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') 15 | default_user = "trustgraph" 16 | 17 | def stop_processing( 18 | url, user, id 19 | ): 20 | 21 | api = Api(url).library() 22 | 23 | api.stop_processing(user = user, id = id) 24 | 25 | def main(): 26 | 27 | parser = argparse.ArgumentParser( 28 | prog='tg-stop-library-processing', 29 | description=__doc__, 30 | ) 31 | 32 | parser.add_argument( 33 | '-u', '--api-url', 34 | default=default_url, 35 | help=f'API URL (default: {default_url})', 36 | ) 37 | 38 | parser.add_argument( 39 | '-U', '--user', 40 | default=default_user, 41 | help=f'User ID (default: {default_user})' 42 | ) 43 | 44 | parser.add_argument( 45 | '--id', '--processing-id', 46 | required=True, 47 | help=f'Processing ID', 48 | ) 49 | 50 | args = parser.parse_args() 51 | 52 | try: 53 | 54 | stop_processing( 55 | url = args.api_url, 56 | user = args.user, 57 | id = args.id, 58 | ) 59 | 60 | except Exception as e: 61 | 62 | print("Exception:", e, flush=True) 63 | 64 | main() 65 | 66 | -------------------------------------------------------------------------------- /trustgraph-embeddings-hf/README.md: -------------------------------------------------------------------------------- 1 | See https://trustgraph.ai/ 2 | -------------------------------------------------------------------------------- /trustgraph-embeddings-hf/scripts/embeddings-hf: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.embeddings.hf import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-embeddings-hf/trustgraph/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-embeddings-hf/trustgraph/__init__.py -------------------------------------------------------------------------------- /trustgraph-embeddings-hf/trustgraph/embeddings/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-embeddings-hf/trustgraph/embeddings/__init__.py -------------------------------------------------------------------------------- /trustgraph-embeddings-hf/trustgraph/embeddings/hf/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . hf import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-embeddings-hf/trustgraph/embeddings/hf/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . hf import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-embeddings-hf/trustgraph/embeddings/hf/hf.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | Embeddings service, applies an embeddings model selected from HuggingFace. 4 | Input is text, output is embeddings vector. 5 | """ 6 | 7 | from ... base import EmbeddingsService 8 | 9 | from langchain_huggingface import HuggingFaceEmbeddings 10 | 11 | default_ident = "embeddings" 12 | 13 | default_model="all-MiniLM-L6-v2" 14 | 15 | class Processor(EmbeddingsService): 16 | 17 | def __init__(self, **params): 18 | 19 | model = params.get("model", default_model) 20 | 21 | super(Processor, self).__init__( 22 | **params | { "model": model } 23 | ) 24 | 25 | print("Get model...", flush=True) 26 | self.embeddings = HuggingFaceEmbeddings(model_name=model) 27 | 28 | async def on_embeddings(self, text): 29 | 30 | embeds = self.embeddings.embed_documents([text]) 31 | print("Done.", flush=True) 32 | return embeds 33 | 34 | @staticmethod 35 | def add_args(parser): 36 | 37 | EmbeddingsService.add_args(parser) 38 | 39 | parser.add_argument( 40 | '-m', '--model', 41 | default="all-MiniLM-L6-v2", 42 | help=f'LLM model (default: all-MiniLM-L6-v2)' 43 | ) 44 | 45 | def run(): 46 | 47 | Processor.launch(default_ident, __doc__) 48 | 49 | -------------------------------------------------------------------------------- /trustgraph-flow/README.md: -------------------------------------------------------------------------------- 1 | See https://trustgraph.ai/ 2 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/agent-manager-react: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.agent.react import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/api-gateway: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.gateway import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/chunker-recursive: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.chunking.recursive import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/chunker-token: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.chunking.token import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/config-svc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.config.service import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/de-query-milvus: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.query.doc_embeddings.milvus import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/de-query-pinecone: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.query.doc_embeddings.pinecone import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/de-query-qdrant: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.query.doc_embeddings.qdrant import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/de-write-milvus: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.storage.doc_embeddings.milvus import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/de-write-pinecone: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.storage.doc_embeddings.pinecone import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/de-write-qdrant: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.storage.doc_embeddings.qdrant import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/document-embeddings: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.embeddings.document_embeddings import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/document-rag: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.retrieval.document_rag import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/embeddings-fastembed: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.embeddings.fastembed import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/embeddings-ollama: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.embeddings.ollama import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/ge-query-milvus: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.query.graph_embeddings.milvus import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/ge-query-pinecone: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.query.graph_embeddings.pinecone import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/ge-query-qdrant: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.query.graph_embeddings.qdrant import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/ge-write-milvus: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.storage.graph_embeddings.milvus import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/ge-write-pinecone: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.storage.graph_embeddings.pinecone import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/ge-write-qdrant: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.storage.graph_embeddings.qdrant import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/graph-embeddings: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.embeddings.graph_embeddings import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/graph-rag: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.retrieval.graph_rag import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/kg-extract-definitions: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.extract.kg.definitions import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/kg-extract-relationships: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.extract.kg.relationships import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/kg-extract-topics: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.extract.kg.topics import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/kg-manager: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.cores import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/kg-store: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.storage.knowledge import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/librarian: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.librarian import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/metering: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.metering import run 4 | 5 | run() -------------------------------------------------------------------------------- /trustgraph-flow/scripts/object-extract-row: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.extract.object.row import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/oe-write-milvus: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.storage.object_embeddings.milvus import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/pdf-decoder: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.decoding.pdf import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/pdf-ocr-mistral: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.decoding.mistral_ocr import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/prompt-generic: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.model.prompt.generic import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/prompt-template: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.model.prompt.template import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/rows-write-cassandra: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.storage.rows.cassandra import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/run-processing: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.processing import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/text-completion-azure: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.model.text_completion.azure import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/text-completion-azure-openai: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.model.text_completion.azure_openai import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/text-completion-claude: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.model.text_completion.claude import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/text-completion-cohere: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.model.text_completion.cohere import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/text-completion-googleaistudio: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.model.text_completion.googleaistudio import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/text-completion-llamafile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.model.text_completion.llamafile import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/text-completion-lmstudio: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.model.text_completion.lmstudio import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/text-completion-mistral: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.model.text_completion.mistral import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/text-completion-ollama: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.model.text_completion.ollama import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/text-completion-openai: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.model.text_completion.openai import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/text-completion-tgi: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.model.text_completion.tgi import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/triples-query-cassandra: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.query.triples.cassandra import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/triples-query-falkordb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.query.triples.falkordb import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/triples-query-memgraph: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.query.triples.memgraph import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/triples-query-neo4j: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.query.triples.neo4j import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/triples-write-cassandra: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.storage.triples.cassandra import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/triples-write-falkordb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.storage.triples.falkordb import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/triples-write-memgraph: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.storage.triples.memgraph import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/triples-write-neo4j: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.storage.triples.neo4j import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/scripts/wikipedia-lookup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.external.wikipedia import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/agent/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/agent/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/agent/react/README.md: -------------------------------------------------------------------------------- 1 | 2 | agent-manager-react \ 3 | -p pulsar://localhost:6650 \ 4 | --tool-type \ 5 | shuttle=knowledge-query:query \ 6 | cats=knowledge-query:query \ 7 | compute=text-completion:computation \ 8 | --tool-description \ 9 | shuttle="Query a knowledge base with information about the space shuttle. The query should be a simple natural language question" \ 10 | cats="Query a knowledge base with information about Mark's cats. The query should be a simple natural language question" \ 11 | compute="A computation engine which can answer questions about maths and computation" \ 12 | --tool-argument \ 13 | cats="query:string:The search query string" \ 14 | shuttle="query:string:The search query string" \ 15 | compute="computation:string:The computation to solve" 16 | 17 | 18 | --context 'The space shuttle challenger final mission was 58-L' 19 | 20 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/agent/react/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . service import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/agent/react/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . service import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/agent/react/tools.py: -------------------------------------------------------------------------------- 1 | 2 | # This tool implementation knows how to put a question to the graph RAG 3 | # service 4 | class KnowledgeQueryImpl: 5 | def __init__(self, context): 6 | self.context = context 7 | async def invoke(self, **arguments): 8 | client = self.context("graph-rag-request") 9 | print("Graph RAG question...", flush=True) 10 | return await client.rag( 11 | arguments.get("question") 12 | ) 13 | 14 | # This tool implementation knows how to do text completion. This uses 15 | # the prompt service, rather than talking to TextCompletion directly. 16 | class TextCompletionImpl: 17 | def __init__(self, context): 18 | self.context = context 19 | async def invoke(self, **arguments): 20 | client = self.context("prompt-request") 21 | print("Prompt question...", flush=True) 22 | return await client.question( 23 | arguments.get("question") 24 | ) 25 | 26 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/agent/react/types.py: -------------------------------------------------------------------------------- 1 | 2 | import dataclasses 3 | from typing import Any, Dict 4 | 5 | @dataclasses.dataclass 6 | class Argument: 7 | name : str 8 | type : str 9 | description : str 10 | 11 | @dataclasses.dataclass 12 | class Tool: 13 | name : str 14 | description : str 15 | arguments : list[Argument] 16 | implementation : Any 17 | config : Dict[str, str] 18 | 19 | @dataclasses.dataclass 20 | class Action: 21 | thought : str 22 | name : str 23 | arguments : dict 24 | observation : str 25 | 26 | @dataclasses.dataclass 27 | class Final: 28 | thought : str 29 | final : str 30 | 31 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/chunking/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/chunking/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/chunking/recursive/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . chunker import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/chunking/recursive/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . chunker import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/chunking/token/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . chunker import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/chunking/token/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . chunker import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/config/service/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . service import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/config/service/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . service import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/cores/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . service import run 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/cores/__main__.py: -------------------------------------------------------------------------------- 1 | 2 | from . service import run 3 | 4 | if __name__ == '__main__': 5 | run() 6 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/decoding/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/decoding/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/decoding/mistral_ocr/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . processor import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/decoding/mistral_ocr/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . processor import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/decoding/pdf/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . pdf_decoder import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/decoding/pdf/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . pdf_decoder import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/direct/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/direct/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/embeddings/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/embeddings/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/embeddings/document_embeddings/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . embeddings import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/embeddings/document_embeddings/__main__.py: -------------------------------------------------------------------------------- 1 | 2 | from . embeddings import run 3 | 4 | if __name__ == '__main__': 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/embeddings/fastembed/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . processor import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/embeddings/fastembed/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . processor import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/embeddings/fastembed/processor.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | Embeddings service, applies an embeddings model using fastembed 4 | Input is text, output is embeddings vector. 5 | """ 6 | 7 | from ... base import EmbeddingsService 8 | 9 | from fastembed import TextEmbedding 10 | 11 | default_ident = "embeddings" 12 | 13 | default_model="sentence-transformers/all-MiniLM-L6-v2" 14 | 15 | class Processor(EmbeddingsService): 16 | 17 | def __init__(self, **params): 18 | 19 | model = params.get("model", default_model) 20 | 21 | super(Processor, self).__init__( 22 | **params | { "model": model } 23 | ) 24 | 25 | print("Get model...", flush=True) 26 | self.embeddings = TextEmbedding(model_name = model) 27 | 28 | async def on_embeddings(self, text): 29 | 30 | vecs = self.embeddings.embed([text]) 31 | 32 | return [ 33 | v.tolist() 34 | for v in vecs 35 | ] 36 | 37 | @staticmethod 38 | def add_args(parser): 39 | 40 | EmbeddingsService.add_args(parser) 41 | 42 | parser.add_argument( 43 | '-m', '--model', 44 | default=default_model, 45 | help=f'Embeddings model (default: {default_model})' 46 | ) 47 | 48 | def run(): 49 | 50 | Processor.launch(default_ident, __doc__) 51 | 52 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/embeddings/graph_embeddings/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . embeddings import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/embeddings/graph_embeddings/__main__.py: -------------------------------------------------------------------------------- 1 | 2 | from . embeddings import run 3 | 4 | if __name__ == '__main__': 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/embeddings/ollama/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . processor import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/embeddings/ollama/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . processor import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/external/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/external/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/external/wikipedia/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . service import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/external/wikipedia/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . service import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/extract/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/extract/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/extract/kg/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/extract/kg/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/extract/kg/definitions/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . extract import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/extract/kg/definitions/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . extract import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/extract/kg/relationships/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . extract import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/extract/kg/relationships/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . extract import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/extract/kg/topics/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . extract import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/extract/kg/topics/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . extract import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/extract/object/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/extract/object/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/extract/object/row/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . extract import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/extract/object/row/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . extract import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . service import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . service import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/auth.py: -------------------------------------------------------------------------------- 1 | 2 | class Authenticator: 3 | 4 | def __init__(self, token=None, allow_all=False): 5 | 6 | if not allow_all and token is None: 7 | raise RuntimeError("Need a token") 8 | 9 | if not allow_all and token == "": 10 | raise RuntimeError("Need a token") 11 | 12 | self.token = token 13 | self.allow_all = allow_all 14 | 15 | def permitted(self, token, roles): 16 | 17 | if self.allow_all: return True 18 | 19 | if self.token != token: return False 20 | 21 | return True 22 | 23 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/dispatch/agent.py: -------------------------------------------------------------------------------- 1 | 2 | from ... schema import AgentRequest, AgentResponse 3 | 4 | from . requestor import ServiceRequestor 5 | 6 | class AgentRequestor(ServiceRequestor): 7 | def __init__( 8 | self, pulsar_client, request_queue, response_queue, timeout, 9 | consumer, subscriber, 10 | ): 11 | 12 | super(AgentRequestor, self).__init__( 13 | pulsar_client=pulsar_client, 14 | request_queue=request_queue, 15 | response_queue=response_queue, 16 | request_schema=AgentRequest, 17 | response_schema=AgentResponse, 18 | subscription = subscriber, 19 | consumer_name = consumer, 20 | timeout=timeout, 21 | ) 22 | 23 | def to_request(self, body): 24 | return AgentRequest( 25 | question=body["question"] 26 | ) 27 | 28 | def from_response(self, message): 29 | resp = { 30 | } 31 | 32 | if message.answer: 33 | resp["answer"] = message.answer 34 | 35 | if message.thought: 36 | resp["thought"] = message.thought 37 | 38 | if message.observation: 39 | resp["observation"] = message.observation 40 | 41 | # The 2nd boolean expression indicates whether we're done responding 42 | return resp, (message.answer is not None) 43 | 44 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/dispatch/document_load.py: -------------------------------------------------------------------------------- 1 | 2 | import base64 3 | 4 | from ... schema import Document, Metadata 5 | 6 | from . sender import ServiceSender 7 | from . serialize import to_subgraph 8 | 9 | class DocumentLoad(ServiceSender): 10 | def __init__(self, pulsar_client, queue): 11 | 12 | super(DocumentLoad, self).__init__( 13 | pulsar_client = pulsar_client, 14 | queue = queue, 15 | schema = Document, 16 | ) 17 | 18 | def to_request(self, body): 19 | 20 | if "metadata" in body: 21 | metadata = to_subgraph(body["metadata"]) 22 | else: 23 | metadata = [] 24 | 25 | # Doing a base64 decoe/encode here to make sure the 26 | # content is valid base64 27 | doc = base64.b64decode(body["data"]) 28 | 29 | print("Document received") 30 | 31 | return Document( 32 | metadata=Metadata( 33 | id=body.get("id"), 34 | metadata=metadata, 35 | user=body.get("user", "trustgraph"), 36 | collection=body.get("collection", "default"), 37 | ), 38 | data=base64.b64encode(doc).decode("utf-8") 39 | ) 40 | 41 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/dispatch/document_rag.py: -------------------------------------------------------------------------------- 1 | 2 | from ... schema import DocumentRagQuery, DocumentRagResponse 3 | 4 | from . requestor import ServiceRequestor 5 | 6 | class DocumentRagRequestor(ServiceRequestor): 7 | def __init__( 8 | self, pulsar_client, request_queue, response_queue, timeout, 9 | consumer, subscriber, 10 | ): 11 | 12 | super(DocumentRagRequestor, self).__init__( 13 | pulsar_client=pulsar_client, 14 | request_queue=request_queue, 15 | response_queue=response_queue, 16 | request_schema=DocumentRagQuery, 17 | response_schema=DocumentRagResponse, 18 | subscription = subscriber, 19 | consumer_name = consumer, 20 | timeout=timeout, 21 | ) 22 | 23 | def to_request(self, body): 24 | return DocumentRagQuery( 25 | query=body["query"], 26 | user=body.get("user", "trustgraph"), 27 | collection=body.get("collection", "default"), 28 | doc_limit=int(body.get("doc-limit", 20)), 29 | ) 30 | 31 | def from_response(self, message): 32 | return { "response": message.response }, True 33 | 34 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/dispatch/embeddings.py: -------------------------------------------------------------------------------- 1 | 2 | from ... schema import EmbeddingsRequest, EmbeddingsResponse 3 | 4 | from . requestor import ServiceRequestor 5 | 6 | class EmbeddingsRequestor(ServiceRequestor): 7 | def __init__( 8 | self, pulsar_client, request_queue, response_queue, timeout, 9 | consumer, subscriber, 10 | ): 11 | 12 | super(EmbeddingsRequestor, self).__init__( 13 | pulsar_client=pulsar_client, 14 | request_queue=request_queue, 15 | response_queue=response_queue, 16 | request_schema=EmbeddingsRequest, 17 | response_schema=EmbeddingsResponse, 18 | subscription = subscriber, 19 | consumer_name = consumer, 20 | timeout=timeout, 21 | ) 22 | 23 | def to_request(self, body): 24 | return EmbeddingsRequest( 25 | text=body["text"] 26 | ) 27 | 28 | def from_response(self, message): 29 | return { "vectors": message.vectors }, True 30 | 31 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/dispatch/graph_embeddings_query.py: -------------------------------------------------------------------------------- 1 | 2 | from ... schema import GraphEmbeddingsRequest, GraphEmbeddingsResponse 3 | 4 | from . requestor import ServiceRequestor 5 | from . serialize import serialize_value 6 | 7 | class GraphEmbeddingsQueryRequestor(ServiceRequestor): 8 | def __init__( 9 | self, pulsar_client, request_queue, response_queue, timeout, 10 | consumer, subscriber, 11 | ): 12 | 13 | super(GraphEmbeddingsQueryRequestor, self).__init__( 14 | pulsar_client=pulsar_client, 15 | request_queue=request_queue, 16 | response_queue=response_queue, 17 | request_schema=GraphEmbeddingsRequest, 18 | response_schema=GraphEmbeddingsResponse, 19 | subscription = subscriber, 20 | consumer_name = consumer, 21 | timeout=timeout, 22 | ) 23 | 24 | def to_request(self, body): 25 | 26 | limit = int(body.get("limit", 20)) 27 | 28 | return GraphEmbeddingsRequest( 29 | vectors = body["vectors"], 30 | limit = limit, 31 | user = body.get("user", "trustgraph"), 32 | collection = body.get("collection", "default"), 33 | ) 34 | 35 | def from_response(self, message): 36 | 37 | return { 38 | "entities": [ 39 | serialize_value(ent) for ent in message.entities 40 | ] 41 | }, True 42 | 43 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/dispatch/graph_rag.py: -------------------------------------------------------------------------------- 1 | 2 | from ... schema import GraphRagQuery, GraphRagResponse 3 | 4 | from . requestor import ServiceRequestor 5 | 6 | class GraphRagRequestor(ServiceRequestor): 7 | def __init__( 8 | self, pulsar_client, request_queue, response_queue, timeout, 9 | consumer, subscriber, 10 | ): 11 | 12 | super(GraphRagRequestor, self).__init__( 13 | pulsar_client=pulsar_client, 14 | request_queue=request_queue, 15 | response_queue=response_queue, 16 | request_schema=GraphRagQuery, 17 | response_schema=GraphRagResponse, 18 | subscription = subscriber, 19 | consumer_name = consumer, 20 | timeout=timeout, 21 | ) 22 | 23 | def to_request(self, body): 24 | return GraphRagQuery( 25 | query=body["query"], 26 | user=body.get("user", "trustgraph"), 27 | collection=body.get("collection", "default"), 28 | entity_limit=int(body.get("entity-limit", 50)), 29 | triple_limit=int(body.get("triple-limit", 30)), 30 | max_subgraph_size=int(body.get("max-subgraph-size", 1000)), 31 | max_path_length=int(body.get("max-path-length", 2)), 32 | ) 33 | 34 | def from_response(self, message): 35 | return { "response": message.response }, True 36 | 37 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/dispatch/prompt.py: -------------------------------------------------------------------------------- 1 | 2 | import json 3 | 4 | from ... schema import PromptRequest, PromptResponse 5 | 6 | from . requestor import ServiceRequestor 7 | 8 | class PromptRequestor(ServiceRequestor): 9 | def __init__( 10 | self, pulsar_client, request_queue, response_queue, timeout, 11 | consumer, subscriber, 12 | ): 13 | 14 | super(PromptRequestor, self).__init__( 15 | pulsar_client=pulsar_client, 16 | request_queue=request_queue, 17 | response_queue=response_queue, 18 | request_schema=PromptRequest, 19 | response_schema=PromptResponse, 20 | subscription = subscriber, 21 | consumer_name = consumer, 22 | timeout=timeout, 23 | ) 24 | 25 | def to_request(self, body): 26 | return PromptRequest( 27 | id=body["id"], 28 | terms={ 29 | k: json.dumps(v) 30 | for k, v in body["variables"].items() 31 | } 32 | ) 33 | 34 | def from_response(self, message): 35 | if message.object: 36 | return { 37 | "object": message.object 38 | }, True 39 | else: 40 | return { 41 | "text": message.text 42 | }, True 43 | 44 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/dispatch/sender.py: -------------------------------------------------------------------------------- 1 | 2 | # Like ServiceRequestor, but just fire-and-forget instead of request/response 3 | 4 | import asyncio 5 | import uuid 6 | import logging 7 | 8 | from ... base import Publisher 9 | 10 | logger = logging.getLogger("sender") 11 | logger.setLevel(logging.INFO) 12 | 13 | class ServiceSender: 14 | 15 | def __init__( 16 | self, 17 | pulsar_client, 18 | queue, schema, 19 | ): 20 | 21 | self.pub = Publisher( 22 | pulsar_client, queue, 23 | schema=schema, 24 | ) 25 | 26 | async def start(self): 27 | await self.pub.start() 28 | 29 | async def stop(self): 30 | await self.pub.stop() 31 | 32 | def to_request(self, request): 33 | raise RuntimeError("Not defined") 34 | 35 | async def process(self, request, responder=None): 36 | 37 | try: 38 | 39 | await self.pub.send(None, self.to_request(request)) 40 | 41 | if responder: 42 | await responder({}, True) 43 | 44 | return {} 45 | 46 | except Exception as e: 47 | 48 | logging.error(f"Exception: {e}") 49 | 50 | err = { "error": str(e) } 51 | 52 | if responder: 53 | await responder(err, True) 54 | 55 | return err 56 | 57 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/dispatch/text_completion.py: -------------------------------------------------------------------------------- 1 | 2 | from ... schema import TextCompletionRequest, TextCompletionResponse 3 | 4 | from . requestor import ServiceRequestor 5 | 6 | class TextCompletionRequestor(ServiceRequestor): 7 | def __init__( 8 | self, pulsar_client, request_queue, response_queue, timeout, 9 | consumer, subscriber, 10 | ): 11 | 12 | super(TextCompletionRequestor, self).__init__( 13 | pulsar_client=pulsar_client, 14 | request_queue=request_queue, 15 | response_queue=response_queue, 16 | request_schema=TextCompletionRequest, 17 | response_schema=TextCompletionResponse, 18 | subscription = subscriber, 19 | consumer_name = consumer, 20 | timeout=timeout, 21 | ) 22 | 23 | def to_request(self, body): 24 | return TextCompletionRequest( 25 | system=body["system"], 26 | prompt=body["prompt"] 27 | ) 28 | 29 | def from_response(self, message): 30 | return { "response": message.response }, True 31 | 32 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/dispatch/text_load.py: -------------------------------------------------------------------------------- 1 | 2 | import base64 3 | 4 | from ... schema import TextDocument, Metadata 5 | 6 | from . sender import ServiceSender 7 | from . serialize import to_subgraph 8 | 9 | class TextLoad(ServiceSender): 10 | def __init__(self, pulsar_client, queue): 11 | 12 | super(TextLoad, self).__init__( 13 | pulsar_client = pulsar_client, 14 | queue = queue, 15 | schema = TextDocument, 16 | ) 17 | 18 | def to_request(self, body): 19 | 20 | if "metadata" in body: 21 | metadata = to_subgraph(body["metadata"]) 22 | else: 23 | metadata = [] 24 | 25 | if "charset" in body: 26 | charset = body["charset"] 27 | else: 28 | charset = "utf-8" 29 | 30 | # Text is base64 encoded 31 | text = base64.b64decode(body["text"]).decode(charset) 32 | 33 | print("Text document received") 34 | 35 | return TextDocument( 36 | metadata=Metadata( 37 | id=body.get("id"), 38 | metadata=metadata, 39 | user=body.get("user", "trustgraph"), 40 | collection=body.get("collection", "default"), 41 | ), 42 | text=text, 43 | ) 44 | 45 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/dispatch/triples_import.py: -------------------------------------------------------------------------------- 1 | 2 | import asyncio 3 | import uuid 4 | from aiohttp import WSMsgType 5 | 6 | from ... schema import Metadata 7 | from ... schema import Triples 8 | from ... base import Publisher 9 | 10 | from . serialize import to_subgraph 11 | 12 | class TriplesImport: 13 | 14 | def __init__( 15 | self, ws, running, pulsar_client, queue 16 | ): 17 | 18 | self.ws = ws 19 | self.running = running 20 | 21 | self.publisher = Publisher( 22 | pulsar_client, topic = queue, schema = Triples 23 | ) 24 | 25 | async def start(self): 26 | await self.publisher.start() 27 | 28 | async def destroy(self): 29 | self.running.stop() 30 | 31 | if self.ws: 32 | await self.ws.close() 33 | 34 | await self.publisher.stop() 35 | 36 | async def receive(self, msg): 37 | 38 | data = msg.json() 39 | 40 | elt = Triples( 41 | metadata=Metadata( 42 | id=data["metadata"]["id"], 43 | metadata=to_subgraph(data["metadata"]["metadata"]), 44 | user=data["metadata"]["user"], 45 | collection=data["metadata"]["collection"], 46 | ), 47 | triples=to_subgraph(data["triples"]), 48 | ) 49 | 50 | await self.publisher.send(None, elt) 51 | 52 | async def run(self): 53 | 54 | while self.running.get(): 55 | await asyncio.sleep(0.5) 56 | 57 | if self.ws: 58 | await self.ws.close() 59 | 60 | self.ws = None 61 | 62 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/dispatch/triples_query.py: -------------------------------------------------------------------------------- 1 | 2 | from ... schema import TriplesQueryRequest, TriplesQueryResponse, Triples 3 | 4 | from . requestor import ServiceRequestor 5 | from . serialize import to_value, serialize_subgraph 6 | 7 | class TriplesQueryRequestor(ServiceRequestor): 8 | def __init__( 9 | self, pulsar_client, request_queue, response_queue, timeout, 10 | consumer, subscriber, 11 | ): 12 | 13 | super(TriplesQueryRequestor, self).__init__( 14 | pulsar_client=pulsar_client, 15 | request_queue=request_queue, 16 | response_queue=response_queue, 17 | request_schema=TriplesQueryRequest, 18 | response_schema=TriplesQueryResponse, 19 | subscription = subscriber, 20 | consumer_name = consumer, 21 | timeout=timeout, 22 | ) 23 | 24 | def to_request(self, body): 25 | 26 | if "s" in body: 27 | s = to_value(body["s"]) 28 | else: 29 | s = None 30 | 31 | if "p" in body: 32 | p = to_value(body["p"]) 33 | else: 34 | p = None 35 | 36 | if "o" in body: 37 | o = to_value(body["o"]) 38 | else: 39 | o = None 40 | 41 | limit = int(body.get("limit", 10000)) 42 | 43 | return TriplesQueryRequest( 44 | s = s, p = p, o = o, 45 | limit = limit, 46 | user = body.get("user", "trustgraph"), 47 | collection = body.get("collection", "default"), 48 | ) 49 | 50 | def from_response(self, message): 51 | return { 52 | "response": serialize_subgraph(message.triples) 53 | }, True 54 | 55 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/endpoint/constant_endpoint.py: -------------------------------------------------------------------------------- 1 | 2 | import asyncio 3 | from aiohttp import web 4 | import uuid 5 | import logging 6 | 7 | logger = logging.getLogger("endpoint") 8 | logger.setLevel(logging.INFO) 9 | 10 | class ConstantEndpoint: 11 | 12 | def __init__(self, endpoint_path, auth, dispatcher): 13 | 14 | self.path = endpoint_path 15 | 16 | self.auth = auth 17 | self.operation = "service" 18 | 19 | self.dispatcher = dispatcher 20 | 21 | async def start(self): 22 | pass 23 | 24 | def add_routes(self, app): 25 | 26 | app.add_routes([ 27 | web.post(self.path, self.handle), 28 | ]) 29 | 30 | async def handle(self, request): 31 | 32 | print(request.path, "...") 33 | 34 | try: 35 | ht = request.headers["Authorization"] 36 | tokens = ht.split(" ", 2) 37 | if tokens[0] != "Bearer": 38 | return web.HTTPUnauthorized() 39 | token = tokens[1] 40 | except: 41 | token = "" 42 | 43 | if not self.auth.permitted(token, self.operation): 44 | return web.HTTPUnauthorized() 45 | 46 | try: 47 | 48 | data = await request.json() 49 | 50 | async def responder(x, fin): 51 | pass 52 | 53 | resp = await self.dispatcher.process(data, responder) 54 | 55 | return web.json_response(resp) 56 | 57 | except Exception as e: 58 | logging.error(f"Exception: {e}") 59 | 60 | return web.json_response( 61 | { "error": str(e) } 62 | ) 63 | 64 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/endpoint/variable_endpoint.py: -------------------------------------------------------------------------------- 1 | 2 | import asyncio 3 | from aiohttp import web 4 | import uuid 5 | import logging 6 | 7 | logger = logging.getLogger("endpoint") 8 | logger.setLevel(logging.INFO) 9 | 10 | class VariableEndpoint: 11 | 12 | def __init__(self, endpoint_path, auth, dispatcher): 13 | 14 | self.path = endpoint_path 15 | 16 | self.auth = auth 17 | self.operation = "service" 18 | 19 | self.dispatcher = dispatcher 20 | 21 | async def start(self): 22 | pass 23 | 24 | def add_routes(self, app): 25 | 26 | app.add_routes([ 27 | web.post(self.path, self.handle), 28 | ]) 29 | 30 | async def handle(self, request): 31 | 32 | print(request.path, "...") 33 | 34 | try: 35 | ht = request.headers["Authorization"] 36 | tokens = ht.split(" ", 2) 37 | if tokens[0] != "Bearer": 38 | return web.HTTPUnauthorized() 39 | token = tokens[1] 40 | except: 41 | token = "" 42 | 43 | if not self.auth.permitted(token, self.operation): 44 | return web.HTTPUnauthorized() 45 | 46 | try: 47 | 48 | data = await request.json() 49 | 50 | async def responder(x, fin): 51 | pass 52 | 53 | resp = await self.dispatcher.process( 54 | data, responder, request.match_info 55 | ) 56 | 57 | return web.json_response(resp) 58 | 59 | except Exception as e: 60 | logging.error(f"Exception: {e}") 61 | 62 | return web.json_response( 63 | { "error": str(e) } 64 | ) 65 | 66 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/running.py: -------------------------------------------------------------------------------- 1 | 2 | class Running: 3 | def __init__(self): self.running = True 4 | def get(self): return self.running 5 | def stop(self): self.running = False 6 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/unused/dbpedia.py: -------------------------------------------------------------------------------- 1 | 2 | from .. schema import LookupRequest, LookupResponse 3 | from .. schema import dbpedia_lookup_request_queue 4 | from .. schema import dbpedia_lookup_response_queue 5 | 6 | from . endpoint import ServiceEndpoint 7 | from . requestor import ServiceRequestor 8 | 9 | class DbpediaRequestor(ServiceRequestor): 10 | def __init__(self, pulsar_client, timeout, auth): 11 | 12 | super(DbpediaRequestor, self).__init__( 13 | pulsar_client=pulsar_client, 14 | request_queue=dbpedia_lookup_request_queue, 15 | response_queue=dbpedia_lookup_response_queue, 16 | request_schema=LookupRequest, 17 | response_schema=LookupResponse, 18 | timeout=timeout, 19 | ) 20 | 21 | def to_request(self, body): 22 | return LookupRequest( 23 | term=body["term"], 24 | kind=body.get("kind", None), 25 | ) 26 | 27 | def from_response(self, message): 28 | return { "text": message.text }, True 29 | 30 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/unused/encyclopedia.py: -------------------------------------------------------------------------------- 1 | 2 | from .. schema import LookupRequest, LookupResponse 3 | from .. schema import encyclopedia_lookup_request_queue 4 | from .. schema import encyclopedia_lookup_response_queue 5 | 6 | from . endpoint import ServiceEndpoint 7 | from . requestor import ServiceRequestor 8 | 9 | class EncyclopediaRequestor(ServiceRequestor): 10 | def __init__(self, pulsar_client, timeout, auth): 11 | 12 | super(EncyclopediaRequestor, self).__init__( 13 | pulsar_client=pulsar_client, 14 | request_queue=encyclopedia_lookup_request_queue, 15 | response_queue=encyclopedia_lookup_response_queue, 16 | request_schema=LookupRequest, 17 | response_schema=LookupResponse, 18 | timeout=timeout, 19 | ) 20 | 21 | def to_request(self, body): 22 | return LookupRequest( 23 | term=body["term"], 24 | kind=body.get("kind", None), 25 | ) 26 | 27 | def from_response(self, message): 28 | return { "text": message.text }, True 29 | 30 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/gateway/unused/internet_search.py: -------------------------------------------------------------------------------- 1 | 2 | from .. schema import LookupRequest, LookupResponse 3 | from .. schema import internet_search_request_queue 4 | from .. schema import internet_search_response_queue 5 | 6 | from . endpoint import ServiceEndpoint 7 | from . requestor import ServiceRequestor 8 | 9 | class InternetSearchRequestor(ServiceRequestor): 10 | def __init__(self, pulsar_client, timeout, auth): 11 | 12 | super(InternetSearchRequestor, self).__init__( 13 | pulsar_client=pulsar_client, 14 | request_queue=internet_search_request_queue, 15 | response_queue=internet_search_response_queue, 16 | request_schema=LookupRequest, 17 | response_schema=LookupResponse, 18 | timeout=timeout, 19 | ) 20 | 21 | def to_request(self, body): 22 | return LookupRequest( 23 | term=body["term"], 24 | kind=body.get("kind", None), 25 | ) 26 | 27 | def from_response(self, message): 28 | return { "text": message.text }, True 29 | 30 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/librarian/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . service import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/librarian/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . service import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/metering/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . counter import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/metering/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . counter import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/model/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/prompt/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/model/prompt/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/prompt/generic/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . service import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/prompt/generic/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . service import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/prompt/template/README.md: -------------------------------------------------------------------------------- 1 | 2 | prompt-template \ 3 | -p pulsar://localhost:6650 \ 4 | --system-prompt 'You are a {{attitude}}, you are called {{name}}' \ 5 | --global-term \ 6 | 'name=Craig' \ 7 | 'attitude=LOUD, SHOUTY ANNOYING BOT' \ 8 | --prompt \ 9 | 'question={{question}}' \ 10 | 'french-question={{question}}' \ 11 | "analyze=Find the name and age in this text, and output a JSON structure containing just the name and age fields: {{description}}. Don't add markup, just output the raw JSON object." \ 12 | "graph-query=Study the following knowledge graph, and then answer the question.\\n\nGraph:\\n{% for edge in knowledge %}({{edge.0}})-[{{edge.1}}]->({{edge.2}})\\n{%endfor%}\\nQuestion:\\n{{question}}" \ 13 | "extract-definition=Analyse the text provided, and then return a list of terms and definitions. The output should be a JSON array, each item in the array is an object with fields 'term' and 'definition'.Don't add markup, just output the raw JSON object. Here is the text:\\n{{text}}" \ 14 | --prompt-response-type \ 15 | 'question=text' \ 16 | 'analyze=json' \ 17 | 'graph-query=text' \ 18 | 'extract-definition=json' \ 19 | --prompt-term \ 20 | 'question=name:Bonny' \ 21 | 'french-question=attitude:French-speaking bot' \ 22 | --prompt-schema \ 23 | 'analyze={ "type" : "object", "properties" : { "age": { "type" : "number" }, "name": { "type" : "string" } } }' \ 24 | 'extract-definition={ "type": "array", "items": { "type": "object", "properties": { "term": { "type": "string" }, "definition": { "type": "string" } }, "required": [ "term", "definition" ] } }' 25 | 26 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/prompt/template/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . service import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/prompt/template/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . service import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/model/text_completion/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/azure/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . llm import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/azure/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . llm import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/azure_openai/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . llm import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/azure_openai/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . llm import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/claude/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . llm import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/claude/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . llm import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/cohere/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . llm import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/cohere/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . llm import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/googleaistudio/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . llm import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/googleaistudio/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . llm import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/llamafile/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . llm import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/llamafile/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . llm import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/lmstudio/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . llm import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/lmstudio/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . llm import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/mistral/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . llm import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/mistral/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . llm import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/ollama/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . llm import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/ollama/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . llm import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/openai/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . llm import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/openai/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . llm import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/tgi/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . llm import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/model/text_completion/tgi/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . llm import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/processing/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . processing import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/processing/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . processing import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/query/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/doc_embeddings/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/query/doc_embeddings/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/doc_embeddings/milvus/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . service import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/doc_embeddings/milvus/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . hf import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/doc_embeddings/pinecone/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . service import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/doc_embeddings/pinecone/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . hf import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/doc_embeddings/qdrant/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . service import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/doc_embeddings/qdrant/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . hf import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/graph_embeddings/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/query/graph_embeddings/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/graph_embeddings/milvus/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . service import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/graph_embeddings/milvus/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . hf import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/graph_embeddings/pinecone/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . service import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/graph_embeddings/pinecone/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . hf import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/graph_embeddings/qdrant/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . service import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/graph_embeddings/qdrant/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . hf import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/triples/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/query/triples/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/triples/cassandra/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . service import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/triples/cassandra/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . hf import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/triples/falkordb/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . service import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/triples/falkordb/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . hf import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/triples/memgraph/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . service import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/triples/memgraph/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . hf import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/triples/neo4j/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . service import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/query/triples/neo4j/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . hf import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/retrieval/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/retrieval/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/retrieval/document_rag/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . rag import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/retrieval/document_rag/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . rag import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/retrieval/graph_rag/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . rag import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/retrieval/graph_rag/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . rag import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/storage/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/doc_embeddings/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/storage/doc_embeddings/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/doc_embeddings/milvus/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . write import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/doc_embeddings/milvus/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . write import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/doc_embeddings/pinecone/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . write import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/doc_embeddings/pinecone/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . write import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/doc_embeddings/qdrant/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . write import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/doc_embeddings/qdrant/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . write import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/graph_embeddings/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/storage/graph_embeddings/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/graph_embeddings/milvus/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . write import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/graph_embeddings/milvus/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . write import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/graph_embeddings/pinecone/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . write import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/graph_embeddings/pinecone/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . write import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/graph_embeddings/qdrant/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . write import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/graph_embeddings/qdrant/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . write import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/knowledge/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . store import run 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/knowledge/__main__.py: -------------------------------------------------------------------------------- 1 | 2 | from . store import run 3 | 4 | if __name__ == '__main__': 5 | run() 6 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/object_embeddings/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/storage/object_embeddings/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/object_embeddings/milvus/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . write import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/object_embeddings/milvus/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . write import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/rows/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/storage/rows/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/rows/cassandra/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . write import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/rows/cassandra/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . write import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/triples/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/storage/triples/__init__.py -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/triples/cassandra/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . write import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/triples/cassandra/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . write import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/triples/falkordb/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . write import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/triples/falkordb/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . write import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/triples/memgraph/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . write import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/triples/memgraph/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . write import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/triples/neo4j/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . write import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/storage/triples/neo4j/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . write import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-flow/trustgraph/tables/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trustgraph-ai/trustgraph/6b24da422baad000a0cecc9e43e7299fef1391d9/trustgraph-flow/trustgraph/tables/__init__.py -------------------------------------------------------------------------------- /trustgraph-ocr/README.md: -------------------------------------------------------------------------------- 1 | See https://trustgraph.ai/ 2 | -------------------------------------------------------------------------------- /trustgraph-ocr/scripts/pdf-ocr: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.decoding.ocr import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-ocr/setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | import os 3 | import importlib 4 | 5 | with open("README.md", "r") as fh: 6 | long_description = fh.read() 7 | 8 | # Load a version number module 9 | spec = importlib.util.spec_from_file_location( 10 | 'version', 'trustgraph/ocr_version.py' 11 | ) 12 | version_module = importlib.util.module_from_spec(spec) 13 | spec.loader.exec_module(version_module) 14 | 15 | version = version_module.__version__ 16 | 17 | setuptools.setup( 18 | name="trustgraph-ocr", 19 | version=version, 20 | author="trustgraph.ai", 21 | author_email="security@trustgraph.ai", 22 | description="TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.", 23 | long_description=long_description, 24 | long_description_content_type="text/markdown", 25 | url="https://github.com/trustgraph-ai/trustgraph", 26 | packages=setuptools.find_namespace_packages( 27 | where='./', 28 | ), 29 | classifiers=[ 30 | "Programming Language :: Python :: 3", 31 | "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", 32 | "Operating System :: OS Independent", 33 | ], 34 | python_requires='>=3.8', 35 | download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz", 36 | install_requires=[ 37 | "trustgraph-base>=1.0,<1.1", 38 | "pulsar-client", 39 | "prometheus-client", 40 | "boto3", 41 | "pdf2image", 42 | "pytesseract", 43 | ], 44 | scripts=[ 45 | "scripts/pdf-ocr", 46 | ] 47 | ) 48 | -------------------------------------------------------------------------------- /trustgraph-ocr/trustgraph/decoding/ocr/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . pdf_decoder import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-ocr/trustgraph/decoding/ocr/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . pdf_decoder import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph-vertexai/README.md: -------------------------------------------------------------------------------- 1 | See https://trustgraph.ai/ 2 | -------------------------------------------------------------------------------- /trustgraph-vertexai/scripts/text-completion-vertexai: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from trustgraph.model.text_completion.vertexai import run 4 | 5 | run() 6 | 7 | -------------------------------------------------------------------------------- /trustgraph-vertexai/setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | import os 3 | import importlib 4 | 5 | with open("README.md", "r") as fh: 6 | long_description = fh.read() 7 | 8 | # Load a version number module 9 | spec = importlib.util.spec_from_file_location( 10 | 'version', 'trustgraph/vertexai_version.py' 11 | ) 12 | version_module = importlib.util.module_from_spec(spec) 13 | spec.loader.exec_module(version_module) 14 | 15 | version = version_module.__version__ 16 | 17 | setuptools.setup( 18 | name="trustgraph-vertexai", 19 | version=version, 20 | author="trustgraph.ai", 21 | author_email="security@trustgraph.ai", 22 | description="TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.", 23 | long_description=long_description, 24 | long_description_content_type="text/markdown", 25 | url="https://github.com/trustgraph-ai/trustgraph", 26 | packages=setuptools.find_namespace_packages( 27 | where='./', 28 | ), 29 | classifiers=[ 30 | "Programming Language :: Python :: 3", 31 | "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", 32 | "Operating System :: OS Independent", 33 | ], 34 | python_requires='>=3.8', 35 | download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz", 36 | install_requires=[ 37 | "trustgraph-base>=1.0,<1.1", 38 | "pulsar-client", 39 | "google-cloud-aiplatform", 40 | "prometheus-client", 41 | ], 42 | scripts=[ 43 | "scripts/text-completion-vertexai", 44 | ] 45 | ) 46 | -------------------------------------------------------------------------------- /trustgraph-vertexai/trustgraph/model/text_completion/vertexai/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from . llm import * 3 | 4 | -------------------------------------------------------------------------------- /trustgraph-vertexai/trustgraph/model/text_completion/vertexai/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from . llm import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | 8 | -------------------------------------------------------------------------------- /trustgraph/README.md: -------------------------------------------------------------------------------- 1 | See https://trustgraph.ai/ 2 | -------------------------------------------------------------------------------- /trustgraph/setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | import os 3 | import importlib 4 | 5 | with open("README.md", "r") as fh: 6 | long_description = fh.read() 7 | 8 | # Load a version number module 9 | spec = importlib.util.spec_from_file_location( 10 | 'version', 'trustgraph/trustgraph_version.py' 11 | ) 12 | version_module = importlib.util.module_from_spec(spec) 13 | spec.loader.exec_module(version_module) 14 | 15 | version = version_module.__version__ 16 | 17 | setuptools.setup( 18 | name="trustgraph", 19 | version=version, 20 | author="trustgraph.ai", 21 | author_email="security@trustgraph.ai", 22 | description="TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.", 23 | long_description=long_description, 24 | long_description_content_type="text/markdown", 25 | url="https://github.com/trustgraph-ai/trustgraph", 26 | packages=setuptools.find_namespace_packages( 27 | where='./', 28 | ), 29 | classifiers=[ 30 | "Programming Language :: Python :: 3", 31 | "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", 32 | "Operating System :: OS Independent", 33 | ], 34 | python_requires='>=3.8', 35 | download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz", 36 | install_requires=[ 37 | "trustgraph-base>=1.0,<1.1", 38 | "trustgraph-bedrock>=1.0,<1.1", 39 | "trustgraph-cli>=1.0,<1.1", 40 | "trustgraph-embeddings-hf>=1.0,<1.1", 41 | "trustgraph-flow>=1.0,<1.1", 42 | "trustgraph-vertexai>=1.0,<1.1", 43 | ], 44 | scripts=[ 45 | ] 46 | ) 47 | --------------------------------------------------------------------------------