├── .gitignore ├── README.md ├── agvector ├── AGVector-example.ipynb ├── GraphRAG-Embedding-with-Properties.ipynb ├── README.md ├── chomsky.nq.tgz ├── contract.def ├── contract.nt ├── create-environment.sh ├── images │ ├── ask_my_contract.png │ ├── ask_my_documents.png │ ├── chomsky-book-layout.png │ ├── chomsky-response-in-graph.png │ ├── contract-compensation.png │ ├── contract-neighbors.png │ ├── nearestneighbors.png │ ├── philosophy-books.png │ ├── webview-llm-embed-contract.png │ └── webview-llm-embed-philosophy.png ├── llm_utils.py ├── philosophy.def └── requirements.txt ├── ansible ├── Makefile ├── README.md ├── agraph.cfg-template ├── agraph.init ├── agraph.yaml ├── ansible.md ├── clean-install.yaml ├── install.yaml ├── inventory.txt ├── make-agraph.cfg ├── start.yaml ├── stop.yaml └── vars.yaml ├── clustering ├── README.md ├── aws-repl │ ├── Makefile │ ├── create.sh │ ├── install.sh │ ├── join.sh │ ├── joincluster │ ├── start.sh │ └── vars.sh ├── kubernetes │ └── mmr │ │ ├── agmmr │ │ ├── Dockerfile │ │ ├── Makefile │ │ ├── agraph.cfg.in │ │ ├── dnspatch.cl │ │ └── runrepl.sh │ │ ├── helm │ │ ├── Makefile │ │ └── agraphmmr │ │ │ ├── .helmignore │ │ │ ├── Chart.yaml │ │ │ ├── templates │ │ │ ├── NOTES.txt │ │ │ ├── controlling-lb.yaml │ │ │ ├── controlling-service.yaml │ │ │ ├── controlling-ss.yaml │ │ │ ├── copy-0-lb.yaml │ │ │ ├── copy-lb.yaml │ │ │ ├── copy-service.yaml │ │ │ ├── copy-ss.yaml │ │ │ └── secrets.yaml │ │ │ └── values.yaml │ │ └── kubernetes-mmr.md ├── misc │ └── using-nginx-load-balancer.md └── terraform-elb │ ├── agelb.tf │ ├── agraph_mmr_elb.svg │ └── using-terraform.md ├── data ├── bitcoin │ ├── README.md │ ├── convert.py │ ├── model.ttl │ └── requirements.txt └── chomsky-llm-example │ ├── README.md │ └── chomsky.nq.gz ├── databricks ├── README.md ├── demo.mapping.obda ├── demo.properties └── img │ ├── cluster-main.png │ ├── data-explorer.png │ ├── gruff.png │ └── sample-data.png ├── generators └── weather │ ├── weather.md │ └── wxdemo.cl ├── geodemo ├── PhoneCalls.ttl ├── generating-phonecalls.cl ├── geodemo.cl ├── geodemo.md └── geodemosteve.png ├── geosparql ├── README.md ├── data │ ├── Mobile_Food_Facility_Permit.csv │ ├── franz.ttl │ └── geosparql-example.ttl ├── environment.yml ├── geosparql-examples.ipynb ├── geosparql-tutorial.ipynb ├── img │ ├── geosparql-etl-layout.png │ ├── geosparql-example.png │ ├── geosparql-ontology.png │ ├── screenshot-1.jpg │ ├── screenshot-2.jpg │ └── screenshot-3.jpg └── requirements.txt ├── gnn └── Events2018.ipynb ├── grafana ├── README.md └── provisioning │ ├── dashboards │ ├── actors-dashboard.json │ └── dashboards.yml │ └── datasources │ └── ag-no-auth.yml ├── gruff ├── embedding.html ├── img │ ├── paths.png │ └── query.png └── readme.md ├── java-http-sessions ├── Makefile ├── Readme.txt ├── com │ └── franz │ │ └── sessionpool │ │ ├── LbDemo.java │ │ ├── LoadBalancer.java │ │ ├── SampleQuery.java │ │ ├── ServerSpec.java │ │ └── SessionPool.java └── lib │ ├── agraph-java-client-3.0.6-SNAPSHOT.jar │ ├── aopalliance-1.0.jar │ ├── asm-4.2.jar │ ├── atomikos-util-4.0.6.jar │ ├── cglib-3.1.jar │ ├── collection-0.6.jar │ ├── commons-beanutils-1.9.3.jar │ ├── commons-cli-1.2.jar │ ├── commons-codec-1.4.jar │ ├── commons-collections-3.2.2.jar │ ├── commons-collections4-4.1.jar │ ├── commons-csv-1.3.jar │ ├── commons-httpclient-3.1.jar │ ├── commons-io-2.4.jar │ ├── commons-lang-2.6.jar │ ├── commons-lang3-3.4.jar │ ├── commons-logging-1.1.1.jar │ ├── commons-math3-3.5.jar │ ├── commons-pool2-2.4.2.jar │ ├── commons-text-1.3.jar │ ├── guava-18.0.jar │ ├── httpclient-4.5.2.jar │ ├── httpclient-cache-4.5.2.jar │ ├── httpcore-4.4.4.jar │ ├── httpmime-4.5.5.jar │ ├── jcl-over-slf4j-1.7.21.jar │ ├── jline-3.5.1.jar │ ├── libthrift-0.9.3.jar │ ├── mapdb-1.0.8.jar │ ├── noggit-0.6.jar │ ├── opencsv-4.2.jar │ ├── slf4j-api-1.7.25.jar │ ├── slf4j-nop-1.7.25.jar │ ├── spatial4j-0.7.jar │ ├── standard-1.1.2.jar │ ├── stax2-api-3.1.4.jar │ └── xml-apis-1.4.01.jar ├── llm ├── README.md ├── environment.yml ├── llm_utils.py ├── nlq.ipynb ├── ollama-sparql-integration.ipynb └── requirements.txt ├── ontop ├── Dockerfile.bootstrap ├── Dockerfile.endpoint ├── Dockerfile.materialize ├── Dockerfile.pg ├── README.md ├── bin │ ├── demo.mini-mapping.obda │ ├── demo.properties │ └── init-db.sh ├── img │ ├── query.jpg │ └── schema.png └── ontop_output │ └── .gitkeep ├── spark ├── .docker │ ├── Dockerfile.jupyter │ ├── docker-compose.yml │ └── kennedy.ntriples ├── AGSpark.ipynb ├── Makefile ├── README.md └── img │ └── kennedy.png ├── streaming └── kafka │ ├── .gitignore │ ├── Dockerfile │ ├── Makefile │ ├── README.md │ ├── docker-compose.yaml │ ├── img │ ├── apache_kafka.png │ ├── data_model.png │ ├── diagram.png │ ├── kafka_web_ui.png │ ├── kafka_web_ui_posts.png │ └── kafka_web_ui_recommendations.png │ ├── poetry.lock │ ├── pyproject.toml │ └── streaming │ ├── __init__.py │ ├── agent.py │ ├── cli.py │ ├── data │ ├── __init__.py │ ├── data.nt.bz2 │ ├── sample_posts_idx.txt.bz2 │ └── sample_users_idx.txt.bz2 │ ├── init_repo.py │ ├── model.py │ └── producer.py └── tutorial-files ├── PhoneCalls.ttl.gz ├── README.md ├── SpaceMissions2.ntriples ├── SpaceMissions2.owl ├── agraph-tutorial.cl ├── federation-examples.cl ├── freetext-index-tutorial.cl ├── kennedy-family.cl ├── kennedy.ntriples ├── lesmis.rdf ├── metarelations.cl ├── reasoner-tutorial.cl ├── remote-client.cl ├── sna-examples.cl ├── sparql-examples.cl ├── stored-proc-example.cl ├── wilburwine.ntriples └── wilburwine.rdf /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | ontop/ontop_output/demo* 3 | .vscode/ 4 | **/.ipynb_checkpoints/ 5 | **/__pycache__/ 6 | **/.ipynb_checkpoints/** 7 | **/.venv/ 8 | **/.venv/** 9 | **/.env 10 | **/venv/ 11 | **/venv/** 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Welcome to AllegroGraph examples! 2 | 3 | This repository contains examples of using AllegroGraph in advanced scenarios. 4 | 5 | ## Featured examples 6 | 7 | * LLM-Vector Database 8 | * [LLM Embedding Examples](agvector/README.md) 9 | * Apache Spark 10 | * [Graph Analytics by AllegroGraph and Apache Spark](spark/AGSpark.ipynb) 11 | * Configuration and setup examples 12 | * [Clustering](clustering) 13 | * [Load balancing multi-master replication cluster on AWS using Terraform](clustering/terraform-elb/using-terraform.md) 14 | * [AllegroGraph Replication using NGINX Load Balancer](clustering/misc/using-nginx-load-balancer.md) 15 | * [MMR Replication Clusters in Kubernetes and Docker Swarm](clustering/kubernetes/mmr/kubernetes-mmr.md) 16 | * [Ansible for multi-server installation](ansible) 17 | * Databricks (Delta Lake) 18 | * [Tutorial - AllegroGraph Semantic Layer for Databricks (Delta Lake)](databricks/README.md) 19 | * Data examples 20 | * [Bitcoin RDF model](data/bitcoin/README.md) 21 | * Generate AllegroGraph databases 22 | * [Weather data](generators/weather/weather.md) 23 | * Graph neural networks 24 | * [Inference over Temporal Knowledge Graphs](gnn/Events2018.ipynb) 25 | * Streaming 26 | * [Construct a Graph Streaming Pipeline by Using Kafka and AllegroGraph](streaming/kafka/README.md) 27 | * Virtual knowledge graphs 28 | * [Creating Virtual and Materialized Graphs in AllegroGraph](ontop/README.md) 29 | * Gruff 30 | * [Embedding Gruff In a Web Page](gruff/readme.md) 31 | * GeoSPARQL 32 | * [GeoSPARQL Tutorial](geosparql/geosparql-tutorial.ipynb) 33 | * Google Maps and nD Search Demo 34 | * [geodemo](geodemo/geodemo.md) 35 | * Grafana 36 | * [Grafana - Business Intelligence Dashboard](grafana/README.md) 37 | 38 | ## AllegroGraph Tutorial files 39 | 40 | These AllegroGraph Tutorial files are referenced from the 41 | [AllegroGraph Documentation](https://franz.com/agraph/support/documentation/current/index.html). 42 | 43 | * [PhoneCalls.ttl.gz](tutorial-files/PhoneCalls.ttl.gz) 44 | * [SpaceMissions2.ntriples](tutorial-files/SpaceMissions2.ntriples) 45 | * [SpaceMissions2.owl](tutorial-files/SpaceMissions2.owl) 46 | * [agraph-tutorial.cl](tutorial-files/agraph-tutorial.cl) 47 | * [federation-examples.cl](tutorial-files/federation-examples.cl) 48 | * [freetext-index-tutorial.cl](tutorial-files/freetext-index-tutorial.cl) 49 | * [kennedy-family.cl](tutorial-files/kennedy-family.cl) 50 | * [kennedy.ntriples](tutorial-files/kennedy.ntriples) 51 | * [lesmis.rdf](tutorial-files/lesmis.rdf) 52 | * [metarelations.cl](tutorial-files/metarelations.cl) 53 | * [reasoner-tutorial.cl](tutorial-files/reasoner-tutorial.cl) 54 | * [remote-client.cl](tutorial-files/remote-client.cl) 55 | * [sna-examples.cl](tutorial-files/sna-examples.cl) 56 | * [sparql-examples.cl](tutorial-files/sparql-examples.cl) 57 | * [stored-proc-example.cl](tutorial-files/stored-proc-example.cl) 58 | * [wilburwine.ntriples](tutorial-files/wilburwine.ntriples) 59 | * [wilburwine.rdf](tutorial-files/wilburwine.rdf) 60 | -------------------------------------------------------------------------------- /agvector/README.md: -------------------------------------------------------------------------------- 1 | # AllegroGraph LLM Embedding Examples 2 | 3 | We recommend you run the [Embedding-with-Properties.ipynb](Embedding_with_Properties.ipynb) notebook to see how embedding and vector stores work in AllegroGraph `>= 8.1.0`. The [AGVector-example.ipynb](AGVector-example.ipynb) notebook works but its methods have been greatly improved upon. It was created for versions `8.0.0` and `8.0.1`. 4 | 5 | ## Installation 6 | Welcome to AllegroGraph's LLM Embedding Example Jupyter notebook. To start, please run the following commands to create a conda environment. 7 | 8 | ### Conda 9 | First, create a new environment 10 | ```shell 11 | conda create -n agvector python=3.10 12 | ``` 13 | 14 | then activate the new conda environment with: 15 | ```shell 16 | conda activate agvector 17 | ``` 18 | 19 | then run the install environment script 20 | 21 | ```shell 22 | ./create-environment.sh 23 | ``` 24 | 25 | ### pip 26 | 27 | first create a new environment with: 28 | 29 | ```shell 30 | python3.11 -m venv venv 31 | ``` 32 | 33 | Activate it using: 34 | 35 | ```shell 36 | source venv/bin/activate 37 | ``` 38 | 39 | then install the requirements with: 40 | 41 | ```shell 42 | pip install -r requirements.txt 43 | ``` 44 | 45 | 46 | ## Setting up OpenAI API Key in AG Webview for the `AGVector-example.ipynb` Notebook only 47 | 48 | This only applies to agraph `8.0.0` and `8.0.1`. 49 | 50 | To actually perform the SPARQL queries you need to set your OpenAI API key as a query option in webview. Please follow the following instructions to do so. 51 | 52 | 1. Please navigate to your local installation of the new webview 53 | 2. Go to the repository where your data is stored (`llm-philosophy` if you're using the repo created from this demo) 54 | 3. Go to `Repository Control` in the left column under `Repository` and search for `query execution options`. Select it. 55 | 4. Select `+ New Query Option` and add **openaiApiKey** as the _name_, and your OpenAI api key as the _value_. Set the `Scope` to **Repository** 56 | 5. Don't forget to save by hitting `Save Query Options`! 57 | 58 | ## Steps to deleting existing vector databases 59 | 60 | Deletion of vector databases has been added to AGWebView. This is only relevant to versions `8.0.0` and `8.0.1` 61 | 62 | 1. In your terminal, navigate to your AllegroGraph installation. 63 | 2. `cd` to `/data/rootcatalog/` and then to the repository, in our case `llm-philosophy` 64 | 3. `rm *.vdb.*` if you want to delete all existing VDBs associated with that repo, otherwise you can manually `rm` the `llm-philosophy.vdb.vec` and `llm-philosophy.vdb.dat` files. 65 | -------------------------------------------------------------------------------- /agvector/chomsky.nq.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/agvector/chomsky.nq.tgz -------------------------------------------------------------------------------- /agvector/contract.def: -------------------------------------------------------------------------------- 1 | gpt 2 | api-key "your-openai-api-key-here" 3 | vector-database-name localhost:10035/contract-def 4 | embedder openai 5 | include-predicates 6 | -------------------------------------------------------------------------------- /agvector/create-environment.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | pip install agraph-python==104.2.0 4 | wait 5 | 6 | conda install -c conda-forge langchain -y 7 | wait 8 | 9 | conda install -c conda-forge shortuuid -y 10 | wait 11 | 12 | conda install -c conda-forge jupyterlab=3.5.0 -y 13 | wait 14 | 15 | conda install -c anaconda nb_conda_kernels -y 16 | wait 17 | 18 | conda install -c anaconda pandas -y 19 | wait 20 | 21 | conda install tqdm -y 22 | wait 23 | -------------------------------------------------------------------------------- /agvector/images/ask_my_contract.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/agvector/images/ask_my_contract.png -------------------------------------------------------------------------------- /agvector/images/ask_my_documents.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/agvector/images/ask_my_documents.png -------------------------------------------------------------------------------- /agvector/images/chomsky-book-layout.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/agvector/images/chomsky-book-layout.png -------------------------------------------------------------------------------- /agvector/images/chomsky-response-in-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/agvector/images/chomsky-response-in-graph.png -------------------------------------------------------------------------------- /agvector/images/contract-compensation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/agvector/images/contract-compensation.png -------------------------------------------------------------------------------- /agvector/images/contract-neighbors.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/agvector/images/contract-neighbors.png -------------------------------------------------------------------------------- /agvector/images/nearestneighbors.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/agvector/images/nearestneighbors.png -------------------------------------------------------------------------------- /agvector/images/philosophy-books.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/agvector/images/philosophy-books.png -------------------------------------------------------------------------------- /agvector/images/webview-llm-embed-contract.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/agvector/images/webview-llm-embed-contract.png -------------------------------------------------------------------------------- /agvector/images/webview-llm-embed-philosophy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/agvector/images/webview-llm-embed-philosophy.png -------------------------------------------------------------------------------- /agvector/llm_utils.py: -------------------------------------------------------------------------------- 1 | from franz.openrdf.connect import ag_connect 2 | from franz.openrdf.query.query import QueryLanguage 3 | from franz.openrdf.model.value import URI 4 | from franz.openrdf.vocabulary import RDF, RDFS 5 | import urllib.request 6 | from pprint import pprint 7 | import datetime 8 | from langchain.text_splitter import RecursiveCharacterTextSplitter 9 | from langchain.schema.document import Document 10 | import shortuuid 11 | import textwrap 12 | 13 | def read_text(url: str): 14 | with urllib.request.urlopen(url) as f: 15 | contents = f.read().decode('utf-8') 16 | return contents 17 | 18 | def print_text(string: str): 19 | string = string.replace('\\n', '').replace('\\r', ' ') 20 | wrapper = textwrap.TextWrapper(width=100) 21 | word_list = wrapper.wrap(text=string) 22 | for element in word_list: 23 | print(element) 24 | 25 | class FindNearestNeighbors: 26 | def __init__(self, conn, phrase, vector_db, number=10, confidence=.5): 27 | self.conn = conn 28 | self.f = conn.namespace('http://franz.com/llm/') 29 | self.phrase = phrase 30 | self.vector_db = vector_db 31 | self.number = number 32 | self.confidence = confidence 33 | self.df = self.query(conn, phrase, vector_db, number, confidence) 34 | try: 35 | print_text(self.df['originalText'][0]) 36 | except: 37 | print(f"No Neighbor found with confidence score {confidence}") 38 | 39 | def query(self, conn, phrase, vector_db, number, confidence): 40 | query_string = f""" 41 | PREFIX llm: 42 | select * where {{ 43 | (?uri ?score ?originalText) llm:nearestNeighbor ("{phrase}" "{vector_db}" {str(number)} {str(confidence)}) }}""" 44 | with conn.executeTupleQuery(query_string) as result: 45 | df = result.toPandas() 46 | return df 47 | 48 | def proof(self): 49 | for i in range(self.df.shape[0]): 50 | print(i, self.df['uri'][i], self.df['score'][i]) 51 | print_text(self.df['originalText'][i]) 52 | print() 53 | 54 | def add_neighbors_to_graph(self): 55 | neighbor_uri = self.conn.createURI(f'http://franz.com/llm/neighbor/{shortuuid.uuid()}') 56 | triples = [ 57 | (neighbor_uri, RDF.TYPE, self.f.Neighbor, neighbor_uri), 58 | (neighbor_uri, self.f.phrase, self.conn.createLiteral(self.phrase), neighbor_uri), 59 | (neighbor_uri, self.f.vectorDB, self.vector_db, neighbor_uri), 60 | (neighbor_uri, self.f.confidence, self.confidence, neighbor_uri), 61 | (neighbor_uri, self.f.datetime, datetime.datetime.now(), neighbor_uri)] 62 | for i in range(self.df.shape[0]): 63 | neighbor_score = self.conn.createBNode() 64 | triples.append((neighbor_uri, self.f.link, neighbor_score, neighbor_uri)) 65 | triples.append((neighbor_score, self.f.confidenceScore, self.df['score'][i], neighbor_uri)) 66 | triples.append((neighbor_score, self.f.index, i, neighbor_uri)) 67 | triples.append((neighbor_score, self.f.neighbor, self.conn.createURI(self.df['uri'][i][1:-1]), neighbor_uri)) 68 | self.conn.addTriples(triples) 69 | 70 | def clear_neighbors(conn): 71 | query_string = """select ?neighbor where { ?neighbor a }""" 72 | with conn.executeTupleQuery(query_string) as result: 73 | df = result.toPandas() 74 | for neighbor in list(df['neighbor']): 75 | conn.remove(None, None, None, neighbor) 76 | 77 | class AskMyDocuments: 78 | def __init__(self, conn, question, vector_db, number=10, confidence=.5): 79 | self.conn = conn 80 | self.f = conn.namespace('http://franz.com/llm/') 81 | self.question = question 82 | self.vector_db = vector_db 83 | self.number = number 84 | self.confidence = confidence 85 | self.df = self.query(conn, question, vector_db, number, confidence) 86 | try: 87 | print_text(self.df['response'][0]) 88 | except: 89 | print(f"No response found with confidence score {confidence}") 90 | 91 | def query(self, conn, question, vector_db, number, confidence): 92 | query_string = f"""PREFIX llm: 93 | select * where {{ 94 | (?response ?score ?citation ?content) llm:askMyDocuments ("{question}" "{vector_db}" {str(number)} {str(confidence)}). }}""" 95 | with conn.executeTupleQuery(query_string) as result: 96 | df = result.toPandas() 97 | return df 98 | 99 | def proof(self): 100 | for i in range(self.df.shape[0]): 101 | print(i, self.df['score'][i], self.df['citation'][i]) 102 | print_text(self.df['content'][i]) 103 | print() 104 | 105 | def add_evidence_to_graph(self): 106 | if self.df.shape[0] > 0: 107 | evidence_uri = self.conn.createURI(f'http://franz.com/llm/evidence/{shortuuid.uuid()}') 108 | triples = [ 109 | (evidence_uri, RDF.TYPE, self.f.Question, evidence_uri), 110 | (evidence_uri, self.f.question, self.conn.createLiteral(self.question), evidence_uri), 111 | (evidence_uri, self.f.vectorDB, self.vector_db, evidence_uri), 112 | (evidence_uri, self.f.confidence, self.confidence, evidence_uri), 113 | (evidence_uri, self.f.datetime, datetime.datetime.now(), evidence_uri), 114 | (evidence_uri, self.f.response, self.conn.createLiteral(self.df['response'][0]), evidence_uri)] 115 | for i in range(self.df.shape[0]): 116 | evidence_score = self.conn.createBNode() 117 | triples.append((evidence_uri, self.f.link, evidence_score, evidence_uri)) 118 | triples.append((evidence_score, self.f.confidenceScore, self.df['score'][i], evidence_uri)) 119 | triples.append((evidence_score, self.f.index, i, evidence_uri)) 120 | triples.append((evidence_score, self.f.evidence, self.conn.createURI(self.df['citation'][i][1:-1]), evidence_uri)) 121 | self.conn.addTriples(triples) 122 | else: 123 | print("No evidence found") 124 | 125 | def clear_questions(conn): 126 | query_string = """select ?question where { ?question a }""" 127 | with conn.executeTupleQuery(query_string) as result: 128 | df = result.toPandas() 129 | for question in list(df['question']): 130 | conn.remove(None, None, None, question) 131 | 132 | class BufferTriples: 133 | def __init__(self, conn, max_size=10000): 134 | self.conn = conn 135 | self.buffer_triples = [] 136 | self.max_size = max_size 137 | def add(self, triple): 138 | if len(self.buffer_triples) < self.max_size: 139 | self.buffer_triples.append(triple) 140 | else: 141 | self.conn.addTriples(self.buffer_triples) 142 | self.buffer_triples = [triple] 143 | def flush_triples(self): 144 | self.conn.addTriples(self.buffer_triples) 145 | self.buffer_triples=[] 146 | 147 | def addArbitraryTextString(conn, f, buffer, text, id, chunk_size=1000, chunk_overlap=10): 148 | documents = [Document(page_content=text)] 149 | text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap) 150 | docs = text_splitter.split_documents(documents) 151 | 152 | if isinstance(id, str): id_uri = conn.createURI(id) 153 | elif isinstance(id, URI): id_uri = id 154 | elif isinstance(id, int): id_uri = conn.createURI(f"http://franz.com/llm/{str(id)}") 155 | id_str = id_uri.localname 156 | 157 | for i, doc in enumerate(docs): 158 | doc_id = conn.createURI(f"http://franz.com/llm/{id_str}_{str(i)}") 159 | buffer.add((id_uri, f.chunk, doc_id, id_uri)) 160 | buffer.add((doc_id, RDF.TYPE, f.Chunk, id_uri)) 161 | content = docs[i].page_content 162 | buffer.add((doc_id, f.text, docs[i].page_content, id_uri)) 163 | buffer.add((doc_id, f.section, i, id_uri)) 164 | return buffer 165 | 166 | 167 | -------------------------------------------------------------------------------- /agvector/philosophy.def: -------------------------------------------------------------------------------- 1 | gpt 2 | api-key "your-openai-api-key-here" 3 | vector-database-name localhost:10035/philosophy-vec 4 | embedder openai 5 | include-predicates 6 | -------------------------------------------------------------------------------- /agvector/requirements.txt: -------------------------------------------------------------------------------- 1 | agraph-python==104.1.1 2 | aiohappyeyeballs==2.4.6 3 | aiohttp==3.11.12 4 | aiosignal==1.3.2 5 | annotated-types==0.7.0 6 | anyio==4.8.0 7 | argon2-cffi==23.1.0 8 | argon2-cffi-bindings==21.2.0 9 | arrow==1.3.0 10 | asttokens==3.0.0 11 | async-lru==2.0.4 12 | attrs==25.1.0 13 | babel==2.17.0 14 | beautifulsoup4==4.13.3 15 | bleach==6.2.0 16 | certifi==2025.1.31 17 | cffi==1.17.1 18 | charset-normalizer==3.4.1 19 | comm==0.2.2 20 | debugpy==1.8.12 21 | decorator==5.1.1 22 | defusedxml==0.7.1 23 | executing==2.2.0 24 | fastjsonschema==2.21.1 25 | fqdn==1.5.1 26 | frozenlist==1.5.0 27 | greenlet==3.1.1 28 | h11==0.14.0 29 | httpcore==1.0.7 30 | httpx==0.28.1 31 | idna==3.10 32 | ipykernel==6.29.5 33 | ipython==8.32.0 34 | iso8601==2.1.0 35 | isoduration==20.11.0 36 | jedi==0.19.2 37 | Jinja2==3.1.5 38 | json5==0.10.0 39 | jsonpatch==1.33 40 | jsonpointer==3.0.0 41 | jsonschema==4.23.0 42 | jsonschema-specifications==2024.10.1 43 | jupyter-events==0.12.0 44 | jupyter-lsp==2.2.5 45 | jupyter_client==8.6.3 46 | jupyter_core==5.7.2 47 | jupyter_server==2.15.0 48 | jupyter_server_terminals==0.5.3 49 | jupyterlab==4.3.5 50 | jupyterlab_pygments==0.3.0 51 | jupyterlab_server==2.27.3 52 | langchain==0.3.18 53 | langchain-core==0.3.35 54 | langchain-text-splitters==0.3.6 55 | langsmith==0.3.8 56 | MarkupSafe==3.0.2 57 | matplotlib-inline==0.1.7 58 | mistune==3.1.1 59 | multidict==6.1.0 60 | nbclient==0.10.2 61 | nbconvert==7.16.6 62 | nbformat==5.10.4 63 | nest-asyncio==1.6.0 64 | notebook_shim==0.2.4 65 | numpy==1.26.4 66 | orjson==3.10.15 67 | overrides==7.7.0 68 | packaging==24.2 69 | pandas==2.2.3 70 | pandocfilters==1.5.1 71 | parso==0.8.4 72 | pexpect==4.9.0 73 | platformdirs==4.3.6 74 | prometheus_client==0.21.1 75 | prompt_toolkit==3.0.50 76 | propcache==0.2.1 77 | psutil==6.1.1 78 | ptyprocess==0.7.0 79 | pure_eval==0.2.3 80 | pycparser==2.22 81 | pydantic==2.10.6 82 | pydantic_core==2.27.2 83 | Pygments==2.19.1 84 | PySocks==1.7.1 85 | python-dateutil==2.9.0.post0 86 | python-json-logger==3.2.1 87 | pytz==2025.1 88 | PyYAML==6.0.2 89 | pyzmq==26.2.1 90 | referencing==0.36.2 91 | requests==2.32.3 92 | requests-toolbelt==1.0.0 93 | rfc3339-validator==0.1.4 94 | rfc3986-validator==0.1.1 95 | rpds-py==0.22.3 96 | Send2Trash==1.8.3 97 | shortuuid==1.0.13 98 | six==1.17.0 99 | sniffio==1.3.1 100 | soupsieve==2.6 101 | SQLAlchemy==2.0.38 102 | stack-data==0.6.3 103 | tenacity==9.0.0 104 | terminado==0.18.1 105 | tinycss2==1.4.0 106 | tornado==6.4.2 107 | tqdm==4.67.1 108 | traitlets==5.14.3 109 | types-python-dateutil==2.9.0.20241206 110 | typing_extensions==4.12.2 111 | tzdata==2025.1 112 | uri-template==1.3.0 113 | urllib3==2.3.0 114 | wcwidth==0.2.13 115 | webcolors==24.11.1 116 | webencodings==0.5.1 117 | websocket-client==1.8.0 118 | yarl==1.18.3 119 | zstandard==0.23.0 120 | -------------------------------------------------------------------------------- /ansible/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Ansible control of AllegroGraph 3 | # 4 | # This is an example of the use of Ansible https://ansible.com to 5 | # install and control AllegroGraph 6 | # 7 | # Ansible is very flexible and we just show one of the 8 | # many ways it can be used to control AllegroGraph. 9 | # Consider these playbooks just an example of how it 10 | # can be done and feel free to edit them to work in your 11 | # particular location 12 | # 13 | # In particular you'll want to edit vars.yaml before runnning 14 | # this script and you'll need to download a version of AllegroGraph 15 | # in tar.gz form and store it in a location accessible to all servers 16 | # (and put that location vars.yaml). 17 | # 18 | # You'll want to edit inventory.yaml to specify the machines on 19 | # which to install AllegroGraph 20 | # 21 | 22 | default: 23 | @echo make rules are start, stop and install 24 | @echo will ping your servers now: 25 | @echo 26 | ansible -i inventory.txt -m ping agservers 27 | 28 | 29 | start: 30 | ansible-playbook -v -i inventory.txt start.yaml 31 | 32 | stop: 33 | ansible-playbook -v -i inventory.txt stop.yaml 34 | 35 | 36 | install: 37 | ansible-playbook -v -i inventory.txt install.yaml 38 | 39 | clean-install: 40 | ansible-playbook -v -i inventory.txt clean-install.yaml 41 | 42 | # create the agraph shell script that is then stored in /etc/init.d 43 | agraph: agraph.init vars.yaml 44 | ansible-playbook -v agraph.yaml 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /ansible/README.md: -------------------------------------------------------------------------------- 1 | # Installing AllegroGraph with ansible 2 | 3 | ## Introduction 4 | This directory contains [ansible](https://www.ansible.com) playbooks for installing, 5 | starting and stopping an AllegroGraph server on one or 6 | more machines. 7 | 8 | You must edit three files to personalize the configuration 9 | and then you can use the Makefile to install, start and stop the 10 | AllegroGraph servers on one or more machines. 11 | 12 | There is also a Makefile rule called `agraph` for creating the `/etc/rc.d/init.d` 13 | shell script for starting and stopping the server when the machine 14 | is started or shutdown. 15 | 16 | ## Configuration 17 | There are a vast number of server configuration parameters 18 | for AllegroGraph, far more than you would want to express as 19 | arguments to a configuration function. 20 | 21 | 22 | 23 | In this directory there is a file `agraph.cfg-template` that 24 | you should edit to add or modify the 25 | configuration options ([Link Here](https://franz.com/agraph/support/documentation/current/daemon-config.html)) 26 | you wish to set. 27 | 28 | The only options you should *not* specify in `agraph.cfg-template` are 29 | ``` 30 | Port 31 | SSLPort 32 | ``` 33 | as these will be added to the final `agraph.cfg` file based on 34 | values you put in `vars.yaml`. 35 | 36 | ## basedir 37 | 38 | One important variable in `vars.yaml` is `basedir`. The server 39 | will be installed in a newly created directory that is the 40 | value of `basedir`. 41 | Also a 42 | ``` 43 | BaseDir 44 | ``` 45 | directive will be put in the agraph.cfg specifying this value. 46 | This means that inside agraph.cfg you can (and should) 47 | use relative pathanmes to refer to directories and files inside 48 | this directory tree. 49 | 50 | ## settings 51 | This line is always in the agraph.cfg 52 | ``` 53 | SettingsDirectory settings 54 | ``` 55 | It places the settings directory as a subdirectory 56 | of the `basedir`. 57 | Do not change this line as the settings directory has to be 58 | here in order for the super user password to be installed correctly. 59 | 60 | 61 | ## Installation 62 | Before starting the installation edit the following files: 63 | 64 | ### inventory.txt 65 | Insert the name of the machines on which you want the AllegroGraph server to be installed. 66 | Replace the sample machine names already in the file 67 | with the names of your machines. 68 | After you edit the `inventory.txt` file you can type 69 | ``` 70 | % make 71 | ``` 72 | to see if the machines you specified are reachable by ansible. 73 | 74 | 75 | ### vars.yaml 76 | 77 | That file contains descriptions of the variables to be set as well 78 | as some sample values that you'll need to change. 79 | 80 | 81 | ### agraph.cfg-template 82 | 83 | This is the file that will be modified to create the agraph.cfg that 84 | will be installed with AllegroGraph. 85 | You should review the [server settings](https://franz.com/agraph/support/documentation/current/daemon-config.html) document to see which additional configuration 86 | parameters you wish to specify. 87 | 88 | ### make install 89 | The command 90 | ``` 91 | % make install 92 | ``` 93 | will run though the installation steps to install the server. 94 | It will do a superseding install meaning it will overwrite the server 95 | executables but it will not remove any repositories. However it is best 96 | to backup your installion before doing the install in case something 97 | unexpected happens and repos are lost. 98 | 99 | ### make clean-install 100 | If you wish to completely remove an installed AllegroGraph server so 101 | that `make install` gives you a totally fresh directory then 102 | ``` 103 | % make clean-install 104 | ``` 105 | will delete everything including repos that are found in subdirectories 106 | of the installation. 107 | 108 | ### make start 109 | To start the server on all machines do 110 | ``` 111 | % make start 112 | ``` 113 | 114 | ### make stop 115 | To stop the server on all machines do 116 | ``` 117 | % make stop 118 | ``` 119 | 120 | If you are going to `make install` be sure to stop all servers before doing so. 121 | 122 | 123 | 124 | ## make agraph 125 | 126 | The command 127 | ``` 128 | % make agraph 129 | ``` 130 | will create the shell script `agraph` that you copy to `/etc/rc.d/init.d` 131 | so you can automatically start and stop the agraph server on machine start up and 132 | shut down. 133 | 134 | Run the command as root 135 | ``` 136 | # chkconfig --help 137 | ``` 138 | to see how you can cause the `/etc/rc.d/init.d/agraph` script to run 139 | at certain run levels. 140 | 141 | 142 | -------------------------------------------------------------------------------- /ansible/agraph.cfg-template: -------------------------------------------------------------------------------- 1 | # AllegroGraph configuration file 2 | BaseDir %BASEDIR% 3 | 4 | ## these will be defined at the bottom of this file if needed 5 | #Port 6 | #SSLPort 7 | SettingsDirectory settings 8 | LogDir log 9 | PidFile data/agraph.pid 10 | InstanceTimeout 604800 11 | 12 | 13 | Main data/rootcatalog 14 | 15 | 16 | 17 | Main data/systemcatalog 18 | InstanceTimeout 10 19 | 20 | -------------------------------------------------------------------------------- /ansible/agraph.init: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # agraph This shell script takes care of starting and stopping 4 | # AllegroGraph (agraph) 5 | # 6 | # chkconfig: - 90 10 7 | # description: AllegroGraph database server. 8 | # processname: agraph 9 | 10 | if [ -f /etc/rc.d/init.d/functions ] 11 | then 12 | # Red Hat (alike) system 13 | 14 | # Source function library. 15 | . /etc/rc.d/init.d/functions 16 | 17 | # Source networking configuration. 18 | . /etc/sysconfig/network 19 | else 20 | action () { 21 | local message rc 22 | message=$1 23 | echo -n "$message" 24 | shift 25 | if $* 26 | then 27 | echo " succeeded" 28 | else 29 | echo " failed" 30 | fi 31 | } 32 | fi 33 | 34 | basedir=%BASEDIR% 35 | 36 | CONFIGFILE=$basedir/lib/agraph.cfg 37 | agraph_control=$basedir/bin/agraph-control 38 | 39 | if [ -f /etc/sysconfig/agraph ]; then 40 | . /etc/sysconfig/agraph 41 | fi 42 | 43 | if [ ! -f $CONFIGFILE ]; then 44 | cat < agraph; chmod 755 agraph" 12 | chdir: "." 13 | warn: false 14 | -------------------------------------------------------------------------------- /ansible/ansible.md: -------------------------------------------------------------------------------- 1 | # Using Ansible with AllegroGraph 2 | 3 | [Ansible](http://ansible.com) is an open source automatation tool 4 | that can be used to install and control [AllegroGraph](http://allegrograph.com) servers. 5 | 6 | We have an example of the use of Ansible in this directory. 7 | Consider it as a starting point for creating your own 8 | Ansible playbooks. 9 | 10 | The Makefile contains rules to **install**, **start** and **stop** 11 | AllegroGraph servers. You will need to edit `vars.yaml` 12 | and `inventory.yaml` for your specific site in order for 13 | the Makefile rules to work. 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /ansible/clean-install.yaml: -------------------------------------------------------------------------------- 1 | # remove the AllegroGraph installation 2 | # 3 | - name: remove agraph installation 4 | hosts: agservers 5 | vars_files: 6 | - ./vars.yaml 7 | tasks: 8 | - name: remove the agraph installation 9 | ansible.builtin.file: 10 | path: "{{basedir}}" 11 | state: absent 12 | -------------------------------------------------------------------------------- /ansible/install.yaml: -------------------------------------------------------------------------------- 1 | # install agraph on the agservers hosts 2 | # 3 | # see vars.yaml for the variables you need to specify 4 | # 5 | - name: install agraph 6 | hosts: agservers 7 | vars_files: 8 | - ./vars.yaml 9 | tasks: 10 | # we will extract tar.gz file containing AllegroGraph into {{tmpdir}}/{{version}} 11 | # and install later into {{basedir}}. 12 | 13 | - name: make sure the temp dir exists 14 | ansible.builtin.file: 15 | path: "{{tmpdir}}" 16 | state: directory 17 | 18 | - name: clean the tar file extract directory 19 | ansible.builtin.file: 20 | path: "{{tmpdir}}/{{version}}" 21 | state: absent 22 | 23 | # this will create {{tmpdir}}/{{version}} which will contain 24 | # the files needed to install AllegroGraph 25 | - name: extract new tar file 26 | ansible.builtin.unarchive: 27 | dest: "{{tmpdir}}" 28 | src: "{{tarfile}}" 29 | 30 | 31 | - name: copy in the script to build agraph.cfg 32 | ansible.builtin.copy: 33 | dest: "{{tmpdir}}/{{version}}" 34 | src: make-agraph.cfg 35 | mode: 0755 36 | 37 | # copy in the agraph.cfg template 38 | # 39 | - name: copy in template for agraph.cfg 40 | ansible.builtin.copy: 41 | dest: "{{tmpdir}}/{{version}}" 42 | src: agraph.cfg-template 43 | 44 | # it is ok to omit a value for license: in vars.yaml 45 | - name: copy in license for later install if given 46 | ansible.builtin.copy: 47 | dest: "{{tmpdir}}/{{version}}/license" 48 | src: "{{licensefile}}" 49 | mode: 0400 50 | when: licensefile | default('',true) | trim != '' 51 | 52 | # copy ssl certificate if it sslcertificate: has 53 | # a value in vars.yaml. 54 | # You only need an ssl certificate specified if you 55 | # specify an sslport and if you haven't modified 56 | # agraph.cfg-template to specify the location of 57 | # the ssl certificate 58 | - name: copy ssl certificate 59 | ansible.builtin.copy: 60 | dest: "{{tmpdir}}/{{version}}" 61 | src: "{{sslcertificate}}" 62 | mode: 0400 63 | when: sslcertificate | default('',true) | trim != '' 64 | 65 | 66 | # create an agraph.cfg 67 | # and build the settings/user file for the super user account 68 | - name: make agraph.cfg 69 | ansible.builtin.shell: 70 | chdir: "{{tmpdir}}/{{version}}" 71 | cmd: "./make-agraph.cfg '{{basedir}}' '{{agport}}' '{{sslport}}' '{{sslcertificate}}' '{{superuser}}' '{{superpassword}}'" 72 | 73 | # install without superseding into {{basedir}} 74 | - name: install new version 75 | ansible.builtin.shell: 76 | chdir: "{{tmpdir}}/{{version}}" 77 | cmd: "./install-agraph {{basedir}} --no-configure --supersede" 78 | 79 | # If a license file is given in vars.yaml add that license 80 | # to agraph.cfg 81 | - name: add my license 82 | ansible.builtin.shell: 83 | chdir: "{{tmpdir}}/{{version}}" 84 | cmd: "cat license >> {{basedir}}/lib/agraph.cfg" 85 | when: licensefile | default('',true) | trim != '' 86 | 87 | # install the super user account 88 | - name: add super user account 89 | ansible.builtin.shell: 90 | chdir: "{{tmpdir}}/{{version}}" 91 | cmd: "cp -rp settings {{basedir}}" 92 | 93 | # remove the installation directory as we no longer need it. 94 | - name: cleanup installation directory 95 | ansible.builtin.file: 96 | path: "{{tmpdir}}/{{version}}" 97 | state: absent 98 | -------------------------------------------------------------------------------- /ansible/inventory.txt: -------------------------------------------------------------------------------- 1 | [agservers] 2 | virgil 3 | homer 4 | -------------------------------------------------------------------------------- /ansible/make-agraph.cfg: -------------------------------------------------------------------------------- 1 | # 2 | # script to convert agraph.cfg-template into agraph.cfg based on variable 3 | # settings in vars.yaml. 4 | # 5 | # Also create the password entry for the superuser 6 | # 7 | # 8 | # make-agraph.cfg basedir port sslport sslcertificate superuser superpassword 9 | # 10 | # basedir is required 11 | # port or sslport are required (and it's ok to specify both) 12 | # If sslport is given then you'll want to either specify sslcertificate 13 | # or put an 14 | # SSLCertificate 15 | # line in agraph.cfg-template 16 | # 17 | 18 | if [ $# -ne 6 ]; then 19 | echo use: make-agraph.cfg basedir port sslport sslcertificate superuser superpassword 20 | exit 1 21 | fi 22 | 23 | basedir=$1 24 | port=$2 25 | sslport=$3 26 | sslcertificate=$4 27 | superuser=$5 28 | superpassword=$6 29 | 30 | if [ z$basedir = z ]; then 31 | echo A non-empty basedir argument must be supplied 32 | fi 33 | 34 | sed -e s,%BASEDIR%,$basedir, < agraph.cfg-template > agraph.cfg 35 | 36 | if [ z$port != z ]; then 37 | echo Port $port >> agraph.cfg 38 | fi 39 | 40 | if [ z$sslport != z ]; then 41 | if [ z$sslcertificate = z ]; then 42 | echo You've specified an sslport but haven't specified an sslcertificate. 43 | echo A ssl certificate is required to start an ssl server. 44 | echo Please be sure to specify a SSLCertificate in 45 | echo the agraph.cfg-template file 46 | else 47 | echo SSLCertificate lib/${sslcertificate##*/} >> agraph.cfg 48 | fi 49 | echo SSLPort $sslport >> agraph.cfg 50 | fi 51 | 52 | ## setup the super user 53 | mkdir -p settings/user 54 | hashed=`echo -n agraph:$superuser:$superpassword | sha1sum | sed 's/\(\S*\).*/\1/'` 55 | echo "(#x$hashed (:super) nil nil nil)" > settings/user/$superuser 56 | 57 | 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /ansible/start.yaml: -------------------------------------------------------------------------------- 1 | # start agraph on the agservers hosts 2 | # 3 | - name: start agraph 4 | hosts: agservers 5 | vars_files: 6 | - ./vars.yaml 7 | tasks: 8 | - name: start agraph 9 | ansible.builtin.shell: "{{basedir}}/bin/agraph-control start" 10 | -------------------------------------------------------------------------------- /ansible/stop.yaml: -------------------------------------------------------------------------------- 1 | # stop agraph on the agservers hosts 2 | # 3 | - name: stop agraph 4 | hosts: agservers 5 | vars_files: 6 | - ./vars.yaml 7 | tasks: 8 | - name: stop agraph 9 | ansible.builtin.shell: "{{basedir}}/bin/agraph-control stop" 10 | -------------------------------------------------------------------------------- /ansible/vars.yaml: -------------------------------------------------------------------------------- 1 | # Variables used to install and configure agraph servers on machines 2 | # listed in the inventory.txt file 3 | # 4 | # We extract the tar file containing an agraph server installation 5 | # into {{tmpdir}}/{{version}}. 6 | # We then build an agraph.cfg file and superuser password entry 7 | # 8 | # Then we install agraph to the directory {{basedir}} 9 | # 10 | 11 | # The tmpdir will just be used to store the installation files 12 | # which will be removed after the install. 13 | # We remove {{tmpdir}}/{{version}} before and after the install 14 | tmpdir: /w/tmp 15 | 16 | # The agraph server and utility files are installed at {{basedir}} 17 | # which is removed before the install. This will remove all 18 | # existing repositories in this directory! 19 | basedir: /w/agraph-7.3.0-test 20 | 21 | # The file containing the agraph server installation. 22 | # This file need only be present on the ansible controlling machine 23 | tarfile: ~/dists/agraph/ag7.3.0/linuxamd64.64/agraph-7.3.0-linuxamd64.64.tar.gz 24 | 25 | # the version must match the tar file. Since the the tar file is 26 | # for version 7.3.0 the version string must be exactly this: 27 | version: agraph-7.3.0 28 | 29 | # 30 | # The agraph license to use 31 | # This file need only be present on the ansible controlling machine 32 | # If you don't want to supply a license or have put the license 33 | # in agraph.cfg-template then don't put any value after licensefile: 34 | licensefile: ~/license.txt 35 | 36 | # set these to the values for the http and https ports 37 | # you must set at least one of them. 38 | agport: 10998 39 | sslport: 12001 40 | 41 | # if you've set an sslport then you can specify the location of the ssl certificate 42 | # file here or put the information in the agraph.cfg-template file 43 | sslcertificate: server.pem 44 | 45 | # you must set the super user name and password here 46 | superuser: test 47 | superpassword: xyzzy 48 | 49 | -------------------------------------------------------------------------------- /clustering/README.md: -------------------------------------------------------------------------------- 1 | # Using AllegroGraph clustering 2 | 3 | These files contain examples of building clustered databases using AllegroGraph. 4 | 5 | ## aws-repl 6 | 7 | These are support files for setting up a multi-master replication cluster on AWS. It's best used in conjunction with terraform-elb. 8 | 9 | ## terraform-elb 10 | 11 | A terraform script for creating a Multi-Master replication cluster with a load balancer distributing work to the instances. 12 | 13 | ## kubernetes/mmr 14 | 15 | Examples of using AllegroGraph Multi-Master Replication with [Kubernetes](https://kubernetes.io) and [Docker Swarm](https://docs.docker.com/engine/swarm/) 16 | 17 | ## misc 18 | 19 | Generic clustering-related examples (load balancing with NGINX, etc). 20 | -------------------------------------------------------------------------------- /clustering/aws-repl/Makefile: -------------------------------------------------------------------------------- 1 | dist: 2 | tar cf aws-repl.tar -C .. aws-repl/create.sh aws-repl/install.sh aws-repl/join.sh aws-repl/start.sh aws-repl/vars.sh aws-repl/joincluster 3 | -------------------------------------------------------------------------------- /clustering/aws-repl/create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # create the initial cluster repo for the controlling instance 4 | # 5 | . ./vars.sh 6 | 7 | curl -X PUT -u $authuser:$authpassword "http://127.0.0.1:$port/repositories/$reponame/repl/createCluster?host=$myip&port=$port&user=$authuser&password=$authpassword&instanceName=controlling" 8 | 9 | exit 0 10 | -------------------------------------------------------------------------------- /clustering/aws-repl/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x 3 | cp joincluster /etc/rc.d/init.d 4 | chkconfig --add joincluster 5 | -------------------------------------------------------------------------------- /clustering/aws-repl/join.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # join the cluster 3 | # This assumes the cwd is the directory holding this file. 4 | # 5 | # This is invoked by the joincluster script in /etc/rc/init.d 6 | # 7 | set -x 8 | . ./vars.sh 9 | # 10 | # 11 | 12 | echo ip is $myip 13 | 14 | controllingspec=$authuser:$authpassword@$controlling:$port/$reponame 15 | 16 | if [ $myip == "$controlling" ] ; then ./create.sh ; exit 0; fi 17 | 18 | # wait for the controlling ag server to be running 19 | until curl -s http://$authuser:$authpassword@$controlling:$port/version ; do echo wait for controlling ; sleep 5; done 20 | 21 | # wait for server on this machine to be running 22 | until curl -s http://$authuser:$authpassword@127.0.0.1:$port/version ; do echo wait for local server ; sleep 5; done 23 | 24 | # wait for cluster repo on the controlling instance to be present 25 | until $agtool repl status $controllingspec > /dev/null ; do echo wait for repo ; sleep 5; done 26 | 27 | 28 | myiname=i-$myip 29 | echo $myiname > instance-name.txt 30 | 31 | # construct the remove-instance.sh shell script to remove this instance 32 | # from the cluster when the instance is terminated. 33 | echo $agtool repl remove $controllingspec $myiname > remove-instance.sh 34 | chmod 755 remove-instance.sh 35 | # 36 | # join the cluster 37 | $agtool repl grow-cluster $controllingspec $authuser:$authpassword@$myip:$port/$reponame $myiname 38 | -------------------------------------------------------------------------------- /clustering/aws-repl/joincluster: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # joincluster joins and leaves the cluster 4 | # 5 | # chkconfig: 345 95 4 6 | # description: Start an agraph server and then joins the cluster 7 | # 8 | 9 | ## 10 | 11 | lockfile=/var/lock/subsys/joincluster 12 | awsrepl=/home/ec2-user/aws-repl 13 | 14 | case "$1" in 15 | start) 16 | cd $awsrepl 17 | su ec2-user -c "(./start.sh; sleep 10; ./join.sh)" > /var/tmp/join.out 2>&1 18 | touch $lockfile 19 | echo cluster joined 20 | exit 0 21 | ;; 22 | stop) 23 | cd $awsrepl 24 | su ec2-user -c ./remove-instance.sh 25 | rm -f $lockfile 26 | echo exiting cluster 27 | exit 0 28 | ;; 29 | *) 30 | echo unknown command 31 | exit 1 32 | esac 33 | 34 | 35 | -------------------------------------------------------------------------------- /clustering/aws-repl/start.sh: -------------------------------------------------------------------------------- 1 | . ./vars.sh 2 | $agraphroot/bin/agraph-control --config $agraphroot/lib/agraph.cfg start 3 | -------------------------------------------------------------------------------- /clustering/aws-repl/vars.sh: -------------------------------------------------------------------------------- 1 | # constants need by scripts 2 | controlling=10.0.0.10 3 | port=10035 4 | authuser=test 5 | authpassword=xyzzy 6 | reponame=myrepl 7 | # 8 | agraphroot=/home/ec2-user/agraph 9 | agtool=$agraphroot/bin/agtool 10 | # compute our ip address 11 | myip=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4) 12 | 13 | -------------------------------------------------------------------------------- /clustering/kubernetes/mmr/agmmr/Dockerfile: -------------------------------------------------------------------------------- 1 | 2 | # NOTE: this is the official Rocky Linux image 3 | FROM rockylinux/rockylinux:8 4 | 5 | # 6 | # agraph root is /app/agraph 7 | # rootcatalog at /app/rootcatalog 8 | # 9 | 10 | # ensure we have a good set of network programs installed 11 | RUN dnf -y update 12 | RUN dnf -y install net-tools iputils bind-utils wget hostname openssl 13 | 14 | ARG agversion=8.4.0 15 | ARG testrelease= 16 | 17 | ENV agdistfile=agraph-${agversion}-linuxamd64.64.tar.gz 18 | ENV agversion=${agversion} 19 | 20 | ## Show the curl command we're about to execute 21 | ## This useful for debugging curl problems due to invalid arguments 22 | RUN echo will do: curl -f -o ${agdistfile} http://franz.com/ftp/pri/acl/ag/ag${agversion}${testrelease}/linuxamd64.64/${agdistfile} 23 | 24 | RUN curl -f -o ${agdistfile} http://franz.com/ftp/pri/acl/ag/ag${agversion}${testrelease}/linuxamd64.64/${agdistfile} 25 | 26 | ## extract the AllegroGraph distribution 27 | RUN tar xfz ${agdistfile} 28 | RUN rm ${agdistfile} 29 | 30 | # so prompts are readable in an emacs window if we're debugging 31 | # this interactively 32 | ENV PROMPT_COMMAND= 33 | 34 | # agraph will be the user running agraph 35 | RUN groupadd agraph && useradd -d /home/agraph -g agraph agraph 36 | RUN mkdir /app /app/rootcatalog /app/misc /app/secrets 37 | 38 | # copy in items we'll need when the docker image runs 39 | COPY . /app/misc 40 | 41 | 42 | # we will attach persistent storage to this directory 43 | # in the controlling instance only. 44 | VOLUME ["/app/rootcatalog"] 45 | 46 | 47 | ENTRYPOINT ["/app/misc/runrepl.sh"] 48 | -------------------------------------------------------------------------------- /clustering/kubernetes/mmr/agmmr/Makefile: -------------------------------------------------------------------------------- 1 | # Two variables you may wish to specify on the make command line. 2 | # While it's unlikely that you'll specify testrelease 3 | # note that the value has the form: .t2 (the period is important). 4 | # 5 | # e.g. make agversion=6.5.0 build 6 | # 7 | dockeraccount=unspecified 8 | agversion=8.3.1 9 | testrelease= 10 | tag=8.3.1 11 | 12 | build: 13 | docker build --build-arg agversion=$(agversion) --build-arg testrelease=$(testrelease) -t $(dockeraccount)/agmmr:$(tag) . 14 | 15 | push: 16 | docker image push $(dockeraccount)/agmmr:$(tag) 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /clustering/kubernetes/mmr/agmmr/agraph.cfg.in: -------------------------------------------------------------------------------- 1 | # AllegroGraph configuration file 2 | Port Config_Port 3 | BaseDir /app/agraph 4 | SettingsDirectory data/settings 5 | LogDir log 6 | PidFile data/agraph.pid 7 | InstanceTimeout 604800 8 | RunAs agraph 9 | 10 | 11 | Main /app/rootcatalog 12 | 13 | 14 | 15 | Main data/systemcatalog 16 | InstanceTimeout 10 17 | 18 | -------------------------------------------------------------------------------- /clustering/kubernetes/mmr/agmmr/dnspatch.cl: -------------------------------------------------------------------------------- 1 | (in-package :acl-socket) 2 | ;; set the stale dns time to near zero so that 3 | ;; when pods are changed to different IP addreses we know about this 4 | (setq *stale-entry-remove-time* 5) ; 5 seconds 5 | 6 | -------------------------------------------------------------------------------- /clustering/kubernetes/mmr/agmmr/runrepl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | ## This starts in a container where the AllegroGraph installation 4 | ## files are present. 5 | ## The agraph server is installed and then either the controlling 6 | ## instance of an MMR cluster is created or the MMR cluster 7 | ## is grown by adding an instance on this server. 8 | ## 9 | ## 10 | 11 | echo secrets 12 | ls /app/secrets 13 | 14 | secretsroot=/app/secrets 15 | 16 | port=$(cat $secretsroot/port) 17 | reponame=$(cat $secretsroot/reponame) 18 | authuser=$(cat $secretsroot/user) 19 | authpassword=$(cat $secretsroot/password) 20 | 21 | myip=$(hostname -I | sed -e 's/ .*$//') 22 | 23 | ## put useful information into the console log 24 | ## to diagnose startup problems 25 | echo port is $port 26 | echo reponame is $reponame 27 | echo authuser is $authuser 28 | echo authpassword is not shown 29 | 30 | echo myip is $myip 31 | 32 | ## install the agraph server 33 | (cd agraph-${agversion} ; ./install-agraph /app/agraph -- --non-interactive \ 34 | --runas-user agraph \ 35 | --super-user $authuser \ 36 | --super-password $authpassword ) 37 | 38 | ## insert dns patch so we cache dns entries for a very short time 39 | ## so that if the controlling instance pod died and is restarted we 40 | ## learn about the new IP address of the controlling instance quickly. 41 | cp /app/misc/dnspatch.cl /app/agraph/lib/patches/dnspatch.cl 42 | 43 | # personalize the agraph.cfg file with the port selected by 44 | # the user found in the secrets file. Also add the agraph license 45 | # to the agraph.cfg file. 46 | sed -e s/Config_Port/$port/ /app/misc/agraph.cfg.in > /app/agraph/lib/agraph.cfg 47 | cat /app/secrets/license >> /app/agraph/lib/agraph.cfg 48 | 49 | 50 | chown -R agraph.agraph /app/agraph /app/rootcatalog 51 | 52 | 53 | agtool=/app/agraph/bin/agtool 54 | 55 | 56 | ## start agraph 57 | /app/agraph/bin/agraph-control --config /app/agraph/lib/agraph.cfg start 58 | 59 | term_handler() { 60 | # this signal is delivered when the pod is 61 | # about to be killed. We remove ourselves 62 | # from the cluster. 63 | echo got term signal 64 | /bin/bash ./remove-instance.sh 65 | exit 66 | } 67 | 68 | sleepforever() { 69 | # This unusual way of sleeping allows 70 | # a TERM signal sent when the pod is to 71 | # die to then cause the shell to invoke 72 | # the term_handler function above. 73 | date 74 | while true 75 | do 76 | sleep 99999 & wait ${!} 77 | done 78 | } 79 | 80 | if [ -e /app/agraph/data/rootcatalog/$reponame ] ; then 81 | echo repository $reponame already exists in this persistent volume 82 | sleepforever 83 | fi 84 | 85 | ## the DNS name of the controlling host is specified in the yaml file 86 | ## starting this container 87 | controllinghost=${ControllingHost} 88 | 89 | controllingspec=$authuser:$authpassword@$controllinghost:$port/$reponame 90 | 91 | if [ x$Controlling == "xyes" ] ; 92 | then 93 | # It may take a little time for the DNS record for $controllinghost to be present 94 | # and we need that DNS lookup to work because the agtool program below will use it 95 | until host $controllinghost ; do echo $controllinghost not in DNS yet; sleep 5 ; done 96 | 97 | ## create the first cluster instance which is the controlling instance 98 | $agtool repl create-cluster $controllingspec controlling 99 | 100 | 101 | else 102 | # wait for the controlling ag server to be running 103 | until curl -s http://$authuser:$authpassword@$controllinghost:$port/version ; do echo wait for controlling ; sleep 5; done 104 | echo 105 | 106 | # wait for server in this container to be running 107 | until curl -s http://$authuser:$authpassword@$myip:$port/version ; do echo wait for local server ; sleep 5; done 108 | echo 109 | 110 | # wait for cluster repo on the controlling instance to be present 111 | until $agtool repl status $controllingspec > /dev/null ; do echo wait for repo ; sleep 5; done 112 | echo 113 | 114 | ## give the instance to be created a unique name within the cluster 115 | myiname=i-$myip 116 | echo $myiname > instance-name.txt 117 | 118 | # construct the remove-instance.sh shell script to remove this instance 119 | # from the cluster when the instance is terminated. 120 | echo $agtool repl remove $controllingspec $myiname > remove-instance.sh 121 | chmod 755 remove-instance.sh 122 | # 123 | 124 | # note that 125 | # % docker kill container 126 | # will send a SIGKILL signal by default. we can't trap on SIGKILL. 127 | # so use 128 | # % docker kill -s TERM container 129 | # in order to test the container's reaction to being commanded to shut down. 130 | 131 | trap term_handler SIGTERM SIGHUP SIGUSR1 132 | trap -p 133 | echo this pid is $$ 134 | 135 | # join the cluster 136 | echo joining the cluster 137 | $agtool repl grow-cluster $controllingspec $authuser:$authpassword@$myip:$port/$reponame $myiname 138 | 139 | fi 140 | sleepforever 141 | -------------------------------------------------------------------------------- /clustering/kubernetes/mmr/helm/Makefile: -------------------------------------------------------------------------------- 1 | # running agraph mmr in kubernetes using helm 2 | # 3 | # type 4 | # % make 5 | # to see the steps you must perform to install the cluster 6 | # 7 | 8 | name ?= --generate-name 9 | 10 | info: 11 | @echo 12 | @echo You can install the mmr cluster in kubernetes 13 | @echo using a generated release name 14 | @echo % make install 15 | @echo 16 | @echo or if you wish to choose a name for the cluster: 17 | @echo % make name=foo install 18 | @echo 19 | @echo 'make install' installs AllegroGraph with no license 20 | @echo so its capabilities are limited. 21 | @echo If you have a license in file ~/license.txt then 22 | @echo use make target install-with-license 23 | @echo for example 24 | @echo % make file=~/license.txt name=foo install-with-license 25 | @echo 26 | 27 | 28 | install: 29 | helm install $(name) ./agraphmmr 30 | 31 | install-with-license: 32 | helm install --set-file license=$(file) $(name) ./agraphmmr 33 | 34 | 35 | ## make a Helm repository with the agraphmmr chart in it 36 | repository: 37 | helm package ./agraphmmr 38 | mkdir -p repo 39 | mv agraphmmr-*.tgz repo 40 | helm repo index repo 41 | -------------------------------------------------------------------------------- /clustering/kubernetes/mmr/helm/agraphmmr/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | .vscode/ 23 | .#* 24 | -------------------------------------------------------------------------------- /clustering/kubernetes/mmr/helm/agraphmmr/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: agraphmmr 3 | description: An AllegroGraph MMR cluster Helm chart for Kubernetes 4 | 5 | type: application 6 | home: https://github.com/franzinc/agraph-examples/tree/master/clustering/kubernetes/mmr 7 | 8 | # This is the chart version. This version number should be incremented each time you make changes 9 | # to the chart and its templates, including the app version. 10 | version: 7.0.0 11 | 12 | # This is the version number of the application being deployed. This version number should be 13 | # incremented each time you make changes to the application. 14 | appVersion: 7.0.0 15 | 16 | icon: https://franz.com/images/logo.png 17 | 18 | maintainers: 19 | - name: "Franz Inc" 20 | email: "support@franz.com" 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /clustering/kubernetes/mmr/helm/agraphmmr/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | {{ .Chart.Name }} - Helm chart for running an AllegroGraph cluster with Multi-Master Replication 2 | 3 | The full details on this chart can be found at 4 | 5 | https://github.com/franzinc/agraph-examples/blob/master/clustering/kubernetes/mmr/kubernetes-mmr.md 6 | 7 | That contains a description of how the chart works as well as how to modify what it does. 8 | 9 | 10 | 11 | Once the chart is installed you can access the controlling instance via a load balancer 12 | 13 | Run 14 | 15 | % kubectl get svc 16 | 17 | To find the address of the controlling-loadbalancer. 18 | 19 | It will take a few minutes for the external IP address to be allocated and 20 | show by this kubectl command. 21 | 22 | For example: 23 | 24 | NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE 25 | sample-controlling-loadbalancer LoadBalancer 10.12.8.217 104.155.148.58 10035:32240/TCP 113s 26 | 27 | Given the above you can access the controlling instance through the load balancer 28 | at: 29 | 30 | http://104.155.148.58:10035 31 | 32 | The default username is "test" and the default password is "xyzzy" 33 | 34 | You can do read and write access of the MMR cluster through the controlling-loadbalancer. 35 | 36 | There is also a copy-loadbalancer. This distributes work to all the copy nodes. 37 | The load balancer does not yet have session support so you should only 38 | use the copy-loadbalancer for queries to the repository. 39 | 40 | We also have a created a copy-0-loadbalancer as a demonstration of how to 41 | communicate with a single specified copy instance. Since this load 42 | balancer is always talking to the same instance one can do read and write accesses through 43 | this load balancer. However the copy instances can be deleted by 44 | kubernetes if you reduce the number of replicas defined. 45 | 46 | One way to change the number of replicas is to edit values.yaml 47 | and specify the desired number of replicas and then 48 | 49 | % helm upgrade releasename chart 50 | 51 | where releasename and chart are the same as 52 | in the command you used to install the chart 53 | 54 | % helm install releasename chart 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /clustering/kubernetes/mmr/helm/agraphmmr/templates/controlling-lb.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # This will create a service with an external IP through which 3 | # one can access the controlling instance. 4 | # Since there's just one controlling instance we don't need to worry 5 | # about the fact that the load balancer has no support for sessions. 6 | # 7 | # The external IP will be chosen by the cloud provider 8 | # 9 | apiVersion: v1 10 | kind: Service 11 | metadata: 12 | name: {{ .Release.Name }}-controlling-loadbalancer 13 | spec: 14 | type: LoadBalancer 15 | ports: 16 | - port: {{ .Values.port }} 17 | targetPort: {{ .Values.port }} 18 | selector: 19 | app: {{ .Release.Name }}-controlling 20 | 21 | 22 | -------------------------------------------------------------------------------- /clustering/kubernetes/mmr/helm/agraphmmr/templates/controlling-service.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # this service exposes the internal IP address 3 | # of the controlling instance and makes it 4 | # available via dns at the 5 | # name 'releasename-controlling' 6 | # using the dns search list 7 | # 8 | apiVersion: v1 9 | kind: Service 10 | metadata: 11 | name: {{ .Release.Name }}-controlling 12 | spec: 13 | clusterIP: None 14 | selector: 15 | app: {{ .Release.Name }}-controlling 16 | ports: 17 | - name: http 18 | port: {{ .Values.port }} 19 | targetPort: {{ .Values.port }} 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /clustering/kubernetes/mmr/helm/agraphmmr/templates/controlling-ss.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # stateful set of controlling instance 3 | # 4 | 5 | apiVersion: apps/v1beta1 6 | kind: StatefulSet 7 | metadata: 8 | name: {{ .Release.Name }}-controlling 9 | spec: 10 | serviceName: {{ .Release.Name }}-controlling 11 | replicas: 1 12 | template: 13 | metadata: 14 | labels: 15 | app: {{ .Release.Name }}-controlling 16 | spec: 17 | containers: 18 | - name: {{ .Release.Name }}-controlling 19 | image: {{ .Values.image.repository }} 20 | imagePullPolicy: {{ .Values.image.pullPolicy }} 21 | livenessProbe: 22 | httpGet: 23 | path: /version 24 | port: {{ .Values.port }} 25 | initialDelaySeconds: 30 26 | volumeMounts: 27 | - name: shm 28 | mountPath: /dev/shm 29 | - name: {{ .Release.Name }}-data 30 | mountPath: /app/rootcatalog 31 | - name: secrets 32 | mountPath: /app/secrets 33 | readOnly: true 34 | env: 35 | - name: Controlling 36 | value: "yes" 37 | - name: ControllingHost 38 | value: "{{ .Release.Name }}-controlling" 39 | volumes: 40 | - name: shm 41 | emptyDir: 42 | medium: Memory 43 | - name: secrets 44 | secret: 45 | secretName: {{ .Release.Name }}-agraphsecrets 46 | volumeClaimTemplates: 47 | - metadata: 48 | name: {{ .Release.Name }}-data 49 | spec: 50 | resources: 51 | requests: 52 | storage: {{ .Values.dataStorage }} 53 | accessModes: 54 | - ReadWriteOnce 55 | 56 | -------------------------------------------------------------------------------- /clustering/kubernetes/mmr/helm/agraphmmr/templates/copy-0-lb.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # This will create a service with an external IP address through which 3 | # one can access the the copy-0 instance. 4 | # Since there's just one copy-0 instance we don't need to worry 5 | # about the fact that the load balancer has no support for sessions. 6 | # 7 | # The external IP will be chosen by the cloud provider 8 | # 9 | # This is just an example of how one specifies a particular instance 10 | # of a StatefulSet. We won't normally use this load balancer 11 | # 12 | apiVersion: v1 13 | kind: Service 14 | metadata: 15 | name: {{ .Release.Name }}-copy-0-loadbalancer 16 | spec: 17 | type: LoadBalancer 18 | ports: 19 | - port: {{ .Values.port }} 20 | targetPort: {{ .Values.port }} 21 | selector: 22 | app: {{ .Release.Name }}-copy 23 | statefulset.kubernetes.io/pod-name: {{ .Release.Name }}-copy-0 24 | 25 | 26 | -------------------------------------------------------------------------------- /clustering/kubernetes/mmr/helm/agraphmmr/templates/copy-lb.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # This will create a service with an external IP address 3 | # through which one can access a randomly chosen copy instance. 4 | # There is no support for sessions so don't create an agraph session 5 | # using the external IP address. This is mainly useful for 6 | # simple non-session transactions or for queries to one of the 7 | # copies of the repo 8 | # 9 | # The external IP will be chosen by the cloud provider 10 | # 11 | apiVersion: v1 12 | kind: Service 13 | metadata: 14 | name: {{ .Release.Name }}-copy-loadbalancer 15 | spec: 16 | type: LoadBalancer 17 | ports: 18 | - port: {{ .Values.port }} 19 | targetPort: {{ .Values.port }} 20 | selector: 21 | app: {{ .Release.Name }}-copy 22 | 23 | -------------------------------------------------------------------------------- /clustering/kubernetes/mmr/helm/agraphmmr/templates/copy-service.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # 3 | apiVersion: v1 4 | kind: Service 5 | metadata: 6 | name: {{ .Release.Name }}-copy 7 | spec: 8 | clusterIP: None 9 | selector: 10 | app: {{ .Release.Name }}-copy 11 | ports: 12 | - name: {{ .Release.Name }}-main 13 | port: {{ .Values.port }} 14 | targetPort: {{ .Values.port }} 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /clustering/kubernetes/mmr/helm/agraphmmr/templates/copy-ss.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # stateful set of copies of the copy instance 3 | # 4 | 5 | apiVersion: apps/v1beta1 6 | kind: StatefulSet 7 | metadata: 8 | name: {{ .Release.Name }}-copy 9 | spec: 10 | serviceName: {{ .Release.Name }}-copy 11 | replicas: {{ .Values.replicas }} 12 | template: 13 | metadata: 14 | labels: 15 | app: {{ .Release.Name }}-copy 16 | spec: 17 | volumes: 18 | - name: shm 19 | emptyDir: 20 | medium: Memory 21 | - name: secrets 22 | secret: 23 | secretName: {{ .Release.Name }}-agraphsecrets 24 | containers: 25 | - name: {{ .Release.Name }}-controlling 26 | image: {{ .Values.image.repository }} 27 | imagePullPolicy: {{ .Values.image.pullPolicy }} 28 | livenessProbe: 29 | httpGet: 30 | path: /version 31 | port: {{ .Values.port }} 32 | initialDelaySeconds: 30 33 | volumeMounts: 34 | - name: shm 35 | mountPath: /dev/shm 36 | - name: secrets 37 | mountPath: /app/secrets 38 | readOnly: true 39 | env: 40 | - name: ControllingHost 41 | value: "{{ .Release.Name }}-controlling" 42 | 43 | 44 | -------------------------------------------------------------------------------- /clustering/kubernetes/mmr/helm/agraphmmr/templates/secrets.yaml: -------------------------------------------------------------------------------- 1 | kind: Secret 2 | apiVersion: v1 3 | metadata: 4 | name: {{ .Release.Name }}-agraphsecrets 5 | stringData: 6 | user: "{{ .Values.user }}" 7 | password: "{{ .Values.password }}" 8 | port: "{{ .Values.port }}" 9 | reponame: "{{ .Values.reponame }}" 10 | license: {{ .Values.license | default "" | quote }} 11 | 12 | 13 | -------------------------------------------------------------------------------- /clustering/kubernetes/mmr/helm/agraphmmr/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for aghelmmmr 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | 5 | # useful values for pullPolicy 6 | # Always - use this during development of the repository 7 | # IfNotPresent - use this when repository is stable 8 | # 9 | 10 | # image.repository is a Docker image containing AllegroGraph and the scripts to set up replication 11 | # image.pullPolicy determines whether Docker will use the cached version of the Docker image 12 | # or download a new version. 13 | # Useful values for image.pullPolicy: 14 | # Always - use this during development of the Docker image 15 | # IfNotPresent - use this when Docker image is stable 16 | # 17 | image: 18 | repository: franzinc/agmmr:7.0.0 19 | pullPolicy: IfNotPresent 20 | 21 | # replicas is the number of AllegroGraph servers to start up that will join the cluster 22 | # and replicate the repository of the controlling instance. 23 | replicas: 2 24 | 25 | # dataStorage is the amount of persistent storage space to allocate for the controlling instance's 26 | # repository data. 27 | dataStorage: 20Gi 28 | 29 | 30 | # these values are put in a secrets file 31 | # user is the username of the admin account for the AllegroGraph server 32 | user: test 33 | # password is the password of the admin account. 34 | password: xyzzy 35 | # port is the HTTP port on which the AllegroGraph server will listen. 36 | port: 10035 37 | # reponame is the name of the repository in the root catalog on the controlling instance 38 | # that will be replicated in each of replicas 39 | reponame: myrepl 40 | # 41 | # to insert the AllegroGraph license here you must 42 | # use the yaml conventions for a multi-line value. 43 | # This means using the pipe character and indenting 44 | # each line, for example: 45 | # 46 | # license: | 47 | # Licensee Spacely Sprockets 48 | # LicenseLimit no-limit 49 | # LicenseExpires no-expiration 50 | # 51 | # 1244234134123412341234123412341234432234 52 | # 3434245251351345234523452345234523452344 53 | # 9087498234148239481239482394123982349123 54 | # 55 | # 56 | # license is the license to run the AllegroGraph server. AllegroGraph will run 57 | # without a license but will then have limited capabilities. 58 | license: 59 | -------------------------------------------------------------------------------- /clustering/misc/using-nginx-load-balancer.md: -------------------------------------------------------------------------------- 1 | # AllegroGraph Replication using NGINX Load Balancer 2 | 3 | [TL;DR: Working configuration example](#allegrograph-and-nginx-configuration-changes) 4 | 5 | 6 | ## Load balancing for AllegroGraph cluster 7 | 8 | AllegroGraph supports a distributed deployment scheme called 9 | [Multi-master Replication (MMR)][mmr], which is useful in cases when 10 | increased availability and data safety is required. As with other 11 | distributed deployment schemes, an MMR cluster consists of multiple AG 12 | server instances that are available for client access. Naturally, the 13 | access load can be distributed among these instances using a load 14 | balancer, but doing it correctly requires some understanding of the 15 | AllegroGraph architecture. 16 | 17 | From a high-level point of view, the problem with using AllegroGraph 18 | behind any kind of proxy is that AllegroGraph clients encourage 19 | stateful, session-based workflow. 20 | 21 | AllegroGraph sessions are dedicated connections from client agents 22 | (either Java, Python or Lisp client libraries) to separate processes 23 | within the running AG server. This approach improves the access 24 | performance by allocating dedicated resources for each client's 25 | operations, but the problem with session-based workflow is that 26 | sessions are inherently stateful: both client and server maintain 27 | information about a particular session used for interacting, and while 28 | the client simply supplies this information as part of every request, 29 | AG server actually allocates certain resources (e.g. separate process 30 | and port number), so this state cannot be moved or replicated across 31 | AG server instances. 32 | 33 | It is clear that in order to resolve this problem in general, the application 34 | using AllegroGraph client libraries must adopt a stateless approach to 35 | interacting with AG deployments. This may require significant changes 36 | and imposes certain restrictions on application side. For example, 37 | a Java application interacting with an AG server via [Java AllegroGraph 38 | client][agraph-java] should not use transactions (more specifically, 39 | `RepositoryConnection.begin()` method), since the Java AllegroGraph client 40 | implicitly requests a dedicated session whenever a transaction is 41 | started. 42 | 43 | In this document, we start with a two-node MMR cluster and a 44 | simplistic NGINX load balancer setup that doesn't work as expected, 45 | and develop a working solution, providing an explanation for each 46 | step, so that user is able to choose a different approach for 47 | bypassing the underlying problem. 48 | 49 | 50 | ## Problems with the simplistic load balancer setup 51 | 52 | Assume we have two AG servers organized into an MMR cluster. The 53 | instances are running on `back1:10035` and `back2:10035`. Now let's 54 | take a look at the following simplistic NGINX configuration: 55 | 56 | ``` 57 | upstream agraph-backends { 58 | server back1:10035; 59 | server back2:10035; 60 | } 61 | 62 | server { 63 | listen 80; 64 | server_name front; 65 | 66 | root /usr/share/nginx/html/; 67 | 68 | location / { 69 | proxy_pass http://agraph-backends; 70 | proxy_http_version 1.1; 71 | } 72 | } 73 | ``` 74 | 75 | Here we are creating a load balancer proxy listening on `front:80` and 76 | redirecting all requests to AG servers `back1:10035` and 77 | `back2:10035` in a round-robin manner. 78 | 79 | Whenever an AG client opens a connection to `front:80`, the instance 80 | that is selected by load balancer to respond will assume its host name 81 | is the one specified in the request's `Host` header and will respond 82 | with a dedicated session URL `http://agraph-backends:/`. In order to use the dedicated session, the 84 | client now must be able to open a direct connection at this location 85 | and perform all the operations over it. 86 | 87 | The first problem that arises is that the `agraph-backends` hostname 88 | cannot be resolved from the client. In order to fix this, AG servers 89 | can be configured to provide fixed hostnames in session-specific URLs, 90 | using the `SessionHost` configuration option: 91 | 92 | ``` 93 | # back1: agraph.cfg 94 | SessionHost back1 95 | 96 | # back2: agraph.cfg 97 | SessionHost back2 98 | ``` 99 | 100 | This will make AG servers provide correct URLs for sessions, but 101 | several other problems arise: 102 | - `back1` and `back2` may be internal hosts and it may be undesirable 103 | to allow connections to them; 104 | - even if `back1` and `back2` can be accessed from outside, under this 105 | setup only the session opening requests will go through the load 106 | balancer, and all consequent operations will bypass the load 107 | balancer altogether, so this will break the assumption about the load 108 | balancer being the entrypoint into the cluster; 109 | - (this applies only to the Java client) credentials specified for 110 | `front:80` will be limited in scope to the `front` hostname, and 111 | won't be used with `back1` and `back2` hosts, which will break 112 | session connections completely. 113 | 114 | 115 | ## How to make it work 116 | 117 | In order to resolve all these problems at once, we will force all 118 | sessions to go through the main port: 119 | 120 | ``` 121 | # agraph.cfg 122 | UseMainPortForSessions true 123 | 124 | # Note that SessionHost option must be removed. 125 | # SessionHost back1 126 | ``` 127 | 128 | We also need AG servers to think that their host name is the one of 129 | the load balancer, so we add this line to the `server.location` 130 | section in the NGINX configuration: 131 | 132 | ``` 133 | server { 134 | ... 135 | location / { 136 | ... 137 | proxy_set_header Host $host; 138 | } 139 | } 140 | ``` 141 | 142 | Now all session URLs will be of the form `http://front:10035/`. The problem is that the load balancer listens on 144 | `front:80`, so we need to change the port to be able to connect to 145 | the session-specific locations (in other words, both load balancer 146 | port and AG server ports must be the same) : 147 | 148 | ``` 149 | server { 150 | listen 10035; 151 | ... 152 | } 153 | ``` 154 | 155 | 156 | Finally, round-robin load balancing won't work in this case. When a 157 | session-opening requests hits one of the AG servers (say, 158 | `back1:10035`), the new session is created, but the other instance 159 | (`back2:10035`) doesn't know about it, so the next request, that will 160 | go to `back2:10035`, will just fail. In order to fix this, we can use 161 | session (HTTP session, not AG session in this case) persistence 162 | methods available in the NGINX Plus. Otherwise, we can use IP hash 163 | load balancing: all request from the same IP will go to the same 164 | instance, so once the AG session is open, all requests from the 165 | corresponding client will go the same host. 166 | 167 | 168 | ## AllegroGraph and NGINX configuration changes 169 | 170 | Here are all AllegroGraph and NGINX configuration changes required for 171 | a two-node MMR cluster behind an NIGNX load balancer: 172 | 173 | - AllegroGraph configuration change: add this line to each AG server's 174 | `agraph.cfg` file: 175 | ``` 176 | UseMainPortForSessions true 177 | ``` 178 | 179 | - NIGNX configuration: add file `agraph.conf` to the NGINX site 180 | configuration directory (usually `/etc/nginx/conf.d`, assuming the 181 | `nginx.conf` contains a `include /etc/nginx/conf.d/*.conf;` line): 182 | ``` 183 | upstream agraph-backends { 184 | ip_hash; 185 | server back1:10035; 186 | server back2:10035; 187 | } 188 | 189 | server { 190 | listen 10035; 191 | server_name front; 192 | 193 | root /usr/share/nginx/html/; 194 | 195 | location / { 196 | proxy_pass http://agraph-backends; 197 | proxy_http_version 1.1; 198 | proxy_set_header Host $host; 199 | } 200 | } 201 | ``` 202 | 203 | [mmr]: https://franz.com/agraph/support/documentation/current/multi-master.html 204 | [agraph-java]: https://github.com/franzinc/agraph-java-client 205 | -------------------------------------------------------------------------------- /clustering/terraform-elb/using-terraform.md: -------------------------------------------------------------------------------- 1 | # AllegroGraph Replication on AWS Using Terraform 2 | 3 | ## Introduction 4 | 5 | In this document we describe how to setup an [AllegroGraph replication cluster](https://franz.com/agraph/support/documentation/current/multi-master.html) on AWS using the [terraform](https://www.terraform.io/) program. The cluster will have one controlling instance and a set of instances controlled by an [Auto Scaling Group](https://docs.aws.amazon.com/autoscaling/) and reached via a [Load Balancer](https://docs.aws.amazon.com/elasticloadbalancing/). 6 | 7 | ![diagram](agraph_mmr_elb.svg) 8 | 9 | Creating such a system on AWS takes a long time if done manually 10 | through their web interface. We have another document that takes you 11 | through the steps. Describing the system in terraform first takes a 12 | little time but once that's done the cluster can be started in less than 13 | five minutes. 14 | 15 | ## Steps 16 | 17 | 1. Obtain an AMI with AllegroGraph and aws-repl (our support code for aws) installed. 18 | 2. Edit the terraform file we supply to suit your needs 19 | 3. Run terraform to build the cluster 20 | 21 | ## Obtain an AMI with AllegroGraph and aws-repl 22 | 23 | An AMI is an image of a virtual machine. You create an AMI by launching an ec2 instance using an AMI, altering the root disk of that instance and then telling AWS to create an AMI based on your instance. You can repeat this process until you create the AMI you need. 24 | 25 | We have a prebuild AMI with all the code installed. It uses AllegroGraph 6.5.0 and doesn't contain a license code so it's limited to 5 million triples. You can use this AMI to test the load balancer or you can use this image as the starting off point for building your own image. 26 | 27 | Alternatively you start from a fresh AMI and install everything yourself as described next. 28 | 29 | We will create an AMI to run AllegroGraph with Replication with the following features 30 | 31 | 1. When an EC2 instance running this AMI is started it starts AllegroGraph and joins the cluster of nodes serving a particular repository. 32 | 2. When the the EC2 instance is terminated the instance sends a message to the controlling instance to ensure that the terminating instance is removed from the cluster 33 | 3. If the EC2 instance is started at a particular IP address it creates the cluster and acts as the controlling instance of the cluster 34 | 35 | This is a very simple setup but will serve many applications. For more complex needs you'll need to write your own tools. Contact support@franz.com to discuss support options. 36 | 37 | The choice of AMI on which to build our AMI is not important except that our scripts assume that the initial account name of the image is ec2-user. Thus we suggest that you use one of the Amazon Linux images. If you use another kind of image you'll need to do extra work (as an example we describe below how to use a Centos AMI). Since the instance we'll build with the AMI are used only for AllegroGraph and not for other uses there's no point in running a different version of Linux that you may use in your development work. 38 | 39 | These are the steps to build an AMI: 40 | 41 | Start an instance using an Amazon Linux AMI with EBS support. 42 | 43 | We can't specify the exact name of the image to start as the names change over time and depending on the region. We will usually pick one of the first images listed. 44 | 45 | You don't need to start a large virtual machine. A t2.micro will do. 46 | 47 | You'll need to specify a VPC and subnet. There should be a default VPC available. If not you'll have to create one. 48 | 49 | Make sure that when you specify that subnet that you want to external IP address. 50 | 51 | Copy an agraph distribution (tar.gz format) to the ec2 instance into the home directory of ec2-user. Also copy the file aws-repl/aws-repl.tar to the home directory of ec2-user on the instance. 52 | aws-repl.tar contains scripts to support replication setup on AWS. 53 | 54 | Extract the agraph repo in a temporary spot and run install-agraph in it, specifying the root of the agraph distribution. 55 | 56 | I put it in /home/ec2-user/agraph 57 | 58 | For example: 59 | ``` 60 | % mkdir tmp 61 | % cd tmp 62 | % tar xfz ../agraph-6.5.0-linuxamd64.64.tar.gz 63 | % cd agraph-6.5.0 64 | % ./install-agraph ~/agraph 65 | ``` 66 | Edit the file ~/agraph/lib/agraph.cfg and add the line 67 | 68 | **UseMainPortForSessions yes** 69 | 70 | This will allow sessions to be tracked through the Load Balancer. 71 | 72 | If you have an agraph license key you should add it to the agraph.cfg file. 73 | 74 | Unpack and install the aws-repl code: 75 | ``` 76 | % tar xf aws-repl.tar 77 | % cd aws-repl 78 | % sudo ./install.sh 79 | ``` 80 | You can delete aws-repl.tar but don't delete the aws-repl directory. It will be used on startup. 81 | 82 | Look at aws-repl/var.sh to see the parameter values. You'll see an agraphroot parameter which should match where you installed agraph. 83 | 84 | At this point the instance is setup. 85 | 86 | You should go to the aws console, select this instance, and from the Action menu select "Image / Create Image". Wait for the AMI to be built. At this time you can terminate the ec2 instance. 87 | 88 | ### Using a CentOS 7 image: 89 | 90 | If you wish to install on top of CentOS then you'll need additional steps. The initial user on CentOS is called 'centos' rather than 'ec2-user'. In order to keep things consistent we'll create the ec2-user account and use that for running agraph just as we do for the Amazon AMI. 91 | 92 | ssh to the ec2 vm as centos and do the following to create the ec2-user account and to allow ssh access to it just like the centos account 93 | 94 | [centos@ip-10-0-1-227 ~]$ sudo sh 95 | ``` 96 | sh-4.2# adduser ec2-user 97 | sh-4.2# cp -rp .ssh ~ec2-user 98 | sh-4.2# chown -R ec2-user ~ec2-user/.ssh 99 | sh-4.2# exit 100 | [centos@ip-10-0-1-227 ~]$ 101 | ``` 102 | At this point you can copy the agraph distribution to the ec2 vm. Scp to ec2-user@x.x.x.x rather than centos@x.x.x.x. Also copy the aws-repl.tar file. 103 | 104 | The only change to the procedure is when you must run install.sh in the aws-repl directory. 105 | 106 | The ec2-user account does not have the ability to sudo. So this command must be run 107 | 108 | when logged in as the user centos; 109 | ``` 110 | centos@ip-10-0-1-227 ~]$ sudo sh 111 | sh-4.2# cd ~ec2-user/aws-repl 112 | sh-4.2# ./install.sh 113 | + cp joincluster /etc/rc.d/init.d 114 | + chkconfig --add joincluster 115 | sh-4.2# exit 116 | [centos@ip-10-0-1-227 ~]$ 117 | ``` 118 | ## Edit the terraform file we supply to suit your needs 119 | 120 | Edit the file agelb.tf. This file contains directives to terraform to create the cluster with load balancer. At the top are the variables you can easily change. Other values are found inside the directives and you can change those as well. 121 | 122 | Two variables you definitely need to change are 123 | 124 | 1. "**ag-elb-ami**" - this is the name of the AMI you created in the previous step or the AMI we supply. 125 | 2. "**ssh-key**" - this is the name of the ssh key pair you want to use in the instances created. 126 | 127 | You may wish to change the region where you want the instances built (that value is in the provider clause at the top of the file) and if you do you'll need to change the variable "azs". 128 | 129 | We suggest you try building the cluster with the minimum changes to verify it works and then customize it to your liking. 130 | 131 | ## Run terraform to build the cluster 132 | 133 | To build the cluster make sure your have an ~/.aws/config file with a default entry, such as 134 | ``` 135 | [default] 136 | aws_access_key_id = AKIAIXXXXXXXXXXXXXXX 137 | aws_secret_access_key = o/dyrxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 138 | ``` 139 | This is what terraform uses as credentials when it contacts AWS. 140 | 141 | In order to use terraform the first time (or any time you change the provider clause in agelb.tf) run this command 142 | ``` 143 | % terraform init 144 | ``` 145 | Terraform will download the files appropriate for the provider you specified. 146 | 147 | After that you can build your cluster with 148 | ``` 149 | % terraform apply 150 | ``` 151 | And watch the messages. If there are no errors terraform will wait for confirmation from you to proceed. Type yes to proceed, anything else to abort. 152 | 153 | After terraform is finished you'll see the address of the load balancer printed. 154 | 155 | You can make changes the agelb.tf file and again 'terraform apply ' and terraform will tell you what it needs to do to change things from how they are now to what the agelb.tf file specifies. 156 | 157 | To delete everything terraform added type the command 158 | ``` 159 | % terraform destroy 160 | ``` 161 | And type yes when prompted. 162 | -------------------------------------------------------------------------------- /data/bitcoin/README.md: -------------------------------------------------------------------------------- 1 | # Bitcoin RDF model 2 | 3 | 4 | ## Introduction 5 | This example demonstrates an [RDF] model for Bitcoin chain data as 6 | well as a Python tool to pull the data from a Bitcoin node into an 7 | instance of an AllegroGraph graph database. The model description itself 8 | can be found in the [Turtle] file [`model.ttl`](model.ttl). 9 | 10 | The following Turtle example demonstrates how this RDF model can be used to 11 | represent complete chain entities (given example is a *genesis* block - the 12 | first block in the mainnet Bitcoin chain; script strings omitted for brevity): 13 | 14 | ```turtle 15 | @prefix : 16 | @prefix btc: 17 | 18 | btc:blk0 19 | :height 0; 20 | :hash "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f"; 21 | :time 1231006505; 22 | :version 1; 23 | :transaction btc:4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b. 24 | 25 | btc:4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b 26 | :lockTime 0; 27 | :input [:unlockScript "...".]; 28 | :output [:amount 5000000000; :lockScript "...".]. 29 | ``` 30 | 31 | 32 | ## Setup 33 | The following examples assume AllegroGraph triple store and assume it is already 34 | installed and running on the target machine. The following AG instance settings 35 | settings are assumed as well: 36 | - *host*: `localhost` (default); 37 | - *port*: `10035` (default); 38 | - *username*: `aguser`; 39 | - *password*: `agpassword`. 40 | 41 | We also assume the following `bitcoind` settings: 42 | - *host*: `localhost` (default); 43 | - *port*: `8332` (default); 44 | - *username*: `btcuser`; 45 | - *password*: `btcpassword`. 46 | 47 | First, install the tool by cloning this repository, setting up virtual 48 | environment and installing the dependencies: 49 | ``` bash 50 | git clone http://github.com/franzinc/agraph-examples 51 | cd agraph-examples/data/bitcoin 52 | python3 -m venv . 53 | source ./bin/activate 54 | pip3 install -r requirements.txt 55 | ``` 56 | 57 | The following command starts the process of loading bitcoin chain data into an 58 | AG repository named `bitcoin` using 4 loader processes: 59 | ``` bash 60 | ./convert.py \ 61 | --source=http://btcuser:btcpassword@localhost:8332 \ 62 | --destination=http://aguser:agpassword@localhost:10035 \ 63 | --name=bitcoin \ 64 | --workers=4 \ 65 | --clear 66 | ``` 67 | 68 | 69 | ## Example queries 70 | Following are the examples of using SPARQL to extract different information 71 | about block data: 72 | - number of known blocks: 73 | ```sparql 74 | PREFIX : 75 | SELECT (COUNT(*) AS ?count) WHERE { ?b a btcm:Block. } 76 | ``` 77 | 78 | - total number of transactions: 79 | ```sparql 80 | PREFIX : 81 | SELECT (COUNT(*) AS ?count) WHERE { ?tx a btcm:Transaction. } 82 | ``` 83 | 84 | - transaction in block 400: 85 | ```sparql 86 | PREFIX : 87 | SELECT ?txid 88 | WHERE { 89 | ?b a :Block. 90 | ?b :height "570001"^^xsd:int. 91 | ?b :transaction ?tx. 92 | ?tx :txid ?txid. 93 | } 94 | ``` 95 | 96 | - transactions sending more than 1000 BTC: 97 | ```sparql 98 | PREFIX : 99 | SELECT ?tx 100 | WHERE { 101 | ?b a :Block. 102 | ?b :transaction ?tx. 103 | ?tx :output ?out. 104 | ?out :amount ?amt. 105 | } 106 | GROUP BY ?tx 107 | HAVING (SUM(?amt) > 100000000000) 108 | ``` 109 | 110 | - transactions sending BTC to Pirate Bay's address: 111 | ```sparql 112 | PREFIX : 113 | SELECT ?tx 114 | WHERE { 115 | ?tx :output ?out. 116 | ?out :lockScript ?s. 117 | FILTER REGEX (?s, ""). 118 | } 119 | ``` 120 | 121 | [model]: https://raw.githubusercontent.com/franzinc/agraph-examples/master/data/bitcoin/model.ttl# 122 | [RDF]: https://en.wikipedia.org/wiki/Resource_Description_Framework 123 | [Turtle]: https://en.wikipedia.org/wiki/Turtle_(syntax) 124 | -------------------------------------------------------------------------------- /data/bitcoin/requirements.txt: -------------------------------------------------------------------------------- 1 | agraph-python==101.0.1 2 | python-bitcoinlib==0.10.1 3 | -------------------------------------------------------------------------------- /data/chomsky-llm-example/README.md: -------------------------------------------------------------------------------- 1 | # Chomsky LLM example 2 | 3 | ## About 4 | 5 | This is an example of RAG (Retrieval Augmented Generation). The folder has the archived triple file (chomsky.nq.gz) containing facts about Chomsky. In this example, you can either restore an already built vector DB or create a vector database of embeddings on your own. After that, AllegroGraph provides a way to ask questions about Chomsky passing the LLM information from which it can get the answer. 6 | 7 | ## Instructions 8 | 9 | Prerequisites: 10 | - Set up AllegroGraph instance (local or remote). 11 | - Locally installed [agtool](https://franz.com/agraph/support/documentation/8.0.1/agtool.html). 12 | - OpenAI API key. 13 | - If you don't want to create the vector database on your own, you need to download the archive with the vector DB backup and unpack it. `curl -O https://s3.amazonaws.com/franz.com/allegrograph/chomsky-vdb.bak.gz && tar -xzf chomsky-vdb.bak.gz` 14 | 15 | 1. Restore from the backup or create the vector database. 16 | - To restore from the backup you need to restore the downloaded backup of vector DB: `agtool archive restore localhost:AG_PORT/chomsky-vdb chomsky-vdb.bak` where `AG_PORT` is the port number of your local AllegroGraph instance. If you have a remote AllegroGraph instance, you need to specify the full URL to it. For example, `https://user:password@your_cloud_provider:10035/chomsky-vdb` where `user:password` credentials to your AllegroGraph instance, `your_cloud_provider` is the hostname to your AllegroGraph instance, and `10035` is the remotely accessible port to your AllegroGraph instance. 17 | - To create the vector database, follow [the instructions](https://franz.com/agraph/support/documentation/8.0.1/llmembed.html) from AllegroGraph documentation. 18 | 2. In WebView of your AllegroGraph instance: 19 | 1. open an existing repo or create a new triple store. The content of the triple store is not important because RAG uses only vector database data, not your triple store data. 20 | 2. click on the "Repository control" link in the left sidebar. 21 | 3. click on the "Manage queries" group, and the "Query execution options" link. 22 | 4. click on the "+ NEW QUERY OPTION" button. 23 | 5. Use "openaiApiKey" as the option name and your OpenAI API key as the option value. 24 | 6. Click on the "SAVE QUERY OPTIONS" button. 25 | 3. Go to the "Query" section of your triple store. 26 | 4. Click on the inverted rectangle button right to the "+ NEW QUERY" button. Click "New query documents". 27 | 5. Write a query for the document. For example, "How old are you?". 28 | 6. Click "Execute". 29 | 30 | The expected result is a quote from the book that references he is in his mid-eighties. If you don't get the expected result, please send your technical questions to support@franz.com. 31 | -------------------------------------------------------------------------------- /data/chomsky-llm-example/chomsky.nq.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/data/chomsky-llm-example/chomsky.nq.gz -------------------------------------------------------------------------------- /databricks/demo.mapping.obda: -------------------------------------------------------------------------------- 1 | [PrefixDeclaration] 2 | : http://example.org/ 3 | rdf: http://www.w3.org/1999/02/22-rdf-syntax-ns# 4 | rdfs: http://www.w3.org/2000/01/rdf-schema# 5 | owl: http://www.w3.org/2002/07/owl# 6 | xsd: http://www.w3.org/2001/XMLSchema# 7 | obda: https://w3id.org/obda/vocabulary# 8 | 9 | [MappingDeclaration] @collection [[ 10 | 11 | mappingId people10m 12 | target :{id} a :Person ; rdfs:label "{firstName} {lastName}" ; :gender "{gender}"; :salary "{salary}"^^xsd:int . 13 | source SELECT * FROM `hive_metastore`.`default`.`people10m` LIMIT 1000 14 | 15 | ]] 16 | -------------------------------------------------------------------------------- /databricks/demo.properties: -------------------------------------------------------------------------------- 1 | jdbc.url= 2 | jdbc.driver=com.databricks.client.jdbc.Driver 3 | -------------------------------------------------------------------------------- /databricks/img/cluster-main.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/databricks/img/cluster-main.png -------------------------------------------------------------------------------- /databricks/img/data-explorer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/databricks/img/data-explorer.png -------------------------------------------------------------------------------- /databricks/img/gruff.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/databricks/img/gruff.png -------------------------------------------------------------------------------- /databricks/img/sample-data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/databricks/img/sample-data.png -------------------------------------------------------------------------------- /generators/weather/weather.md: -------------------------------------------------------------------------------- 1 | # Weather data generator 2 | 3 | This code here generates weather observation data for a given number of weather stations over a certain time period. 4 | This data can be written to a distributed repository that is 5 | partitioned on the graph part of the quad. 6 | 7 | See the file wxdemo.cl for details on how to build the weather observation database. 8 | 9 | 10 | ## quads generated 11 | 12 | The quads generated, in a pseudo-ntriples syntax, are shown here. 13 | For each station there are quads that provide the station name 14 | and location. 15 | For each observation at the station there are quads that provide 16 | the time of the observation and the weather observed. If it was 17 | raining then the rainfall accumulation since the last observation 18 | is provided. 19 | 20 | wx here is "http://demo.com/wx#" 21 | 22 | ``` 23 | wx:st_ rdf:type wx:station wx:st_ 24 | wx:st_ rdf:label "stationname" wx:st_ 25 | wx:st_ wx:location lat-long wx:st_ 26 | _bn2 rdf:type wx:observation wx:st_ 27 | _bn2 wx:time "2018-05-07T18:00:00Z"^^xsd:dateTime wx:st_ 28 | _bn2 wx:temp "20"^^xsd:integer wx:st_ 29 | _bn2 wx:wx wx:rain wx:st_ [[if rain]] 30 | _bn2 wx:precip "5" wx_st:_ [[if rain]] 31 | _bn2 wx:wx ws:clear wx:st_ [[if no rain]] 32 | ``` 33 | -------------------------------------------------------------------------------- /geodemo/generating-phonecalls.cl: -------------------------------------------------------------------------------- 1 | (eval-when (compile load eval) 2 | (require :agraph "~/ag731smp/lib/agraph.fasl")) 3 | 4 | 5 | (in-package :triple-store-user) 6 | 7 | (eval-when (compile load eval) 8 | (enable-!-reader)) 9 | 10 | (defparameter .test-port. 10035) 11 | 12 | (defparameter g3 13 | (geo-register-nd-subtype 14 | :ordinates '((:name :longitude :minimum -123.0 :maximum -122.0 :strip-width 0.001 :resolution 0.00001) 15 | (:name :latitude :minimum +37.0 :maximum +38.0 :strip-width 0.001 :resolution 0.00001) 16 | (:name :time :resolution 1 17 | :minimum "2014-01-01T00:00:00Z" 18 | :maximum "2015-01-01T00:00:00Z")) 19 | :install t)) 20 | 21 | (defun create-store (&key (name "PhoneCalls") (num 1000000)) 22 | (ignore-errors (close-triple-store)) 23 | (register-namespace "phone" "http://spooks.gov/phone-calls#") 24 | (create-triple-store name :port .test-port.) 25 | (add-geospatial-subtype-to-db g3) 26 | (setf (predicate-mapping !phone:phoneCallOriginLocation) g3) 27 | (setf (predicate-mapping !phone:phoneCallTargetLocation) g3) 28 | (time (setup-geo-phone-calls :num num)) 29 | (commit-triple-store) 30 | *db*) 31 | 32 | (defun setup-geo-phone-calls (&key (num 1000)) 33 | (loop for i from 1 to num 34 | as lat-from = (+ 37.750 (random 0.050)) 35 | as lon-from = (+ -122.480 (random 0.090)) 36 | as lat-to = (+ 37.750 (random 0.050)) 37 | as lon-to = (+ -122.480 (random 0.090)) 38 | as time = (+ (load-time-value (string-to-universal-time "Mon Aug 18 08:00:00 2014")) 39 | (random (load-time-value (* 12 3600)))) 40 | as from = (random-phone-number) ; use encoding? 41 | as to = (random-phone-number) ; use encoding? 42 | as event = (new-blank-node) 43 | do 44 | ;; (format t "~a ~a ~a~%" from to o) 45 | (add-triple event !phone:phoneCallOriginator (literal from)) 46 | (add-triple event !phone:phoneCallReceiver (literal to)) 47 | (add-triple event !phone:phoneCallOriginLocation (geo-plist->geo g3 :latitude lat-from :longitude lon-from :time time)) 48 | (add-triple event !phone:phoneCallTargetLocation (geo-plist->geo g3 :latitude lat-to :longitude lon-to :time time)) 49 | when (= 0 (mod i 100000)) do (commit-triple-store)) 50 | (commit-triple-store)) 51 | 52 | (defun random-phone-number () 53 | (macrolet ((r (a) 54 | `(svref ,a (random (load-time-value (length ,a)))))) 55 | (let ((area-code (r #("415" "510" "408"))) 56 | (ex1 (r #("2" "3" "4" "5" "6" "7" "8" "9"))) 57 | (ex2 (r #("1" "2" "3" "4" "5" "6" "7" "8" "9"))) 58 | (ex3 (r #("1" "2" "3" "4" "5" "6" "7" "8" "9"))) 59 | (num (format nil "~4,'0d" (1+ (random 9999))))) 60 | (util.string:string+ #\( area-code #\) ex1 ex2 ex3 #\- num)))) 61 | 62 | #+never 63 | (select (?from ?to) 64 | (get-triples-nd-within-radius ?triple :p !phone:phoneCallTargetLocation 65 | :latitude +37.756 :longitude -122.415 :units :km :radius 0.1 66 | :time-min "2014-08-18T15:20:00Z" 67 | :time-max "2014-08-18T15:25:00Z") 68 | (subject ?phonecall ?triple) 69 | (q ?phonecall !phone:phoneCallOriginator ?from) 70 | (q ?phonecall !phone:phoneCallReceiver ?to)) 71 | 72 | 73 | #+never 74 | (db.agraph.sparql:run-sparql 75 | " 76 | PREFIX ndfn: 77 | PREFIX nd: 78 | PREFIX phone: 79 | 80 | SELECT *{ 81 | 82 | ?call phone:phoneCallOriginLocation ?where . 83 | 84 | } 85 | LIMIT 5 86 | ") 87 | 88 | #+never 89 | (db.agraph.sparql:run-sparql 90 | " 91 | PREFIX ndfn: 92 | PREFIX nd: 93 | PREFIX phone: 94 | 95 | SELECT *{ 96 | 97 | ?call nd:inBoundingBox (phone:phoneCallOriginLocation 98 | nd:latitude-min +37.755 nd:latitude-max +37.758 99 | nd:longitude-mion -122.416 nd:longitude-max -122.414 100 | nd:time-min \"2014-08-18T15:20:00Z\" 101 | nd:time-max \"2014-08-18T15:25:00Z\") . 102 | 103 | } 104 | LIMIT 5 105 | ") 106 | -------------------------------------------------------------------------------- /geodemo/geodemo.md: -------------------------------------------------------------------------------- 1 | # Geodemo 2 | 3 | ## Introduction 4 | 5 | AllegroGraph has a unique n-dimensional indexing capability that works 6 | especially well with geotemporal data. By geotemporal we mean that we 7 | use an index that uses x,y,t coordinates for cartesian 8 | representations, or time,lat,lon for spherical representations. A 9 | tutorial on how to use n-dimensional indexing can be found 10 | [here](https://franz.com/agraph/support/documentation/current/geospatial-nd-tutorial.html#example-run-through). This 11 | page describe a graph that contains phone call records of phone calls 12 | made in the city of San Francisco. Every phone call has a 13 | time/latitude/longitude of the caller and the callee. In the example 14 | we use our n-dimensional indexing to index every record for the 15 | (time,lat,lon) of the caller and the callee and 16 | we show you how to use SPARQL to query the database. 17 | 18 | The example on this github page 19 | is an addition to this example. We show how 20 | you can interact with this demo in Google Maps in your browser. This 21 | is an example of what it might look like (depending on where you 22 | click on the map): 23 | 24 | ![phonecall image](geodemosteve.png) 25 | 26 | 27 | Note that this picture shows you lines between callers and 28 | callees. The circle is determined by the radians widget in the page 29 | and contains all the phone calls that start or terminate in that circle. 30 | 31 | We have a demo version running on [https://flux.franz.com:10027](https://flux.franz.com:10027). Just 32 | click around play with the input boxes to see how fast the indexing 33 | works. 34 | 35 | 36 | ## Running the demo locally 37 | 38 | You will need to obtain a Google Maps API Key in order to run this demo. 39 | The instructions from Google itself to obtain such a key is 40 | 41 | > Go to the Google Maps Platform > Credentials page. 42 | > On the Credentials page, click Create credentials > API key. 43 | > The API key created dialog displays your newly created API key. 44 | 45 | 46 | 47 | In case you want to play with this example yourself, you'll 48 | find the following two files in this github project. 49 | 50 | ### File 1. geodemo.cl 51 | 52 | If you only want to run the demo and not create new triples, then this 53 | is the file that you need to compile in order to run the demo in Google 54 | Maps. Make sure you have access to a running AllegroGraph server and 55 | an Allegro Common Lisp compiler on the same server. 56 | 57 | Before can compile it you have to edit the file in a few places 58 | 59 | - edit the agraph.fasl location 60 | - edit the port that on which AllegroGraph is running 61 | - edit the port on which you are going to run the Maps demo. 62 | - edit the line where the Google Maps API key is specified 63 | 64 | Start up your lisp and `cd` to the location where you have your files 65 | 66 | Compile and load the file 67 | 68 | ``` 69 | cl-user(3): :cl geodemo 70 | ``` 71 | 72 | Start the demo 73 | 74 | ``` 75 | cl-user(4): (start-demo) 76 | ``` 77 | 78 | Assuming you chose to view the demo on port 10027, 79 | open your browser and go to `http://localhost:10027` 80 | 81 | ### File 2: PhoneCalls.ttl 82 | 83 | This file contains the prebuilt phone call records for your 84 | convenience. It is a set of triples written in Turtle format. 85 | 86 | Have fun with the demo! 87 | 88 | 89 | ## Generate your own phone calls 90 | 91 | Lispers might want to create their own nd triple stores. The file 92 | generate-phonecalls.cl shows how the triples in the PhoneCalls demo 93 | were generated. It also provides some prolog and sparql queries. 94 | 95 | Here are the steps to create your own triples.. 96 | 97 | 1. Change the `require` expression to point the right version of the `agraph.fasl` in the `lib` directory where AllegroGraph is installed. 98 | 2. Change `.test-port`. to the port on which your your AllegroGraph server runs. 99 | 3. In your ACL compile and load the file `generate-phonecalls.cl` 100 | 4. Run `(db.agraph.user::create-store :name "SomeName" :num 1000000)` 101 | 102 | `:name` is the name of the repository to create 103 | 104 | `:num` is the number of phonecalls you wat to generate. 105 | 106 | 107 | -------------------------------------------------------------------------------- /geodemo/geodemosteve.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/geodemo/geodemosteve.png -------------------------------------------------------------------------------- /geosparql/README.md: -------------------------------------------------------------------------------- 1 | # Geospatial Queries with AllegroGraph 2 | 3 | In these notebooks we will lay out how to structure geospatial RDF data and then perform a variety of interesting SPARQL queries. By extending the capabilities of SPARQL to handle geospatial data, you can perform spatial queries, such as determining proximity, spatial intersections, and more. These queries are highly useful for applications like geographic information systems (GIS), urban planning, environmental monitoring, and any domain where spatial relationships are essential. 4 | 5 | ## Python Environment 6 | 7 | You can create a new python environment with all the required packages with the following command (NOTE: this assumes you are either in an anconda or minicondan base environment already) 8 | 9 | 10 | ### Conda 11 | 12 | ```shell 13 | conda env create -f environment.yml 14 | ``` 15 | 16 | Please make sure to `activate` your new environment using: 17 | 18 | ```shell 19 | conda activate geo-example 20 | ``` 21 | 22 | ### PIP 23 | 24 | Using at least `python 3.10`, please run the following: 25 | 26 | ```shell 27 | python3 -m venv geo-example 28 | ``` 29 | 30 | Then to activate the environment please run: 31 | 32 | ```shell 33 | source geo-example/bin/activate 34 | ``` 35 | 36 | and then finally run: 37 | 38 | ```shell 39 | pip install -r requirements.txt 40 | ``` 41 | 42 | ### Launching the notebook 43 | 44 | Please launch `Jupyter Lab` by executing in your terminal: 45 | 46 | ```shell 47 | jupyter lab 48 | ``` 49 | 50 | ## The Notebooks 51 | 52 | * [geosparql-tutorial](geosparql-tutorial.ipynb) - Gives a more technical explanation of the GeoSPARQL and the necessary structure of the ontology. This demo also provides a few examples. 53 | 54 | * [geosparql-examples](geosparql-examples.ipynb) - This notebook demonstrates how to **ETL** geopandas dataframes with `geometry` data into AllegroGraph and then shows the structure most GeoSPARQL queries will follow. 55 | -------------------------------------------------------------------------------- /geosparql/data/franz.ttl: -------------------------------------------------------------------------------- 1 | @prefix : . 2 | @prefix geo: . 3 | 4 | :Franz geo:hasGeometry :FranzGeom . 5 | :FranzGeom geo:asWKT "POINT(-122.12956214302625 37.88963389303175)"^^geo:wktLiteral . 6 | 7 | :OyamaSushi a :Restaurant ; 8 | geo:hasGeometry :OyamaSushiGeom . 9 | :OyamaSushiGeom geo:asWKT "POINT(-122.1275488659957 37.89076272510303)"^^geo:wktLiteral . 10 | 11 | :PostinoRestaurant a :Restaurant ; 12 | geo:hasGeometry :PostinoRestaurantGeom . 13 | :PostinoRestaurantGeom geo:asWKT "POINT(-122.12107937769098 37.891406215514536)"^^geo:wktLiteral . 14 | 15 | :TheCooperageAmericanGrille a :Restaurant ; 16 | geo:hasGeometry :TheCooperageAmericanGrilleGeom . 17 | :TheCooperageAmericanGrilleGeom geo:asWKT "POINT(-122.12016742659232 37.89114373984338)"^^geo:wktLiteral . 18 | 19 | :TheHideoutKitchenAndCafe a :Restaurant ; 20 | geo:hasGeometry :TheHideoutKitchenAndCafeGeom . 21 | :TheHideoutKitchenAndCafeGeom geo:asWKT "POINT(-122.10956550861351 37.89419432704863)"^^geo:wktLiteral . 22 | 23 | :TheParkBistroAndBar a :Restaurant ; 24 | geo:hasGeometry :TheParkBistroAndBarGeom . 25 | :TheParkBistroAndBarGeom geo:asWKT "POINT(-122.1009540100645 37.89637380250877)"^^geo:wktLiteral . -------------------------------------------------------------------------------- /geosparql/data/geosparql-example.ttl: -------------------------------------------------------------------------------- 1 | @prefix geo: . 2 | @prefix my: . 3 | @prefix rdf: . 4 | @prefix rdfs: . 5 | @prefix sf: . 6 | 7 | my:PlaceOfInterest a rdfs:Class ; 8 | rdfs:subClassOf geo:Feature . 9 | 10 | my:A a my:PlaceOfInterest ; 11 | my:hasExactGeometry my:AExactGeom ; 12 | my:hasPointGeometry my:APointGeom . 13 | 14 | my:B a my:PlaceOfInterest ; 15 | my:hasExactGeometry my:BExactGeom ; 16 | my:hasPointGeometry my:BPointGeom . 17 | 18 | my:C a my:PlaceOfInterest ; 19 | my:hasExactGeometry my:CExactGeom ; 20 | my:hasPointGeometry my:CPointGeom . 21 | 22 | my:D a my:PlaceOfInterest ; 23 | my:hasExactGeometry my:DExactGeom ; 24 | my:hasPointGeometry my:DPointGeom . 25 | 26 | my:E a my:PlaceOfInterest ; 27 | my:hasExactGeometry my:EExactGeom . 28 | 29 | my:F a my:PlaceOfInterest ; 30 | my:hasExactGeometry my:FExactGeom . 31 | 32 | my:hasExactGeometry a rdf:Property ; 33 | rdfs:subPropertyOf geo:hasDefaultGeometry, 34 | geo:hasGeometry . 35 | 36 | my:hasPointGeometry a rdf:Property ; 37 | rdfs:subPropertyOf geo:hasGeometry . 38 | 39 | my:AExactGeom a sf:Polygon ; 40 | geo:asWKT """ 41 | Polygon((-83.6 34.1, -83.2 34.1, -83.2 34.5, 42 | -83.6 34.5, -83.6 34.1))"""^^geo:wktLiteral. 43 | 44 | my:APointGeom a sf:Point ; 45 | geo:asWKT """ 46 | Point(-83.4 34.3)"""^^geo:wktLiteral. 47 | 48 | my:BExactGeom a sf:Polygon ; 49 | geo:asWKT """ 50 | Polygon((-83.6 34.1, -83.4 34.1, -83.4 34.3, 51 | -83.6 34.3, -83.6 34.1))"""^^geo:wktLiteral. 52 | 53 | my:BPointGeom a sf:Point ; 54 | geo:asWKT """ 55 | Point(-83.5 34.2)"""^^geo:wktLiteral. 56 | 57 | my:CExactGeom a sf:Polygon ; 58 | geo:asWKT """ 59 | Polygon((-83.2 34.3, -83.0 34.3, -83.0 34.5, 60 | -83.2 34.5, -83.2 34.3))"""^^geo:wktLiteral. 61 | 62 | my:CPointGeom a sf:Point ; 63 | geo:asWKT """ 64 | Point(-83.1 34.4)"""^^geo:wktLiteral. 65 | 66 | my:DExactGeom a sf:Polygon ; 67 | geo:asWKT """ 68 | Polygon((-83.3 34.0, -83.1 34.0, -83.1 34.2, 69 | -83.3 34.2, -83.3 34.0))"""^^geo:wktLiteral. 70 | 71 | my:DPointGeom a sf:Point ; 72 | geo:asWKT """ 73 | Point(-83.2 34.1)"""^^geo:wktLiteral. 74 | 75 | my:EExactGeom a sf:LineString ; 76 | geo:asWKT """ 77 | LineString(-83.4 34.0, -83.3 34.3)"""^^geo:wktLiteral. 78 | 79 | my:FExactGeom a sf:Point ; 80 | geo:asWKT """ 81 | Point(-83.4 34.4)"""^^geo:wktLiteral. 82 | -------------------------------------------------------------------------------- /geosparql/img/geosparql-etl-layout.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/geosparql/img/geosparql-etl-layout.png -------------------------------------------------------------------------------- /geosparql/img/geosparql-example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/geosparql/img/geosparql-example.png -------------------------------------------------------------------------------- /geosparql/img/geosparql-ontology.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/geosparql/img/geosparql-ontology.png -------------------------------------------------------------------------------- /geosparql/img/screenshot-1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/geosparql/img/screenshot-1.jpg -------------------------------------------------------------------------------- /geosparql/img/screenshot-2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/geosparql/img/screenshot-2.jpg -------------------------------------------------------------------------------- /geosparql/img/screenshot-3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/geosparql/img/screenshot-3.jpg -------------------------------------------------------------------------------- /geosparql/requirements.txt: -------------------------------------------------------------------------------- 1 | agraph-python==104.1.0 2 | anyio==4.6.0 3 | argon2-cffi==23.1.0 4 | argon2-cffi-bindings==21.2.0 5 | arrow==1.3.0 6 | asttokens==2.4.1 7 | async-lru==2.0.4 8 | attrs==24.2.0 9 | babel==2.16.0 10 | beautifulsoup4==4.12.3 11 | bleach==6.1.0 12 | branca==0.7.2 13 | certifi==2024.8.30 14 | cffi==1.17.1 15 | charset-normalizer==3.3.2 16 | comm==0.2.2 17 | debugpy==1.8.5 18 | decorator==5.1.1 19 | defusedxml==0.7.1 20 | exceptiongroup==1.2.2 21 | executing==2.1.0 22 | fastjsonschema==2.20.0 23 | fqdn==1.5.1 24 | geopandas==1.0.1 25 | h11==0.14.0 26 | httpcore==1.0.5 27 | httpx==0.27.2 28 | idna==3.10 29 | ipykernel==6.29.5 30 | ipyleaflet==0.19.2 31 | ipython==8.27.0 32 | ipywidgets==8.1.5 33 | iso8601==2.1.0 34 | isoduration==20.11.0 35 | jedi==0.19.1 36 | Jinja2==3.1.6 37 | json5==0.9.25 38 | jsonpointer==3.0.0 39 | jsonschema==4.23.0 40 | jsonschema-specifications==2023.12.1 41 | jupyter-events==0.10.0 42 | jupyter-leaflet==0.19.2 43 | jupyter-lsp==2.2.5 44 | jupyter_client==8.6.3 45 | jupyter_core==5.7.2 46 | jupyter_server==2.14.2 47 | jupyter_server_terminals==0.5.3 48 | jupyterlab==4.2.5 49 | jupyterlab_pygments==0.3.0 50 | jupyterlab_server==2.27.3 51 | jupyterlab_widgets==3.0.13 52 | MarkupSafe==2.1.5 53 | matplotlib-inline==0.1.7 54 | mistune==3.0.2 55 | nbclient==0.10.0 56 | nbconvert==7.16.4 57 | nbformat==5.10.4 58 | nest-asyncio==1.6.0 59 | notebook_shim==0.2.4 60 | numpy==2.1.1 61 | overrides==7.7.0 62 | packaging==24.1 63 | pandas==2.2.3 64 | pandocfilters==1.5.1 65 | parso==0.8.4 66 | pexpect==4.9.0 67 | platformdirs==4.3.6 68 | prometheus_client==0.21.0 69 | prompt_toolkit==3.0.47 70 | psutil==6.0.0 71 | ptyprocess==0.7.0 72 | pure_eval==0.2.3 73 | pycparser==2.22 74 | Pygments==2.18.0 75 | pyogrio==0.9.0 76 | pyproj==3.6.1 77 | PySocks==1.7.1 78 | python-dateutil==2.9.0.post0 79 | python-json-logger==2.0.7 80 | pytz==2024.2 81 | PyYAML==6.0.2 82 | pyzmq==26.2.0 83 | referencing==0.35.1 84 | requests==2.32.3 85 | rfc3339-validator==0.1.4 86 | rfc3986-validator==0.1.1 87 | rpds-py==0.20.0 88 | Send2Trash==1.8.3 89 | shapely==2.0.6 90 | shortuuid==1.0.13 91 | six==1.16.0 92 | sniffio==1.3.1 93 | soupsieve==2.6 94 | stack-data==0.6.3 95 | terminado==0.18.1 96 | tinycss2==1.3.0 97 | tomli==2.0.1 98 | tornado==6.4.2 99 | tqdm==4.66.5 100 | traitlets==5.14.3 101 | traittypes==0.2.1 102 | types-python-dateutil==2.9.0.20240906 103 | typing_extensions==4.12.2 104 | tzdata==2024.1 105 | uri-template==1.3.0 106 | urllib3==2.2.3 107 | wcwidth==0.2.13 108 | webcolors==24.8.0 109 | webencodings==0.5.1 110 | websocket-client==1.8.0 111 | widgetsnbextension==4.0.13 112 | xyzservices==2024.9.0 -------------------------------------------------------------------------------- /grafana/README.md: -------------------------------------------------------------------------------- 1 | # AllegroGraph + Grafana 2 | 3 | ## About 4 | 5 | About Grafana from [their website](https://grafana.com/grafana/): 6 | 7 | > Query, visualize, alert on, and understand your data no matter where it’s stored. With Grafana you can create, explore, and share all of your data through beautiful, flexible dashboards. 8 | 9 | This demo shows how users can fetch data from AllegroGraph (without authentication) in Grafana and render it. 10 | 11 | ## Launch instruction 12 | 13 | In this example, we launch grafana inside a docker container. If you're interested in other ways to install Grafana please see [their documentation](https://grafana.com/docs/grafana/latest/setup-grafana/installation/). All your changes to Grafana will be lost on removing container. To keep it across launches, please see [Grafana documentation](https://grafana.com/docs/grafana/latest/setup-grafana/installation/docker/#save-your-grafana-data). 14 | 15 | Execute the following to launch Grafana instance (assuming your current working directory is `agraph-examples/grafana`): 16 | ``` 17 | docker run -d -p 3000:3000 --name=grafana -e "GF_INSTALL_PLUGINS=yesoreyeram-infinity-datasource" -v $(pwd)/provisioning/datasources:/etc/grafana/provisioning/datasources -v $(pwd)/provisioning/dashboards:/etc/grafana/provisioning/dashboards grafana/grafana-oss 18 | ``` 19 | 20 | Where: 21 | 22 | - docker run is a Docker CLI command that runs a new container from an image. 23 | - -d (--detach) runs the container in the background. 24 | - -p : (--publish) publish a container’s port(s) to the host, allowing you to reach the container’s port via a host port. In this case, we can reach the container’s port 3000 via the host’s port 3000. 25 | - --name assign a logical name to the container (e.g. grafana). This allows you to refer to the container by name instead of by ID. 26 | - -e "GF_INSTALL_PLUGINS=yesoreyeram-infinity-datasource" installs [Infinity datasource plugin for Grafana](https://grafana.com/docs/plugins/yesoreyeram-infinity-datasource/latest/) that allows rendering JSON/CSV/GraphQL/XML data from HTTP responses. 27 | - `-v ./provisioning/datasources:/etc/grafana/provisioning/datasources` mounts configured datasource from demo AllegroGraph instance https://gruff.allegrograph.com/webview without authentication. 28 | - `-v ./provisioning/dashboards:/etc/grafana/provisioning/dashboards` mounts simple dashboards with visualization AllegroGraph data. 29 | - grafana/grafana-oss is the image to run. 30 | 31 | After that, you can verify that your Grafana instance is successfully launched by executing `docker ps` and see output similar to the following: 32 | ``` 33 | # This will display a list of containers that looks like the following: 34 | CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 35 | cd48d3994968 grafana/grafana-oss "/run.sh" 8 seconds ago Up 7 seconds 0.0.0.0:3000->3000/tcp grafana 36 | ``` 37 | 38 | If you see the `grafana` container in the output, then you can visit it in your browser (in this example, it's http://localhost:3000). The default user for Grafana instance is admin with `admin` password. If you don't see the `grafana` container please double-check Grafana documentation on launching Grafana instance via docker. 39 | 40 | In Grafana web UI, you can click on "Dashboards" and click on "Simple Example" dashboard to see the demo dashboard. 41 | 42 | ## Grafana dashboard 43 | 44 | With successfully loaded dashboard, you should see: 45 | - "Host" text input with `https://gruff.allegrograph.com` value in it. This value is used as host to send SPARQL queries. 46 | - "Repository" text input with `actors-extended` value in it. This value is used as the repository to send SPARQL queries. 47 | - "Predicate frequency" is a bar chart panel that shows predicates and their occurrences in the "Repository". 48 | - "Class frequency" is a pie chart panel that shows objects from triples with rdf:type predicate and their occurrences in the "Repository". 49 | - "Top 10 actors by starred films" is a bar gauge panel chart that shows top 10 actors ordered by numbers of starred films in the "Repository". 50 | 51 | To see and edit SPARQL query any chart: 52 | 1. Mouse over the top-right corner of the panel and click on the 3 dots icon and click "Edit". 53 | 2. Scroll to the "Query" section of the panel and click on "Headers, Body, Request params" button. 54 | 3. In the "Body Type" widget, there are "Key" and "Value" columns. "Value" column holds the SPARQL query. 55 | 4. To apply post-processing on the SPARQL query response, there is [UQL](https://grafana.com/docs/plugins/yesoreyeram-infinity-datasource/latest/query/uql/) query. Let's see as an example 56 | SPARQL query and appropriate UQL query for the "Predicate frequency" panel. 57 | SPARQL query which counts the number of predicates and returns only the IRI part after the last `/` character: 58 | ``` 59 | SELECT (STR(COUNT(?pred)) as ?predCounter) (REPLACE(STR(?pred), "^.*/", "") AS ?localName) { 60 | ?s ?pred ?o . 61 | } 62 | GROUP BY ?pred 63 | ``` 64 | UQL query which removes excessive `"` characters for numbers and convert it to number type in Grafana: 65 | ``` 66 | parse-json # parses string HTTP response as JSON format. 67 | # For AG instance JSON response has the following structure: 68 | # { "names": ["variable1", "variable2"], 69 | # "values":[["variable-value-1", "variable-value-2"], ...other rows... ] 70 | # } 71 | | project "values" # projects the top-level "values" field from the JSON response 72 | | extend "predicate"="1" # creates "predicate" column from the second items in row array 73 | | extend "predicate_count_1"=replace_string("0",'"','') # replaces `"` character with empty string to convert "\"42\"" into "42" 74 | | extend "predicate_count"=tonumber("predicate_count_1") # converts "42" into 42 75 | | project-away "0" # removes the original first column from response 76 | | project-away "1" # removes the original second column from response 77 | | project-away "predicate_count_1" # removes intermediate column 78 | ``` 79 | -------------------------------------------------------------------------------- /grafana/provisioning/dashboards/dashboards.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: 'default' 5 | orgId: 1 6 | folder: '' 7 | type: file 8 | disableDeletion: false 9 | editable: true 10 | options: 11 | path: /etc/grafana/provisioning/dashboards 12 | -------------------------------------------------------------------------------- /grafana/provisioning/datasources/ag-no-auth.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | datasources: 3 | - id: 2 4 | uid: "fdl6ohpunvvggd" 5 | orgId: 1 6 | name: "yesoreyeram-infinity-datasource" 7 | type: "yesoreyeram-infinity-datasource" 8 | url: "" 9 | basicAuth: false 10 | basicAuthUser: "test" 11 | isDefault: true 12 | jsonData: 13 | allowedHosts: 14 | - "https://gruff.allegrograph.com" 15 | auth_method: "none" 16 | customHealthCheckEnabled: true 17 | customHealthCheckUrl: "https://gruff.allegrograph.com/version" 18 | global_queries: [] 19 | oauthPassThru: false 20 | readOnly: false -------------------------------------------------------------------------------- /gruff/img/paths.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/gruff/img/paths.png -------------------------------------------------------------------------------- /gruff/img/query.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/gruff/img/query.png -------------------------------------------------------------------------------- /gruff/readme.md: -------------------------------------------------------------------------------- 1 | ### Embedding Gruff In a Web Page 2 | 3 | Gruff can be embedded in any web page to let readers use Gruff on 4 | your web site. The file [embedding.html](embedding.html) in the Gruff 5 | installation folder explains how to set up a running example of this. 6 | That file also serves as the example web page itself. You can adapt 7 | its JavaScript code for use on your own web site. 8 | 9 | Gruff needs to be running in a special "launcher" mode on a server 10 | machine that the web browser can reach. Then your web page can send a 11 | message to the Gruff server that tells it to launch another instance 12 | of Gruff for the reader to use in one area of your page. 13 | 14 | Simply embedding Gruff allows the reader to use Gruff by itself as 15 | usual inside the web page. A more advanced feature is that your web 16 | application can also send custom commands to Gruff. For example, your 17 | application could derive a set of triples that it wants Gruff to 18 | display, and then send those triples to Gruff. 19 | 20 | See [embedding.html](embedding.html) for the actual example file that 21 | you can find in your Gruff installation folder. Note that its "Run 22 | Gruff Below" button will not work here, because the Gruff server is 23 | not running. 24 | 25 | This example requires Gruff 8.1.0 to fully work as described here. 26 | 27 | Below are a couple of images of the example web page as it appears in 28 | a browser after Gruff has been embedded in an iframe below the 29 | instructions. The drop-down widget above Gruff contains custom 30 | commands that the web page sends to Gruff. 31 | 32 | ![a visual graph from a query](img/query.png) 33 | 34 | ![finding paths between two nodes](img/paths.png) 35 | 36 | -------------------------------------------------------------------------------- /java-http-sessions/Makefile: -------------------------------------------------------------------------------- 1 | threads ?= 4 2 | queries ?= 50 3 | lifetime ?= 32 4 | initial ?= 7 5 | worker ?= 9999 6 | 7 | 8 | # once the distribution is extracted this can be used to recompile 9 | # the class files 10 | compile: 11 | javac -cp "lib/*" com/franz/sessionpool/*.java 12 | 13 | # once "make compile" is done and the class files are created this can 14 | # be used to run the demo 15 | # you can set the parameters with 16 | # % make queries=100 threads=10 run 17 | # 18 | run: 19 | java -cp "lib/*:." com.franz.sessionpool.LbDemo --threads $(threads)\ 20 | --queries $(queries) --lifetime $(lifetime) --initial $(initial) --worker $(worker) 21 | 22 | 23 | 24 | 25 | # this is used when creating 26 | # a distribution from the main Netbeans directory where the code is developed 27 | 28 | dist: 29 | tar cfz dist.tgz Readme.txt Makefile lib -C src/main/java com 30 | 31 | -------------------------------------------------------------------------------- /java-http-sessions/Readme.txt: -------------------------------------------------------------------------------- 1 | Session Pool and Load Balancer Example 2 | 3 | There are many client APIs for AllegroGraph. Here we use the HTTP API 4 | which forms the basis for the Java and Python APIs. 5 | 6 | The HTTP API is useful if you wish to issue sparql queries to the repository 7 | and can process the query result form returned by the server. There are many 8 | possible result formats but in this code we just accept the default result format. 9 | 10 | This code demonstrates how you can can create a session pool using the HTTP API. 11 | Using a session pool allows you create a session backend with the repository 12 | open and send queries that backend without going through the main AllegroGraph 13 | front end (assuming that you haven't set "UseMainForForSessions true" in your 14 | agraph.cfg file). 15 | 16 | We then implement a load balancer on top of the session pool. This isn't 17 | a typical load balancer that proxies requests to a set of back ends. Instead 18 | this Load Balancer just allocates sessions from a set of session pools. 19 | Once the session is returned the client talks directly to the session backend 20 | and the load balancer is not used again. 21 | 22 | 23 | To use this: 24 | 25 | edit com.franz.sessionpool.LbDemo 26 | and change the setup code in main to specify repositories on your server. 27 | If you specify more than one repository they should have identical 28 | content (either via MMR or warm standby replication). 29 | 30 | 31 | Edit com.franz.sessionpool.SampleQuery 32 | and put in a query you want to execute. 33 | 34 | type 35 | % make compile 36 | 37 | then with the server running 38 | 39 | % make run 40 | 41 | You can specify e parameters to the 'make run' call, e..g 42 | 43 | % make threads=10 queries=20 lifetime=60 initial=10 worker=5 run 44 | 45 | with the meaning of each parameter 46 | 47 | threads - start these many threads to run queries 48 | queries - each thread will run the query this many times. Each time it 49 | runs a query it get a session out of the pool through the load balancer 50 | and puts the session back when the query is over 51 | lifetime - when a session is started it is given an idle time and if the 52 | session is idle for more than this many seconds the session process may be killed 53 | initial - when the session pools is created this many sessions are created 54 | immediately 55 | worker - this is just a number which is printed to identify which worker 56 | is printing a message. 57 | 58 | you may with to run this program simultaneously many times. Here'a 59 | a way to run it 15 times. 60 | 61 | % for i in {1..15} ; do (make queries=200 threads=20 lifetime=500 initial=20 worker=$i run &) \ 62 | ; done 63 | 64 | 65 | In this case you have 15 java programs, each starting 20 threads (so 15*20 = 300 threads) 66 | and each thread running 200 queries (so 300*200 = 60,000 queries). 67 | You start the session pool with 20 sessions which means the 20 threads will 68 | each find a pre-allocated session to use saving a bit of time creating a 69 | new session). 70 | 71 | 72 | 73 | Example: 74 | 75 | jkf@epic java]$ make compile 76 | javac -cp "lib/*" com/franz/sessionpool/*.java 77 | 78 | [jkf@epic java]$ make run 79 | java -cp "lib/*:." com.franz.sessionpool.LbDemo --threads 4\ 80 | --queries 50 --lifetime 32 --initial 7 --worker 9999 81 | 4 threads, 50 queries, 32 lifetime, 7 initial sessions 82 | Thread 9999.0 Started 83 | Thread 9999.1 Started 84 | Thread 9999.2 Started 85 | Thread 9999.3 Started 86 | Thread 9999.0 Finished 87 | Thread 9999.3 Finished 88 | Thread 9999.1 Finished 89 | Thread 9999.2 Finished 90 | [jkf@epic java]$ 91 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /java-http-sessions/com/franz/sessionpool/LbDemo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * To change this license header, choose License Headers in Project Properties. 3 | * To change this template file, choose Tools | Templates 4 | * and open the template in the editor. 5 | */ 6 | package com.franz.sessionpool; 7 | 8 | 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | import org.apache.http.NameValuePair; 12 | import org.apache.http.message.BasicNameValuePair; 13 | 14 | /** 15 | * 16 | * @author jkf 17 | */ 18 | public class LbDemo { 19 | 20 | 21 | public static void main(String[] args) throws Exception { 22 | 23 | int threads = 5; 24 | int queries = 10; 25 | int lifetime = 30; 26 | int initial = 5; //initial sessions 27 | int worker = 0; // for debugging 28 | 29 | for (int i = 0; i < args.length; i++) { 30 | 31 | if (args[i].equals("--threads")) { 32 | threads = Integer.parseInt(args[++i]); 33 | } else if (args[i].equals("--queries")) { 34 | queries = Integer.parseInt(args[++i]); 35 | } else if (args[i].equals("--lifetime")) { 36 | lifetime = Integer.parseInt(args[++i]); 37 | } else if (args[i].equals("--initial")) { 38 | initial = Integer.parseInt(args[++i]); 39 | } else if (args[i].equals("--worker")) { 40 | worker = Integer.parseInt(args[++i]); 41 | } 42 | } 43 | 44 | System.out.println(threads + " threads, " + queries + " queries, " + 45 | lifetime + " lifetime, " + initial + " initial sessions"); 46 | 47 | // create load balancer using ServerSpec class 48 | 49 | // ArrayList sss = new ArrayList<>(); 50 | // sss.add(new ServerSpec("http://localhost:10035", 51 | // null, "demorepo", "test", "xyzzy")); 52 | // sss.add(new ServerSpec("http://localhost:10035", 53 | // null, "secondrepo", "test", "xyzzy")); 54 | // 55 | // LoadBalancer lb = new LoadBalancer(sss, lifetime, initial); 56 | // 57 | 58 | 59 | // create load balancer using array of one server urls 60 | // thus using a single session pool 61 | 62 | ArrayList aaa = new ArrayList<>(); 63 | aaa.add("http://localhost:10035"); 64 | 65 | LoadBalancer lb = new LoadBalancer(aaa,null,"sp2", 66 | "test", "xyzzy", 67 | lifetime, initial); 68 | 69 | 70 | 71 | 72 | // prepare the query argument for the POST command we'll execute 73 | List pairs; 74 | pairs = new ArrayList<>(1); 75 | 76 | pairs.add(new BasicNameValuePair("query", SampleQuery.sampleQueryString)); 77 | 78 | 79 | 80 | 81 | 82 | for (int i = 0; i < threads; i++) { 83 | Thread thread; 84 | final int qcount = queries; 85 | final int qi = i; 86 | final int qworker = worker; 87 | thread = new Thread() { 88 | public void run() { 89 | try { 90 | for (int j = 0; j < qcount; j++) { 91 | SessionPool.Session sess = lb.getSessionFromLoadBalancer(); 92 | sess.sessionPool.httpCall(sess.url, 93 | "/sparql", SessionPool.method_post, pairs); 94 | sess.returnToPool(); 95 | } 96 | 97 | System.out.println("Thread " + qworker + "." + qi + " Finished"); 98 | 99 | } catch (Exception e) { 100 | System.out.println("query thread got exception " + e); 101 | } 102 | } 103 | }; 104 | thread.start(); 105 | System.out.println("Thread " + worker + "." + i + " Started"); 106 | } 107 | } 108 | 109 | } 110 | -------------------------------------------------------------------------------- /java-http-sessions/com/franz/sessionpool/LoadBalancer.java: -------------------------------------------------------------------------------- 1 | 2 | package com.franz.sessionpool; 3 | 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | /** 8 | * 9 | * @author jkf 10 | * The LoadBalancer class holds a set of SessionPool objects which 11 | * should reference repos with the same contents (e.g. through MMR). 12 | * This class will return sessions from one of the SessionPools distributing 13 | * the sessions between the pools in a round robin fashion. 14 | * Unlike a typical load balancer this class does not proxy the request 15 | * and results. It simply returns a session object and from them on 16 | * the client and server are directly connected. 17 | */ 18 | public class LoadBalancer { 19 | 20 | ArrayList servers; 21 | int poolindex = 0; 22 | int poolmax; 23 | 24 | 25 | 26 | /* 27 | This constructor assumes that the same catalog:repo is to be 28 | accessed on all machines given in the urls 29 | The urls are like http://machinex:234234 30 | just specifying the scheme, host and port. 31 | 32 | to use this: 33 | LoadBalancer lb = new LoadBalancer( ... ) // choose a constructor 34 | then when a session is needed: 35 | Session sess = lb.getSessionFromPool(); 36 | and when you no longer need the session 37 | sess.returnToPool(); 38 | */ 39 | 40 | LoadBalancer(List urls, String catalog, String reponame, 41 | String username, String password, int lifetime, 42 | int initialSessions) throws Exception 43 | { 44 | servers = new ArrayList<>(); 45 | 46 | for (String url : urls) { 47 | servers.add(SessionPool.setupSessionPool(url, catalog, reponame, username, 48 | password, lifetime, initialSessions)); 49 | } 50 | 51 | poolindex = 0; 52 | poolmax = urls.size(); 53 | } 54 | 55 | /* 56 | this constructor allows you to load balance against repos and 57 | specify different names and authentication for each repo 58 | */ 59 | LoadBalancer(ArrayList sss, int lifetime, int initialSessions) 60 | throws Exception 61 | { 62 | servers = new ArrayList<>(); 63 | 64 | for (ServerSpec ss :sss) { 65 | servers.add(SessionPool.setupSessionPool(ss.url, ss.catalog, 66 | ss.reponame, ss.username, ss.password, lifetime, 67 | initialSessions)); 68 | } 69 | 70 | poolindex = 0; 71 | poolmax = sss.size(); 72 | } 73 | 74 | SessionPool.Session getSessionFromLoadBalancer() 75 | throws Exception { 76 | 77 | int useindex; 78 | 79 | for (int count = 0; count < 3*poolmax; count++) { 80 | /* try three times to get a session before giving up */ 81 | 82 | synchronized (this) { 83 | poolindex = (poolindex + 1) % poolmax; 84 | useindex = poolindex; 85 | } 86 | 87 | try { 88 | return servers.get(useindex).getSessionFromPool(); 89 | } catch (Exception e) { 90 | } 91 | 92 | } 93 | 94 | /* all servers are down */ 95 | throw new Exception("All servers cannot create a session"); 96 | } 97 | 98 | 99 | } 100 | 101 | -------------------------------------------------------------------------------- /java-http-sessions/com/franz/sessionpool/SampleQuery.java: -------------------------------------------------------------------------------- 1 | /* 2 | * To change this license header, choose License Headers in Project Properties. 3 | * To change this template file, choose Tools | Templates 4 | * and open the template in the editor. 5 | */ 6 | 7 | package com.franz.sessionpool; 8 | 9 | /** 10 | * 11 | * @author jkf 12 | */ 13 | public class SampleQuery { 14 | public static final String sampleQueryString = 15 | String.join( "" 16 | ,"PREFIX rdf: \n" 17 | ,"PREFIX dc: \n" 18 | ,"PREFIX dcterms: \n" 19 | ,"PREFIX bench: \n" 20 | ,"PREFIX xsd: \n" 21 | ,"\n" 22 | ,"SELECT ?yr\n" 23 | ,"WHERE {\n" 24 | ," ?journal rdf:type bench:Journal .\n" 25 | ," ?journal dc:title \"Journal 1 (1940)\"^^xsd:string .\n" 26 | ," ?journal dcterms:issued ?yr \n" 27 | ,"}\n" 28 | ,"\n"); 29 | } 30 | -------------------------------------------------------------------------------- /java-http-sessions/com/franz/sessionpool/ServerSpec.java: -------------------------------------------------------------------------------- 1 | 2 | package com.franz.sessionpool; 3 | 4 | /* 5 | * 6 | * @author jkf 7 | * The information needed to specify a repository on a server 8 | * and the credentials to access that repo. 9 | */ 10 | 11 | public class ServerSpec { 12 | 13 | String url; 14 | String catalog; 15 | String reponame; 16 | String username; 17 | String password; 18 | 19 | public ServerSpec(String url, 20 | String catalog, 21 | String reponame, 22 | String username, 23 | String password) { 24 | this.url = url; 25 | this.catalog = catalog; 26 | this.reponame = reponame; 27 | this.username = username; 28 | this.password = password; 29 | 30 | } 31 | 32 | } -------------------------------------------------------------------------------- /java-http-sessions/lib/agraph-java-client-3.0.6-SNAPSHOT.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/agraph-java-client-3.0.6-SNAPSHOT.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/aopalliance-1.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/aopalliance-1.0.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/asm-4.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/asm-4.2.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/atomikos-util-4.0.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/atomikos-util-4.0.6.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/cglib-3.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/cglib-3.1.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/collection-0.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/collection-0.6.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/commons-beanutils-1.9.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/commons-beanutils-1.9.3.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/commons-cli-1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/commons-cli-1.2.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/commons-codec-1.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/commons-codec-1.4.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/commons-collections-3.2.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/commons-collections-3.2.2.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/commons-collections4-4.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/commons-collections4-4.1.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/commons-csv-1.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/commons-csv-1.3.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/commons-httpclient-3.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/commons-httpclient-3.1.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/commons-io-2.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/commons-io-2.4.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/commons-lang-2.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/commons-lang-2.6.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/commons-lang3-3.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/commons-lang3-3.4.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/commons-logging-1.1.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/commons-logging-1.1.1.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/commons-math3-3.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/commons-math3-3.5.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/commons-pool2-2.4.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/commons-pool2-2.4.2.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/commons-text-1.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/commons-text-1.3.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/guava-18.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/guava-18.0.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/httpclient-4.5.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/httpclient-4.5.2.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/httpclient-cache-4.5.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/httpclient-cache-4.5.2.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/httpcore-4.4.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/httpcore-4.4.4.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/httpmime-4.5.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/httpmime-4.5.5.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/jcl-over-slf4j-1.7.21.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/jcl-over-slf4j-1.7.21.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/jline-3.5.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/jline-3.5.1.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/libthrift-0.9.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/libthrift-0.9.3.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/mapdb-1.0.8.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/mapdb-1.0.8.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/noggit-0.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/noggit-0.6.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/opencsv-4.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/opencsv-4.2.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/slf4j-api-1.7.25.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/slf4j-api-1.7.25.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/slf4j-nop-1.7.25.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/slf4j-nop-1.7.25.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/spatial4j-0.7.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/spatial4j-0.7.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/standard-1.1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/standard-1.1.2.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/stax2-api-3.1.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/stax2-api-3.1.4.jar -------------------------------------------------------------------------------- /java-http-sessions/lib/xml-apis-1.4.01.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/java-http-sessions/lib/xml-apis-1.4.01.jar -------------------------------------------------------------------------------- /llm/README.md: -------------------------------------------------------------------------------- 1 | # AllegroGraph LLM Examples 2 | 3 | ## Python environment 4 | 5 | To use the following notebook we recommend you create a virtual environment. 6 | 7 | ### conda 8 | 9 | ```shell 10 | conda env create -f environment.yml 11 | ``` 12 | 13 | and then: 14 | 15 | ```shell 16 | conda activate ag-examples-llm 17 | ``` 18 | 19 | ### pip 20 | 21 | Please run the following commands 22 | 23 | ```shell 24 | python3 -m venv venv 25 | source venv/bin/activate 26 | pip install -r requirements.txt 27 | ``` 28 | 29 | ## AllegroGraph Natural Language Querying 30 | 31 | AllegroGraph's Natural Language Query system allows users to ask questions in plain language and receive SPARQL query translations and results. The system uses a vector store of successful query pairs and SHACL shapes to improve translation accuracy over time. To try, please go the [nlq.ipynb](nlq.ipynb) 32 | 33 | ### Core Features 34 | 35 | * Translates natural language to SPARQL queries 36 | * Learns from successful translations 37 | * Supports similarity-based query matching 38 | * Includes comprehensive feedback and error tracking 39 | 40 | ## AllegroGraph Ollama Integration 41 | 42 | `Ollama` is a platform designed for deploying, managing, and interacting with AI language models locally. It focuses on making it easy for users to run large language models (LLMs) on their own hardware, without needing to rely on cloud services or external servers. The platform's primary advantage is allowing users to operate AI models while maintaining control over their data and reducing latency associated with cloud-based processing. 43 | 44 | If you use `Ollama`, you can provide information about your `Ollama` setup and set llm query options in the *agraph.cfg* file, or pass it in for each SPARQL query. The notebooks will show how to pass in your connection info as SPARQL Prefix parameters. 45 | 46 | Not all models are supported as supported models need to support function calling. The models `llama3.1`, `mistral`, and `qwen2` do provide such support and others models will also work if they support function calling. If you have questions about other models, please contact support@franz.com. 47 | 48 | [AllegroGraph Ollama Documentation](https://franz.com/agraph/support/documentation/ollama.html) 49 | 50 | ### Installing Ollama 51 | 52 | To download `Ollama` please visit their [download page](https://ollama.com/) 53 | 54 | Once `Ollama` is installed, you can pull models using: 55 | 56 | ```shell 57 | ollama pull llama3.1:latest 58 | ``` 59 | 60 | Test that your model is working using: 61 | 62 | ```shell 63 | ollama run llama3.1:latest 64 | ``` 65 | 66 | Please start with [ollama-sparql-integration.ipynb](ollama-sparql-integration.ipynb) to learn how we've integrated Ollama with our SPARQL engine! 67 | -------------------------------------------------------------------------------- /llm/environment.yml: -------------------------------------------------------------------------------- 1 | name: ag-examples-llm 2 | channels: 3 | - franzinc 4 | - defaults 5 | dependencies: 6 | - _libgcc_mutex=0.1=main 7 | - _openmp_mutex=5.1=1_gnu 8 | - pip 9 | - pip: 10 | - "iso8601>=2.1.0,<2.2.0" 11 | - "agraph-python==104.2.0" 12 | - anyio=4.6.2=py310h06a4308_0 13 | - argon2-cffi=21.3.0=pyhd3eb1b0_0 14 | - argon2-cffi-bindings=21.2.0=py310h7f8727e_0 15 | - asttokens=2.0.5=pyhd3eb1b0_0 16 | - async-lru=2.0.4=py310h06a4308_0 17 | - attrs=24.2.0=py310h06a4308_0 18 | - babel=2.11.0=py310h06a4308_0 19 | - beautifulsoup4=4.12.3=py310h06a4308_0 20 | - blas=1.0=mkl 21 | - bleach=6.2.0=py310h06a4308_0 22 | - bottleneck=1.4.2=py310ha9d4c09_0 23 | - brotli-python=1.0.9=py310h6a678d5_8 24 | - bzip2=1.0.8=h5eee18b_6 25 | - ca-certificates=2024.11.26=h06a4308_0 26 | - certifi=2024.8.30=py310h06a4308_0 27 | - cffi=1.17.1=py310h1fdaa30_0 28 | - charset-normalizer=3.3.2=pyhd3eb1b0_0 29 | - comm=0.2.1=py310h06a4308_0 30 | - debugpy=1.6.7=py310h6a678d5_0 31 | - decorator=5.1.1=pyhd3eb1b0_0 32 | - defusedxml=0.7.1=pyhd3eb1b0_0 33 | - exceptiongroup=1.2.0=py310h06a4308_0 34 | - executing=0.8.3=pyhd3eb1b0_0 35 | - future=1.0.0=py310h06a4308_0 36 | - h11=0.14.0=py310h06a4308_0 37 | - httpcore=1.0.2=py310h06a4308_0 38 | - httpx=0.27.0=py310h06a4308_0 39 | - idna=3.7=py310h06a4308_0 40 | - intel-openmp=2023.1.0=hdb19cb5_46306 41 | - ipykernel=6.29.5=py310h06a4308_0 42 | - ipython=8.27.0=py310h06a4308_0 43 | - jedi=0.19.2=py310h06a4308_0 44 | - jinja2=3.1.4=py310h06a4308_1 45 | - json5=0.9.25=py310h06a4308_0 46 | - jsonschema=4.23.0=py310h06a4308_0 47 | - jsonschema-specifications=2023.7.1=py310h06a4308_0 48 | - jupyter-lsp=2.2.0=py310h06a4308_0 49 | - jupyter_client=8.6.0=py310h06a4308_0 50 | - jupyter_core=5.7.2=py310h06a4308_0 51 | - jupyter_events=0.10.0=py310h06a4308_0 52 | - jupyter_server=2.14.1=py310h06a4308_0 53 | - jupyter_server_terminals=0.4.4=py310h06a4308_1 54 | - jupyterlab=4.2.5=py310h06a4308_0 55 | - jupyterlab_pygments=0.1.2=py_0 56 | - jupyterlab_server=2.27.3=py310h06a4308_0 57 | - ld_impl_linux-64=2.40=h12ee557_0 58 | - libffi=3.4.4=h6a678d5_1 59 | - libgcc-ng=11.2.0=h1234567_1 60 | - libgomp=11.2.0=h1234567_1 61 | - libsodium=1.0.18=h7b6447c_0 62 | - libstdcxx-ng=11.2.0=h1234567_1 63 | - libuuid=1.41.5=h5eee18b_0 64 | - markupsafe=2.1.3=py310h5eee18b_0 65 | - matplotlib-inline=0.1.6=py310h06a4308_0 66 | - mistune=2.0.4=py310h06a4308_0 67 | - mkl=2023.1.0=h213fc3f_46344 68 | - mkl-service=2.4.0=py310h5eee18b_1 69 | - mkl_fft=1.3.11=py310h5eee18b_0 70 | - mkl_random=1.2.8=py310h1128e8f_0 71 | - nb_conda_kernels=2.5.2=py310h06a4308_0 72 | - nbclient=0.8.0=py310h06a4308_0 73 | - nbconvert=7.16.4=hd3eb1b0_1 74 | - nbconvert-core=7.16.4=py310h06a4308_1 75 | - nbconvert-pandoc=7.16.4=hd3eb1b0_1 76 | - nbformat=5.10.4=py310h06a4308_0 77 | - ncurses=6.4=h6a678d5_0 78 | - nest-asyncio=1.6.0=py310h06a4308_0 79 | - notebook-shim=0.2.3=py310h06a4308_0 80 | - numexpr=2.10.1=py310h3c60e43_0 81 | - numpy=2.0.1=py310h5f9d8c6_1 82 | - numpy-base=2.0.1=py310hb5e798b_1 83 | - openssl=3.0.15=h5eee18b_0 84 | - overrides=7.4.0=py310h06a4308_0 85 | - packaging=24.1=py310h06a4308_0 86 | - pandas=2.2.3=py310h6a678d5_0 87 | - pandoc=2.12=h06a4308_3 88 | - pandocfilters=1.5.0=pyhd3eb1b0_0 89 | - parso=0.8.4=py310h06a4308_0 90 | - pexpect=4.8.0=pyhd3eb1b0_3 91 | - pip=24.2=py310h06a4308_0 92 | - platformdirs=3.10.0=py310h06a4308_0 93 | - prometheus_client=0.21.0=py310h06a4308_0 94 | - prompt-toolkit=3.0.43=py310h06a4308_0 95 | - prompt_toolkit=3.0.43=hd3eb1b0_0 96 | - psutil=5.9.0=py310h5eee18b_0 97 | - ptyprocess=0.7.0=pyhd3eb1b0_2 98 | - pure_eval=0.2.2=pyhd3eb1b0_0 99 | - pycparser=2.21=pyhd3eb1b0_0 100 | - pygments=2.15.1=py310h06a4308_1 101 | - pysocks=1.7.1=py310h06a4308_0 102 | - python=3.10.16=he870216_1 103 | - python-dateutil=2.9.0post0=py310h06a4308_2 104 | - python-fastjsonschema=2.20.0=py310h06a4308_0 105 | - python-json-logger=3.2.1=py310h06a4308_0 106 | - python-tzdata=2023.3=pyhd3eb1b0_0 107 | - pytz=2024.1=py310h06a4308_0 108 | - pyyaml=6.0.2=py310h5eee18b_0 109 | - pyzmq=25.1.2=py310h6a678d5_0 110 | - readline=8.2=h5eee18b_0 111 | - referencing=0.30.2=py310h06a4308_0 112 | - requests=2.32.3=py310h06a4308_1 113 | - rfc3339-validator=0.1.4=py310h06a4308_0 114 | - rfc3986-validator=0.1.1=py310h06a4308_0 115 | - rpds-py=0.10.6=py310h4aa5aa6_1 116 | - send2trash=1.8.2=py310h06a4308_0 117 | - setuptools=75.1.0=py310h06a4308_0 118 | - six=1.16.0=pyhd3eb1b0_1 119 | - sniffio=1.3.0=py310h06a4308_0 120 | - soupsieve=2.5=py310h06a4308_0 121 | - sqlite=3.45.3=h5eee18b_0 122 | - stack_data=0.2.0=pyhd3eb1b0_0 123 | - tbb=2021.8.0=hdb19cb5_0 124 | - terminado=0.17.1=py310h06a4308_0 125 | - tinycss2=1.2.1=py310h06a4308_0 126 | - tk=8.6.14=h39e8969_0 127 | - tomli=2.0.1=py310h06a4308_0 128 | - tornado=6.4.2=py310h5eee18b_0 129 | - traitlets=5.14.3=py310h06a4308_0 130 | - typing-extensions=4.11.0=py310h06a4308_0 131 | - typing_extensions=4.11.0=py310h06a4308_0 132 | - tzdata=2024b=h04d1e81_0 133 | - urllib3=2.2.3=py310h06a4308_0 134 | - wcwidth=0.2.5=pyhd3eb1b0_0 135 | - webencodings=0.5.1=py310h06a4308_1 136 | - websocket-client=1.8.0=py310h06a4308_0 137 | - wheel=0.44.0=py310h06a4308_0 138 | - xz=5.4.6=h5eee18b_1 139 | - yaml=0.2.5=h7b6447c_0 140 | - zeromq=4.3.5=h6a678d5_0 141 | - zlib=1.2.13=h5eee18b_1 142 | 143 | -------------------------------------------------------------------------------- /llm/llm_utils.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from franz.openrdf.connect import ag_connect 3 | from typing import Optional 4 | 5 | def create_nlq_vdb( 6 | repo_name, 7 | conn, 8 | vdb_name, 9 | openai_api_key: str = '', 10 | host: str = 'localhost', 11 | port: str = '10035', 12 | user: str = 'test', 13 | password: str = 'xyzzy', 14 | protocol: str = 'http', 15 | embedder: str = 'openai', 16 | embedding_model: str = 'text-embedding-3-small' 17 | ): 18 | 19 | #add shacl to connection 20 | url = ( 21 | f"{protocol}://{user}:{password}@{host}:{port}/repositories/{repo_name}/data-generator/shacl" 22 | ) 23 | response = requests.get(url) 24 | 25 | # add shacl data to repo 26 | conn.addData(response.json()) 27 | 28 | # create vdb 29 | nlq_conn = ag_connect( 30 | vdb_name, 31 | clear=True, 32 | create=True, 33 | host=host, 34 | port=port, 35 | user=user, 36 | password=password, 37 | ) 38 | 39 | #convert to vector store 40 | nlq_conn.convert_to_vector_store( 41 | embedder, 42 | api_key=openai_api_key, 43 | model=embedding_model 44 | ) 45 | 46 | # create the connection between regular connection and vector store 47 | # create a linking repo (just connect if it already exists) 48 | connect_conn = ag_connect( 49 | "nlq-to-store-relationship", 50 | user=user, 51 | password=password, 52 | host=host, 53 | port=port 54 | ) 55 | 56 | # add linking data 57 | connect_conn.executeUpdate( 58 | f""" 59 | prefix gen: 60 | insert data {{ 61 | gen:catalog "/" ; 62 | gen:repository "{repo_name}" ; 63 | gen:nlq-vdb "{vdb_name}" . }} 64 | """ 65 | ) 66 | connect_conn.deleteDuplicates(mode="spo") 67 | connect_conn.close() 68 | 69 | return nlq_conn 70 | -------------------------------------------------------------------------------- /llm/requirements.txt: -------------------------------------------------------------------------------- 1 | agraph-python==104.2.0 2 | anyio==4.8.0 3 | argon2-cffi==23.1.0 4 | argon2-cffi-bindings==21.2.0 5 | arrow==1.3.0 6 | asttokens==3.0.0 7 | async-lru==2.0.4 8 | attrs==25.1.0 9 | babel==2.17.0 10 | beautifulsoup4==4.13.3 11 | bleach==6.2.0 12 | certifi==2025.1.31 13 | cffi==1.17.1 14 | charset-normalizer==3.4.1 15 | comm==0.2.2 16 | debugpy==1.8.12 17 | decorator==5.1.1 18 | defusedxml==0.7.1 19 | executing==2.2.0 20 | fastjsonschema==2.21.1 21 | fqdn==1.5.1 22 | h11==0.14.0 23 | httpcore==1.0.7 24 | httpx==0.28.1 25 | idna==3.10 26 | ipykernel==6.29.5 27 | ipython==8.32.0 28 | iso8601==2.1.0 29 | isoduration==20.11.0 30 | jedi==0.19.2 31 | Jinja2==3.1.5 32 | json5==0.10.0 33 | jsonpointer==3.0.0 34 | jsonschema==4.23.0 35 | jsonschema-specifications==2024.10.1 36 | jupyter-events==0.12.0 37 | jupyter-lsp==2.2.5 38 | jupyter_client==8.6.3 39 | jupyter_core==5.7.2 40 | jupyter_server==2.15.0 41 | jupyter_server_terminals==0.5.3 42 | jupyterlab==4.3.5 43 | jupyterlab_pygments==0.3.0 44 | jupyterlab_server==2.27.3 45 | MarkupSafe==3.0.2 46 | matplotlib-inline==0.1.7 47 | mistune==3.1.2 48 | nbclient==0.10.2 49 | nbconvert==7.16.6 50 | nbformat==5.10.4 51 | nest-asyncio==1.6.0 52 | notebook_shim==0.2.4 53 | numpy==2.2.3 54 | overrides==7.7.0 55 | packaging==24.2 56 | pandas==2.2.3 57 | pandocfilters==1.5.1 58 | parso==0.8.4 59 | pexpect==4.9.0 60 | platformdirs==4.3.6 61 | prometheus_client==0.21.1 62 | prompt_toolkit==3.0.50 63 | psutil==7.0.0 64 | ptyprocess==0.7.0 65 | pure_eval==0.2.3 66 | pycparser==2.22 67 | Pygments==2.19.1 68 | PySocks==1.7.1 69 | python-dateutil==2.9.0.post0 70 | python-json-logger==3.2.1 71 | pytz==2025.1 72 | PyYAML==6.0.2 73 | pyzmq==26.2.1 74 | referencing==0.36.2 75 | requests==2.32.3 76 | rfc3339-validator==0.1.4 77 | rfc3986-validator==0.1.1 78 | rpds-py==0.22.3 79 | Send2Trash==1.8.3 80 | six==1.17.0 81 | sniffio==1.3.1 82 | soupsieve==2.6 83 | stack-data==0.6.3 84 | terminado==0.18.1 85 | tinycss2==1.4.0 86 | tornado==6.4.2 87 | traitlets==5.14.3 88 | types-python-dateutil==2.9.0.20241206 89 | typing_extensions==4.12.2 90 | tzdata==2025.1 91 | uri-template==1.3.0 92 | urllib3==2.3.0 93 | wcwidth==0.2.13 94 | webcolors==24.11.1 95 | webencodings==0.5.1 96 | websocket-client==1.8.0 97 | -------------------------------------------------------------------------------- /ontop/Dockerfile.bootstrap: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-slim-buster 2 | LABEL maintainer="Tianyu Gu " 3 | 4 | RUN apt-get update -yq && apt-get upgrade -yq 5 | RUN apt-get install -yq curl zip unzip 6 | 7 | WORKDIR /opt 8 | ARG ONTOP_VERSION=4.1.0 9 | RUN curl -L https://github.com/ontop/ontop/releases/download/ontop-${ONTOP_VERSION}/ontop-cli-${ONTOP_VERSION}.zip > /tmp/ontop-cli.zip 10 | RUN unzip /tmp/ontop-cli.zip 11 | RUN curl -L https://jdbc.postgresql.org/download/postgresql-42.2.20.jar > jdbc/postgresql-42.2.20.jar 12 | RUN mkdir -p /var/ontop 13 | ADD bin/demo.properties /var 14 | 15 | # boostrap ontology and mapping 16 | ENV BASE_IRI="http://franz.com/ontop-demo/" 17 | CMD /opt/ontop bootstrap \ 18 | --base-iri "${BASE_IRI}" \ 19 | --properties /var/demo.properties \ 20 | --mapping /var/ontop/demo.mapping.obda \ 21 | --ontology /var/ontop/demo.ontology.ttl 22 | -------------------------------------------------------------------------------- /ontop/Dockerfile.endpoint: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-slim-buster 2 | LABEL maintainer="Tianyu Gu " 3 | 4 | RUN apt-get update -yq && apt-get upgrade -yq 5 | RUN apt-get install -yq curl zip unzip 6 | 7 | WORKDIR /opt 8 | ARG ONTOP_VERSION=4.1.0 9 | RUN curl -L https://github.com/ontop/ontop/releases/download/ontop-${ONTOP_VERSION}/ontop-cli-${ONTOP_VERSION}.zip > /tmp/ontop-cli.zip 10 | RUN unzip /tmp/ontop-cli.zip 11 | RUN curl -L https://jdbc.postgresql.org/download/postgresql-42.2.20.jar > jdbc/postgresql-42.2.20.jar 12 | RUN mkdir -p /var/ontop 13 | ADD bin/demo.properties /var 14 | 15 | # endpoint 16 | ENV ONTOP_JAVA_ARGS="-Xms512m -Xmx1g" 17 | CMD /opt/ontop endpoint \ 18 | --properties /var/demo.properties \ 19 | --mapping /var/ontop/demo.mapping.obda \ 20 | --ontology /var/ontop/demo.ontology.ttl \ 21 | --port 8088 \ 22 | --cors-allowed-origins="*" 23 | -------------------------------------------------------------------------------- /ontop/Dockerfile.materialize: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-slim-buster 2 | LABEL maintainer="Tianyu Gu " 3 | 4 | RUN apt-get update -yq && apt-get upgrade -yq 5 | RUN apt-get install -yq curl zip unzip 6 | 7 | WORKDIR /opt 8 | ARG ONTOP_VERSION=4.1.0 9 | RUN curl -L https://github.com/ontop/ontop/releases/download/ontop-${ONTOP_VERSION}/ontop-cli-${ONTOP_VERSION}.zip > /tmp/ontop-cli.zip 10 | RUN unzip /tmp/ontop-cli.zip 11 | RUN curl -L https://jdbc.postgresql.org/download/postgresql-42.2.20.jar > jdbc/postgresql-42.2.20.jar 12 | RUN mkdir -p /var/ontop 13 | ADD bin/demo.properties /var 14 | ADD bin/demo.mini-mapping.obda /var 15 | 16 | # materialize 17 | ENV ONTOP_JAVA_ARGS="-Xms512m -Xmx1g" 18 | CMD /opt/ontop materialize \ 19 | --properties /var/demo.properties \ 20 | --mapping /var/demo.mini-mapping.obda \ 21 | --disable-reasoning \ 22 | --format ntriples \ 23 | --output /var/ontop/demo.materialized.nt 24 | -------------------------------------------------------------------------------- /ontop/Dockerfile.pg: -------------------------------------------------------------------------------- 1 | FROM postgres:14 2 | LABEL maintainer="Tianyu Gu " 3 | 4 | WORKDIR /tmp 5 | 6 | RUN apt-get -qq update -y && apt-get upgrade -y && \ 7 | apt-get install -y curl zip unzip 8 | 9 | ARG demo_size=small 10 | RUN curl -L https://edu.postgrespro.com/demo-${demo_size}-en.zip > demo-${demo_size}-en.zip && \ 11 | unzip demo-${demo_size}-en.zip && rm demo-${demo_size}-en.zip 12 | 13 | COPY bin/init-db.sh /docker-entrypoint-initdb.d/ 14 | RUN chmod a+x /docker-entrypoint-initdb.d/init-db.sh 15 | -------------------------------------------------------------------------------- /ontop/bin/demo.mini-mapping.obda: -------------------------------------------------------------------------------- 1 | [PrefixDeclaration] 2 | r: http://franz.com/ontop-demo/ 3 | p: http://franz.com/ontop-demo/Property# 4 | rdf: http://www.w3.org/1999/02/22-rdf-syntax-ns# 5 | rdfs: http://www.w3.org/2000/01/rdf-schema# 6 | owl: http://www.w3.org/2002/07/owl# 7 | xsd: http://www.w3.org/2001/XMLSchema# 8 | obda: https://w3id.org/obda/vocabulary# 9 | 10 | [MappingDeclaration] @collection [[ 11 | 12 | mappingId MAPPING-Seats 13 | target r:Seat_{aircraft_code}_{seat_no} a r:Seat ; p:aircraft_code {aircraft_code}^^xsd:string ; p:seat_no {seat_no}^^xsd:string ; p:seat_fare_conditions {fare_conditions}^^xsd:string . 14 | source SELECT * FROM "seats" 15 | 16 | mappingId MAPPING-Aircraft 17 | target _:ontop-bnode-1{aircraft_code}/{model}/{range} a r:Aircraft ; p:aircraft_code {aircraft_code}^^xsd:string ; p:aircraft_model {model}^^xsd:string ; p:aircraft_range {range}^^xsd:integer . 18 | source SELECT * FROM "aircrafts" 19 | 20 | ]] 21 | -------------------------------------------------------------------------------- /ontop/bin/demo.properties: -------------------------------------------------------------------------------- 1 | jdbc.url=jdbc\:postgresql\://127.0.0.1\:5432/demo 2 | jdbc.driver=org.postgresql.Driver 3 | jdbc.user=test 4 | jdbc.password=xyzzy 5 | -------------------------------------------------------------------------------- /ontop/bin/init-db.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | ls -1 /tmp | 5 | grep "sql" | 6 | xargs -P 0 -d '\n' -I{} sh -c "psql -d \"$POSTGRES_DB\" -U \"$POSTGRES_USER\" -a -q -f {}" 7 | -------------------------------------------------------------------------------- /ontop/img/query.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/ontop/img/query.jpg -------------------------------------------------------------------------------- /ontop/img/schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/ontop/img/schema.png -------------------------------------------------------------------------------- /ontop/ontop_output/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/ontop/ontop_output/.gitkeep -------------------------------------------------------------------------------- /spark/.docker/Dockerfile.jupyter: -------------------------------------------------------------------------------- 1 | FROM bitnami/spark:3.1.2 2 | 3 | USER root 4 | RUN apt-get update -y && apt-get install -y libyaml-dev 5 | RUN python3.6 -m pip install -U pip 6 | RUN python3.6 -m pip install --no-cache-dir scikit-learn pandas jupyterlab matplotlib requests pyyaml agraph-python 7 | 8 | RUN mkdir /root/work 9 | -------------------------------------------------------------------------------- /spark/.docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | spark-master: 5 | image: docker.io/bitnami/spark:3.1.2 6 | user: 0:0 7 | environment: 8 | - SPARK_MODE=master 9 | - SPARK_RPC_AUTHENTICATION_ENABLED=no 10 | - SPARK_RPC_ENCRYPTION_ENABLED=no 11 | - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no 12 | - SPARK_SSL_ENABLED=no 13 | ports: 14 | - '8080:8080' 15 | spark-worker-1: 16 | image: docker.io/bitnami/spark:3.1.2 17 | user: 0:0 18 | environment: 19 | - SPARK_MODE=worker 20 | - SPARK_MASTER_URL=spark://spark-master:7077 21 | - SPARK_WORKER_MEMORY=8G 22 | - SPARK_WORKER_CORES=4 23 | - SPARK_RPC_AUTHENTICATION_ENABLED=no 24 | - SPARK_RPC_ENCRYPTION_ENABLED=no 25 | - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no 26 | - SPARK_SSL_ENABLED=no 27 | spark-worker-2: 28 | image: docker.io/bitnami/spark:3.1.2 29 | user: 0:0 30 | environment: 31 | - SPARK_MODE=worker 32 | - SPARK_MASTER_URL=spark://spark-master:7077 33 | - SPARK_WORKER_MEMORY=8G 34 | - SPARK_WORKER_CORES=4 35 | - SPARK_RPC_AUTHENTICATION_ENABLED=no 36 | - SPARK_RPC_ENCRYPTION_ENABLED=no 37 | - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no 38 | - SPARK_SSL_ENABLED=no 39 | 40 | jupyter: 41 | build: 42 | context: . 43 | dockerfile: Dockerfile.jupyter 44 | command: pyspark --master spark://spark-master:7077 --packages graphframes:graphframes:0.8.1-spark3.0-s_2.12 --repositories https://repos.spark-packages.org/ 45 | container_name: ag-spark-demo-jupyter 46 | user: 0:0 47 | depends_on: 48 | - spark-master 49 | - spark-worker-1 50 | - spark-worker-2 51 | ports: 52 | - "8088:8888" 53 | environment: 54 | - JUPYTER_ENABLE_LAB=1 55 | - PYSPARK_DRIVER_PYTHON=jupyter 56 | - PYSPARK_DRIVER_PYTHON_OPTS=lab --ip 0.0.0.0 --ServerApp.token='' -y --no-browser --allow-root --ServerApp.notebook_dir="/root/work" 57 | 58 | ag: 59 | image: franzinc/agraph:v7.1.0 60 | container_name: ag-spark-demo-ag 61 | ports: 62 | - "10015:10035" 63 | shm_size: 1g 64 | environment: 65 | - AGRAPH_SUPER_USER=test 66 | - AGRAPH_SUPER_PASSWORD=xyzzy 67 | -------------------------------------------------------------------------------- /spark/Makefile: -------------------------------------------------------------------------------- 1 | ## 2 | # Makefile for starting and stopping the whole demo ... 3 | # including a Spark cluster, an AllegroGraph instance and a Jupyter Notebook server 4 | # 5 | # @file 6 | # @version 0.1 7 | 8 | AG_CONTAINER_NAME=ag-spark-demo-ag 9 | JUPYTER_CONTAINER_NAME=ag-spark-demo-jupyter 10 | 11 | .DEFAULT_GOAL: demo 12 | 13 | .PHONY: demo 14 | demo: 15 | @docker-compose --file .docker/docker-compose.yml up --build --detach --remove-orphans 16 | @echo "Importing kennedy.ntriples ..." 17 | @docker cp .docker/kennedy.ntriples $(AG_CONTAINER_NAME):/tmp 18 | @docker exec $(AG_CONTAINER_NAME) sh -c "agtool load kennedy /tmp/kennedy.ntriples && rm /tmp/kennedy.ntriples" 19 | @echo "Importing jupyter notebook data ..." 20 | @docker cp AGSpark.ipynb $(JUPYTER_CONTAINER_NAME):/root/work 21 | @docker exec $(JUPYTER_CONTAINER_NAME) sh -c "mkdir -p /root/work/img" 22 | @docker cp img/kennedy.png $(JUPYTER_CONTAINER_NAME):/root/work/img 23 | 24 | .PHONY: stop 25 | stop: 26 | @docker-compose --file .docker/docker-compose.yml down --timeout 0 27 | 28 | # end 29 | -------------------------------------------------------------------------------- /spark/README.md: -------------------------------------------------------------------------------- 1 | # Graph Analytics by AllegroGraph and [Apache Spark](https://spark.apache.org/) 2 | 3 | This example shows how to read data from AllegroGraph and then perform graph analytics by [Apache Spark](https://spark.apache.org/). Please start by the Jupyter Notebook `AGSpark.ipynb` from the current directory. 4 | 5 | If you want to run the example interactively, please install **Docker** and **docker-compose**. In the currently directory, run `make` (or `make demo`) to start a small Spark cluster (one master and two workers), an AllegroGraph instance and the Jupyter Notebook server: 6 | 7 | 1. visit [localhost:8080](localhost:8080) for the Spark cluster Web UI 8 | 2. visit [localhost:10015](localhost:10015) for the AllegroGraph WebView 9 | 3. visit [localhost:8088](localhost:8088) for the Jupyter Notebook server 10 | -------------------------------------------------------------------------------- /spark/img/kennedy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/spark/img/kennedy.png -------------------------------------------------------------------------------- /streaming/kafka/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea 3 | *.log 4 | tmp/ 5 | 6 | *.py[cod] 7 | *.egg 8 | build 9 | htmlcov 10 | -------------------------------------------------------------------------------- /streaming/kafka/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8.10-slim-buster AS builder 2 | 3 | ADD streaming /src/streaming 4 | ADD pyproject.toml /src/ 5 | ADD poetry.lock /src/ 6 | 7 | USER root 8 | WORKDIR /src 9 | RUN pip install -U --no-cache-dir --quiet pip setuptools 10 | RUN pip install --no-cache-dir --quiet 'poetry<2.0.0' 11 | RUN poetry build --format wheel 12 | 13 | FROM python:3.8.10-slim-buster AS runner 14 | COPY --from=builder /src/dist/streaming-0.1-py3-none-any.whl /tmp 15 | RUN pip install --quiet /tmp/streaming-0.1-py3-none-any.whl && rm /tmp/streaming-0.1-py3-none-any.whl 16 | 17 | ENV AGRAPH_HOST="localhost" 18 | ENV AGRAPH_PORT="10035" 19 | ENV AGRAPH_USER="test" 20 | ENV AGRAPH_PASSWORD="xyzzy" 21 | ENV AGRAPH_REPO="streaming-demo" 22 | ENV NUM_POSTS="100" 23 | 24 | ENTRYPOINT [ "streaming" ] 25 | -------------------------------------------------------------------------------- /streaming/kafka/Makefile: -------------------------------------------------------------------------------- 1 | ## 2 | # Basic streaming with Kafka 3 | # 4 | # @file 5 | # @version 0.1 6 | 7 | DOCKER=docker 8 | DOCKER_COMPOSE=docker-compose 9 | AG_CONTAINER=ag-kafka-basic_ag 10 | AG_KAFKA_BASIC_IMAGE=ag-kafka-basic_main 11 | PRODUCER_CONTAINER=ag-kafka-basic_producer 12 | AGENT_CONTAINER=ag-kafka-basic_agent 13 | 14 | AGRAPH_VERSION=7.1.0 15 | AGRAPH_HOST=localhost 16 | AGRAPH_PORT=10055 17 | AGRAPH_USER=test 18 | AGRAPH_PASSWORD=xyzzy 19 | AGRAPH_REPO=kafka-basic 20 | 21 | NUM_POSTS=10000 22 | 23 | build: 24 | @$(DOCKER) build -t $(AG_KAFKA_BASIC_IMAGE) -f Dockerfile . 25 | 26 | kafka: 27 | @$(DOCKER_COMPOSE) up --detach --remove-orphans 28 | 29 | stop_kafka: 30 | @$(DOCKER_COMPOSE) down 31 | 32 | ag: build 33 | @$(DOCKER) pull franzinc/agraph:v$(AGRAPH_VERSION) 34 | @$(DOCKER) container inspect $(AG_CONTAINER) >/dev/null 2>&1 || ($(DOCKER) run --detach --rm \ 35 | -p $(AGRAPH_PORT):10035 --shm-size 1g \ 36 | --name $(AG_CONTAINER) \ 37 | --env AGRAPH_SUPER_USER=$(AGRAPH_USER) \ 38 | --env AGRAPH_SUPER_PASSWORD=$(AGRAPH_PASSWORD) \ 39 | franzinc/agraph:v$(AGRAPH_VERSION) && \ 40 | $(DOCKER) run --rm --network host \ 41 | --env AGRAPH_HOST=$(AGRAPH_HOST) \ 42 | --env AGRAPH_PORT=$(AGRAPH_PORT) \ 43 | --env AGRAPH_USER=$(AGRAPH_USER) \ 44 | --env AGRAPH_PASSWORD=$(AGRAPH_PASSWORD) \ 45 | --env AGRAPH_REPO=$(AGRAPH_REPO) \ 46 | $(AG_KAFKA_BASIC_IMAGE) init_repo) 47 | 48 | stop_ag: 49 | @$(DOCKER) container inspect -f '{{.State.Running}}' $(AG_CONTAINER) && $(DOCKER) stop $(AG_CONTAINER) -t 0 50 | 51 | producer: build 52 | @$(DOCKER) run --detach --rm --env NUM_POSTS=$(NUM_POSTS) --network host \ 53 | --name $(PRODUCER_CONTAINER) $(AG_KAFKA_BASIC_IMAGE) producer 54 | 55 | stop_producer: 56 | @$(DOCKER) stop $(PRODUCER_CONTAINER) -t 0 57 | 58 | agent: build 59 | @$(DOCKER) run --detach --rm \ 60 | --env AGRAPH_HOST=$(AGRAPH_HOST) \ 61 | --env AGRAPH_PORT=$(AGRAPH_PORT) \ 62 | --env AGRAPH_USER=$(AGRAPH_USER) \ 63 | --env AGRAPH_PASSWORD=$(AGRAPH_PASSWORD) \ 64 | --env AGRAPH_REPO=$(AGRAPH_REPO) \ 65 | --network host --name $(AGENT_CONTAINER) $(AG_KAFKA_BASIC_IMAGE) agent 66 | 67 | stop_agent: 68 | @$(DOCKER) stop $(AGENT_CONTAINER) -t 0 69 | 70 | stop: stop_kafka stop_ag stop_agent stop_producer 71 | 72 | .PHONY: build kafka stop_kafka ag stop_ag producer stop_producer agent stop_agent stop 73 | 74 | # end 75 | -------------------------------------------------------------------------------- /streaming/kafka/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | services: 3 | kafdrop: 4 | image: obsidiandynamics/kafdrop 5 | restart: "no" 6 | ports: 7 | - "9000:9000" 8 | environment: 9 | KAFKA_BROKERCONNECT: "kafka:29092" 10 | JVM_OPTS: "-Xms16M -Xmx48M -Xss180K -XX:-TieredCompilation -XX:+UseStringDeduplication -noverify" 11 | depends_on: 12 | - "kafka" 13 | kafka: 14 | image: obsidiandynamics/kafka 15 | restart: "no" 16 | ports: 17 | - "2181:2181" 18 | - "9092:9092" 19 | environment: 20 | KAFKA_LISTENERS: "INTERNAL://:29092,EXTERNAL://:9092" 21 | KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka:29092,EXTERNAL://localhost:9092" 22 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT" 23 | KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL" 24 | KAFKA_ZOOKEEPER_SESSION_TIMEOUT: "6000" 25 | KAFKA_RESTART_ATTEMPTS: "10" 26 | KAFKA_RESTART_DELAY: "5" 27 | ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL: "0" 28 | -------------------------------------------------------------------------------- /streaming/kafka/img/apache_kafka.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/streaming/kafka/img/apache_kafka.png -------------------------------------------------------------------------------- /streaming/kafka/img/data_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/streaming/kafka/img/data_model.png -------------------------------------------------------------------------------- /streaming/kafka/img/diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/streaming/kafka/img/diagram.png -------------------------------------------------------------------------------- /streaming/kafka/img/kafka_web_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/streaming/kafka/img/kafka_web_ui.png -------------------------------------------------------------------------------- /streaming/kafka/img/kafka_web_ui_posts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/streaming/kafka/img/kafka_web_ui_posts.png -------------------------------------------------------------------------------- /streaming/kafka/img/kafka_web_ui_recommendations.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/streaming/kafka/img/kafka_web_ui_recommendations.png -------------------------------------------------------------------------------- /streaming/kafka/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "streaming" 3 | version = "0.1" 4 | description = "A basic package demostrating how to integrate Kafka with AllegroGraph." 5 | authors = ["Tianyu Gu "] 6 | 7 | [tool.poetry.dependencies] 8 | python = ">=3.8,<3.13" 9 | bidict = "^0.21.2" 10 | confluent-kafka = "^1.7.0" 11 | tqdm = "^4.61.0" 12 | agraph-python = "^104.0.0" 13 | essential-generators = "1.0" 14 | rdflib = "^5.0.0" 15 | importlib-resources = "^5.1.4" 16 | 17 | [tool.poetry.dev-dependencies] 18 | ipython = "^8.10.0" 19 | black = "^24.3" 20 | 21 | [tool.poetry.scripts] 22 | streaming = 'streaming.cli:main' 23 | 24 | [build-system] 25 | requires = ["poetry-core>=1.0.0"] 26 | build-backend = "poetry.core.masonry.api" 27 | 28 | [tool.black] 29 | line-length = 88 30 | target-version = ['py38', 'py39', 'py310', 'py311', 'py312'] 31 | include = '\.pyi?$' 32 | extend-exclude = ''' 33 | /( 34 | # The following are specific to Black, you probably don't want those. 35 | | blib2to3 36 | | tests/data 37 | | profiling 38 | )/ 39 | ''' 40 | -------------------------------------------------------------------------------- /streaming/kafka/streaming/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.1" 2 | -------------------------------------------------------------------------------- /streaming/kafka/streaming/agent.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | from string import Template 5 | 6 | from confluent_kafka import Consumer, Producer 7 | from franz.openrdf.connect import ag_connect 8 | from franz.openrdf.repository.repositoryconnection import RepositoryConnection 9 | from franz.openrdf.rio.rdfformat import RDFFormat 10 | 11 | from .model import Post 12 | 13 | logger = logging.getLogger(__name__) 14 | logger.setLevel(logging.INFO) 15 | ch = logging.StreamHandler() 16 | formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") 17 | ch.setLevel(logging.INFO) 18 | ch.setFormatter(formatter) 19 | logger.addHandler(ch) 20 | 21 | 22 | AG_ENV = { 23 | "host": os.environ.get("AGRAPH_HOST", "localhost"), 24 | "port": os.environ.get("AGRAPH_PORT", "10035"), 25 | "user": os.environ.get("AGRAPH_USER", "test"), 26 | "password": os.environ.get("AGRAPH_PASSWORD", "xyzzy"), 27 | } 28 | 29 | 30 | _RQ_TEMPLATE = Template( 31 | """PREFIX : 32 | PREFIX sna: 33 | 34 | SELECT DISTINCT ?user WHERE { 35 | ?group sna:egoGroup (:connectedPosts :$pid $depth) . 36 | ?post sna:members ?group ; :postedBy ?user . 37 | FILTER (?post != :$pid) 38 | } LIMIT $limit""" 39 | ) 40 | 41 | 42 | def recommend_users(ag_conn: RepositoryConnection, post: Post, depth=6, limit=10): 43 | data = {"pid": post.pid, "posted_by": post.posted_by, "recommend_users": []} 44 | query = _RQ_TEMPLATE.substitute(pid=post.pid, depth=depth, limit=limit) 45 | with ag_conn.prepareTupleQuery(query=query).evaluate() as res: 46 | for binding_set in res: 47 | user_uri = binding_set.getValue("user") 48 | data["recommend_users"].append(user_uri.localname) 49 | return data 50 | 51 | 52 | def delivered_cb(err, msg): 53 | if err is not None: 54 | logger.error("Message delivery failed: {}".format(err)) 55 | else: 56 | logger.info("Message delivered to topic <{}>".format(msg.topic())) 57 | 58 | 59 | def main(): 60 | ag_conn = ag_connect(os.environ["AGRAPH_REPO"], create=False, clear=False, **AG_ENV) 61 | ag_conn.add_commit_size = 1000 62 | logger.info( 63 | "AllegroGraph connected. ADD_COMMIT_SIZE: {}".format(ag_conn.add_commit_size) 64 | ) 65 | 66 | producer = Producer({"bootstrap.servers": "localhost:9092"}) 67 | 68 | consumer = Consumer( 69 | { 70 | "bootstrap.servers": "localhost:9092", 71 | "group.id": "agStreamingDemo", 72 | "auto.offset.reset": "latest", 73 | } 74 | ) 75 | consumer.subscribe(["posts"]) 76 | logger.info("Topic subscribed.") 77 | 78 | while True: 79 | msg = consumer.poll(1.0) 80 | if msg is None: 81 | continue 82 | if msg.error(): 83 | logger.error("Consumer error: {}".format(msg.error())) 84 | continue 85 | 86 | post = Post(**json.loads(msg.value().decode("utf-8"))) 87 | logger.info("Message of {} consumed.".format(post)) 88 | 89 | # 1. Sink 90 | ag_conn.addData( 91 | post.to_nt(base="http://franz.com/examples/"), rdf_format=RDFFormat.NTRIPLES 92 | ) 93 | logger.info( 94 | "NTriples data added for {}. Repo size: {}.".format(post, ag_conn.size()) 95 | ) 96 | # 2. Recommend 97 | data = recommend_users(ag_conn, post) 98 | if data["recommend_users"]: 99 | producer.produce( 100 | "recommendations", 101 | json.dumps(data).encode("utf-8"), 102 | callback=delivered_cb, 103 | ) 104 | logger.info( 105 | "User recommendations of {} have been sent to topic <{}>".format( 106 | post, "recommendations" 107 | ) 108 | ) 109 | 110 | # consumer.close() 111 | -------------------------------------------------------------------------------- /streaming/kafka/streaming/cli.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | 4 | def main(): 5 | if sys.argv[1] == "init_repo": 6 | from .init_repo import main 7 | 8 | main() 9 | 10 | elif sys.argv[1] == "producer": 11 | from .producer import main 12 | 13 | main() 14 | 15 | elif sys.argv[1] == "agent": 16 | from .agent import main 17 | 18 | main() 19 | 20 | else: 21 | sys.exit( 22 | f"Unknow subcommand: {sys.argv[1]}, must be one of these: init_repo, producer, agent." 23 | ) 24 | -------------------------------------------------------------------------------- /streaming/kafka/streaming/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/streaming/kafka/streaming/data/__init__.py -------------------------------------------------------------------------------- /streaming/kafka/streaming/data/data.nt.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/streaming/kafka/streaming/data/data.nt.bz2 -------------------------------------------------------------------------------- /streaming/kafka/streaming/data/sample_posts_idx.txt.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/streaming/kafka/streaming/data/sample_posts_idx.txt.bz2 -------------------------------------------------------------------------------- /streaming/kafka/streaming/data/sample_users_idx.txt.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/streaming/kafka/streaming/data/sample_users_idx.txt.bz2 -------------------------------------------------------------------------------- /streaming/kafka/streaming/init_repo.py: -------------------------------------------------------------------------------- 1 | import bz2 2 | import logging 3 | import os 4 | import sys 5 | 6 | import importlib_resources 7 | from franz.openrdf.connect import ag_connect 8 | from franz.openrdf.rio.rdfformat import RDFFormat 9 | 10 | from . import data as module_data 11 | 12 | logger = logging.getLogger(__name__) 13 | logger.setLevel(logging.INFO) 14 | ch = logging.StreamHandler() 15 | formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") 16 | ch.setLevel(logging.INFO) 17 | ch.setFormatter(formatter) 18 | logger.addHandler(ch) 19 | 20 | SNA_RQ = """PREFIX : 21 | PREFIX sna: 22 | 23 | DELETE { graph sna:sna { ?id ?p ?o }} 24 | WHERE { 25 | GRAPH sna:sna { 26 | ?id a sna:Generator ; 27 | sna:hasName :connectedPosts ; 28 | ?p ?o . 29 | } 30 | }; 31 | 32 | INSERT data { 33 | GRAPH sna:sna { 34 | [ a sna:Generator ; 35 | sna:hasName :connectedPosts ; 36 | sna:undirected ( :retweetOf :replyTo ) ; 37 | ] 38 | } 39 | }""" 40 | 41 | AG_ENV = { 42 | "host": os.environ.get("AGRAPH_HOST", "localhost"), 43 | "port": os.environ.get("AGRAPH_PORT", "10035"), 44 | "user": os.environ.get("AGRAPH_USER", "test"), 45 | "password": os.environ.get("AGRAPH_PASSWORD", "xyzzy"), 46 | } 47 | 48 | 49 | def main(): 50 | repo = os.environ.get("AGRAPH_REPO", "stremaing-demo") 51 | with ag_connect( 52 | repo, 53 | **AG_ENV, 54 | create=True, 55 | clear=True, 56 | ) as conn: 57 | logger.info("AllegroGraph repo '%s' connected" % repo) 58 | data = bz2.decompress(importlib_resources.read_binary(module_data, "data.nt.bz2")).decode("utf-8") 59 | conn.addData(data, rdf_format=RDFFormat.NTRIPLES) 60 | logger.info("Triples added to the repo") 61 | 62 | flag = conn.prepareUpdate(query=SNA_RQ).evaluate() 63 | if flag: 64 | logger.info("SNA clauses successfully added") 65 | else: 66 | logger.error("Failed to insert SNA clauses\nAbort!") 67 | sys.exit(1) 68 | 69 | conn.deleteDuplicates("spo") 70 | conn.commit() 71 | logger.info( 72 | "Repo is ready at http://%s:%s/#/repositories/%s" 73 | % ( 74 | AG_ENV["host"], 75 | AG_ENV["port"], 76 | repo, 77 | ) 78 | ) 79 | 80 | 81 | if __name__ == "__main__": 82 | main() 83 | -------------------------------------------------------------------------------- /streaming/kafka/streaming/model.py: -------------------------------------------------------------------------------- 1 | import bz2 2 | import json 3 | import random 4 | import uuid 5 | from dataclasses import dataclass 6 | from typing import Dict, List, Optional 7 | 8 | import importlib_resources 9 | from essential_generators import DocumentGenerator 10 | from rdflib import Graph, Literal, Namespace 11 | from rdflib.namespace import RDF, RDFS 12 | 13 | from . import data as module_data 14 | 15 | _GEN = DocumentGenerator() 16 | 17 | 18 | def load_sample_idx(fname: str) -> List[str]: 19 | c = importlib_resources.read_binary(module_data, fname) 20 | data = bz2.decompress(c).decode("utf-8") 21 | return [l for l in data.split("\n") if l] 22 | 23 | 24 | @dataclass 25 | class User: 26 | uid: str 27 | name: str 28 | description: str 29 | 30 | def __hash__(self) -> int: 31 | return hash(self.uid) 32 | 33 | def __repr__(self) -> str: 34 | return f"<{self.uid}>" 35 | 36 | def to_json(self) -> Dict: 37 | data = { 38 | "uid": self.uid, 39 | "name": self.name, 40 | "description": self.description, 41 | } 42 | return json.dumps(data) 43 | 44 | def to_nt(self, base: str) -> str: 45 | ns = Namespace(base) 46 | u = ns[self.uid] 47 | g = Graph() 48 | g.add((u, RDF.type, ns["User"])) 49 | g.add((u, RDFS.label, Literal(self.name))) 50 | g.add((u, ns.name, Literal(self.name))) 51 | g.add((u, ns.description, Literal(self.description))) 52 | return g.serialize(format="nt").decode("utf-8") 53 | 54 | 55 | @dataclass 56 | class Post: 57 | pid: str 58 | content: str 59 | posted_by: str 60 | retweet_of: Optional[str] 61 | reply_to: Optional[str] 62 | 63 | def __hash__(self) -> int: 64 | return hash(self.pid) 65 | 66 | def __repr__(self) -> str: 67 | return f"<{self.pid}>" 68 | 69 | def to_json(self) -> Dict: 70 | data = { 71 | "pid": self.pid, 72 | "content": self.content, 73 | "posted_by": self.posted_by, 74 | "retweet_of": self.retweet_of if self.retweet_of else None, 75 | "reply_to": self.reply_to if self.reply_to else None, 76 | } 77 | return json.dumps(data) 78 | 79 | def to_nt(self, base: str) -> str: 80 | ns = Namespace(base) 81 | p = ns[self.pid] 82 | g = Graph() 83 | g.add((p, RDF.type, ns["Post"])) 84 | g.add((p, ns.content, Literal(self.content))) 85 | g.add((p, ns.postedBy, ns[self.posted_by])) 86 | if self.retweet_of: 87 | g.add((p, ns.retweetOf, ns[self.retweet_of])) 88 | if self.reply_to: 89 | g.add((p, ns.replyTo, ns[self.reply_to])) 90 | return g.serialize(format="nt").decode("utf-8") 91 | 92 | @staticmethod 93 | def generate(sample_users_idx: List[str], sample_posts_idx: List[str]) -> "Post": 94 | data = { 95 | "pid": f"post-{uuid.uuid4()}", 96 | "content": _GEN.sentence(), 97 | "posted_by": random.choice(sample_users_idx), 98 | "retweet_of": None, 99 | "reply_to": None, 100 | } 101 | prob = random.uniform(0, 1) 102 | if prob < 0.5: 103 | return Post(**data) 104 | elif prob < 0.75: 105 | data["retweet_of"] = random.choice(sample_posts_idx) 106 | else: 107 | data["reply_to"] = random.choice(sample_posts_idx) 108 | return Post(**data) 109 | -------------------------------------------------------------------------------- /streaming/kafka/streaming/producer.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import random 4 | import time 5 | 6 | from confluent_kafka import Producer 7 | 8 | from .model import Post, load_sample_idx 9 | 10 | logger = logging.getLogger(__name__) 11 | logger.setLevel(logging.INFO) 12 | ch = logging.StreamHandler() 13 | formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") 14 | ch.setLevel(logging.INFO) 15 | ch.setFormatter(formatter) 16 | logger.addHandler(ch) 17 | 18 | 19 | def delivered_cb(err, msg): 20 | if err is not None: 21 | logger.error("Message delivery failed: {}".format(err)) 22 | else: 23 | logger.info("Message delivered to topic <{}>".format(msg.topic())) 24 | 25 | 26 | def main(): 27 | sample_users_idx = load_sample_idx("sample_users_idx.txt.bz2") 28 | sample_posts_idx = load_sample_idx("sample_posts_idx.txt.bz2") 29 | producer = Producer({"bootstrap.servers": "localhost:9092"}) 30 | topic = "posts" 31 | limit = int(os.environ.get("NUM_POSTS", 100)) 32 | for _ in range(0, limit): 33 | producer.poll(0) 34 | post = Post.generate(sample_users_idx, sample_posts_idx) 35 | data = post.to_json().encode("utf-8") 36 | producer.produce(topic, data, callback=delivered_cb) 37 | time.sleep(random.randint(1, 3)) 38 | -------------------------------------------------------------------------------- /tutorial-files/PhoneCalls.ttl.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/franzinc/agraph-examples/652b46a1d4cf284c25f7b6f39a7a510732c54c33/tutorial-files/PhoneCalls.ttl.gz -------------------------------------------------------------------------------- /tutorial-files/README.md: -------------------------------------------------------------------------------- 1 | # AllegroGraph Tutorial files 2 | 3 | * [PhoneCalls.ttl.gz](PhoneCalls.ttl.gz) 4 | * [SpaceMissions2.ntriples](SpaceMissions2.ntriples) 5 | * [SpaceMissions2.owl](SpaceMissions2.owl) 6 | * [agraph-tutorial.cl](agraph-tutorial.cl) 7 | * [federation-examples.cl](federation-examples.cl) 8 | * [freetext-index-tutorial.cl](freetext-index-tutorial.cl) 9 | * [kennedy-family.cl](kennedy-family.cl) 10 | * [kennedy.ntriples](kennedy.ntriples) 11 | * [lesmis.rdf](lesmis.rdf) 12 | * [metarelations.cl](metarelations.cl) 13 | * [reasoner-tutorial.cl](reasoner-tutorial.cl) 14 | * [remote-client.cl](remote-client.cl) 15 | * [sna-examples.cl](sna-examples.cl) 16 | * [sparql-examples.cl](sparql-examples.cl) 17 | * [stored-proc-example.cl](stored-proc-example.cl) 18 | * [wilburwine.ntriples](wilburwine.ntriples) 19 | * [wilburwine.rdf](wilburwine.rdf) 20 | -------------------------------------------------------------------------------- /tutorial-files/freetext-index-tutorial.cl: -------------------------------------------------------------------------------- 1 | ;; Freetext indexing tuturial for lisp users.. 2 | ;; The idea is that all the following lisp forms are evaluated 3 | ;; from top to bottom, one at a time. 4 | 5 | (require :agraph) ; load Agraph 6 | 7 | (in-package :triple-store-user) ; go in the right package 8 | 9 | (enable-!-reader) 10 | 11 | (enable-print-decoded t) ; make printing look nicer 12 | 13 | (create-triple-store "ag-test" :if-exists :supersede) 14 | 15 | ;; Tell AllegroGraph on which predicates you want to index. 16 | 17 | (create-freetext-index "fti-tutorial" 18 | :predicates '(! 19 | !)) 20 | 21 | ;; Add some triples: 22 | 23 | (add-triple !"Jans" !rdfs:comment !"Born in Amsterdam in the Netherlands") 24 | (add-triple !"Gary" !rdfs:comment !"Born in Springfield in the USA") 25 | (add-triple !"Steve" !rdfs:label !"Born in New Amsterdam in the USA") 26 | 27 | ;; Using the index: 28 | 29 | (freetext-get-unique-subjects '(match "amsterd*") :index "fti-tutorial") 30 | 31 | ;=> ({Steve} {Jans}) 32 | 33 | (freetext-get-unique-subjects '(and "amsterdam" "usa")) ; a boolean expression 34 | 35 | (freetext-get-unique-subjects "\"New Amsterdam\"") ; a phrase, note the quotes 36 | 37 | (freetext-get-triples '(and "usa" "born")) ; return a cursor 38 | 39 | ;=> # 40 | 41 | ;; Oops, this is a cursor. So lets work with this cursor. 42 | ;; First bind it to cursor.. 43 | 44 | (setf cursor (freetext-get-triples '(and "usa" "born"))) 45 | 46 | ;; then loop over the cursor with the handy iterate-cursor macro or 47 | ;; map-cursor function. 48 | 49 | (iterate-cursor (triple cursor) 50 | (print triple)) 51 | 52 | (map-cursor 10 'print (freetext-get-triples '(and "usa" "born"))) 53 | 54 | ;; And sometimes you only want the subjects back, especially in your favorite 55 | ;; query language: 56 | 57 | (freetext-get-unique-subjects '(and "netherlands" "born")) 58 | 59 | ;=> ({"Jans"}) 60 | 61 | 62 | ;;; ====== so now a bigger example. 63 | 64 | (register-namespace "o" "http://www.franz.com/simple#") 65 | 66 | ;; First we add some new triples to our open triple-store, note that the 67 | ;; object of each new triples is a long string filled with random numbers 68 | ;; (in english) 69 | 70 | (defun fill-dummy-ntriple-file-and-load-it (max) 71 | (let ((list '("one " "two " "three " "four " 72 | "five " "six " "seven " "eight " "nine " "ten "))) 73 | (with-open-file (out "dum.ntriples" :direction :output 74 | :if-exists :supersede) 75 | (dotimes (i max) 76 | (let ((subject (string+ ' "))) 77 | (dotimes (j 5) 78 | (let ((predicate " ") 79 | (object (apply 'triple-store::string+ 80 | (let ((li nil)) 81 | (dotimes (i (1+ (random 8))) 82 | (push (nth (random 10) list) li)) 83 | li)))) 84 | (format out "~a~a~s .~%" subject predicate object)))))) 85 | (load-ntriples "dum.ntriples"))) 86 | 87 | (fill-dummy-ntriple-file-and-load-it 10) 88 | 89 | ;; Let's look at the triples 90 | 91 | (dolist (e (get-triples-list)) 92 | (print e)) 93 | 94 | ;; So now we want to play with this file: let us write a little test 95 | ;; function: 96 | 97 | (defun test (arg) 98 | (map-cursor 50 (lambda (a) (print a)) 99 | (freetext-get-triples arg))) 100 | 101 | ;; simple expressions 102 | 103 | (test "eight") 104 | 105 | (test '(and "ten" "eight")) 106 | 107 | (test '(and "ten" "eight" (or "three" "four"))) 108 | 109 | (test '(or (and "five" "one") 110 | (and "ten" "eight" (or "three" "four")))) 111 | 112 | ;; wildcards -> * is zero or more occurrences 113 | ;; ? is one character 114 | ;; no * allowed in phrases 115 | 116 | (test '(match "?i*")) ; five six nine 117 | 118 | (test '(match "?i?e")) ; five nine 119 | 120 | (test '(or (and (match "fiv*") (match "on*")) 121 | (and (match "te*") (match "eigh*") 122 | (or (match "th*ree") (match "fo*ur") (match "\"one five\""))))) 123 | 124 | 125 | ;; And here is an example of a large file, filled with weapon systems, 126 | ;; terrorists, a a lot of common knowledge from the cyc database 127 | ;; (available on request: please mail ssears@franz.com) 128 | 129 | ;; We include this non-trivial example but it will allow us to do some 130 | ;; select queries 131 | 132 | 133 | (defun read-gov () 134 | (format t "~%Add triples") 135 | (load-ntriples "/s/ja/allegrograph-examplefiles/Gov.ntriples") 136 | (format t "~%Commit triples") 137 | (commit-triple-store)) 138 | 139 | (time (read-gov)) 140 | 141 | (register-namespace "c" "http://www.cyc.com/2002/04/08/cyc#") 142 | 143 | (get-triples-list :p !rdfs:comment) 144 | 145 | (test '(and "collection" "people")) 146 | 147 | (test '(phrase "collection of people")) 148 | 149 | ;; use it in prolog 150 | 151 | (select (?person) 152 | (lisp ?list (freetext-get-unique-subjects '(and "collection" "people"))) 153 | (member ?person ?list) 154 | (q- ?person !rdfs:subClassOf !c:AsianCitizenOrSubject)) 155 | 156 | (select (?person) 157 | (lisp ?list (freetext-get-unique-subjects '(phrase "collection of people"))) 158 | (member ?person ?list) 159 | (q- ?person !rdfs:subClassOf !c:AsianCitizenOrSubject)) 160 | 161 | (close-triple-store) 162 | 163 | (open-triple-store "ag-test") 164 | 165 | (select (?person) 166 | (lisp ?list (freetext-get-unique-subjects '(and "collection" "people"))) 167 | (member ?person ?list) 168 | (q- ?person !rdfs:subClassOf !c:AsianCitizenOrSubject)) 169 | -------------------------------------------------------------------------------- /tutorial-files/kennedy-family.cl: -------------------------------------------------------------------------------- 1 | (require :agraph) 2 | 3 | (in-package :triple-store-user) 4 | 5 | (enable-!-reader) 6 | 7 | (register-namespace "ex" "http://www.franz.com/simple#" 8 | :errorp nil) 9 | 10 | (defun fill-kennedy-db () 11 | (create-triple-store "test2" 12 | :if-exists :supersede) 13 | (time 14 | (load-ntriples "kennedy.ntriples" :commit t))) 15 | 16 | (fill-kennedy-db) 17 | 18 | ; Prolog as one retrieval language 19 | 20 | (defun full-name (a b c) 21 | (format nil "~a ~a ~a" 22 | (part->concise a)(part->concise b) (part->concise c))) 23 | 24 | (<-- (full-name ?x ?fullname) 25 | (q- ?x !ex:first-name ?fn) 26 | (q- ?x !ex:middle-initial ?mn) 27 | (q- ?x !ex:last-name ?ln) 28 | (lisp ?fullname (full-name ?fn ?mn ?ln))) 29 | 30 | (<-- (full-name ?x ?fn ?mn ?ln ?fullname) 31 | (q- ?x !ex:first-name ?fn) 32 | (q- ?x !ex:middle-initial ?mn) 33 | (q- ?x !ex:last-name ?ln) 34 | (lisp ?fullname (full-name ?fn ?mn ?ln))) 35 | 36 | (<-- (male ?x) 37 | (q- ?x !ex:sex !ex:male)) 38 | 39 | (<-- (female ?x) 40 | (q- ?x !ex:sex !ex:female)) 41 | 42 | (<-- (father ?x ?y) 43 | (male ?x) 44 | (q- ?x !ex:has-child ?y)) 45 | 46 | (<-- (mother ?x ?y) 47 | (female ?x) 48 | (q- ?x !ex:has-child ?y)) 49 | 50 | (<-- (parent ?x ?y) 51 | (father ?x ?y)) 52 | 53 | (<- (parent ?x ?y) 54 | (mother ?x ?y)) 55 | 56 | (<-- (grandparent ?x ?y) 57 | (parent ?x ?z) 58 | (parent ?z ?y)) 59 | 60 | (<-- (grandchild ?x ?y) 61 | (grandparent ?y ?x)) 62 | 63 | (<-- (ancestor ?x ?y) 64 | (parent ?x ?y)) 65 | 66 | (<- (ancestor ?x ?y) 67 | (parent ?x ?z) 68 | (ancestor ?z ?y)) 69 | 70 | (<-- (descendant ?x ?y) 71 | (ancestor ?y ?x)) 72 | 73 | (<-- (aunt ?x ?y) 74 | (father ?z ?x) 75 | (female ?x) 76 | (father ?z ?w) 77 | (not (= ?x ?w)) 78 | (parent ?w ?y)) 79 | 80 | (<-- (uncle ?x ?y) 81 | (father ?z ?x) 82 | (male ?x) 83 | (father ?z ?w) 84 | (not (= ?x ?w)) 85 | (parent ?w ?y)) 86 | 87 | (<-- (nephew ?x ?y) 88 | (aunt ?y ?x) 89 | (male ?x)) 90 | 91 | (<- (nephew ?x ?y) 92 | (uncle ?y ?x) 93 | (male ?x)) 94 | 95 | (<-- (niece ?x ?y) 96 | (aunt ?y ?x) 97 | (female ?x)) 98 | 99 | (<- (niece ?x ?y) 100 | (uncle ?y ?x) 101 | (female ?x)) 102 | 103 | (<-- (parent-child-have-same-name ?x ?y) 104 | (q- ?x !ex:first-name ?n1) 105 | (parent ?x ?y) 106 | (q- ?y !ex:first-name ?n2) 107 | (= ?n1 ?n2)) 108 | 109 | (<-- (parent-child-went-to-ivy-league-school ?x ?y) 110 | (q- ?x !ex:alma-mater ?am) 111 | (q- ?am !ex:ivy-league !ex:true) 112 | (parent ?x ?y) 113 | (q- ?y !ex:alma-mater ?am2) 114 | (q- ?am2 !ex:ivy-league !ex:true)) 115 | 116 | (<-- (parent-child-went-to-same-ivy-league-school ?x ?y) 117 | (q- ?x !ex:alma-mater ?am) 118 | (q- ?am !ex:ivy-league !ex:true) 119 | (parent ?x ?y) 120 | (q- ?y !ex:alma-mater ?am)) 121 | 122 | (<-- (spouse ?x ?y) 123 | (q- ?x !ex:spouse ?y)) 124 | 125 | ;; ?x has a spouse and children 126 | 127 | (<-- (family ?x ?fam) 128 | (q- ?x !ex:spouse ?sp) 129 | (bagof ?ch (parent ?x ?ch) ?bag) 130 | (append ?bag (?sp) ?fam) 131 | \!) 132 | 133 | ;; ?x has no spouse ... 134 | 135 | (<- (family ?x ?fam) 136 | (mother ?m ?x) 137 | (father ?f ?x) 138 | (siblings-incl ?m ?mlist) 139 | (siblings-incl ?f ?flist) 140 | (append ?mlist ?flist ?fam)) 141 | 142 | (<-- (siblings-incl ?x ?siblist) 143 | (father ?father ?x) 144 | (bagof ?ch (father ?father ?ch) ?siblist)) 145 | 146 | (<- (siblings-incl ?x ?siblist) 147 | (append nil (?x) ?siblist)) 148 | 149 | 150 | ;;; 151 | 152 | #| queries 153 | 154 | (select (?x ?y) 155 | (father ?x ?y)) 156 | 157 | (select (?x ?y) 158 | (aunt ?x ?y)) 159 | 160 | (select (?x ?y) 161 | (nephew ?x ?y)) 162 | 163 | |# 164 | -------------------------------------------------------------------------------- /tutorial-files/metarelations.cl: -------------------------------------------------------------------------------- 1 | 2 | (in-package :triple-store-user) 3 | 4 | (defvar *relations* (make-hash-table)) 5 | 6 | (defun clear-relations () 7 | (setf *relations* (make-hash-table))) 8 | 9 | (defun register-relations (&rest list) 10 | (dolist (e list) 11 | (when (not (gethash e *relations*)) 12 | (let ((form `(<- (relation ?x ?y ,e) 13 | (,e ?x ?y)))) 14 | (when (= 0 (hash-table-count *relations*)) 15 | (setf (car form) '<--)) 16 | (setf (gethash e *relations*) t) 17 | (eval form) 18 | (pprint form))))) 19 | -------------------------------------------------------------------------------- /tutorial-files/remote-client.cl: -------------------------------------------------------------------------------- 1 | (require :agraph) 2 | 3 | (in-package :db.agraph.user) 4 | 5 | (close-all-triple-stores) 6 | 7 | ;;;------------------------------------------------ 8 | ;; Create triple-store on remote AGraph server. 9 | ;;;------------------------------------------------ 10 | (defparameter ground-ts 11 | (create-triple-store "test-remote" 12 | :triple-store-class 'remote-triple-store 13 | :server "localhost" :port 10035 14 | :user "test" :password "xyzzy")) 15 | 16 | ;; As with a local triple-store, the *db* var is set: 17 | *db* 18 | 19 | (close-triple-store) 20 | 21 | (defparameter ground-ts 22 | (open-triple-store "test-remote" 23 | :triple-store-class 'remote-triple-store 24 | :server "localhost" :port 10035 25 | :user "test" :password "xyzzy")) 26 | 27 | ;; Most other functions work the same as with a local triple-store. 28 | ;; For example: 29 | 30 | (register-namespace 31 | "ex" "http://www.franz.com/things#" 32 | :errorp nil) 33 | 34 | (display-namespaces) 35 | 36 | ;; examples copied from reasoner-tutorial.cl 37 | 38 | (add-triple !ex:jans !ex:owns !ex:birra) 39 | 40 | (add-triple !ex:jans !owl:sameAs !ex:jannes) 41 | 42 | ;; no reasoning, ask about ex:owns and get one triple 43 | (get-triples-list :p !ex:owns) 44 | 45 | (defparameter inferred-ts (apply-rdfs++-reasoner :remote-reasoning t 46 | :db ground-ts)) 47 | 48 | ;; we'll get two triples now 49 | (get-triples-list :p !ex:owns) 50 | 51 | ;; using the :db argument 52 | (get-triples-list :p !ex:owns :db ground-ts) 53 | (get-triples-list :p !ex:owns :db (ground-triple-store inferred-ts)) 54 | 55 | (close-all-triple-stores) 56 | -------------------------------------------------------------------------------- /tutorial-files/sna-examples.cl: -------------------------------------------------------------------------------- 1 | (in-package db.agraph.user) 2 | 3 | #| not mentioned 4 | 5 | in-degree, out-degree, nodal-degree 6 | in-neighbors, out-neighbors, nodal-neighbors 7 | 8 | group-degree-centrality 9 | actor-closeness-centrality 10 | group-closeness-centrality 11 | 12 | clique (which also remains unimplemented) 13 | 14 | |# 15 | 16 | ;;;;; Finally, some code to make some different sample networks 17 | 18 | (defun build-florentine-family-graph () 19 | "Silly but it works..." 20 | (make-tutorial-store) 21 | (let ((row 0) 22 | (names (make-array 16)) 23 | (links nil) 24 | (namespace "http://www.franz.com/sna#")) 25 | (register-namespace "ffg" namespace) 26 | (db.agraph::map-lines 27 | " 28 | acciaiuoli 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 29 | albizzi 0 0 0 0 0 1 1 0 1 0 0 0 0 0 0 0 30 | barbadori 0 0 0 0 1 0 0 0 1 0 0 0 0 0 0 0 31 | bischeri 0 0 0 0 0 0 1 0 0 0 1 0 0 0 1 0 32 | castellani 0 0 1 0 0 0 0 0 0 0 1 0 0 0 1 0 33 | ginori 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 34 | guadagni 0 1 0 1 0 0 0 1 0 0 0 0 0 0 0 1 35 | lamberterschi 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 36 | medici 1 1 1 0 0 0 0 0 0 0 0 0 1 1 0 1 37 | pazzi 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 38 | peruzzi 0 0 0 1 1 0 0 0 0 0 0 0 0 0 1 0 39 | pucci 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 40 | ridolfi 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 1 41 | salviati 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 0 42 | strozzi 0 0 0 1 1 0 0 0 0 0 1 0 1 0 0 0 43 | tornabuoni 0 0 0 0 0 0 1 0 1 0 0 0 1 0 0 0 44 | " (lambda (line) 45 | (setf (aref names row) 46 | (resource (string+ namespace (read-from-string line)))) 47 | (add-triple (aref names row) !rdf:type !ffg:family) 48 | (let ((position 14) 49 | (linkp nil)) 50 | (loop while 51 | (setf (values linkp position) 52 | (read-from-string line nil nil :start position)) 53 | for index from 0 54 | when (= linkp 1) do 55 | (push (cons row index) links))) 56 | (incf row))) 57 | (loop for (source . target) in links do 58 | (add-triple (aref names source) 59 | !ffg:marriage 60 | (aref names target))))) 61 | 62 | #+(or) 63 | (build-florentine-family-graph) 64 | 65 | (defun ffg-marriage-generator (node) 66 | "Returns list of nodes to whom node is related by marriage" 67 | ;; ugh! 68 | (union 69 | (collect-cursor (get-triples :s node :p !ffg:marriage) 70 | :transform 'object) 71 | (collect-cursor (get-triples :p !ffg:marriage :o node) 72 | :transform 'subject) 73 | :test 'upi=)) 74 | 75 | ;;;;;;; 76 | 77 | ;;;; 78 | 79 | (defun build-star-graph (size &key (create t)) 80 | (let ((namespace (setup-for-sample-graphs create size))) 81 | (loop for i from 1 below size do 82 | (add-triple 83 | (resource (string+ namespace i)) 84 | !sna:linksTo 85 | (resource (string+ namespace 0)))))) 86 | 87 | (defun build-circle-graph (size &key (create t)) 88 | (let ((namespace (setup-for-sample-graphs create size))) 89 | (loop for i from 0 below size do 90 | (add-triple (resource (string+ namespace (mod i size))) 91 | !sna:linksTo 92 | (resource (string+ namespace (mod (1+ i) size))))))) 93 | 94 | (defun build-line-graph (size &key (create t)) 95 | (let ((namespace (setup-for-sample-graphs create size))) 96 | (loop for i from 0 to (- size 2) do 97 | (add-triple (resource (string+ namespace i)) 98 | !sna:linksTo 99 | (resource (string+ namespace (1+ i))))))) 100 | 101 | (defun linksTo-generator (node) 102 | (collect-cursor (get-triples :s node :p !sna:linksTo) 103 | :transform 'object)) 104 | 105 | (defun linksFrom-generator (node) 106 | (collect-cursor (get-triples :s node :p !sna:linksFrom) 107 | :transform 'object)) 108 | 109 | (defun linksToOrFrom-generator (node) 110 | (union 111 | (collect-cursor (get-triples :s node :p !sna:linksFrom) 112 | :transform 'object) 113 | (collect-cursor (get-triples :s node :p !sna:linksTo) 114 | :transform 'object) 115 | :test 'upi=)) 116 | 117 | (defun setup-for-sample-graphs (create? size) 118 | (when (or create? (not *db*)) 119 | (make-tutorial-store)) 120 | (let ((namespace "http://www.franz.com/sna#")) 121 | (register-namespace "sna" namespace) 122 | (add-triple !sna:linksTo !owl:inverseOf !sna:linksFrom) 123 | (loop for i from 0 below size do 124 | (add-triple (resource (string+ namespace i)) !rdf:type !sna:node)) 125 | (apply-rdfs++-reasoner) 126 | namespace)) 127 | 128 | 129 | ;;; now we exercise it some 130 | 131 | ;; Graph density 132 | (build-star-graph 7) 133 | (density 134 | (select0-distinct ?s (q- ?s !rdf:type !sna:node)) 135 | 'linksToOrFrom-generator) 136 | 137 | (build-circle-graph 7) 138 | (density 139 | (select0-distinct ?s (q- ?s !rdf:type !sna:node)) 140 | 'linksToOrFrom-generator) 141 | 142 | (build-line-graph 7) 143 | (density 144 | (select0-distinct ?s (q- ?s !rdf:type !sna:node)) 145 | 'linksToOrFrom-generator) 146 | 147 | 148 | ;;; Actor degree centrality 149 | ;; star graph 150 | (build-star-graph 7) 151 | (setf group (select0-distinct ?s (q- ?s !rdf:type !sna:node))) 152 | (actor-degree-centrality !sna:0 group 'linksTo-generator) 153 | (actor-degree-centrality !sna:0 group 'linksFrom-generator) 154 | (actor-degree-centrality !sna:1 group 'linksTo-generator) 155 | (actor-degree-centrality !sna:1 group 'linksFrom-generator) 156 | 157 | (build-star-graph 7) 158 | (setf group (select0-distinct ?s (q- ?s !rdf:type !sna:node))) 159 | (actor-degree-centrality !sna:1 group 'linksToOrFrom-generator) 160 | 161 | ;; circle graph 162 | 163 | (build-circle-graph 7) 164 | (setf group (select0-distinct ?s (q- ?s !rdf:type !sna:node))) 165 | (actor-degree-centrality !sna:0 group 'linksTo-generator) 166 | (actor-degree-centrality !sna:1 group 'linksToOrFrom-generator) 167 | 168 | 169 | ;;; ego network 170 | 171 | ;; start 172 | 173 | (build-circle-graph 7) 174 | (setf group (select0-distinct ?s (q- ?s !rdf:type !sna:node))) 175 | (ego-group-select !sna:0 'linksTo-generator 0) 176 | (ego-group-select !sna:0 'linksTo-generator 1) 177 | (ego-group-select !sna:1 'linksToOrFrom-generator 2) 178 | 179 | ;; circle 180 | (build-circle-graph 7) 181 | (ego-group-select !sna:0 'linksTo-generator 0) 182 | (ego-group-select !sna:0 'linksTo-generator 1) 183 | 184 | 185 | ;;; betweenness-centrality 186 | 187 | ;; star 188 | (build-star-graph 7) 189 | 190 | (actor-betweenness-centrality 191 | !sna:0 192 | (select0-distinct ?s (q- ?s !rdf:type !sna:node)) 193 | 'linksToOrFrom-generator) 194 | 195 | (let ((group (select0-distinct ?s (q- ?s !rdf:type !sna:node)))) 196 | (mapcar (lambda (node) 197 | (list node 198 | (actor-betweenness-centrality 199 | node group 'linksToOrFrom-generator))) 200 | group)) 201 | 202 | ;; line 203 | 204 | (build-line-graph 7) 205 | 206 | (let ((group (select0-distinct ?s (q- ?s !rdf:type !sna:node)))) 207 | (mapcar (lambda (node) 208 | (list node 209 | (actor-betweenness-centrality 210 | node group 'linksToOrFrom-generator))) 211 | group)) 212 | 213 | 214 | ;;; group-betweenness-centrality 215 | ;; star 216 | (build-star-graph 7) 217 | (group-betweenness-centrality 218 | (select0-distinct ?s (q- ?s !rdf:type !sna:node)) 219 | 'linksToOrFrom-generator) 220 | 221 | ;; circle 222 | (build-circle-graph 7) 223 | (group-betweenness-centrality 224 | (select0-distinct ?s (q- ?s !rdf:type !sna:node)) 225 | 'linksToOrFrom-generator) 226 | 227 | ;; line 228 | (build-line-graph 7) 229 | (group-betweenness-centrality 230 | (select0-distinct ?s (q- ?s !rdf:type !sna:node)) 231 | 'linksToOrFrom-generator) 232 | 233 | 234 | 235 | 236 | 237 | -------------------------------------------------------------------------------- /tutorial-files/sparql-examples.cl: -------------------------------------------------------------------------------- 1 | (require :agraph) ;load AllegroGraph 2 | 3 | (enable-!-reader) ;turn on the !-reader 4 | 5 | (in-package :db.agraph.user) ;switch to the AllegroGraph package 6 | 7 | (enable-print-decoded t) ;print UPIs and triples readably 8 | 9 | (close-all-triple-stores) 10 | 11 | (create-triple-store "ag-test") 12 | 13 | ;; Examples from sparql-exploration.html 14 | 15 | (load-ntriples "http://www.w3.org/2000/10/rdf-tests/rdfcore/ntriples/test.nt") 16 | 17 | (commit-triple-store) 18 | 19 | (sparql:run-sparql " 20 | SELECT DISTINCT ?o { 21 | ?s ?o . 22 | FILTER (isIRI(?o)) 23 | }" :output-format :sparql-json) 24 | 25 | (sparql.algebra:sparql->algebra " 26 | SELECT DISTINCT ?o { 27 | ?s ?o . 28 | FILTER (isIRI(?o)) 29 | }") 30 | 31 | ;; Free text search in SPARQL 32 | 33 | (create-freetext-index "sparql-tutorial" 34 | :predicates '(!) 35 | :index-resources t) 36 | 37 | ;; The lisp run-sparql API does not use registered or 38 | ;; *standard-namespaces* by default. 39 | ;; Passing a parameter is an alternative to using PREFIX 40 | ;; in the sparql query text. 41 | (setf *prefixes* (append 42 | '(("kdy" "http://www.franz.com/simple#") 43 | ("fti" "http://franz.com/ns/allegrograph/2.2/textindex/")) 44 | *standard-namespaces*)) 45 | 46 | (sparql:run-sparql " 47 | SELECT DISTINCT ?s { 48 | ?s fti:match 'resou*' . 49 | }" :default-prefixes *prefixes* 50 | :output-format :alists) 51 | 52 | ;; Ask, Construct, and Describe queries 53 | 54 | (time 55 | (load-ntriples "kennedy.ntriples")) 56 | 57 | (commit-triple-store) 58 | 59 | (sparql:run-sparql " 60 | select ?s where { ?s rdf:type kdy:person} limit 5 61 | " :default-prefixes *prefixes* 62 | :output-format :alists) 63 | 64 | (sparql:run-sparql " 65 | ask { ?s kdy:first-name 'John' } 66 | " :default-prefixes *prefixes*) 67 | ;; => t 68 | 69 | ;; construct triples, and add to the db 70 | (dolist (tr (sparql:run-sparql " 71 | construct {?a kdy:has-grandchild ?c} 72 | where { ?a kdy:has-child ?b . 73 | ?b kdy:has-child ?c . } 74 | " :default-prefixes *prefixes* 75 | :output-format :triples :engine :allegrograph-2)) 76 | (add-triple (subject tr) (predicate tr) (object tr))) 77 | 78 | (triple-count) 79 | 80 | (commit-triple-store) 81 | 82 | (sparql:run-sparql " 83 | select ?s ?o where { ?s kdy:has-grandchild ?o} limit 5 84 | " :default-prefixes *prefixes*) 85 | 86 | (sparql:run-sparql " 87 | describe ?s ?o where { ?s kdy:has-grandchild ?o . } limit 1 88 | " :default-prefixes *prefixes*) 89 | 90 | ;; with-variables 91 | 92 | (register-namespace "kdy" "http://www.franz.com/simple#" :errorp nil) 93 | 94 | ;; replace variables in the queries with values 95 | (let* ((p !kdy:has-grandchild)) 96 | (sparql:run-sparql "select distinct ?s where { ?s ?p ?o}" 97 | :default-prefixes *prefixes* 98 | :output-format :alists 99 | :with-variables `((?p . ,p)))) 100 | ;; => (((?s . {person1})) ((?s . {person2}))) 101 | 102 | -------------------------------------------------------------------------------- /tutorial-files/stored-proc-example.cl: -------------------------------------------------------------------------------- 1 | ;; sample stored procedure 2 | 3 | (defpackage :my-package 4 | (:use :excl :common-lisp :db.agraph)) 5 | 6 | (in-package :my-package) 7 | 8 | (define-condition my-stored-proc-error (simple-error) ()) 9 | 10 | (defun my-stored-proc-error (fmt &rest args) 11 | (error (make-condition 'my-stored-proc-error 12 | :format-control fmt 13 | :format-arguments args))) 14 | 15 | ;; agraph 3.x style: 16 | ;; (def-kbi-call "addTwo" dbs-add-two-numbers) 17 | ;; 18 | ;; 19 | ;; (defun dbs-add-two-numbers (arg-vec) 20 | ;; ;; takes two values and adds them 21 | ;; (if* (not (eql 2 (length arg-vec))) 22 | ;; then "wrong number of args" 23 | ;; else (write-to-string 24 | ;; (+ (or (parse-integer (aref arg-vec 0) :junk-allowed t) 0) 25 | ;; (or (parse-integer (aref arg-vec 1) :junk-allowed t) 0))))) 26 | ;; 27 | 28 | ; ag 4.x style where the whole argument vector is given 29 | ; to the stored procedure and it pulls out the values 30 | ; 31 | (def-stored-proc addTwo (&whole arg-vec) 32 | ;; takes two values and adds them 33 | (db.agraph::log-info :addtwo "arg vec is ~s" arg-vec) 34 | (if* (not (eql 2 (length arg-vec))) 35 | then (my-stored-proc-error "wrong number of args") 36 | else (write-to-string 37 | (+ (or (parse-integer (aref arg-vec 0) :junk-allowed t) 0) 38 | (or (parse-integer (aref arg-vec 1) :junk-allowed t) 0))))) 39 | 40 | 41 | ; ag 4.x style where we let the def-stored-proc code generate code 42 | ; to check if the correct number of arguments were passed in the 43 | ; argument vector and if not to return an error indication 44 | 45 | (def-stored-proc addTwoVersion2 (a b) 46 | ;; takes two values and adds them 47 | (write-to-string 48 | (+ (or (parse-integer a :junk-allowed t) 0) 49 | (or (parse-integer b :junk-allowed t) 0)))) 50 | --------------------------------------------------------------------------------