├── .vscode ├── settings.json ├── tasks.json └── launch.json ├── .dockerignore ├── Hexastore.Web ├── appsettings.json ├── appsettings.Development.json ├── Queue │ ├── IQueueContainer.cs │ ├── QueueContainer.cs │ ├── QueueWriter.cs │ └── Hasher.cs ├── install.sh ├── EventHubs │ ├── EventType.cs │ ├── Checkpoint.cs │ ├── StoreEvent.cs │ ├── EventSender.cs │ └── EventReceiver.cs ├── Controllers │ ├── UpdateRequest.cs │ └── StoreControllerBase.cs ├── Program.cs ├── LoggingEvents.cs ├── Properties │ └── launchSettings.json ├── PerfHeaderMiddleware.cs ├── StoreConfig.cs ├── Hexastore.Web.csproj ├── Dockerfile ├── Dockerfile.backup └── Startup.cs ├── docker-compose.yaml ├── Hexastore ├── Util │ └── Util.cs ├── Resoner │ ├── IReasoner.cs │ └── Reasoner.cs ├── Store │ ├── IStore.cs │ ├── IStoreProvider.cs │ ├── IGraphProvider.cs │ ├── SetProvider.cs │ ├── Store.cs │ └── MemoryGraphProvider.cs ├── Processor │ ├── IStoreOperationFactory.cs │ └── IStoreProcesor.cs ├── Graph │ ├── IStoreGraph.cs │ ├── ISPIGraph.cs │ ├── Patch.cs │ ├── IPagableGraph.cs │ ├── Triple.cs │ ├── GraphOperator.cs │ ├── IGraph.cs │ ├── DisjointUnion.cs │ ├── SPOIndex.cs │ └── TripleObject.cs ├── Hexastore.csproj ├── Errors │ ├── StoreException.cs │ └── StoreError.cs ├── Constants.cs ├── Query │ └── ObjectQueryModel.cs └── Parser │ └── TripleConverter.cs ├── Hexastore.Rocks ├── Constants.cs ├── Hexastore.Rocks.csproj ├── RocksTest.cs ├── TripleObjectExtensions.cs ├── RocksSubjectGrouping.cs ├── RocksPredicateGrouping.cs ├── TripleExtensions.cs ├── RocksEnumerable.cs ├── RocksGraphProvider.cs └── KeyConfig.cs ├── NuGet.Config ├── Hexastore.ScaleConsole ├── Program.cs ├── Hexastore.ScaleConsole.csproj ├── JsonGenerator.cs ├── TestFolder.cs └── FlatModelTest.cs ├── Hexastore.PerfConsole ├── Hexastore.PerfConsole.csproj ├── Program.cs ├── JsonGenerator.cs ├── TestFolder.cs ├── PatchAddNew.cs ├── PatchUpdateNoChange.cs ├── PatchUpdateSingle.cs └── PatchUpdate.cs ├── omnisharp.json ├── Hexastore.Test ├── ArrayTest.cs ├── Hexastore.Test.csproj ├── TripleObjectTest.cs ├── UnorderedTripleComparer.cs ├── RocksFixture.cs ├── TestFolder.cs ├── TripleConverterTest.cs ├── ContinuationTest.cs ├── ObjectQueryExecutorPagedTest.cs └── ObjectQueryStarPathTest.cs ├── LICENSE ├── Dockerfile ├── hexadb-readme-incoming-path.svg ├── hexadb-readme-outgoing-path.svg ├── hexadb-readme-outgoing-level.svg ├── .gitattributes ├── Hexastore.sln ├── .gitignore ├── hexadb-readme.svg ├── .editorconfig └── README.md /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.formatOnSave": true 3 | } -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .dockerignore 2 | .env 3 | .git 4 | .gitignore 5 | .vs 6 | .vscode 7 | */bin 8 | */obj 9 | **/.toolstarget -------------------------------------------------------------------------------- /Hexastore.Web/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Warning" 5 | } 6 | }, 7 | "AllowedHosts": "*" 8 | } 9 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | hexastore: 4 | build: . 5 | ports: 6 | - "8000:80" 7 | # command: "bash -c 'mount'" 8 | # tmpfs: 9 | # - /var/data 10 | -------------------------------------------------------------------------------- /Hexastore/Util/Util.cs: -------------------------------------------------------------------------------- 1 | using Hexastore.Store; 2 | 3 | public static class Util 4 | { 5 | public static string MakeKey(string id, GraphType type) 6 | { 7 | return $"{type}:{id}"; 8 | } 9 | } -------------------------------------------------------------------------------- /Hexastore.Web/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Debug", 5 | "System": "Information", 6 | "Microsoft": "None" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /Hexastore/Resoner/IReasoner.cs: -------------------------------------------------------------------------------- 1 | using Hexastore.Graph; 2 | 3 | namespace Hexastore.Resoner 4 | { 5 | public interface IReasoner 6 | { 7 | void Spin(IGraph data, IGraph inferred, IGraph meta); 8 | } 9 | } -------------------------------------------------------------------------------- /Hexastore.Rocks/Constants.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Hexastore.Rocks 4 | { 5 | public class Constants 6 | { 7 | public static readonly byte[] TrueByes = BitConverter.GetBytes(true); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /Hexastore/Store/IStore.cs: -------------------------------------------------------------------------------- 1 | using Hexastore.Graph; 2 | using System; 3 | 4 | namespace Hexastore.Store 5 | { 6 | public interface IStore : IDisposable 7 | { 8 | IStoreGraph GetGraph(GraphType type); 9 | } 10 | } -------------------------------------------------------------------------------- /Hexastore/Store/IStoreProvider.cs: -------------------------------------------------------------------------------- 1 | namespace Hexastore.Store 2 | { 3 | public interface IStoreProvider 4 | { 5 | bool Contains(string id); 6 | IStore GetOrAdd(string setId); 7 | IStore GetSet(string id); 8 | } 9 | } -------------------------------------------------------------------------------- /Hexastore/Processor/IStoreOperationFactory.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | 4 | namespace Hexastore.Processor 5 | { 6 | public interface IStoreOperationFactory 7 | { 8 | IDisposable Write(string storeId); 9 | } 10 | } -------------------------------------------------------------------------------- /Hexastore/Graph/IStoreGraph.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace Hexastore.Graph 6 | { 7 | public interface IStoreGraph : IGraph, IPagableGraph, ISPIndexQueryableGraph 8 | { 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /Hexastore.Web/Queue/IQueueContainer.cs: -------------------------------------------------------------------------------- 1 | using Hexastore.Web.EventHubs; 2 | 3 | namespace Hexastore.Web.Queue 4 | { 5 | public interface IQueueContainer 6 | { 7 | public void Send(StoreEvent storeEvent); 8 | public int Count(); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /Hexastore/Graph/ISPIGraph.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace Hexastore.Graph 6 | { 7 | public interface ISPIndexQueryableGraph 8 | { 9 | Triple SPI(string s, string p, int index); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /Hexastore/Graph/Patch.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace Hexastore.Graph 6 | { 7 | public class Patch 8 | { 9 | public IEnumerable Assert { get; set; } 10 | public IEnumerable Retract { get; set; } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /NuGet.Config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /Hexastore.ScaleConsole/Program.cs: -------------------------------------------------------------------------------- 1 | namespace Hexastore.ScaleConsole 2 | { 3 | class Program 4 | { 5 | static void Main(string[] args) 6 | { 7 | FlatModelTest.RunTest(appCount: 1, deviceCount: 1000000, devicePropertyCount: 3, sendCount: 5, senderThreadCount: 32, tryOptimizeRocks: true); 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /Hexastore.ScaleConsole/Hexastore.ScaleConsole.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp3.1 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /Hexastore/Hexastore.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /Hexastore.Web/install.sh: -------------------------------------------------------------------------------- 1 | apt-get update && apt-get install -y libcap2-bin libsnappy1v5 && \ 2 | ln -s /lib/x86_64-linux-gnu/libdl.so.2 /usr/lib/x86_64-linux-gnu/libdl.so && \ 3 | ln -s /lib/x86_64-linux-gnu/libc.so.6 /usr/lib/x86_64-linux-gnu/libc.so && \ 4 | rm -rf /var/lib/apt/lists/* 5 | 6 | # docker rm -f $(docker ps -a -q) 7 | # docker run -d -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.0.0 8 | 9 | -------------------------------------------------------------------------------- /Hexastore.Web/EventHubs/EventType.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | 6 | namespace Hexastore.Web.EventHubs 7 | { 8 | public class EventType 9 | { 10 | public const string POST = "POST"; 11 | public const string PATCH_JSON = "PATCH_JSON"; 12 | public const string PATCH_TRIPLE = "PATCH_TRIPLE"; 13 | public const string DELETE = "DELETE"; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Hexastore.Rocks/Hexastore.Rocks.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /Hexastore.Web/Controllers/UpdateRequest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using Newtonsoft.Json; 6 | using Newtonsoft.Json.Linq; 7 | 8 | namespace Hexastore.Web.Controllers 9 | { 10 | public class UpdateRequest 11 | { 12 | [JsonProperty("partitionKey")] 13 | public string PartitionKey { get; set; } 14 | 15 | [JsonProperty("data")] 16 | public JObject Data { get; set; } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /Hexastore.Web/Program.cs: -------------------------------------------------------------------------------- 1 | using Hexastore.Rocks; 2 | using Microsoft.AspNetCore; 3 | using Microsoft.AspNetCore.Hosting; 4 | using Microsoft.Extensions.Logging; 5 | 6 | namespace Hexastore.Web 7 | { 8 | public class Program 9 | { 10 | public static void Main(string[] args) 11 | { 12 | CreateWebHostBuilder(args).Build().Run(); 13 | } 14 | 15 | public static IWebHostBuilder CreateWebHostBuilder(string[] args) => 16 | WebHost.CreateDefaultBuilder(args) 17 | .UseStartup(); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /Hexastore.PerfConsole/Hexastore.PerfConsole.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp3.1 6 | NU1608 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /omnisharp.json: -------------------------------------------------------------------------------- 1 | { 2 | "FormattingOptions": { 3 | "NewLinesForBracesInLambdaExpressionBody": false, 4 | "NewLinesForBracesInAnonymousMethods": false, 5 | "NewLinesForBracesInAnonymousTypes": false, 6 | "NewLinesForBracesInControlBlocks": false, 7 | "NewLinesForBracesInTypes": false, 8 | "NewLinesForBracesInMethods": false, 9 | "NewLinesForBracesInProperties": false, 10 | "NewLinesForBracesInAccessors": false, 11 | "NewLineForElse": false, 12 | "NewLineForCatch": false, 13 | "NewLineForFinally": false 14 | } 15 | } -------------------------------------------------------------------------------- /Hexastore.PerfConsole/Program.cs: -------------------------------------------------------------------------------- 1 | using BenchmarkDotNet.Running; 2 | 3 | namespace Hexastore.PerfConsole 4 | { 5 | class Program 6 | { 7 | static void Main(string[] args) 8 | { 9 | // These can be executed with: 10 | // dotnet run -c Release 11 | // Results will be in Hexastore.PerfConsole\BenchmarkDotNet.Artifacts\results 12 | BenchmarkRunner.Run(); 13 | BenchmarkRunner.Run(); 14 | BenchmarkRunner.Run(); 15 | BenchmarkRunner.Run(); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /Hexastore/Errors/StoreException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | 6 | namespace Hexastore.Web.Errors 7 | { 8 | public class StoreException : Exception 9 | { 10 | public StoreException(string message, string errorCode) : base(message) 11 | { 12 | ErrorCode = errorCode; 13 | } 14 | 15 | public StoreException(string message, string errorCode, Exception ex) : base(message, ex) 16 | { 17 | ErrorCode = errorCode; 18 | } 19 | 20 | public string ErrorCode { get; set; } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /Hexastore/Graph/IPagableGraph.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace Hexastore.Graph 6 | { 7 | public interface IPagableGraph 8 | { 9 | IEnumerable S(string s, Triple continuation); 10 | IEnumerable P(string p, Triple continuation); 11 | IEnumerable O(TripleObject o, Triple continuation); 12 | 13 | IEnumerable SP(string s, string p, Triple continution); 14 | IEnumerable PO(string p, TripleObject o, Triple continuation); 15 | IEnumerable OS(TripleObject o, string s, Triple continuation); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Hexastore/Store/IGraphProvider.cs: -------------------------------------------------------------------------------- 1 | using Hexastore.Graph; 2 | 3 | namespace Hexastore.Store 4 | { 5 | /// 6 | /// Interface to store and retrieve graphs 7 | /// 8 | public interface IGraphProvider 9 | { 10 | IStoreGraph CreateGraph(string id, GraphType type); 11 | IStoreGraph GetGraph(string id, GraphType type); 12 | bool ContainsGraph(string id, GraphType type); 13 | bool DeleteGraph(string id, GraphType type); 14 | void WriteKey(string id, string key, string value); 15 | string ReadKey(string id, string key); 16 | } 17 | 18 | public enum GraphType 19 | { 20 | Data, 21 | Meta, 22 | Infer, 23 | Checkpoint 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /Hexastore.Web/LoggingEvents.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | 6 | namespace Hexastore.Web 7 | { 8 | public class LoggingEvents 9 | { 10 | public const int ControllerHealth = 100; 11 | public const int ControllerGetSubject = 101; 12 | public const int ControllerPost = 102; 13 | public const int ControllerPatchJson = 103; 14 | public const int ControllerPatchTriple = 104; 15 | public const int ControllerIngest = 105; 16 | public const int ControllerQuery = 106; 17 | public const int ControllerDelete = 107; 18 | public const int ControllerError = 108; 19 | public const int ControllerPredicates = 109; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /Hexastore.PerfConsole/JsonGenerator.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using Newtonsoft.Json.Linq; 4 | 5 | namespace Hexastore.PerfConsole 6 | { 7 | public static class JsonGenerator 8 | { 9 | public static JObject GenerateTelemetry(string id, Dictionary numberValues) 10 | { 11 | return new JObject(new JProperty("id", id), 12 | numberValues.Select(e => new JProperty(e.Key, e.Value))); 13 | } 14 | 15 | public static JObject GenerateTelemetry(string id, Dictionary numberValues, Dictionary stringValues) 16 | { 17 | return new JObject(new JProperty("id", id), 18 | numberValues.Select(e => new JProperty(e.Key, e.Value)), 19 | stringValues.Select(e => new JProperty(e.Key, e.Value))); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /Hexastore/Processor/IStoreProcesor.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using Hexastore.Graph; 3 | using Newtonsoft.Json.Linq; 4 | 5 | namespace Hexastore.Processor 6 | { 7 | public interface IStoreProcesor 8 | { 9 | void Assert(string storeId, JToken value, bool strict); 10 | void PatchJson(string storeId, JToken input); 11 | void PatchTriple(string storeId, JObject input); 12 | void AssertMeta(string storeId, JObject value); 13 | void Delete(string storeId, JToken value); 14 | JObject GetSet(string storeId); 15 | (IStoreGraph, IStoreGraph, IStoreGraph) GetGraphs(string storeId); 16 | JObject GetSubject(string storeId, string subject, string[] expand, int level); 17 | JObject GetType(string storeId, string[] type, string[] expand, int level); 18 | JObject Query(string storeId, JObject query, string[] expand, int level); 19 | JObject GetPredicates(string storeId); 20 | } 21 | } -------------------------------------------------------------------------------- /Hexastore.Test/ArrayTest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using Hexastore.Processor; 5 | using Microsoft.VisualStudio.TestTools.UnitTesting; 6 | using Newtonsoft.Json.Linq; 7 | 8 | namespace Hexastore.Test 9 | { 10 | [TestClass] 11 | public class ArrayTest : RocksFixture 12 | { 13 | private object _item1; 14 | 15 | public ArrayTest() 16 | { 17 | _item1 = new 18 | { 19 | id = "simplearray", 20 | values = new int[] { 10, 20, 10, 11, 29, 12 } 21 | }; 22 | 23 | StoreProcessor.Assert("app1", JToken.FromObject(_item1), false); 24 | } 25 | 26 | [TestMethod] 27 | public void Json_Read_Returns_Array() 28 | { 29 | var rsp = StoreProcessor.GetSubject("app1", "simplearray", null, 1); 30 | Assert.IsTrue(JToken.DeepEquals(JToken.FromObject(_item1), rsp)); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /Hexastore.Web/EventHubs/Checkpoint.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using Hexastore.Graph; 6 | using Hexastore.Store; 7 | using Newtonsoft.Json.Linq; 8 | 9 | namespace Hexastore.Web.EventHubs 10 | { 11 | public class Checkpoint 12 | { 13 | private readonly IGraphProvider _graphProvider; 14 | private const string DBName = "checkpoint"; 15 | 16 | public Checkpoint(IGraphProvider graphProvider) 17 | { 18 | _graphProvider = graphProvider; 19 | } 20 | 21 | public void Write(string key, string offset) 22 | { 23 | _graphProvider.WriteKey(DBName, key, offset); 24 | } 25 | 26 | public string Get(string key) 27 | { 28 | var value = _graphProvider.ReadKey(DBName, key); 29 | if (value == null) { 30 | return "-1"; 31 | } 32 | return value; 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /Hexastore.ScaleConsole/JsonGenerator.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using Newtonsoft.Json.Linq; 4 | 5 | namespace Hexastore.TestCommon 6 | { 7 | public static class JsonGenerator 8 | { 9 | public static JObject GenerateTelemetry(string id, Dictionary numberValues) 10 | { 11 | return new JObject(new JProperty("id", id), 12 | new JProperty("data", new JObject( 13 | numberValues.Select(e => new JProperty(e.Key, e.Value))))); 14 | } 15 | 16 | public static JObject GenerateTelemetry(string id, Dictionary numberValues, Dictionary stringValues) 17 | { 18 | return new JObject(new JProperty("id", id), 19 | new JProperty("data", new JObject( 20 | numberValues.Select(e => new JProperty(e.Key, e.Value)), 21 | stringValues.Select(e => new JProperty(e.Key, e.Value))))); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Hexastore.Test/Hexastore.Test.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp3.1 5 | 6 | false 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | PreserveNewest 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /Hexastore/Store/SetProvider.cs: -------------------------------------------------------------------------------- 1 | using Hexastore.Store; 2 | using System; 3 | using System.Collections.Concurrent; 4 | 5 | public class SetProvider : IStoreProvider, IDisposable 6 | { 7 | private readonly IGraphProvider _graphProvider; 8 | private readonly ConcurrentDictionary _setList; 9 | 10 | public SetProvider(IGraphProvider graphProvider) 11 | { 12 | _graphProvider = graphProvider; 13 | _setList = new ConcurrentDictionary(); 14 | } 15 | 16 | public IStore GetOrAdd(string setId) 17 | { 18 | return _setList.GetOrAdd(setId, (key) => Store.Create(key, _graphProvider)); 19 | } 20 | 21 | public bool Contains(string id) 22 | { 23 | return _setList.ContainsKey(id); 24 | } 25 | 26 | public IStore GetSet(string id) 27 | { 28 | return _setList[id]; 29 | } 30 | 31 | public void Dispose() 32 | { 33 | foreach (var item in _setList) { 34 | item.Value.Dispose(); 35 | } 36 | } 37 | } -------------------------------------------------------------------------------- /Hexastore.Rocks/RocksTest.cs: -------------------------------------------------------------------------------- 1 | using RocksDbSharp; 2 | using System; 3 | using System.IO; 4 | 5 | namespace Hexastore.Rocks 6 | { 7 | public class RocksTest 8 | { 9 | public RocksTest() 10 | { 11 | var dataPath = Path.Combine(Path.GetFullPath("."), "rockstest"); 12 | if (Directory.Exists(dataPath)) { 13 | Directory.Delete(dataPath, true); 14 | } 15 | Directory.CreateDirectory("/var/data/rockstest"); 16 | 17 | var options = new DbOptions().SetCreateIfMissing(true); 18 | using (var db = RocksDb.Open(options, "/var/data/rockstest")) { 19 | // Using strings below, but can also use byte arrays for both keys and values 20 | // much care has been taken to minimize buffer copying 21 | db.Put("key", "value1"); 22 | string value = db.Get("key"); 23 | Console.WriteLine(value); 24 | db.Remove("key"); 25 | } 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /Hexastore.Web/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "iisSettings": { 3 | "windowsAuthentication": false, 4 | "anonymousAuthentication": true, 5 | "iisExpress": { 6 | "applicationUrl": "http://localhost:54069", 7 | "sslPort": 0 8 | } 9 | }, 10 | "$schema": "http://json.schemastore.org/launchsettings.json", 11 | "profiles": { 12 | "IIS Express": { 13 | "commandName": "IISExpress", 14 | "launchBrowser": true, 15 | "launchUrl": "api/values", 16 | "environmentVariables": { 17 | "ASPNETCORE_ENVIRONMENT": "Development" 18 | } 19 | }, 20 | "Hexastore.Web": { 21 | "commandName": "Project", 22 | "launchUrl": "api/values", 23 | "environmentVariables": { 24 | "ASPNETCORE_ENVIRONMENT": "Development" 25 | }, 26 | "applicationUrl": "http://0.0.0.0:5000" 27 | }, 28 | "Docker": { 29 | "commandName": "Docker", 30 | "launchBrowser": true, 31 | "launchUrl": "{Scheme}://{ServiceHost}:{ServicePort}/api/values" 32 | } 33 | } 34 | } -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "label": "build", 6 | "command": "dotnet", 7 | "type": "process", 8 | "args": [ 9 | "build", 10 | "${workspaceFolder}/Hexastore.Web/Hexastore.Web.csproj" 11 | ], 12 | "problemMatcher": "$tsc" 13 | }, 14 | { 15 | "label": "publish", 16 | "command": "dotnet", 17 | "type": "process", 18 | "args": [ 19 | "publish", 20 | "${workspaceFolder}/Hexastore.Web/Hexastore.Web.csproj" 21 | ], 22 | "problemMatcher": "$tsc" 23 | }, 24 | { 25 | "label": "watch", 26 | "command": "dotnet", 27 | "type": "process", 28 | "args": [ 29 | "watch", 30 | "run", 31 | "${workspaceFolder}/Hexastore.Web/Hexastore.Web.csproj" 32 | ], 33 | "problemMatcher": "$tsc" 34 | } 35 | ] 36 | } -------------------------------------------------------------------------------- /Hexastore.Test/TripleObjectTest.cs: -------------------------------------------------------------------------------- 1 | using Hexastore.Graph; 2 | using Hexastore.Rocks; 3 | using Microsoft.VisualStudio.TestTools.UnitTesting; 4 | using Newtonsoft.Json.Linq; 5 | 6 | namespace Hexastore.Test 7 | { 8 | [TestClass] 9 | public class TripleObjectTest 10 | { 11 | [TestMethod] 12 | public void Convert_To_Byte_Roundtrip_Returns() 13 | { 14 | var o1 = new TripleObject("abc", true, JTokenType.String, null); 15 | var bytes1 = o1.ToBytes(); 16 | 17 | var roundtripped = bytes1.ToTripleObject(); 18 | Assert.IsTrue(roundtripped.IsID); 19 | Assert.AreEqual(o1.Value, roundtripped.Value); 20 | Assert.AreEqual(o1.TokenType, roundtripped.TokenType); 21 | 22 | var o2 = TripleObject.FromData(5); 23 | var bytes2 = o2.ToBytes(); 24 | 25 | var roundtripped2 = bytes2.ToTripleObject(); 26 | Assert.IsFalse(roundtripped2.IsID); 27 | Assert.AreEqual(o2.Value, roundtripped2.Value); 28 | Assert.AreEqual(o2.TokenType, roundtripped2.TokenType); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /Hexastore.Web/PerfHeaderMiddleware.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.AspNetCore.Http; 2 | using System; 3 | using System.Diagnostics; 4 | using System.Threading.Tasks; 5 | 6 | namespace Hexastore.Web 7 | { 8 | public class PerfHeaderMiddleware 9 | { 10 | private readonly RequestDelegate _next; 11 | 12 | public PerfHeaderMiddleware(RequestDelegate next) 13 | { 14 | _next = next; 15 | } 16 | 17 | public async Task InvokeAsync(HttpContext context) 18 | { 19 | var stopwatch = new Stopwatch(); 20 | stopwatch.Start(); 21 | context.Response.OnStarting(() => 22 | { 23 | stopwatch.Stop(); 24 | context.Response.Headers.Add("x-response-time", stopwatch.ElapsedMilliseconds.ToString()); 25 | context.Response.Headers.Add("x-cpus", Environment.ProcessorCount.ToString()); 26 | context.Response.Headers.Add("x-working-set", (Environment.WorkingSet / 1000_000).ToString()); 27 | return Task.CompletedTask; 28 | }); 29 | await _next(context); 30 | } 31 | } 32 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Angshuman Sarkar 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Hexastore.Web/StoreConfig.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | 6 | namespace Hexastore.Web 7 | { 8 | public class StoreConfig 9 | { 10 | public string EventHubConnectionString { get; } 11 | 12 | public int EventHubPartitionCount { get; } 13 | 14 | public bool ReplicationIsActive { get; } 15 | 16 | public string EventHubName { get; } 17 | 18 | public StoreConfig() 19 | { 20 | EventHubConnectionString = Environment.GetEnvironmentVariable("HEXASTORE_EVENTHUB_KEY"); 21 | int.TryParse(Environment.GetEnvironmentVariable("HEXASTORE_EVENTHUB_PARTITION_COUNT"), out var paritionCount); 22 | EventHubPartitionCount = paritionCount; 23 | EventHubName = Environment.GetEnvironmentVariable("HEXASTORE_EVENTHUB_NAME"); 24 | if (!string.IsNullOrEmpty(EventHubConnectionString) 25 | && EventHubPartitionCount != 0 26 | && !string.IsNullOrEmpty(EventHubName)) { 27 | ReplicationIsActive = true; 28 | } 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": ".NET Core Launch (web)", 9 | "type": "coreclr", 10 | "request": "launch", 11 | "preLaunchTask": "build", 12 | "program": "${workspaceFolder}/Hexastore.Web/bin/Debug/netcoreapp2.2/Hexastore.Web.dll", 13 | "args": [], 14 | "cwd": "${workspaceFolder}/Hexastore.Web", 15 | "stopAtEntry": false, 16 | "launchBrowser": { 17 | "enabled": true 18 | }, 19 | "env": { 20 | "ASPNETCORE_ENVIRONMENT": "Development" 21 | }, 22 | "sourceFileMap": { 23 | "/Views": "${workspaceFolder}/Views" 24 | } 25 | }, 26 | { 27 | "name": ".NET Core Attach", 28 | "type": "coreclr", 29 | "request": "attach", 30 | "processId": "${command:pickProcess}" 31 | } 32 | ] 33 | } -------------------------------------------------------------------------------- /Hexastore.Test/UnorderedTripleComparer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using Hexastore.Graph; 5 | using System.Collections; 6 | using Microsoft.VisualStudio.TestTools.UnitTesting; 7 | 8 | namespace Hexastore.Test 9 | { 10 | public class UnorderedTripleComparer : IComparer 11 | { 12 | public int Compare(object first, object second) 13 | { 14 | var x = first as Triple; 15 | var y = second as Triple; 16 | 17 | if (x == null || y == null) { 18 | throw new InvalidOperationException("cannot compare null"); 19 | } 20 | 21 | if (x.Subject == y.Subject && x.Predicate == y.Predicate && x.Object.IsID == y.Object.IsID && x.Object.TokenType == y.Object.TokenType && x.Object.Value == y.Object.Value) { 22 | return 0; 23 | } 24 | 25 | return string.Compare(x.Object.Value, y.Object.Value); 26 | } 27 | } 28 | 29 | public static class ListExtension 30 | { 31 | public static void Assert(this List list, string s, string p, TripleObject o) 32 | { 33 | list.Add(new Triple(s, p, o)); 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /Hexastore.Rocks/TripleObjectExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Runtime.InteropServices; 3 | using System.Text; 4 | using Hexastore.Graph; 5 | using Newtonsoft.Json.Linq; 6 | 7 | namespace Hexastore.Rocks 8 | { 9 | public static class TripleObjectExtensions 10 | { 11 | public static byte[] ToBytes(this TripleObject o) 12 | { 13 | var oBytes = KeyConfig.GetBytes(o.ToValue()); 14 | var isIdBytes = o.IsID ? KeyConfig.ByteTrue : KeyConfig.ByteFalse; 15 | var typeBytes = KeyConfig.GetBytes((int)o.TokenType); 16 | 17 | // index is not converted 18 | return KeyConfig.ConcatBytes(isIdBytes, typeBytes, oBytes); 19 | } 20 | 21 | public static TripleObject ToTripleObject(this byte[] input) 22 | { 23 | var isID = input[0] == KeyConfig.ByteTrue[0] ? true : false; 24 | var typeSpan = new ReadOnlySpan(input, 1, 4); 25 | var type = MemoryMarshal.Read(typeSpan); 26 | var valueLength = input.Length - 5; 27 | var value = Encoding.UTF8.GetString(input, 5, valueLength); 28 | return new TripleObject(value, isID, (JTokenType)type, null); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /Hexastore.Web/Hexastore.Web.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp3.1 5 | InProcess 6 | Linux 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | PreserveNewest 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /Hexastore.Web/EventHubs/StoreEvent.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using Hexastore.Errors; 6 | using Hexastore.Web.Errors; 7 | using Newtonsoft.Json; 8 | using Newtonsoft.Json.Linq; 9 | 10 | namespace Hexastore.Web.EventHubs 11 | { 12 | public class StoreEvent 13 | { 14 | [JsonProperty("operation")] 15 | public string Operation { get; set; } 16 | 17 | [JsonProperty("operationId")] 18 | public string OperationId { get; set; } 19 | 20 | [JsonProperty("data")] 21 | public JToken Data { get; set; } 22 | 23 | [JsonProperty("strict")] 24 | public bool Strict { get; set; } 25 | 26 | [JsonProperty("partitionId")] 27 | public string PartitionId { get; set; } 28 | 29 | [JsonProperty("storeId")] 30 | public string StoreId { get; set; } 31 | 32 | public static StoreEvent FromString(string str) 33 | { 34 | try { 35 | return JsonConvert.DeserializeObject(str); 36 | } catch (Exception e) { 37 | throw new InvalidOperationException("Cannot parse store event", e); 38 | } 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /Hexastore/Store/Store.cs: -------------------------------------------------------------------------------- 1 | using Hexastore.Graph; 2 | using System; 3 | using System.Collections.Generic; 4 | 5 | namespace Hexastore.Store 6 | { 7 | public class Store : IStore, IDisposable 8 | { 9 | private readonly string _id; 10 | private readonly IGraphProvider _provider; 11 | private readonly Dictionary _graphs = new Dictionary(); 12 | 13 | public static IStore Create(string id, IGraphProvider provider) 14 | { 15 | return new Store(id, provider); 16 | } 17 | 18 | public Store(string id, IGraphProvider provider) 19 | { 20 | _id = id; 21 | _provider = provider; 22 | _graphs[GraphType.Meta] = provider.CreateGraph(id, GraphType.Meta); 23 | _graphs[GraphType.Data] = provider.CreateGraph(id, GraphType.Data); 24 | _graphs[GraphType.Infer] = provider.CreateGraph(id, GraphType.Infer); 25 | } 26 | 27 | public void Dispose() 28 | { 29 | _graphs[GraphType.Meta].Dispose(); 30 | _graphs[GraphType.Meta].Dispose(); 31 | _graphs[GraphType.Meta].Dispose(); 32 | } 33 | 34 | public IStoreGraph GetGraph(GraphType type) 35 | { 36 | return _graphs[type]; 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /Hexastore/Graph/Triple.cs: -------------------------------------------------------------------------------- 1 | using Newtonsoft.Json; 2 | 3 | namespace Hexastore.Graph 4 | { 5 | public class Triple 6 | { 7 | public Triple(string s, string p, TripleObject o) 8 | { 9 | Subject = s; 10 | Predicate = p; 11 | Object = o; 12 | } 13 | 14 | public string Subject { get; private set; } 15 | 16 | public string Predicate { get; private set; } 17 | 18 | public TripleObject Object { get; private set; } 19 | 20 | public override string ToString() 21 | { 22 | return string.Format("<{0}> <{1}> {2} .", Subject, Predicate, Object); 23 | } 24 | 25 | public override bool Equals(object obj) 26 | { 27 | var t = obj as Triple; 28 | if (t == null) { 29 | return false; 30 | } 31 | if (object.ReferenceEquals(this, t)) { 32 | return true; 33 | } 34 | return t.Subject.Equals(Subject) && t.Predicate.Equals(Predicate) && t.Object.Equals(Object); 35 | } 36 | 37 | public string ToJson() 38 | { 39 | return JsonConvert.SerializeObject(this); 40 | } 41 | 42 | public override int GetHashCode() 43 | { 44 | return ToString().GetHashCode(); 45 | } 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/dotnet/core/sdk:3.1 AS build-env 2 | WORKDIR /app 3 | 4 | # Copy csproj and restore as distinct layers 5 | COPY ./Hexastore.Web/Hexastore.Web.csproj ./Hexastore.Web 6 | COPY ./Hexastore/Hexastore.csproj ./Hexastore 7 | COPY ./Hexastore.Rocks/Hexastore.Rocks.csproj ./Hexastore.Rocks.csproj 8 | 9 | RUN dotnet restore 10 | 11 | COPY . ./ 12 | 13 | RUN apt-get update && apt-get install -y libcap2-bin libsnappy1v5 && \ 14 | ln -s /lib/x86_64-linux-gnu/libdl.so.2 /usr/lib/x86_64-linux-gnu/libdl.so && \ 15 | ln -s /lib/x86_64-linux-gnu/libc.so.6 /usr/lib/x86_64-linux-gnu/libc.so && \ 16 | rm -rf /var/lib/apt/lists/* 17 | 18 | RUN dotnet test Hexastore.Test 19 | 20 | # Copy everything else and build 21 | RUN dotnet publish -c Release ./Hexastore.Web -o /app/out 22 | 23 | # Build runtime image 24 | FROM mcr.microsoft.com/dotnet/core/aspnet:3.1 25 | WORKDIR /app 26 | COPY --from=build-env /app/out . 27 | 28 | RUN apt-get update && apt-get install -y libcap2-bin libsnappy1v5 && \ 29 | ln -s /lib/x86_64-linux-gnu/libdl.so.2 /usr/lib/x86_64-linux-gnu/libdl.so && \ 30 | ln -s /lib/x86_64-linux-gnu/libc.so.6 /usr/lib/x86_64-linux-gnu/libc.so && \ 31 | rm -rf /var/lib/apt/lists/* 32 | 33 | ENV ENV LD_LIBRARY_PATH=ENV LD_LIBRARY_PATH:/app/bin/Debug/netcoreapp3.1/native/amd64/:/app/bin/Release/netcoreapp3.1/native/amd64/ 34 | EXPOSE 80 35 | ENTRYPOINT ["dotnet", "Hexastore.Web.dll"] 36 | -------------------------------------------------------------------------------- /Hexastore.Web/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM microsoft/dotnet:2.2-aspnetcore-runtime AS base 2 | RUN apt-get update && apt-get install -y libcap2-bin libsnappy1v5 && \ 3 | ln -s /lib/x86_64-linux-gnu/libdl.so.2 /usr/lib/x86_64-linux-gnu/libdl.so && \ 4 | ln -s /lib/x86_64-linux-gnu/libc.so.6 /usr/lib/x86_64-linux-gnu/libc.so && \ 5 | rm -rf /var/lib/apt/lists/* 6 | 7 | ENV ENV LD_LIBRARY_PATH=ENV LD_LIBRARY_PATH:/app/bin/Debug/netcoreapp2.2/native/amd64/:/app/bin/Release/netcoreapp2.2/native/amd64/ 8 | WORKDIR /app 9 | EXPOSE 80 10 | 11 | FROM microsoft/dotnet:2.2-sdk AS build 12 | WORKDIR /src 13 | COPY ["Hexastore.Web/Hexastore.Web.csproj", "Hexastore.Web/"] 14 | RUN dotnet restore "Hexastore.Web/Hexastore.Web.csproj" 15 | COPY . . 16 | WORKDIR "/src/Hexastore.Web" 17 | RUN dotnet build "Hexastore.Web.csproj" -c Release -o /app 18 | 19 | FROM build AS publish 20 | RUN dotnet publish "Hexastore.Web.csproj" -c Release -o /app 21 | 22 | FROM base AS final 23 | WORKDIR /app 24 | COPY --from=publish /app . 25 | COPY Hexastore.env /var/data 26 | RUN apt-get update && apt-get install -y libcap2-bin libsnappy1v5 && \ 27 | ln -s /lib/x86_64-linux-gnu/libdl.so.2 /usr/lib/x86_64-linux-gnu/libdl.so && \ 28 | ln -s /lib/x86_64-linux-gnu/libc.so.6 /usr/lib/x86_64-linux-gnu/libc.so && \ 29 | rm -rf /var/lib/apt/lists/* 30 | 31 | ENV ENV LD_LIBRARY_PATH=ENV LD_LIBRARY_PATH:/app/bin/Debug/netcoreapp2.2/native/amd64/:/app/bin/Release/netcoreapp2.2/native/amd64/ 32 | ENTRYPOINT ["dotnet", "Hexastore.Web.dll"] -------------------------------------------------------------------------------- /Hexastore.Web/Dockerfile.backup: -------------------------------------------------------------------------------- 1 | FROM microsoft/dotnet:2.2-aspnetcore-runtime AS base 2 | RUN apt-get update && apt-get install -y libcap2-bin libsnappy1v5 && \ 3 | ln -s /lib/arm-linux-gnueabihf/libdl.so.2 /usr/lib/arm-linux-gnueabihf/libdl.so && \ 4 | ln -s /lib/arm-linux-gnueabihf/libc.so.6 /usr/lib/arm-linux-gnueabihf/libc.so && \ 5 | rm -rf /var/lib/apt/lists/* 6 | WORKDIR /app 7 | EXPOSE 80 8 | 9 | FROM microsoft/dotnet:2.2-sdk AS build 10 | RUN apt-get update && apt-get install -y libcap2-bin libsnappy1v5 && \ 11 | ln -s /lib/arm-linux-gnueabihf/libdl.so.2 /usr/lib/arm-linux-gnueabihf/libdl.so && \ 12 | ln -s /lib/arm-linux-gnueabihf/libc.so.6 /usr/lib/arm-linux-gnueabihf/libc.so && \ 13 | rm -rf /var/lib/apt/lists/* 14 | 15 | WORKDIR /src 16 | COPY ["Hexastore.Web/Hexastore.Web.csproj", "Hexastore.Web/"] 17 | RUN dotnet restore "Hexastore.Web/Hexastore.Web.csproj" 18 | COPY . . 19 | WORKDIR "/src/Hexastore.Web" 20 | RUN dotnet build "Hexastore.Web.csproj" -c Release -o /app 21 | 22 | FROM build AS publish 23 | RUN dotnet publish "Hexastore.Web.csproj" -c Release -o /app 24 | 25 | FROM base AS final 26 | RUN apt-get update && apt-get install -y libcap2-bin libsnappy1v5 && \ 27 | ln -s /lib/arm-linux-gnueabihf/libdl.so.2 /usr/lib/arm-linux-gnueabihf/libdl.so && \ 28 | ln -s /lib/arm-linux-gnueabihf/libc.so.6 /usr/lib/arm-linux-gnueabihf/libc.so && \ 29 | rm -rf /var/lib/apt/lists/* 30 | WORKDIR /app 31 | COPY --from=publish /app . 32 | 33 | ENV LD_DEBUG=/app/bin/Debug/netcoreapp2.2/native/amd64:/app/bin/Release/netcoreapp2.2/native/amd64 34 | 35 | ENTRYPOINT ["dotnet", "Hexastore.Web.dll"] -------------------------------------------------------------------------------- /Hexastore/Constants.cs: -------------------------------------------------------------------------------- 1 | namespace Hexastore 2 | { 3 | public static class Constants 4 | { 5 | public static readonly string ExpandFilter = "expand"; 6 | public static readonly string SubjectFilter = "id"; 7 | public static readonly string TypeFilter = "type"; 8 | public static readonly string StrictFilter = "strict"; 9 | public static readonly string LevelFilter = "level"; 10 | public static readonly string LinkDelimeter = "#"; 11 | public static readonly string PathDelimeter = "/"; 12 | public static readonly string AnyPath = "*"; 13 | 14 | public static readonly string Skip = "skip"; 15 | public static readonly string Take = "take"; 16 | 17 | public static readonly string ID = "id"; 18 | public static readonly string Type = "type"; 19 | public static readonly string Graph = "@graph"; 20 | 21 | public static readonly string Data = "data:"; 22 | public static readonly string Meta = "meta:"; 23 | public static readonly string Infer = "infer:"; 24 | 25 | public static readonly string InverseOf = "inverseOf"; 26 | public static readonly string Domain = "domain"; 27 | public static readonly string Range = "range"; 28 | public static readonly string SubClassOf = "subClassOf"; 29 | public static readonly string SubPropertyOf = "subPropertyOf"; 30 | 31 | public static readonly string EventHubCheckpoint = "#ehc"; 32 | public static readonly string ConfigKeysPrefix = "#config"; 33 | 34 | public static readonly int DefaultPageSize = 25; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /Hexastore.Test/RocksFixture.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Runtime.Serialization; 5 | using System.Text; 6 | using Hexastore.Processor; 7 | using Hexastore.Resoner; 8 | using Hexastore.Rocks; 9 | using Hexastore.Store; 10 | using Microsoft.Extensions.Logging; 11 | using Microsoft.VisualStudio.TestTools.UnitTesting.Logging; 12 | using Moq; 13 | 14 | namespace Hexastore.Test 15 | { 16 | public class RocksFixture : IDisposable 17 | { 18 | public readonly RocksGraphProvider GraphProvider; 19 | public readonly IStoreProvider StoreProvider; 20 | public readonly IStoreProcesor StoreProcessor; 21 | public readonly IStoreOperationFactory StoreOperationFactory; 22 | public readonly string SetId; 23 | public readonly TestFolder TestFolder; 24 | 25 | public RocksFixture() 26 | { 27 | const string testDirectory = "./testdata"; 28 | if (Directory.Exists(testDirectory)) { 29 | Directory.Delete(testDirectory, true); 30 | } 31 | Directory.CreateDirectory(testDirectory); 32 | TestFolder = new TestFolder(); 33 | GraphProvider = new RocksGraphProvider(Mock.Of>(), TestFolder.Root); 34 | StoreProvider = new SetProvider(GraphProvider); 35 | StoreProcessor = new StoreProcessor(StoreProvider, new Reasoner(), Mock.Of>()); 36 | SetId = "testset"; 37 | } 38 | 39 | public void Dispose() 40 | { 41 | GraphProvider.Dispose(); 42 | TestFolder.Dispose(); 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /Hexastore.Test/TestFolder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | 4 | namespace Hexastore.Test 5 | { 6 | /// 7 | /// Temp directory that cleans up on dispose. 8 | /// 9 | public class TestFolder : IDisposable 10 | { 11 | // The actual root 12 | private readonly DirectoryInfo _parent; 13 | 14 | public string Root => RootDirectory.FullName; 15 | 16 | /// 17 | /// Delete the directory after completion. 18 | /// 19 | public bool CleanUp { get; set; } = true; 20 | 21 | /// 22 | /// Root directory, parent of the working directory 23 | /// 24 | public DirectoryInfo RootDirectory { get; } 25 | 26 | public TestFolder() 27 | { 28 | var parts = Guid.NewGuid().ToString().ToLowerInvariant().Split('-'); 29 | 30 | _parent = new DirectoryInfo(Path.Combine(Path.GetTempPath(), parts[0])); 31 | _parent.Create(); 32 | 33 | File.WriteAllText(Path.Combine(_parent.FullName, "trace.txt"), Environment.StackTrace); 34 | 35 | RootDirectory = new DirectoryInfo(Path.Combine(_parent.FullName, parts[1])); 36 | RootDirectory.Create(); 37 | } 38 | 39 | public static implicit operator string(TestFolder folder) 40 | { 41 | return folder.Root; 42 | } 43 | 44 | public override string ToString() 45 | { 46 | return Root; 47 | } 48 | 49 | public void Dispose() 50 | { 51 | if (CleanUp) { 52 | try { 53 | _parent.Delete(true); 54 | } catch { 55 | } 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /hexadb-readme-incoming-path.svg: -------------------------------------------------------------------------------- 1 | type = sensorsensorsname contains "Room"incoming -------------------------------------------------------------------------------- /Hexastore/Errors/StoreError.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using Hexastore.Web.Errors; 5 | 6 | namespace Hexastore.Errors 7 | { 8 | public class StoreError 9 | { 10 | // 400 11 | public string InvalidTypeError => "400.001"; 12 | public string InvalidItemError => "400.002"; 13 | public string MustHaveIdError => "400.003"; 14 | public string AtLestOneFilterError => "400.004"; 15 | public string PathEmptyError => "400.005"; 16 | public string UnknownCOmparatorError => "400.006"; 17 | public string UnableToParseQuery => "400.007"; 18 | public string UnableToParseStoreEvent => "400.008"; 19 | public string MaxQueueSizeError => "429.001"; 20 | 21 | // 409 22 | public string AlreadyContainsIdError => "409.001"; 23 | 24 | // 500 25 | public string Unhandled => "500.001"; 26 | 27 | // Fixed errors 28 | public StoreException InvalidType => new StoreException("Valid input types should be array or object", InvalidTypeError); 29 | public StoreException InvalidItem => new StoreException("Input array should be made of objects", InvalidItemError); 30 | public StoreException MustHaveId => new StoreException("Must have a top level string id", MustHaveIdError); 31 | public StoreException AtLeastOneFilter => new StoreException("need at least one filter", AtLestOneFilterError); 32 | public StoreException PathEmpty => new StoreException("path cannot be empty", PathEmptyError); 33 | public StoreException UnknownComparator => new StoreException("Unknown Comparator Type", UnknownCOmparatorError); 34 | public StoreException MaxQueueSize => new StoreException("Queue size high", MaxQueueSizeError); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /Hexastore.Rocks/RocksSubjectGrouping.cs: -------------------------------------------------------------------------------- 1 | using System.Collections; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using Hexastore.Graph; 5 | using RocksDbSharp; 6 | 7 | namespace Hexastore.Rocks 8 | { 9 | public class RocksSubjectGrouping : IGrouping> 10 | { 11 | private readonly RocksDb _db; 12 | private readonly string _name; 13 | private readonly string _s; 14 | private readonly byte[] _start; 15 | private readonly byte[] _end; 16 | private readonly int _prefixLength; 17 | 18 | public RocksSubjectGrouping(RocksDb db, string name, string s, byte[] start, byte[] end) 19 | { 20 | _db = db; 21 | _name = name; 22 | _s = s; 23 | _start = start; 24 | _end = end; 25 | _prefixLength = name.Length + 3; 26 | } 27 | 28 | public string Key => _s; 29 | 30 | public IEnumerator GetEnumerator() 31 | { 32 | throw new System.NotImplementedException(); 33 | } 34 | 35 | IEnumerator> IEnumerable>.GetEnumerator() 36 | { 37 | var predicates = new RocksEnumerable(_db, _start, _end, (it) => 38 | { 39 | var key = it.Key(); 40 | var splits = KeyConfig.Split(key); 41 | var nextKey = KeyConfig.ConcatBytes(splits[0], KeyConfig.ByteZero, splits[1], KeyConfig.ByteZero, splits[2], KeyConfig.ByteOne); 42 | return it.Seek(nextKey); 43 | }).Select(x => x.Predicate); 44 | 45 | foreach (var p in predicates) { 46 | yield return new RocksPredicateGrouping(_db, _name, _s, p); 47 | } 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /Hexastore.Rocks/RocksPredicateGrouping.cs: -------------------------------------------------------------------------------- 1 | using System.Collections; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using Hexastore.Graph; 5 | using RocksDbSharp; 6 | 7 | namespace Hexastore.Rocks 8 | { 9 | internal class RocksPredicateGrouping : IGrouping 10 | { 11 | private readonly RocksDb _db; 12 | private readonly string _s; 13 | private readonly string _p; 14 | private readonly string _name; 15 | 16 | public RocksPredicateGrouping(RocksDb db, string name, string s, string p) 17 | { 18 | _db = db; 19 | _s = s; 20 | _p = p; 21 | _name = name; 22 | } 23 | 24 | public string Key => _p; 25 | 26 | public IEnumerator GetEnumerator() 27 | { 28 | var sBytes = Hash($"{_name}.S"); 29 | var sh = Hash(_s); 30 | var ph = Hash(_p); 31 | var start = KeyConfig.ConcatBytes(sBytes, KeyConfig.ByteZero, sh, KeyConfig.ByteZero, ph, KeyConfig.ByteZero); 32 | var end = KeyConfig.ConcatBytes(sBytes, KeyConfig.ByteZero, sh, KeyConfig.ByteZero, ph, KeyConfig.ByteOne); 33 | 34 | var baseKey = $"{_name}.S.{Hash(_s)}.{Hash(_p)}"; 35 | IEnumerable idEnum = new RocksEnumerable(_db, start, end, (iterator) => 36 | { 37 | return iterator.Next(); 38 | }); 39 | 40 | foreach (var item in idEnum) { 41 | yield return item.Object; 42 | } 43 | } 44 | 45 | IEnumerator IEnumerable.GetEnumerator() 46 | { 47 | throw new System.NotImplementedException(); 48 | } 49 | 50 | private byte[] Hash(string input) 51 | { 52 | return KeyConfig.GetBytes(input); 53 | } 54 | } 55 | } -------------------------------------------------------------------------------- /Hexastore.PerfConsole/TestFolder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | 4 | namespace Hexastore.TestCommon 5 | { 6 | /// 7 | /// Temp directory that cleans up on dispose. 8 | /// 9 | public class TestFolder : IDisposable 10 | { 11 | // The actual root 12 | private readonly DirectoryInfo _parent; 13 | 14 | public string Root => RootDirectory.FullName; 15 | 16 | /// 17 | /// Delete the directory after completion. 18 | /// 19 | public bool CleanUp { get; set; } = true; 20 | 21 | /// 22 | /// Root directory, parent of the working directory 23 | /// 24 | public DirectoryInfo RootDirectory { get; } 25 | 26 | public TestFolder() 27 | { 28 | var parts = Guid.NewGuid().ToString().ToLowerInvariant().Split('-'); 29 | 30 | _parent = new DirectoryInfo(Path.Combine(Path.GetTempPath(), parts[0])); 31 | _parent.Create(); 32 | 33 | File.WriteAllText(Path.Combine(_parent.FullName, "trace.txt"), Environment.StackTrace); 34 | 35 | RootDirectory = new DirectoryInfo(Path.Combine(_parent.FullName, parts[1])); 36 | RootDirectory.Create(); 37 | } 38 | 39 | public static implicit operator string(TestFolder folder) 40 | { 41 | return folder.Root; 42 | } 43 | 44 | public override string ToString() 45 | { 46 | return Root; 47 | } 48 | 49 | public void Dispose() 50 | { 51 | if (CleanUp) 52 | { 53 | try 54 | { 55 | _parent.Delete(true); 56 | } 57 | catch 58 | { 59 | } 60 | } 61 | } 62 | } 63 | } -------------------------------------------------------------------------------- /Hexastore.ScaleConsole/TestFolder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | 4 | namespace Hexastore.TestCommon 5 | { 6 | /// 7 | /// Temp directory that cleans up on dispose. 8 | /// 9 | public class TestFolder : IDisposable 10 | { 11 | // The actual root 12 | private readonly DirectoryInfo _parent; 13 | 14 | public string Root => RootDirectory.FullName; 15 | 16 | /// 17 | /// Delete the directory after completion. 18 | /// 19 | public bool CleanUp { get; set; } = true; 20 | 21 | /// 22 | /// Root directory, parent of the working directory 23 | /// 24 | public DirectoryInfo RootDirectory { get; } 25 | 26 | public TestFolder() 27 | { 28 | var parts = Guid.NewGuid().ToString().ToLowerInvariant().Split('-'); 29 | 30 | _parent = new DirectoryInfo(Path.Combine(Path.GetTempPath(), parts[0])); 31 | _parent.Create(); 32 | 33 | File.WriteAllText(Path.Combine(_parent.FullName, "trace.txt"), Environment.StackTrace); 34 | 35 | RootDirectory = new DirectoryInfo(Path.Combine(_parent.FullName, parts[1])); 36 | RootDirectory.Create(); 37 | } 38 | 39 | public static implicit operator string(TestFolder folder) 40 | { 41 | return folder.Root; 42 | } 43 | 44 | public override string ToString() 45 | { 46 | return Root; 47 | } 48 | 49 | public void Dispose() 50 | { 51 | if (CleanUp) 52 | { 53 | try 54 | { 55 | _parent.Delete(true); 56 | } 57 | catch 58 | { 59 | } 60 | } 61 | } 62 | } 63 | } -------------------------------------------------------------------------------- /Hexastore.Web/Queue/QueueContainer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Threading.Tasks; 4 | using Hexastore.Errors; 5 | using Hexastore.Web.EventHubs; 6 | using Microsoft.Extensions.Logging; 7 | 8 | namespace Hexastore.Web.Queue 9 | { 10 | public class QueueContainer : IQueueContainer, IDisposable 11 | { 12 | private readonly EventReceiver _eventReceiver; 13 | private readonly QueueWriter[] _queueWriters; 14 | private readonly ILogger _logger; 15 | private readonly int _count = 32; 16 | private bool _running = true; 17 | 18 | public QueueContainer(EventReceiver eventReceiver, ILogger logger, StoreError storeError, int maxQueueSize = 0) 19 | { 20 | _logger = logger; 21 | _eventReceiver = eventReceiver; 22 | _queueWriters = new QueueWriter[_count]; 23 | for(int i=0; i< _count; i++) { 24 | _queueWriters[i] = new QueueWriter(eventReceiver, logger, storeError, maxQueueSize); 25 | } 26 | _ = _eventReceiver.LogCount(); 27 | _ = LogQueueLength(); 28 | } 29 | 30 | public void Dispose() 31 | { 32 | _running = false; 33 | foreach(var q in _queueWriters) { 34 | q.Dispose(); 35 | } 36 | } 37 | 38 | public int Count() 39 | { 40 | return _queueWriters.Sum(x => x.Length); 41 | } 42 | 43 | public void Send(StoreEvent storeEvent) 44 | { 45 | // todo: Add a max size of the queue then fail 46 | var partitionId = Math.Abs(Hasher.GetFnvHash32(storeEvent.PartitionId) % _count); 47 | _queueWriters[partitionId].Send(storeEvent); 48 | } 49 | 50 | private async Task LogQueueLength() 51 | { 52 | while (_running) { 53 | await Task.Delay(10000); 54 | _logger.LogInformation($"{DateTime.Now.ToString("hh':'mm':'ss")} Queue Length: {_queueWriters.Sum(x=> x.Length)}"); 55 | } 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /hexadb-readme-outgoing-path.svg: -------------------------------------------------------------------------------- 1 | type = roomgreen > 9sensorsmarkeroutgoing -------------------------------------------------------------------------------- /Hexastore.Web/Queue/QueueWriter.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Threading; 4 | using System.Threading.Tasks; 5 | using Hexastore.Errors; 6 | using Hexastore.Web.EventHubs; 7 | using Microsoft.Extensions.Logging; 8 | 9 | namespace Hexastore.Web.Queue 10 | { 11 | public class QueueWriter : IDisposable 12 | { 13 | private readonly BlockingCollection _queue; 14 | private readonly EventReceiver _eventReceiver; 15 | private readonly StoreError _storeError; 16 | private readonly ILogger _logger; 17 | private readonly Task _task; 18 | private readonly CancellationTokenSource _cts = new CancellationTokenSource(); 19 | private readonly int DefaultMaxQueueSize = 4096; 20 | private readonly int _maxQueueSize; 21 | 22 | public QueueWriter(EventReceiver eventReceiver, ILogger logger, StoreError storeError, int maxQueueSize) 23 | { 24 | _eventReceiver = eventReceiver; 25 | _storeError = storeError; 26 | _queue = new BlockingCollection(); 27 | _logger = logger; 28 | _maxQueueSize = maxQueueSize > 0 ? maxQueueSize : DefaultMaxQueueSize; 29 | _task = StartReader(); 30 | } 31 | 32 | public int Length => _queue.Count; 33 | 34 | public void Dispose() 35 | { 36 | _cts.Cancel(); 37 | _queue.Dispose(); 38 | } 39 | 40 | public void Send(StoreEvent storeEvent) 41 | { 42 | if (_queue.Count > _maxQueueSize) { 43 | throw _storeError.MaxQueueSize; 44 | } 45 | _queue.Add(storeEvent); 46 | } 47 | 48 | private Task StartReader() 49 | { 50 | return Task.Run(() => { 51 | while (!_cts.IsCancellationRequested) { 52 | try { 53 | var storeEvent = _queue.Take(_cts.Token); 54 | _eventReceiver.ProcessEventsAsync(storeEvent); 55 | } catch(Exception e) { 56 | _logger.LogError("Error in reading from queue {e}", e); 57 | } 58 | } 59 | }); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /Hexastore/Graph/GraphOperator.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using Newtonsoft.Json.Linq; 4 | 5 | namespace Hexastore.Graph 6 | { 7 | public class GraphOperator 8 | { 9 | public static IEnumerable Path(IGraph graph, string id, params string[] paths) 10 | { 11 | return PathImpl(graph, id, paths); 12 | } 13 | 14 | public static IEnumerable PathObjects(IGraph graph, string id, params string[] paths) 15 | { 16 | return PathImpl(graph, id, paths).Select(x => x.Object); 17 | } 18 | 19 | public static IEnumerable Expand(IGraph graph, string id, int level, params string[] expand) 20 | { 21 | if (level == 0) { 22 | yield break; 23 | } 24 | var triples = graph.S(id).ToList(); 25 | foreach (var t in triples) { 26 | yield return t; 27 | if (t.Object.IsID && HasValue(t.Predicate, expand)) { 28 | var rsp = Expand(graph, t.Object.Id, level - 1, expand); 29 | foreach (var expanded in rsp) { 30 | yield return expanded; 31 | } 32 | } 33 | } 34 | } 35 | 36 | private static IEnumerable PathImpl(IGraph graph, string id, string[] paths) 37 | { 38 | if (paths.Length == 1) { 39 | var destination = paths[0]; 40 | var rsp = graph.SP(id, destination); 41 | return rsp; 42 | } 43 | 44 | var destinations = graph.SP(id, paths[0]).Where(x => x.Object.IsID).Select(x => x.Object.Id); 45 | var newPaths = paths.Skip(1).ToArray(); 46 | var all = new List(); 47 | foreach (var item in destinations) { 48 | all.AddRange(PathImpl(graph, item, newPaths)); 49 | } 50 | return all; 51 | } 52 | 53 | private static bool HasValue(string item, IEnumerable list) 54 | { 55 | if (list == null || !list.Any()) { 56 | return true; 57 | } 58 | return list.Contains(item); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /Hexastore/Store/MemoryGraphProvider.cs: -------------------------------------------------------------------------------- 1 | using Hexastore.Graph; 2 | using System; 3 | using System.Collections.Generic; 4 | 5 | namespace Hexastore.Store 6 | { 7 | public class MemoryGraphProvider : IGraphProvider 8 | { 9 | private readonly object _lockObject = new object(); // todo: this needs to be per app 10 | private readonly IDictionary _store; 11 | 12 | public MemoryGraphProvider() 13 | { 14 | _store = new Dictionary(); 15 | } 16 | 17 | public bool ContainsGraph(string id, GraphType type) 18 | { 19 | return _store.ContainsKey(MakeKey(id, type)); 20 | } 21 | 22 | public IStoreGraph CreateGraph(string id, GraphType type) 23 | { 24 | string key = MakeKey(id, type); 25 | lock (_lockObject) { 26 | if (!_store.ContainsKey(key)) { 27 | _store[key] = new MemoryGraph(); 28 | return _store[key]; 29 | } 30 | throw new InvalidOperationException($"Graph exists {id} {type}"); 31 | } 32 | } 33 | 34 | public bool DeleteGraph(string id, GraphType type) 35 | { 36 | string key = MakeKey(id, type); 37 | if (!_store.ContainsKey(key)) { 38 | _store[key] = null; 39 | return true; 40 | } 41 | return false; 42 | } 43 | 44 | public IStoreGraph GetGraph(string id, GraphType type) 45 | { 46 | string key = MakeKey(id, type); 47 | if (!_store.ContainsKey($"{Constants.Data}{id}")) { 48 | throw new InvalidOperationException($"Cannot find graph id:{id} type:{type}"); 49 | } 50 | return _store[key]; 51 | } 52 | 53 | public string ReadKey(string id, string key) 54 | { 55 | throw new NotImplementedException(); 56 | } 57 | 58 | public void WriteKey(string id, string key, string value) 59 | { 60 | throw new NotImplementedException(); 61 | } 62 | 63 | private string MakeKey(string id, GraphType type) 64 | { 65 | return $"{type}:{id}"; ; 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /Hexastore.PerfConsole/PatchAddNew.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using BenchmarkDotNet.Attributes; 4 | using BenchmarkDotNet.Engines; 5 | using Hexastore.Processor; 6 | using Hexastore.Resoner; 7 | using Hexastore.Rocks; 8 | using Hexastore.TestCommon; 9 | using Microsoft.Extensions.Logging; 10 | 11 | namespace Hexastore.PerfConsole 12 | { 13 | /** 14 | * Add only new telemetry to the db using Patch 15 | * 16 | * A new database will be created for each invocation. 17 | */ 18 | [SimpleJob(RunStrategy.Monitoring, invocationCount: 20)] 19 | public class PatchAddNew 20 | { 21 | private List _ids = new List(); 22 | private List _dataPoints = new List(); 23 | private const int _maxIds = 1000; 24 | private const int _maxPoints = 20; 25 | 26 | 27 | [GlobalSetup] 28 | public void Setup() 29 | { 30 | while (_ids.Count < _maxIds) 31 | { 32 | _ids.Add(Guid.NewGuid().ToString()); 33 | } 34 | 35 | while (_dataPoints.Count < _maxPoints) 36 | { 37 | _dataPoints.Add(Guid.NewGuid().ToString()); 38 | } 39 | } 40 | 41 | [Benchmark] 42 | public void RunTest() 43 | { 44 | using (var testFolder = new TestFolder()) 45 | { 46 | var factory = new LoggerFactory(); 47 | var logger = factory.CreateLogger(); 48 | var storeLogger = factory.CreateLogger(); 49 | var provider = new RocksGraphProvider(logger, testFolder); 50 | var storeProvider = new SetProvider(provider); 51 | var storeProcessor = new StoreProcessor(storeProvider, new Reasoner(), storeLogger); 52 | 53 | 54 | var storeId = "test"; 55 | var points = new Dictionary(); 56 | var pointCount = 0; 57 | foreach (var pointId in _dataPoints) 58 | { 59 | pointCount++; 60 | points.Add(pointId, pointCount + 0.234); 61 | } 62 | 63 | foreach (var id in _ids) 64 | { 65 | var json = JsonGenerator.GenerateTelemetry(id, points); 66 | storeProcessor.PatchJson(storeId, json); 67 | } 68 | } 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /hexadb-readme-outgoing-level.svg: -------------------------------------------------------------------------------- 1 | type = room -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /Hexastore.PerfConsole/PatchUpdateNoChange.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using BenchmarkDotNet.Attributes; 4 | using BenchmarkDotNet.Engines; 5 | using Hexastore.Processor; 6 | using Hexastore.Resoner; 7 | using Hexastore.Rocks; 8 | using Hexastore.TestCommon; 9 | using Microsoft.Extensions.Logging; 10 | 11 | namespace Hexastore.PerfConsole 12 | { 13 | /** 14 | * Create a single database for all invocations. 15 | * 16 | * Repeat the same telemetry data multiple times. 17 | */ 18 | [SimpleJob(RunStrategy.Monitoring, invocationCount: 20)] 19 | public class PatchUpdateNoChange 20 | { 21 | private TestFolder _testFolder; 22 | private StoreProcessor _storeProcessor; 23 | private List _ids = new List(); 24 | private List _dataPoints = new List(); 25 | private const int _updateCount = 200; 26 | private const int _maxIds = 10; 27 | private const int _maxPoints = 3; 28 | 29 | 30 | [GlobalSetup] 31 | public void Setup() 32 | { 33 | _testFolder = new TestFolder(); 34 | 35 | while (_ids.Count < _maxIds) 36 | { 37 | _ids.Add(Guid.NewGuid().ToString()); 38 | } 39 | 40 | while (_dataPoints.Count < _maxPoints) 41 | { 42 | _dataPoints.Add(Guid.NewGuid().ToString()); 43 | } 44 | 45 | var factory = new LoggerFactory(); 46 | var logger = factory.CreateLogger(); 47 | var storeLogger = factory.CreateLogger(); 48 | var provider = new RocksGraphProvider(logger, _testFolder); 49 | var storeProvider = new SetProvider(provider); 50 | _storeProcessor = new StoreProcessor(storeProvider, new Reasoner(), storeLogger); 51 | } 52 | 53 | [GlobalCleanup] 54 | public void Cleanup() 55 | { 56 | _testFolder.Dispose(); 57 | } 58 | 59 | [Benchmark] 60 | public void RunTest() 61 | { 62 | var storeId = "test"; 63 | var points = new Dictionary(); 64 | var pointCount = 0; 65 | foreach (var pointId in _dataPoints) 66 | { 67 | pointCount++; 68 | points.Add(pointId, pointCount + 0.234); 69 | } 70 | 71 | for (var i = 0; i < _updateCount; i++) 72 | { 73 | foreach (var id in _ids) 74 | { 75 | var json = JsonGenerator.GenerateTelemetry(id, points); 76 | _storeProcessor.PatchJson(storeId, json); 77 | } 78 | } 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /Hexastore.PerfConsole/PatchUpdateSingle.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using BenchmarkDotNet.Attributes; 4 | using BenchmarkDotNet.Engines; 5 | using Hexastore.Processor; 6 | using Hexastore.Resoner; 7 | using Hexastore.Rocks; 8 | using Hexastore.TestCommon; 9 | using Microsoft.Extensions.Logging; 10 | 11 | namespace Hexastore.PerfConsole 12 | { 13 | /** 14 | * Create a single database for all invocations. 15 | * 16 | * Update telemetry for only 1 id. 17 | */ 18 | [SimpleJob(RunStrategy.Monitoring, invocationCount: 20)] 19 | public class PatchUpdateSingle 20 | { 21 | private TestFolder _testFolder; 22 | private StoreProcessor _storeProcessor; 23 | private List _ids = new List(); 24 | private List _dataPoints = new List(); 25 | private const int _updateCount = 100; 26 | private const int _maxIds = 1; 27 | private const int _maxPoints = 10; 28 | 29 | 30 | [GlobalSetup] 31 | public void Setup() 32 | { 33 | _testFolder = new TestFolder(); 34 | 35 | while (_ids.Count < _maxIds) 36 | { 37 | _ids.Add(Guid.NewGuid().ToString()); 38 | } 39 | 40 | while (_dataPoints.Count < _maxPoints) 41 | { 42 | _dataPoints.Add(Guid.NewGuid().ToString()); 43 | } 44 | 45 | var factory = new LoggerFactory(); 46 | var logger = factory.CreateLogger(); 47 | var storeLogger = factory.CreateLogger(); 48 | var provider = new RocksGraphProvider(logger, _testFolder); 49 | var storeProvider = new SetProvider(provider); 50 | _storeProcessor = new StoreProcessor(storeProvider, new Reasoner(), storeLogger); 51 | } 52 | 53 | [GlobalCleanup] 54 | public void Cleanup() 55 | { 56 | _testFolder.Dispose(); 57 | } 58 | 59 | [Benchmark] 60 | public void RunTest() 61 | { 62 | var storeId = "test"; 63 | var points = new Dictionary(); 64 | var pointCount = 0; 65 | foreach (var pointId in _dataPoints) 66 | { 67 | pointCount++; 68 | points.Add(pointId, pointCount + 0.234); 69 | } 70 | int x = 0; 71 | 72 | for (var i = 0; i < _updateCount; i++) 73 | { 74 | foreach (var id in _ids) 75 | { 76 | x++; 77 | 78 | foreach (var key in _dataPoints) 79 | { 80 | points[key] += x; 81 | } 82 | 83 | var json = JsonGenerator.GenerateTelemetry(id, points); 84 | _storeProcessor.PatchJson(storeId, json); 85 | } 86 | } 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /Hexastore.PerfConsole/PatchUpdate.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using BenchmarkDotNet.Attributes; 4 | using BenchmarkDotNet.Engines; 5 | using Hexastore.Processor; 6 | using Hexastore.Resoner; 7 | using Hexastore.Rocks; 8 | using Hexastore.TestCommon; 9 | using Microsoft.Extensions.Logging; 10 | 11 | namespace Hexastore.PerfConsole 12 | { 13 | /** 14 | * Create a single database for all invocations. 15 | * 16 | * Update multiple ids changing the telemetry data points on each patch. 17 | */ 18 | [SimpleJob(RunStrategy.Monitoring, invocationCount: 20)] 19 | public class PatchUpdate 20 | { 21 | private TestFolder _testFolder; 22 | private StoreProcessor _storeProcessor; 23 | private List _ids = new List(); 24 | private List _dataPoints = new List(); 25 | private const int _updateCount = 50; 26 | private const int _maxIds = 10; 27 | private const int _maxPoints = 3; 28 | 29 | 30 | [GlobalSetup] 31 | public void Setup() 32 | { 33 | _testFolder = new TestFolder(); 34 | 35 | while (_ids.Count < _maxIds) 36 | { 37 | _ids.Add(Guid.NewGuid().ToString()); 38 | } 39 | 40 | while (_dataPoints.Count < _maxPoints) 41 | { 42 | _dataPoints.Add(Guid.NewGuid().ToString()); 43 | } 44 | 45 | var factory = new LoggerFactory(); 46 | var logger = factory.CreateLogger(); 47 | var storeLogger = factory.CreateLogger(); 48 | var provider = new RocksGraphProvider(logger, _testFolder); 49 | var storeProvider = new SetProvider(provider); 50 | _storeProcessor = new StoreProcessor(storeProvider, new Reasoner(), storeLogger); 51 | } 52 | 53 | [GlobalCleanup] 54 | public void Cleanup() 55 | { 56 | _testFolder.Dispose(); 57 | } 58 | 59 | [Benchmark] 60 | public void RunTest() 61 | { 62 | var storeId = "test"; 63 | var points = new Dictionary(); 64 | var pointCount = 0; 65 | foreach (var pointId in _dataPoints) 66 | { 67 | pointCount++; 68 | points.Add(pointId, pointCount + 0.234); 69 | } 70 | int x = 0; 71 | 72 | for (var i = 0; i < _updateCount; i++) 73 | { 74 | foreach (var id in _ids) 75 | { 76 | x++; 77 | 78 | foreach (var key in _dataPoints) 79 | { 80 | points[key] += x; 81 | } 82 | 83 | var json = JsonGenerator.GenerateTelemetry(id, points); 84 | _storeProcessor.PatchJson(storeId, json); 85 | } 86 | } 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /Hexastore.Web/Startup.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using dotenv.net; 4 | using Hexastore.Errors; 5 | using Hexastore.Processor; 6 | using Hexastore.Resoner; 7 | using Hexastore.Rocks; 8 | using Hexastore.Store; 9 | using Hexastore.Web.EventHubs; 10 | using Microsoft.AspNetCore.Builder; 11 | using Microsoft.AspNetCore.Hosting; 12 | using Microsoft.Extensions.Configuration; 13 | using Microsoft.Extensions.DependencyInjection; 14 | using Microsoft.Extensions.Hosting; 15 | using Microsoft.AspNetCore.Mvc.NewtonsoftJson; 16 | using Hexastore.Web.Queue; 17 | 18 | namespace Hexastore.Web 19 | { 20 | public class Startup 21 | { 22 | public Startup(IConfiguration configuration) 23 | { 24 | Configuration = configuration; 25 | } 26 | 27 | public IConfiguration Configuration { get; } 28 | 29 | // This method gets called by the runtime. Use this method to add services to the container. 30 | public void ConfigureServices(IServiceCollection services) 31 | { 32 | services.AddCors(options => { 33 | options.AddPolicy("AllowAnyOrigin", 34 | builder => builder 35 | .AllowAnyOrigin() 36 | .AllowAnyMethod() 37 | .WithExposedHeaders("content-disposition") 38 | .AllowAnyHeader() 39 | .SetPreflightMaxAge(TimeSpan.FromSeconds(3600))); 40 | }); 41 | 42 | 43 | services.AddControllers().AddNewtonsoftJson(); 44 | 45 | services.AddSingleton(); 46 | services.AddSingleton(); 47 | services.AddSingleton(); 48 | services.AddSingleton(); 49 | services.AddSingleton(); 50 | services.AddSingleton(); 51 | services.AddSingleton(); 52 | services.AddSingleton(); 53 | services.AddSingleton(); 54 | services.AddSingleton(); 55 | } 56 | 57 | // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. 58 | public void Configure(IApplicationBuilder app, IWebHostEnvironment env) 59 | { 60 | if (File.Exists("/var/data/Hexastore.env")) { 61 | DotEnv.Config(false, "/var/data/Hexastore.env"); 62 | } else if (File.Exists("Hexastore.env")) { 63 | DotEnv.Config(false, "Hexastore.env"); 64 | } 65 | 66 | if (env.IsDevelopment()) { 67 | app.UseDeveloperExceptionPage(); 68 | } 69 | app.UseCors("AllowAnyOrigin"); 70 | 71 | app.UseMiddleware(); 72 | 73 | // app.UseHttpsRedirection(); 74 | // app.UseAuthorization(); 75 | app.UseRouting(); 76 | 77 | app.UseEndpoints(endpoints => { 78 | endpoints.MapControllers(); 79 | }); 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /Hexastore/Query/ObjectQueryModel.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using Hexastore.Graph; 5 | using Newtonsoft.Json; 6 | using Newtonsoft.Json.Linq; 7 | 8 | namespace Hexastore.Query 9 | { 10 | public class ObjectQueryModel 11 | { 12 | [JsonProperty("id")] 13 | public string Id { get; set; } 14 | [JsonProperty("filter")] 15 | public IDictionary Filter { get; set; } 16 | [JsonProperty("continuation")] 17 | public Continuation Continuation { get; set; } 18 | [JsonProperty("pageSize")] 19 | public int PageSize { get; set; } 20 | [JsonProperty("incoming")] 21 | public LinkQuery[] HasSubject { get; set; } 22 | [JsonProperty("outgoing")] 23 | public LinkQuery[] HasObject { get; set; } 24 | [JsonProperty("aggregates")] 25 | public AggregateQuery[] Aggregates { get; set; } 26 | } 27 | 28 | public class AggregateQuery 29 | { 30 | [JsonProperty("type")] 31 | public AggregateType Type { get; set; } 32 | } 33 | 34 | public enum AggregateType 35 | { 36 | Count 37 | } 38 | 39 | public class ObjectQueryResponse 40 | { 41 | public IEnumerable Values { get; set; } 42 | public Continuation Continuation { get; set; } 43 | public object[] Aggregates { get; set; } 44 | } 45 | 46 | public class Continuation 47 | { 48 | [JsonProperty("s")] 49 | public string S { get; set; } 50 | [JsonProperty("p")] 51 | public string P { get; set; } 52 | [JsonProperty("o")] 53 | public JValue O { get; set; } 54 | [JsonProperty("i")] 55 | public bool IsId { get; set; } 56 | [JsonProperty("c")] 57 | public int Index { get; set; } 58 | 59 | 60 | public static implicit operator Continuation(Triple t) 61 | { 62 | return new Continuation() { S = t.Subject, P = t.Predicate, O = t.Object.ToTypedJSON(), IsId = t.Object.IsID }; 63 | } 64 | 65 | public override bool Equals(object obj) 66 | { 67 | var t = obj as Continuation; 68 | if (t == null) { 69 | return false; 70 | } 71 | if (object.ReferenceEquals(this, t)) { 72 | return true; 73 | } 74 | return t.S.Equals(S) && t.P.Equals(P) && t.O.CompareTo(O) == 0 && t.IsId == IsId; 75 | } 76 | 77 | public override int GetHashCode() 78 | { 79 | return (S + P + O + IsId.ToString()).GetHashCode(); 80 | } 81 | } 82 | 83 | public class QueryUnit 84 | { 85 | [JsonProperty("op")] 86 | public string Operator { get; set; } 87 | [JsonProperty("value")] 88 | public object Value { get; set; } 89 | } 90 | 91 | public class LinkQuery 92 | { 93 | [JsonProperty("level")] 94 | public int Level { get; set; } 95 | [JsonProperty("path")] 96 | public string Path { get; set; } 97 | [JsonProperty("target")] 98 | public ObjectQueryModel Target { get; set; } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /Hexastore.Rocks/TripleExtensions.cs: -------------------------------------------------------------------------------- 1 | using Hexastore.Graph; 2 | using Newtonsoft.Json.Linq; 3 | using System; 4 | using System.Text; 5 | 6 | namespace Hexastore.Rocks 7 | { 8 | public static class TripleExtensions 9 | { 10 | public static Triple ToTriple(this (byte[], byte[]) input) 11 | { 12 | var key = input.Item1; 13 | var value = input.Item2; 14 | 15 | byte z = KeyConfig.ByteZero[0]; 16 | var firstZPos = Array.IndexOf(key, z); 17 | string s; 18 | string p; 19 | string o; 20 | int index; 21 | bool isId; 22 | JTokenType type; 23 | 24 | int sEnd; 25 | int pEnd; 26 | int oEnd; 27 | int isIdEnd; 28 | 29 | switch (key[firstZPos - 1]) { 30 | case KeyConfig.SMark: 31 | sEnd = Array.IndexOf(key, z, firstZPos + 1); 32 | s = Encoding.UTF8.GetString(key, firstZPos + 1, sEnd - firstZPos - 1); 33 | pEnd = Array.IndexOf(key, z, sEnd + 1); 34 | p = Encoding.UTF8.GetString(key, sEnd + 1, pEnd - sEnd - 1); 35 | index = BitConverter.ToInt32(key, pEnd + 1); 36 | var to = value.ToTripleObject(); 37 | o = to.Value; 38 | isId = to.IsID; 39 | type = to.TokenType; 40 | break; 41 | case KeyConfig.PMark: 42 | pEnd = Array.IndexOf(key, z, firstZPos + 1); 43 | p = Encoding.UTF8.GetString(key, firstZPos + 1, pEnd - firstZPos - 1); 44 | isId = key[pEnd + 1] == KeyConfig.ByteTrue[0] ? true : false; 45 | isIdEnd = pEnd + 2; 46 | oEnd = Array.IndexOf(key, z, isIdEnd + 1); 47 | o = Encoding.UTF8.GetString(key, isIdEnd + 1, oEnd - isIdEnd - 1); 48 | sEnd = Array.IndexOf(key, z, oEnd + 1); 49 | s = Encoding.UTF8.GetString(key, oEnd + 1, sEnd - oEnd - 1); 50 | index = BitConverter.ToInt32(key, sEnd + 1); 51 | type = (JTokenType)BitConverter.ToInt32(value, 0); 52 | break; 53 | case KeyConfig.OMark: 54 | isIdEnd = firstZPos + 2; 55 | isId = key[firstZPos + 1] == KeyConfig.ByteTrue[0] ? true : false; 56 | oEnd = Array.IndexOf(key, z, isIdEnd + 1); 57 | o = Encoding.UTF8.GetString(key, isIdEnd + 1, oEnd - isIdEnd - 1); 58 | sEnd = Array.IndexOf(key, z, oEnd + 1); 59 | s = Encoding.UTF8.GetString(key, oEnd + 1, sEnd - oEnd - 1); 60 | pEnd = Array.IndexOf(key, z, sEnd + 1); 61 | p = Encoding.UTF8.GetString(key, sEnd + 1, pEnd - sEnd - 1); 62 | index = BitConverter.ToInt32(key, pEnd + 1); 63 | type = (JTokenType)BitConverter.ToInt32(value, 0); 64 | break; 65 | default: 66 | throw new InvalidOperationException("unmatched key. panic."); 67 | } 68 | 69 | return new Triple(s, p, new TripleObject(o, isId, type, index)); 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /Hexastore/Resoner/Reasoner.cs: -------------------------------------------------------------------------------- 1 | using Hexastore.Graph; 2 | 3 | namespace Hexastore.Resoner 4 | { 5 | public class Reasoner : IReasoner 6 | { 7 | public void Spin(IGraph data, IGraph infer, IGraph meta) 8 | { 9 | var union = new DisjointUnion(data, infer, string.Empty); 10 | bool any = false; 11 | do { 12 | any = false; 13 | any |= InverseOf(union, meta); 14 | any |= Domain(union, meta); 15 | any |= Range(union, meta); 16 | any |= SubClassOf(union, meta); 17 | any |= SubPropertyOf(union, meta); 18 | } while (any); 19 | } 20 | 21 | private bool Range(DisjointUnion graph, IGraph meta) 22 | { 23 | var any = false; 24 | var rules = meta.P(Constants.Range); 25 | foreach (var rule in rules) { 26 | var triples = graph.P(rule.Subject); 27 | foreach (var t in triples) { 28 | any |= graph.Assert(t.Object.ToValue(), Constants.Type, TripleObject.FromData(rule.Object.ToValue())); 29 | } 30 | } 31 | return any; 32 | } 33 | 34 | private bool Domain(DisjointUnion graph, IGraph meta) 35 | { 36 | var any = false; 37 | var rules = meta.P(Constants.Domain); 38 | foreach (var rule in rules) { 39 | var triples = graph.P(rule.Subject); 40 | foreach (var t in triples) { 41 | any |= graph.Assert(t.Subject, Constants.Type, TripleObject.FromData(rule.Object.ToValue())); 42 | } 43 | } 44 | return any; 45 | } 46 | 47 | private bool InverseOf(IGraph graph, IGraph meta) 48 | { 49 | var any = false; 50 | var rules = meta.P(Constants.InverseOf); 51 | foreach (var rule in rules) { 52 | var triples = graph.P(rule.Subject); 53 | foreach (var t in triples) { 54 | any |= graph.Assert(t.Object.ToValue(), rule.Object.ToValue(), t.Subject); 55 | } 56 | } 57 | return any; 58 | } 59 | 60 | private bool SubClassOf(IGraph graph, IGraph meta) 61 | { 62 | var any = false; 63 | var rules = meta.P(Constants.SubClassOf); 64 | foreach (var rule in rules) { 65 | var triples = graph.PO(Constants.Type, TripleObject.FromData(rule.Subject)); 66 | foreach (var t in triples) { 67 | any |= graph.Assert(t.Subject, Constants.Type, TripleObject.FromData(rule.Object.ToValue())); 68 | } 69 | } 70 | return any; 71 | } 72 | 73 | private bool SubPropertyOf(IGraph graph, IGraph meta) 74 | { 75 | var any = false; 76 | var rules = meta.P(Constants.SubPropertyOf); 77 | foreach (var rule in rules) { 78 | var triples = graph.P(rule.Subject); 79 | foreach (var t in triples) { 80 | any |= graph.Assert(t.Subject, rule.Object.ToValue(), t.Object); 81 | } 82 | } 83 | return any; 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /Hexastore.Rocks/RocksEnumerable.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Generic; 4 | using Hexastore.Graph; 5 | using RocksDbSharp; 6 | 7 | namespace Hexastore.Rocks 8 | { 9 | public class RocksEnumerable : IEnumerable 10 | { 11 | private readonly RocksDb _db; 12 | private readonly byte[] _start; 13 | private readonly byte[] _end; 14 | private readonly Func _nextFunction; 15 | 16 | public RocksEnumerable(RocksDb db, byte[] start, byte[] end, Func nextFunction) 17 | { 18 | _db = db; 19 | _start = start; 20 | _end = end; 21 | _nextFunction = nextFunction; 22 | } 23 | 24 | public IEnumerator GetEnumerator() 25 | { 26 | return new RocksEnumerator(_db, _start, _end, _nextFunction); 27 | } 28 | 29 | IEnumerator IEnumerable.GetEnumerator() 30 | { 31 | return new RocksEnumerator(_db, _start, _end, _nextFunction); 32 | } 33 | } 34 | 35 | public class RocksEnumerator : IEnumerator 36 | { 37 | private readonly RocksDb _db; 38 | private readonly byte[] _start; 39 | private readonly byte[] _end; 40 | private byte[] _currentKey; 41 | private readonly Func _nextFunction; 42 | private Iterator _iterator; 43 | 44 | public RocksEnumerator(RocksDb db, byte[] start, byte[] end, Func nextFunction) 45 | { 46 | _db = db; 47 | _start = start; 48 | _end = end; 49 | _currentKey = null; 50 | _nextFunction = nextFunction; 51 | } 52 | 53 | public Triple Current 54 | { 55 | get 56 | { 57 | var t = (_iterator.Key(), _iterator.Value()); 58 | return t.ToTriple(); 59 | } 60 | } 61 | 62 | object IEnumerator.Current => Current; 63 | 64 | public void Dispose() 65 | { 66 | if (_iterator != null) { 67 | _iterator.Dispose(); 68 | } 69 | } 70 | 71 | public bool MoveNext() 72 | { 73 | if (_currentKey == null) { 74 | _currentKey = _start; 75 | // todo: check 76 | _iterator = _db.NewIterator(); 77 | _iterator.Seek(_start); 78 | var firstKey = _iterator.Key(); 79 | if (KeyConfig.ByteCompare(firstKey, _start) < 0) { 80 | return false; 81 | } 82 | } else { 83 | _nextFunction(_iterator); 84 | _currentKey = _iterator.Key(); 85 | } 86 | 87 | if (!_iterator.Valid()) { 88 | return false; 89 | } 90 | 91 | var key = _iterator.Key(); 92 | if (KeyConfig.ByteCompare(key, _end) > 0) { 93 | return false; 94 | } 95 | return true; 96 | } 97 | 98 | public void Reset() 99 | { 100 | _currentKey = null; 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /Hexastore.Web/EventHubs/EventSender.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using System.Threading.Tasks; 5 | using Hexastore.Web.Queue; 6 | using Microsoft.Azure.EventHubs; 7 | using Newtonsoft.Json; 8 | using Newtonsoft.Json.Linq; 9 | 10 | namespace Hexastore.Web.EventHubs 11 | { 12 | public class EventSender : IDisposable 13 | { 14 | private readonly EventReceiver _storeReceiver; 15 | private readonly IQueueContainer _queueContainer; 16 | private readonly Checkpoint _checkpoint; 17 | private readonly StoreConfig _storeConfig; 18 | private readonly EventHubClient _eventHubClient; 19 | private readonly bool _active; 20 | 21 | private PartitionReceiver _receiver; 22 | 23 | public int MaxBatchSize 24 | { 25 | get { 26 | return 1000; 27 | } 28 | set { 29 | } 30 | } 31 | 32 | public EventSender(IQueueContainer queueContainer, EventReceiver storeReceiver, Checkpoint checkpoint, StoreConfig storeConfig) 33 | { 34 | _storeReceiver = storeReceiver; 35 | _queueContainer = queueContainer; 36 | _checkpoint = checkpoint; 37 | _storeConfig = storeConfig; 38 | if (_storeConfig.ReplicationIsActive) { 39 | _eventHubClient = EventHubClient.CreateFromConnectionString(_storeConfig.EventHubConnectionString); 40 | _ = StartListeners(); 41 | _active = true; 42 | } 43 | } 44 | 45 | public async Task SendMessage(StoreEvent storeEvent) 46 | { 47 | if (!_active) { 48 | // pass through 49 | _queueContainer.Send(storeEvent); 50 | return; 51 | } 52 | 53 | try { 54 | int ehPartitionId; 55 | if (!string.IsNullOrEmpty(storeEvent.PartitionId)) { 56 | ehPartitionId = storeEvent.PartitionId.GetHashCode() % _storeConfig.EventHubPartitionCount; 57 | } else { 58 | ehPartitionId = storeEvent.StoreId.GetHashCode() % _storeConfig.EventHubPartitionCount; 59 | } 60 | var content = JsonConvert.SerializeObject(storeEvent, Formatting.None); 61 | var bytes = Encoding.UTF8.GetBytes(content); 62 | await _eventHubClient.SendAsync(new EventData(bytes), ehPartitionId.ToString()); 63 | } catch (Exception e) { 64 | Console.WriteLine(e); 65 | } 66 | } 67 | 68 | public void Dispose() 69 | { 70 | if (_eventHubClient != null) { 71 | _eventHubClient.Close(); 72 | } 73 | } 74 | 75 | private async Task StartListeners() 76 | { 77 | var ehRuntime = await _eventHubClient.GetRuntimeInformationAsync(); 78 | foreach (var partitionId in ehRuntime.PartitionIds) { 79 | var cp = _checkpoint.Get($"{Constants.EventHubCheckpoint}.{_storeConfig.EventHubName}.{partitionId}"); 80 | _receiver = _eventHubClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, partitionId, EventPosition.FromOffset(cp ?? "-1")); 81 | _receiver.SetReceiveHandler(_storeReceiver); 82 | } 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /Hexastore.Test/TripleConverterTest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using Hexastore.Graph; 6 | using Hexastore.Parser; 7 | using Microsoft.VisualStudio.TestTools.UnitTesting; 8 | using Newtonsoft.Json.Linq; 9 | 10 | namespace Hexastore.Test 11 | { 12 | [TestClass] 13 | public class TripleConverterTest 14 | { 15 | public IGraph _set; 16 | 17 | public TripleConverterTest() 18 | { 19 | } 20 | 21 | [TestMethod] 22 | public void To_Triples() 23 | { 24 | var item = new 25 | { 26 | id = "100", 27 | name = "name100", 28 | age = (int)20, 29 | contains = new 30 | { 31 | name = "name200", 32 | values = new object[] { 33 | new 34 | { 35 | name = "name300", 36 | age = (int)30, 37 | }, 38 | new 39 | { 40 | name = "name400" 41 | } 42 | } 43 | } 44 | }; 45 | 46 | var json = JObject.FromObject(item); 47 | var graph = TripleConverter.FromJson(json); 48 | 49 | var expected = new List(); 50 | expected.Assert("100", "name", TripleObject.FromData("name100")); 51 | expected.Assert("100", "age", TripleObject.FromData(20)); 52 | expected.Assert("100", "contains", "100#contains"); 53 | expected.Assert("100#contains", "name", TripleObject.FromData("name200")); 54 | expected.Assert("100#contains", "values", "100#contains#values#0"); 55 | expected.Assert("100#contains#values#0", "name", TripleObject.FromData("name300")); 56 | expected.Assert("100#contains#values#0", "age", TripleObject.FromData(30)); 57 | expected.Assert("100#contains", "values", "100#contains#values#1"); 58 | expected.Assert("100#contains#values#1", "name", TripleObject.FromData("name400")); 59 | CollectionAssert.AreEqual(expected, graph.ToArray(), new UnorderedTripleComparer()); 60 | } 61 | 62 | [TestMethod] 63 | public void To_Patch() 64 | { 65 | var item = new 66 | { 67 | id = "100", 68 | name = "name101", 69 | contains = new 70 | { 71 | name = "name200", 72 | values = new object[] { 73 | new 74 | { 75 | age = (int?)null, 76 | } 77 | } 78 | } 79 | }; 80 | 81 | var json = JObject.FromObject(item); 82 | var graph = TripleConverter.FromJson(json); 83 | 84 | var expected = new List(); 85 | expected.Add(new Triple("100", "name", TripleObject.FromData("name101"))); 86 | expected.Add(new Triple("100", "contains", "100#contains")); 87 | expected.Add(new Triple("100#contains", "name", TripleObject.FromData("name200"))); 88 | expected.Add(new Triple("100#contains", "values", "100#contains#values#0")); 89 | expected.Add(new Triple("100#contains#values#0", "age", TripleObject.FromData(null))); 90 | CollectionAssert.AreEqual(expected, graph.ToArray(), new UnorderedTripleComparer()); 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /Hexastore/Graph/IGraph.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | 5 | namespace Hexastore.Graph 6 | { 7 | public interface IGraph : ICloneable, IDisposable 8 | { 9 | /// 10 | /// Gets the Name of the graph 11 | /// 12 | string Name { get; } 13 | 14 | /// 15 | /// Gets the Count of triples in this graph - this should be efficient 16 | /// 17 | int Count { get; } 18 | 19 | /// 20 | /// Assert adds a triple to the graph if its not already there 21 | /// 22 | bool Assert(Triple t); 23 | void Assert(IEnumerable triples); 24 | 25 | /// 26 | /// Assert adds a triple to the graph if its not already there 27 | /// 28 | bool Assert(string s, string p, TripleObject o); 29 | 30 | /// 31 | /// Merge another graph into this one: Assert all the triples in that graph into this graph 32 | /// 33 | IGraph Merge(IGraph g); 34 | 35 | /// 36 | /// Retracts a triple from the graph if its there 37 | /// 38 | bool Retract(Triple t); 39 | void Retract(IEnumerable t); 40 | 41 | /// 42 | /// Retracts a triple from the graph if its there 43 | /// 44 | bool Retract(string s, string p, TripleObject o); 45 | 46 | void BatchRetractAssert(IEnumerable retract, IEnumerable assert); 47 | 48 | /// 49 | /// Subtracts a whole graph from the graph: Retract all the triples in that graph from this graph 50 | /// 51 | IGraph Minus(IGraph g); 52 | 53 | /// 54 | /// S returns all the triples with this subject 55 | /// 56 | IEnumerable S(string s); 57 | 58 | /// 59 | /// P returns all the triples with this predicate 60 | /// 61 | IEnumerable P(string p); 62 | 63 | /// 64 | /// P returns all the predicates 65 | /// 66 | IEnumerable P(); 67 | 68 | /// 69 | /// O returns all the triples with this object 70 | /// 71 | IEnumerable O(TripleObject o); 72 | 73 | /// 74 | /// SP returns all the triples with this subject and this predicate 75 | /// 76 | IEnumerable SP(string s, string p); 77 | 78 | /// 79 | /// SP returns all the triples with this predicate and this object 80 | /// 81 | IEnumerable PO(string p, TripleObject o); 82 | 83 | /// 84 | /// OS returns all the triples with this object and subject 85 | /// 86 | IEnumerable OS(TripleObject o, string s); 87 | 88 | /// 89 | /// Returns all the triples 90 | /// 91 | IEnumerable GetTriples(); 92 | 93 | /// 94 | /// Tests whether the triple exists in the graph 95 | /// 96 | bool Exists(Triple t); 97 | 98 | /// 99 | /// Tests whether the triple exists in the graph 100 | /// 101 | bool Exists(string s, string p, TripleObject o); 102 | 103 | /// 104 | /// Efficiently enumerate over the whole graph 105 | /// 106 | IEnumerable>> GetGroupings(); 107 | 108 | /// 109 | /// Efficiently enumerate over a particular subject 110 | /// 111 | IEnumerable> GetSubjectGroupings(string s); 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /Hexastore.Test/ContinuationTest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using Hexastore.Graph; 6 | using Hexastore.Rocks; 7 | using Microsoft.VisualStudio.TestTools.UnitTesting; 8 | 9 | namespace Hexastore.Test 10 | { 11 | [TestClass] 12 | public class ContinuationTest : RocksFixture 13 | { 14 | private readonly RocksGraph _set; 15 | 16 | public ContinuationTest() 17 | { 18 | var (set, _, _) = StoreProcessor.GetGraphs(SetId); 19 | _set = (RocksGraph)set; 20 | foreach (var s in Enumerable.Range(0, 10)) { 21 | foreach (var p in Enumerable.Range(0, 10)) { 22 | foreach (var o in Enumerable.Range(0, 10)) { 23 | _set.Assert(s.ToString(), p.ToString(), (o.ToString(), o)); 24 | } 25 | } 26 | } 27 | } 28 | 29 | [TestMethod] 30 | public void SP_Continuation_Returns() 31 | { 32 | var spc = _set.SP("5", "5", new Triple("5", "5", ("1", 1))); 33 | var next = spc.First(); 34 | Assert.AreEqual(next, new Triple("5", "5", ("2", 2))); 35 | 36 | var sc = _set.S("5", new Triple("5", "2", ("3", 3))); 37 | Assert.AreEqual(new Triple("5", "2", ("4", 4)), sc.First()); 38 | 39 | Assert.AreEqual(44, _set.S("5", new Triple("5", "5", ("5", 5))).ToArray().Count()); 40 | Assert.AreEqual(4, _set.SP("5", "5", new Triple("5", "5", ("5", 5))).ToArray().Count()); 41 | 42 | Assert.ThrowsException(() => _set.S("5", new Triple("4", "9", ("9", 9)))); 43 | 44 | Assert.AreEqual(0, _set.S("5", new Triple("6", "5", ("5", 5))).ToArray().Count()); 45 | Assert.AreEqual(0, _set.SP("5", "6", new Triple("5", "7", ("5", 5))).ToArray().Count()); 46 | } 47 | 48 | [TestMethod] 49 | public void PO_Continuation_Returns() 50 | { 51 | var poc = _set.PO("4", "4", new Triple("4", "4", ("4", 4))); 52 | var pocFirst = poc.First(); 53 | Assert.AreEqual(new Triple("5", "4", ("4", 4)), pocFirst); 54 | 55 | var pc = _set.P("4", new Triple("3", "4", ("1", 1))); 56 | Assert.AreEqual(new Triple("4", "4", ("1", 1)), pc.First()); 57 | 58 | Assert.AreEqual(44, _set.P("5", new Triple("5", "5", ("5", 5))).ToArray().Count()); 59 | Assert.AreEqual(4, _set.PO("5", "5", new Triple("5", "5", ("5", 5))).ToArray().Count()); 60 | 61 | Assert.ThrowsException(() => _set.O("5", new Triple("4", "3", ("1", 1)))); 62 | Assert.ThrowsException(() => _set.OS("5", "2", new Triple("1", "3", ("5", 5)))); 63 | 64 | Assert.AreEqual(0, _set.P("5", new Triple("5", "6", ("5", 5))).ToArray().Count()); 65 | Assert.AreEqual(0, _set.PO("6", "4", new Triple("5", "6", ("5", 5))).ToArray().Count()); 66 | } 67 | 68 | [TestMethod] 69 | public void OS_Continuation_Returns() 70 | { 71 | var osc = _set.OS("4", "4", new Triple("4", "4", ("4", 4))); 72 | Assert.AreEqual(new Triple("4", "5", ("4", 4)), osc.First()); 73 | 74 | var oc = _set.O("4", new Triple("3", "4", ("4", 4))); 75 | Assert.AreEqual(new Triple("3", "5", ("4", 4)), oc.First()); 76 | 77 | Assert.AreEqual(44, _set.O("5", new Triple("5", "5", ("5", 5))).ToArray().Count()); 78 | Assert.AreEqual(4, _set.OS("5", "5", new Triple("5", "5", ("5", 5))).ToArray().Count()); 79 | 80 | Assert.ThrowsException(() => _set.O("5", new Triple("4", "3", ("1", 1)))); 81 | Assert.ThrowsException(() => _set.OS("1", "3", new Triple("2", "3", ("1", 1)))); 82 | 83 | Assert.AreEqual(0, _set.O("5", new Triple("5", "5", ("6", 6))).ToArray().Count()); 84 | Assert.AreEqual(0, _set.OS("6", "4", new Triple("5", "5", ("6", 6))).ToArray().Count()); 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /Hexastore.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.29209.152 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Hexastore.Web", "Hexastore.Web\Hexastore.Web.csproj", "{83C88235-3496-4B18-AC09-D0F2F48D965B}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Hexastore", "Hexastore\Hexastore.csproj", "{E20D0035-1E52-41B4-A244-DE4142AEF45E}" 9 | EndProject 10 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Hexastore.Rocks", "Hexastore.Rocks\Hexastore.Rocks.csproj", "{5083D115-4595-41B8-9D6B-8C7474905D2B}" 11 | EndProject 12 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Hexastore.Test", "Hexastore.Test\Hexastore.Test.csproj", "{446D86FC-60E2-4467-BFC8-AE0A22BC38B4}" 13 | EndProject 14 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{5D70A57E-85C5-4860-9278-5DFD5D17A128}" 15 | ProjectSection(SolutionItems) = preProject 16 | .editorconfig = .editorconfig 17 | docker-compose.yaml = docker-compose.yaml 18 | Dockerfile = Dockerfile 19 | README.md = README.md 20 | EndProjectSection 21 | EndProject 22 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Hexastore.PerfConsole", "Hexastore.PerfConsole\Hexastore.PerfConsole.csproj", "{A657D874-2D65-4823-A9FC-1B2B867C56A0}" 23 | EndProject 24 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Hexastore.ScaleConsole", "Hexastore.ScaleConsole\Hexastore.ScaleConsole.csproj", "{CA322610-3B07-416A-B85C-D9175C2D2809}" 25 | EndProject 26 | Global 27 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 28 | Debug|Any CPU = Debug|Any CPU 29 | Release|Any CPU = Release|Any CPU 30 | EndGlobalSection 31 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 32 | {83C88235-3496-4B18-AC09-D0F2F48D965B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 33 | {83C88235-3496-4B18-AC09-D0F2F48D965B}.Debug|Any CPU.Build.0 = Debug|Any CPU 34 | {83C88235-3496-4B18-AC09-D0F2F48D965B}.Release|Any CPU.ActiveCfg = Release|Any CPU 35 | {83C88235-3496-4B18-AC09-D0F2F48D965B}.Release|Any CPU.Build.0 = Release|Any CPU 36 | {E20D0035-1E52-41B4-A244-DE4142AEF45E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 37 | {E20D0035-1E52-41B4-A244-DE4142AEF45E}.Debug|Any CPU.Build.0 = Debug|Any CPU 38 | {E20D0035-1E52-41B4-A244-DE4142AEF45E}.Release|Any CPU.ActiveCfg = Release|Any CPU 39 | {E20D0035-1E52-41B4-A244-DE4142AEF45E}.Release|Any CPU.Build.0 = Release|Any CPU 40 | {5083D115-4595-41B8-9D6B-8C7474905D2B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 41 | {5083D115-4595-41B8-9D6B-8C7474905D2B}.Debug|Any CPU.Build.0 = Debug|Any CPU 42 | {5083D115-4595-41B8-9D6B-8C7474905D2B}.Release|Any CPU.ActiveCfg = Release|Any CPU 43 | {5083D115-4595-41B8-9D6B-8C7474905D2B}.Release|Any CPU.Build.0 = Release|Any CPU 44 | {446D86FC-60E2-4467-BFC8-AE0A22BC38B4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 45 | {446D86FC-60E2-4467-BFC8-AE0A22BC38B4}.Debug|Any CPU.Build.0 = Debug|Any CPU 46 | {446D86FC-60E2-4467-BFC8-AE0A22BC38B4}.Release|Any CPU.ActiveCfg = Release|Any CPU 47 | {446D86FC-60E2-4467-BFC8-AE0A22BC38B4}.Release|Any CPU.Build.0 = Release|Any CPU 48 | {A657D874-2D65-4823-A9FC-1B2B867C56A0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 49 | {A657D874-2D65-4823-A9FC-1B2B867C56A0}.Debug|Any CPU.Build.0 = Debug|Any CPU 50 | {A657D874-2D65-4823-A9FC-1B2B867C56A0}.Release|Any CPU.ActiveCfg = Release|Any CPU 51 | {A657D874-2D65-4823-A9FC-1B2B867C56A0}.Release|Any CPU.Build.0 = Release|Any CPU 52 | {CA322610-3B07-416A-B85C-D9175C2D2809}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 53 | {CA322610-3B07-416A-B85C-D9175C2D2809}.Debug|Any CPU.Build.0 = Debug|Any CPU 54 | {CA322610-3B07-416A-B85C-D9175C2D2809}.Release|Any CPU.ActiveCfg = Release|Any CPU 55 | {CA322610-3B07-416A-B85C-D9175C2D2809}.Release|Any CPU.Build.0 = Release|Any CPU 56 | EndGlobalSection 57 | GlobalSection(SolutionProperties) = preSolution 58 | HideSolutionNode = FALSE 59 | EndGlobalSection 60 | GlobalSection(ExtensibilityGlobals) = postSolution 61 | SolutionGuid = {5A249FF1-EB3D-4F84-8ABF-98C778DD2E9B} 62 | EndGlobalSection 63 | EndGlobal 64 | -------------------------------------------------------------------------------- /Hexastore/Parser/TripleConverter.cs: -------------------------------------------------------------------------------- 1 | using Hexastore.Graph; 2 | using Newtonsoft.Json.Linq; 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Linq; 6 | 7 | namespace Hexastore.Parser 8 | { 9 | public class TripleConverter 10 | { 11 | public static IEnumerable FromJson(JObject obj) 12 | { 13 | var graph = new List(); 14 | 15 | AddObject(graph, null, null, obj, null); 16 | return graph; 17 | } 18 | 19 | public static JObject ToJson(string id, IGraph graph, HashSet seen = null) 20 | { 21 | if (seen == null) { 22 | seen = new HashSet(); 23 | } 24 | 25 | var po = graph.GetSubjectGroupings(id); 26 | var rsp = new JObject { [Constants.ID] = id }; 27 | if (!po.Any()) { 28 | return rsp; 29 | } 30 | 31 | if (seen.Contains(id)) { 32 | return rsp; 33 | } 34 | 35 | seen.Add(id); 36 | foreach (var p in po) { 37 | var toList = new List(); 38 | foreach (var o in p) { 39 | if (o.IsID) { 40 | var obj = ToJson(o.ToValue(), graph, seen); 41 | toList.Add(obj); 42 | } else { 43 | toList.Add(o.ToTypedJSON()); 44 | } 45 | } 46 | if (toList.Count == 1) { 47 | rsp[p.Key] = toList.First(); 48 | } else { 49 | rsp[p.Key] = new JArray(toList); 50 | } 51 | } 52 | return rsp; 53 | } 54 | 55 | public static JObject ToJson(IGraph graph) 56 | { 57 | var rsp = new JArray(); 58 | foreach (var g in graph.GetGroupings()) { 59 | rsp.Add(ToJson(g.Key, graph)); 60 | } 61 | return new JObject { ["@graph"] = rsp }; 62 | } 63 | 64 | private static void AddObject(IList graph, string sourceId, string p, JObject obj, int? index) 65 | { 66 | if (!obj.ContainsKey(Constants.ID)) { 67 | throw new InvalidOperationException("Cannot find id"); 68 | } 69 | 70 | var currentId = (string)obj[Constants.ID]; 71 | if (sourceId != null) { 72 | graph.Add(new Triple(sourceId, p, new TripleObject(currentId, index))); 73 | } 74 | 75 | foreach (var item in obj) { 76 | if (item.Key == Constants.ID) { 77 | continue; 78 | } 79 | AddValue(graph, currentId, item.Key, item.Value, null); 80 | } 81 | } 82 | 83 | private static void AddValue(IList graph, string s, string p, JToken token, int? index) 84 | { 85 | switch (token.Type) { 86 | case JTokenType.Object: 87 | var jobj = (JObject)token; 88 | if (!jobj.ContainsKey(Constants.ID)) { 89 | if (index == null) { 90 | jobj[Constants.ID] = $"{s}{Constants.LinkDelimeter}{p}"; 91 | } else { 92 | jobj[Constants.ID] = $"{s}{Constants.LinkDelimeter}{p}{Constants.LinkDelimeter}{index}"; 93 | } 94 | } 95 | AddObject(graph, s, p, jobj, index); 96 | break; 97 | case JTokenType.Array: 98 | var count = 0; 99 | foreach (var item in (JArray)token) { 100 | AddValue(graph, s, p, item, count++); 101 | } 102 | break; 103 | default: 104 | graph.Add(new Triple(s, p, new TripleObject((JValue)token, false, index))); 105 | break; 106 | } 107 | } 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /Hexastore.Web/Controllers/StoreControllerBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | using Hexastore.Errors; 8 | using Hexastore.Web.Errors; 9 | using Hexastore.Web.EventHubs; 10 | using Microsoft.AspNetCore.Http; 11 | using Microsoft.AspNetCore.Mvc; 12 | using Microsoft.Extensions.Logging; 13 | using Newtonsoft.Json; 14 | using Newtonsoft.Json.Linq; 15 | 16 | namespace Hexastore.Web.Controllers 17 | { 18 | public class StoreControllerBase : ControllerBase 19 | { 20 | private readonly EventReceiver _receiver; 21 | private readonly EventSender _eventProcessor; 22 | private readonly StoreError _storeError; 23 | private readonly ILogger _logger; 24 | 25 | public StoreControllerBase(EventReceiver receiver, EventSender eventProcessor, StoreError storeError, ILogger logger) 26 | { 27 | _receiver = receiver; 28 | _eventProcessor = eventProcessor; 29 | _storeError = storeError; 30 | _logger = logger; 31 | } 32 | 33 | protected (string[], string[], int, bool) GetParams() 34 | { 35 | var expandFilter = Request.HttpContext.Request.Query[Constants.ExpandFilter].ToString(); 36 | string[] expand; 37 | if (string.IsNullOrEmpty(expandFilter)) { 38 | expand = null; 39 | } else { 40 | expand = expandFilter.Split(","); 41 | } 42 | 43 | var type = Request.HttpContext.Request.Query[Constants.TypeFilter].ToString().Split(","); 44 | 45 | if (!int.TryParse(Request.HttpContext.Request.Query[Constants.LevelFilter].ToString(), out int level)) { 46 | level = 1; 47 | } 48 | 49 | 50 | if (!bool.TryParse(Request.HttpContext.Request.Query[Constants.StrictFilter].ToString(), out bool strict)) { 51 | strict = false; 52 | } 53 | return (type, expand, level, strict); 54 | } 55 | 56 | protected (int?, int?) GetSkip() 57 | { 58 | var skipText = Request.HttpContext.Request.Query[Constants.Skip].ToString(); 59 | var takeText = Request.HttpContext.Request.Query[Constants.Take].ToString(); 60 | 61 | int? skip = !string.IsNullOrWhiteSpace(skipText) ? int.Parse(skipText) : (int?)null; 62 | int? take = !string.IsNullOrWhiteSpace(takeText) ? int.Parse(takeText) : (int?)null; 63 | 64 | return (skip, take); 65 | } 66 | 67 | protected async Task SendEvent(string storeId, StoreEvent payload) 68 | { 69 | // todo: resolve the promise when all messages are roundtripped from EH 70 | payload.StoreId = storeId; 71 | await _eventProcessor.SendMessage(payload); 72 | } 73 | 74 | protected IActionResult HandleException(Exception e) 75 | { 76 | _logger.LogError(LoggingEvents.ControllerError, e, e.Message); 77 | if (e is StoreException) { 78 | return new ErrorActionResult(e as StoreException); 79 | } else { 80 | return new ErrorActionResult(new StoreException(e.Message, _storeError.Unhandled)); 81 | } 82 | } 83 | 84 | public class ErrorActionResult : IActionResult 85 | { 86 | private readonly StoreException _e; 87 | 88 | public ErrorActionResult(StoreException e) 89 | { 90 | _e = e; 91 | } 92 | 93 | public async Task ExecuteResultAsync(ActionContext context) 94 | { 95 | int.TryParse(_e.ErrorCode.Split(".").First(), out var status); 96 | 97 | var objectResult = new ObjectResult(_e) 98 | { 99 | StatusCode = status == 0 100 | ? StatusCodes.Status500InternalServerError 101 | : status, 102 | Value = new { message = _e.Message, code = _e.ErrorCode } 103 | }; 104 | 105 | await objectResult.ExecuteResultAsync(context); 106 | } 107 | } 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /Hexastore.Web/Queue/Hasher.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | 3 | using System; 4 | using System.Globalization; 5 | using System.Text; 6 | 7 | namespace Hexastore.Web.Queue 8 | { 9 | /// 10 | /// Provides non-cryptographic hashing functions. 11 | /// 12 | public static class Hasher 13 | { 14 | private const uint FnvPrime32 = 0x01000193; 15 | private const uint FnvOffset32 = 0x811C9DC5; 16 | 17 | private const ulong FnvPrime64 = 0x100000001b3; 18 | private const ulong FnvOffset64 = 0xcbf29ce484222325; 19 | 20 | /// 21 | /// Gets a FNV-1a 32-bit hash of the provided . The FNV-1a algorithm 22 | /// is used in many context including DNS servers, database indexing hashes, non-cryptographic file 23 | /// fingerprints to name a few. For more information about FNV, please see the IETF document 24 | /// The FNV Non-Cryptographic Hash Algorithm as well as http://isthe.com/chongo/tech/comp/fnv/. 25 | /// 26 | /// The content to hash. 27 | /// The computed hash. 28 | public static int GetFnvHash32(string content) 29 | { 30 | if (content == null) { 31 | throw new ArgumentNullException("content"); 32 | } 33 | byte[] data = Encoding.UTF8.GetBytes(content); 34 | 35 | uint hash = FnvOffset32; 36 | for (int cnt = 0; cnt < data.Length; cnt++) { 37 | unchecked { 38 | hash ^= data[cnt]; 39 | hash = hash * FnvPrime32; 40 | } 41 | } 42 | return (int)hash; 43 | } 44 | 45 | /// 46 | /// Gets a string representation of a FNV-1a 32-bit hash of the provided . The FNV-1a 47 | /// algorithm is used in many context including DNS servers, database indexing hashes, non-cryptographic file 48 | /// fingerprints to name a few. For more information about FNV, please see the IETF document 49 | /// The FNV Non-Cryptographic Hash Algorithm as well as http://isthe.com/chongo/tech/comp/fnv/. 50 | /// 51 | /// The content to hash. 52 | /// A string representation of the computed hash. 53 | public static string GetFnvHash32AsString(string content) 54 | { 55 | return ((uint)GetFnvHash32(content)).ToString("x8", CultureInfo.InvariantCulture); 56 | } 57 | 58 | /// 59 | /// Gets a FNV-1a 64-bit hash of the provided . The FNV-1a algorithm 60 | /// is used in many context including DNS servers, database indexing hashes, non-cryptographic file 61 | /// fingerprints to name a few. For more information about FNV, please see the IETF document 62 | /// The FNV Non-Cryptographic Hash Algorithm as well as http://isthe.com/chongo/tech/comp/fnv/. 63 | /// 64 | /// The content to hash. 65 | /// The computed hash. 66 | public static long GetFnvHash64(string content) 67 | { 68 | if (content == null) { 69 | throw new ArgumentNullException("content"); 70 | } 71 | byte[] data = Encoding.UTF8.GetBytes(content); 72 | 73 | ulong hash = FnvOffset64; 74 | for (int cnt = 0; cnt < data.Length; cnt++) { 75 | unchecked { 76 | hash ^= data[cnt]; 77 | hash = hash * FnvPrime64; 78 | } 79 | } 80 | return (long)hash; 81 | } 82 | 83 | /// 84 | /// Gets a string representation of a FNV-1a 64-bit hash of the provided . The FNV-1a 85 | /// algorithm is used in many context including DNS servers, database indexing hashes, non-cryptographic file 86 | /// fingerprints to name a few. For more information about FNV, please see the IETF document 87 | /// The FNV Non-Cryptographic Hash Algorithm as well as http://isthe.com/chongo/tech/comp/fnv/. 88 | /// 89 | /// The content to hash. 90 | /// A string representation of the computed hash. 91 | public static string GetFnvHash64AsString(string content) 92 | { 93 | return ((ulong)GetFnvHash64(content)).ToString("x16", CultureInfo.InvariantCulture); 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /Hexastore/Graph/DisjointUnion.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | 6 | namespace Hexastore.Graph 7 | { 8 | public class DisjointUnion : IGraph 9 | { 10 | private readonly IGraph _read; 11 | private readonly IGraph _write; 12 | 13 | public DisjointUnion(IGraph read, IGraph write, string name) 14 | { 15 | _read = read; 16 | _write = write; 17 | Name = name; 18 | } 19 | 20 | public int Count 21 | { 22 | get 23 | { 24 | return _read.Count + _write.Count; 25 | } 26 | } 27 | 28 | public string Name 29 | { 30 | get; 31 | private set; 32 | } 33 | 34 | public bool Assert(Triple t) 35 | { 36 | return Assert(t.Subject, t.Predicate, t.Object); 37 | } 38 | 39 | public bool Assert(string s, string p, TripleObject o) 40 | { 41 | if (_read.Exists(s, p, o)) { 42 | return false; 43 | } 44 | 45 | return _write.Assert(s, p, o); 46 | } 47 | 48 | public object Clone() 49 | { 50 | throw new NotImplementedException(); 51 | } 52 | 53 | public bool Exists(Triple t) 54 | { 55 | if (_read.Exists(t) || _write.Exists(t)) { 56 | return true; 57 | } 58 | return false; 59 | } 60 | 61 | public bool Exists(string s, string p, TripleObject o) 62 | { 63 | if (_read.Exists(s, p, o) || _write.Exists(s, p, o)) { 64 | return true; 65 | } 66 | return false; 67 | } 68 | 69 | public IEnumerable O(TripleObject o) 70 | { 71 | return _read.O(o).Concat(_write.O(o)); 72 | } 73 | 74 | public IEnumerable OS(TripleObject o, string s) 75 | { 76 | return _read.OS(o, s).Concat(_write.OS(o, s)); 77 | } 78 | 79 | public IEnumerable P(string p) 80 | { 81 | return _read.P(p).Concat(_write.P(p)); 82 | } 83 | 84 | public IEnumerable P() 85 | { 86 | return _read.P().Concat(_write.P()); 87 | } 88 | 89 | public IEnumerable PO(string p, TripleObject o) 90 | { 91 | return _read.PO(p, o).Concat(_write.PO(p, o)); 92 | } 93 | 94 | public IEnumerable S(string s) 95 | { 96 | return _read.S(s).Concat(_write.S(s)); 97 | } 98 | 99 | public IEnumerable SP(string s, string p) 100 | { 101 | return _read.SP(s, p).Concat(_write.SP(s, p)); 102 | } 103 | 104 | public IEnumerable GetTriples() 105 | { 106 | return _read.GetTriples().Concat(_write.GetTriples()); 107 | } 108 | 109 | public IGraph Merge(IGraph g) 110 | { 111 | throw new NotImplementedException(); 112 | } 113 | 114 | public IGraph Minus(IGraph g) 115 | { 116 | throw new NotImplementedException(); 117 | } 118 | 119 | public bool Retract(Triple t) 120 | { 121 | throw new NotImplementedException(); 122 | } 123 | 124 | public void Retract(IEnumerable triples) 125 | { 126 | throw new NotImplementedException(); 127 | } 128 | 129 | public bool Retract(string s, string p, TripleObject o) 130 | { 131 | throw new NotImplementedException(); 132 | } 133 | 134 | public IEnumerable>> GetGroupings() 135 | { 136 | return _read.GetGroupings().Concat(_write.GetGroupings()); 137 | } 138 | 139 | public IEnumerable> GetSubjectGroupings(string s) 140 | { 141 | return _read.GetSubjectGroupings(s).Concat(_write.GetSubjectGroupings(s)); 142 | } 143 | 144 | #if DEBUG 145 | public override string ToString() 146 | { 147 | var sb = new StringBuilder(); 148 | foreach (var t in GetTriples()) { 149 | sb.AppendLine(t.ToString()); 150 | } 151 | return sb.ToString(); 152 | } 153 | #endif 154 | 155 | public override bool Equals(object obj) 156 | { 157 | if (obj is IGraph) { 158 | return Equals(this, (IGraph)obj); 159 | } 160 | return false; 161 | } 162 | 163 | public override int GetHashCode() 164 | { 165 | return 2; 166 | } 167 | 168 | public void Dispose() 169 | { 170 | } 171 | 172 | public IEnumerable Assert(IEnumerable triples) 173 | { 174 | throw new NotImplementedException(); 175 | } 176 | 177 | void IGraph.Assert(IEnumerable triples) 178 | { 179 | throw new NotImplementedException(); 180 | } 181 | 182 | public void BatchRetractAssert(IEnumerable retract, IEnumerable assert) 183 | { 184 | throw new NotImplementedException(); 185 | } 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /Hexastore.Web/EventHubs/EventReceiver.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Net.Http; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | using Hexastore.Processor; 8 | using Microsoft.Azure.EventHubs; 9 | using Microsoft.Extensions.Logging; 10 | using Newtonsoft.Json; 11 | using Newtonsoft.Json.Linq; 12 | 13 | namespace Hexastore.Web.EventHubs 14 | { 15 | 16 | public class EventReceiver : IPartitionReceiveHandler, IDisposable 17 | { 18 | private readonly ConcurrentDictionary> _completions; 19 | private readonly ILogger _logger; 20 | private readonly Checkpoint _checkpoint; 21 | private readonly IStoreProcesor _storeProcessor; 22 | private readonly StoreConfig _storeConfig; 23 | private bool _running = true; 24 | 25 | private int _eventCount; 26 | 27 | public EventReceiver(IStoreProcesor storeProcessor, Checkpoint checkpoint, StoreConfig storeConfig, ILogger logger) 28 | { 29 | _completions = new ConcurrentDictionary>(); 30 | _logger = logger; 31 | _checkpoint = checkpoint; 32 | _storeProcessor = storeProcessor; 33 | _storeConfig = storeConfig; 34 | 35 | _eventCount = 0; 36 | } 37 | 38 | public int MaxBatchSize 39 | { 40 | get 41 | { 42 | return 1000; 43 | } 44 | set 45 | { 46 | } 47 | } 48 | 49 | public async Task LogCount() 50 | { 51 | var lastCount = 0; 52 | while (_running) { 53 | await Task.Delay(10000); 54 | _logger.LogInformation($"{DateTime.Now.ToString("hh':'mm':'ss")} Events: {_eventCount} Diff: {_eventCount - lastCount}"); 55 | lastCount = _eventCount; 56 | } 57 | } 58 | 59 | public async Task ProcessEventsAsync(IEnumerable events) 60 | { 61 | if (events == null) { 62 | return; 63 | } 64 | 65 | foreach (var e in events) { 66 | _eventCount++; 67 | var content = Encoding.UTF8.GetString(e.Body); 68 | var partitionId = e.SystemProperties["x-opt-partition-key"].ToString(); 69 | var offset = e.SystemProperties["x-opt-offset"].ToString(); 70 | StoreEvent storeEvent; 71 | try { 72 | storeEvent = StoreEvent.FromString(content); 73 | } catch (Exception exception) { 74 | _logger.LogError(exception, "Unable to process event {offset} {partition} {}", offset, partitionId); 75 | continue; 76 | } 77 | 78 | await ProcessEventsAsync(storeEvent); 79 | _checkpoint.Write($"{Constants.EventHubCheckpoint}.{_storeConfig.EventHubName}.{partitionId}", offset); 80 | } 81 | 82 | return; 83 | } 84 | 85 | public Task ProcessEventsAsync(StoreEvent storeEvent) 86 | { 87 | var storeId = storeEvent.StoreId; 88 | 89 | string operation = storeEvent.Operation; 90 | var opId = storeEvent.OperationId; 91 | var strict = storeEvent.Strict; 92 | 93 | TaskCompletionSource tc = null; 94 | if (opId != null) { 95 | _completions.TryGetValue(opId, out tc); 96 | } 97 | 98 | try { 99 | var data = storeEvent.Data; 100 | switch (operation) { 101 | case EventType.POST: 102 | _storeProcessor.Assert(storeId, data, strict); 103 | break; 104 | case EventType.PATCH_JSON: 105 | _storeProcessor.PatchJson(storeId, data); 106 | break; 107 | case EventType.PATCH_TRIPLE: 108 | var patch = (JObject)storeEvent.Data; 109 | _storeProcessor.PatchTriple(storeId, patch); 110 | break; 111 | case EventType.DELETE: 112 | _storeProcessor.Delete(storeId, data); 113 | break; 114 | default: 115 | throw new InvalidOperationException($"Unknown operation {operation}"); 116 | } 117 | tc?.SetResult(true); 118 | } catch (Exception exception) { 119 | tc?.SetException(exception); 120 | } finally { 121 | if (tc != null) { 122 | _completions.Remove(opId, out _); 123 | } 124 | } 125 | return Task.CompletedTask; 126 | } 127 | 128 | public Task ProcessErrorAsync(Exception error) 129 | { 130 | Console.WriteLine(JsonConvert.SerializeObject(error)); 131 | return Task.CompletedTask; 132 | } 133 | 134 | public void SetCompletion(string guid, TaskCompletionSource tc) 135 | { 136 | _completions.TryAdd(guid, tc); 137 | } 138 | 139 | public void Dispose() 140 | { 141 | _running = false; 142 | } 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /Hexastore/Graph/SPOIndex.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using Hexastore.Parser; 7 | using Newtonsoft.Json.Linq; 8 | 9 | namespace Hexastore.Graph 10 | { 11 | public class SPOIndex 12 | { 13 | private readonly IDictionary>> _spo; 14 | 15 | public SPOIndex() 16 | { 17 | _spo = new Dictionary>>(); 18 | } 19 | 20 | public void Assert(Triple t) 21 | { 22 | Assert(t.Subject, t.Predicate, t.Object); 23 | } 24 | 25 | public void Assert(IEnumerable triples) 26 | { 27 | foreach (var t in triples) { 28 | Assert(t.Subject, t.Predicate, t.Object); 29 | } 30 | } 31 | 32 | public void Assert(string s, string p, TripleObject o) 33 | { 34 | Assert(_spo, s, p, o); 35 | } 36 | 37 | public JObject ToJson(string id, HashSet seen = null) 38 | { 39 | if (seen == null) { 40 | seen = new HashSet(); 41 | } 42 | 43 | var po = GetSubjectGroupings(id); 44 | var rsp = new JObject { [Constants.ID] = id }; 45 | if (!po.Any()) { 46 | return rsp; 47 | } 48 | 49 | if (seen.Contains(id)) { 50 | return rsp; 51 | } 52 | 53 | seen.Add(id); 54 | foreach (var p in po) { 55 | List toList = null; 56 | foreach (var o in p) { 57 | if (o.Index == -1) { 58 | if (o.IsID) { 59 | var obj = ToJson(o.ToValue(), seen); 60 | rsp[p.Key] = obj; 61 | } else { 62 | rsp[p.Key] = o.ToTypedJSON(); 63 | } 64 | } else { 65 | if (toList == null) { 66 | toList = new List(); 67 | } 68 | if (o.IsID) { 69 | var obj = ToJson(o.ToValue(), seen); 70 | toList.Add(obj); 71 | } else { 72 | toList.Add(o.ToTypedJSON()); 73 | } 74 | } 75 | } 76 | if (toList != null) { 77 | rsp[p.Key] = new JArray(toList); 78 | } 79 | } 80 | return rsp; 81 | } 82 | 83 | private static void Assert(IDictionary>> xyz, TX tx, TY ty, TZ tz) 84 | { 85 | if (!xyz.TryGetValue(tx, out IDictionary> yz)) { 86 | yz = new Dictionary>(); 87 | xyz[tx] = yz; 88 | } 89 | 90 | if (!yz.TryGetValue(ty, out IList z)) { 91 | z = new List(); 92 | yz[ty] = z; 93 | } 94 | if (!z.Contains(tz)) { 95 | // todo: keyed collection / ordered hashset 96 | z.Add(tz); 97 | } 98 | } 99 | 100 | public IEnumerable> GetSubjectGroupings(string s) 101 | { 102 | IDictionary> po; 103 | if (_spo.TryGetValue(s, out po)) { 104 | foreach (var p in po) { 105 | yield return new PredicateGrouping(p); 106 | } 107 | } 108 | } 109 | 110 | private class SubjectGrouping : IGrouping> 111 | { 112 | private readonly KeyValuePair>> _kv; 113 | 114 | public SubjectGrouping(KeyValuePair>> kv) 115 | { 116 | _kv = kv; 117 | } 118 | 119 | public string Key 120 | { 121 | get { return _kv.Key; } 122 | } 123 | 124 | public IEnumerator> GetEnumerator() 125 | { 126 | foreach (var p in _kv.Value) { 127 | yield return new PredicateGrouping(p); 128 | } 129 | } 130 | 131 | IEnumerator IEnumerable.GetEnumerator() 132 | { 133 | return GetEnumerator(); 134 | } 135 | } 136 | 137 | private class PredicateGrouping : IGrouping 138 | { 139 | private readonly KeyValuePair> _kv; 140 | 141 | public PredicateGrouping(KeyValuePair> kv) 142 | { 143 | _kv = kv; 144 | } 145 | 146 | public string Key 147 | { 148 | get { return _kv.Key; } 149 | } 150 | 151 | public IEnumerator GetEnumerator() 152 | { 153 | foreach (var o in _kv.Value) { 154 | yield return o; 155 | } 156 | } 157 | 158 | IEnumerator IEnumerable.GetEnumerator() 159 | { 160 | return GetEnumerator(); 161 | } 162 | } 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | **/data/ 5 | **/out/ 6 | *.env 7 | **/BenchmarkDotNet.Artifacts/ 8 | 9 | # User-specific files 10 | *.suo 11 | *.user 12 | *.userosscache 13 | *.sln.docstates 14 | 15 | # User-specific files (MonoDevelop/Xamarin Studio) 16 | *.userprefs 17 | 18 | # Build results 19 | [Dd]ebug/ 20 | [Dd]ebugPublic/ 21 | [Rr]elease/ 22 | [Rr]eleases/ 23 | x64/ 24 | x86/ 25 | bld/ 26 | [Bb]in/ 27 | [Oo]bj/ 28 | [Ll]og/ 29 | 30 | # Visual Studio 2015 cache/options directory 31 | .vs/ 32 | # Uncomment if you have tasks that create the project's static files in wwwroot 33 | #wwwroot/ 34 | 35 | # MSTest test Results 36 | [Tt]est[Rr]esult*/ 37 | [Bb]uild[Ll]og.* 38 | 39 | # NUNIT 40 | *.VisualState.xml 41 | TestResult.xml 42 | 43 | # Build Results of an ATL Project 44 | [Dd]ebugPS/ 45 | [Rr]eleasePS/ 46 | dlldata.c 47 | 48 | # DNX 49 | project.lock.json 50 | project.fragment.lock.json 51 | artifacts/ 52 | 53 | *_i.c 54 | *_p.c 55 | *_i.h 56 | *.ilk 57 | *.meta 58 | *.obj 59 | *.pch 60 | *.pdb 61 | *.pgc 62 | *.pgd 63 | *.rsp 64 | *.sbr 65 | *.tlb 66 | *.tli 67 | *.tlh 68 | *.tmp 69 | *.tmp_proj 70 | *.log 71 | *.vspscc 72 | *.vssscc 73 | .builds 74 | *.pidb 75 | *.svclog 76 | *.scc 77 | 78 | # Chutzpah Test files 79 | _Chutzpah* 80 | 81 | # Visual C++ cache files 82 | ipch/ 83 | *.aps 84 | *.ncb 85 | *.opendb 86 | *.opensdf 87 | *.sdf 88 | *.cachefile 89 | *.VC.db 90 | *.VC.VC.opendb 91 | 92 | # Visual Studio profiler 93 | *.psess 94 | *.vsp 95 | *.vspx 96 | *.sap 97 | 98 | # TFS 2012 Local Workspace 99 | $tf/ 100 | 101 | # Guidance Automation Toolkit 102 | *.gpState 103 | 104 | # ReSharper is a .NET coding add-in 105 | _ReSharper*/ 106 | *.[Rr]e[Ss]harper 107 | *.DotSettings.user 108 | 109 | # JustCode is a .NET coding add-in 110 | .JustCode 111 | 112 | # TeamCity is a build add-in 113 | _TeamCity* 114 | 115 | # DotCover is a Code Coverage Tool 116 | *.dotCover 117 | 118 | # NCrunch 119 | _NCrunch_* 120 | .*crunch*.local.xml 121 | nCrunchTemp_* 122 | 123 | # MightyMoose 124 | *.mm.* 125 | AutoTest.Net/ 126 | 127 | # Web workbench (sass) 128 | .sass-cache/ 129 | 130 | # Installshield output folder 131 | [Ee]xpress/ 132 | 133 | # DocProject is a documentation generator add-in 134 | DocProject/buildhelp/ 135 | DocProject/Help/*.HxT 136 | DocProject/Help/*.HxC 137 | DocProject/Help/*.hhc 138 | DocProject/Help/*.hhk 139 | DocProject/Help/*.hhp 140 | DocProject/Help/Html2 141 | DocProject/Help/html 142 | 143 | # Click-Once directory 144 | publish/ 145 | 146 | # Publish Web Output 147 | *.[Pp]ublish.xml 148 | *.azurePubxml 149 | # TODO: Comment the next line if you want to checkin your web deploy settings 150 | # but database connection strings (with potential passwords) will be unencrypted 151 | #*.pubxml 152 | *.publishproj 153 | 154 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 155 | # checkin your Azure Web App publish settings, but sensitive information contained 156 | # in these scripts will be unencrypted 157 | PublishScripts/ 158 | 159 | # NuGet Packages 160 | *.nupkg 161 | # The packages folder can be ignored because of Package Restore 162 | **/packages/* 163 | # except build/, which is used as an MSBuild target. 164 | !**/packages/build/ 165 | # Uncomment if necessary however generally it will be regenerated when needed 166 | #!**/packages/repositories.config 167 | # NuGet v3's project.json files produces more ignoreable files 168 | *.nuget.props 169 | *.nuget.targets 170 | 171 | # Microsoft Azure Build Output 172 | csx/ 173 | *.build.csdef 174 | 175 | # Microsoft Azure Emulator 176 | ecf/ 177 | rcf/ 178 | 179 | # Windows Store app package directories and files 180 | AppPackages/ 181 | BundleArtifacts/ 182 | Package.StoreAssociation.xml 183 | _pkginfo.txt 184 | 185 | # Visual Studio cache files 186 | # files ending in .cache can be ignored 187 | *.[Cc]ache 188 | # but keep track of directories ending in .cache 189 | !*.[Cc]ache/ 190 | 191 | # Others 192 | ClientBin/ 193 | ~$* 194 | *~ 195 | *.dbmdl 196 | *.dbproj.schemaview 197 | *.jfm 198 | *.pfx 199 | *.publishsettings 200 | node_modules/ 201 | orleans.codegen.cs 202 | 203 | # Since there are multiple workflows, uncomment next line to ignore bower_components 204 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 205 | #bower_components/ 206 | 207 | # RIA/Silverlight projects 208 | Generated_Code/ 209 | 210 | # Backup & report files from converting an old project file 211 | # to a newer Visual Studio version. Backup files are not needed, 212 | # because we have git ;-) 213 | _UpgradeReport_Files/ 214 | Backup*/ 215 | UpgradeLog*.XML 216 | UpgradeLog*.htm 217 | 218 | # SQL Server files 219 | *.mdf 220 | *.ldf 221 | 222 | # Business Intelligence projects 223 | *.rdl.data 224 | *.bim.layout 225 | *.bim_*.settings 226 | 227 | # Microsoft Fakes 228 | FakesAssemblies/ 229 | 230 | # GhostDoc plugin setting file 231 | *.GhostDoc.xml 232 | 233 | # Node.js Tools for Visual Studio 234 | .ntvs_analysis.dat 235 | 236 | # Visual Studio 6 build log 237 | *.plg 238 | 239 | # Visual Studio 6 workspace options file 240 | *.opt 241 | 242 | # Visual Studio LightSwitch build output 243 | **/*.HTMLClient/GeneratedArtifacts 244 | **/*.DesktopClient/GeneratedArtifacts 245 | **/*.DesktopClient/ModelManifest.xml 246 | **/*.Server/GeneratedArtifacts 247 | **/*.Server/ModelManifest.xml 248 | _Pvt_Extensions 249 | 250 | # Paket dependency manager 251 | .paket/paket.exe 252 | paket-files/ 253 | 254 | # FAKE - F# Make 255 | .fake/ 256 | 257 | # JetBrains Rider 258 | .idea/ 259 | *.sln.iml 260 | 261 | # CodeRush 262 | .cr/ 263 | 264 | # Python Tools for Visual Studio (PTVS) 265 | __pycache__/ 266 | *.pyc 267 | -------------------------------------------------------------------------------- /Hexastore.ScaleConsole/FlatModelTest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Diagnostics; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | using Hexastore.Errors; 8 | using Hexastore.Processor; 9 | using Hexastore.Resoner; 10 | using Hexastore.Rocks; 11 | using Hexastore.TestCommon; 12 | using Hexastore.Web; 13 | using Hexastore.Web.EventHubs; 14 | using Hexastore.Web.Queue; 15 | using Microsoft.Extensions.Logging; 16 | using RocksDbSharp; 17 | 18 | namespace Hexastore.ScaleConsole 19 | { 20 | public static class FlatModelTest 21 | { 22 | private static readonly Random _random = new Random(234234); 23 | 24 | public static void RunTest(int appCount, int deviceCount, int devicePropertyCount, int sendCount, int senderThreadCount, bool tryOptimizeRocks) 25 | { 26 | Console.WriteLine("Creating Messages"); 27 | var apps = new List(appCount); 28 | var deviceIds = new List(deviceCount); 29 | var devicePropertyNames = new List(devicePropertyCount); 30 | var tasks = new List(); 31 | var sendQueue = new ConcurrentQueue(); 32 | 33 | while (apps.Count < appCount) 34 | { 35 | apps.Add(Guid.NewGuid().ToString()); 36 | } 37 | 38 | while (deviceIds.Count < deviceCount) 39 | { 40 | deviceIds.Add(Guid.NewGuid().ToString()); 41 | } 42 | 43 | while (devicePropertyNames.Count < devicePropertyCount) 44 | { 45 | devicePropertyNames.Add(Guid.NewGuid().ToString()); 46 | } 47 | 48 | using (var testFolder = new TestFolder()) 49 | { 50 | var factory = LoggerFactory.Create(builder => builder.AddConsole()); 51 | var logger = factory.CreateLogger(); 52 | var storeLogger = factory.CreateLogger(); 53 | 54 | var dbOptions = new DbOptions(); 55 | var provider = !tryOptimizeRocks ? 56 | new RocksGraphProvider(logger, testFolder) : 57 | new RocksGraphProvider(logger, testFolder, dbOptions.SetCreateIfMissing(true) 58 | .SetAllowConcurrentMemtableWrite(true) 59 | //.SetAllowMmapReads(true) 60 | //.SetAllowMmapWrites(true) 61 | //.SetUseFsync(0) 62 | .IncreaseParallelism(Environment.ProcessorCount) 63 | .SetMaxBackgroundCompactions(Environment.ProcessorCount) 64 | .SetMaxBackgroundFlushes(Environment.ProcessorCount)); 65 | 66 | var storeProvider = new SetProvider(provider); 67 | var storeProcessor = new StoreProcessor(storeProvider, new Reasoner(), storeLogger); 68 | var storeConfig = new StoreConfig(); 69 | var storeError = new StoreError(); 70 | var eventReceiver = new EventReceiver(storeProcessor, null, storeConfig, factory.CreateLogger()); 71 | var queueContainer = new QueueContainer(eventReceiver, factory.CreateLogger(), storeError, 1_000_000); 72 | var eventSender = new EventSender(queueContainer, null, null, storeConfig); 73 | 74 | for (var i = 0; i < sendCount; i++) 75 | { 76 | foreach (var id in deviceIds) 77 | { 78 | foreach (var app in apps) 79 | { 80 | var points = GetPropertyValues(devicePropertyNames, _random); 81 | var e = new StoreEvent 82 | { 83 | Operation = EventType.PATCH_JSON, 84 | Data = JsonGenerator.GenerateTelemetry(id, points), 85 | PartitionId = id, 86 | StoreId = app 87 | }; 88 | sendQueue.Enqueue(e); 89 | } 90 | } 91 | } 92 | 93 | Console.WriteLine($"Starting send of {sendQueue.Count} messages"); 94 | 95 | var timer = Stopwatch.StartNew(); 96 | 97 | for (var i = 0; i < senderThreadCount; i++) 98 | { 99 | tasks.Add(Task.Run(() => RunSender(eventSender, sendQueue))); 100 | } 101 | 102 | Task.WhenAll(tasks).Wait(); 103 | 104 | Console.WriteLine($"Completed writing to queues in {timer.Elapsed}"); 105 | 106 | while (queueContainer.Count() > 0) 107 | { 108 | Thread.Sleep(1000); 109 | } 110 | Console.WriteLine($"Completed writing to storage in {timer.Elapsed}"); 111 | } 112 | } 113 | 114 | private static async void RunSender(EventSender eventSender, ConcurrentQueue sendQueue) 115 | { 116 | while (sendQueue.TryDequeue(out var message)) 117 | { 118 | await eventSender.SendMessage(message); 119 | } 120 | } 121 | 122 | private static Dictionary GetPropertyValues(List devicePropertyNames, Random random) 123 | { 124 | var points = new Dictionary(); 125 | foreach (var pointId in devicePropertyNames) 126 | { 127 | points.Add(pointId, random.NextDouble()); 128 | } 129 | 130 | return points; 131 | } 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /Hexastore.Rocks/RocksGraphProvider.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.IO; 4 | using System.Runtime.InteropServices; 5 | using Hexastore.Graph; 6 | using Hexastore.Store; 7 | using Microsoft.Extensions.Logging; 8 | using RocksDbSharp; 9 | 10 | namespace Hexastore.Rocks 11 | { 12 | public class RocksGraphProvider : IGraphProvider, IDisposable 13 | { 14 | private static readonly WriteOptions _writeOptions = (new WriteOptions()).SetSync(false); 15 | private readonly ConcurrentDictionary _dbs = new ConcurrentDictionary(); 16 | private readonly ILogger _logger; 17 | private readonly string _rootPath = null; 18 | private readonly DbOptions _dbOptions = new DbOptions().SetCreateIfMissing(true); 19 | 20 | 21 | public RocksGraphProvider(ILogger logger, string path = null, DbOptions optionInput = null) 22 | { 23 | _logger = logger; 24 | _rootPath = path; 25 | 26 | if (_rootPath == null) { 27 | var configPath = Environment.GetEnvironmentVariable("HEXASTORE_DATA_PATH"); 28 | if (!string.IsNullOrEmpty(configPath)) { 29 | _rootPath = Path.GetFullPath(configPath); 30 | } 31 | } 32 | 33 | if (_rootPath == null) { 34 | if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { 35 | _rootPath = "/var/data/hexastore"; 36 | } else { 37 | _rootPath = "C:\\data\\hexastore"; 38 | } 39 | } 40 | 41 | Directory.CreateDirectory(_rootPath); 42 | 43 | if (optionInput != null) { 44 | _dbOptions = optionInput; 45 | } else { 46 | _dbOptions.SetAllowConcurrentMemtableWrite(true); 47 | } 48 | 49 | /* 50 | .SetAllowConcurrentMemtableWrite(true) 51 | .SetAllowMmapReads(true) 52 | .SetAllowMmapWrites(true) 53 | .SetUseFsync(0) 54 | .IncreaseParallelism(6) 55 | .SetCompression(Compression.No); 56 | */ 57 | } 58 | 59 | 60 | public bool ContainsGraph(string id, GraphType type) 61 | { 62 | var key = MakeKey(id, type); 63 | var db = GetDBOrNull(id); 64 | return db?.Get(key) == "1"; 65 | } 66 | 67 | public IStoreGraph CreateGraph(string id, GraphType type) 68 | { 69 | var key = MakeKey(id, type); 70 | var db = GetOrCreateDB(id); 71 | db.Put(key, "1"); 72 | return new RocksGraph(key, db); 73 | } 74 | 75 | public bool DeleteGraph(string id, GraphType type) 76 | { 77 | var db = GetDBOrNull(id); 78 | 79 | if (db == null || !ContainsGraph(id, type)) { 80 | return false; 81 | } 82 | db.Remove(MakeKey(id, type)); 83 | return true; 84 | } 85 | 86 | public void WriteKey(string id, string key, string value) 87 | { 88 | GetDB(id).Put(key, value, null, _writeOptions); 89 | } 90 | 91 | public string ReadKey(string id, string key) 92 | { 93 | return GetDB(id).Get(key); 94 | } 95 | 96 | public void Dispose() 97 | { 98 | _logger.LogInformation("disposing rocksdb"); 99 | 100 | foreach (var pair in _dbs) { 101 | pair.Value.Dispose(); 102 | } 103 | 104 | _dbs.Clear(); 105 | } 106 | 107 | public IStoreGraph GetGraph(string id, GraphType type) 108 | { 109 | var db = GetDB(id); 110 | string key = MakeKey(id, type); 111 | return new RocksGraph(key, db); 112 | } 113 | 114 | private static string MakeKey(string id, GraphType type) 115 | { 116 | return $"{(int)type}"; ; 117 | } 118 | 119 | private RocksDb GetDB(string id) 120 | { 121 | if (string.IsNullOrEmpty(id)) { 122 | throw new ArgumentException("message", nameof(id)); 123 | } 124 | 125 | var db = GetDBOrNull(id); 126 | 127 | if (db == null) { 128 | throw new FileNotFoundException($"Unable to find rocks db: {id}"); 129 | } 130 | 131 | return db; 132 | } 133 | 134 | private RocksDb GetDBOrNull(string id) 135 | { 136 | if (string.IsNullOrEmpty(id)) { 137 | throw new ArgumentException("message", nameof(id)); 138 | } 139 | 140 | // Get the db if it exists, otherwise null is returned 141 | _dbs.TryGetValue(id, out var db); 142 | 143 | return db; 144 | } 145 | 146 | private RocksDb GetOrCreateDB(string id) 147 | { 148 | if (string.IsNullOrEmpty(id)) { 149 | throw new ArgumentException("message", nameof(id)); 150 | } 151 | 152 | var db = GetDBOrNull(id); 153 | 154 | if (db == null) { 155 | // Allow only one db to be created at a time to avoid conflicts 156 | lock (_dbs) { 157 | // Double check lock 158 | if (!_dbs.TryGetValue(id, out db)) { 159 | var dbPath = Path.Combine(_rootPath, id); 160 | 161 | db = RocksDb.Open(_dbOptions, dbPath); 162 | _logger.LogInformation($"created rocksdb at {dbPath}"); 163 | 164 | if (!_dbs.TryAdd(id, db)) { 165 | // this should never happen 166 | throw new Exception("unable to add db"); 167 | } 168 | } 169 | } 170 | } 171 | 172 | return db; 173 | } 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /Hexastore.Test/ObjectQueryExecutorPagedTest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using Hexastore.Graph; 6 | using Hexastore.Query; 7 | using Hexastore.Rocks; 8 | using Microsoft.VisualStudio.TestTools.UnitTesting; 9 | using Newtonsoft.Json.Linq; 10 | 11 | namespace Hexastore.Test 12 | { 13 | [TestClass] 14 | public class ObjectQueryExecutorPagedTest : RocksFixture 15 | { 16 | public ObjectQueryExecutorPagedTest() 17 | { 18 | for (int i = 0; i < 100; i++) { 19 | var item = new 20 | { 21 | id = $"i{i.ToString("D2")}", 22 | name = $"name{i.ToString("D2")}", 23 | param = i, 24 | contains = new 25 | { 26 | id = "c0", 27 | name = "namec0", 28 | values = new 29 | { 30 | id = "v0", 31 | name = "namev0" 32 | } 33 | } 34 | }; 35 | StoreProcessor.Assert("app1", JToken.FromObject(item), false); 36 | } 37 | 38 | for (int j = 0; j < 100; j++) { 39 | var item = new 40 | { 41 | id = $"j{j.ToString("D2")}", 42 | name = $"name{j.ToString("D2")}", 43 | param = j, 44 | contains = new 45 | { 46 | id = "c1", 47 | name = "namec1", 48 | values = new 49 | { 50 | id = "v1", 51 | name = "namev1" 52 | } 53 | } 54 | }; 55 | StoreProcessor.Assert("app1", JToken.FromObject(item), false); 56 | } 57 | } 58 | 59 | [TestMethod] 60 | public void Query_With_PageSize_Returns() 61 | { 62 | var (set, _, _) = StoreProcessor.GetGraphs("app1"); 63 | 64 | var query = new ObjectQueryModel 65 | { 66 | Filter = new Dictionary() 67 | { 68 | ["name"] = new QueryUnit { Operator = "contains", Value = "name" } 69 | }, 70 | PageSize = 10, 71 | HasObject = new LinkQuery[] 72 | { 73 | new LinkQuery 74 | { 75 | Path = "contains/values", 76 | Target = new ObjectQueryModel 77 | { 78 | Filter = new Dictionary() 79 | { 80 | ["name"] = new QueryUnit { Operator = "eq", Value = "namev0" } 81 | } 82 | } 83 | } 84 | } 85 | }; 86 | 87 | var rsp = new ObjectQueryExecutor().Query(query, (RocksGraph)set); 88 | var values = rsp.Values.Select(x => x.Subject).ToArray(); 89 | CollectionAssert.AreEqual(values, new string[] { "i00", "i01", "i02", "i03", "i04", "i05", "i06", "i07", "i08", "i09" }); 90 | Assert.AreEqual(new Triple("i09", "name", TripleObject.FromData("name09")), rsp.Continuation); 91 | 92 | var query2 = new ObjectQueryModel 93 | { 94 | Continuation = rsp.Continuation, 95 | Filter = new Dictionary() 96 | { 97 | ["name"] = new QueryUnit { Operator = "contains", Value = "name" } 98 | }, 99 | PageSize = 10, 100 | HasObject = new LinkQuery[] 101 | { 102 | new LinkQuery 103 | { 104 | Path = "contains/values", 105 | Target = new ObjectQueryModel 106 | { 107 | Filter = new Dictionary() 108 | { 109 | ["name"] = new QueryUnit { Operator = "eq", Value = "namev0" } 110 | } 111 | } 112 | } 113 | } 114 | }; 115 | 116 | var rsp2 = new ObjectQueryExecutor().Query(query2, (RocksGraph)set); 117 | var values2 = rsp2.Values.Select(x => x.Subject).ToArray(); 118 | CollectionAssert.AreEqual(values2, new string[] { "i10", "i11", "i12", "i13", "i14", "i15", "i16", "i17", "i18", "i19" }); 119 | Assert.AreEqual(new Triple("i19", "name", TripleObject.FromData("name19")), rsp2.Continuation); 120 | 121 | var query3 = new ObjectQueryModel 122 | { 123 | Continuation = rsp2.Continuation, 124 | Filter = new Dictionary() 125 | { 126 | ["name"] = new QueryUnit { Operator = "contains", Value = "name" } 127 | }, 128 | PageSize = 1000, 129 | HasObject = new LinkQuery[] 130 | { 131 | new LinkQuery 132 | { 133 | Path = "contains/values", 134 | Target = new ObjectQueryModel 135 | { 136 | Filter = new Dictionary() 137 | { 138 | ["name"] = new QueryUnit { Operator = "eq", Value = "namev0" } 139 | } 140 | } 141 | } 142 | } 143 | }; 144 | 145 | var rsp3 = new ObjectQueryExecutor().Query(query3, (RocksGraph)set); 146 | var values3 = rsp3.Values.Select(x => x.Subject).ToArray(); 147 | Assert.AreEqual(80, values3.Length); 148 | Assert.AreEqual(null, rsp3.Continuation); 149 | } 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /hexadb-readme.svg: -------------------------------------------------------------------------------- 1 | RoomSensor 0Sensor 1MarkerMarker{"id":"sensor:0","type":"sensor","name":"Entitysensor0","temperature":64.44,"humidity":14.65,"pressure":957.1}{"id":"room:0","name":"RoomZero","type":"room","description":"Hastwosensors"}{"id":"sensor:1","type":"sensor","name":"Entitysensor1","temperature":65.86,"humidity":12.29,"pressure":945.19}{"status":"stopped","red":9.95,"blue":7.16,"green":2.02}{"status":"running","red":9.12,"blue":4.53,"green":9.85} -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # To learn more about .editorconfig see https://aka.ms/editorconfigdocs 2 | ############################### 3 | # Core EditorConfig Options # 4 | ############################### 5 | # All files 6 | [*] 7 | indent_style = space 8 | # Code files 9 | [*.{cs,csx,vb,vbx}] 10 | indent_size = 4 11 | insert_final_newline = true 12 | charset = utf-8-bom 13 | ############################### 14 | # .NET Coding Conventions # 15 | ############################### 16 | [*.{cs,vb}] 17 | # Organize usings 18 | dotnet_sort_system_directives_first = true 19 | # this. preferences 20 | dotnet_style_qualification_for_field = false:silent 21 | dotnet_style_qualification_for_property = false:silent 22 | dotnet_style_qualification_for_method = false:silent 23 | dotnet_style_qualification_for_event = false:silent 24 | # Language keywords vs BCL types preferences 25 | dotnet_style_predefined_type_for_locals_parameters_members = true:silent 26 | dotnet_style_predefined_type_for_member_access = true:silent 27 | # Parentheses preferences 28 | dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent 29 | dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent 30 | dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent 31 | dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent 32 | # Modifier preferences 33 | dotnet_style_require_accessibility_modifiers = for_non_interface_members:silent 34 | dotnet_style_readonly_field = true:suggestion 35 | # Expression-level preferences 36 | dotnet_style_object_initializer = true:suggestion 37 | dotnet_style_collection_initializer = true:suggestion 38 | dotnet_style_explicit_tuple_names = true:suggestion 39 | dotnet_style_null_propagation = true:suggestion 40 | dotnet_style_coalesce_expression = true:suggestion 41 | dotnet_style_prefer_is_null_check_over_reference_equality_method = true:silent 42 | dotnet_style_prefer_inferred_tuple_names = true:suggestion 43 | dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion 44 | dotnet_style_prefer_auto_properties = true:silent 45 | dotnet_style_prefer_conditional_expression_over_assignment = true:silent 46 | dotnet_style_prefer_conditional_expression_over_return = true:silent 47 | ############################### 48 | # Naming Conventions # 49 | ############################### 50 | # Style Definitions 51 | dotnet_naming_style.pascal_case_style.capitalization = pascal_case 52 | # Use PascalCase for constant fields 53 | dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion 54 | dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields 55 | dotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style 56 | dotnet_naming_symbols.constant_fields.applicable_kinds = field 57 | dotnet_naming_symbols.constant_fields.applicable_accessibilities = * 58 | dotnet_naming_symbols.constant_fields.required_modifiers = const 59 | ############################### 60 | # C# Coding Conventions # 61 | ############################### 62 | [*.cs] 63 | # var preferences 64 | csharp_style_var_for_built_in_types = true:silent 65 | csharp_style_var_when_type_is_apparent = true:silent 66 | csharp_style_var_elsewhere = true:silent 67 | # Expression-bodied members 68 | csharp_style_expression_bodied_methods = false:silent 69 | csharp_style_expression_bodied_constructors = false:silent 70 | csharp_style_expression_bodied_operators = false:silent 71 | csharp_style_expression_bodied_properties = true:silent 72 | csharp_style_expression_bodied_indexers = true:silent 73 | csharp_style_expression_bodied_accessors = true:silent 74 | # Pattern matching preferences 75 | csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion 76 | csharp_style_pattern_matching_over_as_with_null_check = true:suggestion 77 | # Null-checking preferences 78 | csharp_style_throw_expression = true:suggestion 79 | csharp_style_conditional_delegate_call = true:suggestion 80 | # Modifier preferences 81 | csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:suggestion 82 | # Expression-level preferences 83 | csharp_prefer_braces = true:silent 84 | csharp_style_deconstructed_variable_declaration = true:suggestion 85 | csharp_prefer_simple_default_expression = true:suggestion 86 | csharp_style_pattern_local_over_anonymous_function = true:suggestion 87 | csharp_style_inlined_variable_declaration = true:suggestion 88 | ############################### 89 | # C# Formatting Rules # 90 | ############################### 91 | # New line preferences 92 | csharp_new_line_before_open_brace = methods, properties, types 93 | csharp_new_line_before_else = false 94 | csharp_new_line_before_catch = false 95 | csharp_new_line_before_finally = false 96 | csharp_new_line_before_members_in_object_initializers = true 97 | csharp_new_line_before_members_in_anonymous_types = true 98 | csharp_new_line_between_query_expression_clauses = true 99 | # Indentation preferences 100 | csharp_indent_case_contents = true 101 | csharp_indent_switch_labels = true 102 | csharp_indent_labels = flush_left 103 | # Space preferences 104 | csharp_space_after_cast = false 105 | csharp_space_after_keywords_in_control_flow_statements = true 106 | csharp_space_between_method_call_parameter_list_parentheses = false 107 | csharp_space_between_method_declaration_parameter_list_parentheses = false 108 | csharp_space_between_parentheses = false 109 | csharp_space_before_colon_in_inheritance_clause = true 110 | csharp_space_after_colon_in_inheritance_clause = true 111 | csharp_space_around_binary_operators = before_and_after 112 | csharp_space_between_method_declaration_empty_parameter_list_parentheses = false 113 | csharp_space_between_method_call_name_and_opening_parenthesis = false 114 | csharp_space_between_method_call_empty_parameter_list_parentheses = false 115 | # Wrapping preferences 116 | csharp_preserve_single_line_statements = true 117 | csharp_preserve_single_line_blocks = true 118 | ############################### 119 | # VB Coding Conventions # 120 | ############################### 121 | [*.vb] 122 | # Modifier preferences 123 | visual_basic_preferred_modifier_order = Partial,Default,Private,Protected,Public,Friend,NotOverridable,Overridable,MustOverride,Overloads,Overrides,MustInherit,NotInheritable,Static,Shared,Shadows,ReadOnly,WriteOnly,Dim,Const,WithEvents,Widening,Narrowing,Custom,Async:suggestion 124 | -------------------------------------------------------------------------------- /Hexastore/Graph/TripleObject.cs: -------------------------------------------------------------------------------- 1 | using Newtonsoft.Json; 2 | using Newtonsoft.Json.Linq; 3 | using System; 4 | using System.IO; 5 | 6 | namespace Hexastore.Graph 7 | { 8 | public class TripleObject 9 | { 10 | public string Value { get; } 11 | public JTokenType TokenType { get; } 12 | public bool IsID { get; } 13 | public int Index { get; } 14 | 15 | public TripleObject(string id, int? arrayIndex) 16 | : this(id, true, JTokenType.String, arrayIndex) 17 | { 18 | } 19 | 20 | public TripleObject(JValue data, bool isId, int? arrayIndex) 21 | { 22 | IsID = isId; 23 | TokenType = data.Type; 24 | Value = data.Value(); 25 | Index = arrayIndex == null ? -1 : (int)arrayIndex; 26 | } 27 | 28 | public TripleObject(string value, bool isID, JTokenType tokenType, int? arrayIndex) 29 | { 30 | Value = value; 31 | IsID = isID; 32 | TokenType = tokenType; 33 | Index = arrayIndex == null ? -1 : (int)arrayIndex; 34 | } 35 | 36 | [JsonIgnore] 37 | public string Id 38 | { 39 | get 40 | { 41 | if (IsID) { 42 | return Value; 43 | } 44 | throw new InvalidOperationException("object is not an Id"); 45 | } 46 | } 47 | [JsonIgnore] 48 | public bool IsNull 49 | { 50 | get 51 | { 52 | return Value == null; 53 | } 54 | } 55 | 56 | public static implicit operator TripleObject(JToken jToken) 57 | { 58 | return new TripleObject((JValue)jToken, false, null); 59 | } 60 | 61 | public static implicit operator TripleObject(ValueTuple jToken) 62 | { 63 | return new TripleObject((JValue)jToken.Item1, false, jToken.Item2); 64 | } 65 | 66 | public static implicit operator TripleObject(string id) 67 | { 68 | return new TripleObject(id, true, JTokenType.String, null); 69 | } 70 | 71 | public static implicit operator TripleObject(ValueTuple id) 72 | { 73 | return new TripleObject(id.Item1, true, JTokenType.String, id.Item2); 74 | } 75 | 76 | public static TripleObject FromId(string s) 77 | { 78 | return new TripleObject(s, null); 79 | } 80 | 81 | public static TripleObject FromData(string s) 82 | { 83 | return new TripleObject(s, false, JTokenType.String, null); 84 | } 85 | 86 | public static TripleObject FromData(string s, int index) 87 | { 88 | return new TripleObject(s, false, JTokenType.String, index); 89 | } 90 | 91 | public static TripleObject FromData(int n) 92 | { 93 | return new TripleObject($"{n}", false, JTokenType.Integer, null); 94 | } 95 | 96 | public static TripleObject FromData(long n) 97 | { 98 | return new TripleObject($"{n}", false, JTokenType.Float, null); 99 | } 100 | 101 | public static TripleObject FromData(double n) 102 | { 103 | return new TripleObject($"{n}", false, JTokenType.Float, null); 104 | } 105 | 106 | public static TripleObject FromData(bool f) 107 | { 108 | return new TripleObject(f ? "true" : "false", false, JTokenType.Boolean, null); 109 | } 110 | 111 | public static TripleObject FromRaw(string json) 112 | { 113 | return new TripleObject(json, false, JTokenType.String, null); 114 | } 115 | 116 | public static string Stringify(JValue jValue) 117 | { 118 | using (var textWriter = new StringWriter()) { 119 | using (var jsonWriter = new JsonTextWriter(textWriter)) { 120 | jValue.WriteTo(jsonWriter); 121 | jsonWriter.Flush(); 122 | return textWriter.ToString(); 123 | } 124 | } 125 | } 126 | 127 | public override string ToString() 128 | { 129 | if (IsID) { 130 | return $"<{Value}> c:{Index}"; 131 | } else { 132 | return $"{Value} c:{Index}"; 133 | } 134 | } 135 | 136 | public string ToValue() 137 | { 138 | return Value; 139 | } 140 | 141 | public JValue ToTypedJSON() 142 | { 143 | switch (TokenType) { 144 | case JTokenType.String: 145 | case JTokenType.Date: 146 | case JTokenType.TimeSpan: 147 | case JTokenType.Guid: 148 | case JTokenType.Uri: 149 | return new JValue(Value); 150 | case JTokenType.Integer: 151 | return new JValue(int.Parse(Value)); 152 | case JTokenType.Boolean: 153 | return new JValue(bool.Parse(Value)); 154 | case JTokenType.Float: 155 | return new JValue(float.Parse(Value)); 156 | default: 157 | throw new InvalidOperationException($"{TokenType} not support as object"); 158 | } 159 | } 160 | 161 | public string DataAsString() 162 | { 163 | if (IsID) { 164 | throw new InvalidOperationException("object is an id but should be data"); 165 | } 166 | return JValue.Parse(Value).ToString(); 167 | } 168 | 169 | public override bool Equals(object obj) 170 | { 171 | var t = obj as TripleObject; 172 | if (t == null) { 173 | return false; 174 | } 175 | 176 | if (Value == null && t.Value == null) { 177 | return true; 178 | } 179 | 180 | //if (t.Index == -1 || Index == -1) { 181 | // // unordered comparison 182 | // return t.IsID == IsID && t.Value == Value && t.TokenType == TokenType; 183 | //} 184 | 185 | return t.IsID == IsID && t.Value == Value && t.Index == Index && t.TokenType == TokenType; 186 | } 187 | 188 | public override int GetHashCode() 189 | { 190 | return (IsID ? "1" : "0" + Value).GetHashCode(); 191 | } 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /Hexastore.Test/ObjectQueryStarPathTest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Text; 6 | using Hexastore.Graph; 7 | using Hexastore.Query; 8 | using Hexastore.Rocks; 9 | using Microsoft.VisualStudio.TestTools.UnitTesting; 10 | using Newtonsoft.Json.Linq; 11 | 12 | namespace Hexastore.Test 13 | { 14 | [TestClass] 15 | public class ObjectQueryStarPathTest : RocksFixture 16 | { 17 | 18 | public ObjectQueryStarPathTest() 19 | { 20 | var text = File.ReadAllText(Path.Combine("TestInput", "region.json")); 21 | 22 | StoreProcessor.Assert("app1", JToken.Parse(text), true); 23 | } 24 | 25 | [TestMethod] 26 | public void Query_Outgoing_One_Level_Star_Returns() 27 | { 28 | var (set, _, _) = StoreProcessor.GetGraphs("app1"); 29 | 30 | var query = new ObjectQueryModel { 31 | Filter = new Dictionary() { 32 | ["type"] = new QueryUnit { Operator = "eq", Value = "building" } 33 | }, 34 | PageSize = 10, 35 | HasObject = new LinkQuery[] 36 | { 37 | new LinkQuery 38 | { 39 | Path = "*", 40 | Target = new ObjectQueryModel 41 | { 42 | Filter = new Dictionary() 43 | { 44 | ["type"] = new QueryUnit { Operator = "eq", Value = "floor" } 45 | } 46 | } 47 | } 48 | } 49 | }; 50 | 51 | var rsp = new ObjectQueryExecutor().Query(query, (RocksGraph)set); 52 | var values = rsp.Values.ToArray(); 53 | 54 | CollectionAssert.AreEquivalent(new string[] { "oBZ_JoNOBC", "zKbQyTeF" }, rsp.Values.Select(x => x.Subject).ToArray()); 55 | } 56 | 57 | [TestMethod] 58 | public void Query_Outgoing_Two_Level_Star_Returns() 59 | { 60 | var (set, _, _) = StoreProcessor.GetGraphs("app1"); 61 | 62 | var query = new ObjectQueryModel { 63 | Filter = new Dictionary() { 64 | ["type"] = new QueryUnit { Operator = "eq", Value = "building" } 65 | }, 66 | PageSize = 10, 67 | HasObject = new LinkQuery[] 68 | { 69 | new LinkQuery 70 | { 71 | Path = "*/*", 72 | Target = new ObjectQueryModel 73 | { 74 | Filter = new Dictionary() 75 | { 76 | ["name"] = new QueryUnit { Operator = "eq", Value = "Entity room 4" } 77 | } 78 | } 79 | } 80 | } 81 | }; 82 | 83 | var rsp = new ObjectQueryExecutor().Query(query, (RocksGraph)set); 84 | var values = rsp.Values.ToArray(); 85 | 86 | Assert.AreEqual(1, values.Count()); 87 | Assert.AreEqual(rsp.Continuation, null); 88 | Assert.AreEqual("oBZ_JoNOBC", values.First().Subject); 89 | } 90 | 91 | [TestMethod] 92 | public void Query_Outgoing_Partial_Star_Returns() 93 | { 94 | var (set, _, _) = StoreProcessor.GetGraphs("app1"); 95 | 96 | var query = new ObjectQueryModel { 97 | Filter = new Dictionary() { 98 | ["type"] = new QueryUnit { Operator = "eq", Value = "building" } 99 | }, 100 | PageSize = 10, 101 | HasObject = new LinkQuery[] 102 | { 103 | new LinkQuery 104 | { 105 | Path = "*/rooms", 106 | Target = new ObjectQueryModel 107 | { 108 | Filter = new Dictionary() 109 | { 110 | ["name"] = new QueryUnit { Operator = "eq", Value = "Entity room 4" } 111 | } 112 | } 113 | } 114 | } 115 | }; 116 | 117 | var rsp = new ObjectQueryExecutor().Query(query, (RocksGraph)set); 118 | var values = rsp.Values.ToArray(); 119 | 120 | Assert.AreEqual(1, values.Count()); 121 | Assert.AreEqual(rsp.Continuation, null); 122 | Assert.AreEqual("oBZ_JoNOBC", values.First().Subject); 123 | } 124 | 125 | [TestMethod] 126 | public void Query_Incoming_Three_Level_Star_Returns() 127 | { 128 | var (set, _, _) = StoreProcessor.GetGraphs("app1"); 129 | 130 | var query = new ObjectQueryModel { 131 | Filter = new Dictionary() { 132 | ["type"] = new QueryUnit { Operator = "eq", Value = "sensor" } 133 | }, 134 | PageSize = 64, 135 | HasSubject = new LinkQuery[] 136 | { 137 | new LinkQuery 138 | { 139 | Path = "*/*/*", 140 | Target = new ObjectQueryModel 141 | { 142 | Id = "oBZ_JoNOBC" 143 | } 144 | } 145 | } 146 | }; 147 | 148 | var rsp = new ObjectQueryExecutor().Query(query, (RocksGraph)set); 149 | var values = rsp.Values.ToArray(); 150 | 151 | Assert.AreEqual(8, values.Count()); 152 | Assert.AreEqual(rsp.Continuation, null); 153 | } 154 | 155 | [TestMethod] 156 | public void Query_Incoming_Partial_Star_Returns() 157 | { 158 | var (set, _, _) = StoreProcessor.GetGraphs("app1"); 159 | 160 | var query = new ObjectQueryModel { 161 | Filter = new Dictionary() { 162 | ["type"] = new QueryUnit { Operator = "eq", Value = "sensor" } 163 | }, 164 | PageSize = 64, 165 | HasSubject = new LinkQuery[] 166 | { 167 | new LinkQuery 168 | { 169 | Path = "floors/*/sensors", 170 | Target = new ObjectQueryModel 171 | { 172 | Id = "oBZ_JoNOBC" 173 | } 174 | } 175 | } 176 | }; 177 | 178 | var rsp = new ObjectQueryExecutor().Query(query, (RocksGraph)set); 179 | var values = rsp.Values.ToArray(); 180 | 181 | Assert.AreEqual(8, values.Count()); 182 | Assert.AreEqual(rsp.Continuation, null); 183 | } 184 | 185 | [TestMethod] 186 | public void Query_Incoming_Two_Level_Star_Returns() 187 | { 188 | var (set, _, _) = StoreProcessor.GetGraphs("app1"); 189 | 190 | var query = new ObjectQueryModel { 191 | Filter = new Dictionary() { 192 | ["type"] = new QueryUnit { Operator = "eq", Value = "sensor" } 193 | }, 194 | PageSize = 64, 195 | HasSubject = new LinkQuery[] 196 | { 197 | new LinkQuery 198 | { 199 | Path = "*/*", 200 | Target = new ObjectQueryModel 201 | { 202 | Id = "fR5pgeHPpH" 203 | } 204 | } 205 | } 206 | }; 207 | 208 | var rsp = new ObjectQueryExecutor().Query(query, (RocksGraph)set); 209 | var values = rsp.Values.ToArray(); 210 | 211 | Assert.AreEqual(4, values.Count()); 212 | Assert.AreEqual(rsp.Continuation, null); 213 | } 214 | } 215 | } 216 | -------------------------------------------------------------------------------- /Hexastore.Rocks/KeyConfig.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using Hexastore.Graph; 6 | 7 | namespace Hexastore.Rocks 8 | { 9 | public class KeySegments 10 | { 11 | public byte[] S { get; private set; } 12 | public byte[] P { get; private set; } 13 | public byte[] O { get; private set; } 14 | public byte[] IsId { get; private set; } 15 | public byte[] Index { get; private set; } 16 | public byte[] Type { get; private set; } 17 | 18 | private byte[] _sKey; 19 | private byte[] _pKey; 20 | private byte[] _oKey; 21 | private byte[] _name; 22 | 23 | public KeySegments(string name, string s, string p, TripleObject o) 24 | { 25 | _name = KeyConfig.GetBytes(name); 26 | S = KeyConfig.GetBytes(s); 27 | P = KeyConfig.GetBytes(p); 28 | O = KeyConfig.GetBytes(o.ToValue()); 29 | IsId = o.IsID ? KeyConfig.ByteTrue : KeyConfig.ByteFalse; 30 | Index = BitConverter.GetBytes(o.Index); 31 | Type = BitConverter.GetBytes((int)o.TokenType); 32 | } 33 | 34 | public KeySegments(string name, Triple t) : this(name, t.Subject, t.Predicate, t.Object) 35 | { 36 | } 37 | 38 | public (byte[], byte[], byte[]) GetKeys() 39 | { 40 | if (_sKey == null) { 41 | var z = KeyConfig.ByteZero; 42 | _sKey = KeyConfig.ConcatBytes(_name, KeyConfig.ByteS, z, S, z, P, z, Index); 43 | _pKey = KeyConfig.ConcatBytes(_name, KeyConfig.ByteP, z, P, z, IsId, z, O, z, S, z, Index); 44 | _oKey = KeyConfig.ConcatBytes(_name, KeyConfig.ByteO, z, IsId, z, O, z, S, z, P, z, Index); 45 | } 46 | 47 | return (_sKey, _pKey, _oKey); 48 | } 49 | 50 | public byte[] GetPPrefix() 51 | { 52 | var z = KeyConfig.ByteZero; 53 | return KeyConfig.ConcatBytes(_name, KeyConfig.ByteP, z, P, z, IsId, z, O, z, S); 54 | } 55 | 56 | public static byte[] GetNameSKey(string name) 57 | { 58 | return KeyConfig.ConcatBytes(KeyConfig.GetBytes(name), KeyConfig.ByteS); 59 | } 60 | 61 | public static byte[] GetNameSKeySubject(string name, string subject) 62 | { 63 | var nameBytes = KeyConfig.GetBytes(name); 64 | var z = KeyConfig.ByteZero; 65 | var sBytes = KeyConfig.GetBytes(subject); 66 | return KeyConfig.ConcatBytes(nameBytes, KeyConfig.ByteS, z, sBytes); 67 | } 68 | 69 | public static byte[] GetNameSKeySubjectPredicate(string name, string subject, string predicate) 70 | { 71 | var nameBytes = KeyConfig.GetBytes(name); 72 | var z = KeyConfig.ByteZero; 73 | var sBytes = KeyConfig.GetBytes(subject); 74 | var pBytes = KeyConfig.GetBytes(predicate); 75 | return KeyConfig.ConcatBytes(nameBytes, KeyConfig.ByteS, z, sBytes, z, pBytes); 76 | } 77 | 78 | public static byte[] GetNameSKeySubjectPredicateIndex(string name, string subject, string predicate, int index) 79 | { 80 | var nameBytes = KeyConfig.GetBytes(name); 81 | var z = KeyConfig.ByteZero; 82 | var sBytes = KeyConfig.GetBytes(subject); 83 | var pBytes = KeyConfig.GetBytes(predicate); 84 | var indexBytes = BitConverter.GetBytes(index); 85 | return KeyConfig.ConcatBytes(nameBytes, KeyConfig.ByteS, z, sBytes, z, pBytes, z, indexBytes); 86 | } 87 | 88 | public static byte[] GetNameOKeyObject(string name, TripleObject o) 89 | { 90 | var nameBytes = KeyConfig.GetBytes(name); 91 | var z = KeyConfig.ByteZero; 92 | var oBytes = KeyConfig.GetBytes(o.ToValue()); 93 | var isIdBytes = o.IsID ? KeyConfig.ByteTrue : KeyConfig.ByteFalse; 94 | return KeyConfig.ConcatBytes(nameBytes, KeyConfig.ByteO, z, isIdBytes, z, oBytes); 95 | } 96 | 97 | public static byte[] GetNameOKeyObjectSubject(string name, TripleObject o, string subject) 98 | { 99 | var nameBytes = KeyConfig.GetBytes(name); 100 | var z = KeyConfig.ByteZero; 101 | var oBytes = KeyConfig.GetBytes(o.ToValue()); 102 | var isIdBytes = o.IsID ? KeyConfig.ByteTrue : KeyConfig.ByteFalse; 103 | var sBytes = KeyConfig.GetBytes(subject); 104 | 105 | return KeyConfig.ConcatBytes(nameBytes, KeyConfig.ByteO, z, isIdBytes, z, oBytes, z, sBytes); 106 | } 107 | 108 | public static byte[] GetNamePKeyPredicate(string name, string predicate) 109 | { 110 | var nameBytes = KeyConfig.GetBytes(name); 111 | var z = KeyConfig.ByteZero; 112 | var pBytes = KeyConfig.GetBytes(predicate); 113 | return KeyConfig.ConcatBytes(nameBytes, KeyConfig.ByteP, z, pBytes); 114 | } 115 | 116 | public static byte[] GetNamePPredicate(string name) 117 | { 118 | var nameBytes = KeyConfig.GetBytes(name); 119 | return KeyConfig.ConcatBytes(nameBytes, KeyConfig.ByteP); 120 | } 121 | 122 | public static byte[] GetNamePKeyPredicateObject(string name, string predicate, TripleObject o) 123 | { 124 | var nameBytes = KeyConfig.GetBytes(name); 125 | var z = KeyConfig.ByteZero; 126 | var pBytes = KeyConfig.GetBytes(predicate); 127 | var oBytes = KeyConfig.GetBytes(o.ToValue()); 128 | var isIdBytes = o.IsID ? KeyConfig.ByteTrue : KeyConfig.ByteFalse; 129 | return KeyConfig.ConcatBytes(nameBytes, KeyConfig.ByteP, z, pBytes, z, isIdBytes, z, oBytes); 130 | } 131 | } 132 | 133 | public class KeyConfig 134 | { 135 | public static byte[] GetBytes(string input) 136 | { 137 | return Encoding.UTF8.GetBytes(input); 138 | } 139 | 140 | public static byte[] GetBytes(int input) 141 | { 142 | return BitConverter.GetBytes(input); 143 | } 144 | 145 | public static byte[] ConcatBytes(params byte[][] inputs) 146 | { 147 | var totalLength = inputs.Sum(x => x.Length); 148 | var output = new byte[totalLength]; 149 | 150 | var outputIndex = 0; 151 | foreach (var i in inputs) { 152 | Buffer.BlockCopy(i, 0, output, outputIndex, i.Length); 153 | outputIndex += i.Length; 154 | } 155 | return output; 156 | } 157 | 158 | public static int ByteCompare(byte[] first, byte[] second) 159 | { 160 | for (int i = 0; i < first.Length; i++) { 161 | if (i >= second.Length) { 162 | return 1; 163 | } 164 | var firstByte = first[i]; 165 | var secondByte = second[i]; 166 | if (firstByte < secondByte) { 167 | return -1; 168 | } else if (firstByte > secondByte) { 169 | return 1; 170 | } else { 171 | continue; 172 | } 173 | } 174 | if (first.Length == second.Length) { 175 | return 0; 176 | } 177 | return -1; 178 | } 179 | 180 | public static IList Split(byte[] input) 181 | { 182 | if (input.Length == 1 && input[0] != 0) { 183 | return new byte[][] { input }; 184 | } else if (input.Length == 1 && input[0] == 0) { 185 | return new byte[][] { }; 186 | } 187 | var list = new List(); 188 | var lastIndex = -1; 189 | for (int i = 0; i < input.Length; i++) { 190 | if (input[i] != 0) { 191 | continue; 192 | } 193 | var item = new byte[i - lastIndex - 1]; 194 | Buffer.BlockCopy(input, lastIndex + 1, item, 0, item.Length); 195 | list.Add(item); 196 | lastIndex = i; 197 | } 198 | return list; 199 | } 200 | 201 | public static readonly byte[] ByteZero = new byte[] { 0 }; 202 | public static readonly byte[] ByteOne = new byte[] { 1 }; 203 | public static readonly byte[] ByteFalse = new byte[] { 48 }; 204 | public static readonly byte[] ByteTrue = new byte[] { 49 }; 205 | public static readonly byte[] ByteS = new byte[] { 46, SMark }; // .S 206 | public static readonly byte[] ByteP = new byte[] { 46, PMark }; // .P 207 | public static readonly byte[] ByteO = new byte[] { 46, OMark }; // .O 208 | public const int SMark = 83; 209 | public const int PMark = 80; 210 | public const int OMark = 79; 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HexaDb 2 | ## A schemaless graph database based on RocksDb 3 | 4 | HexaDb is a triple based graph data store created on RocksDb storage. It can be used to store, retrieve and query JSON documents. 5 | HexaDb does not require a schema. It also supports relational query through providing nested path or nesting level 6 | in any arbitrary JSON document. 7 | 8 | ## Naming 9 | 10 | HexaDb breaks JSON documents to RDF triples in the format of `(S,P,O)` and creates six indices. All triples are queryable by those six indices 11 | `S, 12 | P, 13 | O, 14 | SP, 15 | PO, 16 | OS` 17 | 18 | Thus **HexaDb**. 19 | 20 | ## Building and running locally 21 | 22 | `$ docker-compose up` 23 | 24 | For more detailed documentation take a look at the [wiki](https://github.com/angshuman/hexadb/wiki/Simple-Relationships "Simple Relationships") 25 | 26 | You can also find example scenarios at [hexadb-examples](https://github.com/angshuman/hexadb-examples "Hexadb Examples") 27 | 28 | ## Getting started with the API's 29 | 30 | Let's create a graph. First we will create two object's with nested JSON objects. Later, we will create another object that references these objects by id. 31 | 32 | ### Creating your first objects 33 | 34 | `POST /api/store/app01` 35 | 36 | ```json 37 | [ 38 | { 39 | "id": "sensor:0", 40 | "type": "sensor", 41 | "name": "Entity sensor 0", 42 | "temperature": 64.44, 43 | "humidity": 14.65, 44 | "pressure": 957.1, 45 | "marker": { 46 | "status": "running", 47 | "red": 9.12, 48 | "blue": 4.53, 49 | "green": 9.85 50 | } 51 | }, 52 | { 53 | "id": "sensor:1", 54 | "type": "sensor", 55 | "name": "Entity sensor 1", 56 | "temperature": 65.86, 57 | "humidity": 12.29, 58 | "pressure": 945.19, 59 | "marker": { 60 | "status": "stopped", 61 | "red": 9.95, 62 | "blue": 7.16, 63 | "green": 2.02 64 | } 65 | } 66 | ] 67 | ``` 68 | The top level objects in the POST call need to have an id field. The nested objects will be assigned id's based on the top level id. This POST call creates four different objectswith links. Nested objects are considered top level and can be queried indepently. 69 | 70 | ### Get an object by id 71 | 72 | `GET /api/store/app01/sensor:1` 73 | 74 | ### Find an object by a top level property comparison 75 | 76 | `POST /api/store/app01/query` 77 | 78 | This query finds all object with `type` of `sensor` and with a `temperature` property > `65`. 79 | 80 | ```json 81 | { 82 | "filter": { 83 | "type": { 84 | "op": "eq", 85 | "value": "sensor" 86 | }, 87 | "temperature" : { 88 | "op" : "gt", 89 | "value" : 65, 90 | } 91 | } 92 | } 93 | ``` 94 | 95 | ### Create a relationship 96 | 97 | Posting a nested object automatically creates relationships. In HexaDb all objects are considered to be a top-level object. New objects and relationships can be created with just another POST with id references to existing objects. This is similar to POST with inline objects. But the object does not need to be inline. An id reference to an existing object is good enough. 98 | 99 | `POST /api/store/app01` 100 | 101 | ```json 102 | { 103 | "id" : "room:0", 104 | "name" : "Room Zero", 105 | "type" : "room", 106 | "description" : "Has two sensors", 107 | "sensors" : [ 108 | { 109 | "id" : "sensor:0" 110 | }, 111 | { 112 | "id" : "sensor:1" 113 | } 114 | ] 115 | } 116 | ``` 117 | 118 | This creates the following structure 119 | 120 | ![Graph](hexadb-readme.svg) 121 | 122 | 123 | ### Find an object by relationship query 124 | 125 | We are trying to find rooms that have a sensor with a `marker` object that has a property of `green > 9`. This can be thought of as the following pattern search in the graph. To query multiple nesting levels with named paths, a `#` separated list of path names are used. In this case it is `sensors#marker` 126 | 127 | ![Outgoing Path](hexadb-readme-outgoing-path.svg) 128 | 129 | `POST /api/store/app01/query` 130 | 131 | ```json 132 | { 133 | "filter": { 134 | "type": { 135 | "op": "eq", 136 | "value": "room" 137 | } 138 | }, 139 | "outgoing": [ 140 | { 141 | "path": "sensors#marker", 142 | "target": { 143 | "filter": { 144 | "green": { 145 | "op": "gt", 146 | "value": 9 147 | } 148 | } 149 | } 150 | } 151 | ] 152 | } 153 | ``` 154 | 155 | ### Find an object by nesting level 156 | 157 | Here is the same query without specifying the explicit relationships. This is querying all outgoing objects in the nesting level of `3`. Notice that the `path` is `*` so it will match any objects in that vicinity. 158 | 159 | `POST /api/store/app01/query` 160 | 161 | ```json 162 | { 163 | "filter": { 164 | "type": { 165 | "op": "eq", 166 | "value": "room" 167 | } 168 | }, 169 | "outgoing": [ 170 | { 171 | "path": "*", 172 | "level": 3, 173 | "target": { 174 | "filter": { 175 | "green": { 176 | "op": "gt", 177 | "value": 9 178 | } 179 | } 180 | } 181 | } 182 | ] 183 | } 184 | ``` 185 | 186 | Note that without specifying the path all the objects in the nesting level will be considered for a pattern match. For example this query may look like below 187 | 188 | ![Outgoing Level](hexadb-readme-outgoing-level.svg) 189 | 190 | ### Find an object with incoming relationship 191 | 192 | Similar to the outgoing queries it is also possible to query objects that are pointed to by other objects. E.q. in a `parent->child` relatioship children can be found by `incoming` query from the parent. 193 | 194 | Incoming queries can also be done with nesting level with `*` as path. 195 | 196 | ```json 197 | { 198 | "filter": { 199 | "type": { 200 | "op": "eq", 201 | "value": "sensor" 202 | } 203 | }, 204 | "incoming": [ 205 | { 206 | "path": "sensors", 207 | "target": { 208 | "filter": { 209 | "name": { 210 | "op": "contains", 211 | "value": "Room" 212 | } 213 | } 214 | } 215 | } 216 | ] 217 | } 218 | ``` 219 | 220 | This is querying for the following pattern in the graph 221 | 222 | ![Incomng Path](hexadb-readme-incoming-path.svg) 223 | 224 | 225 | ### Update an object 226 | 227 | `PATCH /api/store/app01/json` 228 | 229 | A json-merge-patch style endpoint is available. Below is an example that changes the name of the object pointed to by `sensor:0` and modifies the `marker` relationship to contain a single object with no `status` (deleted) and value of `red` with `1.0`. 230 | 231 | ```json 232 | { 233 | "id": "sensor:0", 234 | "name": "Another name", 235 | "marker": { 236 | "status": null, 237 | "red": 1.0 238 | } 239 | } 240 | ``` 241 | 242 | `PATCH /api/store/app01/triple` 243 | 244 | Hexadb also supports patching at the triple level. The patches are performed with two separate `add` and `remove` sections. The `remove` is performed before `add`. 245 | 246 | ```json 247 | { 248 | "remove": { 249 | "id": "room:0", 250 | "sensors": [ 251 | { 252 | "id": "sensor:0" 253 | } 254 | ] 255 | }, 256 | "add": { 257 | "id": "room:1", 258 | "sensors": [ 259 | { 260 | "id": "sensor:2", 261 | "temperature" : 50, 262 | "description" : "some other sensor" 263 | } 264 | ] 265 | } 266 | } 267 | ``` 268 | 269 | ### Replication 270 | 271 | The data store can be replicated to multiple instances using event hubs. The store supports the following environment variables to use for replication. 272 | 273 | ``` 274 | HEXASTORE_EVENTHUB_KEY 275 | HEXASTORE_EVENTHUB_PARTITION_COUNT 276 | HEXASTORE_EVENTHUB_NAME 277 | ``` 278 | 279 | The `HEXASTORE_EVENTHUB_KEY` must contain `EntityPath`. `HEXASTORE_EVENTHUB_PARTITION_COUNT` denotes the number of partitions to send replication data to. 280 | `HEXASTORE_EVENTHUB_NAME` identifies the event hub as a name. If this is changed the event hub configuration will be treated as a new configuration with 281 | no checkpoints. The checkpoints are used to remember the offset to which the database contains information. On restart the writes are read from the already 282 | saved offset in the database. 283 | 284 | 285 | --------------------------------------------------------------------------------