├── .nuke ├── .gitignore ├── src ├── OutCode.EscapeTeams.ObjectRepository │ ├── Attributes.cs │ ├── ChangeType.cs │ ├── BaseEntity.cs │ ├── ITableDictionary.cs │ ├── ObjectRepositoryException.cs │ ├── IStorage.cs │ ├── LoadingInProgressException.cs │ ├── OutCode.EscapeTeams.ObjectRepository.csproj │ ├── ModelChangedEventArgs.cs │ ├── ConcurrentList.cs │ ├── ReflectionExtensions.cs │ ├── TableDictionary`1.cs │ ├── ModelExtensions.cs │ ├── ModelBase.cs │ ├── TableDictionary.cs │ └── ObjectRepositoryBase.cs ├── OutCode.EscapeTeams.ObjectRepository.Hangfire.Tests │ ├── appsettings.json │ ├── appsettings.Development.json │ ├── Program.cs │ ├── Properties │ │ └── launchSettings.json │ ├── OutCode.EscapeTeams.ObjectRepository.Hangfire.Tests.csproj │ └── Startup.cs ├── OutCode.EscapeTeams.ObjectRepository.Hangfire │ ├── EnqueuedAndFetchedCountDto.cs │ ├── ServerData.cs │ ├── CollectionHelpers.cs │ ├── ObjectRepositoryJobQueueProvider.cs │ ├── OutCode.EscapeTeams.ObjectRepository.Hangfire.csproj │ ├── ObjectRepositoryStorage.cs │ ├── Entities │ │ ├── ListModel.cs │ │ ├── CounterModel.cs │ │ ├── Server.cs │ │ ├── JobQueueModel.cs │ │ ├── JobParameter.cs │ │ ├── HashModel.cs │ │ ├── SetModel.cs │ │ ├── StateModel.cs │ │ └── JobModel.cs │ ├── ObjectRepositoryFetchedJob.cs │ ├── ObjectRepositoryExtensions.cs │ ├── ObjectRepositoryJobQueueMonitoringApi.cs │ ├── ExpirationManager.cs │ ├── ObjectRepositoryWriteOnlyTransaction.cs │ ├── ObjectRepositoryConnection.cs │ └── ObjectRepositoryMonitoringApi.cs ├── OutCode.EscapeTeams.ObjectRepository.LiteDB │ ├── ObjectIdExtensions.cs │ ├── OutCode.EscapeTeams.ObjectRepository.LiteDB.csproj │ └── LiteDbStorage.cs ├── OutCode.EscapeTeams.ObjectRepository.AzureTableStorage │ ├── AzureTableMigration.cs │ ├── OutCode.EscapeTeams.ObjectRepository.AzureTableStorage.csproj │ ├── AzureExtensions.cs │ ├── DateTimeAwareTableEntityAdapter.cs │ └── AzureTableContext.cs ├── OutCode.EscapeTeams.ObjectRepository.Tests │ ├── TestObjectRepository.cs │ ├── TestStorage.cs │ ├── OutCode.EscapeTeams.ObjectRepository.Tests.csproj │ ├── FileTests.cs │ ├── TestModel.cs │ ├── LiteDbTests.cs │ ├── AzureStorageTests.cs │ ├── ObjectRepositoryTests.cs │ └── ProviderTestBase.cs ├── OutCode.EscapeTeams.ObjectRepository.File │ ├── OutCode.EscapeTeams.ObjectRepository.File.csproj │ └── FileStorage.cs ├── build.sh ├── build.ps1 └── OutCode.EscapeTeams.ObjectRepository.sln ├── .github └── workflows │ └── build.yml └── README.md /.nuke: -------------------------------------------------------------------------------- 1 | src/OutCode.EscapeTeams.ObjectRepository.sln -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /output/ 2 | .vs/ 3 | bin/ 4 | obj/ 5 | *.user 6 | .tmp/ 7 | .idea 8 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository/Attributes.cs: -------------------------------------------------------------------------------- 1 | 2 | using System.Runtime.CompilerServices; 3 | 4 | [assembly:InternalsVisibleTo("OutCode.EscapeTeams.ObjectRepository.Tests")] -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire.Tests/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Warning" 5 | } 6 | }, 7 | "AllowedHosts": "*" 8 | } 9 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository/ChangeType.cs: -------------------------------------------------------------------------------- 1 | namespace OutCode.EscapeTeams.ObjectRepository 2 | { 3 | public enum ChangeType 4 | { 5 | Add, 6 | Remove, 7 | Update 8 | } 9 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository/BaseEntity.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace OutCode.EscapeTeams.ObjectRepository 4 | { 5 | public class BaseEntity 6 | { 7 | public Guid Id { get; set; } 8 | } 9 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire.Tests/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Debug", 5 | "System": "Information", 6 | "Microsoft": "Information" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository/ITableDictionary.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace OutCode.EscapeTeams.ObjectRepository 5 | { 6 | public interface ITableDictionary : IEnumerable 7 | { 8 | T Find(Guid id); 9 | } 10 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/EnqueuedAndFetchedCountDto.cs: -------------------------------------------------------------------------------- 1 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire 2 | { 3 | public class EnqueuedAndFetchedCountDto 4 | { 5 | public long? EnqueuedCount { get; set; } 6 | public long? FetchedCount { get; set; } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/ServerData.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire 4 | { 5 | internal class ServerData 6 | { 7 | public int WorkerCount { get; set; } 8 | public string[] Queues { get; set; } 9 | public DateTime? StartedAt { get; set; } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository/ObjectRepositoryException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace OutCode.EscapeTeams.ObjectRepository 4 | { 5 | public class ObjectRepositoryException : Exception 6 | { 7 | public ObjectRepositoryException(Guid? id, string callingProperty, string where):base($"bad ID {id} for type {typeof(T).FullName} at {callingProperty} ({where})") 8 | { 9 | 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository/IStorage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Threading.Tasks; 4 | 5 | namespace OutCode.EscapeTeams.ObjectRepository 6 | { 7 | public interface IStorage 8 | { 9 | Task SaveChanges(); 10 | Task> GetAll(); 11 | void Track(ObjectRepositoryBase objectRepository, bool isReadonly); 12 | event Action OnError; 13 | } 14 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/CollectionHelpers.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire 5 | { 6 | internal static class CollectionHelpers 7 | { 8 | public static void ForEach(this IEnumerable collection, Action action) 9 | { 10 | foreach (var item in collection) 11 | action(item); 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.LiteDB/ObjectIdExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using LiteDB; 3 | 4 | namespace OutCode.EscapeTeams.ObjectRepository.LiteDB 5 | { 6 | public static class ObjectIdExtensions 7 | { 8 | public static Guid ToGuid(this ObjectId id) 9 | { 10 | var byteArray = id.ToByteArray(); 11 | Array.Resize(ref byteArray, 16); 12 | return new Guid(byteArray); 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository/LoadingInProgressException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace OutCode.EscapeTeams.ObjectRepository 4 | { 5 | public class LoadingInProgressException : Exception 6 | { 7 | public double Progress { get; set; } 8 | 9 | public LoadingInProgressException(double progress) 10 | { 11 | Progress = progress; 12 | } 13 | 14 | public override string Message => Progress + "% loading done..."; 15 | } 16 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.AzureTableStorage/AzureTableMigration.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace OutCode.EscapeTeams.ObjectRepository.AzureTableStorage 4 | { 5 | public class AzureTableMigration 6 | { 7 | public string Name { get; private set; } 8 | 9 | public AzureTableMigration(string name) 10 | { 11 | Name = name; 12 | } 13 | 14 | public virtual Task Execute(AzureTableContext context) => Task.CompletedTask; 15 | } 16 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire.Tests/Program.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.AspNetCore; 2 | using Microsoft.AspNetCore.Hosting; 3 | 4 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire.Tests 5 | { 6 | public static class Program 7 | { 8 | public static void Main(string[] args) 9 | { 10 | CreateWebHostBuilder(args).Build().Run(); 11 | } 12 | 13 | public static IWebHostBuilder CreateWebHostBuilder(string[] args) => 14 | WebHost.CreateDefaultBuilder(args) 15 | .UseStartup(); 16 | } 17 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Tests/TestObjectRepository.cs: -------------------------------------------------------------------------------- 1 | using System.Threading; 2 | using Microsoft.Extensions.Logging.Abstractions; 3 | 4 | namespace OutCode.EscapeTeams.ObjectRepository.Tests 5 | { 6 | public class TestObjectRepository : ObjectRepositoryBase 7 | { 8 | public TestObjectRepository(IStorage storage) : base(storage, NullLogger.Instance) 9 | { 10 | IsReadOnly = true; 11 | AddType((ParentEntity x) => new ParentModel(x)); 12 | AddType((ChildEntity x) => new ChildModel(x)); 13 | Initialize(); 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Tests/TestStorage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | 6 | namespace OutCode.EscapeTeams.ObjectRepository.Tests 7 | { 8 | public class TestStorage : List, IStorage 9 | { 10 | public Task SaveChanges() => Task.CompletedTask; 11 | 12 | public Task> GetAll() => Task.FromResult(this.OfType()); 13 | 14 | public void Track(ObjectRepositoryBase objectRepository, bool isReadonly) 15 | { 16 | } 17 | 18 | public event Action OnError = delegate { }; 19 | } 20 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/ObjectRepositoryJobQueueProvider.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire 4 | { 5 | internal class ObjectRepositoryJobQueueProvider 6 | { 7 | private readonly ObjectRepositoryJobQueueMonitoringApi _monitoringApi; 8 | 9 | public ObjectRepositoryJobQueueProvider(ObjectRepositoryBase storage) 10 | { 11 | if (storage == null) throw new ArgumentNullException(nameof(storage)); 12 | 13 | _monitoringApi = new ObjectRepositoryJobQueueMonitoringApi(storage); 14 | } 15 | 16 | public ObjectRepositoryJobQueueMonitoringApi GetJobQueueMonitoringApi() => _monitoringApi; 17 | } 18 | } -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: [push, pull_request, workflow_dispatch] 4 | 5 | jobs: 6 | build: 7 | runs-on: windows-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | 11 | - name: Setup .NET Core 12 | uses: actions/setup-dotnet@v1 13 | with: 14 | dotnet-version: 5.0.x 15 | 16 | - name: Build with dotnet 17 | run: | 18 | cd src 19 | dotnet build 20 | dotnet test 21 | 22 | - name: Publish to NuGet.org 23 | if: github.event_name == 'push' && github.ref == 'refs/heads/master' 24 | env: 25 | NUGET: ${{ secrets.NUGET }} 26 | run: | 27 | cd src 28 | dotnet pack -p:PackageVersion=2.0.${env:GITHUB_RUN_NUMBER} -o nugets -c Release 29 | dotnet nuget push nugets\*.nupkg -k ${env:NUGET} -s https://api.nuget.org/v3/index.json -n true 30 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire.Tests/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "iisSettings": { 3 | "windowsAuthentication": false, 4 | "anonymousAuthentication": true, 5 | "iisExpress": { 6 | "applicationUrl": "http://localhost:27468", 7 | "sslPort": 44340 8 | } 9 | }, 10 | "profiles": { 11 | "IIS Express": { 12 | "commandName": "IISExpress", 13 | "launchBrowser": true, 14 | "environmentVariables": { 15 | "ASPNETCORE_ENVIRONMENT": "Development" 16 | } 17 | }, 18 | "OutCode.EscapeTeams.ObjectRepository.Hangfire.Tests": { 19 | "commandName": "Project", 20 | "launchBrowser": true, 21 | "applicationUrl": "https://localhost:5001;http://localhost:5000", 22 | "environmentVariables": { 23 | "ASPNETCORE_ENVIRONMENT": "Development" 24 | } 25 | } 26 | } 27 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.File/OutCode.EscapeTeams.ObjectRepository.File.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net5.0 5 | 1.0.0.0 6 | Kirill Orlov 7 | EscapeTeams 8 | ObjectRepository File-backed storage provider 9 | File backed storage provider for EscapeTeams ObjectRepository 10 | 1.0.0.0 11 | 1.0.0.0 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.AzureTableStorage/OutCode.EscapeTeams.ObjectRepository.AzureTableStorage.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net5.0 5 | 1.0.0.0 6 | Kirill Orlov 7 | EscapeTeams 8 | ObjectRepository AzureTable-backed storage provider 9 | Azure Tables backed storage provider for EscapeTeams ObjectRepository 10 | 1.0.0.0 11 | 1.0.0.0 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire.Tests/OutCode.EscapeTeams.ObjectRepository.Hangfire.Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net5.0 5 | InProcess 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/OutCode.EscapeTeams.ObjectRepository.Hangfire.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net5.0 5 | 1.0.0.0 6 | Kirill Orlov 7 | EscapeTeams 8 | ObjectRepository 9 | Hangfire plugin for ObjectRepository 10 | 1.0.0.0 11 | 1.0.0.0 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.LiteDB/OutCode.EscapeTeams.ObjectRepository.LiteDB.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net5.0 5 | 1.0.0.0 6 | Kirill Orlov 7 | EscapeTeams 8 | ObjectRepository LiteDB-backed storage provider 9 | LiteDB backed storage provider for EscapeTeams ObjectRepository 10 | 1.0.0.0 11 | 1.0.0.0 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository/OutCode.EscapeTeams.ObjectRepository.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net5.0 5 | 1.0.0.0 6 | Kirill Orlov 7 | EscapeTeams 8 | ObjectRepository 9 | In-memory database used by EscapeTeams. 10 | 1.0.0.0 11 | 1.0.0.0 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/ObjectRepositoryStorage.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using Hangfire; 3 | using Hangfire.Server; 4 | using Hangfire.Storage; 5 | 6 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire 7 | { 8 | public class ObjectRepositoryStorage : JobStorage 9 | { 10 | internal ObjectRepositoryJobQueueMonitoringApi MonitoringApi { get; set; } 11 | internal ObjectRepositoryBase ObjectRepository { get; } 12 | 13 | public ObjectRepositoryStorage(ObjectRepositoryBase objectRepository) 14 | { 15 | ObjectRepository = objectRepository; 16 | MonitoringApi = new ObjectRepositoryJobQueueMonitoringApi(objectRepository); 17 | } 18 | 19 | public override IMonitoringApi GetMonitoringApi() 20 | { 21 | return new ObjectRepositoryMonitoringApi(this); 22 | } 23 | 24 | public override IStorageConnection GetConnection() 25 | { 26 | return new ObjectRepositoryConnection(this); 27 | } 28 | 29 | #pragma warning disable 618 30 | public override IEnumerable GetComponents() 31 | #pragma warning restore 618 32 | { 33 | yield return new ExpirationManager(this); 34 | } 35 | } 36 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/Entities/ListModel.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire.Entities 4 | { 5 | internal class ListModel : ModelBase 6 | { 7 | internal class ListEntity : BaseEntity 8 | { 9 | public string Key { get; set; } 10 | public string Value { get; set; } 11 | public DateTime? ExpireAt { get; set; } 12 | } 13 | 14 | private readonly ListEntity _list; 15 | 16 | public ListModel(ListEntity list) 17 | { 18 | _list = list; 19 | } 20 | 21 | public ListModel(string key, string value) 22 | { 23 | _list = new ListEntity 24 | { 25 | Id = Guid.NewGuid(), 26 | Key = key, 27 | Value = value 28 | }; 29 | } 30 | 31 | protected override BaseEntity Entity => _list; 32 | 33 | public DateTime? ExpireAt 34 | { 35 | get => _list.ExpireAt; 36 | set => UpdateProperty(_list, () => x => x.ExpireAt, value); 37 | } 38 | 39 | public string Key 40 | { 41 | get => _list.Key; 42 | set => UpdateProperty(_list, () => x => x.Key, value); 43 | } 44 | 45 | public string Value 46 | { 47 | get => _list.Value; 48 | set => UpdateProperty(_list, () => x => x.Value, value); 49 | } 50 | } 51 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/Entities/CounterModel.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire.Entities 4 | { 5 | internal class CounterModel : ModelBase 6 | { 7 | internal class CounterEntity : BaseEntity 8 | { 9 | public string Key { get; set; } 10 | public int Value { get; set; } 11 | public DateTime? ExpireAt { get; set; } 12 | } 13 | 14 | private readonly CounterEntity _counter; 15 | 16 | public CounterModel(CounterEntity counter) 17 | { 18 | _counter = counter; 19 | } 20 | 21 | public CounterModel(string key) 22 | { 23 | _counter = new CounterEntity 24 | { 25 | Id = Guid.NewGuid(), 26 | Key = key 27 | }; 28 | } 29 | 30 | protected override BaseEntity Entity => _counter; 31 | 32 | public DateTime? ExpireAt 33 | { 34 | get => _counter.ExpireAt; 35 | set => UpdateProperty(_counter, () => x => x.ExpireAt, value); 36 | } 37 | 38 | public string Key 39 | { 40 | get => _counter.Key; 41 | set => UpdateProperty(_counter, () => x => x.Key, value); 42 | } 43 | 44 | public int Value 45 | { 46 | get => _counter.Value; 47 | set => UpdateProperty(_counter, () => x => _counter.Value, value); 48 | } 49 | } 50 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/Entities/Server.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire.Entities 4 | { 5 | internal class ServerModel : ModelBase 6 | { 7 | internal class ServerEntity : BaseEntity 8 | { 9 | public string Data { get; set; } 10 | public DateTime LastHeartbeat { get; set; } 11 | public string Name { get; set; } 12 | } 13 | 14 | private readonly ServerEntity _server; 15 | 16 | public ServerModel(ServerEntity server) 17 | { 18 | _server = server; 19 | } 20 | 21 | public ServerModel(string serverId) 22 | { 23 | _server = new ServerEntity 24 | { 25 | Id = Guid.NewGuid(), 26 | Name = serverId 27 | }; 28 | } 29 | 30 | protected override BaseEntity Entity => _server; 31 | 32 | public string Data 33 | { 34 | get => _server.Data; 35 | set => UpdateProperty(_server, () => x => x.Data, value); 36 | } 37 | 38 | public DateTime LastHeartbeat 39 | { 40 | get => _server.LastHeartbeat; 41 | set => UpdateProperty(_server, () => x => x.LastHeartbeat, value); 42 | } 43 | 44 | public string Name 45 | { 46 | get => _server.Name; 47 | set => UpdateProperty(_server, () => x => x.Name, value); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/Entities/JobQueueModel.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire.Entities 4 | { 5 | internal class JobQueueModel : ModelBase 6 | { 7 | internal class JobQueueEntity : BaseEntity 8 | { 9 | public Guid JobId { get; set; } 10 | public string Queue { get; set; } 11 | public DateTime? FetchedAt { get; set; } 12 | } 13 | 14 | private readonly JobQueueEntity _jobQueue; 15 | 16 | public JobQueueModel(JobQueueEntity jobQueue) 17 | { 18 | _jobQueue = jobQueue; 19 | } 20 | 21 | public JobQueueModel() 22 | { 23 | _jobQueue = new JobQueueEntity 24 | { 25 | Id = Guid.NewGuid() 26 | }; 27 | } 28 | 29 | protected override BaseEntity Entity => _jobQueue; 30 | 31 | public Guid JobId 32 | { 33 | get => _jobQueue.JobId; 34 | set => UpdateProperty(_jobQueue, () => x => x.JobId, value); 35 | } 36 | public string Queue 37 | { 38 | get => _jobQueue.Queue; 39 | set => UpdateProperty(_jobQueue, () => x => x.Queue, value); 40 | } 41 | public DateTime? FetchedAt 42 | { 43 | get => _jobQueue.FetchedAt; 44 | set => UpdateProperty(_jobQueue, () => x => x.FetchedAt, value); 45 | } 46 | 47 | public JobModel Job => Single(JobId); 48 | } 49 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Tests/OutCode.EscapeTeams.ObjectRepository.Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net5.0 5 | false 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | ..\..\..\..\..\.nuget\packages\litedb\4.1.4\lib\netstandard2.0\LiteDB.dll 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/Entities/JobParameter.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire.Entities 4 | { 5 | internal class JobParameterModel : ModelBase 6 | { 7 | internal class JobParameterEntity : BaseEntity 8 | { 9 | public Guid JobId { get; set; } 10 | public string Name { get; set; } 11 | public string Value { get; set; } 12 | } 13 | 14 | private readonly JobParameterEntity _jobParameter; 15 | 16 | public JobParameterModel(JobParameterEntity jobParameter) 17 | { 18 | _jobParameter = jobParameter; 19 | } 20 | 21 | public JobParameterModel(Guid jobId, string name) 22 | { 23 | _jobParameter = new JobParameterEntity 24 | { 25 | Id = Guid.NewGuid(), 26 | JobId = jobId, 27 | Name = name 28 | }; 29 | } 30 | 31 | protected override BaseEntity Entity => _jobParameter; 32 | 33 | public Guid JobId 34 | { 35 | get => _jobParameter.JobId; 36 | set => UpdateProperty(_jobParameter, () => x => x.JobId, value); 37 | } 38 | 39 | public string Name 40 | { 41 | get => _jobParameter.Name; 42 | set => UpdateProperty(_jobParameter, () => x => x.Name, value); 43 | } 44 | 45 | public string Value 46 | { 47 | get => _jobParameter.Value; 48 | set => UpdateProperty(_jobParameter, () => x => x.Value, value); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/ObjectRepositoryFetchedJob.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using Hangfire.Annotations; 6 | using Hangfire.Storage; 7 | using OutCode.EscapeTeams.ObjectRepository.Hangfire.Entities; 8 | 9 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire 10 | { 11 | internal class ObjectRepositoryFetchedJob : IFetchedJob 12 | { 13 | private readonly ConcurrentList _jobsTakenOut; 14 | private readonly ObjectRepositoryBase _storage; 15 | private readonly JobQueueModel _job; 16 | 17 | public ObjectRepositoryFetchedJob(ConcurrentList jobsTakenOut, 18 | [NotNull] ObjectRepositoryBase storage, 19 | JobQueueModel job) 20 | { 21 | _storage = storage; 22 | _jobsTakenOut = jobsTakenOut; 23 | _job = job; 24 | 25 | jobsTakenOut.Add(job); 26 | } 27 | 28 | public Guid Id => _job.Id; 29 | public string JobId => _job.JobId.ToString(); 30 | public string Queue => _job.Queue; 31 | 32 | public void RemoveFromQueue() 33 | { 34 | _jobsTakenOut.Remove(_job); 35 | _storage.Remove(s => s.Id == Id); 36 | } 37 | 38 | public void Requeue() 39 | { 40 | _jobsTakenOut.Remove(_job); 41 | _storage.Set().Find(Id).FetchedAt = null; 42 | } 43 | 44 | public void Dispose() 45 | { 46 | _jobsTakenOut.Remove(_job); 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository/ModelChangedEventArgs.cs: -------------------------------------------------------------------------------- 1 | namespace OutCode.EscapeTeams.ObjectRepository 2 | { 3 | public class ModelChangedEventArgs 4 | { 5 | public static ModelChangedEventArgs Added(ModelBase source) 6 | { 7 | return new ModelChangedEventArgs 8 | { 9 | Source = source, 10 | Entity = source.Entity, 11 | ChangeType = ChangeType.Add 12 | }; 13 | } 14 | 15 | public static ModelChangedEventArgs Removed(ModelBase source) 16 | { 17 | return new ModelChangedEventArgs 18 | { 19 | Source = source, 20 | Entity = source.Entity, 21 | ChangeType = ChangeType.Remove 22 | }; 23 | } 24 | 25 | public static ModelChangedEventArgs PropertyChange(ModelBase source, string propertyName, object from, 26 | object to) 27 | { 28 | return new ModelChangedEventArgs 29 | { 30 | Source = source, 31 | Entity = source.Entity, 32 | ChangeType = ChangeType.Update, 33 | PropertyName = propertyName, 34 | OldValue = from, 35 | NewValue = to 36 | }; 37 | } 38 | 39 | public ModelBase Source { get; private set; } 40 | public BaseEntity Entity { get; private set; } 41 | public ChangeType ChangeType { get; private set; } 42 | public string PropertyName { get; private set; } 43 | public object OldValue { get; private set; } 44 | public object NewValue { get; private set; } 45 | } 46 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/Entities/HashModel.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire.Entities 4 | { 5 | internal class HashModel : ModelBase 6 | { 7 | internal class HashEntity : BaseEntity 8 | { 9 | public string Key { get; set; } 10 | public string Field { get; set; } 11 | public string Value { get; set; } 12 | public DateTime? ExpireAt { get; set; } 13 | } 14 | 15 | private readonly HashEntity _hash; 16 | 17 | public HashModel(HashEntity hashEntity) 18 | { 19 | _hash = hashEntity; 20 | } 21 | 22 | public HashModel(string key, string field) 23 | { 24 | _hash = new HashEntity 25 | { 26 | Id = Guid.NewGuid(), 27 | Key = key, 28 | Field = field 29 | }; 30 | } 31 | 32 | protected override BaseEntity Entity => _hash; 33 | 34 | public DateTime? ExpireAt 35 | { 36 | get => _hash.ExpireAt; 37 | set => UpdateProperty(_hash, () => x => x.ExpireAt, value); 38 | } 39 | 40 | public string Field 41 | { 42 | get => _hash.Field; 43 | set => UpdateProperty(_hash, () => x => x.Field, value); 44 | } 45 | 46 | public string Key 47 | { 48 | get => _hash.Key; 49 | set => UpdateProperty(_hash, () => x => x.Key, value); 50 | } 51 | 52 | public string Value 53 | { 54 | get => _hash.Value; 55 | set => UpdateProperty(_hash, () => x => x.Value, value); 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.AzureTableStorage/AzureExtensions.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Threading.Tasks; 3 | using Microsoft.WindowsAzure.Storage.Blob; 4 | using Microsoft.WindowsAzure.Storage.Table; 5 | 6 | namespace OutCode.EscapeTeams.ObjectRepository.AzureTableStorage 7 | { 8 | public static class AzureExtensions 9 | { 10 | public static async Task> ListBlobsAsync(this CloudBlobContainer container) 11 | { 12 | var realResult = new List(); 13 | 14 | var result = await container.ListBlobsSegmentedAsync(null, true, BlobListingDetails.All, null, null, null, null); 15 | realResult.AddRange(result.Results); 16 | while (result.ContinuationToken != null) 17 | { 18 | result = await container.ListBlobsSegmentedAsync(null, true, BlobListingDetails.All, null, result.ContinuationToken, null, null); 19 | realResult.AddRange(result.Results); 20 | } 21 | 22 | return realResult; 23 | } 24 | 25 | public static async Task> ExecuteQueryAsync(this CloudTable table, TableQuery query) where T : ITableEntity, new() 26 | { 27 | var realResult = new List(); 28 | 29 | var result = await table.ExecuteQuerySegmentedAsync(query, null); 30 | realResult.AddRange(result.Results); 31 | while (result.ContinuationToken != null) 32 | { 33 | result = await table.ExecuteQuerySegmentedAsync(query, result.ContinuationToken); 34 | realResult.AddRange(result.Results); 35 | } 36 | 37 | return realResult; 38 | } 39 | } 40 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/Entities/SetModel.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire.Entities 4 | { 5 | internal class SetModel : ModelBase 6 | { 7 | internal class SetEntity : BaseEntity 8 | { 9 | public string Key { get; set; } 10 | 11 | public double Score { get; set; } 12 | 13 | public string Value { get; set; } 14 | 15 | public DateTime? ExpireAt { get; set; } 16 | } 17 | 18 | private readonly SetEntity _set; 19 | 20 | public SetModel(SetEntity set) 21 | { 22 | _set = set; 23 | } 24 | 25 | public SetModel(string key, string value) 26 | { 27 | _set = new SetEntity 28 | { 29 | Id = Guid.NewGuid(), 30 | Key = key, 31 | Value = value, 32 | Score = 0.0 33 | }; 34 | } 35 | 36 | protected override BaseEntity Entity => _set; 37 | 38 | public DateTime? ExpireAt 39 | { 40 | get => _set.ExpireAt; 41 | set => UpdateProperty(_set, () => x => x.ExpireAt, value); 42 | } 43 | 44 | public double Score 45 | { 46 | get => _set.Score; 47 | set => UpdateProperty(_set, () => x => x.Score, value); 48 | } 49 | 50 | public string Key 51 | { 52 | get => _set.Key; 53 | set => UpdateProperty(_set, () => x => x.Key, value); 54 | } 55 | 56 | public string Value 57 | { 58 | get => _set.Value; 59 | set => UpdateProperty(_set, () => x => x.Value, value); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/ObjectRepositoryExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Hangfire; 3 | using Hangfire.Annotations; 4 | using OutCode.EscapeTeams.ObjectRepository.Hangfire.Entities; 5 | 6 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire 7 | { 8 | public static class ObjectRepositoryExtensions 9 | { 10 | internal const int QueuePollInterval = 1000; 11 | 12 | public static void RegisterHangfireScheme(this ObjectRepositoryBase objectRepository) 13 | { 14 | objectRepository.AddType((CounterModel.CounterEntity x) => new CounterModel(x)); 15 | objectRepository.AddType((HashModel.HashEntity x) => new HashModel(x)); 16 | objectRepository.AddType((JobModel.JobEntity x) => new JobModel(x)); 17 | objectRepository.AddType((JobParameterModel.JobParameterEntity x) => new JobParameterModel(x)); 18 | objectRepository.AddType((JobQueueModel.JobQueueEntity x) => new JobQueueModel(x)); 19 | objectRepository.AddType((ListModel.ListEntity x) => new ListModel(x)); 20 | objectRepository.AddType((ServerModel.ServerEntity x) => new ServerModel(x)); 21 | objectRepository.AddType((SetModel.SetEntity x) => new SetModel(x)); 22 | objectRepository.AddType((StateModel.StateEntity x) => new StateModel(x)); 23 | } 24 | 25 | public static IGlobalConfiguration UseHangfireStorage( 26 | [NotNull] this IGlobalConfiguration configuration, ObjectRepositoryBase objectRepository) 27 | { 28 | if (configuration == null) throw new ArgumentNullException(nameof(configuration)); 29 | 30 | var storage = new ObjectRepositoryStorage(objectRepository); 31 | return configuration.UseStorage(storage); 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.AzureTableStorage/DateTimeAwareTableEntityAdapter.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using Microsoft.WindowsAzure.Storage; 4 | using Microsoft.WindowsAzure.Storage.Table; 5 | 6 | namespace OutCode.EscapeTeams.ObjectRepository.AzureTableStorage 7 | { 8 | internal class DateTimeAwareTableEntityAdapter : TableEntityAdapter where T:BaseEntity 9 | { 10 | public DateTimeAwareTableEntityAdapter() 11 | { 12 | } 13 | 14 | public DateTimeAwareTableEntityAdapter(T entity) : base(entity) 15 | { 16 | ETag = "*"; 17 | RowKey = entity.Id.ToString(); 18 | PartitionKey = entity.GetType().Name; 19 | } 20 | 21 | public override void ReadEntity(IDictionary properties, 22 | OperationContext operationContext) 23 | { 24 | OriginalEntity = ConvertBack(properties, 25 | new EntityPropertyConverterOptions {PropertyNameDelimiter = "."}, operationContext); 26 | OriginalEntity.Id = Guid.Parse(RowKey); 27 | } 28 | 29 | public override IDictionary WriteEntity(OperationContext operationContext) 30 | { 31 | var result = Flatten(OriginalEntity, new EntityPropertyConverterOptions() {PropertyNameDelimiter = "."}, 32 | operationContext); 33 | 34 | foreach (var item in result) 35 | { 36 | if (item.Value.PropertyType == EdmType.DateTime && item.Value.DateTime == DateTime.MinValue) 37 | item.Value.DateTime = null; 38 | } 39 | 40 | result.Remove(nameof(BaseEntity.Id)); 41 | return result; 42 | } 43 | 44 | public bool InconsistentPartitionKey => PartitionKey != OriginalEntity.GetType().Name; 45 | } 46 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/Entities/StateModel.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire.Entities 4 | { 5 | internal class StateModel : ModelBase 6 | { 7 | internal class StateEntity : BaseEntity 8 | { 9 | public Guid JobId { get; set; } 10 | public string Name { get; set; } 11 | public string Reason { get; set; } 12 | public DateTime CreatedAt { get; set; } 13 | public string Data { get; set; } 14 | } 15 | 16 | private readonly StateEntity _state; 17 | 18 | public StateModel(StateEntity state) 19 | { 20 | _state = state; 21 | } 22 | 23 | public StateModel() 24 | { 25 | _state = new StateEntity 26 | { 27 | Id = Guid.NewGuid() 28 | }; 29 | } 30 | 31 | protected override BaseEntity Entity => _state; 32 | 33 | public DateTime CreatedAt 34 | { 35 | get => _state.CreatedAt; 36 | set => UpdateProperty(_state, () => x => x.CreatedAt, value); 37 | } 38 | 39 | public Guid JobId 40 | { 41 | get => _state.JobId; 42 | set => UpdateProperty(_state, () => x => x.JobId, value); 43 | } 44 | 45 | public JobModel Job => Single(JobId); 46 | 47 | public string Name 48 | { 49 | get => _state.Name; 50 | set => UpdateProperty(_state, () => x => x.Name, value); 51 | } 52 | 53 | public string Reason 54 | { 55 | get => _state.Reason; 56 | set => UpdateProperty(_state, () => x => x.Reason, value); 57 | } 58 | 59 | public string Data 60 | { 61 | get => _state.Data; 62 | set => UpdateProperty(_state, () => x => x.Data, value); 63 | } 64 | } 65 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/ObjectRepositoryJobQueueMonitoringApi.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using OutCode.EscapeTeams.ObjectRepository.Hangfire.Entities; 5 | 6 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire 7 | { 8 | internal class ObjectRepositoryJobQueueMonitoringApi 9 | { 10 | private readonly ObjectRepositoryBase _storage; 11 | 12 | public ObjectRepositoryJobQueueMonitoringApi(ObjectRepositoryBase storage) 13 | { 14 | _storage = storage ?? throw new ArgumentNullException(nameof(storage)); 15 | } 16 | 17 | public IEnumerable GetQueues() 18 | { 19 | return _storage.Set().Select(v => v.Queue).Distinct().ToList(); 20 | } 21 | 22 | public IEnumerable GetEnqueuedJobIds(string queue, int @from, int perPage) 23 | { 24 | return _storage.Set().Where(v => v.Queue == queue) 25 | .Skip(from).Take(perPage).Select(s => s.JobId).ToList(); 26 | } 27 | 28 | public IEnumerable GetFetchedJobIds(string queue, int @from, int perPage) 29 | { 30 | return _storage.Set() 31 | .Where(s => s.Queue == queue && s.FetchedAt.HasValue) 32 | .Skip(from) 33 | .Take(perPage) 34 | .Select(v => v.JobId) 35 | .ToList(); 36 | } 37 | 38 | public EnqueuedAndFetchedCountDto GetEnqueuedAndFetchedCount(string queue) 39 | { 40 | return _storage.Set().Where(v => v.Queue == queue).Aggregate( 41 | new EnqueuedAndFetchedCountDto(), (a, b) => 42 | { 43 | a.EnqueuedCount += b.FetchedAt == null ? 1 : 0; 44 | a.FetchedCount += b.FetchedAt != null ? 1 : 0; 45 | return a; 46 | }); 47 | } 48 | } 49 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Tests/FileTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Threading; 4 | using Microsoft.Extensions.Logging.Abstractions; 5 | using Microsoft.VisualStudio.TestTools.UnitTesting; 6 | using OutCode.EscapeTeams.ObjectRepository.File; 7 | 8 | namespace OutCode.EscapeTeams.ObjectRepository.Tests 9 | { 10 | [TestClass] 11 | public class FileTests : ProviderTestBase 12 | { 13 | private readonly String _filename = Path.GetTempFileName(); 14 | private bool _firstTime = true; 15 | 16 | protected override ObjectRepositoryBase CreateRepository() 17 | { 18 | var dbStorage = new FileStorage(_filename); 19 | var objectRepo = new FileTestObjectRepository(dbStorage); 20 | objectRepo.OnException += ex => Console.WriteLine(ex.ToString()); 21 | objectRepo.WaitForInitialize().GetAwaiter().GetResult(); 22 | 23 | if (_firstTime) 24 | { 25 | _firstTime = false; 26 | 27 | objectRepo.Add(_testModel); 28 | objectRepo.Add(_parentModel); 29 | objectRepo.Add(_childModel); 30 | } 31 | 32 | return objectRepo; 33 | } 34 | 35 | protected override IStorage GetStorage(ObjectRepositoryBase objectRepository) 36 | { 37 | return ((FileTestObjectRepository) objectRepository).FileStorage; 38 | } 39 | 40 | internal class FileTestObjectRepository : ObjectRepositoryBase 41 | { 42 | public FileTestObjectRepository(FileStorage dbLiteStorage) : base(dbLiteStorage, NullLogger.Instance) 43 | { 44 | FileStorage = dbLiteStorage; 45 | AddType((TestEntity x) => new TestModel(x)); 46 | AddType((ParentEntity x) => new ParentModel(x)); 47 | AddType((ChildEntity x) => new ChildModel(x)); 48 | Initialize(); 49 | } 50 | 51 | public FileStorage FileStorage { get; } 52 | } 53 | } 54 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/Entities/JobModel.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire.Entities 4 | { 5 | internal class JobModel : ModelBase 6 | { 7 | internal class JobEntity : BaseEntity 8 | { 9 | public Guid? StateId { get; set; } 10 | public string InvocationData { get; set; } 11 | public string Arguments { get; set; } 12 | public DateTime CreatedAt { get; set; } 13 | public DateTime? ExpireAt { get; set; } 14 | } 15 | 16 | private readonly JobEntity _job; 17 | 18 | public JobModel(JobEntity job) 19 | { 20 | _job = job; 21 | } 22 | 23 | public JobModel() 24 | { 25 | _job = new JobEntity 26 | { 27 | Id = Guid.NewGuid() 28 | }; 29 | } 30 | 31 | protected override BaseEntity Entity => _job; 32 | 33 | public string InvocationData 34 | { 35 | get => _job.InvocationData; 36 | set => UpdateProperty(_job, () => x => x.InvocationData, value); 37 | } 38 | 39 | public string Arguments 40 | { 41 | get => _job.Arguments; 42 | set => UpdateProperty(_job, () => x => x.Arguments, value); 43 | } 44 | 45 | public DateTime CreatedAt 46 | { 47 | get => _job.CreatedAt; 48 | set => UpdateProperty(_job, () => x => x.CreatedAt, value); 49 | } 50 | 51 | public Guid? StateId 52 | { 53 | get => _job.StateId; 54 | set => UpdateProperty(_job, () => x => x.StateId, value); 55 | } 56 | 57 | public DateTime? ExpireAt 58 | { 59 | get => _job.ExpireAt; 60 | set => UpdateProperty(_job, () => x => x.ExpireAt, value); 61 | } 62 | 63 | public StateModel State 64 | { 65 | get => Single(StateId); 66 | set => UpdateProperty(_job, () => x => x.StateId, value?.Id); 67 | } 68 | } 69 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository/ConcurrentList.cs: -------------------------------------------------------------------------------- 1 | using System.Collections; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | 6 | namespace OutCode.EscapeTeams.ObjectRepository 7 | { 8 | public class ConcurrentList : IEnumerable 9 | { 10 | private readonly ConcurrentDictionary _dictionary; 11 | 12 | public ConcurrentList(IEnumerable source) 13 | { 14 | _dictionary = new ConcurrentDictionary(source.Select(v => new KeyValuePair(v, v))); 15 | } 16 | 17 | public ConcurrentList() 18 | { 19 | _dictionary = new ConcurrentDictionary(); 20 | } 21 | 22 | public void Add(T item) => _dictionary.TryAdd(item, item); 23 | 24 | public void Remove(T item) 25 | { 26 | T unused; 27 | _dictionary.TryRemove(item, out unused); 28 | } 29 | 30 | public bool TryTake(out T result) 31 | { 32 | result = _dictionary.Keys.FirstOrDefault(); 33 | 34 | if (result != null) 35 | { 36 | Remove(result); 37 | return true; 38 | } 39 | return false; 40 | } 41 | 42 | public IEnumerator GetEnumerator() => new EnumeratorWrapper(_dictionary.GetEnumerator()); 43 | 44 | IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); 45 | 46 | 47 | private class EnumeratorWrapper : IEnumerator 48 | { 49 | private readonly IEnumerator> _source; 50 | 51 | public EnumeratorWrapper(IEnumerator> source) 52 | { 53 | _source = source; 54 | } 55 | 56 | public void Dispose() => _source.Dispose(); 57 | 58 | public bool MoveNext() => _source.MoveNext(); 59 | 60 | public void Reset() => _source.Reset(); 61 | 62 | public T Current => _source.Current.Key; 63 | 64 | object IEnumerator.Current => Current; 65 | } 66 | } 67 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Tests/TestModel.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace OutCode.EscapeTeams.ObjectRepository.Tests 5 | { 6 | public class ParentEntity : BaseEntity 7 | { 8 | public ParentEntity(Guid id) => Id = id; 9 | } 10 | 11 | public class ChildEntity : BaseEntity 12 | { 13 | public ChildEntity(Guid id) => Id = id; 14 | public Guid ParentId { get; set; } 15 | 16 | public String Property { get; set; } 17 | } 18 | 19 | public class ParentModel : ModelBase 20 | { 21 | public ParentModel(ParentEntity entity) 22 | { 23 | Entity = entity; 24 | } 25 | 26 | public Guid? NullableId => null; 27 | 28 | public IEnumerable Children => Multiple(() => x => x.ParentId); 29 | public IEnumerable OptionalChildren => Multiple(() => x => x.NullableTestId); 30 | 31 | protected internal override BaseEntity Entity { get; } 32 | } 33 | 34 | public class ChildModel : ModelBase 35 | { 36 | private readonly ChildEntity _myEntity; 37 | 38 | public ChildModel(ChildEntity entity) 39 | { 40 | _myEntity = entity; 41 | } 42 | 43 | public Guid? NullableTestId => null; 44 | 45 | public string Property 46 | { 47 | get => ((ChildEntity) Entity).Property; 48 | set => UpdateProperty(_myEntity, () => x => x.Property, value); 49 | } 50 | 51 | public Guid ParentId 52 | { 53 | get => ((ChildEntity) Entity).ParentId; 54 | set => UpdateProperty(_myEntity, () => x => x.ParentId, value); 55 | } 56 | 57 | public ParentModel Parent 58 | { 59 | get => Single(ParentId); 60 | set => UpdateProperty(_myEntity, () => x => x.ParentId, value.Id); 61 | } 62 | 63 | public ParentModel ParentOptional => Single(NullableTestId); 64 | 65 | protected internal override BaseEntity Entity => _myEntity; 66 | } 67 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository/ReflectionExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Reflection; 4 | using System.Reflection.Emit; 5 | 6 | namespace OutCode.EscapeTeams.ObjectRepository 7 | { 8 | internal static class ReflectionExtensions 9 | { 10 | private static readonly ConcurrentDictionary> GetterCache = new ConcurrentDictionary>(); 11 | private static readonly ConcurrentDictionary SetterCache = new ConcurrentDictionary(); 12 | 13 | public static Func GetOrCreateGetDelegate(this FieldInfo entityInfo) 14 | { 15 | return GetterCache.GetOrAdd(entityInfo, x => 16 | { 17 | string methodName = entityInfo.ReflectedType.FullName + ".get_" + x.Name; 18 | var getterMethod = new DynamicMethod(methodName, typeof(object), new[] {typeof(object)}, true); 19 | var gen = getterMethod.GetILGenerator(); 20 | gen.Emit(OpCodes.Ldarg_0); 21 | gen.Emit(OpCodes.Castclass, entityInfo.DeclaringType); 22 | gen.Emit(OpCodes.Ldfld, x); 23 | gen.Emit(OpCodes.Castclass, typeof(object)); 24 | gen.Emit(OpCodes.Ret); 25 | 26 | return (Func) getterMethod.CreateDelegate(typeof(Func)); 27 | }); 28 | } 29 | 30 | public static Delegate GetOrCreateGetter(this PropertyInfo propertyInfo) => propertyInfo.GetMethod.GetOrCreateMethodDelegate(typeof(Func<,>).MakeGenericType(propertyInfo.DeclaringType, propertyInfo.PropertyType)); 31 | public static Delegate GetOrCreateSetter(this PropertyInfo propertyInfo) => propertyInfo.SetMethod.GetOrCreateMethodDelegate(typeof(Action<,>).MakeGenericType(propertyInfo.DeclaringType, propertyInfo.PropertyType)); 32 | 33 | private static Delegate GetOrCreateMethodDelegate(this MethodInfo entityInfo, Type delegateType) => SetterCache.GetOrAdd(entityInfo, x => entityInfo.CreateDelegate(delegateType)); 34 | } 35 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Tests/LiteDbTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Linq; 4 | using System.Threading; 5 | using LiteDB; 6 | using Microsoft.Extensions.Logging.Abstractions; 7 | using Microsoft.VisualStudio.TestTools.UnitTesting; 8 | using OutCode.EscapeTeams.ObjectRepository.LiteDB; 9 | 10 | // ReSharper disable UnusedAutoPropertyAccessor.Global 11 | 12 | namespace OutCode.EscapeTeams.ObjectRepository.Tests 13 | { 14 | [TestClass] 15 | public class LiteDbTests : ProviderTestBase 16 | { 17 | private bool firstTime = true; 18 | private MemoryStream _memory = new MemoryStream(); 19 | 20 | protected override ObjectRepositoryBase CreateRepository() 21 | { 22 | var db = new LiteDatabase(_memory); 23 | var dbStorage = new LiteDbStorage(db); 24 | var objectRepo = new LiteDbTestObjectRepository(dbStorage); 25 | objectRepo.OnException += ex => Console.WriteLine(ex.ToString()); 26 | objectRepo.WaitForInitialize().GetAwaiter().GetResult(); 27 | 28 | if (firstTime) 29 | { 30 | firstTime = false; 31 | objectRepo.Add(_testModel); 32 | objectRepo.Add(_parentModel); 33 | objectRepo.Add(_childModel); 34 | GetStorage(objectRepo).SaveChanges().GetAwaiter().GetResult(); 35 | } 36 | 37 | return objectRepo; 38 | } 39 | 40 | protected override IStorage GetStorage(ObjectRepositoryBase objectRepository) => ((LiteDbTestObjectRepository) objectRepository).LiteStorage; 41 | 42 | internal class LiteDbTestObjectRepository : ObjectRepositoryBase 43 | { 44 | public LiteDbTestObjectRepository(LiteDbStorage dbLiteStorage) : base(dbLiteStorage, NullLogger.Instance) 45 | { 46 | LiteStorage = dbLiteStorage; 47 | IsReadOnly = true; 48 | AddType((TestEntity x) => new TestModel(x)); 49 | AddType((ParentEntity x) => new ParentModel(x)); 50 | AddType((ChildEntity x) => new ChildModel(x)); 51 | Initialize(); 52 | } 53 | 54 | public LiteDbStorage LiteStorage { get; } 55 | } 56 | } 57 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/ExpirationManager.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Threading; 4 | using Hangfire.Common; 5 | using Hangfire.Logging; 6 | using Hangfire.Server; 7 | using OutCode.EscapeTeams.ObjectRepository.Hangfire.Entities; 8 | #pragma warning disable 618 9 | 10 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire 11 | { 12 | internal class ExpirationManager : IServerComponent 13 | { 14 | private static readonly ILog Logger = LogProvider.For(); 15 | 16 | private readonly ObjectRepositoryStorage _storage; 17 | 18 | public ExpirationManager(ObjectRepositoryStorage storage) 19 | { 20 | _storage = storage ?? throw new ArgumentNullException(nameof(storage)); 21 | } 22 | 23 | public void Execute(CancellationToken cancellationToken) 24 | { 25 | Logger.Debug($"Removing outdated records..."); 26 | 27 | _storage.ObjectRepository.Remove(v => v.ExpireAt < DateTime.UtcNow); 28 | _storage.ObjectRepository.Remove(v => _storage.ObjectRepository.Set().Find(v.JobId) == null); 29 | _storage.ObjectRepository.Remove(v => _storage.ObjectRepository.Set().Find(v.JobId) == null); 30 | 31 | _storage.ObjectRepository.Remove(v => 32 | _storage.ObjectRepository.Set().Find(v.JobId) == null); 33 | 34 | _storage.ObjectRepository.Remove(s=>s.ExpireAt < DateTime.UtcNow); 35 | _storage.ObjectRepository.Remove(s => s.ExpireAt < DateTime.UtcNow); 36 | _storage.ObjectRepository.Remove(s => s.ExpireAt < DateTime.UtcNow); 37 | _storage.ObjectRepository.Remove(s => s.ExpireAt < DateTime.UtcNow); 38 | 39 | // Hangfire does clever job on storing retries in totally strange way. 40 | _storage.ObjectRepository.Remove(s => 41 | s.Key == "retries" && Guid.TryParse(s.Value, out var jobId) && 42 | _storage.ObjectRepository.Set().Find(jobId) == null); 43 | 44 | cancellationToken.WaitHandle.WaitOne(TimeSpan.FromMinutes(1)); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Tests/AzureStorageTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading; 3 | using Microsoft.Extensions.Logging.Abstractions; 4 | using Microsoft.VisualStudio.TestTools.UnitTesting; 5 | using Microsoft.WindowsAzure.Storage; 6 | using OutCode.EscapeTeams.ObjectRepository.AzureTableStorage; 7 | 8 | namespace OutCode.EscapeTeams.ObjectRepository.Tests 9 | { 10 | [TestClass, Ignore] 11 | public class AzureStorageTests : ProviderTestBase 12 | { 13 | protected override ObjectRepositoryBase CreateRepository() 14 | { 15 | var account = CloudStorageAccount.DevelopmentStorageAccount; 16 | var client = account.CreateCloudTableClient(); 17 | var storage = new AzureTableContext(client); 18 | 19 | client.GetTableReference("ChildEntity").DeleteIfExistsAsync().GetAwaiter().GetResult(); 20 | client.GetTableReference("ParentEntity").DeleteIfExistsAsync().GetAwaiter().GetResult(); 21 | client.GetTableReference("TestEntity").DeleteIfExistsAsync().GetAwaiter().GetResult(); 22 | 23 | var objectRepo = new AzureObjectRepository(storage); 24 | 25 | objectRepo.OnException += ex => Console.WriteLine(ex.ToString()); 26 | objectRepo.WaitForInitialize().GetAwaiter().GetResult(); 27 | 28 | objectRepo.Add(_testModel); 29 | objectRepo.Add(_parentModel); 30 | objectRepo.Add(_childModel); 31 | 32 | return objectRepo; 33 | } 34 | 35 | protected override IStorage GetStorage(ObjectRepositoryBase objectRepository) => ((AzureObjectRepository) objectRepository).AzureTableContext; 36 | 37 | internal class AzureObjectRepository : ObjectRepositoryBase 38 | { 39 | public AzureObjectRepository(AzureTableContext dbAzureTableContext) : base(dbAzureTableContext, NullLogger.Instance) 40 | { 41 | IsReadOnly = true; 42 | AzureTableContext = dbAzureTableContext; 43 | AddType((TestEntity x) => new TestModel(x)); 44 | AddType((ParentEntity x) => new ParentModel(x)); 45 | AddType((ChildEntity x) => new ChildModel(x)); 46 | Initialize(); 47 | } 48 | 49 | public AzureTableContext AzureTableContext { get; } 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /src/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo $(bash --version 2>&1 | head -n 1) 4 | 5 | #CUSTOMPARAM=0 6 | BUILD_ARGUMENTS=() 7 | for i in "$@"; do 8 | case $(echo $1 | awk '{print tolower($0)}') in 9 | # -custom-param) CUSTOMPARAM=1;; 10 | *) BUILD_ARGUMENTS+=("$1") ;; 11 | esac 12 | shift 13 | done 14 | 15 | set -eo pipefail 16 | SCRIPT_DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd) 17 | 18 | ########################################################################### 19 | # CONFIGURATION 20 | ########################################################################### 21 | 22 | BUILD_PROJECT_FILE="$SCRIPT_DIR/..\build/_build.csproj" 23 | TEMP_DIRECTORY="$SCRIPT_DIR/../.tmp" 24 | 25 | DOTNET_GLOBAL_FILE="$SCRIPT_DIR/../global.json" 26 | DOTNET_INSTALL_URL="https://raw.githubusercontent.com/dotnet/cli/master/scripts/obtain/dotnet-install.sh" 27 | DOTNET_RELEASES_URL="https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json" 28 | 29 | export DOTNET_CLI_TELEMETRY_OPTOUT=1 30 | export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 31 | export NUGET_XMLDOC_MODE="skip" 32 | 33 | ########################################################################### 34 | # EXECUTION 35 | ########################################################################### 36 | 37 | function FirstJsonValue { 38 | perl -nle 'print $1 if m{"'$1'": "([^"\-]+)",?}' <<< ${@:2} 39 | } 40 | 41 | # If global.json exists, load expected version 42 | if [ -f "$DOTNET_GLOBAL_FILE" ]; then 43 | DOTNET_VERSION=$(FirstJsonValue "version" $(cat "$DOTNET_GLOBAL_FILE")) 44 | fi 45 | 46 | # If dotnet is installed locally, and expected version is not set or installation matches the expected version 47 | if [[ -x "$(command -v dotnet)" && (-z ${DOTNET_VERSION+x} || $(dotnet --version) == "$DOTNET_VERSION") ]]; then 48 | export DOTNET_EXE="$(command -v dotnet)" 49 | else 50 | DOTNET_DIRECTORY="$TEMP_DIRECTORY/dotnet-unix" 51 | export DOTNET_EXE="$DOTNET_DIRECTORY/dotnet" 52 | 53 | # If expected version is not set, get latest version 54 | if [ -z ${DOTNET_VERSION+x} ]; then 55 | DOTNET_VERSION=$(FirstJsonValue "version-sdk" $(curl -s "$DOTNET_RELEASES_URL")) 56 | fi 57 | 58 | # Download and execute install script 59 | DOTNET_INSTALL_FILE="$TEMP_DIRECTORY/dotnet-install.sh" 60 | mkdir -p "$TEMP_DIRECTORY" 61 | curl -Lsfo "$DOTNET_INSTALL_FILE" "$DOTNET_INSTALL_URL" 62 | chmod +x "$DOTNET_INSTALL_FILE" 63 | "$DOTNET_INSTALL_FILE" --install-dir "$DOTNET_DIRECTORY" --version "$DOTNET_VERSION" --no-path 64 | fi 65 | 66 | echo "Microsoft (R) .NET Core SDK version $("$DOTNET_EXE" --version)" 67 | 68 | "$DOTNET_EXE" run --project "$BUILD_PROJECT_FILE" -- ${BUILD_ARGUMENTS[@]} 69 | -------------------------------------------------------------------------------- /src/build.ps1: -------------------------------------------------------------------------------- 1 | [CmdletBinding()] 2 | Param( 3 | #[switch]$CustomParam, 4 | [Parameter(Position=0,Mandatory=$false,ValueFromRemainingArguments=$true)] 5 | [string[]]$BuildArguments 6 | ) 7 | 8 | Write-Output "Windows PowerShell $($Host.Version)" 9 | 10 | Set-StrictMode -Version 2.0; $ErrorActionPreference = "Stop"; $ConfirmPreference = "None"; trap { $host.SetShouldExit(1) } 11 | $PSScriptRoot = Split-Path $MyInvocation.MyCommand.Path -Parent 12 | 13 | ########################################################################### 14 | # CONFIGURATION 15 | ########################################################################### 16 | 17 | $BuildProjectFile = "$PSScriptRoot\..\build\_build.csproj" 18 | $TempDirectory = "$PSScriptRoot\..\.tmp" 19 | 20 | $DotNetGlobalFile = "$PSScriptRoot\..\global.json" 21 | $DotNetInstallUrl = "https://raw.githubusercontent.com/dotnet/cli/master/scripts/obtain/dotnet-install.ps1" 22 | $DotNetReleasesUrl = "https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json" 23 | 24 | $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = 1 25 | $env:DOTNET_CLI_TELEMETRY_OPTOUT = 1 26 | $env:NUGET_XMLDOC_MODE = "skip" 27 | 28 | ########################################################################### 29 | # EXECUTION 30 | ########################################################################### 31 | 32 | function ExecSafe([scriptblock] $cmd) { 33 | & $cmd 34 | if ($LASTEXITCODE) { exit $LASTEXITCODE } 35 | } 36 | 37 | # If global.json exists, load expected version 38 | if (Test-Path $DotNetGlobalFile) { 39 | $DotNetVersion = $(Get-Content $DotNetGlobalFile | Out-String | ConvertFrom-Json).sdk.version 40 | } 41 | 42 | # If dotnet is installed locally, and expected version is not set or installation matches the expected version 43 | if ((Get-Command "dotnet" -ErrorAction SilentlyContinue) -ne $null -and ` 44 | (!(Test-Path variable:DotNetVersion) -or $(& dotnet --version) -eq $DotNetVersion)) { 45 | $env:DOTNET_EXE = (Get-Command "dotnet").Path 46 | } 47 | else { 48 | $DotNetDirectory = "$TempDirectory\dotnet-win" 49 | $env:DOTNET_EXE = "$DotNetDirectory\dotnet.exe" 50 | 51 | # If expected version is not set, get latest version 52 | if (!(Test-Path variable:DotNetVersion)) { 53 | $DotNetVersion = $(Invoke-WebRequest -UseBasicParsing $DotNetReleasesUrl | ConvertFrom-Json)[0]."version-sdk" 54 | } 55 | 56 | # Download and execute install script 57 | $DotNetInstallFile = "$TempDirectory\dotnet-install.ps1" 58 | md -force $TempDirectory > $null 59 | (New-Object System.Net.WebClient).DownloadFile($DotNetInstallUrl, $DotNetInstallFile) 60 | ExecSafe { & $DotNetInstallFile -InstallDir $DotNetDirectory -Version $DotNetVersion -NoPath } 61 | } 62 | 63 | Write-Output "Microsoft (R) .NET Core SDK version $(& $env:DOTNET_EXE --version)" 64 | 65 | ExecSafe { & $env:DOTNET_EXE run --project $BuildProjectFile -- $BuildArguments } 66 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.File/FileStorage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.IO; 5 | using System.Linq; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | using Newtonsoft.Json; 9 | 10 | namespace OutCode.EscapeTeams.ObjectRepository.File 11 | { 12 | public class FileStorage : IStorage, IDisposable 13 | { 14 | private readonly ConcurrentDictionary> _store; 15 | private readonly string _filename; 16 | private Timer _saveTimer; 17 | private bool _isDirty; 18 | 19 | public FileStorage(String filename) 20 | { 21 | _filename = filename; 22 | if (System.IO.File.Exists(_filename) && new FileInfo(_filename).Length > 0) 23 | { 24 | var text = System.IO.File.ReadAllText(_filename); 25 | 26 | var baseEntities = JsonConvert.DeserializeObject>>(text, 27 | new JsonSerializerSettings 28 | { 29 | TypeNameHandling = TypeNameHandling.All 30 | }); 31 | 32 | _store = baseEntities; 33 | } 34 | else 35 | { 36 | _store = new ConcurrentDictionary>(); 37 | } 38 | } 39 | 40 | public Task SaveChanges() 41 | { 42 | if (!_isDirty) 43 | return Task.CompletedTask; 44 | 45 | var contents = JsonConvert.SerializeObject(_store, Formatting.Indented, 46 | new JsonSerializerSettings 47 | { 48 | TypeNameHandling = TypeNameHandling.All 49 | }); 50 | System.IO.File.WriteAllText(_filename, contents); 51 | _isDirty = false; 52 | 53 | return Task.CompletedTask; 54 | } 55 | 56 | public Task> GetAll() => 57 | Task.FromResult((IEnumerable)_store.GetOrAdd(typeof(T), x => new ConcurrentList()).Cast().ToList()); 58 | 59 | public void Track(ObjectRepositoryBase objectRepository, bool isReadonly) 60 | { 61 | if (!isReadonly) 62 | { 63 | _saveTimer = new Timer(_ => SaveChanges(), null, 0, 5000); 64 | } 65 | 66 | objectRepository.ModelChanged += (change) => 67 | { 68 | _isDirty = true; 69 | 70 | var itemsList = _store.GetOrAdd(change.Entity.GetType(), x => new ConcurrentList()); 71 | 72 | switch (change.ChangeType) 73 | { 74 | case ChangeType.Add: 75 | itemsList.Add(change.Entity); 76 | break; 77 | case ChangeType.Remove: 78 | itemsList.Remove(change.Entity); 79 | break; 80 | } 81 | }; 82 | } 83 | 84 | public event Action OnError = delegate { }; 85 | 86 | public void Dispose() => _saveTimer?.Dispose(); 87 | } 88 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository/TableDictionary`1.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Concurrent; 4 | using System.Collections.Generic; 5 | using System.Linq; 6 | using System.Linq.Expressions; 7 | 8 | namespace OutCode.EscapeTeams.ObjectRepository 9 | { 10 | public class TableDictionary : TableDictionary, ITableDictionary where T : ModelBase 11 | { 12 | private readonly ObjectRepositoryBase _owner; 13 | private readonly ConcurrentDictionary> _columnsForIndex; 14 | private readonly ConcurrentDictionary> _indexes; 15 | private readonly ConcurrentDictionary _dictionary; 16 | 17 | public TableDictionary(ObjectRepositoryBase owner, IEnumerable source):base(owner) 18 | { 19 | _owner = owner; 20 | _dictionary = new ConcurrentDictionary(source.Select(v => new KeyValuePair(v.Entity, v))); 21 | 22 | _columnsForIndex = new ConcurrentDictionary>(); 23 | 24 | _indexes = new ConcurrentDictionary>(); 25 | 26 | AddIndex(() => x => x.Id); 27 | } 28 | 29 | public void AddIndex(Func>> index) 30 | { 31 | var key = GetPropertyName(index); 32 | var func = index().Compile(); 33 | 34 | _columnsForIndex.TryAdd(key, func); 35 | 36 | _owner.ModelChanged += change => 37 | { 38 | if (change.Source is T model && change.PropertyName == key) 39 | { 40 | _indexes[key].TryRemove(change.OldValue, out var _); 41 | _indexes[key].TryAdd(change.NewValue, model); 42 | } 43 | }; 44 | 45 | var dic = new ConcurrentDictionary(_dictionary.Select(v => new KeyValuePair(func(v.Value), v.Value))); 46 | _indexes.TryAdd(key, dic); 47 | } 48 | 49 | public T Find(Func>> index, object value) 50 | { 51 | if (_indexes[GetPropertyName(index)].TryGetValue(value, out T result)) 52 | { 53 | return result; 54 | } 55 | 56 | return default; 57 | } 58 | 59 | public T Find(Guid id) => Find(() => x => x.Id, id); 60 | 61 | IEnumerator IEnumerable.GetEnumerator() => _dictionary.Values.GetEnumerator(); 62 | 63 | public override IEnumerator GetEnumerator() => _dictionary.Values.GetEnumerator(); 64 | 65 | public void Add(T instance) 66 | { 67 | instance.SetOwner(_owner); 68 | _dictionary.TryAdd(instance.Entity, instance); 69 | foreach (var index in _columnsForIndex) 70 | { 71 | _indexes[index.Key].TryAdd(index.Value(instance), instance); 72 | } 73 | } 74 | 75 | public void Remove(T itemEntity) 76 | { 77 | _dictionary.TryRemove(itemEntity.Entity, out _); 78 | foreach (var index in _columnsForIndex) 79 | { 80 | _indexes[index.Key].TryRemove(index.Value(itemEntity), out _); 81 | } 82 | } 83 | } 84 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository/ModelExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Reflection; 6 | using System.Security.Cryptography; 7 | using System.Text; 8 | 9 | namespace OutCode.EscapeTeams.ObjectRepository 10 | { 11 | public static class ModelExtensions 12 | { 13 | public static string GetPropertiesAsRawData(this object obj) 14 | { 15 | if (obj == null) 16 | { 17 | return ""; 18 | } 19 | 20 | if (obj.GetType().GetTypeInfo().IsPrimitive || obj is string || obj is Enum) 21 | { 22 | return obj.ToString(); 23 | } 24 | 25 | var sb = new StringBuilder(); 26 | 27 | foreach (var item in obj.GetType().GetProperties().Where(v => v.PropertyType.GetTypeInfo().IsPrimitive || v.PropertyType == typeof(string) || v.PropertyType.IsEnum)) 28 | sb.Append("
" + item.Name + ": " + (item.GetValue(obj) ?? "")); 29 | 30 | foreach (var item in obj.GetType().GetMethods(BindingFlags.Public | BindingFlags.Instance)) 31 | { 32 | sb.Append("
Method " + item.Name + "(" + 33 | string.Join(", ", item.GetParameters().Select(v => v.ParameterType.FullName + " " + v.Name)) + 34 | ")"); 35 | } 36 | 37 | return sb.ToString(); 38 | } 39 | 40 | public static TValue GetOrDefault(this ConcurrentDictionary dictionary, TKey key) 41 | { 42 | TValue value; 43 | dictionary.TryGetValue(key, out value); 44 | return value; 45 | } 46 | 47 | public static TableDictionary ToConcurrentTable(this IEnumerable source, ObjectRepositoryBase owner) where T : ModelBase => new TableDictionary(owner, source); 48 | 49 | public static string ToMD5(this string str) 50 | { 51 | using (var md5 = MD5.Create()) 52 | { 53 | var srcBytes = Encoding.UTF8.GetBytes(str ?? string.Empty); 54 | var encodedBytes = md5.ComputeHash(srcBytes); 55 | 56 | var sb = new StringBuilder(); 57 | foreach (var b in encodedBytes) 58 | sb.Append(b.ToString("x2")); 59 | 60 | return sb.ToString(); 61 | } 62 | } 63 | 64 | public static string[] SplitLongString(this string str) 65 | { 66 | // Azure Table Storage cannot handle a string longer than 64 kb 67 | // Therefore in UTF-16 it is 32k symbols 68 | 69 | const int PieceSize = 32*1024; 70 | 71 | if (str == null) 72 | return new string[0]; 73 | 74 | var pieceCount = (int) Math.Ceiling((double) str.Length/PieceSize); 75 | var pieces = new string[pieceCount]; 76 | 77 | for (var i = 0; i < pieceCount; i++) 78 | { 79 | var start = i*PieceSize; 80 | var length = PieceSize; 81 | if (start + length > str.Length) 82 | length = str.Length - start; 83 | 84 | pieces[i] = str.Substring(start, length); 85 | } 86 | 87 | return pieces; 88 | } 89 | } 90 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire.Tests/Startup.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | using Hangfire; 7 | using Microsoft.AspNetCore.Builder; 8 | using Microsoft.AspNetCore.Hosting; 9 | using Microsoft.AspNetCore.Http; 10 | using Microsoft.Extensions.DependencyInjection; 11 | using Microsoft.Extensions.Logging; 12 | using Microsoft.Extensions.Logging.Abstractions; 13 | 14 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire.Tests 15 | { 16 | public class DummyObjectRepository : ObjectRepositoryBase 17 | { 18 | public DummyObjectRepository(ILogger logger): base(CreateStorage(), logger) 19 | { 20 | this.RegisterHangfireScheme(); 21 | Initialize(); 22 | } 23 | 24 | private static IStorage CreateStorage() => new DummyStorage(); 25 | 26 | private class DummyStorage : IStorage 27 | { 28 | private ConcurrentList items = new ConcurrentList(); 29 | 30 | public Task SaveChanges() => Task.CompletedTask; 31 | public Task> GetAll() => Task.FromResult>(items.OfType().ToList()); 32 | 33 | public void Track(ObjectRepositoryBase objectRepository, bool isReadonly) 34 | { 35 | objectRepository.ModelChanged += handler; 36 | } 37 | 38 | private void handler(ModelChangedEventArgs obj) 39 | { 40 | switch (obj.ChangeType) 41 | { 42 | case ChangeType.Add: 43 | items.Add(obj.Source); 44 | break; 45 | case ChangeType.Remove: 46 | items.Remove(obj.Source); 47 | break; 48 | } 49 | } 50 | 51 | public event Action OnError = delegate { }; 52 | } 53 | } 54 | 55 | public class Startup 56 | { 57 | // This method gets called by the runtime. Use this method to add services to the container. 58 | // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 59 | public void ConfigureServices(IServiceCollection services) 60 | { 61 | var objectRepository = new DummyObjectRepository(NullLogger.Instance); 62 | services.AddLogging(); 63 | services.AddHangfire(s => s.UseHangfireStorage(objectRepository).UseColouredConsoleLogProvider()); 64 | } 65 | 66 | // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. 67 | public void Configure(IApplicationBuilder app, IHostingEnvironment env) 68 | { 69 | if (env.IsDevelopment()) 70 | { 71 | app.UseDeveloperExceptionPage(); 72 | } 73 | 74 | app.UseHangfireServer(); 75 | app.UseHangfireDashboard(); 76 | RecurringJob.AddOrUpdate("testjob", () => TestMethod(1, "my param"), Cron.Daily); 77 | RecurringJob.AddOrUpdate("longjob", () => LongTestMethod(1, "my param"), Cron.Daily); 78 | RecurringJob.AddOrUpdate("failjob", () => FailMethod(2, "fail"), Cron.Daily); 79 | 80 | for (char a = 'a'; a <= 'z'; a++) 81 | { 82 | var a1 = a; 83 | var name = "TestSorting_" + a; 84 | RecurringJob.AddOrUpdate(name, () => TestMethod(a1, name), Cron.Yearly); 85 | } 86 | 87 | app.Run(async (context) => { await context.Response.WriteAsync("Hello World!"); }); 88 | } 89 | 90 | public void FailMethod(int i, string fail) 91 | { 92 | throw new Exception(fail); 93 | } 94 | 95 | public void TestMethod(int i, string myParam) 96 | { 97 | Thread.Sleep(5000); 98 | } 99 | 100 | public void LongTestMethod(int i, string myParam) 101 | { 102 | Thread.Sleep(60000); 103 | } 104 | 105 | } 106 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository/ModelBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.ComponentModel; 5 | using System.Linq.Expressions; 6 | using System.Reflection; 7 | using System.Runtime.CompilerServices; 8 | 9 | namespace OutCode.EscapeTeams.ObjectRepository 10 | { 11 | public abstract class ModelBase : INotifyPropertyChanged 12 | { 13 | /// 14 | /// Primary key of this object. 15 | /// 16 | public Guid Id => Entity.Id; 17 | 18 | /// 19 | /// Underlying database item associated with the model. 20 | /// 21 | protected internal abstract BaseEntity Entity { get; } 22 | 23 | protected ObjectRepositoryBase ObjectRepository { get; private set; } 24 | 25 | internal void SetOwner(ObjectRepositoryBase owner) 26 | { 27 | if (ObjectRepository != null) 28 | throw new InvalidOperationException("ObjectRepository already set!"); 29 | ObjectRepository = owner; 30 | } 31 | 32 | /// 33 | /// Returns a list of related entities by mathing the current object's ID to specified property. 34 | /// 35 | protected IEnumerable Multiple(Func>> propertyGetter, [CallerMemberName] string callingProperty = "") where T : ModelBase 36 | { 37 | var multiple = ObjectRepository.Set(GetType()).GetMultiple(propertyGetter); 38 | 39 | if (!multiple.ContainsKey(Id)) 40 | { 41 | multiple.TryAdd(Id, new ConcurrentList()); 42 | } 43 | return multiple[Id]; 44 | } 45 | 46 | /// 47 | /// Returns a single entity by matching the specified value to it's ID. 48 | /// 49 | protected T Single(Guid? id, [CallerMemberName] string callingProperty = "", [CallerFilePath] string file = "", [CallerLineNumber] int line = 0) where T : ModelBase 50 | { 51 | if (id == null) 52 | return null; 53 | 54 | var set = ObjectRepository.Set(); 55 | 56 | var value = set.Find(id.Value); 57 | 58 | if (value == null) 59 | { 60 | throw new ObjectRepositoryException(id, callingProperty, $"(file {file} at line {line})"); 61 | } 62 | 63 | return value; 64 | } 65 | 66 | public event Action PropertyChanging; 67 | 68 | protected void UpdateProperty(TEntity entity, Func>> expressionGetter, TValue newValue) 69 | { 70 | var c = PropertyUpdater.GetPropertyUpdater(expressionGetter); 71 | 72 | var oldValue = c.UpdateValue(entity, newValue); 73 | 74 | if (!Equals(oldValue, newValue)) 75 | { 76 | PropertyChanging?.Invoke(ModelChangedEventArgs.PropertyChange(this, c.Name, oldValue, newValue)); 77 | PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(c.Name)); 78 | } 79 | } 80 | 81 | internal class PropertyUpdater 82 | { 83 | internal static readonly ConcurrentDictionary>>, PropertyUpdater> Cache = new ConcurrentDictionary>>, PropertyUpdater>(); 84 | private readonly PropertyInfo _propertyInfo; 85 | 86 | public static PropertyUpdater GetPropertyUpdater(Func>> expressionGetter) => Cache.GetOrAdd(expressionGetter, x => new PropertyUpdater(x)); 87 | 88 | private PropertyUpdater(Func>> expressionGetter) 89 | { 90 | var propertyExpr = (MemberExpression) expressionGetter().Body; 91 | 92 | _propertyInfo = (PropertyInfo) propertyExpr.Member; 93 | } 94 | 95 | public string Name => _propertyInfo.Name; 96 | 97 | 98 | public TValue UpdateValue(TEntity entity, TValue newValue) 99 | { 100 | var oldValue = (TValue)_propertyInfo.GetOrCreateGetter().DynamicInvoke(entity); 101 | _propertyInfo.GetOrCreateSetter().DynamicInvoke(entity, newValue); 102 | return oldValue; 103 | } 104 | } 105 | 106 | public event PropertyChangedEventHandler PropertyChanged; 107 | } 108 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.27130.2010 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OutCode.EscapeTeams.ObjectRepository", "OutCode.EscapeTeams.ObjectRepository\OutCode.EscapeTeams.ObjectRepository.csproj", "{1611AEF8-0CD6-49C3-B545-18A78185EADC}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OutCode.EscapeTeams.ObjectRepository.AzureTableStorage", "OutCode.EscapeTeams.ObjectRepository.AzureTableStorage\OutCode.EscapeTeams.ObjectRepository.AzureTableStorage.csproj", "{C8AD63F7-56C5-4BEF-8B07-05770A4F8A17}" 9 | EndProject 10 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OutCode.EscapeTeams.ObjectRepository.Tests", "OutCode.EscapeTeams.ObjectRepository.Tests\OutCode.EscapeTeams.ObjectRepository.Tests.csproj", "{C4EB5579-3BFB-4770-86FA-A4031B4816AE}" 11 | EndProject 12 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OutCode.EscapeTeams.ObjectRepository.LiteDB", "OutCode.EscapeTeams.ObjectRepository.LiteDB\OutCode.EscapeTeams.ObjectRepository.LiteDB.csproj", "{398FF6EF-D0D9-4C38-A542-73A93952FF00}" 13 | EndProject 14 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "_build", "..\build\_build.csproj", "{0AADB1E7-813C-4016-BF30-65380EC26CE2}" 15 | EndProject 16 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OutCode.EscapeTeams.ObjectRepository.File", "OutCode.EscapeTeams.ObjectRepository.File\OutCode.EscapeTeams.ObjectRepository.File.csproj", "{AF5064E1-5FD9-440E-AA4E-700CAAD764B0}" 17 | EndProject 18 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OutCode.EscapeTeams.ObjectRepository.Hangfire", "OutCode.EscapeTeams.ObjectRepository.Hangfire\OutCode.EscapeTeams.ObjectRepository.Hangfire.csproj", "{2646E7BC-66FC-429C-B1F9-9817325C931D}" 19 | EndProject 20 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OutCode.EscapeTeams.ObjectRepository.Hangfire.Tests", "OutCode.EscapeTeams.ObjectRepository.Hangfire.Tests\OutCode.EscapeTeams.ObjectRepository.Hangfire.Tests.csproj", "{619E2D38-E517-412F-9E56-C780C67343D6}" 21 | EndProject 22 | Global 23 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 24 | Debug|Any CPU = Debug|Any CPU 25 | Release|Any CPU = Release|Any CPU 26 | EndGlobalSection 27 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 28 | {0AADB1E7-813C-4016-BF30-65380EC26CE2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 29 | {0AADB1E7-813C-4016-BF30-65380EC26CE2}.Release|Any CPU.ActiveCfg = Release|Any CPU 30 | {1611AEF8-0CD6-49C3-B545-18A78185EADC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 31 | {1611AEF8-0CD6-49C3-B545-18A78185EADC}.Debug|Any CPU.Build.0 = Debug|Any CPU 32 | {1611AEF8-0CD6-49C3-B545-18A78185EADC}.Release|Any CPU.ActiveCfg = Release|Any CPU 33 | {1611AEF8-0CD6-49C3-B545-18A78185EADC}.Release|Any CPU.Build.0 = Release|Any CPU 34 | {C8AD63F7-56C5-4BEF-8B07-05770A4F8A17}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 35 | {C8AD63F7-56C5-4BEF-8B07-05770A4F8A17}.Debug|Any CPU.Build.0 = Debug|Any CPU 36 | {C8AD63F7-56C5-4BEF-8B07-05770A4F8A17}.Release|Any CPU.ActiveCfg = Release|Any CPU 37 | {C8AD63F7-56C5-4BEF-8B07-05770A4F8A17}.Release|Any CPU.Build.0 = Release|Any CPU 38 | {C4EB5579-3BFB-4770-86FA-A4031B4816AE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 39 | {C4EB5579-3BFB-4770-86FA-A4031B4816AE}.Debug|Any CPU.Build.0 = Debug|Any CPU 40 | {C4EB5579-3BFB-4770-86FA-A4031B4816AE}.Release|Any CPU.ActiveCfg = Release|Any CPU 41 | {C4EB5579-3BFB-4770-86FA-A4031B4816AE}.Release|Any CPU.Build.0 = Release|Any CPU 42 | {398FF6EF-D0D9-4C38-A542-73A93952FF00}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 43 | {398FF6EF-D0D9-4C38-A542-73A93952FF00}.Debug|Any CPU.Build.0 = Debug|Any CPU 44 | {398FF6EF-D0D9-4C38-A542-73A93952FF00}.Release|Any CPU.ActiveCfg = Release|Any CPU 45 | {398FF6EF-D0D9-4C38-A542-73A93952FF00}.Release|Any CPU.Build.0 = Release|Any CPU 46 | {AF5064E1-5FD9-440E-AA4E-700CAAD764B0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 47 | {AF5064E1-5FD9-440E-AA4E-700CAAD764B0}.Debug|Any CPU.Build.0 = Debug|Any CPU 48 | {AF5064E1-5FD9-440E-AA4E-700CAAD764B0}.Release|Any CPU.ActiveCfg = Release|Any CPU 49 | {AF5064E1-5FD9-440E-AA4E-700CAAD764B0}.Release|Any CPU.Build.0 = Release|Any CPU 50 | {2646E7BC-66FC-429C-B1F9-9817325C931D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 51 | {2646E7BC-66FC-429C-B1F9-9817325C931D}.Debug|Any CPU.Build.0 = Debug|Any CPU 52 | {2646E7BC-66FC-429C-B1F9-9817325C931D}.Release|Any CPU.ActiveCfg = Release|Any CPU 53 | {2646E7BC-66FC-429C-B1F9-9817325C931D}.Release|Any CPU.Build.0 = Release|Any CPU 54 | {619E2D38-E517-412F-9E56-C780C67343D6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 55 | {619E2D38-E517-412F-9E56-C780C67343D6}.Debug|Any CPU.Build.0 = Debug|Any CPU 56 | {619E2D38-E517-412F-9E56-C780C67343D6}.Release|Any CPU.ActiveCfg = Release|Any CPU 57 | {619E2D38-E517-412F-9E56-C780C67343D6}.Release|Any CPU.Build.0 = Release|Any CPU 58 | EndGlobalSection 59 | GlobalSection(SolutionProperties) = preSolution 60 | HideSolutionNode = FALSE 61 | EndGlobalSection 62 | GlobalSection(ExtensibilityGlobals) = postSolution 63 | SolutionGuid = {88488B02-132A-495C-A025-489FC09B4960} 64 | EndGlobalSection 65 | EndGlobal 66 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository/TableDictionary.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Concurrent; 4 | using System.Collections.Generic; 5 | using System.Linq; 6 | using System.Linq.Expressions; 7 | 8 | namespace OutCode.EscapeTeams.ObjectRepository 9 | { 10 | public abstract class TableDictionary : IEnumerable 11 | { 12 | private readonly ObjectRepositoryBase _owner; 13 | 14 | // Foreign types which are using this table. Type -> PropertyName -> Getter + ConcurrentDictionary> 15 | private readonly ConcurrentDictionary>> _foreignIndexes; 16 | 17 | protected TableDictionary(ObjectRepositoryBase owner) 18 | { 19 | _owner = owner; 20 | _foreignIndexes = new ConcurrentDictionary>>(); 21 | } 22 | 23 | public ConcurrentDictionary> GetMultiple(Func>> foreignKey) 24 | where TForeign : ModelBase 25 | { 26 | var type = typeof(TForeign); 27 | 28 | var guidMember = GetPropertyName(foreignKey); 29 | 30 | ConcurrentDictionary> t; 31 | 32 | if (!_foreignIndexes.TryGetValue(type, out t)) 33 | { 34 | lock (this) 35 | { 36 | if (!_foreignIndexes.TryGetValue(type, out t)) 37 | { 38 | t = new ConcurrentDictionary>(); 39 | _foreignIndexes.TryAdd(type, t); 40 | } 41 | } 42 | } 43 | 44 | if (!t.TryGetValue(guidMember, out var value)) 45 | { 46 | lock (this) 47 | { 48 | if (!t.TryGetValue(guidMember, out value)) 49 | { 50 | var func = foreignKey().Compile(); 51 | var other = 52 | new ConcurrentDictionary>( 53 | _owner.Set() 54 | .GroupBy(func) 55 | .Where(v => v.Key.HasValue) 56 | .Select(v => new KeyValuePair>(v.Key.Value, new ConcurrentList(v)))); 57 | 58 | _owner.ModelChanged += (change) => 59 | { 60 | if (type != change.Source.GetType()) 61 | return; 62 | 63 | Guid? removeKey = null; 64 | Guid? addKey = null; 65 | 66 | 67 | switch (change.ChangeType) 68 | { 69 | case ChangeType.Add: 70 | addKey = func((TForeign)change.Source); 71 | break; 72 | case ChangeType.Remove: 73 | removeKey = func((TForeign)change.Source); 74 | break; 75 | case ChangeType.Update when change.PropertyName == guidMember: 76 | removeKey = (Guid?) change.OldValue; 77 | addKey = (Guid?) change.NewValue; 78 | break; 79 | } 80 | 81 | if (removeKey != null) 82 | { 83 | other[removeKey.Value].Remove((TForeign) change.Source); 84 | } 85 | 86 | if (addKey != null) 87 | { 88 | if (!other.TryGetValue(addKey.Value, out var list)) 89 | { 90 | lock (this) 91 | { 92 | if (!other.TryGetValue(addKey.Value, out list)) 93 | { 94 | list = new ConcurrentList(); 95 | other.TryAdd(addKey.Value, list); 96 | } 97 | } 98 | } 99 | 100 | other[addKey.Value].Add((TForeign) change.Source); 101 | 102 | } 103 | }; 104 | 105 | value = Tuple.Create((Delegate) func, (object) other); 106 | t.TryAdd(guidMember, value); 107 | } 108 | } 109 | } 110 | 111 | return (ConcurrentDictionary>) value.Item2; 112 | } 113 | 114 | static readonly ConcurrentDictionary PropertyNameCache = new ConcurrentDictionary(); 115 | 116 | protected static string GetPropertyName(Func>> index) 117 | { 118 | return PropertyNameCache.GetOrAdd(index, func => 119 | { 120 | var expression = index().Body; 121 | 122 | if (expression is UnaryExpression unary) 123 | { 124 | expression = unary.Operand; 125 | } 126 | 127 | return ((MemberExpression) expression).Member.Name; 128 | }); 129 | } 130 | 131 | public abstract IEnumerator GetEnumerator(); 132 | } 133 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.LiteDB/LiteDbStorage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | using LiteDB; 8 | using Newtonsoft.Json; 9 | 10 | namespace OutCode.EscapeTeams.ObjectRepository.LiteDB 11 | { 12 | public class LiteDbStorage : IStorage, IDisposable 13 | { 14 | private readonly LiteDatabase _database; 15 | private Timer _saveTimer; 16 | private int _saveInProgress; 17 | 18 | private readonly ConcurrentDictionary> _entitiesToAdd = 19 | new ConcurrentDictionary>(); 20 | 21 | private readonly ConcurrentDictionary> _entitiesToRemove = 22 | new ConcurrentDictionary>(); 23 | 24 | private readonly ConcurrentDictionary> _entitiesToUpdate = 25 | new ConcurrentDictionary>(); 26 | 27 | private readonly BsonMapper _mapper; 28 | 29 | public LiteDbStorage(LiteDatabase database) 30 | { 31 | _database = database; 32 | _mapper = new BsonMapper(); 33 | } 34 | 35 | public Task SaveChanges() 36 | { 37 | if (Interlocked.CompareExchange(ref _saveInProgress, 1, 0) == 0) 38 | { 39 | try 40 | { 41 | foreach (var remove in _entitiesToRemove.ToList()) 42 | { 43 | foreach (var removeItem in remove.Value.ToList()) 44 | { 45 | if (_entitiesToAdd.TryGetValue(remove.Key, out var toAddList)) 46 | { 47 | if (toAddList?.Contains(removeItem) == true) 48 | { 49 | remove.Value.Remove(removeItem); 50 | toAddList.Remove(removeItem); 51 | } 52 | } 53 | 54 | if (_entitiesToUpdate.TryGetValue(remove.Key, out var removeList)) 55 | { 56 | removeList.Remove(removeItem); 57 | } 58 | } 59 | } 60 | 61 | ProcessAction(_entitiesToAdd, (x, y) => y.Upsert(_mapper.ToDocument(x))); 62 | ProcessAction(_entitiesToUpdate, (x, y) => y.Update(_mapper.ToDocument(x))); 63 | ProcessAction(_entitiesToRemove, (x, y) => 64 | { 65 | if (!y.Delete(x.Id)) 66 | { 67 | throw new InvalidOperationException( 68 | "Failed to remove entity from storage " + _mapper.ToDocument(x)); 69 | } 70 | }); 71 | _database.Checkpoint(); 72 | } 73 | finally 74 | { 75 | _saveInProgress = 0; 76 | } 77 | } 78 | 79 | return Task.CompletedTask; 80 | } 81 | 82 | public string ExportStream() 83 | { 84 | var result = new 85 | { 86 | add = _entitiesToAdd.ToDictionary(v => v.Key, v => v.Value.ToList()), 87 | remove = _entitiesToRemove.ToDictionary(v => v.Key, v => v.Value.ToList()), 88 | mod = _entitiesToUpdate.ToDictionary(v => v.Key, v => v.Value.ToList()), 89 | }; 90 | 91 | return JsonConvert.SerializeObject(result); 92 | } 93 | 94 | private void ProcessAction(ConcurrentDictionary> dictionary, 95 | Action> entity) 96 | { 97 | foreach (var addPair in dictionary) 98 | { 99 | var collection = _database.GetCollection(addPair.Key.Name); 100 | var itemsToAction = addPair.Value.ToList(); 101 | foreach (var item in itemsToAction) 102 | { 103 | try 104 | { 105 | addPair.Value.Remove(item); 106 | entity(item, collection); 107 | } 108 | catch (Exception ex) 109 | { 110 | OnError(ex); 111 | addPair.Value.Add(item); 112 | } 113 | } 114 | } 115 | } 116 | 117 | public Task> GetAll() => 118 | Task.FromResult((IEnumerable) _database.GetCollection().FindAll().ToList()); 119 | 120 | public void Track(ObjectRepositoryBase objectRepository, bool isReadonly) 121 | { 122 | if (!isReadonly) 123 | { 124 | _saveTimer = new Timer(_ => SaveChanges(), null, 0, 5000); 125 | } 126 | 127 | objectRepository.ModelChanged += (change) => 128 | { 129 | switch (change.ChangeType) 130 | { 131 | case ChangeType.Update: 132 | AddEntityToLookup((BaseEntity) change.Entity, _entitiesToUpdate); 133 | break; 134 | case ChangeType.Add: 135 | AddEntityToLookup((BaseEntity) change.Entity, _entitiesToAdd); 136 | break; 137 | case ChangeType.Remove: 138 | AddEntityToLookup((BaseEntity) change.Entity, _entitiesToRemove); 139 | break; 140 | } 141 | }; 142 | } 143 | 144 | public event Action OnError = delegate { }; 145 | 146 | public void Dispose() => _saveTimer?.Dispose(); 147 | 148 | /// 149 | /// Registers an entity for one of the operations. 150 | /// 151 | private void AddEntityToLookup(T entity, ConcurrentDictionary> lookup) 152 | where T : BaseEntity 153 | { 154 | var set = lookup.GetOrAdd(entity.GetType(), type => new ConcurrentList()); 155 | set.Add(entity); 156 | } 157 | } 158 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository/ObjectRepositoryBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Diagnostics; 5 | using System.Linq; 6 | using System.Reflection; 7 | using System.Threading.Tasks; 8 | using Microsoft.Extensions.Logging; 9 | 10 | namespace OutCode.EscapeTeams.ObjectRepository 11 | { 12 | public class ObjectRepositoryBase 13 | { 14 | private readonly ILogger _logger; 15 | private readonly List _tasks = new List(); 16 | private bool _typeAdded; 17 | 18 | internal bool _setsAreReady; 19 | 20 | protected readonly ConcurrentDictionary> _sets = new ConcurrentDictionary>(); 21 | private bool _isReadOnly; 22 | 23 | private readonly TaskCompletionSource _taskCompletionSource = new TaskCompletionSource(); 24 | 25 | public event Action ModelChanged = delegate {}; 26 | 27 | public ObjectRepositoryBase(IStorage storage, ILogger logger) 28 | { 29 | Storage = storage; 30 | _logger = logger; 31 | Storage.OnError += RaiseOnException; 32 | } 33 | 34 | protected IStorage Storage { get; } 35 | 36 | protected bool ThrowIfBadItems { get; set; } 37 | 38 | public bool IsLoading => !_taskCompletionSource.Task.IsCompleted; 39 | 40 | public bool IsReadOnly 41 | { 42 | get => _isReadOnly; 43 | protected set 44 | { 45 | if (_typeAdded) 46 | throw new NotSupportedException("Is readonly should be set before adding types!"); 47 | _isReadOnly = value; 48 | } 49 | } 50 | 51 | public void AddType(Func converter) 52 | where TModel:ModelBase 53 | { 54 | if (!_typeAdded) 55 | { 56 | _typeAdded = true; 57 | Storage.Track(this, IsReadOnly); 58 | } 59 | 60 | _tasks.Add(AddTypeAndLoad(()=>Storage.GetAll(), converter)); 61 | } 62 | 63 | private async Task AddTypeAndLoad(Func>> sourceFunc, Func converter) where TModel : ModelBase 64 | { 65 | var sw = new Stopwatch(); 66 | sw.Start(); 67 | 68 | var source = sourceFunc(); 69 | 70 | _logger.LogInformation($"Loading entities for {typeof(TStoreEntity).Name}..."); 71 | var value = await source; 72 | sw.Stop(); 73 | 74 | _sets.TryAdd(typeof(TModel), value.Select(converter).Select(v => 75 | { 76 | v.PropertyChanging += InstancePropertyChangingHandler; 77 | v.SetOwner(this); 78 | return v; 79 | }).ToList().ToConcurrentTable(this)); 80 | 81 | _logger.LogInformation($"Loaded entities for {typeof(TStoreEntity).Name} in {sw.Elapsed.TotalSeconds} sec..."); 82 | } 83 | 84 | public Task WaitForInitialize() => _taskCompletionSource.Task; 85 | 86 | protected void Initialize() 87 | { 88 | if (_tasks.Count == 0) 89 | throw new InvalidOperationException("No AddType was called before Initialize!"); 90 | 91 | Task.WaitAll(_tasks.ToArray()); 92 | 93 | _setsAreReady = true; 94 | 95 | Parallel.ForEach(_sets.Keys, key => 96 | { 97 | try 98 | { 99 | var sw = new Stopwatch(); 100 | 101 | var properties = 102 | key.GetTypeInfo() 103 | .GetProperties() 104 | .Select(v => v.GetMethod.CreateDelegate(typeof(Func<,>).MakeGenericType(key, v.PropertyType))) 105 | .ToList(); 106 | 107 | sw.Start(); 108 | 109 | foreach (var item in (dynamic) _sets[key]) 110 | { 111 | foreach (dynamic p in properties) 112 | { 113 | p(item); 114 | } 115 | } 116 | 117 | sw.Stop(); 118 | _logger.LogInformation( 119 | $"Warming up of type {key.Name} completed, took {sw.Elapsed.TotalSeconds} seconds."); 120 | } 121 | catch (Exception) 122 | { 123 | if (ThrowIfBadItems) 124 | { 125 | throw; 126 | } 127 | } 128 | }); 129 | 130 | _taskCompletionSource.SetResult(null); 131 | } 132 | 133 | public T Add(T instance) 134 | where T : ModelBase 135 | { 136 | ThrowIfLoading(); 137 | 138 | Set().Add(instance); 139 | 140 | instance.PropertyChanging += InstancePropertyChangingHandler; 141 | ModelChanged(ModelChangedEventArgs.Added(instance)); 142 | 143 | return instance; 144 | } 145 | 146 | private void InstancePropertyChangingHandler(ModelChangedEventArgs modelChangedEventArgs) => ModelChanged(modelChangedEventArgs); 147 | 148 | private void ThrowIfLoading() 149 | { 150 | if (IsLoading && !_setsAreReady) 151 | { 152 | var total = _tasks?.Count ?? 1; 153 | var finished = _tasks?.Count(s => s.IsCompleted) ?? 0; 154 | throw new LoadingInProgressException((double)finished / total); 155 | } 156 | } 157 | 158 | public void Remove(Func func) 159 | where T : ModelBase 160 | { 161 | ThrowIfLoading(); 162 | 163 | var badItems = Set().Where(func).ToList(); 164 | foreach (var item in badItems) 165 | { 166 | Remove(item); 167 | } 168 | } 169 | 170 | public void Remove(T item) 171 | where T : ModelBase 172 | { 173 | ThrowIfLoading(); 174 | 175 | Set().Remove(item); 176 | item.PropertyChanging -= InstancePropertyChangingHandler; 177 | ModelChanged(ModelChangedEventArgs.Removed(item)); 178 | } 179 | 180 | public void RemoveRange(IEnumerable item) 181 | where T : ModelBase 182 | { 183 | ThrowIfLoading(); 184 | foreach (var i in item) 185 | { 186 | Remove(i); 187 | } 188 | } 189 | 190 | public event Action OnException; 191 | 192 | protected void RaiseOnException(Exception ex) => OnException?.Invoke(ex); 193 | 194 | public TableDictionary Set() where T : ModelBase => (TableDictionary) Set(typeof(T)); 195 | 196 | public TableDictionary Set(Type t) 197 | { 198 | ThrowIfLoading(); 199 | if (_sets.TryGetValue(t, out var result)) 200 | { 201 | return result as TableDictionary; 202 | } 203 | 204 | throw new NotSupportedException("Failed to get ObjectRepository's set for type " + t?.FullName); 205 | } 206 | 207 | public async void SaveChanges() 208 | { 209 | if (IsLoading) 210 | { 211 | return; 212 | } 213 | 214 | try 215 | { 216 | await Storage.SaveChanges(); 217 | } 218 | catch (Exception ex) 219 | { 220 | RaiseOnException(ex); 221 | } 222 | } 223 | } 224 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Tests/ObjectRepositoryTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using Microsoft.VisualStudio.TestTools.UnitTesting; 4 | 5 | namespace OutCode.EscapeTeams.ObjectRepository.Tests 6 | { 7 | [TestClass] 8 | public class ObjectRepositoryTests 9 | { 10 | [TestMethod] 11 | public void CtorShouldNotThrowException() 12 | { 13 | // Given 14 | var instance = new TestObjectRepository(new TestStorage()); 15 | 16 | // When 17 | instance.WaitForInitialize().GetAwaiter().GetResult(); 18 | 19 | // Then 20 | // no exceptions 21 | } 22 | 23 | [TestMethod] 24 | public void TestThatRelationsWorks() 25 | { 26 | // Given 27 | var id = Guid.NewGuid(); 28 | var testStorage = new TestStorage 29 | { 30 | new ParentEntity(id), 31 | new ChildEntity(Guid.NewGuid()) {ParentId = id} 32 | }; 33 | 34 | var instance = new TestObjectRepository(testStorage); 35 | 36 | // When 37 | instance.WaitForInitialize().GetAwaiter().GetResult(); 38 | 39 | // Then 40 | // no exceptions 41 | 42 | var parentModel = instance.Set().Single(); 43 | var childModel = instance.Set().Single(); 44 | Assert.AreEqual(parentModel.Children.Single(), childModel); 45 | Assert.AreEqual(parentModel.OptionalChildren.Count(), 0); 46 | Assert.AreEqual(childModel.Parent, parentModel); 47 | Assert.AreEqual(childModel.ParentOptional, null); 48 | } 49 | 50 | [TestMethod 51 | ] 52 | public void TestThatSingleOnUnattachedItemDoesNotThrows() 53 | { 54 | // Given 55 | var id = Guid.NewGuid(); 56 | var testStorage = new TestStorage 57 | { 58 | new ParentEntity(id), 59 | }; 60 | 61 | var instance = new TestObjectRepository(testStorage); 62 | 63 | // When 64 | instance.WaitForInitialize().GetAwaiter().GetResult(); 65 | 66 | // Then 67 | // no exceptions 68 | 69 | var parentModel = instance.Set().Single(); 70 | 71 | var childModel = new ChildModel(new ChildEntity(Guid.NewGuid())); 72 | childModel.Parent = parentModel; 73 | } 74 | 75 | [TestMethod] 76 | public void TestThatDeletingChildrenDoesntBreaks() 77 | { 78 | // Given 79 | var id = Guid.NewGuid(); 80 | var testStorage = new TestStorage 81 | { 82 | new ParentEntity(id), 83 | new ChildEntity(Guid.NewGuid()) {ParentId = id} 84 | }; 85 | 86 | var instance = new TestObjectRepository(testStorage); 87 | 88 | // When 89 | instance.WaitForInitialize().GetAwaiter().GetResult(); 90 | instance.Remove(v => true); 91 | 92 | // Then 93 | // no exceptions 94 | var parentModel = instance.Set().Single(); 95 | var childModel = instance.Set().ToArray(); 96 | Assert.AreEqual(parentModel.Children.Count(), 0); 97 | Assert.AreEqual(parentModel.OptionalChildren.Count(), 0); 98 | Assert.AreEqual(childModel.Length, 0); 99 | } 100 | 101 | [TestMethod] 102 | public void TestThatFindWorks() 103 | { 104 | // Given 105 | var id = Guid.NewGuid(); 106 | var testStorage = new TestStorage 107 | { 108 | new ParentEntity(id), 109 | }; 110 | 111 | var instance = new TestObjectRepository(testStorage); 112 | 113 | // When 114 | instance.WaitForInitialize().GetAwaiter().GetResult(); 115 | 116 | var set = instance.Set(); 117 | 118 | Assert.AreEqual(set.Find(id), set.Single()); 119 | Assert.AreEqual(set.Find(Guid.Empty), null); 120 | } 121 | 122 | [TestMethod] 123 | public void TestThatCustomIndexesWorks() 124 | { 125 | var id = Guid.NewGuid(); 126 | var testStorage = new TestStorage 127 | { 128 | new ParentEntity(id), 129 | new ChildEntity(Guid.NewGuid()) {ParentId = id, Property = "1"}, 130 | new ChildEntity(Guid.NewGuid()) {ParentId = id, Property = "2"} 131 | }; 132 | 133 | var instance = new TestObjectRepository(testStorage); 134 | 135 | // When 136 | instance.WaitForInitialize().GetAwaiter().GetResult(); 137 | 138 | instance.Set().AddIndex(() => x => x.Property); 139 | var child = instance.Set().Find(() => x => x.Property, "2"); 140 | 141 | Assert.IsNotNull(child); 142 | Assert.AreEqual(child.Property, "2"); 143 | } 144 | 145 | [TestMethod] 146 | public void TestThatCustomIndexesWorksAfterPropertyChange() 147 | { 148 | var id = Guid.NewGuid(); 149 | var testStorage = new TestStorage 150 | { 151 | new ParentEntity(id), 152 | new ChildEntity(Guid.NewGuid()) {ParentId = id, Property = "1"}, 153 | new ChildEntity(Guid.NewGuid()) {ParentId = id, Property = "2"} 154 | }; 155 | 156 | var instance = new TestObjectRepository(testStorage); 157 | 158 | // When 159 | instance.WaitForInitialize().GetAwaiter().GetResult(); 160 | 161 | instance.Set().AddIndex(() => x => x.Property); 162 | var child = instance.Set().Find(() => x => x.Property, "2"); 163 | 164 | Assert.IsNotNull(child); 165 | Assert.AreEqual(child.Property, "2"); 166 | 167 | child.Property = "3"; 168 | 169 | Assert.IsNull(instance.Set().Find(() => x => x.Property, "2")); 170 | 171 | Assert.AreEqual(child, instance.Set().Find(() => x => x.Property, "3")); 172 | } 173 | 174 | [TestMethod] 175 | public void TestThatPropertyUpdaterNotLeaks() 176 | { 177 | var id = Guid.NewGuid(); 178 | var testStorage = new TestStorage 179 | { 180 | new ParentEntity(id), 181 | new ChildEntity(Guid.NewGuid()) {ParentId = id, Property = "1"}, 182 | new ChildEntity(Guid.NewGuid()) {ParentId = id, Property = "2"} 183 | }; 184 | 185 | var instance = new TestObjectRepository(testStorage); 186 | 187 | // When 188 | instance.WaitForInitialize().GetAwaiter().GetResult(); 189 | 190 | instance.Set().AddIndex(() => x => x.Property); 191 | var child = instance.Set().Find(() => x => x.Property, "2"); 192 | 193 | Assert.IsNotNull(child); 194 | Assert.AreEqual(child.Property, "2"); 195 | 196 | child.Property = "3"; 197 | 198 | var count = ModelBase.PropertyUpdater.Cache.Count; 199 | 200 | child.Property = "2"; 201 | 202 | var newCount = ModelBase.PropertyUpdater.Cache.Count; 203 | 204 | Assert.AreEqual(count, newCount, "Memory leak!"); 205 | } 206 | 207 | [TestMethod] 208 | public void TestThatNoNotifyWhenValueNotChanged() 209 | { 210 | var id = Guid.NewGuid(); 211 | var testStorage = new TestStorage 212 | { 213 | new ChildEntity(Guid.NewGuid()) {ParentId = id, Property = "2"} 214 | }; 215 | 216 | var instance = new TestObjectRepository(testStorage); 217 | 218 | // When 219 | var child = instance.Set().First(); 220 | 221 | Assert.IsNotNull(child); 222 | var shouldFail = false; 223 | instance.ModelChanged += (a) => 224 | { 225 | if (shouldFail) 226 | throw new Exception(); 227 | }; 228 | 229 | child.Property = "3"; 230 | shouldFail = true; 231 | child.Property = "3"; 232 | shouldFail = false; 233 | child.Property = "2"; 234 | } 235 | 236 | [TestMethod, Ignore("TODO finds out how to find which property on which object needs to be reset when such happens.")] 237 | public void TestThatDeletingParentDoesntBreaks() 238 | { 239 | // Given 240 | var id = Guid.NewGuid(); 241 | var testStorage = new TestStorage 242 | { 243 | new ParentEntity(id), 244 | new ChildEntity(Guid.NewGuid()) {ParentId = id} 245 | }; 246 | 247 | var instance = new TestObjectRepository(testStorage); 248 | 249 | // When 250 | instance.WaitForInitialize().GetAwaiter().GetResult(); 251 | instance.Remove(v => true); 252 | 253 | // Then 254 | // no exceptions 255 | var parentModel = instance.Set().ToArray(); 256 | var childModel = instance.Set().Single(); 257 | Assert.AreEqual(childModel.Parent, null); 258 | Assert.AreEqual(childModel.ParentOptional, null); 259 | Assert.AreEqual(parentModel.Length, 0); 260 | } 261 | } 262 | } 263 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Tests/ProviderTestBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using Microsoft.VisualStudio.TestTools.UnitTesting; 5 | 6 | namespace OutCode.EscapeTeams.ObjectRepository.Tests 7 | { 8 | public abstract class ProviderTestBase 9 | { 10 | protected readonly TestModel _testModel; 11 | protected readonly ParentModel _parentModel; 12 | protected readonly ChildModel _childModel; 13 | 14 | public ProviderTestBase() 15 | { 16 | _testModel = new TestModel(); 17 | _parentModel = new ParentModel(); 18 | _childModel = new ChildModel(_parentModel); 19 | } 20 | 21 | protected abstract ObjectRepositoryBase CreateRepository(); 22 | 23 | protected abstract IStorage GetStorage(ObjectRepositoryBase objectRepository); 24 | 25 | [TestMethod] 26 | public void TestThatAfterRestartWorks() 27 | { 28 | var objectRepo = CreateRepository(); 29 | 30 | var storage = GetStorage(objectRepo); 31 | 32 | storage.SaveChanges().GetAwaiter().GetResult(); 33 | 34 | objectRepo = CreateRepository(); 35 | 36 | Assert.AreEqual(objectRepo.Set().Count(), 1); 37 | Assert.AreEqual(objectRepo.Set().Count(), 1); 38 | Assert.AreEqual(objectRepo.Set().Count(), 1); 39 | 40 | Assert.AreEqual(objectRepo.Set().First().Id, _testModel.TestId); 41 | Assert.AreEqual(objectRepo.Set().First().Id, _parentModel.TestId); 42 | Assert.AreEqual(objectRepo.Set().First().Id, _childModel.TestId); 43 | Assert.AreEqual(objectRepo.Set().First().ParentId, _parentModel.Id); 44 | 45 | } 46 | 47 | [TestMethod] 48 | public void TestThatAddWorks() 49 | { 50 | var objectRepo = CreateRepository(); 51 | 52 | var storage = GetStorage(objectRepo); 53 | 54 | storage.SaveChanges().GetAwaiter().GetResult(); 55 | 56 | var testsStored = storage.GetAll().GetAwaiter().GetResult().ToList(); 57 | var parentsStored = storage.GetAll().GetAwaiter().GetResult().ToList(); 58 | var childStored = storage.GetAll().GetAwaiter().GetResult().ToList(); 59 | 60 | Assert.AreEqual(testsStored.Count(), 1); 61 | Assert.AreEqual(parentsStored.Count(), 1); 62 | Assert.AreEqual(childStored.Count(), 1); 63 | 64 | Assert.AreEqual(testsStored.First().Id, _testModel.TestId); 65 | Assert.AreEqual(parentsStored.First().Id, _parentModel.TestId); 66 | Assert.AreEqual(childStored.First().Id, _childModel.TestId); 67 | Assert.AreEqual(childStored.First().ParentId, _parentModel.Id); 68 | } 69 | 70 | [TestMethod] 71 | public void TestThatRemoveWorks() 72 | { 73 | var objectRepo = CreateRepository(); 74 | 75 | objectRepo.Remove(_testModel); 76 | objectRepo.Remove(_childModel); 77 | 78 | var storage = GetStorage(objectRepo); 79 | storage.SaveChanges().GetAwaiter().GetResult(); 80 | 81 | var testsStored = storage.GetAll().GetAwaiter().GetResult().ToList(); 82 | var parentsStored = storage.GetAll().GetAwaiter().GetResult().ToList(); 83 | var childStored = storage.GetAll().GetAwaiter().GetResult().ToList(); 84 | 85 | Assert.AreEqual(testsStored.Count(), 0); 86 | Assert.AreEqual(parentsStored.Count(), 1); 87 | Assert.AreEqual(childStored.Count(), 0); 88 | 89 | Assert.AreEqual(parentsStored.First().Id, _parentModel.TestId); 90 | } 91 | 92 | public class TestEntity : BaseEntity 93 | { 94 | public string Property { get; set; } 95 | } 96 | 97 | public class ParentEntity : BaseEntity 98 | { 99 | public Guid? NullableId { get; set; } 100 | } 101 | 102 | public class ChildEntity : BaseEntity 103 | { 104 | public Guid ParentId { get; set; } 105 | public Guid? NullableId { get; set; } 106 | } 107 | 108 | public class TestModel : ModelBase 109 | { 110 | private readonly TestEntity _entity; 111 | 112 | public TestModel(TestEntity entity) 113 | { 114 | _entity = entity; 115 | } 116 | 117 | public TestModel() 118 | { 119 | _entity = new TestEntity {Id = Guid.NewGuid()}; 120 | } 121 | 122 | public string Property 123 | { 124 | get => _entity.Property; 125 | set => UpdateProperty(_entity, () => x => _entity.Property, value); 126 | } 127 | 128 | public Guid TestId 129 | { 130 | get => _entity.Id; 131 | set => UpdateProperty(_entity, () => x => _entity.Id, value); 132 | } 133 | 134 | protected internal override BaseEntity Entity => _entity; 135 | } 136 | 137 | public class ParentModel : ModelBase 138 | { 139 | private readonly ParentEntity _entity; 140 | 141 | public ParentModel(ParentEntity entity) 142 | { 143 | _entity = entity; 144 | } 145 | 146 | public ParentModel() 147 | { 148 | _entity = new ParentEntity {Id = Guid.NewGuid()}; 149 | } 150 | 151 | public Guid? NullableId 152 | { 153 | get => _entity.NullableId; 154 | set => UpdateProperty(_entity, () => x => _entity.NullableId, value); 155 | } 156 | 157 | public Guid TestId 158 | { 159 | get => _entity.Id; 160 | set => UpdateProperty(_entity, () => x => _entity.Id, value); 161 | } 162 | 163 | public IEnumerable Children => Multiple(() => x => x.ParentId); 164 | public IEnumerable OptionalChildren => Multiple(() => x => x.NullableTestId); 165 | 166 | protected internal override BaseEntity Entity => _entity; 167 | } 168 | 169 | public class ChildModel : ModelBase 170 | { 171 | private readonly ChildEntity _entity; 172 | 173 | public ChildModel(ChildEntity entity) 174 | { 175 | _entity = entity; 176 | } 177 | 178 | public ChildModel(ParentModel parent) 179 | { 180 | _entity = new ChildEntity {Id = Guid.NewGuid()}; 181 | if (parent != null) 182 | { 183 | _entity.ParentId = parent.Id; 184 | } 185 | } 186 | 187 | public Guid TestId 188 | { 189 | get => _entity.Id; 190 | set => UpdateProperty(_entity, () => x => _entity.Id, value); 191 | } 192 | 193 | public Guid? NullableTestId 194 | { 195 | get => _entity.NullableId; 196 | set => UpdateProperty(_entity, () => x => _entity.NullableId, value); 197 | } 198 | 199 | public Guid ParentId 200 | { 201 | get => _entity.ParentId; 202 | set => UpdateProperty(_entity, () => x => _entity.ParentId, value); 203 | } 204 | 205 | public ParentModel Parent => Single(ParentId); 206 | public ParentModel ParentOptional => Single(NullableTestId); 207 | 208 | protected internal override BaseEntity Entity => _entity; 209 | } 210 | 211 | [TestMethod] 212 | public void TestThatUpdateWorks() 213 | { 214 | var objectRepo = CreateRepository(); 215 | 216 | objectRepo.Remove(_testModel); 217 | var newTestModel = new TestModel {Property = "123"}; 218 | 219 | objectRepo.Add(newTestModel); 220 | 221 | var storage = GetStorage(objectRepo); 222 | storage.SaveChanges().GetAwaiter().GetResult(); 223 | 224 | var testsStored = storage.GetAll().GetAwaiter().GetResult().ToList(); 225 | 226 | Assert.AreEqual(testsStored.Count, 1); 227 | Assert.AreEqual(testsStored.First().Property, "123"); 228 | 229 | newTestModel.Property = "234"; 230 | 231 | storage.SaveChanges().GetAwaiter().GetResult(); 232 | testsStored = storage.GetAll().GetAwaiter().GetResult().ToList(); 233 | 234 | Assert.AreEqual(testsStored.Count, 1); 235 | Assert.AreEqual(testsStored.First().Property, "234"); 236 | } 237 | 238 | [TestMethod] 239 | public void TestThatFastAddRemoveNotBreaks() 240 | { 241 | var objectRepo = CreateRepository(); 242 | 243 | var newTestModel = new TestModel(); 244 | 245 | objectRepo.Add(newTestModel); 246 | newTestModel.Property = "123"; 247 | objectRepo.Remove(newTestModel); 248 | 249 | var storage = GetStorage(objectRepo); 250 | storage.SaveChanges().GetAwaiter().GetResult(); 251 | 252 | var testsStored = storage.GetAll().GetAwaiter().GetResult().ToList(); 253 | 254 | Assert.AreEqual(testsStored.Count, 1); 255 | Assert.AreEqual(testsStored.First().Id, _testModel.TestId); 256 | } 257 | } 258 | } -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/ObjectRepositoryWriteOnlyTransaction.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using Hangfire.Common; 5 | using Hangfire.States; 6 | using Hangfire.Storage; 7 | using OutCode.EscapeTeams.ObjectRepository.Hangfire.Entities; 8 | 9 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire 10 | { 11 | internal class ObjectRepositoryWriteOnlyTransaction : JobStorageTransaction 12 | { 13 | private readonly Queue _commandQueue = new Queue(); 14 | private readonly ObjectRepositoryStorage _storage; 15 | 16 | public ObjectRepositoryWriteOnlyTransaction(ObjectRepositoryStorage storage) 17 | { 18 | _storage = storage ?? throw new ArgumentNullException(nameof(storage)); 19 | } 20 | 21 | public override void Commit() 22 | { 23 | foreach (var command in _commandQueue) 24 | { 25 | command(); 26 | } 27 | } 28 | 29 | public override void ExpireJob(string jobId, TimeSpan expireIn) 30 | { 31 | QueueCommand(() => _storage.ObjectRepository.Set() 32 | .Where(v => v.Id == Guid.Parse(jobId)).ForEach(s => s.ExpireAt = DateTime.UtcNow.Add(expireIn))); 33 | } 34 | 35 | public override void PersistJob(string jobId) 36 | { 37 | QueueCommand(() => _storage.ObjectRepository.Set() 38 | .Where(v => v.Id == Guid.Parse(jobId)).ForEach(s => s.ExpireAt = null)); 39 | } 40 | 41 | public override void SetJobState(string jobId, IState state) 42 | { 43 | var jobState = AddJobStateImpl(jobId, state); 44 | 45 | QueueCommand(() => 46 | { 47 | var jobIdGuid = Guid.Parse(jobId); 48 | 49 | _storage.ObjectRepository.Set().Find(jobIdGuid) 50 | .StateId = jobState.Id; 51 | }); 52 | } 53 | 54 | public override void AddJobState(string jobId, IState state) 55 | { 56 | AddJobStateImpl(jobId, state); 57 | } 58 | 59 | private StateModel AddJobStateImpl(string jobId, IState state) 60 | { 61 | var jobIdGuid = Guid.Parse(jobId); 62 | var job = _storage.ObjectRepository.Set().Find(jobIdGuid); 63 | 64 | if (job == null) 65 | return null; 66 | 67 | var model = new StateModel 68 | { 69 | JobId = job.Id, 70 | Name = state.Name, 71 | Reason = state.Reason, 72 | CreatedAt = DateTime.UtcNow, 73 | Data = JobHelper.ToJson(state.SerializeData()) 74 | }; 75 | QueueCommand(() => _storage.ObjectRepository.Add( 76 | model)); 77 | return model; 78 | } 79 | 80 | public override void AddToQueue(string queue, string jobId) 81 | { 82 | var jobGuid = Guid.Parse(jobId); 83 | QueueCommand(() => 84 | { 85 | _storage.ObjectRepository.Add(new JobQueueModel 86 | { 87 | JobId = jobGuid, 88 | Queue = queue 89 | }); 90 | }); 91 | } 92 | 93 | public override void IncrementCounter(string key) 94 | { 95 | UpdateCounter(key, +1, null); 96 | } 97 | 98 | public override void IncrementCounter(string key, TimeSpan expireIn) 99 | { 100 | UpdateCounter(key, +1, expireIn); 101 | } 102 | 103 | public override void DecrementCounter(string key) 104 | { 105 | UpdateCounter(key, -1, null); 106 | } 107 | 108 | public override void DecrementCounter(string key, TimeSpan expireIn) 109 | { 110 | UpdateCounter(key, -1, expireIn); 111 | } 112 | 113 | private void UpdateCounter(string key, int adjustment, TimeSpan? expireIn) 114 | { 115 | QueueCommand(() => 116 | { 117 | var counter = _storage.ObjectRepository.Set().FirstOrDefault(v => v.Key == key && v.ExpireAt.HasValue == expireIn.HasValue); 118 | 119 | if (counter == null) 120 | { 121 | counter = new CounterModel(key); 122 | _storage.ObjectRepository.Add(counter); 123 | } 124 | 125 | counter.Value += adjustment; 126 | if (expireIn.HasValue) 127 | { 128 | counter.ExpireAt = DateTime.UtcNow.Add(expireIn.Value); 129 | } 130 | }); 131 | } 132 | 133 | public override void AddToSet(string key, string value) 134 | { 135 | AddToSet(key, value, 0.0); 136 | } 137 | 138 | public override void AddToSet(string key, string value, double score) 139 | { 140 | QueueCommand(() => 141 | { 142 | var fetchedSet = 143 | _storage.ObjectRepository.Set().FirstOrDefault(v => v.Key == key && v.Value == value); 144 | 145 | if (fetchedSet == null) 146 | { 147 | fetchedSet = new SetModel(key, value); 148 | _storage.ObjectRepository.Add(fetchedSet); 149 | } 150 | 151 | fetchedSet.Score = score; 152 | }); 153 | } 154 | 155 | public override void RemoveFromSet(string key, string value) 156 | { 157 | QueueCommand(() => 158 | _storage.ObjectRepository.Remove(v => v.Key == key && v.Value == value) 159 | ); 160 | } 161 | 162 | public override void InsertToList(string key, string value) 163 | { 164 | QueueCommand(() => 165 | _storage.ObjectRepository.Add(new ListModel(key, value))); 166 | } 167 | 168 | public override void RemoveFromList(string key, string value) 169 | { 170 | QueueCommand(() => _storage.ObjectRepository.Remove(v => v.Key == key && v.Value == value)); 171 | } 172 | 173 | public override void TrimList(string key, int keepStartingFrom, int keepEndingAt) 174 | { 175 | QueueCommand(() => 176 | { 177 | var listModels = _storage.ObjectRepository.Set().Where(v => v.Key == key).Skip(keepStartingFrom) 178 | .Take(keepEndingAt - keepStartingFrom + 1) 179 | .ToList(); 180 | _storage.ObjectRepository.RemoveRange( 181 | listModels); 182 | }); 183 | } 184 | 185 | public override void SetRangeInHash(string key, IEnumerable> keyValuePairs) 186 | { 187 | QueueCommand(() => 188 | { 189 | foreach (var keyValuePair in keyValuePairs) 190 | { 191 | var fetchedHash = _storage.ObjectRepository.Set() 192 | .FirstOrDefault(v => v.Key == key && v.Field == keyValuePair.Key); 193 | 194 | if (fetchedHash == null) 195 | { 196 | fetchedHash = new HashModel(key, keyValuePair.Key); 197 | _storage.ObjectRepository.Add(fetchedHash); 198 | } 199 | 200 | fetchedHash.Value = keyValuePair.Value; 201 | } 202 | }); 203 | } 204 | 205 | public override void RemoveHash(string key) 206 | { 207 | QueueCommand(() => _storage.ObjectRepository.Remove(v => v.Key == key)); 208 | } 209 | 210 | public override void AddRangeToSet(string key, IList items) 211 | { 212 | QueueCommand(() => items.Select(v => new SetModel(key, v)) 213 | .ForEach(v => _storage.ObjectRepository.Add(v))); 214 | } 215 | 216 | public override void RemoveSet(string key) 217 | { 218 | QueueCommand(() => _storage.ObjectRepository.Remove(v => v.Key == key)); 219 | } 220 | 221 | public override void ExpireHash(string key, TimeSpan expireIn) 222 | { 223 | var when = DateTime.UtcNow.Add(expireIn); 224 | QueueCommand(() => _storage.ObjectRepository.Set() 225 | .Where(v=>v.Key == key) 226 | .ForEach(s=>s.ExpireAt = when)); 227 | } 228 | 229 | public override void ExpireSet(string key, TimeSpan expireIn) 230 | { 231 | var when = DateTime.UtcNow.Add(expireIn); 232 | QueueCommand(() => _storage.ObjectRepository.Set() 233 | .Where(v => v.Key == key) 234 | .ForEach(s => s.ExpireAt = when)); 235 | } 236 | 237 | public override void ExpireList(string key, TimeSpan expireIn) 238 | { 239 | var when = DateTime.UtcNow.Add(expireIn); 240 | QueueCommand(() => _storage.ObjectRepository.Set() 241 | .Where(v => v.Key == key) 242 | .ForEach(s => s.ExpireAt = when)); 243 | } 244 | 245 | public override void PersistHash(string key) 246 | { 247 | QueueCommand(() => _storage.ObjectRepository.Set() 248 | .Where(v => v.Key == key).ForEach(s => s.ExpireAt = null)); 249 | } 250 | 251 | public override void PersistSet(string key) 252 | { 253 | QueueCommand(() => _storage.ObjectRepository.Set() 254 | .Where(v => v.Key == key).ForEach(s => s.ExpireAt = null)); 255 | } 256 | 257 | public override void PersistList(string key) 258 | { 259 | QueueCommand(() => _storage.ObjectRepository.Set() 260 | .Where(v => v.Key == key).ForEach(s => s.ExpireAt = null)); 261 | } 262 | 263 | internal void QueueCommand(Action action) => _commandQueue.Enqueue(action); 264 | } 265 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ObjectRepository 2 | Generic In-Memory Object Database (Repository pattern) 3 | 4 | ## Why store anything in memory? 5 | 6 | Most people would use SQL-based database for backend. 7 | But sometimes SQL just don't fit well - i.e. when you're building a search engine or when you need to query social graph in eloquent way. 8 | 9 | **Worst of all is when your teammate doesn't know how to write fast queries. How much time was spent debugging N+1 issues and building additional indexes just for the sake of main page load speed?** 10 | 11 | Another approach would be to use NoSQL. Several years ago there was a big hype about it - every microservice had used MongoDB and everyone was happy getting JSON documents *(btw, how about circular references?)* 12 | 13 | Why not store everything in-memory, sometimes flushing all on the underlying storage (i.e. file, remote database)? 14 | 15 | Memory is cheap, and all kind of small and medium-sized projects would take no more than 1 Gb of memory. *(i.e. my favorite home project - [BudgetTracker](https://github.com/DiverOfDark/BudgetTracker), which stores daily stats of all my transcations and balances uses just 45 mb after 1.5 years of history)* 16 | 17 | Pros: 18 | 19 | - Access to data is easier - you don't need to think about writing queries, eager loading or ORM-dependent stuff. You work regular C# objects; 20 | - No issues due to multithreading; 21 | - Very fast - no network calls, no generating queries, no (de)serialization; 22 | - You can store data in any way you like - be it XML file on disk, SQL Server, or Azure Table Storage. 23 | 24 | Cons: 25 | 26 | - You can't scale horizontally, thus no blue-green deployment; 27 | - If app crashes - you can lost you latest data. *(But YOUR app never crashes, right?)* 28 | 29 | 30 | ## How it works? 31 | 32 | In a nutshell: 33 | 34 | - On application start connection to data storage is established, and initial load begins; 35 | - Object model is created, (primary) indexes are calculated; 36 | - Subscription to model's property changes (INotifyPropertyChanged) and collection changes (INotifyCollectionChanged) is created; 37 | - When something changes - event is raised and changed object is added to queue for persisting; 38 | - Persisting occurs by timer in background thread; 39 | - When application exits - additional save is called. 40 | 41 | ## Usage: 42 | 43 | ```cs 44 | // Required dependencies: 45 | 46 | // Core library 47 | Install-Package OutCode.EscapeTeams.ObjectRepository 48 |      49 | // Storage - you one which you need. 50 | Install-Package OutCode.EscapeTeams.ObjectRepository.File 51 | Install-Package OutCode.EscapeTeams.ObjectRepository.LiteDb 52 | Install-Package OutCode.EscapeTeams.ObjectRepository.AzureTableStorage 53 |      54 | // Optional - it is possible to store hangfire data in ObjectRepository 55 | // Install-Package OutCode.EscapeTeams.ObjectRepository.Hangfire 56 | ``` 57 | 58 | ```cs 59 | // Data Model - it is how all will be stored. 60 | 61 | public class ParentEntity : BaseEntity 62 | { 63 |     public ParentEntity(Guid id) => Id = id; 64 | } 65 |      66 | public class ChildEntity : BaseEntity 67 | { 68 |     public ChildEntity(Guid id) => Id = id; 69 |     public Guid ParentId { get; set; } 70 |     public string Value { get; set; } 71 | } 72 | ``` 73 | 74 | ```cs 75 | // Object Model - something your app will work with 76 | 77 | public class ParentModel : ModelBase 78 | { 79 |     public ParentModel(ParentEntity entity) 80 |     { 81 |         Entity = entity; 82 |     } 83 |      84 |     public ParentModel() 85 |     { 86 |         Entity = new ParentEntity(Guid.NewGuid()); 87 |     } 88 |      89 |     // 1-Many relation 90 |     public IEnumerable Children => Multiple(() => x => x.ParentId); 91 |      92 |     protected override BaseEntity Entity { get; } 93 | } 94 |      95 | public class ChildModel : ModelBase 96 | { 97 |     private ChildEntity _childEntity; 98 |      99 |     public ChildModel(ChildEntity entity) 100 |     { 101 |         _childEntity = entity; 102 |     } 103 |      104 |     public ChildModel()  105 |     { 106 |         _childEntity = new ChildEntity(Guid.NewGuid()); 107 |     } 108 |      109 |     public Guid ParentId 110 |     { 111 |         get => _childEntity.ParentId; 112 |         set => UpdateProperty(_childEntity, () => x => x.ParentId, value); 113 |     } 114 |      115 |     public string Value 116 |     { 117 |         get => _childEntity.Value; 118 |         set => UpdateProperty(_childEntity, () => x => x.Value, value); 119 |     } 120 |      121 |     // Indexed access 122 |     public ParentModel Parent => Single(ParentId); 123 |      124 |     protected override BaseEntity Entity => _childEntity; 125 | } 126 | ``` 127 | 128 | ```cs 129 | // ObjectRepository itself 130 | 131 | public class MyObjectRepository : ObjectRepositoryBase 132 | { 133 |     public MyObjectRepository(IStorage storage) : base(storage, NullLogger.Instance) 134 |     { 135 |         IsReadOnly = true; // For testing purposes. Allows to not save changes to database. 136 |      137 |         AddType((ParentEntity x) => new ParentModel(x)); 138 |         AddType((ChildEntity x) => new ChildModel(x)); 139 |      140 |         //// If you are using hangfire and want to store it's data in this objectrepo - uncomment this 141 |         // this.RegisterHangfireScheme();  142 |      143 |         Initialize(); 144 |     } 145 | } 146 | ``` 147 | 148 | Create ObjectRepository: 149 | 150 | ```cs 151 | var memory = new MemoryStream(); 152 | var db = new LiteDatabase(memory); 153 | var dbStorage = new LiteDbStorage(db); 154 |      155 | var repository = new MyObjectRepository(dbStorage); 156 | await repository.WaitForInitialize(); 157 | 158 | /* if you need HangFire 159 | public void ConfigureServices(IServiceCollection services, ObjectRepository objectRepository) 160 | { 161 |     services.AddHangfire(s => s.UseHangfireStorage(objectRepository)); 162 | } 163 | */ 164 | ``` 165 | 166 | Inserting new object: 167 | 168 | ```cs 169 | var newParent = new ParentModel() 170 | repository.Add(newParent); 171 | ``` 172 | 173 | After this **ParentModel** will be added to both local cache and to the queue to persist. Thus this op is O(1) and you can continue you work right away. 174 | 175 | To check that this object is added and is the same you added: 176 | 177 | ```cs 178 | var parents = repository.Set(); 179 | var myParent = parents.Find(newParent.Id); 180 | Assert.IsTrue(ReferenceEquals(myParent, newParent)); 181 | ``` 182 | 183 | What happens here? *Set<ParentModel>()* returns *TableDictionary<ParentModel>* which is essentially *ConcurrentDictionary<ParentModel, ParentModel>* and provides additional methods for indexes. This allows to have a Find methods to search by Id (or other fields) without iterating all objects. 184 | 185 | When you add something to *ObjectRepository* subscription to property changes is created, thus any property change also add object to write queue. 186 | Property updating looks just like regular POCO object:: 187 | 188 | ```cs 189 | myParent.Children.First().Property = "Updated value"; 190 | ``` 191 | 192 | You can delete object in following ways: 193 | 194 | ```cs 195 | repository.Remove(myParent); 196 | repository.RemoveRange(otherParents); 197 | repository.Remove(x => !x.Children.Any()); 198 | ``` 199 | 200 | Deletion also happens via queue in background thread. 201 | 202 | ## How saving actually works? 203 | 204 | When any object set tracked by *ObjectRepository* is changed (i.e. added, removed, property update) then event *ModelChanged* is raised. 205 | *IStorage*, which is used for persistence, is subscribed to this event. All implementations of *IStorage* are enqueueing all *ModelChanged* events to 3 different queues - for addition, update, and removal. 206 | 207 | Also each kind of *IStorage* creates timer which every 5 secs invokes actual saving. 208 | 209 | *BTW, there exists separate API for explicit saving: **ObjectRepository.Save()**.* 210 | 211 | Before each saving unneccessary operations are removed from the queue (i.e. multiple property changes, adding and removal of same object). After queue is sanitized actual saving is performed.  212 | 213 | *In all cases when object is persisted - it is persisted as a whole. So it is possible a scenario when objects are saving in different order than they were changed, including objects being saved with newer property values than were at the time of adding to queue.* 214 | 215 | ## What else? 216 | 217 | - All libraries are targeted to .NET Standard 2.0. ObjectRepository can be used in any modern .NET app. 218 | - All API is thread-safe. Inner collections are based on *ConcurrentDictionary* and all handlers are either have locks or don't need them.  219 | - Only thing you should remember - don't forget to call *ObjectRepository.Save();* when your app is going to shutdown 220 | - If you need fast search - you can use custom indexes (works only for unique values): 221 | 222 | ```cs 223 | repository.Set().AddIndex(() => x => x.Value); 224 | repository.Set().Find(() => x => x.Value, "myValue"); 225 | ``` 226 | 227 | ## Who uses this? 228 | 229 | I am using this library in all my hobby projects because it is simple and handy. In most cases I don't have to set up SQL Server or use some pricey cloud service - LiteDB/file-based approach is fine. 230 | 231 | A while ago, when I was bootstrapping EscapeTeams startup - we used Azure Table Storage as backing storage. 232 | 233 | ## Plans for future 234 | 235 | We want to solve major pain point of current approach - horizontal scaling. For this to happen we need to either implement distributed transactions(sic!) or to accept the fact that same objects in different instances should be not changed at the same moment of time (or latest who changed wins). 236 | 237 | From tech point of view this can be solved in following way: 238 | 239 | - Store event log and snapshot instead of actual model 240 | - Find other instances (add endpoints to settings? use udp discovery? master/slave or peers?) 241 | - Replicate eventlog between instances using any consensus algo, i.e. raft. 242 | 243 | Other issue that exists (and worries me) is cascade deletion(and finding cases when you are deleting object that is being references by some other object). It is just not implemented, and currently exceptions may be thrown when such issue happens. 244 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.AzureTableStorage/AzureTableContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Reflection; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | using Microsoft.WindowsAzure.Storage.Table; 9 | using Newtonsoft.Json; 10 | 11 | namespace OutCode.EscapeTeams.ObjectRepository.AzureTableStorage 12 | { 13 | /// 14 | /// Data Context bound to Azure Table Storage. 15 | /// 16 | public class AzureTableContext : IStorage, IDisposable 17 | { 18 | private readonly CloudTableClient _client; 19 | private Timer _saveTimer; 20 | 21 | private readonly CloudTable _migrationTables; 22 | private readonly ConcurrentDictionary> _entitiesToAdd; 23 | private readonly ConcurrentDictionary> _entitiesToRemove; 24 | private readonly ConcurrentDictionary> _entitiesToUpdate; 25 | 26 | private int _saveInProgress; 27 | 28 | public AzureTableContext(CloudTableClient client) 29 | { 30 | _client = client; 31 | _migrationTables = _client.GetTableReference("MigrationTable"); 32 | 33 | _entitiesToAdd = new ConcurrentDictionary>(); 34 | _entitiesToRemove = new ConcurrentDictionary>(); 35 | _entitiesToUpdate = new ConcurrentDictionary>(); 36 | } 37 | 38 | public event Action OnError = delegate {}; 39 | 40 | public string ExportStream() 41 | { 42 | var result = new 43 | { 44 | add = _entitiesToAdd.ToDictionary(v => v.Key, v => v.Value.ToList()), 45 | remove = _entitiesToRemove.ToDictionary(v => v.Key, v => v.Value.ToList()), 46 | mod = _entitiesToUpdate.ToDictionary(v => v.Key, v => v.Value.ToList()), 47 | }; 48 | 49 | return JsonConvert.SerializeObject(result); 50 | } 51 | 52 | /// 53 | /// Saves all data changes to the tables. 54 | /// 55 | public async Task SaveChanges() 56 | { 57 | if (Interlocked.CompareExchange(ref _saveInProgress, 1, 0) == 0) 58 | { 59 | try 60 | { 61 | foreach (var remove in _entitiesToRemove.ToList()) 62 | { 63 | foreach (var removeItem in remove.Value.ToList()) 64 | { 65 | if (_entitiesToAdd.TryGetValue(remove.Key, out var toAddList)) 66 | { 67 | if (toAddList?.Contains(removeItem) == true) 68 | { 69 | remove.Value.Remove(removeItem); 70 | toAddList.Remove(removeItem); 71 | } 72 | } 73 | 74 | if (_entitiesToUpdate.TryGetValue(remove.Key, out var removeList)) 75 | { 76 | removeList.Remove(removeItem); 77 | } 78 | } 79 | } 80 | 81 | await ProcessAction(_entitiesToAdd, (batch, entity) => batch.Insert(new DateTimeAwareTableEntityAdapter(entity))); 82 | await ProcessAction(_entitiesToUpdate, (batch, entity) => batch.Replace(new DateTimeAwareTableEntityAdapter(entity))); 83 | await ProcessAction(_entitiesToRemove, (batch, entity) => batch.Delete(new DateTimeAwareTableEntityAdapter(entity))); 84 | } 85 | finally 86 | { 87 | _saveInProgress = 0; 88 | } 89 | } 90 | } 91 | 92 | /// 93 | /// Returns the entire contents of a table. 94 | /// 95 | public Task> GetAll() 96 | { 97 | var task = GetType().GetMethod(nameof(ExecuteQuery), BindingFlags.Instance | BindingFlags.NonPublic).MakeGenericMethod(typeof(T)).Invoke(this, new object[0]); 98 | return (Task>) task; 99 | } 100 | 101 | private async Task> ExecuteQuery() where T:BaseEntity, new() 102 | { 103 | var tableReference = _client.GetTableReference(typeof(T).Name); 104 | await tableReference.CreateIfNotExistsAsync(); 105 | 106 | var result = await tableReference.ExecuteQueryAsync(new TableQuery>()); 107 | 108 | var adaptersWithInconsistentPartitionKey = 109 | new ConcurrentList>(result.Where(x => x.InconsistentPartitionKey)); 110 | 111 | foreach (var batch in SplitToBatches(adaptersWithInconsistentPartitionKey)) 112 | { 113 | var deleteOp = new TableBatchOperation(); 114 | var insertOp = new TableBatchOperation(); 115 | foreach (var item in batch) 116 | { 117 | deleteOp.Delete(item); 118 | insertOp.InsertOrMerge(new DateTimeAwareTableEntityAdapter(item.OriginalEntity)); 119 | } 120 | 121 | await tableReference.ExecuteBatchAsync(insertOp); 122 | await tableReference.ExecuteBatchAsync(deleteOp); 123 | } 124 | 125 | return result.Select(v=>v.OriginalEntity).ToList(); 126 | } 127 | 128 | public async Task ApplyMigration(AzureTableMigration migration) 129 | { 130 | await _migrationTables.CreateIfNotExistsAsync(); 131 | var appliedMigrations = await _migrationTables.ExecuteQueryAsync(new TableQuery()); 132 | 133 | if (appliedMigrations.Any(s => s.PartitionKey == "migration" && s.RowKey == migration.Name)) 134 | { 135 | return; 136 | } 137 | 138 | await migration.Execute(this); 139 | var entity = new TableEntity {PartitionKey = "migration", RowKey = migration.Name}; 140 | await _migrationTables.ExecuteAsync(TableOperation.Insert(entity)); 141 | await SaveChanges(); 142 | } 143 | 144 | /// 145 | /// Registers an entity for one of the operations. 146 | /// 147 | private void AddEntityToLookup(T entity, ConcurrentDictionary> lookup) 148 | where T: BaseEntity 149 | { 150 | var set = lookup.GetOrAdd(entity.GetType(), type => new ConcurrentList()); 151 | set.Add(entity); 152 | } 153 | 154 | /// 155 | /// Performs an action on the group of objects. 156 | /// 157 | private async Task ProcessAction(ConcurrentDictionary> lookup, Action entityAction) 158 | { 159 | foreach (var type in lookup.Keys) 160 | { 161 | var batches = SplitToBatches(lookup[type]).ToList(); 162 | foreach(var batch in batches) 163 | try 164 | { 165 | await ProcessObjectList(lookup[type], type, batch, entityAction); 166 | } 167 | catch (Exception ex) 168 | { 169 | OnError(ex); 170 | } 171 | } 172 | } 173 | 174 | /// 175 | /// Processes a single action on the objects. 176 | /// 177 | private async Task ProcessObjectList(ConcurrentList concurrentQueue, Type type, IEnumerable list, Action entityAction) 178 | where T: BaseEntity 179 | { 180 | var tableName = type.Name; 181 | var tableRef = _client.GetTableReference(tableName); 182 | var exs = new List(); 183 | list = list.ToList(); 184 | try 185 | { 186 | var batch = new TableBatchOperation(); 187 | foreach (var obj in list) 188 | entityAction(batch, obj); 189 | 190 | await tableRef.ExecuteBatchAsync(batch); 191 | } 192 | catch 193 | { 194 | foreach (var item in list) 195 | { 196 | try 197 | { 198 | var batch = new TableBatchOperation(); 199 | entityAction(batch, item); 200 | await tableRef.ExecuteBatchAsync(batch); 201 | } 202 | catch(Exception exception) 203 | { 204 | concurrentQueue.Add(item); 205 | 206 | var data = item.GetPropertiesAsRawData(); 207 | 208 | exs.Add(new Exception($"Failed to update {item.Id}\r\n{data}", exception)); 209 | } 210 | } 211 | 212 | if (exs.Any()) 213 | { 214 | throw new AggregateException("Azure batch failed", exs); 215 | } 216 | } 217 | } 218 | 219 | /// 220 | /// Splits a sequence into a list of subsequences of given length. 221 | /// 222 | private static IEnumerable> SplitToBatches(ConcurrentList sequence) 223 | { 224 | const int BATCH_SIZE = 100; // enforced by Azure Table Storage 225 | 226 | var batch = new List(BATCH_SIZE); 227 | 228 | while(sequence.TryTake(out var curr)) 229 | { 230 | if (batch.Count < BATCH_SIZE) 231 | { 232 | if (!batch.Contains(curr)) 233 | batch.Add(curr); 234 | } 235 | else 236 | { 237 | yield return batch; 238 | batch = new List(BATCH_SIZE) {curr}; 239 | } 240 | } 241 | 242 | if(batch.Count > 0) 243 | yield return batch; 244 | } 245 | 246 | public Dictionary GetStatistics() 247 | { 248 | var d = new Dictionary(); 249 | foreach (var item in _entitiesToAdd) 250 | { 251 | d[item.Key] = item.Value.Count(); 252 | } 253 | 254 | foreach (var item in _entitiesToRemove) 255 | { 256 | if (!d.ContainsKey(item.Key)) 257 | d[item.Key] = 0; 258 | d[item.Key] += item.Value.Count(); 259 | } 260 | 261 | foreach (var item in _entitiesToUpdate) 262 | { 263 | if (!d.ContainsKey(item.Key)) 264 | d[item.Key] = 0; 265 | d[item.Key] += item.Value.Count(); 266 | } 267 | 268 | return d; 269 | } 270 | 271 | public void Track(ObjectRepositoryBase objectRepository, bool isReadonly) 272 | { 273 | if (!isReadonly) 274 | { 275 | _saveTimer = new Timer(_ => SaveChanges().GetAwaiter().GetResult(), null, 0, 5000); 276 | } 277 | 278 | objectRepository.ModelChanged += (change) => 279 | { 280 | switch (change.ChangeType) 281 | { 282 | case ChangeType.Update: 283 | AddEntityToLookup((BaseEntity) change.Entity, _entitiesToUpdate); 284 | break; 285 | case ChangeType.Add: 286 | AddEntityToLookup((BaseEntity)change.Entity, _entitiesToAdd); 287 | break; 288 | case ChangeType.Remove: 289 | AddEntityToLookup((BaseEntity)change.Entity, _entitiesToRemove); 290 | break; 291 | } 292 | }; 293 | 294 | } 295 | 296 | public void Dispose() => _saveTimer?.Dispose(); 297 | } 298 | } 299 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/ObjectRepositoryConnection.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Threading; 6 | using Hangfire.Common; 7 | using Hangfire.Server; 8 | using Hangfire.Storage; 9 | using Hangfire.Storage.Monitoring; 10 | using OutCode.EscapeTeams.ObjectRepository.Hangfire.Entities; 11 | 12 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire 13 | { 14 | internal class ObjectRepositoryConnection : JobStorageConnection 15 | { 16 | private class InProcessLockDisposable : IDisposable 17 | { 18 | public static readonly InProcessLockDisposable Instance = new InProcessLockDisposable(); 19 | 20 | private InProcessLockDisposable() 21 | { 22 | } 23 | 24 | public void Dispose() => Monitor.Exit(Instance); 25 | } 26 | 27 | private readonly ObjectRepositoryStorage _storage; 28 | private static readonly ConcurrentList _jobsTakenOut = new ConcurrentList(); 29 | 30 | public ObjectRepositoryConnection(ObjectRepositoryStorage storage) 31 | { 32 | _storage = storage ?? throw new ArgumentNullException(nameof(storage)); 33 | } 34 | 35 | public override IWriteOnlyTransaction CreateWriteTransaction() 36 | { 37 | return new ObjectRepositoryWriteOnlyTransaction(_storage); 38 | } 39 | 40 | public override IDisposable AcquireDistributedLock(string resource, TimeSpan timeout) 41 | { 42 | Monitor.Enter(InProcessLockDisposable.Instance); 43 | return InProcessLockDisposable.Instance; 44 | } 45 | 46 | public override IFetchedJob FetchNextJob(string[] queues, CancellationToken cancellationToken) 47 | { 48 | lock (_jobsTakenOut) 49 | { 50 | if (queues == null || queues.Length == 0) throw new ArgumentNullException(nameof(queues)); 51 | 52 | if (queues == null) throw new ArgumentNullException(nameof(queues)); 53 | if (queues.Length == 0) throw new ArgumentException("Queue array must be non-empty.", nameof(queues)); 54 | 55 | JobQueueModel fetchedJob; 56 | 57 | do 58 | { 59 | cancellationToken.ThrowIfCancellationRequested(); 60 | 61 | fetchedJob = _storage.ObjectRepository.Set() 62 | .Where(s => s.FetchedAt == null || s.FetchedAt < DateTime.UtcNow) 63 | .Where(v => !_jobsTakenOut.Contains(v)) 64 | .FirstOrDefault(s => queues.Contains(s.Queue)); 65 | 66 | if (fetchedJob != null) 67 | { 68 | fetchedJob.FetchedAt = DateTime.UtcNow; 69 | } 70 | 71 | if (fetchedJob == null) 72 | { 73 | cancellationToken.WaitHandle.WaitOne(ObjectRepositoryExtensions.QueuePollInterval); 74 | cancellationToken.ThrowIfCancellationRequested(); 75 | } 76 | } while (fetchedJob == null); 77 | 78 | return new ObjectRepositoryFetchedJob(_jobsTakenOut, _storage.ObjectRepository, 79 | fetchedJob); 80 | } 81 | } 82 | 83 | public override string CreateExpiredJob( 84 | Job job, 85 | IDictionary parameters, 86 | DateTime createdAt, 87 | TimeSpan expireIn) 88 | { 89 | var invocationData = InvocationData.Serialize(job); 90 | 91 | var jobModel = new JobModel(); 92 | _storage.ObjectRepository.Add(jobModel); 93 | jobModel.InvocationData = JobHelper.ToJson(invocationData); 94 | jobModel.Arguments = invocationData.Arguments; 95 | jobModel.CreatedAt = createdAt; 96 | jobModel.ExpireAt = createdAt.Add(expireIn); 97 | 98 | var jobId = jobModel.Id; 99 | 100 | foreach (var parameter in parameters) 101 | { 102 | var jpm = new JobParameterModel(jobId, parameter.Key) {Value = parameter.Value}; 103 | _storage.ObjectRepository.Add(jpm); 104 | } 105 | 106 | return jobId.ToString(); 107 | } 108 | 109 | public override JobData GetJobData(string id) 110 | { 111 | if (!Guid.TryParse(id, out var guid)) 112 | return null; 113 | 114 | var jobData = _storage.ObjectRepository.Set().Find(guid); 115 | 116 | if (jobData == null) return null; 117 | 118 | Job job = null; 119 | JobLoadException loadException = null; 120 | 121 | try 122 | { 123 | var invocationData = JobHelper.FromJson(jobData.InvocationData); 124 | invocationData.Arguments = jobData.Arguments; 125 | 126 | job = invocationData.Deserialize(); 127 | } 128 | catch (JobLoadException ex) 129 | { 130 | loadException = ex; 131 | } 132 | 133 | var state = _storage.ObjectRepository.Set().FirstOrDefault(v => v.Id == jobData.StateId); 134 | 135 | return new JobData 136 | { 137 | Job = job, 138 | State = state?.Name, 139 | CreatedAt = jobData.CreatedAt, 140 | LoadException = loadException 141 | }; 142 | } 143 | 144 | public override StateData GetStateData(string jobId) 145 | { 146 | if (!Guid.TryParse(jobId, out var jobIdGuid)) 147 | return null; 148 | 149 | var stateId = _storage.ObjectRepository.Set().Find(jobIdGuid); 150 | if (stateId?.StateId == null) 151 | return null; 152 | 153 | var state = _storage.ObjectRepository.Set().Find(stateId.StateId.Value); 154 | if (state == null) 155 | return null; 156 | 157 | var data = new Dictionary( 158 | JobHelper.FromJson>(state.Data), 159 | StringComparer.OrdinalIgnoreCase); 160 | 161 | return new StateData 162 | { 163 | Name = state.Name, 164 | Reason = state.Reason, 165 | Data = data 166 | }; 167 | } 168 | 169 | public override void SetJobParameter(string id, string name, string value) 170 | { 171 | var jobId = Guid.Parse(id); 172 | 173 | var jobParam = _storage.ObjectRepository.Set() 174 | .FirstOrDefault(v => v.JobId == jobId && v.Name == name); 175 | 176 | if (jobParam == null) 177 | { 178 | jobParam = new JobParameterModel(jobId, name); 179 | _storage.ObjectRepository.Add(jobParam); 180 | } 181 | 182 | jobParam.Value = value; 183 | } 184 | 185 | public override string GetJobParameter(string id, string name) 186 | { 187 | if (!Guid.TryParse(id, out var jobId)) 188 | return null; 189 | return _storage.ObjectRepository.Set() 190 | .Where(v => v.JobId == jobId && v.Name == name) 191 | .Select(v => v.Value) 192 | .FirstOrDefault(); 193 | } 194 | 195 | public override HashSet GetAllItemsFromSet(string key) 196 | { 197 | return new HashSet(_storage.ObjectRepository.Set().Where(v => v.Key == key) 198 | .Select(v => v.Value)); 199 | } 200 | 201 | public override string GetFirstByLowestScoreFromSet(string key, double fromScore, double toScore) 202 | { 203 | return _storage.ObjectRepository.Set() 204 | .Where(v => v.Key == key && v.Score >= fromScore && v.Score <= toScore) 205 | .OrderBy(v => v.Score) 206 | .Select(v => v.Value) 207 | .FirstOrDefault(); 208 | } 209 | 210 | public override void SetRangeInHash(string key, IEnumerable> keyValuePairs) 211 | { 212 | foreach (var keyValuePair in keyValuePairs) 213 | { 214 | var fetchedHash = _storage.ObjectRepository 215 | .Set() 216 | .FirstOrDefault(v => v.Key == key && v.Field == keyValuePair.Key); 217 | 218 | if (fetchedHash == null) 219 | { 220 | fetchedHash = new HashModel(key, keyValuePair.Key); 221 | _storage.ObjectRepository.Add(fetchedHash); 222 | } 223 | 224 | fetchedHash.Value = keyValuePair.Value; 225 | } 226 | } 227 | 228 | public override Dictionary GetAllEntriesFromHash(string key) 229 | { 230 | return _storage.ObjectRepository.Set() 231 | .Where(v => v.Key == key) 232 | .ToDictionary(v => v.Field, v => v.Value); 233 | } 234 | 235 | public override void AnnounceServer(string serverId, ServerContext context) 236 | { 237 | if (serverId == null) throw new ArgumentNullException(nameof(serverId)); 238 | if (context == null) throw new ArgumentNullException(nameof(context)); 239 | 240 | var data = new ServerData 241 | { 242 | WorkerCount = context.WorkerCount, 243 | Queues = context.Queues, 244 | StartedAt = DateTime.UtcNow, 245 | }; 246 | 247 | var existing = _storage.ObjectRepository.Set().FirstOrDefault(v => v.Name == serverId); 248 | 249 | if (existing == null) 250 | { 251 | existing = new ServerModel(serverId); 252 | _storage.ObjectRepository.Add(existing); 253 | } 254 | 255 | existing.Data = JobHelper.ToJson(data); 256 | existing.LastHeartbeat = DateTime.UtcNow; 257 | } 258 | 259 | public override void RemoveServer(string serverId) 260 | { 261 | _storage.ObjectRepository.Remove(v=>v.Name == serverId); 262 | } 263 | 264 | public override void Heartbeat(string serverId) 265 | { 266 | _storage.ObjectRepository 267 | .Set() 268 | .First(v => v.Name == serverId).LastHeartbeat = DateTime.UtcNow; 269 | } 270 | 271 | public override int RemoveTimedOutServers(TimeSpan timeOut) 272 | { 273 | var badServers = _storage.ObjectRepository.Set().Where(v => v.LastHeartbeat < DateTime.UtcNow.Add(timeOut.Negate())) 274 | .ToList(); 275 | 276 | _storage.ObjectRepository.RemoveRange(badServers); 277 | return badServers.Count; 278 | } 279 | 280 | public override long GetSetCount(string key) 281 | { 282 | return _storage.ObjectRepository 283 | .Set() 284 | .Count(v => v.Key == key); 285 | } 286 | 287 | public override List GetRangeFromSet(string key, int startingFrom, int endingAt) 288 | { 289 | return _storage.ObjectRepository.Set() 290 | .Where(v => v.Key == key) 291 | .Select(s => s.Value) 292 | .OrderBy(v => v) 293 | .Skip(startingFrom) 294 | .Take(endingAt - startingFrom + 1) 295 | .ToList(); 296 | } 297 | 298 | public override TimeSpan GetSetTtl(string key) 299 | { 300 | var result = _storage.ObjectRepository.Set() 301 | .Where(v => v.Key == key && v.ExpireAt != null) 302 | .OrderBy(v => v.ExpireAt) 303 | .Select(v => v.ExpireAt) 304 | .FirstOrDefault(); 305 | 306 | if (!result.HasValue) return TimeSpan.FromSeconds(-1); 307 | 308 | return result.Value - DateTime.UtcNow; 309 | } 310 | 311 | public override long GetCounter(string key) 312 | { 313 | return _storage.ObjectRepository.Set().Where(v => v.Key == key) 314 | .Select(v => v.Value).Sum(); 315 | } 316 | 317 | public override long GetHashCount(string key) 318 | { 319 | return _storage.ObjectRepository 320 | .Set() 321 | .Count(v => v.Key == key); 322 | } 323 | 324 | public override TimeSpan GetHashTtl(string key) 325 | { 326 | var result = _storage.ObjectRepository.Set() 327 | .Where(v => v.Key == key && v.ExpireAt != null) 328 | .OrderBy(v => v.ExpireAt) 329 | .Select(v => v.ExpireAt) 330 | .FirstOrDefault(); 331 | 332 | if (!result.HasValue) return TimeSpan.FromSeconds(-1); 333 | 334 | return result.Value - DateTime.UtcNow; 335 | } 336 | 337 | public override string GetValueFromHash(string key, string name) 338 | { 339 | return _storage.ObjectRepository.Set() 340 | .Where(v => v.Key == key && v.Field == name) 341 | .Select(v => v.Value) 342 | .FirstOrDefault(); 343 | } 344 | 345 | public override long GetListCount(string key) 346 | { 347 | return _storage.ObjectRepository.Set() 348 | .Count(s => s.Key == key); 349 | } 350 | 351 | public override TimeSpan GetListTtl(string key) 352 | { 353 | var result = _storage.ObjectRepository.Set() 354 | .OrderBy(v => v.ExpireAt) 355 | .Select(v => v.ExpireAt) 356 | .FirstOrDefault(); 357 | if (!result.HasValue) return TimeSpan.FromSeconds(-1); 358 | 359 | return result.Value - DateTime.UtcNow; 360 | } 361 | 362 | public override List GetRangeFromList(string key, int startingFrom, int endingAt) 363 | { 364 | return _storage.ObjectRepository 365 | .Set() 366 | .Where(v => v.Key == key) 367 | .Select(v => v.Value) 368 | .OrderBy(v => v) 369 | .Skip(startingFrom) 370 | .Take(endingAt - startingFrom + 1) 371 | .ToList(); 372 | } 373 | 374 | public override List GetAllItemsFromList(string key) 375 | { 376 | return _storage.ObjectRepository.Set() 377 | .Where(v => v.Key == key) 378 | .Select(v => v.Value) 379 | .OrderBy(v => v) 380 | .ToList(); 381 | } 382 | } 383 | } 384 | -------------------------------------------------------------------------------- /src/OutCode.EscapeTeams.ObjectRepository.Hangfire/ObjectRepositoryMonitoringApi.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using Hangfire.Annotations; 5 | using Hangfire.Common; 6 | using Hangfire.States; 7 | using Hangfire.Storage; 8 | using Hangfire.Storage.Monitoring; 9 | using OutCode.EscapeTeams.ObjectRepository.Hangfire.Entities; 10 | 11 | namespace OutCode.EscapeTeams.ObjectRepository.Hangfire 12 | { 13 | internal class ObjectRepositoryMonitoringApi : IMonitoringApi 14 | { 15 | private readonly ObjectRepositoryBase _repository; 16 | private ObjectRepositoryStorage _storage; 17 | 18 | public ObjectRepositoryMonitoringApi([NotNull] ObjectRepositoryStorage storage) 19 | { 20 | _storage = storage; 21 | _repository = storage.ObjectRepository; 22 | } 23 | 24 | public long ScheduledCount() 25 | { 26 | return GetNumberOfJobsByStateName(ScheduledState.StateName); 27 | } 28 | 29 | public long EnqueuedCount(string queue) 30 | { 31 | var queueApi = _storage.MonitoringApi; 32 | var counters = queueApi.GetEnqueuedAndFetchedCount(queue); 33 | 34 | return counters.EnqueuedCount ?? 0; 35 | } 36 | 37 | public long FetchedCount(string queue) 38 | { 39 | var queueApi = _storage.MonitoringApi; 40 | var counters = queueApi.GetEnqueuedAndFetchedCount(queue); 41 | 42 | return counters.FetchedCount ?? 0; 43 | } 44 | 45 | public long FailedCount() 46 | { 47 | return GetNumberOfJobsByStateName(FailedState.StateName); 48 | } 49 | 50 | public long ProcessingCount() 51 | { 52 | return GetNumberOfJobsByStateName(ProcessingState.StateName); 53 | } 54 | 55 | public JobList ProcessingJobs(int @from, int count) 56 | { 57 | return GetJobs( 58 | from, count, 59 | ProcessingState.StateName, 60 | (sqlJob, job, stateData) => new ProcessingJobDto 61 | { 62 | Job = job, 63 | ServerId = stateData.ContainsKey("ServerId") ? stateData["ServerId"] : stateData["ServerName"], 64 | StartedAt = JobHelper.DeserializeDateTime(stateData["StartedAt"]), 65 | }); 66 | } 67 | 68 | public JobList ScheduledJobs(int @from, int count) 69 | { 70 | return GetJobs( 71 | from, count, 72 | ScheduledState.StateName, 73 | (sqlJob, job, stateData) => new ScheduledJobDto 74 | { 75 | Job = job, 76 | EnqueueAt = JobHelper.DeserializeDateTime(stateData["EnqueueAt"]), 77 | ScheduledAt = JobHelper.DeserializeDateTime(stateData["ScheduledAt"]) 78 | }); 79 | } 80 | 81 | public IDictionary SucceededByDatesCount() => GetTimelineStats("succeeded"); 82 | 83 | public IDictionary FailedByDatesCount() => GetTimelineStats("failed"); 84 | 85 | public IList Servers() 86 | { 87 | var servers = _repository.Set().ToList(); 88 | 89 | var result = new List(); 90 | 91 | foreach (var server in servers) 92 | { 93 | var data = JobHelper.FromJson(server.Data); 94 | result.Add(new ServerDto 95 | { 96 | Name = server.Name, 97 | Heartbeat = server.LastHeartbeat, 98 | Queues = data.Queues, 99 | StartedAt = data.StartedAt ?? DateTime.MinValue, 100 | WorkersCount = data.WorkerCount 101 | }); 102 | } 103 | 104 | return result; 105 | } 106 | 107 | public JobList FailedJobs(int @from, int count) 108 | { 109 | return GetJobs( 110 | @from, 111 | count, 112 | FailedState.StateName, 113 | (sqlJob, job, stateData) => new FailedJobDto 114 | { 115 | Job = job, 116 | Reason = sqlJob.State.Reason, 117 | ExceptionDetails = stateData["ExceptionDetails"], 118 | ExceptionMessage = stateData["ExceptionMessage"], 119 | ExceptionType = stateData["ExceptionType"], 120 | FailedAt = JobHelper.DeserializeNullableDateTime(stateData["FailedAt"]) 121 | }); 122 | } 123 | 124 | public JobList SucceededJobs(int @from, int count) 125 | { 126 | return GetJobs( 127 | from, 128 | count, 129 | SucceededState.StateName, 130 | (sqlJob, job, stateData) => new SucceededJobDto 131 | { 132 | Job = job, 133 | Result = stateData.ContainsKey("Result") ? stateData["Result"] : null, 134 | TotalDuration = stateData.ContainsKey("PerformanceDuration") && stateData.ContainsKey("Latency") 135 | ? (long?)long.Parse(stateData["PerformanceDuration"]) + (long?)long.Parse(stateData["Latency"]) 136 | : null, 137 | SucceededAt = JobHelper.DeserializeNullableDateTime(stateData["SucceededAt"]) 138 | }); 139 | } 140 | 141 | public JobList DeletedJobs(int @from, int count) 142 | { 143 | return GetJobs( 144 | from, 145 | count, 146 | DeletedState.StateName, 147 | (sqlJob, job, stateData) => new DeletedJobDto 148 | { 149 | Job = job, 150 | DeletedAt = JobHelper.DeserializeNullableDateTime(stateData.ContainsKey("DeletedAt") 151 | ? stateData["DeletedAt"] 152 | : null) 153 | }); 154 | } 155 | 156 | public IList Queues() 157 | { 158 | var monitoring = _storage.MonitoringApi; 159 | var queues = monitoring 160 | .GetQueues() 161 | .OrderBy(x => x) 162 | .ToArray(); 163 | 164 | var result = new List(queues.Length); 165 | 166 | foreach (var queue in queues) 167 | { 168 | var enqueuedJobIds = monitoring.GetEnqueuedJobIds(queue, 0, 5); 169 | var counters = monitoring.GetEnqueuedAndFetchedCount(queue); 170 | 171 | var firstJobs = EnqueuedJobs(enqueuedJobIds); 172 | 173 | result.Add(new QueueWithTopEnqueuedJobsDto 174 | { 175 | Name = queue, 176 | Length = counters.EnqueuedCount ?? 0, 177 | Fetched = counters.FetchedCount, 178 | FirstJobs = firstJobs 179 | }); 180 | } 181 | 182 | return result; 183 | } 184 | 185 | public JobList EnqueuedJobs(string queue, int @from, int perPage) 186 | { 187 | var queueApi = _storage.MonitoringApi; 188 | var enqueuedJobIds = queueApi.GetEnqueuedJobIds(queue, from, perPage); 189 | 190 | return EnqueuedJobs(enqueuedJobIds); 191 | } 192 | 193 | public JobList FetchedJobs(string queue, int @from, int perPage) 194 | { 195 | var queueApi = _storage.MonitoringApi; 196 | var fetchedJobIds = queueApi.GetFetchedJobIds(queue, from, perPage); 197 | 198 | var jobs = _repository.Set() 199 | .Where(v => fetchedJobIds.Contains(v.Id)) 200 | .ToList(); 201 | 202 | var result = new List>(jobs.Count); 203 | 204 | foreach (var job in jobs) 205 | { 206 | result.Add(new KeyValuePair( 207 | job.Id.ToString(), 208 | new FetchedJobDto 209 | { 210 | Job = DeserializeJob(job.InvocationData, job.Arguments), 211 | State = job.State?.Name 212 | })); 213 | } 214 | 215 | return new JobList(result); 216 | } 217 | 218 | public IDictionary HourlySucceededJobs() 219 | { 220 | return GetHourlyTimelineStats("succeeded"); 221 | } 222 | 223 | public IDictionary HourlyFailedJobs() 224 | { 225 | return GetHourlyTimelineStats("failed"); 226 | } 227 | 228 | public JobDetailsDto JobDetails(String jobIdString) 229 | { 230 | if (!Guid.TryParse(jobIdString, out var jobId)) 231 | return null; 232 | var job = _repository.Set().Find(jobId); 233 | if (job == null) 234 | return null; 235 | 236 | var parameters = _repository.Set() 237 | .Where(v => v.JobId == jobId).Aggregate(new Dictionary(), (a, b) => 238 | { 239 | a[b.Name] = b.Value; 240 | return a; 241 | }); 242 | var history = _repository.Set() 243 | .Where(v => v.JobId == jobId) 244 | .OrderByDescending(s => s.CreatedAt) 245 | .Select(x => new StateHistoryDto 246 | { 247 | StateName = x.Name, 248 | CreatedAt = x.CreatedAt, 249 | Reason = x.Reason, 250 | Data = new Dictionary( 251 | JobHelper.FromJson>(x.Data), 252 | StringComparer.OrdinalIgnoreCase), 253 | }) 254 | .ToList(); 255 | 256 | return new JobDetailsDto 257 | { 258 | CreatedAt = job.CreatedAt, 259 | ExpireAt = job.ExpireAt, 260 | Job = DeserializeJob(job.InvocationData, job.Arguments), 261 | History = history, 262 | Properties = parameters 263 | }; 264 | } 265 | 266 | public long SucceededListCount() => GetNumberOfJobsByStateName(SucceededState.StateName); 267 | 268 | public long DeletedListCount() => GetNumberOfJobsByStateName(DeletedState.StateName); 269 | 270 | public StatisticsDto GetStatistics() 271 | { 272 | var jobs = _repository.Set(); 273 | 274 | Func count = name => 275 | { 276 | var counters = _repository.Set(); 277 | return counters.Where(v => v.Key == name).Sum(s => s.Value); 278 | }; 279 | 280 | var stats = new StatisticsDto 281 | { 282 | Enqueued = jobs.Count(v=> v.State?.Name == EnqueuedState.StateName), 283 | Failed = jobs.Count(v=> v.State?.Name == FailedState.StateName), 284 | Processing = jobs.Count(v=> v.State?.Name == ProcessingState.StateName), 285 | Scheduled = jobs.Count(v=> v.State?.Name == ScheduledState.StateName), 286 | Servers = _repository.Set().Count(), 287 | Succeeded = count("stats:succeeded"), 288 | Deleted = count("stats:deleted"), 289 | Recurring = _repository.Set().Count(v => v.Key == "recurring-jobs"), 290 | Queues = _storage.MonitoringApi.GetQueues().Count() 291 | }; 292 | 293 | return stats; 294 | } 295 | 296 | private Dictionary GetHourlyTimelineStats(string type) 297 | { 298 | var endDate = DateTime.UtcNow; 299 | var dates = new List(); 300 | for (var i = 0; i < 24; i++) 301 | { 302 | dates.Add(endDate); 303 | endDate = endDate.AddHours(-1); 304 | } 305 | 306 | var keyMaps = dates.ToDictionary(x => $"stats:{type}:{x:yyyy-MM-dd-HH}", x => x); 307 | 308 | return GetTimelineStats(keyMaps); 309 | } 310 | 311 | private Dictionary GetTimelineStats(string type) 312 | { 313 | var endDate = DateTime.UtcNow.Date; 314 | var dates = new List(); 315 | for (var i = 0; i < 7; i++) 316 | { 317 | dates.Add(endDate); 318 | endDate = endDate.AddDays(-1); 319 | } 320 | 321 | var keyMaps = dates.ToDictionary(x => $"stats:{type}:{x.ToString("yyyy-MM-dd")}", x => x); 322 | 323 | return GetTimelineStats(keyMaps); 324 | } 325 | 326 | private Dictionary GetTimelineStats(IDictionary keyMaps) 327 | { 328 | var valuesMap = 329 | _repository.Set() 330 | .Where(v => keyMaps.Keys.Contains(v.Key)) 331 | .ToList() 332 | .Aggregate(new Dictionary(), (a, b) => 333 | { 334 | if (!a.ContainsKey(b.Key)) 335 | { 336 | a[b.Key] = b.Value; 337 | } 338 | else 339 | { 340 | a[b.Key] += b.Value; 341 | } 342 | 343 | return a; 344 | }); 345 | 346 | foreach (var key in keyMaps.Keys) 347 | { 348 | if (!valuesMap.ContainsKey(key)) valuesMap.Add(key, 0); 349 | } 350 | 351 | var result = new Dictionary(); 352 | for (var i = 0; i < keyMaps.Count; i++) 353 | { 354 | var value = valuesMap[keyMaps.ElementAt(i).Key]; 355 | result.Add(keyMaps.ElementAt(i).Value, value); 356 | } 357 | 358 | return result; 359 | } 360 | 361 | private JobList EnqueuedJobs(IEnumerable jobIds) 362 | { 363 | var jobs = _repository.Set() 364 | .Where(v => jobIds.Contains(v.Id)) 365 | .ToDictionary(v=>v.Id, v=>v); 366 | 367 | var sortedSqlJobs = jobIds 368 | .Select(v=>jobs[v]) 369 | .ToList(); 370 | 371 | return DeserializeJobs( 372 | sortedSqlJobs, 373 | (sqlJob, job, stateData) => new EnqueuedJobDto 374 | { 375 | Job = job, 376 | State = sqlJob.State?.Name, 377 | EnqueuedAt = sqlJob.State?.Name == EnqueuedState.StateName 378 | ? JobHelper.DeserializeNullableDateTime(stateData["EnqueuedAt"]) 379 | : null 380 | }); 381 | } 382 | 383 | private long GetNumberOfJobsByStateName(string stateName) 384 | { 385 | return _repository.Set().Count(v => v.State?.Name == stateName); 386 | } 387 | 388 | private static Job DeserializeJob(string invocationData, string arguments) 389 | { 390 | var data = JobHelper.FromJson(invocationData); 391 | data.Arguments = arguments; 392 | 393 | try 394 | { 395 | return data.Deserialize(); 396 | } 397 | catch (JobLoadException) 398 | { 399 | return null; 400 | } 401 | } 402 | 403 | private JobList GetJobs( 404 | int from, 405 | int count, 406 | string stateName, 407 | Func, TDto> selector) 408 | { 409 | var jobs = _repository.Set() 410 | .Where(s => s.State?.Name == stateName) 411 | .Skip(from) 412 | .Take(count) 413 | .ToList(); 414 | 415 | return DeserializeJobs(jobs, selector); 416 | } 417 | 418 | private static JobList DeserializeJobs(ICollection jobs, 419 | Func, TDto> selector) 420 | { 421 | var result = new List>(jobs.Count()); 422 | 423 | foreach (var job in jobs) 424 | { 425 | var dto = default(TDto); 426 | 427 | if (job.InvocationData != null) 428 | { 429 | var deserializedData = JobHelper.FromJson>(job.State?.Data); 430 | var stateData = deserializedData != null 431 | ? new Dictionary(deserializedData, StringComparer.OrdinalIgnoreCase) 432 | : null; 433 | 434 | dto = selector(job, DeserializeJob(job.InvocationData, job.Arguments), stateData); 435 | } 436 | 437 | result.Add(new KeyValuePair( 438 | job.Id.ToString(), dto)); 439 | } 440 | 441 | return new JobList(result); 442 | } 443 | } 444 | } 445 | --------------------------------------------------------------------------------