├── logo-large.png ├── ssd-saturation.png ├── .env ├── reset.ps1 ├── benchmark ├── mmq-post.lua ├── mmq-post-json.lua ├── service-nodejs │ ├── package.json │ ├── Dockerfile │ └── src │ │ ├── logger.js │ │ └── index.js ├── service-express │ ├── tslint.json │ ├── Dockerfile │ ├── package.json │ ├── src │ │ ├── logger.ts │ │ └── index.ts │ └── tsconfig.json ├── service-hapi │ ├── tslint.json │ ├── Dockerfile │ ├── package.json │ ├── src │ │ ├── logger.ts │ │ └── index.ts │ └── tsconfig.json ├── ARM64v8-atomic-op-fix.diff ├── Dockerfile ├── mmq-post-json-2.lua ├── install.sh ├── mmq-post-large.lua └── mmq-post-xml.lua ├── .gitignore ├── setup.ps1 ├── setup.sh ├── service ├── MinMQ.Service │ ├── Controllers.Dto │ │ ├── MessageRequestDto.cs │ │ ├── StatusResponseDto.cs │ │ ├── PeekResponseDto.cs │ │ └── ListResponseDto.cs │ ├── Repository │ │ ├── IMessage.cs │ │ ├── FindMessagesQuery.cs │ │ ├── IMimeTypeRepository.cs │ │ ├── IMessageRepository.cs │ │ ├── ICursorRepository.cs │ │ ├── IQueueRepository.cs │ │ ├── MimeTypeRepository.cs │ │ ├── QueueRepository.cs │ │ ├── CursorRepository.cs │ │ └── MessageRepository.cs │ ├── Configuration │ │ └── MinMQConfiguration.cs │ ├── Entities │ │ ├── MimeType.cs │ │ ├── Queue.cs │ │ ├── DebugCursor.cs │ │ ├── Cursor.cs │ │ └── Message.cs │ ├── Models │ │ ├── MessagesContext.cs │ │ ├── Message.cs │ │ └── MessageQueueContext.cs │ ├── appsettings.Development.json │ ├── Faster │ │ ├── IFasterWriter.cs │ │ ├── MimeTypeTable.cs │ │ ├── FasterHostedServiceCommitState.cs │ │ ├── FasterLogWriterExtensions.cs │ │ ├── FasterHostedServiceCommit.cs │ │ ├── FasterHostedServiceMoveData.cs │ │ └── FasterOps.cs │ ├── Migrations │ │ ├── 20191204211704_Minor changes.cs │ │ ├── 20191204211817_Add MimeType.cs │ │ ├── 20191204215508_Add a lot more mime types.cs │ │ ├── 20191204203610_Initial database contruction.cs │ │ ├── MessageQueueContextModelSnapshot.cs │ │ ├── 20191204211704_Minor changes.Designer.cs │ │ ├── 20191204211817_Add MimeType.Designer.cs │ │ ├── 20191204215508_Add a lot more mime types.Designer.cs │ │ └── 20191204203610_Initial database contruction.Designer.cs │ ├── appsettings.Docker.json │ ├── appsettings.json │ ├── AsyncEnumerable.cs │ ├── Properties │ │ └── launchSettings.json │ ├── Hashing.cs │ ├── HttpRequestHandlers │ │ └── FasterHttpHandler.cs │ ├── MinMQ.Service.csproj.user │ ├── Filters │ │ └── CustomExceptionFilterAttribute.cs │ ├── Program.cs │ ├── Controllers │ │ ├── ControllerExtensions.cs │ │ └── DefaultController.cs │ ├── Startup.cs │ └── MinMQ.Service.csproj ├── .dockerignore ├── MinMQ.ScanConsole │ ├── MinMQ.ScanConsole.csproj │ ├── Program.cs │ ├── PrintSystemClockTimer.cs │ └── FasterLogReader.cs ├── Dockerfile ├── MinMQ.BenchmarkConsole │ ├── HttpCustomHeaderHandler.cs │ ├── GeneratorBase.cs │ ├── RandomExtensions.cs │ ├── BenchmarkHostedService.cs │ ├── Program.cs │ ├── MinMQ.BenchmarkConsole.csproj │ ├── JsonGenerator.cs │ ├── XmlGenerator.cs │ ├── Words.cs │ └── Benchmarker.cs ├── MinMQ.sln └── StyleCop.ruleset ├── benchmark.ps1 ├── docs ├── ntiered.md ├── perf.md ├── design_decisions.md ├── ntiered-diagram.nomnoml ├── development_work.md ├── web-performance.md └── ntiered-diagram.svg ├── run.ps1 ├── LICENSE.md ├── .editorconfig ├── docker-compose.yaml └── README.md /logo-large.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jmrnilsson/MinMQ/HEAD/logo-large.png -------------------------------------------------------------------------------- /ssd-saturation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jmrnilsson/MinMQ/HEAD/ssd-saturation.png -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | POSTGRES_USER=5a4ba2e9-6c44-49dd-bc6c-b9ea2b901114 2 | POSTGRES_PASSWORD=effe908d-158d-47c5-a2eb-ad6814ce6083 -------------------------------------------------------------------------------- /reset.ps1: -------------------------------------------------------------------------------- 1 | Set-Location .\MinMQ 2 | docker-compose.exe down 3 | docker volume rm minmq_postgresdata 4 | docker-compose.exe up mmq-db 5 | -------------------------------------------------------------------------------- /benchmark/mmq-post.lua: -------------------------------------------------------------------------------- 1 | wrk.method = "POST" 2 | wrk.body = "message=13371337" 3 | wrk.headers["Content-Type"] = "application/x-www-form-urlencoded" -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | obj 4 | bin 5 | .DS_Store 6 | .ionide 7 | .vs 8 | **/*.xcf 9 | logo-worker-2.png 10 | #.env 11 | artworks 12 | -------------------------------------------------------------------------------- /setup.ps1: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env pwsh 2 | docker volume create --name=fasterdbo --driver local --opt o=size=2200m --opt device="K://docker-hlog-1337h" --opt type=tmpfs -------------------------------------------------------------------------------- /benchmark/mmq-post-json.lua: -------------------------------------------------------------------------------- 1 | wrk.method = "POST" 2 | wrk.body = "{\"content\": \"Some story about things and serializers\"}" 3 | wrk.headers["Content-Type"] = "application/json" -------------------------------------------------------------------------------- /setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | mkdir -p /data 3 | docker volume create --name=fasterdbo --driver local --opt o=size=1200m --opt device="/data/docker-hlog" --opt type=tmpfs 4 | 5 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Controllers.Dto/MessageRequestDto.cs: -------------------------------------------------------------------------------- 1 | namespace MinMQ.Service.Controllers.Dto 2 | { 3 | public class MessageRequestDto 4 | { 5 | public string Content { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Controllers.Dto/StatusResponseDto.cs: -------------------------------------------------------------------------------- 1 | namespace MinMQ.Service.Controllers.Dto 2 | { 3 | public class StatusResponseDto 4 | { 5 | public string Text { get; set; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /benchmark.ps1: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env pwsh 2 | 3 | docker-compose build; 4 | docker-compose down; 5 | docker-compose up -d; 6 | docker-compose run mmq-benchmark -- status.sh 7 | docker-compose run mmq-benchmark -- post_message.sh -------------------------------------------------------------------------------- /benchmark/service-nodejs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "license": "", 3 | "scripts": { 4 | "start": "node ./src/index.js" 5 | }, 6 | "dependencies": { 7 | "winston": "^3.2.1" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Repository/IMessage.cs: -------------------------------------------------------------------------------- 1 | namespace MinMq.Service.Repository 2 | { 3 | public interface IMessage 4 | { 5 | public long ReferenceId { get; } 6 | public string HashCode { get; } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /benchmark/service-express/tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "tslint-config-airbnb", 3 | "rules": { 4 | "import-name": false, 5 | "max-line-length": [true, 120], 6 | "align": false, 7 | "no-console": true 8 | } 9 | } -------------------------------------------------------------------------------- /benchmark/service-hapi/tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "tslint-config-airbnb", 3 | "rules": { 4 | "import-name": false, 5 | "max-line-length": [true, 120], 6 | "align": false, 7 | "no-console": true 8 | } 9 | } -------------------------------------------------------------------------------- /service/MinMQ.Service/Controllers.Dto/PeekResponseDto.cs: -------------------------------------------------------------------------------- 1 | namespace MinMQ.Service.Controllers.Dto 2 | { 3 | public class PeekResponseDto 4 | { 5 | public long CurrentAddress { get; set; } 6 | public long NextAddress { get; set; } 7 | public string Content { get; set; } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /docs/ntiered.md: -------------------------------------------------------------------------------- 1 | ## N-tiered queue provisioning 2 | _Note: Work in progress._ 3 | 4 | This serves as an outline for a distributed approach of the FASTER Log rather than a KV-store in the end. It requires message metadata to be included prepended to the written log message. 5 | 6 | ![ntiered-diagram.svg](ntiered-diagram.svg) -------------------------------------------------------------------------------- /service/MinMQ.Service/Configuration/MinMQConfiguration.cs: -------------------------------------------------------------------------------- 1 | namespace MinMQ.Service.Configuration 2 | { 3 | public class MinMQConfiguration 4 | { 5 | public string FasterDevice { get; set; } 6 | public int ScanFlushSize { get; set; } 7 | public string ConnectionStringPostgres { get; set; } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /run.ps1: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env pwsh 2 | 3 | $workingPath = $PWD; 4 | 5 | try 6 | { 7 | Set-Location .\service\MinMQ.Service\ 8 | dotnet run 9 | 10 | Set-Location $workingPath 11 | 12 | Set-Location .\service\BenchmarkConsole\ 13 | dotnet run 14 | } 15 | finally 16 | { 17 | Set-Location $workingPath; 18 | } 19 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Entities/MimeType.cs: -------------------------------------------------------------------------------- 1 | namespace MinMq.Service.Entities 2 | { 3 | public class MimeType 4 | { 5 | public MimeType(short mimeTypeId, string expression) 6 | { 7 | MimeTypeId = mimeTypeId; 8 | Expression = expression; 9 | } 10 | 11 | public short MimeTypeId { get; } 12 | public string Expression { get; set; } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Models/MessagesContext.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore; 2 | 3 | namespace MinMQ.Service.Models 4 | { 5 | public class MessagesContext : DbContext 6 | { 7 | public MessagesContext(DbContextOptions options) 8 | : base(options) 9 | { 10 | } 11 | 12 | public DbSet Messages { get; set; } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /service/MinMQ.Service/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "System": "Information", 6 | "Microsoft": "Error", 7 | "Microsoft.Hosting.Lifetime": "Error" 8 | } 9 | }, 10 | "FasterDevice": "K:\\hlog.log", 11 | "ScanFlushSize": "500" 12 | } 13 | -------------------------------------------------------------------------------- /service/.dockerignore: -------------------------------------------------------------------------------- 1 | **/.dockerignore 2 | **/.env 3 | **/.git 4 | **/.gitignore 5 | **/.vs 6 | **/.vscode 7 | **/*.*proj.user 8 | **/azds.yaml 9 | **/charts 10 | **/bin 11 | **/obj 12 | **/Dockerfile 13 | **/Dockerfile.develop 14 | **/docker-compose.yml 15 | **/docker-compose.*.yml 16 | **/*.dbmdl 17 | **/*.jfm 18 | **/secrets.dev.yaml 19 | **/values.dev.yaml 20 | **/.toolstarget 21 | .vs/ 22 | */.vs -------------------------------------------------------------------------------- /service/MinMQ.Service/Repository/FindMessagesQuery.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using MinMq.Service.Entities; 3 | 4 | namespace MinMq.Service.Repository 5 | { 6 | public class FindMessagesQuery 7 | { 8 | public FindMessagesQuery(List messages) 9 | { 10 | Messages = messages; 11 | } 12 | 13 | public List Messages { get; } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Repository/IMimeTypeRepository.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using MinMq.Service.Entities; 4 | using Optional; 5 | 6 | namespace MinMq.Service.Repository 7 | { 8 | public interface IMimeTypeRepository : IDisposable 9 | { 10 | Task Add(MimeType mimeType); 11 | Task> Find(string expression); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Repository/IMessageRepository.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Threading.Tasks; 4 | using MinMq.Service.Entities; 5 | using Optional; 6 | 7 | namespace MinMq.Service.Repository 8 | { 9 | public interface IMessageRepository : IDisposable 10 | { 11 | public Task> AddRange(List messages); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Entities/Queue.cs: -------------------------------------------------------------------------------- 1 | namespace MinMq.Service.Entities 2 | { 3 | public class Queue 4 | { 5 | public Queue(short byteKey, string name) 6 | { 7 | QueueId = byteKey; 8 | Name = name; 9 | } 10 | 11 | public Queue(string name) 12 | { 13 | QueueId = null; 14 | Name = name; 15 | } 16 | 17 | public short? QueueId { get; } 18 | public string Name { get; set; } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Entities/DebugCursor.cs: -------------------------------------------------------------------------------- 1 | namespace MinMq.Service.Entities 2 | { 3 | public class DebugCursor : Cursor 4 | { 5 | private int iteration = 0; 6 | 7 | public DebugCursor(int id, long nextAddress) 8 | : base(id, nextAddress) 9 | { 10 | } 11 | 12 | public int Iteration => iteration; 13 | 14 | public void Increment() 15 | { 16 | iteration++; 17 | } 18 | 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /benchmark/service-nodejs/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:11-slim 2 | 3 | ENV DEBIAN_FRONTEND noninteractive 4 | 5 | RUN apt-get update 6 | RUN apt-get install -y --no-install-recommends ca-certificates 7 | RUN rm -rf /var/lib/apt/lists/* 8 | RUN apt-get purge --auto-remove -y curl 9 | RUN rm -rf /src/*.deb 10 | RUN apt-get clean 11 | 12 | COPY . /app/ 13 | WORKDIR /app 14 | 15 | RUN npm install 16 | 17 | EXPOSE 8000 18 | 19 | CMD ["node", "./src/index.js"] -------------------------------------------------------------------------------- /service/MinMQ.Service/Controllers.Dto/ListResponseDto.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace MinMQ.Service.Controllers.Dto 4 | { 5 | public class ListResponseDto 6 | { 7 | public long FirstAddress { get; set; } 8 | public long LastAddress { get; set; } 9 | public long NextAddress { get; set; } 10 | public List Addresses { get; set; } 11 | public List Contents { get; set; } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /docs/perf.md: -------------------------------------------------------------------------------- 1 | 2 | ## Some preliminary benchmarks 3 | 4 | Some ad-hoc performance comparisons have been made [comparing both different web servers](web-performance.md); Mostly those kinds that rely on Libuv. Moreover, benchmarks are mainly focused towards 5 | seeing commit-over-commit performance hits. Metrics in focus are really just _throughput_ and _latency_. Some 6 | unstructured logs of benchmarks runs can be found in [performance.md](performance.md). 7 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Faster/IFasterWriter.cs: -------------------------------------------------------------------------------- 1 | using System.Threading; 2 | using System.Threading.Tasks; 3 | 4 | namespace MinMQ.Service 5 | { 6 | public interface IFasterWriter 7 | { 8 | ValueTask CommitAsync(CancellationToken token = default); 9 | ValueTask WaitForCommitAsync(long untilAddress = 0, CancellationToken token = default); 10 | ValueTask EnqueueAsync(byte[] entry, CancellationToken token = default); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Repository/ICursorRepository.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using MinMq.Service.Entities; 4 | 5 | namespace MinMq.Service.Repository 6 | { 7 | public interface ICursorRepository : IDisposable 8 | { 9 | Task Add(Cursor cursor); 10 | Task Find(int cursorId); 11 | Task FindOr(int cursorId, Func> valueFactory); 12 | Task Update(Cursor cursor); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Migrations/20191204211704_Minor changes.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace MinMq.Service.Migrations 4 | { 5 | public partial class Minorchanges : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | } 10 | 11 | protected override void Down(MigrationBuilder migrationBuilder) 12 | { 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Repository/IQueueRepository.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using MinMq.Service.Entities; 4 | 5 | namespace MinMq.Service.Repository 6 | { 7 | public interface IQueueRepository : IDisposable 8 | { 9 | Task Find(string queueName); 10 | Task FindOr(string queueName, Func> valueFactory); 11 | Task Add(Queue queue); 12 | Task Update(Queue queue); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Faster/MimeTypeTable.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Threading.Tasks; 6 | 7 | namespace MinMq.Service.Faster 8 | { 9 | public class MimeTypeTable 10 | { 11 | private readonly Dictionary lookup; 12 | public MimeTypeTable() 13 | { 14 | lookup = new Dictionary(); 15 | lookup.Add(0, ""); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /benchmark/service-express/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:11-slim 2 | 3 | ENV DEBIAN_FRONTEND noninteractive 4 | 5 | RUN apt-get update 6 | RUN apt-get install -y --no-install-recommends ca-certificates 7 | RUN rm -rf /var/lib/apt/lists/* 8 | RUN apt-get purge --auto-remove -y curl 9 | RUN rm -rf /src/*.deb 10 | RUN apt-get clean 11 | 12 | COPY . /app/ 13 | WORKDIR /app 14 | 15 | RUN npm install 16 | RUN npm run build 17 | RUN rm -dr src 18 | 19 | EXPOSE 4000 20 | 21 | CMD ["node", "./dist/index.js"] -------------------------------------------------------------------------------- /benchmark/service-hapi/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:11-slim 2 | 3 | ENV DEBIAN_FRONTEND noninteractive 4 | 5 | RUN apt-get update 6 | RUN apt-get install -y --no-install-recommends ca-certificates 7 | RUN rm -rf /var/lib/apt/lists/* 8 | RUN apt-get purge --auto-remove -y curl 9 | RUN rm -rf /src/*.deb 10 | RUN apt-get clean 11 | 12 | COPY . /app/ 13 | WORKDIR /app 14 | 15 | RUN npm install 16 | RUN npm run build 17 | RUN rm -dr src 18 | 19 | EXPOSE 1000 20 | 21 | CMD ["node", "./dist/index.js"] 22 | -------------------------------------------------------------------------------- /benchmark/service-express/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "license": "", 3 | "scripts": { 4 | "start": "node ./dist/index.js", 5 | "build": "tsc" 6 | }, 7 | "dependencies": { 8 | "body-parser": "^1.19.0", 9 | "express": "^4.17.0", 10 | "typescript": "^3.8.3", 11 | "winston": "^3.2.1" 12 | }, 13 | "devDependencies": { 14 | "@types/express": "^4.17.11", 15 | "@types/node": "^12.20.4", 16 | "tslint": "^5.18.0" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /docs/design_decisions.md: -------------------------------------------------------------------------------- 1 | ## Libuv 2 | For the time beeing this version reverts back from Managed sockets to widely adopted Libuv-transport previously which 3 | was used in AspNetCore 1.0 and replaced and later on restored [restored](https://github.com/aspnet/KestrelHttpServer/issues/2104) 4 | in AspNet 2.1. Similarl to [this article](https://github.com/aspnet/KestrelHttpServer/issues/2104) it has been found 5 | that it performs significantly better in high-contention scenarios but slightly worse under balanced load. 6 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Models/Message.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.ComponentModel.DataAnnotations.Schema; 3 | using NodaTime; 4 | 5 | namespace MinMQ.Service.Models 6 | { 7 | public sealed class Message 8 | { 9 | public Message() 10 | { 11 | Instant = SystemClock.Instance.GetCurrentInstant(); 12 | } 13 | 14 | [DatabaseGenerated(DatabaseGeneratedOption.Identity)] 15 | public Guid Id { get; set; } 16 | public Instant Instant { get; set; } 17 | public string Content { get; set; } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /benchmark/service-hapi/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "license": "", 3 | "scripts": { 4 | "start": "node ./dist/index.js", 5 | "build": "tsc" 6 | }, 7 | "dependencies": { 8 | "@hapi/hapi": "^18.4.1", 9 | "@types/hapi__hapi": "^18.2.6", 10 | "body-parser": "^1.19.0", 11 | "typescript": "^3.8.3", 12 | "winston": "^3.2.1" 13 | }, 14 | "devDependencies": { 15 | "@types/express": "^4.17.6", 16 | "@types/node": "^12.12.35", 17 | "tslint": "^5.18.0" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /service/MinMQ.Service/appsettings.Docker.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "System": "Error", 6 | "Microsoft": "Error", 7 | "Microsoft.Hosting.Lifetime": "Error" 8 | } 9 | }, 10 | "FasterDevice": "/opt/faster/hlog.log", 11 | "ScanFlushSize": "250", 12 | "ConnectionStrings": { 13 | "MessageQueueContext": "Host=mmq-db;Database=mmq;Username=5a4ba2e9-6c44-49dd-bc6c-b9ea2b901114;Password=effe908d-158d-47c5-a2eb-ad6814ce6083" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /service/MinMQ.ScanConsole/MinMQ.ScanConsole.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp3.0 6 | Debug;Release;Troubleshoot 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /docs/ntiered-diagram.nomnoml: -------------------------------------------------------------------------------- 1 | [message_recieved]->[instance 1] 2 | [instance 1]->[queue] 3 | [queue]->[instance 2] 4 | [queue]->[instance 4] 5 | [queue]->[instance 3] 6 | [instance 2]->[mime-type] 7 | [mime-type]->[instance 5] 8 | [mime-type]->[instance 6] 9 | [instance 2]->[validate] 10 | [validate]->[instance 7] 11 | [validate]->[instance 8] 12 | [instance 6]->[e6] 13 | [instance 3]->[e3] 14 | [instance 4]->[e4] 15 | [instance 5]->[e5] 16 | [instance 7]->[e7] 17 | [instance 8]->[e8] 18 | -------------------------------------------------------------------------------- /service/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/dotnet/core/sdk AS build 2 | WORKDIR /src 3 | COPY ./MinMQ.Service . 4 | COPY ./StyleCop.ruleset ../ 5 | RUN dotnet build "MinMQ.Service.csproj" 6 | RUN dotnet publish "MinMQ.Service.csproj" -c Release -o /app 7 | 8 | FROM mcr.microsoft.com/dotnet/core/aspnet AS mmq 9 | # ENV ASPNETCORE_ENVIRONMENT Docker 10 | # ENV ASPNETCORE_URLS "http://0.0.0.0:9000" 11 | ENV ASPNETCORE_ENVIRONMENT "Docker" 12 | ENV ASPNETCORE_URLS "http://0.0.0.0:9000" 13 | 14 | WORKDIR /app 15 | EXPOSE 9000 16 | COPY --from=build /app . 17 | ENTRYPOINT ["dotnet", "MinMQ.Service.dll"] 18 | # ENTRYPOINT ["bash"] 19 | -------------------------------------------------------------------------------- /service/MinMQ.ScanConsole/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | 4 | namespace MinMQ.ScanConsole 5 | { 6 | public class Program 7 | { 8 | public static async Task Main(string[] args) 9 | { 10 | string devicePath = "K:\\hlog.log"; 11 | 12 | if (args.Length > 0) 13 | { 14 | Console.Error.WriteLine("Options not supported ATM"); 15 | } 16 | 17 | var timer = new PrintSystemClockTimer(); 18 | timer.StartTimer(); 19 | 20 | var reader = new FasterLogReader(); 21 | await reader.StartScan(devicePath); 22 | 23 | Console.WriteLine("Done!"); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /service/MinMQ.Service/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "System": "Information", 6 | "Microsoft": "Information", 7 | "Microsoft.Hosting.Lifetime": "Information" 8 | } 9 | }, 10 | "AllowedHosts": "*", 11 | "FasterDevice": "C:\\hlog.log", 12 | "LogCommitPollingEverySeconds": "3000", 13 | "ScanFlushSize": "50", 14 | "ConnectionStrings": { 15 | "MessageQueueContext": "Host=localhost;Database=mmq;Username=5a4ba2e9-6c44-49dd-bc6c-b9ea2b901114;Password=effe908d-158d-47c5-a2eb-ad6814ce6083" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /service/MinMQ.BenchmarkConsole/HttpCustomHeaderHandler.cs: -------------------------------------------------------------------------------- 1 | using System.Net.Http; 2 | using System.Threading; 3 | using System.Threading.Tasks; 4 | 5 | namespace MinMQ.BenchmarkConsole 6 | { 7 | public class HttpCustomHeaderHandler : DelegatingHandler 8 | { 9 | protected override Task SendAsync 10 | ( 11 | HttpRequestMessage request, 12 | CancellationToken cancellationToken 13 | ) 14 | { 15 | return base.SendAsync(request, cancellationToken).ContinueWith( 16 | (task) => 17 | { 18 | HttpResponseMessage response = task.Result; 19 | response.Headers.Add("X-Custom-Header", "Some value"); 20 | return response; 21 | } 22 | ); 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Migrations/20191204211817_Add MimeType.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace MinMq.Service.Migrations 4 | { 5 | public partial class AddMimeType : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.Sql("insert into \"tMimeTypes\" (\"Expression\", \"Added\", \"Changed\") values ('application/json', current_timestamp, current_timestamp), ('application/xml', current_timestamp, current_timestamp), ('text/xml', current_timestamp, current_timestamp)"); 10 | } 11 | 12 | protected override void Down(MigrationBuilder migrationBuilder) 13 | { 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Faster/FasterHostedServiceCommitState.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Logging; 2 | 3 | namespace MinMQ.Service.Faster 4 | { 5 | internal class FasterHostedServiceCommitState 6 | { 7 | public FasterHostedServiceCommitState(CommitAsyncDelegate commitAsync, ILogger logger, int loggingInterval, int periodMs) 8 | { 9 | CommitAsync = commitAsync; 10 | Logger = logger; 11 | LoggingInterval = loggingInterval; 12 | PeriodMs = periodMs; 13 | InvokationCount = 0; 14 | } 15 | 16 | public CommitAsyncDelegate CommitAsync { get; } 17 | public ILogger Logger { get; } 18 | public int LoggingInterval { get; } 19 | public int PeriodMs { get; } 20 | public int InvokationCount { get; set; } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /benchmark/ARM64v8-atomic-op-fix.diff: -------------------------------------------------------------------------------- 1 | diff --git a/src/zmalloc.c b/src/zmalloc.c 2 | index 094dd80..4dd084b 100644 3 | --- a/src/zmalloc.c 4 | +++ b/src/zmalloc.c 5 | @@ -45,13 +45,19 @@ void zlibc_free(void *ptr) { 6 | #include "zmalloc.h" 7 | #include "atomicvar.h" 8 | 9 | +#ifdef _LP64 10 | +#define ALIGMENT (16) 11 | +#else 12 | +#define ALIGMENT (8) 13 | +#endif 14 | +#define ROUND_UP(n,r) (((n + r - 1) / r ) * r) 15 | #ifdef HAVE_MALLOC_SIZE 16 | #define PREFIX_SIZE (0) 17 | #else 18 | #if defined(__sun) || defined(__sparc) || defined(__sparc__) 19 | -#define PREFIX_SIZE (sizeof(long long)) 20 | +#define PREFIX_SIZE (ROUND_UP(sizeof(long long), ALIGMENT)) 21 | #else 22 | -#define PREFIX_SIZE (sizeof(size_t)) 23 | +#define PREFIX_SIZE (ROUND_UP(sizeof(size_t), ALIGMENT)) 24 | #endif 25 | #endif 26 | 27 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Entities/Cursor.cs: -------------------------------------------------------------------------------- 1 | using Optional; 2 | 3 | namespace MinMq.Service.Entities 4 | { 5 | public class Cursor 6 | { 7 | // private int currentAddress; 8 | private long nextAddress; 9 | private int? id; 10 | 11 | public Cursor(int id, long nextAddress) 12 | { 13 | this.id = id; 14 | this.nextAddress = nextAddress; 15 | } 16 | public Cursor() 17 | { 18 | this.nextAddress = 0; 19 | } 20 | 21 | public int? Id => id; 22 | // public int CurrentAddress => id; 23 | public long NextAddress => nextAddress; 24 | 25 | public void Set(Option nextAddress) 26 | { 27 | nextAddress.MatchSome(next => this.nextAddress = next); 28 | } 29 | 30 | public void Set(long nextAddress) 31 | { 32 | this.nextAddress = nextAddress; 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /benchmark/service-nodejs/src/logger.js: -------------------------------------------------------------------------------- 1 | const winston = require('winston'); 2 | 3 | let _logger = null; 4 | 5 | (function () { 6 | if (_logger != null) return; 7 | const { combine, timestamp, printf, splat, label } = winston.format; 8 | 9 | const customFormat = printf(({ level, label, message, timestamp }) => { 10 | return `${level}: [${label}] ${timestamp} ${message}`; 11 | }); 12 | 13 | _logger = winston.createLogger({ 14 | level: process.env.LOGLEVEL || 'info', 15 | format: combine( 16 | splat(), 17 | label({ label: 'Service.Nodejs' }), 18 | timestamp(), 19 | customFormat, 20 | ), 21 | defaultMeta: { service: 'user-service' }, 22 | transports: [ 23 | new winston.transports.Console(), 24 | ], 25 | }); 26 | }()); 27 | 28 | module.exports = _logger; 29 | -------------------------------------------------------------------------------- /benchmark/service-hapi/src/logger.ts: -------------------------------------------------------------------------------- 1 | import winston from 'winston'; 2 | 3 | let _logger: winston.Logger | null = null; 4 | 5 | (function () { 6 | if (_logger != null) return; 7 | const { combine, timestamp, printf, splat, label } = winston.format; 8 | 9 | const customFormat = printf(({ level, label, message, timestamp }) => { 10 | return `${level}: [${label}] ${timestamp} ${message}`; 11 | }); 12 | 13 | _logger = winston.createLogger({ 14 | level: process.env.LOGLEVEL || 'info', 15 | format: combine( 16 | splat(), 17 | label({ label: 'Service.Hapi' }), 18 | timestamp(), 19 | customFormat, 20 | ), 21 | defaultMeta: { service: 'user-service' }, 22 | transports: [ 23 | new winston.transports.Console(), 24 | ], 25 | }); 26 | }()); 27 | 28 | export let logger: winston.Logger = _logger; -------------------------------------------------------------------------------- /benchmark/service-express/src/logger.ts: -------------------------------------------------------------------------------- 1 | import winston from 'winston'; 2 | 3 | let _logger: winston.Logger | null = null; 4 | 5 | (function () { 6 | if (_logger != null) return; 7 | const { combine, timestamp, printf, splat, label } = winston.format; 8 | 9 | const customFormat = printf(({ level, label, message, timestamp }) => { 10 | return `${level}: [${label}] ${timestamp} ${message}`; 11 | }); 12 | 13 | _logger = winston.createLogger({ 14 | level: process.env.LOGLEVEL || 'info', 15 | format: combine( 16 | splat(), 17 | label({ label: 'Service.Express' }), 18 | timestamp(), 19 | customFormat, 20 | ), 21 | defaultMeta: { service: 'user-service' }, 22 | transports: [ 23 | new winston.transports.Console(), 24 | ], 25 | }); 26 | }()); 27 | 28 | export let logger: winston.Logger = _logger; -------------------------------------------------------------------------------- /service/MinMQ.BenchmarkConsole/GeneratorBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | 5 | namespace MinMQ.BenchmarkConsole 6 | { 7 | public abstract class GeneratorBase 8 | { 9 | private readonly int n; 10 | 11 | public GeneratorBase(int n) 12 | { 13 | this.n = n; 14 | } 15 | 16 | protected Random Seed { get; set; } = new Random(); 17 | protected abstract T GenerateChild(IEnumerable innerChildren); 18 | public abstract string GenerateObject(); 19 | 20 | protected IEnumerable GenerateChildren(int depth) 21 | { 22 | if (depth < 10) 23 | { 24 | var count = Seed.Next(0, n); 25 | for (int i = 0; i < count; i++) 26 | { 27 | yield return GenerateChild(GenerateChildren(++depth)); 28 | } 29 | } 30 | } 31 | 32 | protected int NumberOfProperties() 33 | { 34 | return Seed.Next(0, 6); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /service/MinMQ.Service/AsyncEnumerable.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | 6 | namespace MinMq.Service 7 | { 8 | // TODO: Keep for now 9 | public class AsyncEnumerable 10 | { 11 | private async Task> ToListAsync(IAsyncEnumerable scan, Func valueFactory) 12 | { 13 | List result = new List(); 14 | 15 | await foreach (T value in scan) 16 | { 17 | result.Add(valueFactory(value)); 18 | } 19 | 20 | return result; 21 | } 22 | 23 | private async Task> ToDictionaryAsync(IAsyncEnumerable scan, Func keyFactory) 24 | { 25 | Dictionary result = new Dictionary(); 26 | 27 | await foreach (T value in scan) 28 | { 29 | result[keyFactory(value)] = value; 30 | } 31 | 32 | return result; 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "iisSettings": { 3 | "windowsAuthentication": false, 4 | "anonymousAuthentication": true, 5 | "iisExpress": { 6 | "applicationUrl": "http://localhost:9000", 7 | "sslPort": 0 8 | } 9 | }, 10 | "$schema": "http://json.schemastore.org/launchsettings.json", 11 | "profiles": { 12 | "IIS Express": { 13 | "commandName": "IISExpress", 14 | "launchBrowser": true, 15 | "launchUrl": "status", 16 | "environmentVariables": { 17 | "ASPNETCORE_ENVIRONMENT": "Development" 18 | } 19 | }, 20 | "Service_Kestrel": { 21 | "commandName": "Project", 22 | "launchBrowser": true, 23 | "launchUrl": "status", 24 | "environmentVariables": { 25 | "ASPNETCORE_ENVIRONMENT": "Development" 26 | }, 27 | "applicationUrl": "http://localhost:9001;http://localhost:9000" 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Faster/FasterLogWriterExtensions.cs: -------------------------------------------------------------------------------- 1 | using System.Threading; 2 | using System.Threading.Tasks; 3 | using FASTER.core; 4 | 5 | namespace MinMQ.Service 6 | { 7 | // TODO: Not used. Perhaps not great but I like the proxy/nearest neighbour approach rather than direct logger access. 8 | public static class FasterLogWriterExtensions 9 | { 10 | public static async ValueTask CommitAsync(this FasterLog logger, CancellationToken token = default) 11 | { 12 | await logger.CommitAsync(token); 13 | } 14 | 15 | public static async ValueTask WaitForCommitAsync(this FasterLog logger, long untilAddress = 0, CancellationToken token = default) 16 | { 17 | await logger.WaitForCommitAsync(untilAddress, token); 18 | } 19 | 20 | public static async ValueTask EnqueueAsync(this FasterLog logger, byte[] entry, CancellationToken token = default) 21 | { 22 | return await logger.EnqueueAsync(entry, token); 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Hashing.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Text; 3 | 4 | namespace MinMq.Service 5 | { 6 | public static class Hashing 7 | { 8 | /// 9 | /// Fowler-Noll-Vo 1 a in 64 bit flavor as base 64 10 | /// 11 | /// Text to be hashed 12 | /// A base 64 encoded string 13 | public static string ToFnv1aHashInt64(this string text) 14 | { 15 | string Fnv1a(byte[] bytes_) 16 | { 17 | const ulong offset = 14695981039346656037; 18 | const ulong prime = 1099511628211; 19 | ulong hash = offset; 20 | 21 | for (var i = 0; i < bytes_.Length; i++) 22 | { 23 | unchecked 24 | { 25 | hash ^= bytes_[i]; 26 | hash *= prime; 27 | } 28 | } 29 | 30 | return Convert.ToBase64String(BitConverter.GetBytes(hash)); 31 | } 32 | 33 | byte[] bytes = Encoding.UTF8.GetBytes(text); 34 | return Fnv1a(bytes); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /service/MinMQ.ScanConsole/PrintSystemClockTimer.cs: -------------------------------------------------------------------------------- 1 | using NodaTime; 2 | using System; 3 | using System.Threading; 4 | 5 | namespace MinMQ.ScanConsole 6 | { 7 | public class PrintSystemClockTimer 8 | { 9 | Instant start; 10 | 11 | public PrintSystemClockTimer() 12 | { 13 | start = SystemClock.Instance.GetCurrentInstant(); 14 | } 15 | 16 | public void StartTimer() 17 | { 18 | var timer = new Timer(WriteProgress, start, TimeSpan.Zero, TimeSpan.FromMilliseconds(500)); 19 | } 20 | 21 | public void WriteProgress(object stateInfo) 22 | { 23 | if (stateInfo is Instant start) 24 | { 25 | var duration = SystemClock.Instance.GetCurrentInstant() - start; 26 | Console.WriteLine("Uptime={0}", duration.ToTimeSpan()); 27 | } 28 | else 29 | { 30 | DateTimeZone tz = DateTimeZoneProviders.Tzdb.GetSystemDefault(); 31 | Console.WriteLine("State unknown. Time=", SystemClock.Instance.GetCurrentInstant().InZone(tz).TimeOfDay); 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /service/MinMQ.Service/HttpRequestHandlers/FasterHttpHandler.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Text; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | using Microsoft.AspNetCore.Http; 7 | 8 | namespace MinMQ.Service.HttpRequestHandlers 9 | { 10 | public static class FasterHttpHandler 11 | { 12 | public static async Task HandleRequest(HttpContext context) 13 | { 14 | using (StreamReader reader = new StreamReader(context.Request.Body)) 15 | { 16 | var body = await reader.ReadToEndAsync(); 17 | var bytes = Encoding.UTF8.GetBytes(body); 18 | CancellationTokenSource cts = new CancellationTokenSource(); 19 | long address = await FasterOps.Instance.Value.EnqueueAsync(bytes, cts.Token); 20 | // await FasterOps.Instance.Value.CommitAsync(cts.Token); 21 | await FasterOps.Instance.Value.WaitForCommitAsync(address, cts.Token); 22 | context.Response.StatusCode = 201; 23 | await context.Response.WriteAsync("Created"); 24 | } 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /benchmark/service-nodejs/src/index.js: -------------------------------------------------------------------------------- 1 | const http = require('http'); 2 | const logger = require('./logger.js'); 3 | const port = 8000 4 | 5 | const requestHandler = (req, res) => { 6 | const url = req.url || ""; 7 | if (url.match(/^\/status\/?$/)) { 8 | res.writeHead(200); 9 | return res.end('{text: "ok"}'); 10 | } 11 | res.writeHead(404); 12 | return res.end('{error: 404}'); 13 | 14 | } 15 | 16 | const server = http.createServer(requestHandler) 17 | 18 | server.listen(port, (err) => { 19 | if (err) { 20 | return logger.error('Something bad happened', err) 21 | } 22 | 23 | logger.info('Starting service. Port=%s', port); 24 | }); 25 | 26 | function shutdown(reason) { 27 | return () => { 28 | logger.on('finish', () => { process.exit(3); }); 29 | logger.info('Process exiting. Reason=%s', reason); 30 | logger.end(); 31 | setTimeout(() => process.exit(2), 2000); 32 | }; 33 | } 34 | 35 | process.on('SIGHUP', shutdown('SIGHUP')); 36 | process.on('SIGINT', shutdown('SIGINT')); 37 | process.on('SIGTERM', shutdown('SIGTERM')); 38 | process.on('uncaughtException', e => shutdown(String(e))); 39 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Entities/Message.cs: -------------------------------------------------------------------------------- 1 | namespace MinMq.Service.Entities 2 | { 3 | public class Message 4 | { 5 | public Message(string content, long referenceId, long nextReferenceId, short mimeTypeId, short queueId) 6 | { 7 | Content = content; 8 | ReferenceId = referenceId; 9 | NextReferenceId = nextReferenceId; 10 | MimeTypeId = mimeTypeId; 11 | QueueId = queueId; 12 | HashCode = content.ToFnv1aHashInt64(); 13 | } 14 | 15 | public Message(string content, long referenceId, long nextReferenceId, string hashCode, short mimeTypeId, short queueId) 16 | { 17 | Content = content; 18 | ReferenceId = referenceId; 19 | NextReferenceId = nextReferenceId; 20 | MimeTypeId = mimeTypeId; 21 | QueueId = queueId; 22 | HashCode = hashCode; 23 | } 24 | 25 | public string Content { get; } 26 | public long ReferenceId { get; } 27 | public long NextReferenceId { get; } 28 | public short MimeTypeId { get; } 29 | // Work-around since queue should be probably be created before 30 | public short QueueId { get; set; } 31 | public string HashCode { get; set; } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 jmrnilsson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /benchmark/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:focal-20210217 AS build 2 | 3 | ENV DEBIAN_FRONTEND noninteractive 4 | 5 | RUN apt-get update 6 | 7 | RUN apt-get install -y --no-install-recommends \ 8 | ca-certificates \ 9 | git \ 10 | build-essential \ 11 | wget \ 12 | unzip 13 | 14 | WORKDIR /source 15 | # RUN git clone https://github.com/wg/wrk.git app/wrk 16 | RUN wget https://github.com/wg/wrk/archive/master.zip 17 | # RUN wget https://github.com/jmrnilsson/wrk/archive/master.zip 18 | RUN unzip master.zip 19 | COPY ARM64v8-atomic-op-fix.diff /source/wrk-master/ 20 | RUN (cd wrk-master; patch -p1 < ./ARM64v8-atomic-op-fix.diff; make) 21 | 22 | FROM ubuntu:focal-20210217 AS mmq-benchmark 23 | 24 | RUN apt-get update 25 | 26 | RUN apt-get install -y --no-install-recommends \ 27 | ca-certificates \ 28 | curl 29 | 30 | COPY --from=build /source/wrk-master/ /app/wrk/ 31 | COPY install.sh /opt/install.sh 32 | COPY *.lua /app/wrk/scripts/ 33 | RUN chmod +x /opt/install.sh 34 | 35 | WORKDIR /app/wrk 36 | 37 | RUN rm -rf /var/lib/apt/lists/* 38 | RUN rm -rf /src/*.deb 39 | RUN apt-get clean 40 | 41 | RUN sh /opt/install.sh 2>&1 | tee -a "install_sh.log" 42 | 43 | ENTRYPOINT [ "bash" ] 44 | -------------------------------------------------------------------------------- /service/MinMQ.Service/MinMQ.Service.csproj.user: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ApiControllerEmptyScaffolder 5 | root/Controller 6 | 600 7 | True 8 | False 9 | True 10 | 11 | False 12 | Service_Kestrel 13 | 14 | 15 | ProjectDebugger 16 | 17 | 18 | ProjectDebugger 19 | 20 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Filters/CustomExceptionFilterAttribute.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using Microsoft.AspNetCore.Mvc.Filters; 4 | using Microsoft.AspNetCore.Server.Kestrel.Core; 5 | using Microsoft.Extensions.Logging; 6 | 7 | namespace MinMQ.Service.Filters 8 | { 9 | public class CustomExceptionFilterAttribute : ExceptionFilterAttribute 10 | { 11 | private readonly CustomExceptionHandler handler; 12 | 13 | public CustomExceptionFilterAttribute(CustomExceptionHandler handler) 14 | { 15 | this.handler = handler; 16 | } 17 | 18 | public override void OnException(ExceptionContext context) 19 | { 20 | handler.ActOn(context.Exception); 21 | } 22 | 23 | public async override Task OnExceptionAsync(ExceptionContext context) 24 | { 25 | OnException(context); 26 | await Task.CompletedTask; 27 | } 28 | } 29 | 30 | public class CustomExceptionHandler 31 | { 32 | private readonly ILogger logger; 33 | 34 | public CustomExceptionHandler(ILogger logger) 35 | { 36 | this.logger = logger; 37 | } 38 | 39 | public void ActOn(Exception exception) 40 | { 41 | if (exception is BadHttpRequestException) 42 | { 43 | logger.LogInformation("A suspected thread starvation exceptions occured."); 44 | return; 45 | } 46 | logger.LogError("Unhandled error={0}", exception); 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # https://editorconfig.org/ 2 | # https://docs.microsoft.com/en-us/visualstudio/ide/create-portable-custom-editor-options?view=vs-2017 3 | # https://www.hanselman.com/blog/TabsVsSpacesAPeacefulResolutionWithEditorConfigInVisualStudioPlusNETExtensions.aspx 4 | 5 | # Supported things in vs pro 2017 6 | # - indent_style 7 | # - indent_size 8 | # - tab_width 9 | # - end_of_line 10 | # - charset 11 | # - trim_trailing_whitespace 12 | # - insert_final_newline 13 | # - root 14 | 15 | [*] 16 | max_line_length = 120 17 | 18 | [Dockerfile] 19 | end_of_line = lf 20 | indent_style = space 21 | indent_size = 4 22 | trim_trailing_whitespace = true 23 | 24 | [*.cs] 25 | end_of_line = crlf 26 | indent_style = tab 27 | indent_size = 4 28 | insert_final_newline = true 29 | trim_trailing_whitespace = true 30 | charset = utf-8-bom 31 | 32 | [*.ts] 33 | end_of_line = lf 34 | indent_style = space 35 | indent_size = 2 36 | insert_final_newline = true 37 | trim_trailing_whitespace = true 38 | 39 | [*.json] 40 | end_of_line = crlf 41 | indent_style = space 42 | indent_size = 4 43 | insert_final_newline = true 44 | trim_trailing_whitespace = true 45 | 46 | [*.{yaml,yml}] 47 | end_of_line = lf 48 | indent_style = space 49 | indent_size = 2 50 | insert_final_newline = true 51 | trim_trailing_whitespace = true 52 | 53 | [*.config] 54 | end_of_line = crlf 55 | indent_style = tab 56 | indent_size = 4 57 | insert_final_newline = true 58 | 59 | [*.bash, *.sh] 60 | end_of_line = lf 61 | indent_style = space 62 | insert_final_newline = true 63 | trim_trailing_whitespace = true 64 | -------------------------------------------------------------------------------- /service/MinMQ.BenchmarkConsole/RandomExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace MinMQ.BenchmarkConsole 5 | { 6 | public static class RandomExtensions 7 | { 8 | /// 9 | /// Python inspired random choice 10 | /// 11 | /// Object or value 12 | /// Random 13 | /// Options 14 | /// Returns a random choice out of several options 15 | public static T Choice(this Random random, params T[] options) 16 | { 17 | var choice = random.Next(1, options.Length) - 1; 18 | return options[choice]; 19 | } 20 | 21 | /// 22 | /// Python inspired random choice for enums 23 | /// 24 | /// enum 25 | /// Random 26 | /// Returns one enum 27 | public static T Choice(this Random random) 28 | where T : struct, IConvertible 29 | { 30 | if (!typeof(T).IsEnum) throw new ArgumentException("Type must be an enumerated type"); 31 | 32 | List options = new List(); 33 | 34 | foreach (T option in (T[])Enum.GetValues(typeof(T))) 35 | { 36 | options.Add(option); 37 | } 38 | 39 | var choice = random.Next(1, options.Count) - 1; 40 | return options[choice]; 41 | } 42 | 43 | public static bool OneIn(this Random random, int count) 44 | { 45 | return random.Next(0, count) < 1; 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /benchmark/service-express/src/index.ts: -------------------------------------------------------------------------------- 1 | import express from 'express'; 2 | import bodyParser from 'body-parser'; 3 | import { logger } from './logger'; 4 | 5 | const config = { 6 | name: 'mmq-service-nodejs (express)', 7 | port: 4000, 8 | host: '0.0.0.0', 9 | }; 10 | 11 | const app = express(); 12 | 13 | app.use(bodyParser.json()); 14 | 15 | app.get('/status', (req, res) => { 16 | return res.status(200).json({ 17 | text: 'ok' 18 | }); 19 | }); 20 | 21 | // server maybe used closing down later 22 | const server = app.listen(config.port, config.host, async () => { 23 | const { name, host, port} = config; 24 | logger.info('Starting service. Name=%s Host=%s Port=%s', name, host, port); 25 | }); 26 | 27 | // Take it cool and shutdown with ease. 28 | // https://gist.github.com/jmrnilsson/2b2775e18207e52cf19d2544382f0943 29 | // https://stackoverflow.com/questions/18771707/how-to-flush-winston-logs 30 | // https://github.com/winstonjs/winston/issues/228 31 | // https://medium.com/@becintec/building-graceful-node-applications-in-docker-4d2cd4d5d392 32 | // --> https://github.com/winstonjs/winston/issues/1250 33 | function shutdown(reason: string) { 34 | return () => { 35 | logger.on('finish', () => { process.exit(3); }); 36 | logger.info('Process exiting. Reason=%s', reason); 37 | logger.end(); 38 | setTimeout(() => process.exit(2), 2000); 39 | }; 40 | } 41 | 42 | process.on('SIGHUP', shutdown('SIGHUP')); 43 | process.on('SIGINT', shutdown('SIGINT')); 44 | process.on('SIGTERM', shutdown('SIGTERM')); 45 | process.on('uncaughtException', e => shutdown(String(e))); -------------------------------------------------------------------------------- /benchmark/service-hapi/src/index.ts: -------------------------------------------------------------------------------- 1 | import Hapi from '@hapi/hapi'; 2 | import { logger } from './logger'; 3 | 4 | const config = { 5 | name: 'mmq-service-hapi', 6 | port: 1000, 7 | host: '0.0.0.0', 8 | // host: 'localhost' 9 | }; 10 | 11 | const init = async () => { 12 | const {port, name, host} = config; 13 | 14 | const server = new Hapi.Server({ 15 | port: port, 16 | host: host 17 | }); 18 | 19 | server.route({ 20 | method: 'GET', 21 | path:'/status', 22 | handler: (request, h) => { 23 | 24 | return h.response('ok').code(200); 25 | } 26 | }); 27 | 28 | await server.start(); 29 | logger.info('Starting service. Name=%s Host=%s Port=%s', name, host, port); 30 | }; 31 | 32 | init(); 33 | 34 | // Take it cool and shutdown with ease. 35 | // https://gist.github.com/jmrnilsson/2b2775e18207e52cf19d2544382f0943 36 | // https://stackoverflow.com/questions/18771707/how-to-flush-winston-logs 37 | // https://github.com/winstonjs/winston/issues/228 38 | // https://medium.com/@becintec/building-graceful-node-applications-in-docker-4d2cd4d5d392 39 | // --> https://github.com/winstonjs/winston/issues/1250 40 | function shutdown(reason: string) { 41 | return () => { 42 | logger.on('finish', () => { process.exit(3); }); 43 | logger.info('Process exiting. Reason=%s', reason); 44 | logger.end(); 45 | setTimeout(() => process.exit(2), 2000); 46 | }; 47 | } 48 | 49 | process.on('SIGHUP', shutdown('SIGHUP')); 50 | process.on('SIGINT', shutdown('SIGINT')); 51 | process.on('SIGTERM', shutdown('SIGTERM')); 52 | process.on('uncaughtException', e => shutdown(String(e))); -------------------------------------------------------------------------------- /service/MinMQ.BenchmarkConsole/BenchmarkHostedService.cs: -------------------------------------------------------------------------------- 1 | using System.Net.Http; 2 | using System.Threading; 3 | using System.Threading.Tasks; 4 | using Microsoft.Extensions.Hosting; 5 | 6 | namespace MinMQ.BenchmarkConsole 7 | { 8 | public class BenchmarkHostedService : IHostedService 9 | { 10 | private readonly IHttpClientFactory httpClientFactory; 11 | private readonly IHostApplicationLifetime hostApplicationLifetime; 12 | 13 | public BenchmarkHostedService(IHttpClientFactory httpClientFactory, IHostApplicationLifetime hostApplicationLifetime) 14 | { 15 | this.httpClientFactory = httpClientFactory; 16 | this.hostApplicationLifetime = hostApplicationLifetime; 17 | } 18 | 19 | public async Task StartAsync(CancellationToken cancellationToken) 20 | { 21 | // hostApplicationLifetime.ApplicationStarted.Register(OnStarted); 22 | hostApplicationLifetime.ApplicationStopping.Register(OnStopping); 23 | // hostApplicationLifetime.ApplicationStopped.Register(OnStopped); 24 | 25 | var benchmarker = new Benchmarker(httpClientFactory, Program.NTree, Program.NumberOfObjects, cancellationToken); 26 | benchmarker.OnComplete += Program.OnCompletedEvent; 27 | await benchmarker.Start(); 28 | await StopAsync(cancellationToken); 29 | } 30 | 31 | public async Task StopAsync(CancellationToken cancellationToken) 32 | { 33 | await Task.CompletedTask; 34 | } 35 | 36 | // private void OnStarted() 37 | // { 38 | // } 39 | 40 | private void OnStopping() 41 | { 42 | } 43 | 44 | // private void OnStopped() 45 | // { 46 | // } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /service/MinMQ.BenchmarkConsole/Program.cs: -------------------------------------------------------------------------------- 1 | using System.Threading; 2 | using System.Threading.Tasks; 3 | using Microsoft.Extensions.DependencyInjection; 4 | using Microsoft.Extensions.Hosting; 5 | using Optional; 6 | using Serilog; 7 | 8 | namespace MinMQ.BenchmarkConsole 9 | { 10 | public delegate void OnCompleteDelegate(); 11 | 12 | public class Program 13 | { 14 | public static readonly int NTree = 5; // NTree = 2 == binary tree 15 | private static readonly CancellationTokenSource CancellationTokenSource = new CancellationTokenSource(); 16 | public static int NumberOfObjects { get; set; } = 1000; 17 | 18 | public static async Task Main(string[] args) 19 | { 20 | ParseArguments(args).MatchSome(value => NumberOfObjects = value); 21 | 22 | Log.Logger = new LoggerConfiguration() 23 | .MinimumLevel.Information() 24 | .WriteTo.Console() 25 | .CreateLogger(); 26 | 27 | var builder = new HostBuilder() 28 | .ConfigureServices((hostContext, services) => 29 | { 30 | services.AddHttpClient(); 31 | services.AddHostedService(); 32 | }); 33 | 34 | await builder.RunConsoleAsync(CancellationTokenSource.Token); 35 | } 36 | 37 | public static void OnCompletedEvent() 38 | { 39 | CancellationTokenSource.Cancel(); 40 | } 41 | 42 | private static Option ParseArguments(string[] args) 43 | { 44 | if (args.Length == 1 && int.TryParse(args[0], out int numberOfObjects_)) 45 | { 46 | return numberOfObjects_.Some(); 47 | } 48 | 49 | return Option.None(); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /service/MinMQ.BenchmarkConsole/MinMQ.BenchmarkConsole.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | ..\StyleCop.ruleset 6 | Debug;Release;Troubleshoot 7 | 8 | 9 | 10 | Exe 11 | netcoreapp3.0 12 | 13 | 14 | 15 | latest 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | all 34 | runtime; build; native; contentfiles; analyzers; buildtransitive 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /docs/development_work.md: -------------------------------------------------------------------------------- 1 | # Development work details 2 | ## Setup database 3 | Start the databaste with 4 | 5 | docker-compose up mmq-db 6 | 7 | Start Visual Studio since the dotnet core tools from 2.* doesn't work any more or [documenation](https://docs.microsoft.com/en-us/ef/core/get-started/?tabs=netcore-cli) is not up-to-date. 8 | 9 | Tools > Nuget Package Mannager 10 | 11 | Update-Database -Context MessageQueueContext 12 | 13 | ## Benchmarking 14 | The benchmarking solution currently makes comparisons agains nodejs's http module and nodejs express. Moreover, FASTER 15 | is compared agains in-memory databases. 16 | 17 | ### How to benchmark 18 | With Docker-compose. `Sudo` is system dependant doesn't have to apply. 19 | 20 | docker-compose build; docker-compose down; docker-compose up; 21 | sudo docker-compose run mmq-benchmarks -- status.sh 22 | 23 | Or, 24 | sudo docker-compose run mmq-benchmarks -- post_message.sh 25 | 26 | Or just enter benchmark container, 27 | 28 | docker-compose run mmq-benchmark -- 29 | 30 | Checkout out the Nodejs expression server 31 | 32 | curl -X GET http://localhost:4000/status 33 | 34 | Check the log 35 | 36 | curl -X GET http://localhost:9000/list 37 | 38 | 39 | Troubleshoot with curl 40 | 41 | curl -H "Accept: application/json" -H "Content-type: application/json" -X POST -d '{"id":"asdad"}' http://localhost:9000/faster --trace-ascii /dev/stdout 42 | 43 | ## Links 44 | - https://docs.microsoft.com/en-us/aspnet/core/fundamentals/middleware/write?view=aspnetcore-3.0#per-request-middleware-dependencies 45 | - https://docs.microsoft.com/en-us/aspnet/core/fundamentals/dependency-injection?view=aspnetcore-3.0 -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | # mmq-db: 4 | # image: mongo:4.2.1-bionic 5 | # ports: 6 | # - "27017:27017" 7 | # volumes: 8 | # - mongodata:/data/db 9 | mmq-service: 10 | build: ./service 11 | ports: 12 | - "9000:9000" 13 | environment: 14 | - ASPNETCORE_ENVIRONMENT=Docker 15 | - ASPNETCORE_URLS=http://0.0.0.0:9000 16 | volumes: 17 | - fasterdbo:/opt/faster 18 | # shm_size: "2gb" 19 | links: 20 | - "mmq-db" 21 | mmq-service-express: 22 | build: ./benchmark/service-express 23 | ports: 24 | - "4000:4000" 25 | mmq-service-hapi: 26 | build: ./benchmark/service-hapi 27 | ports: 28 | - "1000:1000" 29 | mmq-service-nodejs: 30 | build: ./benchmark/service-nodejs 31 | # links: 32 | # - "mmq-db:database" 33 | ports: 34 | - "8000:8000" 35 | mmq-benchmarks: 36 | build: ./benchmark 37 | links: 38 | - "mmq-service" 39 | - "mmq-service-nodejs" 40 | - "mmq-service-express" 41 | mmq-db: 42 | # https://docs.docker.com/compose/environment-variables/ 43 | # https://hub.docker.com/_/postgres 44 | image: postgres:12.1 45 | restart: always 46 | ports: 47 | - "5432:5432" 48 | volumes: 49 | - postgresdata:/var/lib/postgresql/data 50 | env_file: 51 | - .env 52 | environment: 53 | # - PGDATA=/var/lib/postgresql/data/pgdata 54 | - POSTGRES_DB=mmq 55 | volumes: 56 | # mongodata: {} 57 | fasterdbo: 58 | # DOESN'T WORK FROM INSIDE DOCKER-COMPOSE TO ALLOCATE BIG VOLUME 59 | # driver: local 60 | # driver_opts: 61 | # type: 62 | # o: size=1200m 63 | # device: K://docker-faster-hlog 64 | external: true 65 | postgresdata: {} 66 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Repository/MimeTypeRepository.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using System.Threading.Tasks; 3 | using MinMq.Service.Entities; 4 | using MinMq.Service.Models; 5 | using NodaTime; 6 | using Optional; 7 | 8 | namespace MinMq.Service.Repository 9 | { 10 | public class MimeTypeRepository : IMimeTypeRepository 11 | { 12 | private readonly MessageQueueContext messageQueueContext; 13 | 14 | public MimeTypeRepository(MessageQueueContext messageQueueContext) 15 | { 16 | this.messageQueueContext = messageQueueContext; 17 | } 18 | 19 | public async Task Add(MimeType mimeType) 20 | { 21 | var mimeTypeDo = await messageQueueContext.tMimeTypes.SingleOrDefaultAsync(q => q.Expression == mimeType.Expression); 22 | 23 | var now = SystemClock.Instance.GetCurrentInstant().InUtc().ToDateTimeUtc(); 24 | 25 | if (mimeTypeDo != null) 26 | { 27 | mimeTypeDo.Changed = now; 28 | await messageQueueContext.SaveChangesAsync(); 29 | return mimeTypeDo.MimeTypeId; 30 | } 31 | 32 | mimeTypeDo = new tMimeType 33 | { 34 | Expression = mimeType.Expression, 35 | Changed = now, 36 | Added = now 37 | }; 38 | 39 | await messageQueueContext.AddAsync(mimeTypeDo); 40 | await messageQueueContext.SaveChangesAsync(); 41 | mimeTypeDo = await messageQueueContext.tMimeTypes.SingleOrDefaultAsync(q => q.Expression == mimeType.Expression); 42 | return mimeTypeDo.MimeTypeId; 43 | } 44 | 45 | public async Task> Find(string expression) 46 | { 47 | var mimeType = (await messageQueueContext.tMimeTypes.SingleOrDefaultAsync(q => q.Expression == expression)).SomeNotNull(); 48 | 49 | return mimeType.Match 50 | ( 51 | some: mt => new MimeType(mt.MimeTypeId, mt.Expression).Some(), 52 | none: () => Option.None() 53 | ); 54 | } 55 | 56 | public void Dispose() 57 | { 58 | messageQueueContext?.Dispose(); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using Microsoft.AspNetCore.Hosting; 4 | using Microsoft.Extensions.DependencyInjection; 5 | using Microsoft.Extensions.Hosting; 6 | using Microsoft.Extensions.Logging; 7 | using MinMQ.Service.Faster; 8 | 9 | namespace MinMQ.Service 10 | { 11 | internal delegate void LogOrPrint(string message, params object[] args); 12 | 13 | public class Program 14 | { 15 | public static async Task Main(string[] args) 16 | { 17 | var host = CreateHostBuilder(args).Build(); 18 | 19 | using (var scope = host.Services.CreateScope()) 20 | { 21 | LogOrPrint logOrPrint; 22 | 23 | try 24 | { 25 | logOrPrint = new LogOrPrint(host.Services.GetRequiredService>().LogInformation); 26 | } 27 | catch 28 | { 29 | logOrPrint = new LogOrPrint(Console.WriteLine); 30 | } 31 | 32 | var urls = Environment.GetEnvironmentVariable("ASPNETCORE_URLS"); 33 | var env = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT"); 34 | logOrPrint("Starting service. Env='{0}' Urls='{1}'", env, urls); 35 | } 36 | 37 | await host.RunAsync(); 38 | } 39 | 40 | public static IHostBuilder CreateHostBuilder(string[] args) => 41 | Host.CreateDefaultBuilder(args) 42 | .ConfigureWebHostDefaults(webBuilder => 43 | { 44 | // Filter functions could be useful in future: 45 | // - https://docs.microsoft.com/en-us/aspnet/core/fundamentals/logging/?view=aspnetcore-3.0#filter-functions 46 | // - https://docs.microsoft.com/en-us/aspnet/core/fundamentals/host/hosted-services?view=aspnetcore-3.0&tabs=visual-studio 47 | webBuilder.UseLibuv(); 48 | webBuilder.UseStartup(); 49 | }).ConfigureServices(services => 50 | { 51 | services.AddHostedService(); 52 | services.AddHostedService(); 53 | }); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Controllers/ControllerExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using MinMQ.Service.Controllers.Dto; 6 | using Optional; 7 | 8 | namespace MinMQ.Service.Controllers 9 | { 10 | public delegate IAsyncEnumerable<(string, long, long)> ScanCollection(); 11 | 12 | public static class ControllerExtensions 13 | { 14 | public static PeekResponseDto ToPeekResponse(this (string, long, long) peek) 15 | { 16 | var (content, currentAddress, nextAddress) = peek; 17 | return new PeekResponseDto 18 | { 19 | Content = content, 20 | CurrentAddress = currentAddress, 21 | NextAddress = nextAddress, 22 | }; 23 | } 24 | 25 | /// 26 | /// Hopefully runtime optimizers will have a look at this.. 27 | /// 28 | /// Scanner items 29 | /// Returns an optional list of items 30 | public static async Task> ToListResponse(this IAsyncEnumerable<(string, long, long)> scan) 31 | { 32 | long min = long.MaxValue; 33 | long max = long.MinValue; 34 | long next = long.MinValue; 35 | List addresses = new List(); 36 | List contents = new List(); 37 | 38 | await foreach (var item in scan) 39 | { 40 | var (content, currentAddress, nextAddress) = item; 41 | min = Math.Min(min, currentAddress); 42 | max = Math.Max(max, currentAddress); 43 | next = Math.Max(next, nextAddress); 44 | addresses.Add(currentAddress); 45 | contents.Add(content); 46 | } 47 | 48 | if (addresses.Count() < 1) 49 | { 50 | return Option.None(); 51 | } 52 | 53 | return new ListResponseDto 54 | { 55 | FirstAddress = min, 56 | LastAddress = max, 57 | NextAddress = next, 58 | Addresses = addresses, 59 | Contents = contents 60 | }.Some(); 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Repository/QueueRepository.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using MinMq.Service.Entities; 6 | using MinMq.Service.Models; 7 | using NodaTime; 8 | 9 | namespace MinMq.Service.Repository 10 | { 11 | public class QueueRepository : IQueueRepository 12 | { 13 | private readonly MessageQueueContext messageQueueContext; 14 | 15 | public QueueRepository(MessageQueueContext messageQueueContext) 16 | { 17 | this.messageQueueContext = messageQueueContext; 18 | } 19 | 20 | public async Task Update(Queue queue) 21 | { 22 | var now = SystemClock.Instance.GetCurrentInstant().InUtc().ToDateTimeUtc(); 23 | 24 | tQueue queueDo = await messageQueueContext.tQueues.SingleOrDefaultAsync(q => q.Name == queue.Name); 25 | queueDo.Changed = now; 26 | await messageQueueContext.SaveChangesAsync(); 27 | return queueDo.QueueId; 28 | } 29 | 30 | public async Task Add(Queue queue) 31 | { 32 | var now = SystemClock.Instance.GetCurrentInstant().InUtc().ToDateTimeUtc(); 33 | 34 | tQueue queue_ = new tQueue 35 | { 36 | Name = queue.Name, 37 | Changed = now, 38 | Added = now 39 | }; 40 | 41 | await messageQueueContext.AddAsync(queue_); 42 | await messageQueueContext.SaveChangesAsync(); 43 | return queue_.QueueId; 44 | } 45 | 46 | public void Dispose() 47 | { 48 | messageQueueContext?.Dispose(); 49 | } 50 | 51 | public async Task Find(string queueName) 52 | { 53 | return await 54 | ( 55 | from q in (IAsyncEnumerable)messageQueueContext.tQueues 56 | where q.Name == queueName 57 | select (short?)q.QueueId 58 | ).SingleOrDefaultAsync(); 59 | } 60 | 61 | public async Task FindOr(string queueName, Func> valueFactory) 62 | { 63 | var queueId = await Find(queueName); 64 | return queueId.HasValue ? queueId.Value : await valueFactory(); 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /service/MinMQ.BenchmarkConsole/JsonGenerator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using Newtonsoft.Json.Linq; 5 | 6 | namespace MinMQ.BenchmarkConsole 7 | { 8 | public class JsonGenerator : GeneratorBase 9 | { 10 | private Words words = new Words(); 11 | 12 | public JsonGenerator(int n) 13 | : base(n) 14 | { 15 | } 16 | 17 | /// 18 | /// A JSON generator. 19 | /// 20 | /// JSON with random objects and values 21 | public override string GenerateObject() 22 | { 23 | JObject child = new JObject(new JProperty(words.Pick(), new JValue(words.Pick()))); 24 | int depth = 0; 25 | IEnumerable children = GenerateChildren(++depth); 26 | AddChildren(children, child); 27 | 28 | return child.ToString(); 29 | } 30 | 31 | private static JToken GenerateJValue(Random seed) 32 | { 33 | switch (seed.Next(0, 5)) 34 | { 35 | case 0: 36 | var values = Enumerable.Range(0, seed.Next(10)).Select(_ => new JValue(seed.Next(0, 999))); 37 | return new JArray(values); 38 | case 1: return new JValue(DateTime.Now); 39 | case 2: return new JValue(seed.Next(0, 10_000)); 40 | case 3: return new JValue(seed.NextDouble()); 41 | case 4: return new JValue(seed.Next(0, 10_000)); 42 | default: return null; 43 | } 44 | } 45 | 46 | protected override JObject GenerateChild(IEnumerable innerChildren) 47 | { 48 | JToken value = GenerateJValue(Seed); 49 | JProperty prop = new JProperty(words.Pick(), value); 50 | JObject child = new JObject(prop); 51 | 52 | AddChildren(innerChildren, child); 53 | 54 | return child; 55 | } 56 | 57 | private void AddChildren(IEnumerable innerChildren, JObject child) 58 | { 59 | foreach (JObject innerChild in innerChildren) 60 | { 61 | try 62 | { 63 | child.Add(words.Pick(), innerChild); 64 | } 65 | catch (ArgumentException) 66 | { 67 | child.Add(Guid.NewGuid().ToString(), innerChild); 68 | } 69 | } 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Repository/CursorRepository.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using MinMq.Service.Entities; 6 | using MinMq.Service.Models; 7 | using NodaTime; 8 | using Optional; 9 | 10 | namespace MinMq.Service.Repository 11 | { 12 | public class CursorRepository : ICursorRepository 13 | { 14 | private readonly MessageQueueContext messageQueueContext; 15 | 16 | public CursorRepository(MessageQueueContext messageQueueContext) 17 | { 18 | this.messageQueueContext = messageQueueContext; 19 | } 20 | 21 | public async Task Update(Cursor cursor) 22 | { 23 | var now = SystemClock.Instance.GetCurrentInstant().InUtc().ToDateTimeUtc(); 24 | 25 | tCursor cursorDo = await messageQueueContext.tCursors.SingleOrDefaultAsync(q => q.CursorId == cursor.Id); 26 | cursorDo.Changed = now; 27 | cursorDo.NextReferenceId = cursor.NextAddress; 28 | await messageQueueContext.SaveChangesAsync(); 29 | return cursorDo.CursorId; 30 | } 31 | 32 | public async Task Add(Cursor cursor) 33 | { 34 | var now = SystemClock.Instance.GetCurrentInstant().InUtc().ToDateTimeUtc(); 35 | 36 | tCursor cursor_ = new tCursor 37 | { 38 | Changed = now, 39 | Added = now, 40 | NextReferenceId = cursor.NextAddress 41 | }; 42 | 43 | await messageQueueContext.AddAsync(cursor_); 44 | await messageQueueContext.SaveChangesAsync(); 45 | return cursor_.CursorId; 46 | } 47 | 48 | public void Dispose() 49 | { 50 | messageQueueContext?.Dispose(); 51 | } 52 | 53 | public async Task Find(int cursorId) 54 | { 55 | return await 56 | ( 57 | from q in (IAsyncEnumerable)messageQueueContext.tCursors 58 | where q.CursorId == cursorId 59 | select new DebugCursor(q.CursorId, q.NextReferenceId) 60 | ).SingleOrDefaultAsync(); 61 | } 62 | 63 | public async Task FindOr(int cursorId, Func> valueFactory) 64 | { 65 | var cursor = await Find(cursorId); 66 | return cursor != null ? cursor : await valueFactory(); 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /service/MinMQ.ScanConsole/FasterLogReader.cs: -------------------------------------------------------------------------------- 1 | using FASTER.core; 2 | using NodaTime; 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Text; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | 9 | namespace MinMQ.ScanConsole 10 | { 11 | public class FasterLogReader 12 | { 13 | public async Task> StartScan(string devicePath) 14 | { 15 | IDevice device = Devices.CreateLogDevice(devicePath); 16 | FasterLog logger = new FasterLog(new FasterLogSettings { LogDevice = device }); 17 | long nextAddress = 0; 18 | bool keepGoing = true; 19 | int i = 0; 20 | 21 | var result = new List<(string, long, long)>(); 22 | 23 | // using (FasterLogScanIterator iter = logger.Scan(logger.BeginAddress, 100_000_000, name: nameof(GetListAsync))) 24 | using (FasterLogScanIterator iter = logger.Scan(nextAddress, 1_000_000_000)) 25 | { 26 | while(keepGoing) 27 | { 28 | Console.WriteLine("Going"); 29 | LocalTime timeOfDay; 30 | await foreach ((byte[] bytes, int length) in iter.GetAsyncEnumerable()) 31 | { 32 | 33 | DateTimeZone tz = DateTimeZoneProviders.Tzdb.GetSystemDefault(); 34 | timeOfDay = SystemClock.Instance.GetCurrentInstant().InZone(tz).TimeOfDay; 35 | nextAddress = iter.NextAddress; 36 | Console.WriteLine("Time={1} NextAddress={0}, Count={2}", iter.NextAddress, timeOfDay, i++); 37 | var cts = new CancellationTokenSource(); 38 | UTF8Encoding encoding = new UTF8Encoding(); 39 | 40 | try 41 | { 42 | await Task.WhenAny(WaitAsync(iter), SetTimeout(cts)); 43 | } 44 | catch (Exception e) 45 | { 46 | Console.Error.WriteLine($"Error={e.GetType()}, Message={e.ToString()}"); 47 | break; 48 | } 49 | 50 | timeOfDay = SystemClock.Instance.GetCurrentInstant().InZone(tz).TimeOfDay; 51 | Console.WriteLine("Time={2} ContentLength={0}", bytes.Length, iter.NextAddress, timeOfDay); 52 | } 53 | await Task.Delay(5000); 54 | } 55 | 56 | } 57 | 58 | return result; 59 | } 60 | 61 | public async Task SetTimeout(CancellationTokenSource cancellationTokenSource) 62 | { 63 | await Task.Delay(300); 64 | cancellationTokenSource.Cancel(); 65 | } 66 | 67 | public async Task WaitAsync(FasterLogScanIterator iter) 68 | { 69 | return await iter.WaitAsync(); 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /service/MinMQ.BenchmarkConsole/XmlGenerator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Text; 5 | using System.Xml; 6 | 7 | namespace MinMQ.BenchmarkConsole 8 | { 9 | public class XmlGenerator : GeneratorBase 10 | { 11 | private XmlDocument doc; 12 | private Words words = new Words(); 13 | 14 | public XmlGenerator(int n) 15 | : base(n) 16 | { 17 | } 18 | 19 | /// 20 | /// XML-generator. Combines variable depth with attributes and sets a text value at the lowest child. 21 | /// 22 | /// XML with random attributes and elements 23 | public override string GenerateObject() 24 | { 25 | doc = new XmlDocument(); 26 | doc.AppendChild(doc.CreateElement(words.Pick())); 27 | var root = doc.DocumentElement; 28 | 29 | int depth = 0; 30 | var children = GenerateChildren(++depth); 31 | 32 | foreach (XmlElement child in children) 33 | { 34 | root.AppendChild(child); 35 | } 36 | 37 | return PrintXml(doc.InnerXml); 38 | } 39 | 40 | protected override XmlElement GenerateChild(IEnumerable innerChildren) 41 | { 42 | Func wordFactory = () => words.Pick(); 43 | int numberOfProps = NumberOfProperties(); 44 | var child = doc.CreateElement(wordFactory()); 45 | 46 | for (int i = 0; i < numberOfProps; i++) 47 | { 48 | child.SetAttribute(wordFactory(), wordFactory()); 49 | } 50 | 51 | foreach (XmlElement innerChild in innerChildren) 52 | { 53 | child.AppendChild(innerChild); 54 | } 55 | 56 | if (child.ChildNodes.Count < 1) 57 | { 58 | child.InnerText = $"{wordFactory()} {wordFactory()} {wordFactory()} {wordFactory()}"; 59 | } 60 | 61 | return child; 62 | } 63 | 64 | public static string PrintXml(string xml) 65 | { 66 | string formattedXml = ""; 67 | 68 | MemoryStream memoryStream = new MemoryStream(); 69 | XmlTextWriter writer = new XmlTextWriter(memoryStream, Encoding.Unicode); 70 | XmlDocument document = new XmlDocument(); 71 | StreamReader reader = new StreamReader(memoryStream); 72 | 73 | try 74 | { 75 | document.LoadXml(xml); 76 | writer.Formatting = Formatting.Indented; 77 | document.WriteContentTo(writer); 78 | writer.Flush(); 79 | memoryStream.Flush(); 80 | memoryStream.Position = 0; 81 | formattedXml = reader.ReadToEnd(); 82 | } 83 | finally 84 | { 85 | reader.Close(); 86 | memoryStream.Close(); 87 | writer.Close(); 88 | } 89 | 90 | return formattedXml; 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Faster/FasterHostedServiceCommit.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading; 3 | using System.Threading.Tasks; 4 | using Microsoft.Extensions.Hosting; 5 | using Microsoft.Extensions.Logging; 6 | 7 | namespace MinMQ.Service.Faster 8 | { 9 | internal delegate ValueTask CommitAsyncDelegate(CancellationToken token = default); 10 | 11 | /// 12 | /// A hosted service that runs in the background. It's responsible for flushing commits every 5ms. It's required 13 | /// in high-contention scenarios because the eventloop gets completely bogged down with requests. In such scenarios 14 | /// all threads get stuck at some other step than FasterLog.CommitAsync(). However, this service's Timer is always 15 | /// granted some CPU-time. 16 | /// 17 | public class FasterHostedServiceCommit : IHostedService, IDisposable 18 | { 19 | private const int PeriodMs = 5; 20 | private const int LogCommitTimerEverySeconds = 3000; 21 | private readonly ILogger logger; 22 | private Timer timer; 23 | 24 | public FasterHostedServiceCommit(ILogger logger) 25 | { 26 | this.logger = logger; 27 | } 28 | 29 | private async void ExecuteAsync(object stateInfo) 30 | { 31 | FasterHostedServiceCommitState state = stateInfo as FasterHostedServiceCommitState; 32 | 33 | try 34 | { 35 | await state.CommitAsync(); 36 | } 37 | catch (Exception e) 38 | { 39 | logger.LogError("Commit failed: {0}", e); 40 | } 41 | 42 | if (state.InvokationCount > 0 && state.InvokationCount % state.LoggingInterval == 0) 43 | { 44 | state.InvokationCount = 0; 45 | state.Logger.LogInformation("Polling commits every {0} ms", state.PeriodMs); 46 | } 47 | else 48 | { 49 | state.InvokationCount++; 50 | } 51 | } 52 | 53 | public Task StartAsync(CancellationToken stoppingToken) 54 | { 55 | logger.LogInformation("{0} service running.", nameof(FasterHostedServiceCommit)); 56 | var state = new FasterHostedServiceCommitState(FasterOps.Instance.Value.CommitAsync, logger, GetLoggingInterval(), PeriodMs); 57 | timer = new Timer(ExecuteAsync, state, TimeSpan.Zero, TimeSpan.FromMilliseconds(PeriodMs)); 58 | return Task.CompletedTask; 59 | } 60 | 61 | public Task StopAsync(CancellationToken stoppingToken) 62 | { 63 | logger.LogInformation("Timed Hosted' Service is stopping."); 64 | timer?.Change(Timeout.Infinite, 0); 65 | return Task.CompletedTask; 66 | } 67 | 68 | public void Dispose() 69 | { 70 | timer?.Dispose(); 71 | } 72 | 73 | private int GetLoggingInterval() 74 | { 75 | int interval = LogCommitTimerEverySeconds / PeriodMs; 76 | interval = Math.Max(1, interval); 77 | return Math.Min(2000, interval); 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Controllers/DefaultController.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using Microsoft.AspNetCore.Mvc; 3 | using MinMQ.Service.Controllers.Dto; 4 | using MinMq.Service.Entities; 5 | using MinMQ.Service.Models; 6 | using MinMq.Service.Repository; 7 | 8 | namespace MinMQ.Service.Controllers 9 | { 10 | [Route("api")] 11 | [ApiController] 12 | public class DefaultController : ControllerBase 13 | { 14 | private readonly MessagesContext messagesContext; 15 | private readonly IQueueRepository queueRepository; 16 | 17 | public DefaultController(MessagesContext messagesContext, IQueueRepository queueRepository) 18 | { 19 | this.messagesContext = messagesContext; 20 | this.queueRepository = queueRepository; 21 | } 22 | 23 | [HttpGet("/status")] 24 | public IActionResult Status() 25 | { 26 | return Ok(new StatusResponseDto { Text = "ok" }); 27 | } 28 | 29 | [HttpPost("/efcore-in-mem-dto")] 30 | public async Task PostAsync(MessageRequestDto message) 31 | { 32 | await messagesContext.Messages.AddAsync(new Models.Message { Content = message.Content }); 33 | await messagesContext.SaveChangesAsync(); 34 | } 35 | 36 | [HttpPost("/efcore-in-mem-text")] 37 | public async Task PostAsync(string message) 38 | { 39 | await messagesContext.Messages.AddAsync(new Models.Message { Content = message }); 40 | await messagesContext.SaveChangesAsync(); 41 | } 42 | 43 | [HttpGet("/peek")] 44 | public async Task Peek() 45 | { 46 | var option = await FasterOps.Instance.Value.GetNext(); 47 | 48 | return option.Match 49 | ( 50 | some: value => Ok(value.ToPeekResponse()), 51 | none: () => NotFound() 52 | ); 53 | } 54 | 55 | [HttpGet("/list")] 56 | public async Task List() 57 | { 58 | var scanner = FasterOps.Instance.Value.GetListAsync(); 59 | return (await ControllerExtensions.ToListResponse(scanner)).Match 60 | ( 61 | some: values => Ok(values), 62 | none: () => NotFound() 63 | ); 64 | } 65 | 66 | [HttpGet("/list/{name:int}")] 67 | public async Task ListFor(int queueId) 68 | { 69 | var scanner = FasterOps.Instance.Value.GetListAsync(); 70 | return (await ControllerExtensions.ToListResponse(scanner)).Match 71 | ( 72 | some: values => Ok(values), 73 | none: () => NotFound() 74 | ); 75 | } 76 | 77 | // In contrast, the URI in a PUT request identifies the entity enclosed with the request. 78 | [HttpPut("/queue/{name:regex(^\\w+)}")] 79 | public async Task Add(string name) 80 | { 81 | { 82 | short? queueId = await queueRepository.Find(name); 83 | if (queueId.HasValue) return Redirect("/list/{queueId}"); 84 | } 85 | { 86 | short queueId = await queueRepository.Add(new Queue(name)); 87 | return Created($"/list/{queueId}", queueId); 88 | } 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /benchmark/mmq-post-json-2.lua: -------------------------------------------------------------------------------- 1 | wrk.method = "POST" 2 | wrk.body = "{\"web-app\": { \"servlet\": [ { \"servlet-name\": \"cofaxCDS\", \"servlet-class\": \"org.cofax.cds.CDSServlet\", \"init-param\": { \"configGlossary:installationAt\": \"Philadelphia, PA\", \"configGlossary:adminEmail\": \"ksm@pobox.com\", \"configGlossary:poweredBy\": \"Cofax\", \"configGlossary:poweredByIcon\": \"/images/cofax.gif\", \"configGlossary:staticPath\": \"/content/static\", \"templateProcessorClass\": \"org.cofax.WysiwygTemplate\", \"templateLoaderClass\": \"org.cofax.FilesTemplateLoader\", \"templatePath\": \"templates\", \"templateOverridePath\": \"\", \"defaultListTemplate\": \"listTemplate.htm\", \"defaultFileTemplate\": \"articleTemplate.htm\", \"useJSP\": false, \"jspListTemplate\": \"listTemplate.jsp\", \"jspFileTemplate\": \"articleTemplate.jsp\", \"cachePackageTagsTrack\": 200, \"cachePackageTagsStore\": 200, \"cachePackageTagsRefresh\": 60, \"cacheTemplatesTrack\": 100, \"cacheTemplatesStore\": 50, \"cacheTemplatesRefresh\": 15, \"cachePagesTrack\": 200, \"cachePagesStore\": 100, \"cachePagesRefresh\": 10, \"cachePagesDirtyRead\": 10, \"searchEngineListTemplate\": \"forSearchEnginesList.htm\", \"searchEngineFileTemplate\": \"forSearchEngines.htm\", \"searchEngineRobotsDb\": \"WEB-INF/robots.db\", \"useDataStore\": true, \"dataStoreClass\": \"org.cofax.SqlDataStore\", \"redirectionClass\": \"org.cofax.SqlRedirection\", \"dataStoreName\": \"cofax\", \"dataStoreDriver\": \"com.microsoft.jdbc.sqlserver.SQLServerDriver\", \"dataStoreUrl\": \"jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon\", \"dataStoreUser\": \"sa\", \"dataStorePassword\": \"dataStoreTestQuery\", \"dataStoreTestQuery\": \"SET NOCOUNT ON;select test='test';\", \"dataStoreLogFile\": \"/usr/local/tomcat/logs/datastore.log\", \"dataStoreInitConns\": 10, \"dataStoreMaxConns\": 100, \"dataStoreConnUsageLimit\": 100, \"dataStoreLogLevel\": \"debug\", \"maxUrlLength\": 500}}, { \"servlet-name\": \"cofaxEmail\", \"servlet-class\": \"org.cofax.cds.EmailServlet\", \"init-param\": { \"mailHost\": \"mail1\", \"mailHostOverride\": \"mail2\"}}, { \"servlet-name\": \"cofaxAdmin\", \"servlet-class\": \"org.cofax.cds.AdminServlet\"}, { \"servlet-name\": \"fileServlet\", \"servlet-class\": \"org.cofax.cds.FileServlet\"}, { \"servlet-name\": \"cofaxTools\", \"servlet-class\": \"org.cofax.cms.CofaxToolsServlet\", \"init-param\": { \"templatePath\": \"toolstemplates/\", \"log\": 1, \"logLocation\": \"/usr/local/tomcat/logs/CofaxTools.log\", \"logMaxSize\": \"\", \"dataLog\": 1, \"dataLogLocation\": \"/usr/local/tomcat/logs/dataLog.log\", \"dataLogMaxSize\": \"\", \"removePageCache\": \"/content/admin/remove?cache=pages&id=\", \"removeTemplateCache\": \"/content/admin/remove?cache=templates&id=\", \"fileTransferFolder\": \"/usr/local/tomcat/webapps/content/fileTransferFolder\", \"lookInContext\": 1, \"adminGroupID\": 4, \"betaServer\": true}}], \"servlet-mapping\": { \"cofaxCDS\": \"/\", \"cofaxEmail\": \"/cofaxutil/aemail/*\", \"cofaxAdmin\": \"/admin/*\", \"fileServlet\": \"/static/*\", \"cofaxTools\": \"/tools/*\"}, \"taglib\": { \"taglib-uri\": \"cofax.tld\", \"taglib-location\": \"/WEB-INF/tlds/cofax.tld\"}}}" 3 | wrk.headers["Content-Type"] = "application/json" -------------------------------------------------------------------------------- /service/MinMQ.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.29324.140 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MinMQ.Service", "MinMQ.Service\MinMQ.Service.csproj", "{0806F4F4-011D-4841-A2EA-E784C0224EB5}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MinMQ.BenchmarkConsole", "MinMQ.BenchmarkConsole\MinMQ.BenchmarkConsole.csproj", "{BAEBB1F1-F13A-456D-8F7E-48A8BB72D098}" 9 | EndProject 10 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{22BAF7F9-35AD-49D4-9BD9-ABB6498757EC}" 11 | ProjectSection(SolutionItems) = preProject 12 | StyleCop.ruleset = StyleCop.ruleset 13 | EndProjectSection 14 | EndProject 15 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MinMQ.ScanConsole", "MinMQ.ScanConsole\MinMQ.ScanConsole.csproj", "{B083EB4B-99DB-4F8B-BCCB-61ABC6EE884D}" 16 | EndProject 17 | Global 18 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 19 | Debug|Any CPU = Debug|Any CPU 20 | Release|Any CPU = Release|Any CPU 21 | Troubleshoot|Any CPU = Troubleshoot|Any CPU 22 | EndGlobalSection 23 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 24 | {0806F4F4-011D-4841-A2EA-E784C0224EB5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 25 | {0806F4F4-011D-4841-A2EA-E784C0224EB5}.Debug|Any CPU.Build.0 = Debug|Any CPU 26 | {0806F4F4-011D-4841-A2EA-E784C0224EB5}.Release|Any CPU.ActiveCfg = Release|Any CPU 27 | {0806F4F4-011D-4841-A2EA-E784C0224EB5}.Release|Any CPU.Build.0 = Release|Any CPU 28 | {0806F4F4-011D-4841-A2EA-E784C0224EB5}.Troubleshoot|Any CPU.ActiveCfg = Troubleshoot|Any CPU 29 | {0806F4F4-011D-4841-A2EA-E784C0224EB5}.Troubleshoot|Any CPU.Build.0 = Troubleshoot|Any CPU 30 | {BAEBB1F1-F13A-456D-8F7E-48A8BB72D098}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 31 | {BAEBB1F1-F13A-456D-8F7E-48A8BB72D098}.Debug|Any CPU.Build.0 = Debug|Any CPU 32 | {BAEBB1F1-F13A-456D-8F7E-48A8BB72D098}.Release|Any CPU.ActiveCfg = Release|Any CPU 33 | {BAEBB1F1-F13A-456D-8F7E-48A8BB72D098}.Release|Any CPU.Build.0 = Release|Any CPU 34 | {BAEBB1F1-F13A-456D-8F7E-48A8BB72D098}.Troubleshoot|Any CPU.ActiveCfg = Troubleshoot|Any CPU 35 | {BAEBB1F1-F13A-456D-8F7E-48A8BB72D098}.Troubleshoot|Any CPU.Build.0 = Troubleshoot|Any CPU 36 | {B083EB4B-99DB-4F8B-BCCB-61ABC6EE884D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 37 | {B083EB4B-99DB-4F8B-BCCB-61ABC6EE884D}.Debug|Any CPU.Build.0 = Debug|Any CPU 38 | {B083EB4B-99DB-4F8B-BCCB-61ABC6EE884D}.Release|Any CPU.ActiveCfg = Release|Any CPU 39 | {B083EB4B-99DB-4F8B-BCCB-61ABC6EE884D}.Release|Any CPU.Build.0 = Release|Any CPU 40 | {B083EB4B-99DB-4F8B-BCCB-61ABC6EE884D}.Troubleshoot|Any CPU.ActiveCfg = Troubleshoot|Any CPU 41 | {B083EB4B-99DB-4F8B-BCCB-61ABC6EE884D}.Troubleshoot|Any CPU.Build.0 = Troubleshoot|Any CPU 42 | EndGlobalSection 43 | GlobalSection(SolutionProperties) = preSolution 44 | HideSolutionNode = FALSE 45 | EndGlobalSection 46 | GlobalSection(ExtensibilityGlobals) = postSolution 47 | SolutionGuid = {9C8264A1-F808-47BE-A17C-861B6B3323F9} 48 | EndGlobalSection 49 | EndGlobal 50 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Startup.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading; 3 | using Microsoft.AspNetCore.Builder; 4 | using Microsoft.AspNetCore.Hosting; 5 | using Microsoft.EntityFrameworkCore; 6 | using Microsoft.Extensions.Configuration; 7 | using Microsoft.Extensions.DependencyInjection; 8 | using Microsoft.Extensions.Hosting; 9 | using MinMQ.Service.Configuration; 10 | using MinMQ.Service.Filters; 11 | using MinMQ.Service.HttpRequestHandlers; 12 | using MinMq.Service.Models; 13 | using MinMQ.Service.Models; 14 | using MinMq.Service.Repository; 15 | 16 | namespace MinMQ.Service 17 | { 18 | public delegate CancellationTokenSource CancellationTokenSourceFactory(); 19 | 20 | public class Startup 21 | { 22 | public Startup(IConfiguration configuration) 23 | { 24 | Configuration = configuration; 25 | } 26 | 27 | public static IConfiguration Configuration { get; private set; } 28 | public void ConfigureServices(IServiceCollection services) 29 | { 30 | services.AddControllers(); 31 | 32 | services.AddControllers(configure => 33 | { 34 | configure.Filters.Add(); 35 | }); 36 | services.AddTransient(); 37 | services.AddDbContext(options => options.UseInMemoryDatabase(databaseName: "Messages")); 38 | services.AddOptions().Configure(SetOptions).ValidateDataAnnotations(); 39 | services.AddHealthChecks(); 40 | services.AddScoped(_ => () => new CancellationTokenSource()); 41 | // services.AddDbContext(options => options.UseNpgsql(Configuration.GetConnectionString("MessageQueueContext"))); 42 | services.AddDbContext(options => options.UseNpgsql(Configuration.GetConnectionString("MessageQueueContext"))); 43 | services.AddScoped(); 44 | services.AddScoped(); 45 | services.AddScoped(); 46 | services.AddScoped(); 47 | } 48 | 49 | private static void SetOptions(MinMQConfiguration o) 50 | { 51 | o.FasterDevice = Configuration[nameof(o.FasterDevice)]; 52 | o.ScanFlushSize = int.Parse(Configuration[nameof(o.ScanFlushSize)]); 53 | o.ConnectionStringPostgres = Configuration.GetSection("ConnectionStrings")["MessageQueueContext"]; 54 | } 55 | 56 | public void Configure(IApplicationBuilder app, IWebHostEnvironment env) 57 | { 58 | Console.WriteLine("Environment setting {0}", env.EnvironmentName); 59 | 60 | new ConfigurationBuilder() 61 | .AddJsonFile("appsettings.json") 62 | .AddJsonFile($"appsettings.{env.EnvironmentName}.json") 63 | .AddEnvironmentVariables() 64 | .Build(); 65 | 66 | if (env.IsDevelopment()) 67 | { 68 | app.UseDeveloperExceptionPage(); 69 | } 70 | 71 | UpdateDatabase(app); 72 | 73 | app.UseRouting(); 74 | 75 | // app.UseAuthorization(); 76 | 77 | app.UseEndpoints(endpoints => 78 | { 79 | endpoints.MapControllers(); 80 | endpoints.MapPost("/send", FasterHttpHandler.HandleRequest); 81 | endpoints.MapHealthChecks("/healthcheck"); 82 | }); 83 | } 84 | 85 | private static void UpdateDatabase(IApplicationBuilder app) 86 | { 87 | using (var serviceScope = app.ApplicationServices.GetRequiredService().CreateScope()) 88 | { 89 | using (var context = serviceScope.ServiceProvider.GetService()) 90 | { 91 | context.Database.Migrate(); 92 | } 93 | } 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Repository/MessageRepository.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using MinMq.Service.Models; 7 | using NodaTime; 8 | using Optional; 9 | 10 | namespace MinMq.Service.Repository 11 | { 12 | public class MessageRepository : IMessageRepository 13 | { 14 | private readonly MessageQueueContext messageQueueContext; 15 | 16 | public MessageRepository(MessageQueueContext messageQueueContext) 17 | { 18 | this.messageQueueContext = messageQueueContext; 19 | } 20 | 21 | public async Task> AddRange(List messages) 22 | { 23 | Option nextReferenceId = Option.None(); 24 | var savedMessages = await Find(new FindMessagesQuery(messages)); 25 | var newMessages = messages.Except(savedMessages, new MessageComparer()); 26 | 27 | foreach (var message in newMessages) 28 | { 29 | var queue = await messageQueueContext.tQueues.SingleOrDefaultAsync(q => q.QueueId == message.QueueId); 30 | var now = SystemClock.Instance.GetCurrentInstant().InUtc().ToDateTimeUtc(); 31 | 32 | var messageDo = new tMessage 33 | { 34 | ReferenceId = message.ReferenceId, 35 | NextReferenceId = message.NextReferenceId, 36 | Content = message.Content, 37 | Queue = queue, 38 | HashCode = message.HashCode, 39 | Added = now, 40 | Changed = now, 41 | MimeTypeId = message.MimeTypeId 42 | }; 43 | 44 | messageQueueContext.tMessages.Add(messageDo); 45 | 46 | #if TROUBLESHOOT 47 | try 48 | { 49 | await messageQueueContext.SaveChangesAsync(); 50 | } 51 | catch (Microsoft.EntityFrameworkCore.DbUpdateException error) when (error.InnerException.Message == "22021: invalid byte sequence for encoding \"UTF8\": 0x00") 52 | { 53 | logger.LogError("Invalid document. Skipping.."); 54 | continue; 55 | } 56 | #endif 57 | nextReferenceId = nextReferenceId.Match 58 | ( 59 | none: () => message.NextReferenceId.Some(), 60 | some: m => Math.Max(message.NextReferenceId, m).Some() 61 | ); 62 | } 63 | #if RELEASE || DEBUG 64 | await messageQueueContext.SaveChangesAsync(); 65 | #endif 66 | return nextReferenceId; 67 | } 68 | 69 | private async Task> Find(FindMessagesQuery findMessageQuery) 70 | { 71 | IAsyncEnumerable messages = messageQueueContext.tMessages; 72 | 73 | var query = 74 | from m in messages 75 | where findMessageQuery.Messages.Any(pm => pm.ReferenceId == m.ReferenceId && pm.HashCode == m.HashCode) 76 | select new Entities.Message 77 | ( 78 | m.Content, 79 | m.ReferenceId, 80 | m.NextReferenceId, 81 | m.HashCode, 82 | m.MimeTypeId, 83 | m.Queue.QueueId 84 | ); 85 | 86 | return await query.ToListAsync(); 87 | } 88 | 89 | public void Dispose() 90 | { 91 | messageQueueContext?.Dispose(); 92 | } 93 | 94 | public class MessageComparer : IEqualityComparer 95 | { 96 | bool IEqualityComparer.Equals(Entities.Message x, Entities.Message y) 97 | { 98 | return x.ReferenceId == y.ReferenceId 99 | && x.HashCode == y.HashCode; 100 | } 101 | 102 | int IEqualityComparer.GetHashCode(Entities.Message obj) 103 | { 104 | return new StringBuilder().Append(obj.ReferenceId).Append(obj.HashCode).ToString().GetHashCode(); 105 | } 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Models/MessageQueueContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.ComponentModel.DataAnnotations; 4 | using Microsoft.EntityFrameworkCore; 5 | using Microsoft.Extensions.Options; 6 | using MinMQ.Service.Configuration; 7 | using Optional; 8 | 9 | namespace MinMq.Service.Models 10 | { 11 | public class MessageQueueContext : DbContext 12 | { 13 | private readonly IOptions configuration; 14 | 15 | public MessageQueueContext(DbContextOptions options, IOptions configuration) 16 | : base(options) 17 | { 18 | this.configuration = configuration; 19 | } 20 | 21 | public DbSet tQueues { get; set; } 22 | public DbSet tMessages { get; set; } 23 | public DbSet tMimeTypes { get; set; } 24 | public DbSet tCursors { get; set; } 25 | 26 | protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) 27 | { 28 | var connectionString = configuration.Value.ConnectionStringPostgres.SomeNotNull(); 29 | var defaultConnectionString = "Host=localhost;Database=mmq;Username=5a4ba2e9-6c44-49dd-bc6c-b9ea2b901114;Password=effe908d-158d-47c5-a2eb-ad6814ce6083"; 30 | optionsBuilder.UseNpgsql(connectionString.ValueOr(defaultConnectionString)); 31 | } 32 | 33 | protected override void OnModelCreating(ModelBuilder modelBuilder) 34 | { 35 | modelBuilder.Entity() 36 | .HasOne(p => p.Queue) 37 | .WithMany(b => b.Messages) 38 | .HasForeignKey(p => p.QueueId) 39 | .HasConstraintName("FK_Message_Queue"); 40 | 41 | modelBuilder.Entity() 42 | .Property(f => f.MessageId) 43 | .ValueGeneratedOnAdd(); 44 | 45 | modelBuilder.Entity() 46 | .Property(f => f.QueueId) 47 | .ValueGeneratedOnAdd(); 48 | 49 | modelBuilder.Entity() 50 | .Property(f => f.MimeTypeId) 51 | .ValueGeneratedOnAdd(); 52 | 53 | modelBuilder.Entity() 54 | .HasOne(p => p.MimeType) 55 | .WithMany(b => b.tMessages) 56 | .HasForeignKey(p => p.MimeTypeId) 57 | .HasConstraintName("FK_Message_MimeType"); 58 | } 59 | } 60 | 61 | public class tQueue 62 | { 63 | [Key] 64 | public short QueueId { get; set; } 65 | public string Name { get; set; } 66 | 67 | public List Messages { get; set; } 68 | public DateTime Added { get; set; } 69 | public DateTime Changed { get; set; } 70 | } 71 | 72 | public class tMessage 73 | { 74 | [Key] 75 | public int MessageId { get; set; } 76 | public long ReferenceId { get; set; } 77 | public long NextReferenceId { get; set; } 78 | public string Content { get; set; } 79 | public short QueueId { get; set; } 80 | public tQueue Queue { get; set; } 81 | public short MimeTypeId { get; set; } 82 | public tMimeType MimeType { get; set; } 83 | public string HashCode { get; set; } 84 | public DateTime Added { get; set; } 85 | public DateTime Changed { get; set; } 86 | } 87 | 88 | public class tMimeType 89 | { 90 | [Key] 91 | public short MimeTypeId { get; set; } 92 | public string Expression { get; set; } 93 | public List tMessages { get; set; } 94 | public DateTime Added { get; set; } 95 | public DateTime Changed { get; set; } 96 | } 97 | 98 | public class tCursor 99 | { 100 | [Key] 101 | public int CursorId { get; set; } 102 | public DateTime Added { get; set; } 103 | public DateTime Changed { get; set; } 104 | public long NextReferenceId { get; set; } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /benchmark/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -u 4 | 5 | echo "Run once" 6 | if [ -f /opt/install_done ]; then 7 | echo "Error. Installation already completed once." 8 | exit 0 9 | fi 10 | 11 | # https://stackoverflow.com/questions/9869902/prevent-bash-from-interpreting-without-quoting-everything 12 | cat >> /app/wrk/http-ready.sh << 'EOF' 13 | #!/bin/bash 14 | while [ $(curl -o -I -L -s -w "%{http_code}" http://mmq-service-express:4000/status) -ne 200 ] 15 | do 16 | echo -n "express" 17 | sleep 1 18 | done 19 | 20 | while [ $(curl -o -I -L -s -w "%{http_code}" http://mmq-service-nodejs:8000/status) -ne 200 ] 21 | do 22 | echo -n "node" 23 | sleep 1 24 | done 25 | 26 | while [ $(curl -o -I -L -s -w "%{http_code}" http://mmq-service:9000/healthcheck) -ne 200 ] 27 | do 28 | echo -n "kerstel" 29 | sleep 1 30 | done 31 | 32 | while [ $(curl -o -I -L -s -w "%{http_code}" http://mmq-service-hapi:1000/status) -ne 200 ] 33 | do 34 | echo -n "hapi" 35 | sleep 1 36 | done 37 | 38 | EOF 39 | 40 | chmod +x /app/wrk/http-ready.sh 41 | 42 | echo 'Creating status.sh' 43 | cat >> /app/wrk/status.sh << EOF 44 | #!/bin/bash 45 | echo '' 46 | echo '- Make sure to check CPU and RAM saturation.' 47 | echo '' 48 | ./http-ready.sh 49 | ./wrk -t12 -c400 -d10s http://mmq-service-nodejs:8000/status 50 | ./wrk -t12 -c400 -d10s http://mmq-service-express:4000/status 51 | ./wrk -t12 -c400 -d10s http://mmq-service-hapi:1000/status 52 | ./wrk -t12 -c400 -d10s http://mmq-service:9000/status 53 | ./wrk -t12 -c400 -d10s http://mmq-service:9000/healthcheck 54 | EOF 55 | 56 | chmod +x /app/wrk/status.sh 57 | 58 | 59 | echo 'Creating misc.sh' 60 | cat >> /app/wrk/misc.sh << EOF 61 | #!/bin/bash 62 | set -e 63 | echo '' 64 | echo '- Make sure to check CPU and RAM saturation.' 65 | echo '' 66 | ./http-ready.sh 67 | echo '' 68 | echo '** EF CORE IN-MEMORY DB **' 69 | ./wrk -t1 -c5 -d5s -s ./scripts/mmq-post.lua http://mmq-service:9000/efcore-in-mem-text 70 | echo '' 71 | echo '** XML 15C **' 72 | ./wrk -t2 -c15 -d7s -s ./scripts/mmq-post-xml.lua http://mmq-service:9000/send 73 | echo '' 74 | echo '** LARGE JSON 15C **' 75 | ./wrk -t2 -c15 -d7s -s ./scripts/mmq-post-json-2.lua http://mmq-service:9000/send 76 | echo '' 77 | echo '** JSON 400C **' 78 | ./wrk -t12 -c400 -d7s -s ./scripts/mmq-post.lua http://mmq-service:9000/send 79 | EOF 80 | 81 | chmod +x /app/wrk/misc.sh 82 | 83 | echo 'Creating post_message.sh' 84 | cat >> /app/wrk/post_message.sh << EOF 85 | #!/bin/bash 86 | set -e 87 | echo '' 88 | echo '- Make sure to check CPU and RAM saturation.' 89 | echo '' 90 | ./http-ready.sh 91 | echo '' 92 | echo '** EF CORE IN-MEMORY DB **' 93 | ./wrk -t1 -c5 -d5s -s ./scripts/mmq-post.lua http://mmq-service:9000/efcore-in-mem-text 94 | echo '' 95 | echo '** JSON 400C **' 96 | ./wrk -t12 -c400 -d12s -s ./scripts/mmq-post.lua http://mmq-service:9000/send 97 | EOF 98 | 99 | chmod +x /app/wrk/post_message.sh 100 | 101 | echo 'Creating arm.sh' 102 | cat >> /app/wrk/arm.sh << EOF 103 | #!/bin/bash 104 | set -v 105 | echo '' 106 | echo '- Make sure to check CPU and RAM saturation.' 107 | echo '' 108 | ./http-ready.sh 109 | ./wrk -t1 -c5 -d7s -s ./scripts/mmq-post.lua http://mmq-service:9000/send 110 | EOF 111 | 112 | chmod +x /app/wrk/arm.sh 113 | 114 | echo 'Creating arm1.sh' 115 | cat >> /app/wrk/arm1.sh << EOF 116 | #!/bin/bash 117 | set -v 118 | echo '' 119 | echo '- Make sure to check CPU and RAM saturation.' 120 | echo '' 121 | ./http-ready.sh 122 | ./wrk -t1 -c5 -d5s -s ./scripts/mmq-post.lua http://mmq-service:9000/efcore-in-mem-text 123 | ./wrk -t4 -c40 -d3s -s ./scripts/mmq-post.lua http://mmq-service:9000/faster-get 124 | EOF 125 | 126 | chmod +x /app/wrk/arm1.sh 127 | 128 | 129 | touch /opt/install_done 130 | echo 'Done!' 131 | -------------------------------------------------------------------------------- /docs/web-performance.md: -------------------------------------------------------------------------------- 1 | # Benchmark machine 2 | As a general case tests are normally conducted on a _Kingston SSDNow_. However it's intended to add plenty more storage options (and even SD-cards) along with more architectures like ARM64v8 and ARM32v7 to the suite. 3 | 4 | ## EFCore In-Memory Database 5 | 6 | Running 5s test @ http://mmq-service-kestrel:9000/message-text 7 | 2 threads and 3 connections 8 | Thread Stats Avg Stdev Max +/- Stdev 9 | Latency 9.01ms 44.44ms 369.75ms 95.97% 10 | Req/Sec 2.79k 758.75 3.78k 86.46% 11 | 26670 requests in 5.10s, 2.34MB read 12 | Requests/sec: 5229.63 13 | Transfer/sec: 469.85KB 14 | 15 | ## Faster 16 | 17 | Running 5s test @ http://mmq-service-kestrel:9000/faster 18 | 2 threads and 13 connections 19 | Thread Stats Avg Stdev Max +/- Stdev 20 | Latency 1.12ms 2.69ms 45.34ms 97.36% 21 | Req/Sec 6.87k 2.12k 9.85k 75.00% 22 | 32854 requests in 5.01s, 4.01MB read 23 | Requests/sec: 6561.83 24 | Transfer/sec: 820.23KB 25 | 26 | ## HTTP GET Compared 27 | 28 | ``` 29 | Running 10s test @ http://mmq-service-nodejs:8000/status 30 | 12 threads and 400 connections 31 | Thread Stats Avg Stdev Max +/- Stdev 32 | Latency 32.55ms 5.61ms 129.52ms 94.83% 33 | Req/Sec 1.02k 158.32 1.99k 83.97% 34 | 120047 requests in 10.08s, 14.88MB read 35 | Requests/sec: 11910.95 36 | Transfer/sec: 1.48MB 37 | 38 | Running 10s test @ http://mmq-service-express:4000/status 39 | 12 threads and 400 connections 40 | Thread Stats Avg Stdev Max +/- Stdev 41 | Latency 49.60ms 8.15ms 398.64ms 94.28% 42 | Req/Sec 655.91 132.34 1.00k 86.10% 43 | 76784 requests in 10.07s, 16.40MB read 44 | Requests/sec: 7621.72 45 | Transfer/sec: 1.63MB 46 | 47 | Running 10s test @ http://mmq-service-kestrel:9000/status 48 | 12 threads and 400 connections 49 | Thread Stats Avg Stdev Max +/- Stdev 50 | Latency 22.98ms 5.09ms 68.42ms 72.30% 51 | Req/Sec 1.43k 181.25 2.69k 74.83% 52 | 171200 requests in 10.08s, 27.92MB read 53 | Requests/sec: 16985.24 54 | Transfer/sec: 2.77MB 55 | PS M:\devwork\MinMQ> 56 | ``` 57 | 58 | ## Kestrel HTTP POST - In-memory EFCore database - 1000 pre-loaded records 59 | ### A test message 60 | 61 | Running 10s test @ http://mmq-service-kestrel:9000/message-text 62 | 12 threads and 400 connections 63 | Thread Stats Avg Stdev Max +/- Stdev 64 | Latency 48.23ms 16.36ms 258.85ms 78.46% 65 | Req/Sec 672.28 170.82 1.03k 76.64% 66 | 78927 requests in 10.07s, 6.92MB read 67 | Requests/sec: 7841.49 68 | Transfer/sec: 704.51KB 69 | 70 | ### A very large message 71 | 72 | Running 10s test @ http://mmq-service-kestrel:9000/message-text 73 | 12 threads and 400 connections 74 | Thread Stats Avg Stdev Max +/- Stdev 75 | Latency 52.20ms 18.95ms 290.34ms 82.02% 76 | Req/Sec 624.62 219.15 2.00k 91.12% 77 | 70918 requests in 10.06s, 6.22MB read 78 | Socket errors: connect 0, read 22, write 0, timeout 0 79 | Requests/sec: 7052.28 80 | Transfer/sec: 633.60KB 81 | 82 | ### JSON body 83 | 84 | Running 10s test @ http://mmq-service-kestrel:9000/message 85 | 12 threads and 400 connections 86 | Thread Stats Avg Stdev Max +/- Stdev 87 | Latency 47.49ms 9.30ms 99.34ms 71.41% 88 | Req/Sec 682.04 121.71 1.29k 76.21% 89 | 79843 requests in 10.02s, 7.01MB read 90 | Requests/sec: 7965.02 91 | Transfer/sec: 715.61KB 92 | -------------------------------------------------------------------------------- /service/MinMQ.BenchmarkConsole/Words.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text.RegularExpressions; 5 | 6 | namespace MinMQ.BenchmarkConsole 7 | { 8 | public class Words 9 | { 10 | private int wordIndex = 0; 11 | private string[] words = WordFactory(); 12 | 13 | public static string[] WordFactory() 14 | { 15 | static IEnumerable GetWords() 16 | { 17 | var phrase = @"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris eleifend leo ut velit pellentesque 18 | dictum. In venenatis ipsum non lacinia luctus. Aliquam pretium mauris nec quam imperdiet, in euismod ligula 19 | mollis. Integer risus diam, facilisis at tellus eget, fringilla hendrerit orci. Pellentesque at ornare diam. 20 | Quisque semper gravida erat. Sed feugiat felis vel massa consectetur suscipit nec bibendum ligula. Nullam 21 | pulvinar metus quis porta ornare. Pellentesque sodales nisi sit amet rutrum faucibus. Vestibulum pulvinar 22 | rhoncus purus, ut bibendum ante pretium in. Suspendisse potenti. 23 | 24 | Vestibulum facilisis ullamcorper lacus id iaculis. Nunc laoreet odio non nisi vulputate fringilla. Etiam erat 25 | nisi, viverra ut aliquam ac, dignissim eget felis. Integer sit amet augue suscipit, dignissim turpis semper, 26 | consectetur leo. Phasellus venenatis, libero ac pulvinar facilisis, elit augue ullamcorper ante, quis egestas mi 27 | justo ut dolor. Fusce nec diam nec est pulvinar faucibus. Vestibulum ante ipsum primis in faucibus orci luctus 28 | et ultrices posuere cubilia Curae; Morbi sit amet condimentum mi. Pellentesque habitant morbi tristique senectus 29 | et netus et malesuada fames ac turpis egestas. 30 | 31 | In rutrum ultrices sapien et venenatis. Nulla ornare euismod lectus ac facilisis. Curabitur imperdiet dignissim 32 | massa quis congue. Ut varius, dui et ornare finibus, lacus ligula porttitor ex, vel fringilla nisl risus et 33 | odio. Morbi ultricies volutpat mauris non bibendum. Aliquam at arcu sed massa venenatis sodales vitae nec ipsum. 34 | Suspendisse euismod lobortis massa eu molestie. Vivamus quis erat quis erat sollicitudin tristique. Praesent 35 | ornare consequat ipsum vel porta. Vivamus dignissim at diam sed faucibus. 36 | 37 | Duis nisi purus, lacinia eget magna vel, cursus ultricies erat. Suspendisse id sapien ullamcorper, maximus nisi 38 | ut, sodales dolor. Aenean consequat, est vitae venenatis rutrum, est massa dapibus magna, ac faucibus mauris 39 | augue eget purus. Suspendisse vel eros sapien. Curabitur vel sem eu sapien suscipit commodo. Vivamus vulputate 40 | nisl et ligula aliquet, nec dapibus lacus hendrerit. Etiam lobortis ornare nulla rutrum dignissim. Nulla 41 | sollicitudin non ipsum id ullamcorper. Nulla facilisi. 42 | 43 | Cras neque diam, dapibus eu felis sit amet, sollicitudin pellentesque diam. Ut et tincidunt dolor, vel varius 44 | urus. Etiam ut nibh sit amet tellus vestibulum pulvinar. Phasellus ante felis, venenatis interdum tempor eu, 45 | ornare vitae mi. Duis porttitor ipsum ac sapien dictum, in scelerisque lacus imperdiet. Nullam venenatis 46 | pellentesque elit a volutpat. Sed hendrerit tristique felis nec sagittis. Quisque in diam vulputate, porta purus 47 | nec, viverra mauris. Morbi accumsan consectetur lectus, eget semper sapien sagittis in. Donec quam lacus, 48 | consequat semper nunc eu, suscipit scelerisque magna. Curabitur porta arcu nec iaculis convallis. Duis mattis 49 | tempor mi tristique commodo. Pellentesque nec consequat dolor. "; 50 | 51 | foreach (Match match in Regex.Matches(phrase, "\\w+")) 52 | { 53 | yield return match.Value; 54 | } 55 | } 56 | 57 | return GetWords().ToArray(); 58 | } 59 | 60 | public string Pick() 61 | { 62 | if (++wordIndex > 0 && wordIndex % words.Length == 0) 63 | { 64 | wordIndex = 0; 65 | } 66 | 67 | return words[wordIndex]; 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Migrations/20191204215508_Add a lot more mime types.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore.Migrations; 2 | 3 | namespace MinMq.Service.Migrations 4 | { 5 | public partial class Addalotmoremimetypes : Migration 6 | { 7 | protected override void Up(MigrationBuilder migrationBuilder) 8 | { 9 | migrationBuilder.Sql(@"insert into ""tMimeTypes"" (""Expression"", ""Added"", ""Changed"") values 10 | ('application/x-7z-compressed', current_timestamp, current_timestamp), 11 | ('application/zip', current_timestamp, current_timestamp), 12 | ('font/woff2', current_timestamp, current_timestamp), 13 | ('font/woff', current_timestamp, current_timestamp), 14 | ('audio/wav', current_timestamp, current_timestamp), 15 | ('text/plain', current_timestamp, current_timestamp), 16 | ('font/ttf', current_timestamp, current_timestamp), 17 | ('image/tiff', current_timestamp, current_timestamp), 18 | ('application/x-tar', current_timestamp, current_timestamp), 19 | ('image/svg+xml', current_timestamp, current_timestamp), 20 | ('application/x-sh', current_timestamp, current_timestamp), 21 | ('application/rtf', current_timestamp, current_timestamp), 22 | ('application/x-rar-compressed', current_timestamp, current_timestamp), 23 | ('application/php', current_timestamp, current_timestamp), 24 | ('application/pdf', current_timestamp, current_timestamp), 25 | ('image/png', current_timestamp, current_timestamp), 26 | ('font/otf', current_timestamp, current_timestamp), 27 | ('application/ogg', current_timestamp, current_timestamp), 28 | ('video/ogg', current_timestamp, current_timestamp), 29 | ('audio/ogg', current_timestamp, current_timestamp), 30 | ('application/vnd.oasis.opendocument.text', current_timestamp, current_timestamp), 31 | ('application/vnd.oasis.opendocument.spreadsheet', current_timestamp, current_timestamp), 32 | ('application/vnd.oasis.opendocument.presentation', current_timestamp, current_timestamp), 33 | ('application/vnd.apple.installer+xml', current_timestamp, current_timestamp), 34 | ('video/mpeg', current_timestamp, current_timestamp), 35 | ('audio/mpeg', current_timestamp, current_timestamp), 36 | ('text/javascript', current_timestamp, current_timestamp), 37 | ('audio/x-midi', current_timestamp, current_timestamp), 38 | ('audio/midi', current_timestamp, current_timestamp), 39 | ('application/ld+json', current_timestamp, current_timestamp), 40 | ('text/javascript', current_timestamp, current_timestamp), 41 | ('image/jpeg', current_timestamp, current_timestamp), 42 | ('application/java-archive', current_timestamp, current_timestamp), 43 | ('text/calendar', current_timestamp, current_timestamp), 44 | ('image/vnd.microsoft.icon', current_timestamp, current_timestamp), 45 | ('text/html', current_timestamp, current_timestamp), 46 | ('image/gif', current_timestamp, current_timestamp), 47 | ('application/gzip', current_timestamp, current_timestamp), 48 | ('application/epub+zip', current_timestamp, current_timestamp), 49 | ('application/vnd.ms-fontobject', current_timestamp, current_timestamp), 50 | ('application/vnd.openxmlformats-officedocument.wordprocessingml.document', current_timestamp, current_timestamp), 51 | ('application/msword', current_timestamp, current_timestamp), 52 | ('text/csv', current_timestamp, current_timestamp), 53 | ('text/css', current_timestamp, current_timestamp), 54 | ('application/x-csh', current_timestamp, current_timestamp), 55 | ('application/x-bzip2', current_timestamp, current_timestamp), 56 | ('application/x-bzip', current_timestamp, current_timestamp), 57 | ('image/bmp', current_timestamp, current_timestamp), 58 | ('application/octet-stream', current_timestamp, current_timestamp), 59 | ('video/x-msvideo', current_timestamp, current_timestamp), 60 | ('audio/aac', current_timestamp, current_timestamp)"); 61 | } 62 | 63 | protected override void Down(MigrationBuilder migrationBuilder) 64 | { 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /benchmark/mmq-post-large.lua: -------------------------------------------------------------------------------- 1 | wrk.method = "POST" 2 | wrk.body = "message=Aliquam%20quis%20orci%20justo.%20Aenean%20lacinia%20hendrerit%20ante%20ac%20cursus.%20Proin%20lobortis%20justo%20non%20consectetur%20mattis.%20Maecenas%20ac%20libero%20eu%20massa%20tempor%20porttitor%20at%20eu%20magna.%20Ut%20tristique%20dictum%20odio%2C%20sed%20tristique%20urna%20molestie%20ultricies.%20Morbi%20arcu%20sapien%2C%20lobortis%20quis%20euismod%20quis%2C%20vestibulum%20eu%20odio.%20Fusce%20aliquet%20neque%20massa%2C%20at%20efficitur%20sem%20elementum%20nec.%20Phasellus%20ipsum%20dui%2C%20faucibus%20ac%20quam%20eu%2C%20ullamcorper%20suscipit%20est.%20Morbi%20lectus%20ligula%2C%20interdum%20eu%20molestie%20nec%2C%20cursus%20id%20elit.%20Nam%20vel%20quam%20a%20risus%20congue%20egestas.%20Ut%20maximus%20massa%20purus%2C%20id%20semper%20enim%20sagittis%20vel.%20Phasellus%20nunc%20massa%2C%20porta%20non%20neque%20ut%2C%20pretium%20egestas%20purus.Sed%20vitae%20nibh%20non%20mauris%20tristique%20convallis.%20Aliquam%20est%20nunc%2C%20sodales%20quis%20posuere%20congue%2C%20malesuada%20efficitur%20tortor.%20Praesent%20ornare%2C%20turpis%20nec%20feugiat%20fermentum%2C%20diam%20ligula%20ultrices%20ante%2C%20ut%20varius%20lectus%20enim%20in%20nulla.%20Nulla%20hendrerit%2C%20sem%20ac%20volutpat%20luctus%2C%20orci%20dui%20consectetur%20erat%2C%20a%20pulvinar%20nibh%20dolor%20vitae%20nisl.%20Maecenas%20venenatis%20lacus%20ipsum%2C%20et%20tincidunt%20orci%20egestas%20vitae.%20Aliquam%20dapibus%20commodo%20nunc.%20Nunc%20faucibus%20at%20turpis%20et%20iaculis.Etiam%20ornare%20mi%20ut%20porta%20volutpat.%20Pellentesque%20habitant%20morbi%20tristique%20senectus%20et%20netus%20et%20malesuada%20fames%20ac%20turpis%20egestas.%20Ut%20volutpat%20suscipit%20aliquet.%20Nam%20consectetur%20mattis%20felis%2C%20non%20placerat%20turpis%20feugiat%20nec.%20Integer%20in%20neque%20consequat%2C%20aliquet%20felis%20nec%2C%20sodales%20ante.%20Aenean%20consectetur%20lacus%20id%20egestas%20tristique.%20Etiam%20non%20lacus%20sed%20elit%20pulvinar%20placerat.%20Vestibulum%20vel%20sagittis%20ex.%20Ut%20imperdiet%20molestie%20dolor%2C%20ut%20mattis%20quam%20mattis%20et.%20Vestibulum%20et%20orci%20sed%20odio%20faucibus%20hendrerit%20sit%20amet%20nec%20mauris.%20Vestibulum%20sed%20ipsum%20in%20magna%20iaculis%20eleifend%20in%20in%20lectus.Nullam%20auctor%20consectetur%20quam%20a%20malesuada.%20Nam%20vestibulum%20vulputate%20mollis.%20Etiam%20rhoncus%20porta%20sodales.%20Proin%20ullamcorper%20dapibus%20nulla%20id%20molestie.%20Nunc%20maximus%2C%20enim%20sed%20imperdiet%20dapibus%2C%20dolor%20dui%20semper%20orci%2C%20sed%20maximus%20diam%20justo%20faucibus%20purus.%20Fusce%20ac%20vestibulum%20velit.%20Nulla%20facilisi.%20Integer%20augue%20ex%2C%20consequat%20sed%20tempor%20non%2C%20accumsan%20vel%20elit.%20Donec%20sed%20mi%20felis.%20Proin%20blandit%20nec%20ipsum%20quis%20eleifend.%20Aliquam%20sed%20sem%20eu%20lacus%20sollicitudin%20auctor.%20Orci%20varius%20natoque%20penatibus%20et%20magnis%20dis%20parturient%20montes%2C%20nascetur%20ridiculus%20mus.%20Sed%20a%20nisl%20volutpat%2C%20finibus%20libero%20at%2C%20molestie%20dolor.%20Pellentesque%20volutpat%2C%20turpis%20et%20dignissim%20finibus%2C%20eros%20velit%20tincidunt%20eros%2C%20ac%20tristique%20orci%20ex%20nec%20justo.Morbi%20vitae%20lobortis%20tortor%2C%20sit%20amet%20tristique%20lacus.%20In%20at%20magna%20elit.%20Sed%20scelerisque%20nibh%20et%20risus%20dignissim%2C%20nec%20ultrices%20metus%20sollicitudin.%20Orci%20varius%20natoque%20penatibus%20et%20magnis%20dis%20parturient%20montes%2C%20nascetur%20ridiculus%20mus.%20Pellentesque%20vulputate%20magna%20quis%20commodo%20porta.%20Etiam%20sit%20amet%20lectus%20risus.%20Class%20aptent%20taciti%20sociosqu%20ad%20litora%20torquent%20per%20conubia%20nostra%2C%20per%20inceptos%20himenaeos.%20Vestibulum%20at%20auctor%20ligula%2C%20nec%20tempor%20ipsum.%20Aliquam%20id%20faucibus%20lectus.%20Suspendisse%20ut%20iaculis%20lorem%2C%20sit%20amet%20dapibus%20sem.%20Vivamus%20pharetra%20lobortis%20felis%20ut%20cursus.%20Nulla%20laoreet%20odio%20nec%20sapien%20porttitor%2C%20eu%20porttitor%20justo%20condimentum.%20Phasellus%20urna%20ligula%2C%20tempor%20vel%20efficitur%20in%2C%20blandit%20non%20nibh.%20Phasellus%20dapibus%20eu%20sem%20at%20maximus.%20Sed%20eu%20purus%20non%20lacus%20egestas%20semper%20quis%20sit%20amet%20neque.%20Maecenas%20tempus%20neque%20sed%20venenatis%20faucibus." 3 | wrk.headers["Content-Type"] = "application/x-www-form-urlencoded" 4 | 5 | -------------------------------------------------------------------------------- /service/MinMQ.BenchmarkConsole/Benchmarker.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Net.Http; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | using Microsoft.Extensions.Logging; 8 | using NodaTime; 9 | using Serilog; 10 | 11 | namespace MinMQ.BenchmarkConsole 12 | { 13 | public sealed class Benchmarker 14 | { 15 | private const int ConcurrentHttpRequests = 400; 16 | private readonly IHttpClientFactory httpClientFactory; 17 | private readonly int ntree; 18 | private readonly Duration showProgressEvery = Duration.FromMilliseconds(400); 19 | private readonly int numberOfObjects; 20 | private readonly CancellationToken cancellationToken; 21 | 22 | internal Benchmarker(IHttpClientFactory httpClientFactory, int ntree, int numberOfObjects, CancellationToken cancellationToken) 23 | { 24 | this.httpClientFactory = httpClientFactory; 25 | this.ntree = ntree; 26 | this.numberOfObjects = numberOfObjects; 27 | this.cancellationToken = cancellationToken; 28 | } 29 | 30 | public event OnCompleteDelegate OnComplete; 31 | 32 | internal async Task Start() 33 | { 34 | int jsonCount, xmlCount; 35 | (List jsons, List xmls) = TimedFunction(() => GenerateObjects(numberOfObjects, out jsons, out xmls)); 36 | jsonCount = jsons.Count; 37 | xmlCount = xmls.Count; 38 | 39 | Log.Information("Sending JSON and XML.."); 40 | Instant start = SystemClock.Instance.GetCurrentInstant(); 41 | 42 | // Old-school non-blocking 43 | await PostSendAsStringContent(jsons); 44 | await PostSendAsStringContent(xmls); 45 | Duration duration = SystemClock.Instance.GetCurrentInstant() - start; 46 | decimal throughtput = (jsonCount + xmlCount) / (decimal)duration.TotalSeconds; 47 | Log.Information("Done! {0:N2} documents/s (Xmls: {1}, Jsons={2}))", throughtput, xmlCount, jsonCount); 48 | OnComplete?.Invoke(); 49 | } 50 | 51 | private (List, List) GenerateObjects(int numberOfObjects, out List jsons, out List xmls) 52 | { 53 | Instant lastShowProgress = SystemClock.Instance.GetCurrentInstant(); 54 | jsons = new List(); 55 | xmls = new List(); 56 | var jsonGenerator = new JsonGenerator(ntree); 57 | var xmlGenerator = new XmlGenerator(ntree); 58 | 59 | Log.Information("Preparing payload"); 60 | for (int i = 0; i < numberOfObjects; i++) 61 | { 62 | if (cancellationToken.IsCancellationRequested) return (new List(), new List()); 63 | 64 | Instant now = SystemClock.Instance.GetCurrentInstant(); 65 | if (now - lastShowProgress > showProgressEvery) 66 | { 67 | lastShowProgress = now; 68 | Log.Information("{0} %", Math.Floor((decimal)i * 100 / numberOfObjects)); 69 | } 70 | 71 | jsons.Add(jsonGenerator.GenerateObject()); 72 | xmls.Add(xmlGenerator.GenerateObject()); 73 | } 74 | 75 | return (jsons, xmls); 76 | } 77 | 78 | private (List, List) TimedFunction(Func<(List, List)> action) 79 | { 80 | Instant start = SystemClock.Instance.GetCurrentInstant(); 81 | var objects = action(); 82 | Duration duration = SystemClock.Instance.GetCurrentInstant() - start; 83 | var perf = (objects.Item1.Count + objects.Item2.Count) / (decimal)duration.TotalSeconds; 84 | Log.Information("Done! {0:N2} documents/s", perf); 85 | return objects; 86 | } 87 | 88 | private async Task TimedFunction(Func> action, string name) 89 | { 90 | Log.Information("Sending XML.."); 91 | Instant start = SystemClock.Instance.GetCurrentInstant(); 92 | var count = await action(); 93 | Duration duration = SystemClock.Instance.GetCurrentInstant() - start; 94 | Log.Information("Done! {0:N2} documents/s", count / (decimal)duration.TotalSeconds); 95 | } 96 | 97 | private async Task PostSendAsStringContent(List documents) 98 | { 99 | List tasks = new List(); 100 | 101 | for (int j = 0; j < documents.Count; j++) 102 | { 103 | if (cancellationToken.IsCancellationRequested) return; 104 | 105 | HttpClient httpClient = httpClientFactory.CreateClient(); 106 | StringContent content = new StringContent(documents[j]); 107 | tasks.Add(httpClient.PostAsync("http://localhost:9000/send", content)); // It seems a CancellationToken here will fail the service. 108 | 109 | if (j % ConcurrentHttpRequests == 0 && j > 0) 110 | { 111 | await Task.WhenAll(tasks); 112 | tasks.Clear(); 113 | } 114 | } 115 | 116 | // Don't miss tail documents. 117 | if (tasks.Count > 0) 118 | { 119 | await Task.WhenAll(tasks); 120 | tasks.Clear(); 121 | } 122 | } 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Migrations/20191204203610_Initial database contruction.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.EntityFrameworkCore.Migrations; 3 | using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; 4 | 5 | namespace MinMq.Service.Migrations 6 | { 7 | public partial class Initialdatabasecontruction : Migration 8 | { 9 | protected override void Up(MigrationBuilder migrationBuilder) 10 | { 11 | migrationBuilder.CreateTable( 12 | name: "tCursors", 13 | columns: table => new 14 | { 15 | CursorId = table.Column(nullable: false) 16 | .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), 17 | Added = table.Column(nullable: false), 18 | Changed = table.Column(nullable: false), 19 | NextReferenceId = table.Column(nullable: false) 20 | }, 21 | constraints: table => 22 | { 23 | table.PrimaryKey("PK_tCursors", x => x.CursorId); 24 | }); 25 | 26 | migrationBuilder.CreateTable( 27 | name: "tMimeTypes", 28 | columns: table => new 29 | { 30 | MimeTypeId = table.Column(nullable: false) 31 | .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), 32 | Expression = table.Column(nullable: true), 33 | Added = table.Column(nullable: false), 34 | Changed = table.Column(nullable: false) 35 | }, 36 | constraints: table => 37 | { 38 | table.PrimaryKey("PK_tMimeTypes", x => x.MimeTypeId); 39 | }); 40 | 41 | migrationBuilder.CreateTable( 42 | name: "tQueues", 43 | columns: table => new 44 | { 45 | QueueId = table.Column(nullable: false) 46 | .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), 47 | Name = table.Column(nullable: true), 48 | Added = table.Column(nullable: false), 49 | Changed = table.Column(nullable: false) 50 | }, 51 | constraints: table => 52 | { 53 | table.PrimaryKey("PK_tQueues", x => x.QueueId); 54 | }); 55 | 56 | migrationBuilder.CreateTable( 57 | name: "tMessages", 58 | columns: table => new 59 | { 60 | MessageId = table.Column(nullable: false) 61 | .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), 62 | ReferenceId = table.Column(nullable: false), 63 | NextReferenceId = table.Column(nullable: false), 64 | Content = table.Column(nullable: true), 65 | QueueId = table.Column(nullable: false), 66 | MimeTypeId = table.Column(nullable: false), 67 | HashCode = table.Column(nullable: true), 68 | Added = table.Column(nullable: false), 69 | Changed = table.Column(nullable: false) 70 | }, 71 | constraints: table => 72 | { 73 | table.PrimaryKey("PK_tMessages", x => x.MessageId); 74 | table.ForeignKey( 75 | name: "FK_Message_MimeType", 76 | column: x => x.MimeTypeId, 77 | principalTable: "tMimeTypes", 78 | principalColumn: "MimeTypeId", 79 | onDelete: ReferentialAction.Cascade); 80 | table.ForeignKey( 81 | name: "FK_Message_Queue", 82 | column: x => x.QueueId, 83 | principalTable: "tQueues", 84 | principalColumn: "QueueId", 85 | onDelete: ReferentialAction.Cascade); 86 | }); 87 | 88 | migrationBuilder.CreateIndex( 89 | name: "IX_tMessages_MimeTypeId", 90 | table: "tMessages", 91 | column: "MimeTypeId"); 92 | 93 | migrationBuilder.CreateIndex( 94 | name: "IX_tMessages_QueueId", 95 | table: "tMessages", 96 | column: "QueueId"); 97 | } 98 | 99 | protected override void Down(MigrationBuilder migrationBuilder) 100 | { 101 | migrationBuilder.DropTable( 102 | name: "tCursors"); 103 | 104 | migrationBuilder.DropTable( 105 | name: "tMessages"); 106 | 107 | migrationBuilder.DropTable( 108 | name: "tMimeTypes"); 109 | 110 | migrationBuilder.DropTable( 111 | name: "tQueues"); 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /service/StyleCop.ruleset: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | -------------------------------------------------------------------------------- /service/MinMQ.Service/MinMQ.Service.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp3.0 5 | MinMq.Service 6 | Debug;Release;Troubleshoot 7 | 8 | 9 | 10 | 11 | 12 | ..\StyleCop.ruleset 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | all 63 | runtime; build; native; contentfiles; analyzers; buildtransitive 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | all 73 | runtime; build; native; contentfiles; analyzers; buildtransitive 74 | 75 | 76 | 77 | 78 | 79 | 80 | Always 81 | 82 | 83 | Always 84 | 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /benchmark/service-hapi/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Basic Options */ 4 | "target": "ES2018", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */ 5 | "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ 6 | // "lib": [], /* Specify library files to be included in the compilation. */ 7 | // "allowJs": true, /* Allow javascript files to be compiled. */ 8 | // "checkJs": true, /* Report errors in .js files. */ 9 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ 10 | // "declaration": true, /* Generates corresponding '.d.ts' file. */ 11 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ 12 | "sourceMap": true, /* Generates corresponding '.map' file. */ 13 | // "outFile": "./", /* Concatenate and emit output to single file. */ 14 | "outDir": "dist", /* Redirect output structure to the directory. */ 15 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 16 | // "composite": true, /* Enable project compilation */ 17 | // "incremental": true, /* Enable incremental compilation */ 18 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ 19 | // "removeComments": true, /* Do not emit comments to output. */ 20 | // "noEmit": true, /* Do not emit outputs. */ 21 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */ 22 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 23 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 24 | /* Strict Type-Checking Options */ 25 | "strict": true, /* Enable all strict type-checking options. */ 26 | "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ 27 | "strictNullChecks": true, /* Enable strict null checks. */ 28 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */ 29 | // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ 30 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ 31 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 32 | "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 33 | /* Additional Checks */ 34 | // "noUnusedLocals": true, /* Report errors on unused locals. */ 35 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 36 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 37 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 38 | /* Module Resolution Options */ 39 | // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 40 | // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ 41 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ 42 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 43 | // "typeRoots": [], /* List of folders to include type definitions from. */ 44 | // "types": [], /* Type declaration files to be included in compilation. */ 45 | "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 46 | "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ 47 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 48 | /* Source Map Options */ 49 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 50 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 51 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 52 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 53 | /* Experimental Options */ 54 | // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ 55 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 56 | } 57 | } -------------------------------------------------------------------------------- /benchmark/service-express/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Basic Options */ 4 | "target": "ES2018", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */ 5 | "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ 6 | // "lib": [], /* Specify library files to be included in the compilation. */ 7 | // "allowJs": true, /* Allow javascript files to be compiled. */ 8 | // "checkJs": true, /* Report errors in .js files. */ 9 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ 10 | // "declaration": true, /* Generates corresponding '.d.ts' file. */ 11 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ 12 | "sourceMap": true, /* Generates corresponding '.map' file. */ 13 | // "outFile": "./", /* Concatenate and emit output to single file. */ 14 | "outDir": "dist", /* Redirect output structure to the directory. */ 15 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 16 | // "composite": true, /* Enable project compilation */ 17 | // "incremental": true, /* Enable incremental compilation */ 18 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ 19 | // "removeComments": true, /* Do not emit comments to output. */ 20 | // "noEmit": true, /* Do not emit outputs. */ 21 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */ 22 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 23 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 24 | /* Strict Type-Checking Options */ 25 | "strict": true, /* Enable all strict type-checking options. */ 26 | "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ 27 | "strictNullChecks": true, /* Enable strict null checks. */ 28 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */ 29 | // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ 30 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ 31 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 32 | "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 33 | /* Additional Checks */ 34 | // "noUnusedLocals": true, /* Report errors on unused locals. */ 35 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 36 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 37 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 38 | /* Module Resolution Options */ 39 | // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 40 | // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ 41 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ 42 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 43 | // "typeRoots": [], /* List of folders to include type definitions from. */ 44 | // "types": [], /* Type declaration files to be included in compilation. */ 45 | "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 46 | "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ 47 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 48 | /* Source Map Options */ 49 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 50 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 51 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 52 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 53 | /* Experimental Options */ 54 | // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ 55 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 56 | } 57 | } -------------------------------------------------------------------------------- /service/MinMQ.Service/Migrations/MessageQueueContextModelSnapshot.cs: -------------------------------------------------------------------------------- 1 | // 2 | using System; 3 | using Microsoft.EntityFrameworkCore; 4 | using Microsoft.EntityFrameworkCore.Infrastructure; 5 | using Microsoft.EntityFrameworkCore.Storage.ValueConversion; 6 | using MinMq.Service.Models; 7 | using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; 8 | 9 | namespace MinMq.Service.Migrations 10 | { 11 | [DbContext(typeof(MessageQueueContext))] 12 | partial class MessageQueueContextModelSnapshot : ModelSnapshot 13 | { 14 | protected override void BuildModel(ModelBuilder modelBuilder) 15 | { 16 | #pragma warning disable 612, 618 17 | modelBuilder 18 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn) 19 | .HasAnnotation("ProductVersion", "3.0.0") 20 | .HasAnnotation("Relational:MaxIdentifierLength", 63); 21 | 22 | modelBuilder.Entity("MinMq.Service.Models.tCursor", b => 23 | { 24 | b.Property("CursorId") 25 | .ValueGeneratedOnAdd() 26 | .HasColumnType("integer") 27 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 28 | 29 | b.Property("Added") 30 | .HasColumnType("timestamp without time zone"); 31 | 32 | b.Property("Changed") 33 | .HasColumnType("timestamp without time zone"); 34 | 35 | b.Property("NextReferenceId") 36 | .HasColumnType("bigint"); 37 | 38 | b.HasKey("CursorId"); 39 | 40 | b.ToTable("tCursors"); 41 | }); 42 | 43 | modelBuilder.Entity("MinMq.Service.Models.tMessage", b => 44 | { 45 | b.Property("MessageId") 46 | .ValueGeneratedOnAdd() 47 | .HasColumnType("integer") 48 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 49 | 50 | b.Property("Added") 51 | .HasColumnType("timestamp without time zone"); 52 | 53 | b.Property("Changed") 54 | .HasColumnType("timestamp without time zone"); 55 | 56 | b.Property("Content") 57 | .HasColumnType("text"); 58 | 59 | b.Property("HashCode") 60 | .HasColumnType("text"); 61 | 62 | b.Property("MimeTypeId") 63 | .HasColumnType("smallint"); 64 | 65 | b.Property("NextReferenceId") 66 | .HasColumnType("bigint"); 67 | 68 | b.Property("QueueId") 69 | .HasColumnType("smallint"); 70 | 71 | b.Property("ReferenceId") 72 | .HasColumnType("bigint"); 73 | 74 | b.HasKey("MessageId"); 75 | 76 | b.HasIndex("MimeTypeId"); 77 | 78 | b.HasIndex("QueueId"); 79 | 80 | b.ToTable("tMessages"); 81 | }); 82 | 83 | modelBuilder.Entity("MinMq.Service.Models.tMimeType", b => 84 | { 85 | b.Property("MimeTypeId") 86 | .ValueGeneratedOnAdd() 87 | .HasColumnType("smallint") 88 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 89 | 90 | b.Property("Added") 91 | .HasColumnType("timestamp without time zone"); 92 | 93 | b.Property("Changed") 94 | .HasColumnType("timestamp without time zone"); 95 | 96 | b.Property("Expression") 97 | .HasColumnType("text"); 98 | 99 | b.HasKey("MimeTypeId"); 100 | 101 | b.ToTable("tMimeTypes"); 102 | }); 103 | 104 | modelBuilder.Entity("MinMq.Service.Models.tQueue", b => 105 | { 106 | b.Property("QueueId") 107 | .ValueGeneratedOnAdd() 108 | .HasColumnType("smallint") 109 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 110 | 111 | b.Property("Added") 112 | .HasColumnType("timestamp without time zone"); 113 | 114 | b.Property("Changed") 115 | .HasColumnType("timestamp without time zone"); 116 | 117 | b.Property("Name") 118 | .HasColumnType("text"); 119 | 120 | b.HasKey("QueueId"); 121 | 122 | b.ToTable("tQueues"); 123 | }); 124 | 125 | modelBuilder.Entity("MinMq.Service.Models.tMessage", b => 126 | { 127 | b.HasOne("MinMq.Service.Models.tMimeType", "MimeType") 128 | .WithMany("tMessages") 129 | .HasForeignKey("MimeTypeId") 130 | .HasConstraintName("FK_Message_MimeType") 131 | .OnDelete(DeleteBehavior.Cascade) 132 | .IsRequired(); 133 | 134 | b.HasOne("MinMq.Service.Models.tQueue", "Queue") 135 | .WithMany("Messages") 136 | .HasForeignKey("QueueId") 137 | .HasConstraintName("FK_Message_Queue") 138 | .OnDelete(DeleteBehavior.Cascade) 139 | .IsRequired(); 140 | }); 141 | #pragma warning restore 612, 618 142 | } 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Migrations/20191204211704_Minor changes.Designer.cs: -------------------------------------------------------------------------------- 1 | // 2 | using System; 3 | using Microsoft.EntityFrameworkCore; 4 | using Microsoft.EntityFrameworkCore.Infrastructure; 5 | using Microsoft.EntityFrameworkCore.Migrations; 6 | using Microsoft.EntityFrameworkCore.Storage.ValueConversion; 7 | using MinMq.Service.Models; 8 | using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; 9 | 10 | namespace MinMq.Service.Migrations 11 | { 12 | [DbContext(typeof(MessageQueueContext))] 13 | [Migration("20191204211704_Minor changes")] 14 | partial class Minorchanges 15 | { 16 | protected override void BuildTargetModel(ModelBuilder modelBuilder) 17 | { 18 | #pragma warning disable 612, 618 19 | modelBuilder 20 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn) 21 | .HasAnnotation("ProductVersion", "3.0.0") 22 | .HasAnnotation("Relational:MaxIdentifierLength", 63); 23 | 24 | modelBuilder.Entity("MinMq.Service.Models.tCursor", b => 25 | { 26 | b.Property("CursorId") 27 | .ValueGeneratedOnAdd() 28 | .HasColumnType("integer") 29 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 30 | 31 | b.Property("Added") 32 | .HasColumnType("timestamp without time zone"); 33 | 34 | b.Property("Changed") 35 | .HasColumnType("timestamp without time zone"); 36 | 37 | b.Property("NextReferenceId") 38 | .HasColumnType("bigint"); 39 | 40 | b.HasKey("CursorId"); 41 | 42 | b.ToTable("tCursors"); 43 | }); 44 | 45 | modelBuilder.Entity("MinMq.Service.Models.tMessage", b => 46 | { 47 | b.Property("MessageId") 48 | .ValueGeneratedOnAdd() 49 | .HasColumnType("integer") 50 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 51 | 52 | b.Property("Added") 53 | .HasColumnType("timestamp without time zone"); 54 | 55 | b.Property("Changed") 56 | .HasColumnType("timestamp without time zone"); 57 | 58 | b.Property("Content") 59 | .HasColumnType("text"); 60 | 61 | b.Property("HashCode") 62 | .HasColumnType("text"); 63 | 64 | b.Property("MimeTypeId") 65 | .HasColumnType("smallint"); 66 | 67 | b.Property("NextReferenceId") 68 | .HasColumnType("bigint"); 69 | 70 | b.Property("QueueId") 71 | .HasColumnType("smallint"); 72 | 73 | b.Property("ReferenceId") 74 | .HasColumnType("bigint"); 75 | 76 | b.HasKey("MessageId"); 77 | 78 | b.HasIndex("MimeTypeId"); 79 | 80 | b.HasIndex("QueueId"); 81 | 82 | b.ToTable("tMessages"); 83 | }); 84 | 85 | modelBuilder.Entity("MinMq.Service.Models.tMimeType", b => 86 | { 87 | b.Property("MimeTypeId") 88 | .ValueGeneratedOnAdd() 89 | .HasColumnType("smallint") 90 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 91 | 92 | b.Property("Added") 93 | .HasColumnType("timestamp without time zone"); 94 | 95 | b.Property("Changed") 96 | .HasColumnType("timestamp without time zone"); 97 | 98 | b.Property("Expression") 99 | .HasColumnType("text"); 100 | 101 | b.HasKey("MimeTypeId"); 102 | 103 | b.ToTable("tMimeTypes"); 104 | }); 105 | 106 | modelBuilder.Entity("MinMq.Service.Models.tQueue", b => 107 | { 108 | b.Property("QueueId") 109 | .ValueGeneratedOnAdd() 110 | .HasColumnType("smallint") 111 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 112 | 113 | b.Property("Added") 114 | .HasColumnType("timestamp without time zone"); 115 | 116 | b.Property("Changed") 117 | .HasColumnType("timestamp without time zone"); 118 | 119 | b.Property("Name") 120 | .HasColumnType("text"); 121 | 122 | b.HasKey("QueueId"); 123 | 124 | b.ToTable("tQueues"); 125 | }); 126 | 127 | modelBuilder.Entity("MinMq.Service.Models.tMessage", b => 128 | { 129 | b.HasOne("MinMq.Service.Models.tMimeType", "MimeType") 130 | .WithMany("tMessages") 131 | .HasForeignKey("MimeTypeId") 132 | .HasConstraintName("FK_Message_MimeType") 133 | .OnDelete(DeleteBehavior.Cascade) 134 | .IsRequired(); 135 | 136 | b.HasOne("MinMq.Service.Models.tQueue", "Queue") 137 | .WithMany("Messages") 138 | .HasForeignKey("QueueId") 139 | .HasConstraintName("FK_Message_Queue") 140 | .OnDelete(DeleteBehavior.Cascade) 141 | .IsRequired(); 142 | }); 143 | #pragma warning restore 612, 618 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Migrations/20191204211817_Add MimeType.Designer.cs: -------------------------------------------------------------------------------- 1 | // 2 | using System; 3 | using Microsoft.EntityFrameworkCore; 4 | using Microsoft.EntityFrameworkCore.Infrastructure; 5 | using Microsoft.EntityFrameworkCore.Migrations; 6 | using Microsoft.EntityFrameworkCore.Storage.ValueConversion; 7 | using MinMq.Service.Models; 8 | using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; 9 | 10 | namespace MinMq.Service.Migrations 11 | { 12 | [DbContext(typeof(MessageQueueContext))] 13 | [Migration("20191204211817_Add MimeType")] 14 | partial class AddMimeType 15 | { 16 | protected override void BuildTargetModel(ModelBuilder modelBuilder) 17 | { 18 | #pragma warning disable 612, 618 19 | modelBuilder 20 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn) 21 | .HasAnnotation("ProductVersion", "3.0.0") 22 | .HasAnnotation("Relational:MaxIdentifierLength", 63); 23 | 24 | modelBuilder.Entity("MinMq.Service.Models.tCursor", b => 25 | { 26 | b.Property("CursorId") 27 | .ValueGeneratedOnAdd() 28 | .HasColumnType("integer") 29 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 30 | 31 | b.Property("Added") 32 | .HasColumnType("timestamp without time zone"); 33 | 34 | b.Property("Changed") 35 | .HasColumnType("timestamp without time zone"); 36 | 37 | b.Property("NextReferenceId") 38 | .HasColumnType("bigint"); 39 | 40 | b.HasKey("CursorId"); 41 | 42 | b.ToTable("tCursors"); 43 | }); 44 | 45 | modelBuilder.Entity("MinMq.Service.Models.tMessage", b => 46 | { 47 | b.Property("MessageId") 48 | .ValueGeneratedOnAdd() 49 | .HasColumnType("integer") 50 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 51 | 52 | b.Property("Added") 53 | .HasColumnType("timestamp without time zone"); 54 | 55 | b.Property("Changed") 56 | .HasColumnType("timestamp without time zone"); 57 | 58 | b.Property("Content") 59 | .HasColumnType("text"); 60 | 61 | b.Property("HashCode") 62 | .HasColumnType("text"); 63 | 64 | b.Property("MimeTypeId") 65 | .HasColumnType("smallint"); 66 | 67 | b.Property("NextReferenceId") 68 | .HasColumnType("bigint"); 69 | 70 | b.Property("QueueId") 71 | .HasColumnType("smallint"); 72 | 73 | b.Property("ReferenceId") 74 | .HasColumnType("bigint"); 75 | 76 | b.HasKey("MessageId"); 77 | 78 | b.HasIndex("MimeTypeId"); 79 | 80 | b.HasIndex("QueueId"); 81 | 82 | b.ToTable("tMessages"); 83 | }); 84 | 85 | modelBuilder.Entity("MinMq.Service.Models.tMimeType", b => 86 | { 87 | b.Property("MimeTypeId") 88 | .ValueGeneratedOnAdd() 89 | .HasColumnType("smallint") 90 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 91 | 92 | b.Property("Added") 93 | .HasColumnType("timestamp without time zone"); 94 | 95 | b.Property("Changed") 96 | .HasColumnType("timestamp without time zone"); 97 | 98 | b.Property("Expression") 99 | .HasColumnType("text"); 100 | 101 | b.HasKey("MimeTypeId"); 102 | 103 | b.ToTable("tMimeTypes"); 104 | }); 105 | 106 | modelBuilder.Entity("MinMq.Service.Models.tQueue", b => 107 | { 108 | b.Property("QueueId") 109 | .ValueGeneratedOnAdd() 110 | .HasColumnType("smallint") 111 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 112 | 113 | b.Property("Added") 114 | .HasColumnType("timestamp without time zone"); 115 | 116 | b.Property("Changed") 117 | .HasColumnType("timestamp without time zone"); 118 | 119 | b.Property("Name") 120 | .HasColumnType("text"); 121 | 122 | b.HasKey("QueueId"); 123 | 124 | b.ToTable("tQueues"); 125 | }); 126 | 127 | modelBuilder.Entity("MinMq.Service.Models.tMessage", b => 128 | { 129 | b.HasOne("MinMq.Service.Models.tMimeType", "MimeType") 130 | .WithMany("tMessages") 131 | .HasForeignKey("MimeTypeId") 132 | .HasConstraintName("FK_Message_MimeType") 133 | .OnDelete(DeleteBehavior.Cascade) 134 | .IsRequired(); 135 | 136 | b.HasOne("MinMq.Service.Models.tQueue", "Queue") 137 | .WithMany("Messages") 138 | .HasForeignKey("QueueId") 139 | .HasConstraintName("FK_Message_Queue") 140 | .OnDelete(DeleteBehavior.Cascade) 141 | .IsRequired(); 142 | }); 143 | #pragma warning restore 612, 618 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Migrations/20191204215508_Add a lot more mime types.Designer.cs: -------------------------------------------------------------------------------- 1 | // 2 | using System; 3 | using Microsoft.EntityFrameworkCore; 4 | using Microsoft.EntityFrameworkCore.Infrastructure; 5 | using Microsoft.EntityFrameworkCore.Migrations; 6 | using Microsoft.EntityFrameworkCore.Storage.ValueConversion; 7 | using MinMq.Service.Models; 8 | using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; 9 | 10 | namespace MinMq.Service.Migrations 11 | { 12 | [DbContext(typeof(MessageQueueContext))] 13 | [Migration("20191204215508_Add a lot more mime types")] 14 | partial class Addalotmoremimetypes 15 | { 16 | protected override void BuildTargetModel(ModelBuilder modelBuilder) 17 | { 18 | #pragma warning disable 612, 618 19 | modelBuilder 20 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn) 21 | .HasAnnotation("ProductVersion", "3.0.0") 22 | .HasAnnotation("Relational:MaxIdentifierLength", 63); 23 | 24 | modelBuilder.Entity("MinMq.Service.Models.tCursor", b => 25 | { 26 | b.Property("CursorId") 27 | .ValueGeneratedOnAdd() 28 | .HasColumnType("integer") 29 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 30 | 31 | b.Property("Added") 32 | .HasColumnType("timestamp without time zone"); 33 | 34 | b.Property("Changed") 35 | .HasColumnType("timestamp without time zone"); 36 | 37 | b.Property("NextReferenceId") 38 | .HasColumnType("bigint"); 39 | 40 | b.HasKey("CursorId"); 41 | 42 | b.ToTable("tCursors"); 43 | }); 44 | 45 | modelBuilder.Entity("MinMq.Service.Models.tMessage", b => 46 | { 47 | b.Property("MessageId") 48 | .ValueGeneratedOnAdd() 49 | .HasColumnType("integer") 50 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 51 | 52 | b.Property("Added") 53 | .HasColumnType("timestamp without time zone"); 54 | 55 | b.Property("Changed") 56 | .HasColumnType("timestamp without time zone"); 57 | 58 | b.Property("Content") 59 | .HasColumnType("text"); 60 | 61 | b.Property("HashCode") 62 | .HasColumnType("text"); 63 | 64 | b.Property("MimeTypeId") 65 | .HasColumnType("smallint"); 66 | 67 | b.Property("NextReferenceId") 68 | .HasColumnType("bigint"); 69 | 70 | b.Property("QueueId") 71 | .HasColumnType("smallint"); 72 | 73 | b.Property("ReferenceId") 74 | .HasColumnType("bigint"); 75 | 76 | b.HasKey("MessageId"); 77 | 78 | b.HasIndex("MimeTypeId"); 79 | 80 | b.HasIndex("QueueId"); 81 | 82 | b.ToTable("tMessages"); 83 | }); 84 | 85 | modelBuilder.Entity("MinMq.Service.Models.tMimeType", b => 86 | { 87 | b.Property("MimeTypeId") 88 | .ValueGeneratedOnAdd() 89 | .HasColumnType("smallint") 90 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 91 | 92 | b.Property("Added") 93 | .HasColumnType("timestamp without time zone"); 94 | 95 | b.Property("Changed") 96 | .HasColumnType("timestamp without time zone"); 97 | 98 | b.Property("Expression") 99 | .HasColumnType("text"); 100 | 101 | b.HasKey("MimeTypeId"); 102 | 103 | b.ToTable("tMimeTypes"); 104 | }); 105 | 106 | modelBuilder.Entity("MinMq.Service.Models.tQueue", b => 107 | { 108 | b.Property("QueueId") 109 | .ValueGeneratedOnAdd() 110 | .HasColumnType("smallint") 111 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 112 | 113 | b.Property("Added") 114 | .HasColumnType("timestamp without time zone"); 115 | 116 | b.Property("Changed") 117 | .HasColumnType("timestamp without time zone"); 118 | 119 | b.Property("Name") 120 | .HasColumnType("text"); 121 | 122 | b.HasKey("QueueId"); 123 | 124 | b.ToTable("tQueues"); 125 | }); 126 | 127 | modelBuilder.Entity("MinMq.Service.Models.tMessage", b => 128 | { 129 | b.HasOne("MinMq.Service.Models.tMimeType", "MimeType") 130 | .WithMany("tMessages") 131 | .HasForeignKey("MimeTypeId") 132 | .HasConstraintName("FK_Message_MimeType") 133 | .OnDelete(DeleteBehavior.Cascade) 134 | .IsRequired(); 135 | 136 | b.HasOne("MinMq.Service.Models.tQueue", "Queue") 137 | .WithMany("Messages") 138 | .HasForeignKey("QueueId") 139 | .HasConstraintName("FK_Message_Queue") 140 | .OnDelete(DeleteBehavior.Cascade) 141 | .IsRequired(); 142 | }); 143 | #pragma warning restore 612, 618 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Migrations/20191204203610_Initial database contruction.Designer.cs: -------------------------------------------------------------------------------- 1 | // 2 | using System; 3 | using Microsoft.EntityFrameworkCore; 4 | using Microsoft.EntityFrameworkCore.Infrastructure; 5 | using Microsoft.EntityFrameworkCore.Migrations; 6 | using Microsoft.EntityFrameworkCore.Storage.ValueConversion; 7 | using MinMq.Service.Models; 8 | using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; 9 | 10 | namespace MinMq.Service.Migrations 11 | { 12 | [DbContext(typeof(MessageQueueContext))] 13 | [Migration("20191204203610_Initial database contruction")] 14 | partial class Initialdatabasecontruction 15 | { 16 | protected override void BuildTargetModel(ModelBuilder modelBuilder) 17 | { 18 | #pragma warning disable 612, 618 19 | modelBuilder 20 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn) 21 | .HasAnnotation("ProductVersion", "3.0.0") 22 | .HasAnnotation("Relational:MaxIdentifierLength", 63); 23 | 24 | modelBuilder.Entity("MinMq.Service.Models.tCursor", b => 25 | { 26 | b.Property("CursorId") 27 | .ValueGeneratedOnAdd() 28 | .HasColumnType("integer") 29 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 30 | 31 | b.Property("Added") 32 | .HasColumnType("timestamp without time zone"); 33 | 34 | b.Property("Changed") 35 | .HasColumnType("timestamp without time zone"); 36 | 37 | b.Property("NextReferenceId") 38 | .HasColumnType("bigint"); 39 | 40 | b.HasKey("CursorId"); 41 | 42 | b.ToTable("tCursors"); 43 | }); 44 | 45 | modelBuilder.Entity("MinMq.Service.Models.tMessage", b => 46 | { 47 | b.Property("MessageId") 48 | .ValueGeneratedOnAdd() 49 | .HasColumnType("integer") 50 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 51 | 52 | b.Property("Added") 53 | .HasColumnType("timestamp without time zone"); 54 | 55 | b.Property("Changed") 56 | .HasColumnType("timestamp without time zone"); 57 | 58 | b.Property("Content") 59 | .HasColumnType("text"); 60 | 61 | b.Property("HashCode") 62 | .HasColumnType("text"); 63 | 64 | b.Property("MimeTypeId") 65 | .HasColumnType("smallint"); 66 | 67 | b.Property("NextReferenceId") 68 | .HasColumnType("bigint"); 69 | 70 | b.Property("QueueId") 71 | .HasColumnType("smallint"); 72 | 73 | b.Property("ReferenceId") 74 | .HasColumnType("bigint"); 75 | 76 | b.HasKey("MessageId"); 77 | 78 | b.HasIndex("MimeTypeId"); 79 | 80 | b.HasIndex("QueueId"); 81 | 82 | b.ToTable("tMessages"); 83 | }); 84 | 85 | modelBuilder.Entity("MinMq.Service.Models.tMimeType", b => 86 | { 87 | b.Property("MimeTypeId") 88 | .ValueGeneratedOnAdd() 89 | .HasColumnType("smallint") 90 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 91 | 92 | b.Property("Added") 93 | .HasColumnType("timestamp without time zone"); 94 | 95 | b.Property("Changed") 96 | .HasColumnType("timestamp without time zone"); 97 | 98 | b.Property("Expression") 99 | .HasColumnType("text"); 100 | 101 | b.HasKey("MimeTypeId"); 102 | 103 | b.ToTable("tMimeTypes"); 104 | }); 105 | 106 | modelBuilder.Entity("MinMq.Service.Models.tQueue", b => 107 | { 108 | b.Property("QueueId") 109 | .ValueGeneratedOnAdd() 110 | .HasColumnType("smallint") 111 | .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); 112 | 113 | b.Property("Added") 114 | .HasColumnType("timestamp without time zone"); 115 | 116 | b.Property("Changed") 117 | .HasColumnType("timestamp without time zone"); 118 | 119 | b.Property("Name") 120 | .HasColumnType("text"); 121 | 122 | b.HasKey("QueueId"); 123 | 124 | b.ToTable("tQueues"); 125 | }); 126 | 127 | modelBuilder.Entity("MinMq.Service.Models.tMessage", b => 128 | { 129 | b.HasOne("MinMq.Service.Models.tMimeType", "MimeType") 130 | .WithMany("tMessages") 131 | .HasForeignKey("MimeTypeId") 132 | .HasConstraintName("FK_Message_MimeType") 133 | .OnDelete(DeleteBehavior.Cascade) 134 | .IsRequired(); 135 | 136 | b.HasOne("MinMq.Service.Models.tQueue", "Queue") 137 | .WithMany("Messages") 138 | .HasForeignKey("QueueId") 139 | .HasConstraintName("FK_Message_Queue") 140 | .OnDelete(DeleteBehavior.Cascade) 141 | .IsRequired(); 142 | }); 143 | #pragma warning restore 612, 618 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Faster/FasterHostedServiceMoveData.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | using Microsoft.Extensions.DependencyInjection; 7 | using Microsoft.Extensions.Hosting; 8 | using Microsoft.Extensions.Logging; 9 | using Microsoft.Extensions.Options; 10 | using MinMQ.Service.Configuration; 11 | using MinMq.Service.Entities; 12 | using MinMq.Service.Repository; 13 | using NodaTime; 14 | 15 | namespace MinMQ.Service.Faster 16 | { 17 | internal delegate void EndOfFileCallback(long address); 18 | internal delegate short MimeTypeDecider(string content); 19 | 20 | /// 21 | /// A hosted service that moves stuff from the FASTER log to some EF-providers data context. 22 | /// 23 | public class FasterHostedServiceMoveData : IHostedService, IDisposable 24 | { 25 | private const int DelayMs = 1000; 26 | private readonly ILogger logger; 27 | private readonly IServiceScopeFactory scopeFactory; 28 | private readonly IOptionsMonitor optionsMonitor; 29 | private int delayCoefficient = 1; 30 | private Cursor currentCursor = null; 31 | 32 | public FasterHostedServiceMoveData(ILogger logger, IServiceScopeFactory scopeFactory, IOptionsMonitor optionsMonitor) 33 | { 34 | this.logger = logger; 35 | this.scopeFactory = scopeFactory; 36 | this.optionsMonitor = optionsMonitor; 37 | } 38 | 39 | public async Task StartAsync(CancellationToken stoppingToken) 40 | { 41 | logger.LogInformation("{0} service running.", nameof(FasterHostedServiceMoveData)); 42 | 43 | using (var scope = scopeFactory.CreateScope()) 44 | { 45 | using (var queueRepository = scope.ServiceProvider.GetRequiredService()) 46 | using (var mimeTypeRepository = scope.ServiceProvider.GetRequiredService()) 47 | using (var messageRepository = scope.ServiceProvider.GetRequiredService()) 48 | using (var cursorRepository = scope.ServiceProvider.GetRequiredService()) 49 | { 50 | short queueId = await queueRepository.FindOr("some", async () => await queueRepository.Add(new Queue("some"))); 51 | 52 | if (currentCursor == null) 53 | { 54 | int cursorId = await cursorRepository.Add(new Cursor()); 55 | currentCursor = await cursorRepository.Find(cursorId); 56 | } 57 | 58 | while (true) 59 | { 60 | await Task.Delay(DelayMs * delayCoefficient); 61 | delayCoefficient = 1; 62 | // Have a sneaking suspicions we should keep the nextAddress from the last iteration. If we remove truncate 63 | // then the cursor doesn't progress and flush next batch. 64 | Instant start = SystemClock.Instance.GetCurrentInstant(); 65 | var scanner = FasterOps.Instance.Value.Listen(optionsMonitor.CurrentValue.ScanFlushSize, currentCursor); 66 | // var messages = await ToListAsync(scanner, FasterOps.Instance.Value.TruncateUntil); 67 | 68 | // Sloppy write (queue name and mime type should probably be known before) 69 | MimeType mimeTypeXml = (await mimeTypeRepository.Find("text/xml")).ValueOr(() => throw new ApplicationException("xml")); 70 | MimeType mimeTypeJson = (await mimeTypeRepository.Find("application/json")).ValueOr(() => throw new ApplicationException("json")); 71 | MimeTypeDecider decider = c => c.StartsWith("<") ? mimeTypeXml.MimeTypeId : mimeTypeJson.MimeTypeId; 72 | var messages = ToList(scanner, decider, queueId); 73 | 74 | if (!messages.Any()) 75 | { 76 | delayCoefficient = 5; 77 | logger.LogInformation("Nothing to flush"); 78 | continue; 79 | } 80 | 81 | var lastReferenceId = await messageRepository.AddRange(messages); 82 | currentCursor.Set(lastReferenceId); 83 | lastReferenceId.MatchSome(referenceId => FasterOps.Instance.Value.TruncateUntil(referenceId)); 84 | await cursorRepository.Update(currentCursor); 85 | Duration elapsed = SystemClock.Instance.GetCurrentInstant() - start; 86 | logger.LogInformation("Flushed {0} records at {0:N2} documents/s", messages.Count, messages.Count / elapsed.TotalSeconds); 87 | } 88 | } 89 | } 90 | } 91 | 92 | // Maybe this also should be IAsyncEnumerable. Or, maybe not yet.. 93 | // private async Task> ToListAsync(IAsyncEnumerable<(string, long, long)> scan, EndOfFileCallback endOfFileCallback) 94 | // { 95 | // var messages = new List(); 96 | 97 | // await foreach ((string content, long referenceId, long nextReferenceId) in scan) 98 | // { 99 | // // I'm guessing this is out of bounds for the current storage config 100 | // if (nextReferenceId > 1_000_000_000) 101 | // { 102 | // logger.LogError("Reached end of IDevice"); 103 | // // Debugger.Break(); 104 | // endOfFileCallback(nextReferenceId); 105 | // continue; 106 | // } 107 | // messages.Add(new Message(content, referenceId, nextReferenceId)); 108 | // } 109 | 110 | // return messages; 111 | // } 112 | 113 | // private List ToList(List<(string, long, long)> scan, EndOfFileCallback endOfFileCallback, MimeTypeDecider mimeTypeDecider, short queueId) 114 | private List ToList(List<(string, long, long)> scan, MimeTypeDecider mimeTypeDecider, short queueId) 115 | { 116 | var messages = new List(); 117 | 118 | foreach ((string content, long referenceId, long nextReferenceId) in scan) 119 | { 120 | // I'm guessing this is out of bounds for the current storage config 121 | //if (nextReferenceId > 1_000_000_000) 122 | //{ 123 | // logger.LogError("Reached end of IDevice"); 124 | // // Debugger.Break(); 125 | // endOfFileCallback(nextReferenceId); 126 | // continue; 127 | //} 128 | messages.Add(new Message(content, referenceId, nextReferenceId, mimeTypeDecider(content), queueId)); 129 | } 130 | 131 | return messages; 132 | } 133 | 134 | public Task StopAsync(CancellationToken stoppingToken) 135 | { 136 | logger.LogInformation("{0} Service is stopping.", nameof(FasterHostedServiceMoveData)); 137 | return Task.CompletedTask; 138 | } 139 | 140 | public void Dispose() 141 | { 142 | // Nothing to do 143 | } 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /service/MinMQ.Service/Faster/FasterOps.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Text; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | using FASTER.core; 8 | using MinMQ.Service.Configuration; 9 | using MinMq.Service.Entities; 10 | using Optional; 11 | 12 | namespace MinMQ.Service 13 | { 14 | public sealed class FasterOps : IFasterWriter 15 | { 16 | private readonly IDevice device; 17 | private readonly FasterLog logger; 18 | private long nextAddress = 0; 19 | 20 | private FasterOps() 21 | { 22 | string devicePath = Startup.Configuration[nameof(MinMQConfiguration.FasterDevice)]; 23 | device = Devices.CreateLogDevice(devicePath); 24 | logger = new FasterLog(new FasterLogSettings { LogDevice = device, }); 25 | } 26 | 27 | public static Lazy Instance { get; set; } = new Lazy(() => new FasterOps()); 28 | 29 | #region Write 30 | public async ValueTask CommitAsync(CancellationToken token = default) 31 | { 32 | await logger.CommitAsync(token); 33 | } 34 | 35 | public async ValueTask WaitForCommitAsync(long untilAddress = 0, CancellationToken token = default) 36 | { 37 | await logger.WaitForCommitAsync(untilAddress, token); 38 | } 39 | 40 | public async ValueTask EnqueueAsync(byte[] entry, CancellationToken token = default) 41 | { 42 | return await logger.EnqueueAsync(entry, token); 43 | } 44 | 45 | public void TruncateUntil(long address) 46 | { 47 | logger.TruncateUntil(address); 48 | } 49 | #endregion 50 | 51 | #region Read 52 | public async Task> GetNext() 53 | { 54 | using FasterLogScanIterator iter = logger.Scan(nextAddress, 100_000_000); 55 | while (true) 56 | { 57 | byte[] entry; 58 | int length; 59 | 60 | while (!iter.GetNext(out entry, out length)) 61 | { 62 | if (iter.CurrentAddress >= 100_000_000) return Option.None<(string, long, long)>(); 63 | } 64 | 65 | UTF8Encoding encoding = new UTF8Encoding(); 66 | await iter.WaitAsync(); 67 | nextAddress = iter.NextAddress; 68 | return Option.Some((encoding.GetString(entry), iter.CurrentAddress, iter.NextAddress)); // Possible to pipe 69 | } 70 | } 71 | 72 | public async Task> GetList() 73 | { 74 | var result = new List<(string, long, long)>(); 75 | using (FasterLogScanIterator iter = logger.Scan(nextAddress, 100_000_000)) 76 | { 77 | int i = 0; 78 | byte[] entry; 79 | int length; 80 | while (iter.GetNext(out entry, out length)) 81 | { 82 | UTF8Encoding encoding = new UTF8Encoding(); 83 | if (iter.CurrentAddress >= 1568) Debugger.Break(); 84 | await iter.WaitAsync(); 85 | result.Add((encoding.GetString(entry), iter.CurrentAddress, iter.NextAddress)); 86 | i++; 87 | if (i > 50) 88 | { 89 | nextAddress = iter.NextAddress; 90 | break; 91 | } 92 | } 93 | } 94 | return result; 95 | } 96 | 97 | public async IAsyncEnumerable<(string, long, long)> GetListAsync() 98 | { 99 | using (FasterLogScanIterator iter = logger.Scan(nextAddress, 100_000_000)) 100 | { 101 | int i = 0; 102 | await foreach ((byte[] bytes, int length) in iter.GetAsyncEnumerable()) 103 | { 104 | if (i > 50) 105 | { 106 | nextAddress = iter.NextAddress; 107 | break; 108 | } 109 | 110 | CancellationTokenSource cts = new CancellationTokenSource(); 111 | UTF8Encoding encoding = new UTF8Encoding(); 112 | 113 | try 114 | { 115 | await Task.WhenAny(WaitAsync(iter, cts.Token), SetTimeout(cts)); 116 | i++; 117 | } 118 | catch (Exception) 119 | { 120 | break; 121 | } 122 | 123 | yield return (encoding.GetString(bytes), iter.CurrentAddress, iter.NextAddress); 124 | } 125 | } 126 | } 127 | 128 | // TODO: Currenly the last items are omitted with this variant. Use Listen() instead. 129 | public async IAsyncEnumerable<(string, long, long)> ListenAsync_(int flushSize) 130 | { 131 | // Always start from beginning. Assume it is refilled or truncate works. 132 | using (FasterLogScanIterator iter = logger.Scan(0, 1_000_000_000, name: "listen")) 133 | { 134 | int i = 0; 135 | await foreach ((byte[] bytes, int length) in iter.GetAsyncEnumerable()) 136 | { 137 | if (i >= flushSize) 138 | { 139 | nextAddress = iter.NextAddress; 140 | break; 141 | } 142 | 143 | CancellationTokenSource cts = new CancellationTokenSource(); 144 | UTF8Encoding encoding = new UTF8Encoding(); 145 | 146 | i++; 147 | 148 | // Probably shouldn't wait 149 | // - https://microsoft.github.io/FASTER/docs/fasterlog#iteration 150 | 151 | yield return (encoding.GetString(bytes), iter.CurrentAddress, iter.NextAddress); 152 | } 153 | } 154 | } 155 | 156 | public List<(string, long, long)> Listen(int flushSize, Cursor cursor) 157 | { 158 | var nextAddress_ = cursor.NextAddress; 159 | 160 | List<(string, long, long)> entries = new List<(string, long, long)>(); 161 | // CancellationTokenSource cts = new CancellationTokenSource(); 162 | UTF8Encoding encoding = new UTF8Encoding(); 163 | 164 | int i = 0; 165 | using (FasterLogScanIterator iter = logger.Scan(0, 1_000_000_000)) 166 | { 167 | while (true) 168 | { 169 | byte[] bytes; 170 | while (!iter.GetNext(out bytes, out int entryLength)) 171 | { 172 | // TODO: Cursor keeps an item at the end for some reason. Fix this! 173 | if (entries.Count > 1) 174 | { 175 | return entries; 176 | } 177 | return new List<(string, long, long)>(); 178 | // TODO: for the moment make sure to return the final items instead of awaiting more results. 179 | // if (iter.CurrentAddress >= 1_000_000_000) break; 180 | // await iter.WaitAsync(cts.Token); 181 | } 182 | 183 | entries.Add((encoding.GetString(bytes), iter.CurrentAddress, iter.NextAddress)); 184 | 185 | if (cursor is DebugCursor debugCursor) 186 | { 187 | debugCursor.Increment(); 188 | 189 | if (debugCursor.Iteration > 1841) 190 | { 191 | int j = 0; 192 | } 193 | } 194 | 195 | i++; 196 | 197 | if (i >= flushSize) 198 | { 199 | return entries; 200 | } 201 | } 202 | } 203 | } 204 | 205 | private async Task SetTimeout(CancellationTokenSource cancellationTokenSource) 206 | { 207 | await Task.Delay(300); 208 | cancellationTokenSource.Cancel(); 209 | } 210 | 211 | private async Task WaitAsync(FasterLogScanIterator iter, CancellationToken cancellationToken) 212 | { 213 | return await iter.WaitAsync(cancellationToken); 214 | } 215 | #endregion 216 | } 217 | } 218 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # MinMQ 4 | 5 | **Note:** _This is work-in-progress. A prototype API is in place to write queries. Nothing exists 6 | in terms of formal API for sending or retrieving messages. At the moment messages are being flushed 7 | to Postgres for review and follow-up purposes; Making sure sent messages eventually show up._ 8 | 9 | MinMQ is a minimal message queue for private networks, on-premise, or non-public networks for the 10 | time being. It targets virtual machines, Docker and physical hosts. It's designed for low ceremony, 11 | high throughput, medium-to-low latency, and has a HTTP-transport for comfortable transmission of 12 | messages. 13 | 14 | ## This effort focuses on: 15 | - Low latency 16 | - High throughput (storage dependant) 17 | - Durable, transactional commits 18 | - In-order processing 19 | - Continuous benchmarking 20 | 21 | This implementation merely combines the efforts of [microsoft.FASTER](https://github.com/microsoft/FASTER) and 22 | [AspNet Core 3.0](https://docs.microsoft.com/en-us/aspnet/core/?view=aspnetcore-3.0). A HTTP-transports on top of a *very* 23 | fast log. FASTER provides "group commits" with [Concurrent Prefix Recovery](https://www.microsoft.com/en-us/research/uploads/prod/2019/01/cpr-sigmod19.pdf) rather than a [Work Ahead Log](https://wiki.postgresql.org/wiki/Improve_the_performance_of_ALTER_TABLE_SET_LOGGED_UNLOGGED_statement). This approach is reminiscent to that of [Microsoft Message Queue](https://support.microsoft.com/ms-my/help/256096/how-to-install-msmq-2-0-to-enable-queued-components) for 24 | messages transacted in bulk using Microsoft Distributed Transaction Coordinator. Albeit, this is quite diffent from "unlogged" tables or in-memory database 25 | flushing to a durable disk later on. 26 | 27 | ## Eventually the following things will be explored 28 | - A formal API (perhaps something reminiscent to MSMQ or IronMQ) 29 | - `Send` (or Post) 30 | - `Peek` (or Reserve) 31 | - `Delete` (or Recieve or Get) 32 | - Implementation per specification pattern for bootstrapping onto Kestrel. 33 | - Named queues. 34 | - Mime-Types (probably pre-defined). 35 | - Deleting queues. 36 | - Message content limit (<1 MB). 37 | - Should there be Error-queues? 38 | - Create a C# client: 39 | - Should be a separate project and possibly another solution and/or repository. 40 | - Wraps https requests effectively. 41 | - Add Reactive Extensions. 42 | - What to do about bulk operations e.g. on-the-fly log compaction via delegate or interface and the specification pattern? 43 | 44 | ## In a more distant future the following things may also be explored: 45 | - A queue management tools, that supports mime types and named queues and a formal token based security API for admin provisioning. 46 | _Only allow for empty queues to be deleted?_ 47 | - Read models: Faster KV, SQL Materalized views or cached responses. 48 | - Some kind of tiered solution: 49 | - Log splicing (it's likely that dealing with errors or unread message will require logs to be entirely rewritten 50 | possibly even [compacted](http://cloudurable.com/blog/kafka-architecture-log-compaction/index.html). 51 | - Internal relaying [N-tiered service provisioning](docs/ntiered.md). 52 | - Multiple IDevices and a commit-schedular settings. 53 | - Admin tool authentication and queue privilages (w/r/d) 54 | 55 | ## Unresolved talking points 56 | - Should the constant commit interval cater for high contension scenario only or dynamically change depending on the load. 57 | - How to manage storage or IDevice overflow? 58 | - Is there a point to adding an API for inbound FlatBuffers or Protobuf since this project is closely related to Kestrel. 59 | - Is there point to creating a Docker image and/or [Helm charts](https://helm.sh/) as a stand-alone Kestrel+MinMQ-host? 60 | - Should [advanced benchmarks suites](https://github.com/aspnet/Benchmarks) per AspNetCore's recommendation be used? 61 | - Should non-empty queues allow for deletion? 62 | 63 | ## Setup 64 | ### Setup a volume or disk area for FASTER IDevice 65 | FASTER allocates disk preemptively. Around 1.1 GB is used per default. Consequently a large docker volume, or path on 66 | disk that comfortably can allocate more than 1.1GB have to be assigned, preferably an SSD. For the time beeing only 67 | local IDevices can be configured. 68 | 69 | *Docker users* 70 | > Inspect the `setup.ps1` and change path so corresponds some disk space. For comfort i may be simpler change it to 71 | > shell-script instead. Docker-compose volume is defined with `external: true` so a disk _won't be_ created automatically. 72 | 73 | *Docker on macOS* 74 | Possibly? 75 | 76 | docker volume create --name=fasterdbo --driver local --opt o=size=1200m --opt device=tmpfs --opt type=tmpfs 77 | 78 | *Everyone else* 79 | > If you plan to run the service without a container service a FasterDevice-path must be set as in 80 | > [appsettings.Development.json](./service/MinMQ.Service/appsettings.Development.json). It 81 | > must be assigned before starting. 82 | 83 | 84 | ## Running on macOS 85 | This is on Docker. 86 | 87 | docker-compose up 88 | 89 | The benchmarker will fail as it requires a specific script to be selected. There is no default benchmark script that will be used at this point. More information on 90 | running and benchmarking are pending a branch merger. 91 | 92 | Benchmarking can be done with the following (verified on macOS). 93 | 94 | docker-compose run mmq-benchmarks -- post_message.sh 95 | 96 | The benchmark scripts to choose from are available by typing (from the repository root). 97 | 98 | cat benchmark/install.sh | grep cat | cut -d" " -f3 99 | 100 | > This will output something like this: 101 | > 102 | > ```/app/wrk/http-ready.sh 103 | > /app/wrk/status.sh 104 | > /app/wrk/misc.sh 105 | > /app/wrk/post_message.sh 106 | > /app/wrk/arm.sh 107 | > /app/wrk/arm1.sh``` 108 | 109 | The performance of the web stack can be compared to a few javascript variants. 110 | 111 | docker-compose run mmq-benchmarks -- status.sh 112 | 113 | On a recent (202103) run the figures were the following. 114 | 115 | | Web stack | Requests/sec | Transfer/sec | 116 | |-------|--------------|--------| 117 | |NodeJS (httpd)|20439.95|2.53MB| 118 | |Express|10180.03|2.17MB| 119 | |Hapi|7815.26|1.40MB| 120 | |Kestrel|36354.56|5.93MB| 121 | |Kestrel (healthcheck) |80230.80|18.13MB| 122 | 123 | ## Performance 124 | This is continiously measured and some sparse unstructed working documenets are available in [docs/perf.md](docs/perf.md). 125 | 126 | More information on how to continue the development work can be found [here](docs/development_work.md). 127 | 128 | But overall the with the custom made benchmarker about 30-50% saturation of a SATA SSD seems to be plausible. 129 | 130 | 131 | 132 | 133 | ## TL;DR 134 | Here are some useful commands. _Create a new queue._ 135 | 136 | curl -X PUT -d "" http://localhost:9000/queue/merde --trace-asci /dev/stdout 137 | 138 | Clear out the database and starting it again: 139 | 140 | docker-compose.exe down 141 | docker volume rm minmq_postgresdata 142 | docker-compose.exe up mmq-db 143 | 144 | Then open Visual Studio Pro/Community and PMC: 145 | 146 | update-database -context messagequeuecontext 147 | 148 | -------------------------------------------------------------------------------- /benchmark/mmq-post-xml.lua: -------------------------------------------------------------------------------- 1 | wrk.method = "POST" 2 | wrk.body = " cofaxCDS org.cofax.cds.CDSServlet configGlossary:installationAt Philadelphia, PA configGlossary:adminEmail ksm@pobox.com configGlossary:poweredBy Cofax configGlossary:poweredByIcon /images/cofax.gif configGlossary:staticPath /content/static templateProcessorClass org.cofax.WysiwygTemplate templateLoaderClass org.cofax.FilesTemplateLoader templatePath templates templateOverridePath defaultListTemplate listTemplate.htm defaultFileTemplate articleTemplate.htm useJSP false jspListTemplate listTemplate.jsp jspFileTemplate articleTemplate.jsp cachePackageTagsTrack 200 cachePackageTagsStore 200 cachePackageTagsRefresh 60 cacheTemplatesTrack 100 cacheTemplatesStore 50 cacheTemplatesRefresh 15 cachePagesTrack 200 cachePagesStore 100 cachePagesRefresh 10 cachePagesDirtyRead 10 searchEngineListTemplate forSearchEnginesList.htm searchEngineFileTemplate forSearchEngines.htm searchEngineRobotsDb WEB-INF/robots.db useDataStore true dataStoreClass org.cofax.SqlDataStore redirectionClass org.cofax.SqlRedirection dataStoreName cofax dataStoreDriver com.microsoft.jdbc.sqlserver.SQLServerDriver dataStoreUrl jdbc:microsoft:sqlserver://LOCALHOST:1433;DatabaseName=goon dataStoreUser sa dataStorePassword dataStoreTestQuery SET NOCOUNT ON;select test='test'; dataStoreLogFile /usr/local/tomcat/logs/datastore.log dataStoreInitConns 10 dataStoreMaxConns 100 dataStoreConnUsageLimit 100 dataStoreLogLevel debug maxUrlLength 500 cofaxEmail org.cofax.cds.EmailServlet mailHost mail1 mailHostOverride mail2 cofaxAdmin org.cofax.cds.AdminServlet fileServlet org.cofax.cds.FileServlet cofaxTools org.cofax.cms.CofaxToolsServlet templatePath toolstemplates/ log 1 logLocation /usr/local/tomcat/logs/CofaxTools.log logMaxSize dataLog 1 dataLogLocation /usr/local/tomcat/logs/dataLog.log dataLogMaxSize removePageCache /content/admin/remove?cache=pages&id= removeTemplateCache /content/admin/remove?cache=templates&id= fileTransferFolder /usr/local/tomcat/webapps/content/fileTransferFolder lookInContext 1 adminGroupID 4 betaServer true cofaxCDS / cofaxEmail /cofaxutil/aemail/* cofaxAdmin /admin/* fileServlet /static/* cofaxTools /tools/* cofax.tld /WEB-INF/tlds/cofax.tld " 3 | wrk.headers["Content-Type"] = "text/xml" -------------------------------------------------------------------------------- /docs/ntiered-diagram.svg: -------------------------------------------------------------------------------- 1 | 2 | nomnoml 3 | [<start>message_recieved]->[instance 1] 4 | [instance 1]->[<choice>queue] 5 | [<choice>queue]->[instance 2] 6 | [<choice>queue]->[instance 4] 7 | [<choice>queue]->[instance 3] 8 | [instance 2]->[<choice>mime-type] 9 | [<choice>mime-type]->[instance 5] 10 | [<choice>mime-type]->[instance 6] 11 | [instance 2]->[<choice>validate] 12 | [<choice>validate]->[instance 7] 13 | [<choice>validate]->[instance 8] 14 | [instance 6]->[<end>e6] 15 | [instance 3]->[<end>e3] 16 | [instance 4]->[<end>e4] 17 | [instance 5]->[<end>e5] 18 | [instance 7]->[<end>e7] 19 | [instance 8]->[<end>e8] 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | instance 1 58 | 59 | queue 60 | 61 | instance 2 62 | 63 | instance 4 64 | 65 | instance 3 66 | 67 | mime-type 68 | 69 | instance 5 70 | 71 | instance 6 72 | 73 | validate 74 | 75 | instance 7 76 | 77 | instance 8 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | --------------------------------------------------------------------------------