├── .editorconfig ├── .gitattributes ├── .gitignore ├── Changelog.md ├── Jering.KeyValueStore.sln ├── License.md ├── NuGet.Config ├── ReadMe.md ├── ThirdPartyLicenses.md ├── azure-pipelines.yml ├── codecov.yml ├── generators └── Jering.KeyValueStore.Generators │ ├── ApiDocumentationGenerator.cs │ ├── Jering.KeyValueStore.Generators.csproj │ └── SourceGenerator.cs ├── keypair.snk ├── nuget_icon.png ├── perf └── KeyValueStore │ ├── Jering.KeyValueStore.Performance.csproj │ ├── LowMemoryUsageBenchmarks.cs │ └── Program.cs ├── src └── KeyValueStore │ ├── AssemblyInfo.cs │ ├── IMixedStorageKVStore.cs │ ├── Jering.KeyValueStore.csproj │ ├── MixedStorageKVStore.cs │ ├── MixedStorageKVStoreOptions.cs │ ├── Strings.Designer.cs │ ├── Strings.resx │ └── packages.lock.json └── test └── KeyValueStore ├── Helpers ├── StringBuilderLogger.cs └── StringBuilderProvider.cs ├── Jering.KeyValueStore.Tests.csproj └── MixedStorageKVStoreIntegrationTests.cs /.editorconfig: -------------------------------------------------------------------------------- 1 | # Remove the line below if you want to inherit .editorconfig settings from higher directories 2 | root = true 3 | 4 | # Files 5 | [*.{cs,md,yml,txt,js,json}] 6 | 7 | #### Core EditorConfig Options #### 8 | 9 | # Indentation and spacing 10 | indent_size = 4 11 | indent_style = space 12 | tab_width = 4 13 | 14 | # New line preferences 15 | end_of_line = lf 16 | insert_final_newline = true 17 | 18 | [*.cs] 19 | # Analyzers 20 | dotnet_diagnostic.RS2000.severity = none 21 | dotnet_diagnostic.RS2008.severity = none 22 | 23 | # Substring 24 | dotnet_diagnostic.IDE0057.severity = none 25 | 26 | # var 27 | dotnet_diagnostic.IDE0007.severity = error 28 | dotnet_diagnostic.IDE0008.severity = error 29 | csharp_style_var_when_type_is_apparent = true 30 | csharp_style_var_elsewhere = false 31 | 32 | # async/await 33 | dotnet_diagnostic.CA2007.severity = error 34 | dotnet_diagnostic.CS4014.severity = none 35 | 36 | # Naming 37 | dotnet_naming_symbols.const_fields.applicable_kinds = field 38 | dotnet_naming_symbols.const_fields.applicable_accessibilities = * 39 | dotnet_naming_symbols.const_fields.required_modifiers = const 40 | 41 | dotnet_naming_style.uppercase_snake.required_prefix = 42 | dotnet_naming_style.uppercase_snake.required_suffix = 43 | dotnet_naming_style.uppercase_snake.word_separator = _ 44 | dotnet_naming_style.uppercase_snake.capitalization = all_upper 45 | 46 | dotnet_naming_rule.const_fields_should_be_uppercase_snake.severity = error 47 | dotnet_naming_rule.const_fields_should_be_uppercase_snake.symbols = const_fields 48 | dotnet_naming_rule.const_fields_should_be_uppercase_snake.style = uppercase_snake 49 | 50 | dotnet_naming_symbols.private_or_internal_field.applicable_kinds = field 51 | dotnet_naming_symbols.private_or_internal_field.applicable_accessibilities = internal, private, private_protected 52 | dotnet_naming_symbols.private_or_internal_field.required_modifiers = 53 | 54 | dotnet_naming_style._fieldname.required_prefix = _ 55 | dotnet_naming_style._fieldname.required_suffix = 56 | dotnet_naming_style._fieldname.word_separator = 57 | dotnet_naming_style._fieldname.capitalization = camel_case 58 | 59 | dotnet_naming_rule.private_or_internal_field_should_be__fieldname.severity = error 60 | dotnet_naming_rule.private_or_internal_field_should_be__fieldname.symbols = private_or_internal_field 61 | dotnet_naming_rule.private_or_internal_field_should_be__fieldname.style = _fieldname -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # https://github.com/alexkaratarakis/gitattributes/blob/master/CSharp.gitattributes 3 | ############################################################################### 4 | *.cs diff=csharp 5 | 6 | ############################################################################### 7 | # https://github.com/alexkaratarakis/gitattributes/blob/master/VisualStudio.gitattributes 8 | ############################################################################### 9 | # Set default behavior to automatically normalize line endings. 10 | ############################################################################### 11 | * text=auto 12 | ############################################################################### 13 | # Set the merge driver for project and solution files 14 | # 15 | # Merging from the command prompt will add diff markers to the files if there 16 | # are conflicts (Merging from VS is not affected by the settings below, in VS 17 | # the diff markers are never inserted). Diff markers may cause the following 18 | # file extensions to fail to load in VS. An alternative would be to treat 19 | # these files as binary and thus will always conflict and require user 20 | # intervention with every merge. To do so, just comment the entries below and 21 | # uncomment the group further below 22 | ############################################################################### 23 | *.sln text eol=crlf 24 | *.csproj text eol=crlf 25 | *.vbproj text eol=crlf 26 | *.vcxproj text eol=crlf 27 | *.vcproj text eol=crlf 28 | *.dbproj text eol=crlf 29 | *.fsproj text eol=crlf 30 | *.lsproj text eol=crlf 31 | *.wixproj text eol=crlf 32 | *.modelproj text eol=crlf 33 | *.sqlproj text eol=crlf 34 | *.wmaproj text eol=crlf 35 | 36 | *.xproj text eol=crlf 37 | *.props text eol=crlf 38 | *.filters text eol=crlf 39 | *.vcxitems text eol=crlf 40 | 41 | 42 | #*.sln merge=binary 43 | #*.csproj merge=binary 44 | #*.vbproj merge=binary 45 | #*.vcxproj merge=binary 46 | #*.vcproj merge=binary 47 | #*.dbproj merge=binary 48 | #*.fsproj merge=binary 49 | #*.lsproj merge=binary 50 | #*.wixproj merge=binary 51 | #*.modelproj merge=binary 52 | #*.sqlproj merge=binary 53 | #*.wwaproj merge=binary 54 | 55 | #*.xproj merge=binary 56 | #*.props merge=binary 57 | #*.filters merge=binary 58 | #*.vcxitems merge=binary -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.suo 8 | *.user 9 | *.userosscache 10 | *.sln.docstates 11 | 12 | # User-specific files (MonoDevelop/Xamarin Studio) 13 | *.userprefs 14 | 15 | # Build results 16 | [Dd]ebug/ 17 | [Dd]ebugPublic/ 18 | [Rr]elease/ 19 | [Rr]eleases/ 20 | x64/ 21 | x86/ 22 | bld/ 23 | [Bb]in/ 24 | [Oo]bj/ 25 | [Ll]og/ 26 | 27 | # Visual Studio 2015 cache/options directory 28 | .vs/ 29 | # Uncomment if you have tasks that create the project's static files in wwwroot 30 | #wwwroot/ 31 | 32 | # MSTest test Results 33 | [Tt]est[Rr]esult*/ 34 | [Bb]uild[Ll]og.* 35 | 36 | # NUNIT 37 | *.VisualState.xml 38 | TestResult.xml 39 | 40 | # Build Results of an ATL Project 41 | [Dd]ebugPS/ 42 | [Rr]eleasePS/ 43 | dlldata.c 44 | 45 | # .NET Core 46 | project.lock.json 47 | project.fragment.lock.json 48 | artifacts/ 49 | **/Properties/launchSettings.json 50 | 51 | *_i.c 52 | *_p.c 53 | *_i.h 54 | *.ilk 55 | *.meta 56 | *.obj 57 | *.pch 58 | *.pdb 59 | *.pgc 60 | *.pgd 61 | *.rsp 62 | *.sbr 63 | *.tlb 64 | *.tli 65 | *.tlh 66 | *.tmp 67 | *.tmp_proj 68 | *.log 69 | *.vspscc 70 | *.vssscc 71 | .builds 72 | *.pidb 73 | *.svclog 74 | *.scc 75 | 76 | # Chutzpah Test files 77 | _Chutzpah* 78 | 79 | # Visual C++ cache files 80 | ipch/ 81 | *.aps 82 | *.ncb 83 | *.opendb 84 | *.opensdf 85 | *.sdf 86 | *.cachefile 87 | *.VC.db 88 | *.VC.VC.opendb 89 | 90 | # Visual Studio profiler 91 | *.psess 92 | *.vsp 93 | *.vspx 94 | *.sap 95 | 96 | # TFS 2012 Local Workspace 97 | $tf/ 98 | 99 | # Guidance Automation Toolkit 100 | *.gpState 101 | 102 | # ReSharper is a .NET coding add-in 103 | _ReSharper*/ 104 | *.[Rr]e[Ss]harper 105 | *.DotSettings.user 106 | 107 | # JustCode is a .NET coding add-in 108 | .JustCode 109 | 110 | # TeamCity is a build add-in 111 | _TeamCity* 112 | 113 | # DotCover is a Code Coverage Tool 114 | *.dotCover 115 | 116 | # Visual Studio code coverage results 117 | *.coverage 118 | *.coveragexml 119 | 120 | # NCrunch 121 | _NCrunch_* 122 | .*crunch*.local.xml 123 | nCrunchTemp_* 124 | 125 | # MightyMoose 126 | *.mm.* 127 | AutoTest.Net/ 128 | 129 | # Web workbench (sass) 130 | .sass-cache/ 131 | 132 | # Installshield output folder 133 | [Ee]xpress/ 134 | 135 | # DocProject is a documentation generator add-in 136 | DocProject/buildhelp/ 137 | DocProject/Help/*.HxT 138 | DocProject/Help/*.HxC 139 | DocProject/Help/*.hhc 140 | DocProject/Help/*.hhk 141 | DocProject/Help/*.hhp 142 | DocProject/Help/Html2 143 | DocProject/Help/html 144 | 145 | # Click-Once directory 146 | publish/ 147 | 148 | # Publish Web Output 149 | *.[Pp]ublish.xml 150 | *.azurePubxml 151 | # TODO: Comment the next line if you want to checkin your web deploy settings 152 | # but database connection strings (with potential passwords) will be unencrypted 153 | *.pubxml 154 | *.publishproj 155 | 156 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 157 | # checkin your Azure Web App publish settings, but sensitive information contained 158 | # in these scripts will be unencrypted 159 | PublishScripts/ 160 | 161 | # NuGet Packages 162 | *.nupkg 163 | # The packages folder can be ignored because of Package Restore 164 | **/packages/* 165 | # except build/, which is used as an MSBuild target. 166 | !**/packages/build/ 167 | # Uncomment if necessary however generally it will be regenerated when needed 168 | #!**/packages/repositories.config 169 | # NuGet v3's project.json files produces more ignoreable files 170 | *.nuget.props 171 | *.nuget.targets 172 | 173 | # Microsoft Azure Build Output 174 | csx/ 175 | *.build.csdef 176 | 177 | # Microsoft Azure Emulator 178 | ecf/ 179 | rcf/ 180 | 181 | # Windows Store app package directories and files 182 | AppPackages/ 183 | BundleArtifacts/ 184 | Package.StoreAssociation.xml 185 | _pkginfo.txt 186 | 187 | # Visual Studio cache files 188 | # files ending in .cache can be ignored 189 | *.[Cc]ache 190 | # but keep track of directories ending in .cache 191 | !*.[Cc]ache/ 192 | 193 | # Others 194 | ClientBin/ 195 | ~$* 196 | *~ 197 | *.dbmdl 198 | *.dbproj.schemaview 199 | *.jfm 200 | *.pfx 201 | *.publishsettings 202 | orleans.codegen.cs 203 | 204 | # Since there are multiple workflows, uncomment next line to ignore bower_components 205 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 206 | #bower_components/ 207 | 208 | # RIA/Silverlight projects 209 | Generated_Code/ 210 | 211 | # Backup & report files from converting an old project file 212 | # to a newer Visual Studio version. Backup files are not needed, 213 | # because we have git ;-) 214 | _UpgradeReport_Files/ 215 | Backup*/ 216 | UpgradeLog*.XML 217 | UpgradeLog*.htm 218 | 219 | # SQL Server files 220 | *.mdf 221 | *.ldf 222 | 223 | # Business Intelligence projects 224 | *.rdl.data 225 | *.bim.layout 226 | *.bim_*.settings 227 | 228 | # Microsoft Fakes 229 | FakesAssemblies/ 230 | 231 | # GhostDoc plugin setting file 232 | *.GhostDoc.xml 233 | 234 | # Node.js Tools for Visual Studio 235 | .ntvs_analysis.dat 236 | node_modules/ 237 | 238 | # Visual Studio 6 build log 239 | *.plg 240 | 241 | # Visual Studio 6 workspace options file 242 | *.opt 243 | 244 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 245 | *.vbw 246 | 247 | # Visual Studio LightSwitch build output 248 | **/*.HTMLClient/GeneratedArtifacts 249 | **/*.DesktopClient/GeneratedArtifacts 250 | **/*.DesktopClient/ModelManifest.xml 251 | **/*.Server/GeneratedArtifacts 252 | **/*.Server/ModelManifest.xml 253 | _Pvt_Extensions 254 | 255 | # Paket dependency manager 256 | .paket/paket.exe 257 | paket-files/ 258 | 259 | # FAKE - F# Make 260 | .fake/ 261 | 262 | # JetBrains Rider 263 | .idea/ 264 | *.sln.iml 265 | 266 | # CodeRush 267 | .cr/ 268 | 269 | # Python Tools for Visual Studio (PTVS) 270 | __pycache__/ 271 | *.pyc 272 | 273 | # Cake - Uncomment if you are using it 274 | # tools/** 275 | # !tools/packages.config 276 | /perf/KeyValueStore/BenchmarkDotNet.Artifacts/results 277 | -------------------------------------------------------------------------------- /Changelog.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | This project uses [semantic versioning](http://semver.org/spec/v2.0.0.html). Refer to 3 | *[Semantic Versioning in Practice](https://www.jering.tech/articles/semantic-versioning-in-practice)* 4 | for an overview of semantic versioning. 5 | 6 | ## [Unreleased](https://github.com/JeringTech/KeyValueStore/compare/1.1.0...HEAD) 7 | 8 | ## [1.1.0](https://github.com/JeringTech/KeyValueStore/compare/1.0.0...1.1.0) - May 15, 2021 9 | ### Changes 10 | - Improved how `MixedStorageKVStore` handles memory. ([#3](https://github.com/JeringTech/KeyValueStore/pull/3)) 11 | 12 | ## [1.0.0](https://github.com/JeringTech/KeyValueStore/compare/1.0.0...1.0.0) - May 14, 2021 13 | Initial release. 14 | -------------------------------------------------------------------------------- /Jering.KeyValueStore.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.31004.235 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jering.KeyValueStore", "src\KeyValueStore\Jering.KeyValueStore.csproj", "{14EC15F7-A4A1-4EDA-BBF1-1552BFE5D3DE}" 7 | EndProject 8 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{ACBC1C23-65DD-43D8-A2CD-E1C31BBBA269}" 9 | ProjectSection(SolutionItems) = preProject 10 | .editorconfig = .editorconfig 11 | .gitattributes = .gitattributes 12 | .gitignore = .gitignore 13 | azure-pipelines.yml = azure-pipelines.yml 14 | Changelog.md = Changelog.md 15 | codecov.yml = codecov.yml 16 | keypair.snk = keypair.snk 17 | License.md = License.md 18 | NuGet.Config = NuGet.Config 19 | ReadMe.md = ReadMe.md 20 | ThirdPartyLicenses.md = ThirdPartyLicenses.md 21 | EndProjectSection 22 | EndProject 23 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jering.KeyValueStore.Tests", "test\KeyValueStore\Jering.KeyValueStore.Tests.csproj", "{E9076F85-1A7C-4F15-84EF-BE8A0F1C39C2}" 24 | EndProject 25 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jering.KeyValueStore.Performance", "perf\KeyValueStore\Jering.KeyValueStore.Performance.csproj", "{598002AB-E266-407F-9587-D14CE341C04B}" 26 | EndProject 27 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Jering.KeyValueStore.Generators", "generators\Jering.KeyValueStore.Generators\Jering.KeyValueStore.Generators.csproj", "{78818480-0469-44C7-97A7-998710651DFD}" 28 | EndProject 29 | Global 30 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 31 | Debug|Any CPU = Debug|Any CPU 32 | Debug|x64 = Debug|x64 33 | Release|Any CPU = Release|Any CPU 34 | Release|x64 = Release|x64 35 | EndGlobalSection 36 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 37 | {14EC15F7-A4A1-4EDA-BBF1-1552BFE5D3DE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 38 | {14EC15F7-A4A1-4EDA-BBF1-1552BFE5D3DE}.Debug|Any CPU.Build.0 = Debug|Any CPU 39 | {14EC15F7-A4A1-4EDA-BBF1-1552BFE5D3DE}.Debug|x64.ActiveCfg = Debug|Any CPU 40 | {14EC15F7-A4A1-4EDA-BBF1-1552BFE5D3DE}.Debug|x64.Build.0 = Debug|Any CPU 41 | {14EC15F7-A4A1-4EDA-BBF1-1552BFE5D3DE}.Release|Any CPU.ActiveCfg = Release|Any CPU 42 | {14EC15F7-A4A1-4EDA-BBF1-1552BFE5D3DE}.Release|Any CPU.Build.0 = Release|Any CPU 43 | {14EC15F7-A4A1-4EDA-BBF1-1552BFE5D3DE}.Release|x64.ActiveCfg = Release|Any CPU 44 | {14EC15F7-A4A1-4EDA-BBF1-1552BFE5D3DE}.Release|x64.Build.0 = Release|Any CPU 45 | {E9076F85-1A7C-4F15-84EF-BE8A0F1C39C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 46 | {E9076F85-1A7C-4F15-84EF-BE8A0F1C39C2}.Debug|Any CPU.Build.0 = Debug|Any CPU 47 | {E9076F85-1A7C-4F15-84EF-BE8A0F1C39C2}.Debug|x64.ActiveCfg = Debug|Any CPU 48 | {E9076F85-1A7C-4F15-84EF-BE8A0F1C39C2}.Debug|x64.Build.0 = Debug|Any CPU 49 | {E9076F85-1A7C-4F15-84EF-BE8A0F1C39C2}.Release|Any CPU.ActiveCfg = Release|Any CPU 50 | {E9076F85-1A7C-4F15-84EF-BE8A0F1C39C2}.Release|Any CPU.Build.0 = Release|Any CPU 51 | {E9076F85-1A7C-4F15-84EF-BE8A0F1C39C2}.Release|x64.ActiveCfg = Release|Any CPU 52 | {E9076F85-1A7C-4F15-84EF-BE8A0F1C39C2}.Release|x64.Build.0 = Release|Any CPU 53 | {598002AB-E266-407F-9587-D14CE341C04B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 54 | {598002AB-E266-407F-9587-D14CE341C04B}.Debug|Any CPU.Build.0 = Debug|Any CPU 55 | {598002AB-E266-407F-9587-D14CE341C04B}.Debug|x64.ActiveCfg = Debug|Any CPU 56 | {598002AB-E266-407F-9587-D14CE341C04B}.Debug|x64.Build.0 = Debug|Any CPU 57 | {598002AB-E266-407F-9587-D14CE341C04B}.Release|Any CPU.ActiveCfg = Release|Any CPU 58 | {598002AB-E266-407F-9587-D14CE341C04B}.Release|Any CPU.Build.0 = Release|Any CPU 59 | {598002AB-E266-407F-9587-D14CE341C04B}.Release|x64.ActiveCfg = Release|Any CPU 60 | {598002AB-E266-407F-9587-D14CE341C04B}.Release|x64.Build.0 = Release|Any CPU 61 | {78818480-0469-44C7-97A7-998710651DFD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 62 | {78818480-0469-44C7-97A7-998710651DFD}.Debug|Any CPU.Build.0 = Debug|Any CPU 63 | {78818480-0469-44C7-97A7-998710651DFD}.Debug|x64.ActiveCfg = Debug|Any CPU 64 | {78818480-0469-44C7-97A7-998710651DFD}.Debug|x64.Build.0 = Debug|Any CPU 65 | {78818480-0469-44C7-97A7-998710651DFD}.Release|Any CPU.ActiveCfg = Release|Any CPU 66 | {78818480-0469-44C7-97A7-998710651DFD}.Release|Any CPU.Build.0 = Release|Any CPU 67 | {78818480-0469-44C7-97A7-998710651DFD}.Release|x64.ActiveCfg = Release|Any CPU 68 | {78818480-0469-44C7-97A7-998710651DFD}.Release|x64.Build.0 = Release|Any CPU 69 | EndGlobalSection 70 | GlobalSection(SolutionProperties) = preSolution 71 | HideSolutionNode = FALSE 72 | EndGlobalSection 73 | GlobalSection(ExtensibilityGlobals) = postSolution 74 | SolutionGuid = {333D3ECF-5EEE-4057-A2C6-3D2CE7618E4B} 75 | EndGlobalSection 76 | EndGlobal 77 | -------------------------------------------------------------------------------- /License.md: -------------------------------------------------------------------------------- 1 | # Licenses 2 | Jering.KeyValueStore, copyright © 2021 Jering. All rights reserved. 3 | 4 | ## Source Code and Documentation Code-Examples 5 | Source code and documentation code-examples are licensed under the following license: 6 | 7 | Apache License 8 | Version 2.0, January 2004 9 | http://www.apache.org/licenses/ 10 | 11 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 12 | 13 | 1. Definitions. 14 | 15 | "License" shall mean the terms and conditions for use, reproduction, 16 | and distribution as defined by Sections 1 through 9 of this document. 17 | 18 | "Licensor" shall mean the copyright owner or entity authorized by 19 | the copyright owner that is granting the License. 20 | 21 | "Legal Entity" shall mean the union of the acting entity and all 22 | other entities that control, are controlled by, or are under common 23 | control with that entity. For the purposes of this definition, 24 | "control" means (i) the power, direct or indirect, to cause the 25 | direction or management of such entity, whether by contract or 26 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 27 | outstanding shares, or (iii) beneficial ownership of such entity. 28 | 29 | "You" (or "Your") shall mean an individual or Legal Entity 30 | exercising permissions granted by this License. 31 | 32 | "Source" form shall mean the preferred form for making modifications, 33 | including but not limited to software source code, documentation 34 | source, and configuration files. 35 | 36 | "Object" form shall mean any form resulting from mechanical 37 | transformation or translation of a Source form, including but 38 | not limited to compiled object code, generated documentation, 39 | and conversions to other media types. 40 | 41 | "Work" shall mean the work of authorship, whether in Source or 42 | Object form, made available under the License, as indicated by a 43 | copyright notice that is included in or attached to the work 44 | (an example is provided in the Appendix below). 45 | 46 | "Derivative Works" shall mean any work, whether in Source or Object 47 | form, that is based on (or derived from) the Work and for which the 48 | editorial revisions, annotations, elaborations, or other modifications 49 | represent, as a whole, an original work of authorship. For the purposes 50 | of this License, Derivative Works shall not include works that remain 51 | separable from, or merely link (or bind by name) to the interfaces of, 52 | the Work and Derivative Works thereof. 53 | 54 | "Contribution" shall mean any work of authorship, including 55 | the original version of the Work and any modifications or additions 56 | to that Work or Derivative Works thereof, that is intentionally 57 | submitted to Licensor for inclusion in the Work by the copyright owner 58 | or by an individual or Legal Entity authorized to submit on behalf of 59 | the copyright owner. For the purposes of this definition, "submitted" 60 | means any form of electronic, verbal, or written communication sent 61 | to the Licensor or its representatives, including but not limited to 62 | communication on electronic mailing lists, source code control systems, 63 | and issue tracking systems that are managed by, or on behalf of, the 64 | Licensor for the purpose of discussing and improving the Work, but 65 | excluding communication that is conspicuously marked or otherwise 66 | designated in writing by the copyright owner as "Not a Contribution." 67 | 68 | "Contributor" shall mean Licensor and any individual or Legal Entity 69 | on behalf of whom a Contribution has been received by Licensor and 70 | subsequently incorporated within the Work. 71 | 72 | 2. Grant of Copyright License. Subject to the terms and conditions of 73 | this License, each Contributor hereby grants to You a perpetual, 74 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 75 | copyright license to reproduce, prepare Derivative Works of, 76 | publicly display, publicly perform, sublicense, and distribute the 77 | Work and such Derivative Works in Source or Object form. 78 | 79 | 3. Grant of Patent License. Subject to the terms and conditions of 80 | this License, each Contributor hereby grants to You a perpetual, 81 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 82 | (except as stated in this section) patent license to make, have made, 83 | use, offer to sell, sell, import, and otherwise transfer the Work, 84 | where such license applies only to those patent claims licensable 85 | by such Contributor that are necessarily infringed by their 86 | Contribution(s) alone or by combination of their Contribution(s) 87 | with the Work to which such Contribution(s) was submitted. If You 88 | institute patent litigation against any entity (including a 89 | cross-claim or counterclaim in a lawsuit) alleging that the Work 90 | or a Contribution incorporated within the Work constitutes direct 91 | or contributory patent infringement, then any patent licenses 92 | granted to You under this License for that Work shall terminate 93 | as of the date such litigation is filed. 94 | 95 | 4. Redistribution. You may reproduce and distribute copies of the 96 | Work or Derivative Works thereof in any medium, with or without 97 | modifications, and in Source or Object form, provided that You 98 | meet the following conditions: 99 | 100 | (a) You must give any other recipients of the Work or 101 | Derivative Works a copy of this License; and 102 | 103 | (b) You must cause any modified files to carry prominent notices 104 | stating that You changed the files; and 105 | 106 | (c) You must retain, in the Source form of any Derivative Works 107 | that You distribute, all copyright, patent, trademark, and 108 | attribution notices from the Source form of the Work, 109 | excluding those notices that do not pertain to any part of 110 | the Derivative Works; and 111 | 112 | (d) If the Work includes a "NOTICE" text file as part of its 113 | distribution, then any Derivative Works that You distribute must 114 | include a readable copy of the attribution notices contained 115 | within such NOTICE file, excluding those notices that do not 116 | pertain to any part of the Derivative Works, in at least one 117 | of the following places: within a NOTICE text file distributed 118 | as part of the Derivative Works; within the Source form or 119 | documentation, if provided along with the Derivative Works; or, 120 | within a display generated by the Derivative Works, if and 121 | wherever such third-party notices normally appear. The contents 122 | of the NOTICE file are for informational purposes only and 123 | do not modify the License. You may add Your own attribution 124 | notices within Derivative Works that You distribute, alongside 125 | or as an addendum to the NOTICE text from the Work, provided 126 | that such additional attribution notices cannot be construed 127 | as modifying the License. 128 | 129 | You may add Your own copyright statement to Your modifications and 130 | may provide additional or different license terms and conditions 131 | for use, reproduction, or distribution of Your modifications, or 132 | for any such Derivative Works as a whole, provided Your use, 133 | reproduction, and distribution of the Work otherwise complies with 134 | the conditions stated in this License. 135 | 136 | 5. Submission of Contributions. Unless You explicitly state otherwise, 137 | any Contribution intentionally submitted for inclusion in the Work 138 | by You to the Licensor shall be under the terms and conditions of 139 | this License, without any additional terms or conditions. 140 | Notwithstanding the above, nothing herein shall supersede or modify 141 | the terms of any separate license agreement you may have executed 142 | with Licensor regarding such Contributions. 143 | 144 | 6. Trademarks. This License does not grant permission to use the trade 145 | names, trademarks, service marks, or product names of the Licensor, 146 | except as required for reasonable and customary use in describing the 147 | origin of the Work and reproducing the content of the NOTICE file. 148 | 149 | 7. Disclaimer of Warranty. Unless required by applicable law or 150 | agreed to in writing, Licensor provides the Work (and each 151 | Contributor provides its Contributions) on an "AS IS" BASIS, 152 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 153 | implied, including, without limitation, any warranties or conditions 154 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 155 | PARTICULAR PURPOSE. You are solely responsible for determining the 156 | appropriateness of using or redistributing the Work and assume any 157 | risks associated with Your exercise of permissions under this License. 158 | 159 | 8. Limitation of Liability. In no event and under no legal theory, 160 | whether in tort (including negligence), contract, or otherwise, 161 | unless required by applicable law (such as deliberate and grossly 162 | negligent acts) or agreed to in writing, shall any Contributor be 163 | liable to You for damages, including any direct, indirect, special, 164 | incidental, or consequential damages of any character arising as a 165 | result of this License or out of the use or inability to use the 166 | Work (including but not limited to damages for loss of goodwill, 167 | work stoppage, computer failure or malfunction, or any and all 168 | other commercial damages or losses), even if such Contributor 169 | has been advised of the possibility of such damages. 170 | 171 | 9. Accepting Warranty or Additional Liability. While redistributing 172 | the Work or Derivative Works thereof, You may choose to offer, 173 | and charge a fee for, acceptance of support, warranty, indemnity, 174 | or other liability obligations and/or rights consistent with this 175 | License. However, in accepting such obligations, You may act only 176 | on Your own behalf and on Your sole responsibility, not on behalf 177 | of any other Contributor, and only if You agree to indemnify, 178 | defend, and hold each Contributor harmless for any liability 179 | incurred by, or claims asserted against, such Contributor by reason 180 | of your accepting any such warranty or additional liability. 181 | 182 | END OF TERMS AND CONDITIONS 183 | 184 | ## Documentation (Excluding Code-Examples) 185 | Documentation (excluding code-examples), is licensed under a [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) license. 186 | 187 | ## Brand Assets 188 | Brand assets such as the Jering logo are not licensed under the above-mentioned licenses. You may not use these assets in 189 | a manner that might mislead users about the origin of your work. You may use these assets to refer to Jering. 190 | -------------------------------------------------------------------------------- /NuGet.Config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /ReadMe.md: -------------------------------------------------------------------------------- 1 | # Jering.KeyValueStore 2 | [![Build Status](https://dev.azure.com/JeringTech/KeyValueStore/_apis/build/status/JeringTech.KeyValueStore?repoName=JeringTech%2FKeyValueStore&branchName=refs%2Fpull%2F2%2Fmerge)](https://dev.azure.com/JeringTech/KeyValueStore/_build/latest?definitionId=11&repoName=JeringTech%2FKeyValueStore&branchName=refs%2Fpull%2F2%2Fmerge) 3 | [![codecov](https://codecov.io/gh/JeringTech/KeyValueStore/branch/main/graph/badge.svg?token=5STAAJJ4Q4)](https://codecov.io/gh/JeringTech/KeyValueStore) 4 | [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/JeringTech/KeyValueStore/blob/main/License.md) 5 | [![NuGet](https://img.shields.io/nuget/vpre/Jering.KeyValueStore.svg?label=nuget)](https://www.nuget.org/packages/Jering.KeyValueStore/) 6 | 7 | ## Table of Contents 8 | [Overview](#overview) 9 | [Target Frameworks](#target-frameworks) 10 | [Platforms](#platforms) 11 | [Installation](#installation) 12 | [Usage](#usage) 13 | [API](#api) 14 | [Performance](#performance) 15 | [Building and Testing](#building-and-testing) 16 | [Alternatives](#alternatives) 17 | [Related Concepts](#related-concepts) 18 | [Contributing](#contributing) 19 | [About](#about) 20 | 21 | ## Overview 22 | Jering.KeyValueStore enables you to store key-value data across memory and disk. 23 | 24 | Usage: 25 | 26 | ```csharp 27 | var mixedStorageKVStore = new MixedStorageKVStore(); // Stores data across memory (primary storage) and disk (secondary storage) 28 | 29 | // Insert 30 | await mixedStorageKVStore.UpsertAsync(0, "dummyString1").ConfigureAwait(false); // Insert a key-value pair (record) 31 | 32 | // Verify inserted 33 | (Status status, string? result) = await mixedStorageKVStore.ReadAsync(0).ConfigureAwait(false); 34 | Assert.Equal(Status.OK, status); // Status.NOTFOUND if no record with key 0 35 | Assert.Equal("dummyString1", result); 36 | 37 | // Update 38 | await mixedStorageKVStore.UpsertAsync(0, "dummyString2").ConfigureAwait(false); 39 | 40 | // Verify updated 41 | (status, result) = await mixedStorageKVStore.ReadAsync(0).ConfigureAwait(false); 42 | Assert.Equal(Status.OK, status); 43 | Assert.Equal("dummyString2", result); 44 | 45 | // Delete 46 | await mixedStorageKVStore.DeleteAsync(0).ConfigureAwait(false); 47 | 48 | // Verify deleted 49 | (status, result) = await mixedStorageKVStore.ReadAsync(0).ConfigureAwait(false); 50 | Assert.Equal(Status.NOTFOUND, status); 51 | Assert.Null(result); 52 | ``` 53 | 54 | This library is a wrapper of Microsoft's [Faster key-value store](https://github.com/microsoft/FASTER). Faster is a low-level key-value store that introduces a novel, lock-free concurrency system. 55 | You'll need a basic understanding of Faster to use this library. Refer to [Faster Basics](#faster-basics) for a quick primer and an overview of features this library provides on top of Faster. 56 | 57 | ## Target Frameworks 58 | - .NET Standard 2.1 59 | 60 | ## Platforms 61 | - Windows 62 | - macOS 63 | - Linux 64 | 65 | ## Installation 66 | Using Package Manager: 67 | ``` 68 | PM> Install-Package Jering.KeyValueStore 69 | ``` 70 | Using .Net CLI: 71 | ``` 72 | > dotnet add package Jering.KeyValueStore 73 | ``` 74 | 75 | ## Usage 76 | This section explains how to use this library. Topics: 77 | 78 | [Choosing Key and Value Types](#choosing-key-and-value-types) 79 | [Using This Library in Highly Concurrent Logic](#using-this-library-in-highly-concurrent-logic) 80 | [Configuring](#configuring) 81 | [Creating and Managing On-Disk Data](#creating-and-managing-on-disk-data) 82 | 83 | ### Choosing Key and Value Types 84 | [MessagePack C#](https://github.com/neuecc/MessagePack-CSharp) must be able to serialize your `MixedStorageKVStore` key and value types. 85 | 86 | The list of types MessagePack C# can serialize includes [built-in types](https://github.com/neuecc/MessagePack-CSharp#built-in-supported-types) and custom types annotated according to 87 | [MessagePack C# conventions](https://github.com/neuecc/MessagePack-CSharp#object-serialization). 88 | 89 | #### Common Key and Value Types 90 | The following are examples of common key and value types. 91 | 92 | ##### Reference Types 93 | The following custom reference type is annotated according to MessagePack C# conventions: 94 | 95 | ```csharp 96 | [MessagePackObject] // MessagePack C# attribute 97 | public class DummyClass 98 | { 99 | [Key(0)] // MessagePack C# attribute 100 | public string? DummyString { get; set; } 101 | 102 | [Key(1)] 103 | public string[]? DummyStringArray { get; set; } 104 | 105 | [Key(2)] 106 | public int DummyInt { get; set; } 107 | 108 | [Key(3)] 109 | public int[]? DummyIntArray { get; set; } 110 | } 111 | ``` 112 | We can use it, together with the built-in reference type `string` as key and value types: 113 | ```csharp 114 | var mixedStorageKVStore = new MixedStorageKVStore(); // string key, DummyClass value 115 | var dummyClassInstance = new DummyClass() 116 | { 117 | DummyString = "dummyString", 118 | DummyStringArray = new[] { "dummyString1", "dummyString2", "dummyString3", "dummyString4", "dummyString5" }, 119 | DummyInt = 10, 120 | DummyIntArray = new[] { 10, 100, 1000, 10000, 100000, 1000000, 10000000 } 121 | }; 122 | 123 | // Insert 124 | await mixedStorageKVStore.UpsertAsync("dummyKey", dummyClassInstance).ConfigureAwait(false); 125 | 126 | // Read 127 | (Status status, DummyClass? result) = await mixedStorageKVStore.ReadAsync("dummyKey").ConfigureAwait(false); 128 | 129 | // Verify 130 | Assert.Equal(Status.OK, status); 131 | Assert.Equal(dummyClassInstance.DummyString, result!.DummyString); // result is only null if status is Status.NOTFOUND 132 | Assert.Equal(dummyClassInstance.DummyStringArray, result!.DummyStringArray); 133 | Assert.Equal(dummyClassInstance.DummyInt, result!.DummyInt); 134 | Assert.Equal(dummyClassInstance.DummyIntArray, result!.DummyIntArray); 135 | ``` 136 | ##### Value Types 137 | The following custom value-type is annotated according to MessagePack C# conventions: 138 | ```csharp 139 | [MessagePackObject] 140 | public struct DummyStruct 141 | { 142 | [Key(0)] 143 | public byte DummyByte { get; set; } 144 | 145 | [Key(1)] 146 | public short DummyShort { get; set; } 147 | 148 | [Key(2)] 149 | public int DummyInt { get; set; } 150 | 151 | [Key(3)] 152 | public long DummyLong { get; set; } 153 | } 154 | ``` 155 | We can use it, together with the built-in value type `int` as key and value types: 156 | ```csharp 157 | var mixedStorageKVStore = new MixedStorageKVStore(); // int key, DummyStruct value 158 | var dummyStructInstance = new DummyStruct() 159 | { 160 | // Populate with dummy values 161 | DummyByte = byte.MaxValue, 162 | DummyShort = short.MaxValue, 163 | DummyInt = int.MaxValue, 164 | DummyLong = long.MaxValue 165 | }; 166 | 167 | // Insert 168 | await mixedStorageKVStore.UpsertAsync(0, dummyStructInstance).ConfigureAwait(false); 169 | 170 | // Read 171 | (Status status, DummyStruct result) = await mixedStorageKVStore.ReadAsync(0).ConfigureAwait(false); 172 | 173 | // Verify 174 | Assert.Equal(Status.OK, status); 175 | Assert.Equal(dummyStructInstance.DummyByte, result.DummyByte); 176 | Assert.Equal(dummyStructInstance.DummyShort, result.DummyShort); 177 | Assert.Equal(dummyStructInstance.DummyInt, result.DummyInt); 178 | Assert.Equal(dummyStructInstance.DummyLong, result.DummyLong); 179 | ``` 180 | 181 | #### Mutable Type as Key Type 182 | Before we conclude this section on key and value types, a word of caution on using mutable types (type with members you can modify after creation) 183 | as key types: 184 | 185 | Under-the-hood, the binary serialized form of what you pass as keys are the actual keys. 186 | This means that if you pass an instance of a mutable type as a key, then modify a member, you can no longer use it retrieve the original record. 187 | 188 | For example, consider the situation where you insert a value using a `DummyClass` instance (defined [above](#common-key-and-value-types)) as key, and then change a member of the instance. 189 | When you try to read the value using the same instance, you either read nothing or a different value: 190 | 191 | ```csharp 192 | var mixedStorageKVStore = new MixedStorageKVStore(); 193 | var dummyClassInstance = new DummyClass() 194 | { 195 | DummyString = "dummyString", 196 | DummyStringArray = new[] { "dummyString1", "dummyString2", "dummyString3", "dummyString4", "dummyString5" }, 197 | DummyInt = 10, 198 | DummyIntArray = new[] { 10, 100, 1000, 10000, 100000, 1000000, 10000000 } 199 | }; 200 | 201 | // Insert 202 | await mixedStorageKVStore.UpsertAsync(dummyClassInstance, "dummyKey").ConfigureAwait(false); 203 | 204 | // Read 205 | dummyClassInstance.DummyInt = 11; // Change a member 206 | (Status status, string? result) = await mixedStorageKVStore.ReadAsync(dummyClassInstance).ConfigureAwait(false); 207 | 208 | // Verify 209 | Assert.Equal(Status.NOTFOUND, status); // No value for given key 210 | Assert.Null(result); 211 | ``` 212 | 213 | We suggest avoiding mutable object types as key types. 214 | 215 | ### Using This Library in Highly Concurrent Logic 216 | `MixedStorageKVStore.UpsertAsync`, `MixedStorageKVStore.DeleteAsync` and `MixedStorageKVStore.ReadAsync` are thread-safe and suitable for highly concurrent situations 217 | situations. Some example usage: 218 | 219 | ```csharp 220 | var mixedStorageKVStore = new MixedStorageKVStore(); 221 | int numRecords = 100_000; 222 | 223 | // Concurrent inserts 224 | ConcurrentQueue upsertTasks = new(); 225 | Parallel.For(0, numRecords, key => upsertTasks.Enqueue(mixedStorageKVStore.UpsertAsync(key, "dummyString1"))); 226 | await Task.WhenAll(upsertTasks).ConfigureAwait(false); 227 | 228 | // Concurrent reads 229 | ConcurrentQueue> readTasks = new(); 230 | Parallel.For(0, numRecords, key => readTasks.Enqueue(mixedStorageKVStore.ReadAsync(key))); 231 | foreach (ValueTask<(Status, string?)> task in readTasks) 232 | { 233 | // Verify 234 | Assert.Equal((Status.OK, "dummyString1"), await task.ConfigureAwait(false)); 235 | } 236 | 237 | // Concurrent updates 238 | upsertTasks.Clear(); 239 | Parallel.For(0, numRecords, key => upsertTasks.Enqueue(mixedStorageKVStore.UpsertAsync(key, "dummyString2"))); 240 | await Task.WhenAll(upsertTasks).ConfigureAwait(false); 241 | 242 | // Read again so we can verify updates 243 | readTasks.Clear(); 244 | Parallel.For(0, numRecords, key => readTasks.Enqueue(mixedStorageKVStore.ReadAsync(key))); 245 | foreach (ValueTask<(Status, string?)> task in readTasks) 246 | { 247 | // Verify 248 | Assert.Equal((Status.OK, "dummyString2"), await task.ConfigureAwait(false)); 249 | } 250 | 251 | // Concurrent deletes 252 | ConcurrentQueue> deleteTasks = new(); 253 | Parallel.For(0, numRecords, key => deleteTasks.Enqueue(mixedStorageKVStore.DeleteAsync(key))); 254 | foreach (ValueTask task in deleteTasks) 255 | { 256 | Status result = await task.ConfigureAwait(false); 257 | 258 | // Verify 259 | Assert.Equal(Status.OK, result); 260 | } 261 | 262 | // Read again so we can verify deletes 263 | readTasks.Clear(); 264 | Parallel.For(0, numRecords, key => readTasks.Enqueue(mixedStorageKVStore.ReadAsync(key))); 265 | foreach (ValueTask<(Status, string?)> task in readTasks) 266 | { 267 | // Verify 268 | Assert.Equal((Status.NOTFOUND, null), await task.ConfigureAwait(false)); 269 | } 270 | ``` 271 | 272 | ### Configuring 273 | To configure a `MixedStorageKVStore`, pass it a `MixedStorageKVStoreOptions` instance: 274 | 275 | ```csharp 276 | var mixedStorageKVStoreOptions = new MixedStorageKVStoreOptions() 277 | { 278 | // Specify options 279 | LogDirectory = "my/log/directory", 280 | ... 281 | }; 282 | var mixedStorageKVStore = new MixedStorageKVStore(mixedStorageKVStoreOptions); 283 | ``` 284 | 285 | We've listed all of the options in the API section: [`MixedStorageKVStoreOptions`](#mixedstoragekvstoreoptions-class). 286 | 287 | #### Advanced Configuration 288 | If you want greater control over faster, you can pass a manually configured `FasterKV` instance to `MixedStorageKVStore`: 289 | 290 | ```csharp 291 | var logSettings = new LogSettings() // Faster options type 292 | { 293 | // Specify options 294 | ... 295 | }; 296 | var fasterKV = new FasterKV(1L << 20, logSettings)); // Manually configured FasterKV 297 | var mixedStorageKVStoreOptions = new MixedStorageKVStoreOptions() 298 | { 299 | // Specify options 300 | LogDirectory = "my/log/directory", 301 | ... 302 | }; 303 | var mixedStorageKVStore = new MixedStorageKVStore(mixedStorageKVStoreOptions, fasterKVStore: fasterKV); 304 | ``` 305 | 306 | ### Creating and Managing On-Disk Data 307 | `MixedStorageKVStore` stores data across memory and disk. This section briefly covers on-disk data. 308 | 309 | - When is data written to disk? `MixedStorageKVStore` writes to disk when 310 | the in-memory region of your store is full. You can configure the size of the in-memory region using 311 | [`MixedStorageKVStoreOptions.MemorySizeBits`](#MixedStorageKVStoreOptionsMemorySizeBits). 312 | 313 | - Where is on-disk data located? By default, it is located in `/FasterLogs`, where `` is the value returned by `Path.GetTempPath()`. 314 | You can specify `` using [`MixedStorageKVStoreOptions.LogDirectory`](#MixedStorageKVStoreOptionsLogDirectory). 315 | 316 | - Can I recreate a `MixedStorageKVStore` from on-disk data? You can do this using Faster's checkpointing system. This library 317 | doesn't wrap the system, so you'll have to do it manually. 318 | 319 | The following example writes data to disk: 320 | 321 | ```csharp 322 | var mixedStorageKVStoreOptions = new MixedStorageKVStoreOptions() 323 | { 324 | PageSizeBits = 12, // See MixedStorageKVStoreOptions.PageSizeBits in the MixedStorageKVStoreOptions section above 325 | MemorySizeBits = 13, 326 | DeleteLogOnClose = false // Disables automatic deleting of files on disk. See MixedStorageKVStoreOptions.DeleteLogOnClose in the MixedStorageKVStoreOptions section above 327 | }; 328 | var mixedStorageKVStore = new MixedStorageKVStore(mixedStorageKVStoreOptions); 329 | 330 | // Insert 331 | ConcurrentQueue upsertTasks = new(); 332 | Parallel.For(0, 100_000, key => upsertTasks.Enqueue(mixedStorageKVStore.UpsertAsync(key, "dummyString1"))); 333 | await Task.WhenAll(upsertTasks).ConfigureAwait(false); 334 | ``` 335 | 336 | You will find a file in `/FasterLogs` named `.log.0`. An example absolute filepath on windows might look like 337 | `C:/Users/UserName/AppData/Local/Temp/FasterLogs/836b4239-ab56-4fa8-b3a5-833cbd198044.log.0`. 338 | 339 | #### Managing Files 340 | By default, `MixedStorageKVStore` deletes files on disposal or finalization. 341 | If your program terminates abruptly, `MixedStorageKVStore` may not delete files. 342 | We suggest: 343 | 344 | - Placing all files in the same directory. Do this by specifying the same [`MixedStorageKVStoreOptions.LogDirectory`](#MixedStorageKVStoreOptionsLogDirectory) for all `MixedStorageKVStore`s. This is the default behaviour: 345 | all files are placed in `/FasterLogs`. 346 | - On application initialization, delete the directory if it exists: 347 | ```csharp 348 | try 349 | { 350 | Directory.Delete(Path.Combine(Path.GetTempPath(), "FasterLogs"), true); 351 | } 352 | catch 353 | { 354 | // Do nothing 355 | } 356 | ``` 357 | 358 | #### Managing Disk Space 359 | `MixedStorageKVStore` performs log compaction periodically, however, data can only be so compact - the size of your data can grow boundlessly as long as you're adding new records. 360 | Therefore, we recommend monitoring disk space the same way you would monitor any other metric. 361 | 362 | ## API 363 | 364 | 365 | ### MixedStorageKVStore Class 366 | #### Constructors 367 | ##### MixedStorageKVStore(MixedStorageKVStoreOptions, ILogger>, FasterKV) 368 | 369 | Creates a `MixedStorageKVStore`. 370 | ```csharp 371 | public MixedStorageKVStore([MixedStorageKVStoreOptions? mixedStorageKVStoreOptions = null], [ILogger>? logger = null], [FasterKV? fasterKVStore = null]) 372 | ``` 373 | ###### Parameters 374 | mixedStorageKVStoreOptions `MixedStorageKVStoreOptions` 375 | The options for the `MixedStorageKVStore`. 376 | 377 | logger `ILogger>` 378 | The logger for log compaction events. 379 | 380 | fasterKVStore `FasterKV` 381 | The underlying `FasterKV` for the `MixedStorageKVStore`. 382 | This parameter allows you to use a manually configured Faster instance. 383 | #### Properties 384 | ##### MixedStorageKVStore.FasterKV 385 | 386 | Gets the underlying `FasterKV` instance. 387 | ```csharp 388 | public FasterKV FasterKV { get; } 389 | ``` 390 | #### Methods 391 | ##### MixedStorageKVStore.UpsertAsync(TKey, TValue) 392 | 393 | Updates or inserts a record asynchronously. 394 | ```csharp 395 | public Task UpsertAsync(TKey key, TValue obj) 396 | ``` 397 | ###### Parameters 398 | key `TKey` 399 | The record's key. 400 | 401 | obj `TValue` 402 | The record's new value. 403 | ###### Returns 404 | The task representing the asynchronous operation. 405 | ###### Exceptions 406 | `ObjectDisposedException` 407 | Thrown if the instance or a dependency is disposed. 408 | ###### Remarks 409 | This method is thread-safe. 410 | ##### MixedStorageKVStore.DeleteAsync(TKey) 411 | 412 | Deletes a record asynchronously. 413 | ```csharp 414 | public ValueTask DeleteAsync(TKey key) 415 | ``` 416 | ###### Parameters 417 | key `TKey` 418 | The record's key. 419 | ###### Returns 420 | The task representing the asynchronous operation. 421 | ###### Exceptions 422 | `ObjectDisposedException` 423 | Thrown if the instance or a dependency is disposed. 424 | ###### Remarks 425 | This method is thread-safe. 426 | ##### MixedStorageKVStore.ReadAsync(TKey) 427 | 428 | Reads a record asynchronously. 429 | ```csharp 430 | public ValueTask<(Status, TValue?)> ReadAsync(TKey key) 431 | ``` 432 | ###### Parameters 433 | key `TKey` 434 | The record's key. 435 | ###### Returns 436 | The task representing the asynchronous operation. 437 | ###### Exceptions 438 | `ObjectDisposedException` 439 | Thrown if the instance or a dependency is disposed. 440 | ###### Remarks 441 | This method is thread-safe. 442 | ##### MixedStorageKVStore.Dispose() 443 | 444 | Disposes this instance. 445 | ```csharp 446 | public void Dispose() 447 | ``` 448 | 449 | 450 | 451 | 452 | ### MixedStorageKVStoreOptions Class 453 | #### Constructors 454 | ##### MixedStorageKVStoreOptions() 455 | ```csharp 456 | public MixedStorageKVStoreOptions() 457 | ``` 458 | #### Properties 459 | ##### MixedStorageKVStoreOptions.IndexNumBuckets 460 | The number of buckets in Faster's index. 461 | ```csharp 462 | public long IndexNumBuckets { get; set; } 463 | ``` 464 | ###### Remarks 465 | Each bucket is 64 bits. 466 | 467 | This value is ignored if a `FasterKV` instance is supplied to the `MixedStorageKVStore` constructor. 468 | 469 | Defaults to 1048576 (64 MB index). 470 | ##### MixedStorageKVStoreOptions.PageSizeBits 471 | The size of a page in Faster's log. 472 | ```csharp 473 | public int PageSizeBits { get; set; } 474 | ``` 475 | ###### Remarks 476 | A page is a contiguous block of in-memory or on-disk storage. 477 | 478 | This value is ignored if a `FasterKV` instance is supplied to the `MixedStorageKVStore` constructor. 479 | 480 | Defaults to 25 (2^25 = 33.5 MB). 481 | ##### MixedStorageKVStoreOptions.MemorySizeBits 482 | The size of the in-memory region of Faster's log. 483 | ```csharp 484 | public int MemorySizeBits { get; set; } 485 | ``` 486 | ###### Remarks 487 | If the log outgrows this region, overflow is moved to its on-disk region. 488 | 489 | This value is ignored if a `FasterKV` instance is supplied to the `MixedStorageKVStore` constructor. 490 | 491 | Defaults to 26 (2^26 = 67 MB). 492 | ##### MixedStorageKVStoreOptions.SegmentSizeBits 493 | The size of a segment of the on-disk region of Faster's log. 494 | ```csharp 495 | public int SegmentSizeBits { get; set; } 496 | ``` 497 | ###### Remarks 498 | What is a segment? Records on disk are split into groups called segments. Each segment corresponds to a file. 499 | 500 | For performance reasons, segments are "pre-allocated". This means they are not created empty and left to grow gradually, instead they are created at the size specified by this value and populated gradually. 501 | 502 | This value is ignored if a `FasterKV` instance is supplied to the `MixedStorageKVStore` constructor. 503 | 504 | Defaults to 28 (268 MB). 505 | ##### MixedStorageKVStoreOptions.LogDirectory 506 | The directory containing the on-disk region of Faster's log. 507 | ```csharp 508 | public string? LogDirectory { get; set; } 509 | ``` 510 | ###### Remarks 511 | If this value is `null` or an empty string, log files are placed in "<temporary path>/FasterLogs" where 512 | "<temporary path>" is the value returned by `Path.GetTempPath`. 513 | 514 | Note that nothing is written to disk while your log fits in-memory. 515 | 516 | This value is ignored if a `FasterKV` instance is supplied to the `MixedStorageKVStore` constructor. 517 | 518 | Defaults to `null`. 519 | ##### MixedStorageKVStoreOptions.LogFileNamePrefix 520 | The Faster log filename prefix. 521 | ```csharp 522 | public string? LogFileNamePrefix { get; set; } 523 | ``` 524 | ###### Remarks 525 | The on-disk region of the log is stored across multiple files. Each file is referred to as a segment. 526 | Each segment has file name "<log file name prefix>.log.<segment number>". 527 | 528 | 529 | If this value is `null` or an empty string, a random `Guid` is used as the prefix. 530 | 531 | This value is ignored if a `FasterKV` instance is supplied to the `MixedStorageKVStore` constructor. 532 | 533 | Defaults to `null`. 534 | ##### MixedStorageKVStoreOptions.TimeBetweenLogCompactionsMS 535 | The time between Faster log compaction attempts. 536 | ```csharp 537 | public int TimeBetweenLogCompactionsMS { get; set; } 538 | ``` 539 | ###### Remarks 540 | If this value is negative, log compaction is disabled. 541 | 542 | Defaults to 60000. 543 | ##### MixedStorageKVStoreOptions.InitialLogCompactionThresholdBytes 544 | The initial log compaction threshold. 545 | ```csharp 546 | public long InitialLogCompactionThresholdBytes { get; internal set; } 547 | ``` 548 | ###### Remarks 549 | Initially, log compactions only run when the Faster log's safe-readonly region's size is larger than or equal to this value. 550 | 551 | If log compaction runs 5 times in a row, this value is doubled. Why? Consider the situation where the safe-readonly region is already 552 | compact, but still larger than the threshold. Not increasing the threshold would result in redundant compaction runs. 553 | 554 | If this value is less than or equal to 0, the initial log compaction threshold is 2 * memory size in bytes (`MixedStorageKVStoreOptions.MemorySizeBits`). 555 | 556 | Defaults to 0. 557 | ##### MixedStorageKVStoreOptions.DeleteLogOnClose 558 | The value specifying whether log files are deleted when the `MixedStorageKVStore` is disposed or finalized (at which points underlying log files are closed). 559 | ```csharp 560 | public bool DeleteLogOnClose { get; set; } 561 | ``` 562 | ###### Remarks 563 | This value is ignored if a `FasterKV` instance is supplied to the `MixedStorageKVStore` constructor. 564 | 565 | Defaults to `true`. 566 | ##### MixedStorageKVStoreOptions.MessagePackSerializerOptions 567 | The options for serializing data using MessagePack C#. 568 | ```csharp 569 | public MessagePackSerializerOptions MessagePackSerializerOptions { get; set; } 570 | ``` 571 | ###### Remarks 572 | MessagePack C# is a performant binary serialization library. Refer to [MessagePack C# documentation](https://github.com/neuecc/MessagePack-CSharp) 573 | for details. 574 | 575 | Defaults to `MessagePackSerializerOptions.Standard` with compression using `MessagePackCompression.Lz4BlockArray`. 576 | 577 | 578 | ## Performance 579 | ### Benchmarks 580 | The following benchmarks use `MixedStorageKVStore`s with key type `int` and value type `DummyClass` as defined and populated in [this section](#common-key-and-value-types). 581 | The `MixedStorageKVStore`s are configured to provide basis for comparison with disk-based alternatives like Sqlite and LiteDB: 582 | 583 | - MessagePack C# compression is disabled 584 | - The vast majority of the store is on-disk (8 KB in-memory region, multi-MB on-disk region) 585 | - Log compaction is disabled 586 | 587 | Benchmarks: 588 | 589 | - Inserts_WithoutCompression performs 350,000 single-record insertions 590 | - Reads_WithoutCompression performs 75,000 single-record reads 591 | 592 | View source [here](https://github.com/JeringTech/KeyValueStore/blob/main/perf/KeyValueStore/LowMemoryUsageBenchmarks.cs). 593 | 594 | Results: 595 | 596 | | Method | Mean | Error | StdDev | Median | Gen 0 | Gen 1 | Gen 2 | Allocated | 597 | |--------------------------- |-----------:|---------:|----------:|-----------:|-----------:|-----------:|----------:|----------:| 598 | | Inserts_WithoutCompression | 685.6 ms | 73.33 ms | 201.98 ms | 615.5 ms | 52000.0000 | 17000.0000 | 4000.0000 | 217.97 MB | 599 | | Reads_WithoutCompression | 1,197.2 ms | 23.69 ms | 26.33 ms | 1,190.0 ms | 38000.0000 | 13000.0000 | - | 156.28 MB | 600 | 601 | ``` ini 602 | 603 | BenchmarkDotNet=v0.12.1, OS=Windows 10.0.19041.928 (2004/?/20H1) 604 | Intel Core i7-7700 CPU 3.60GHz (Kaby Lake), 1 CPU, 8 logical and 4 physical cores 605 | .NET Core SDK=5.0.300-preview.21180.15 606 | [Host] : .NET Core 5.0.6 (CoreCLR 5.0.621.22011, CoreFX 5.0.621.22011), X64 RyuJIT 607 | Job-JXJRVC : .NET Core 5.0.6 (CoreCLR 5.0.621.22011, CoreFX 5.0.621.22011), X64 RyuJIT 608 | 609 | InvocationCount=1 UnrollFactor=1 610 | 611 | ``` 612 | 613 | #### Cursory analysis 614 | Insert performance is excellent. Per-second insertion rate beats disk-based alternatives by an order of magnitude or more. 615 | Read performance is good - similar to the fastest disk-based alternatives. 616 | 617 | ### Future Performance Improvements 618 | Performance of the current `MixedStorageKeyValueStore` implementation exceeds our requirements. That said, we're open to pull-requests improving performance. 619 | Several low-hanging fruit: 620 | 621 | - Support [Faster's read only cache](https://microsoft.github.io/FASTER/docs/fasterkv-tuning/#configuring-the-read-cache). This is an in-memory 622 | cache of recently read records. Depending on read-patterns, this could reduce average read latency significantly. 623 | 624 | - Fast-path for blittable types: [Blittable types](https://docs.microsoft.com/en-us/dotnet/framework/interop/blittable-and-non-blittable-types) are fixed-length value-types. 625 | Instances of these types can be converted to their binary forms without going through MessagePack C#. Also they are fixed-length and so do not need `SpanByte` wrappers. 626 | We ran internal benchmarks for a `MixedStorageKeyValueStore` store with a fast path for its `int` keys. Read performance improved by ~20%. 627 | 628 | - Use object log for mostly-in-memory situations. 629 | 630 | ## Building and Testing 631 | You can build and test this project in Visual Studio 2019. 632 | 633 | ## Alternatives 634 | - [Faster](https://microsoft.github.io/FASTER) 635 | 636 | - [ManagedEsent `PersistentDictionary`](https://github.com/microsoft/ManagedEsent/blob/master/Documentation/PersistentDictionaryDocumentation.md) 637 | 638 | - [SQLite](https://docs.microsoft.com/en-us/dotnet/standard/data/sqlite/?tabs=netcore-cli) 639 | 640 | - [LiteDB](https://www.litedb.org/) 641 | 642 | - [MemoryCache](https://docs.microsoft.com/en-us/dotnet/api/system.runtime.caching.memorycache?view=dotnet-plat-ext-5.0) 643 | 644 | ## Related Concepts 645 | ### Faster Basics 646 | This section provides enough information about Faster to use this library effectively. Refer to the 647 | [official Faster documentation](https://microsoft.github.io/FASTER/docs/fasterkv-basics/) for complete information on Faster. 648 | 649 | Faster is a key-value store library. `FasterKV` is the key-value store type Faster exposes. 650 | A `FasterKV` instance is composed of an **index** and a **log**. 651 | 652 | #### Index 653 | The index is a simple [hash table](https://en.wikipedia.org/wiki/Hash_table) that maps keys to locations in the log. 654 | 655 | #### Log 656 | You can think of the log as a list of key-value pairs (records). For example, say we insert 3 records with keys 0, 1, and 2 and value "dummyString1". We insert them in order 657 | of increasing key value. Our log will look like this: 658 | 659 | ``` 660 | // Head 661 | key: 0, value: "dummyString1" 662 | key: 1, value: "dummyString1" 663 | key: 2, value: "dummyString1" 664 | // Tail - records are added here 665 | ``` 666 | 667 | Mutiple records can have the same key. Say we update values to "dummyString2", our log will now look like this: 668 | 669 | ``` 670 | // Head 671 | key: 0, value: "dummyString1" 672 | key: 1, value: "dummyString1" 673 | key: 2, value: "dummyString1" 674 | // Index points to these - the most recent records for each key 675 | key: 0, value: "dummyString2" 676 | key: 1, value: "dummyString2" 677 | key: 2, value: "dummyString2" 678 | // Tail 679 | ``` 680 | 681 | **Log compaction** removes redundant records. After log compaction: 682 | 683 | ``` 684 | key: 0, value: "dummyString2" 685 | key: 1, value: "dummyString2" 686 | key: 2, value: "dummyString2" 687 | ``` 688 | 689 | By default, `MixedStorageKeyValueStore` performs periodic log compactions for you. 690 | 691 | The log can span memory and disk. Say we configure the in-memory region to fit 3 records. If we have 6 records, 3 end up on disk: 692 | 693 | ``` 694 | // In-memory region 695 | key: 0, value: "dummyString2" 696 | key: 1, value: "dummyString2" 697 | key: 2, value: "dummyString2" 698 | // On-disk region 699 | key: 3, value: "dummyString2" 700 | key: 4, value: "dummyString2" 701 | key: 5, value: "dummyString2" 702 | ``` 703 | 704 | Records on disk are split into groups called **segments**. Each segment corresponds to a fixed-size file. Say we configure segments to fit 3 records. 705 | If we have 4 records, the on-disk region of our log will look like this: 706 | 707 | ``` 708 | // On-disk region 709 | // Segment 0, full 710 | key: 3, value: "dummyString2" 711 | key: 4, value: "dummyString2" 712 | key: 5, value: "dummyString2" 713 | // Segment 1, partially filled 714 | key: 6, value: "dummyString2" 715 | empty 716 | empty 717 | ``` 718 | 719 | `MixedStorageKeyValueStore` has Faster "pre-allocate" files to speeds up inserts. This means files are not created empty and left to grow gradually, 720 | instead they are created at the segment size of our choosing (see [`MixedStorageKVStoreOptions.SegmentSizeBits`](#mixedstoragekvstoreoptions-class)) and populated gradually. 721 | Choosing a larger segment size means more empty, "reserved" disk space. Choosing a smaller segment size means creating more files. It's up to you to 722 | to weigh tradeoffs. 723 | 724 | The log is also subdivided into **pages**. Pages are a separate concept from segments - segments typically consist of multiple pages. 725 | Pages are contiguous blocks of in-memory or on-disk storage. What are pages for? Records are moved around as pages. For example, when we add records, 726 | they are held in memory and only written to the log after we've added enough to fill a page. You can specify page size using [`MixedStorageKVStoreOptions.PageSizeBits`](#mixedstoragekvstoreoptions-class). 727 | 728 | Note: Both segments and pages affect Faster in ways that aren't relevant to the current `MixedStorageKeyValueStore` implementation. Refer to the official Faster documentation 729 | to learn more about these concepts. 730 | 731 | #### Sessions 732 | Most interactions with a `FasterKV` instance are done through sessions. A session's members are not thread safe: 733 | 734 | ```csharp 735 | // Create FasterKV instance 736 | FasterKV fasterKV = new(1L << 20, logSettings); // logSettings is an instance of LogSettings 737 | 738 | // Create session 739 | var session = fasterKV.For(simpleFunctions).NewSession>(); // simpleFunction is an instance of SimpleFunctions 740 | 741 | // Perform operations 742 | await session.UpsertAsync(0, "dummyString").ConfigureAwait(false); // Not thread-safe. You need to manage a pool of sessions for multi-threaded logic. 743 | ``` 744 | 745 | `MixedStorageKeyValueStore` abstracts sessions away: 746 | ```csharp 747 | // Create MixedStorageKeyValueStore instance 748 | MixedStorageKeyValueStore mixedStorageKeyValueStore = new(); 749 | 750 | // Perform operations 751 | await mixedStorageKeyValueStore.UpsertAsync().ConfigureAwait(false); // Thread-safe 752 | ``` 753 | 754 | #### Serialization 755 | Faster differentiates between fixed-length types (primitives, structs with only primitive members etc), 756 | and variable-length types (reference types, structs with variable-length members etc). 757 | 758 | By default, `FasterKV` serializes variable-length types using [`DataContractSerializer`](https://docs.microsoft.com/en-us/dotnet/api/system.runtime.serialization.datacontractserializer?view=net-5.0), 759 | a slow, space-inefficient XML serializer. It writes serialized variable-length data to a secondary log, the "object log". 760 | Use of the object log slows down reads and writes. 761 | 762 | To keep all data in the primary log, the Faster team added the `SpanByte` struct. 763 | `SpanByte` can be thought of as a wrapper for "integer specifying data length" + "serialized variable-length data". 764 | If a `FasterKV` instance is instantiated with `Spanbyte` in place of variable-length types, for example, `FasterKV` 765 | instead of `FasterKV`, *all data* is written to the primary log. 766 | 767 | To use `SpanByte`, you need to manually serialize variable-length types and instantiate `SpanByte`s around the resultant data. 768 | 769 | `MixedStorageKeyValueStore` abstracts all that away: 770 | ```csharp 771 | var mixedStorageKVStore = new MixedStorageKVStore(); 772 | 773 | // Upsert updates or inserts records. 774 | // 775 | // Under-the-hood, MixedStorageKeyValueStore serializes dummyClassInstance using the MessagePack C# library, 776 | // creates a `SpanByte` around the resultant data, and passes the `SpanByte` to the underlying FasterKV instance. 777 | mixedStorageKVStore.Upsert(0, dummyClassInstance); 778 | ``` 779 | 780 | Note: Writing to the object log is more performant than the `SpanByte` system when most of the log is in memory. 781 | Supporting object log for such situations is listed under [Future Performance Improvements](#future-performance-improvements). 782 | 783 | ## Contributing 784 | Contributions are welcome! 785 | 786 | ### Contributors 787 | - [JeremyTCD](https://github.com/JeremyTCD) 788 | 789 | Thanks to [badrishc](https://github.com/badrishc) for help getting started with Faster. Quite a bit of this library is based 790 | on his suggestions. 791 | 792 | ## About 793 | Follow [@JeringTech](https://twitter.com/JeringTech) for updates and more. 794 | -------------------------------------------------------------------------------- /ThirdPartyLicenses.md: -------------------------------------------------------------------------------- 1 | # Third Party Licenses 2 | 3 | ## [Microsoft.Faster](https://github.com/microsoft/FASTER/blob/master/LICENSE) 4 | 5 | MIT License 6 | 7 | Copyright (c) Microsoft Corporation. All rights reserved. 8 | 9 | Permission is hereby granted, free of charge, to any person obtaining a copy 10 | of this software and associated documentation files (the "Software"), to deal 11 | in the Software without restriction, including without limitation the rights 12 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 13 | copies of the Software, and to permit persons to whom the Software is 14 | furnished to do so, subject to the following conditions: 15 | 16 | The above copyright notice and this permission notice shall be included in all 17 | copies or substantial portions of the Software. 18 | 19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 22 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 24 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 25 | SOFTWARE 26 | -------------------------------------------------------------------------------- /azure-pipelines.yml: -------------------------------------------------------------------------------- 1 | trigger: 2 | - main 3 | 4 | resources: 5 | repositories: 6 | - repository: templates 7 | type: github 8 | name: JeringTech/DevOps.AzurePipelines 9 | endpoint: JeringTech 10 | 11 | jobs: 12 | - template: templates/nuget/main.yml@templates 13 | parameters: 14 | codecovKey: "67424e1a-1c96-4db4-8874-bfa35e2d302c" 15 | - template: templates/docs/main.yml@templates 16 | parameters: 17 | nugetRestorePats: "$(nugetRestorePats)" 18 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | # Disable project and patch statuses - https://docs.codecov.io/docs/commit-status. 2 | # Developers should use their own discretion when accepting/rejecting code coverage statistics. 3 | coverage: 4 | status: 5 | project: off 6 | patch: off -------------------------------------------------------------------------------- /generators/Jering.KeyValueStore.Generators/ApiDocumentationGenerator.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.CodeAnalysis; 2 | using Microsoft.CodeAnalysis.CSharp; 3 | using Microsoft.CodeAnalysis.CSharp.Syntax; 4 | using System.Collections.Generic; 5 | using System.Collections.Immutable; 6 | using System.IO; 7 | using System.Linq; 8 | using System.Text; 9 | using System.Text.RegularExpressions; 10 | using System.Threading; 11 | using System.Web; 12 | using System.Xml; 13 | using System.Xml.Linq; 14 | 15 | #nullable enable 16 | 17 | namespace Jering.KeyValueStore.Generators 18 | { 19 | /// 20 | /// Generates API documenation and inserts it into ReadMe. 21 | /// To add API documentation for a type to ReadMe, simply add a "<!-- typeName generated docs --><!-- typeName generated docs -->" pair. 22 | /// The generator locates all "<!-- typeName generated docs --><!-- typeName generated docs -->" pairs in ReadMe. 23 | /// It extracts typeNames, using them to retrieve type metadata from the compilation's public types. The metadata is used to generate API documentation for the types, 24 | /// which is then inserted within "<!-- typeName generated docs --><!-- typeName generated docs -->" pairs. 25 | /// 26 | [Generator] 27 | public class ApiDocumentationGenerator : SourceGenerator 28 | { 29 | private static readonly DiagnosticDescriptor _missingClassDeclaration = new("G0013", 30 | "Missing class declaration", 31 | "Missing class declaration for: \"{0}\"", 32 | "Code generation", 33 | DiagnosticSeverity.Error, 34 | true); 35 | private static readonly DiagnosticDescriptor _missingInterfaceDeclaration = new("G0014", 36 | "Missing interface declaration", 37 | "Missing interface declaration for: \"{0}\"", 38 | "Code generation", 39 | DiagnosticSeverity.Error, 40 | true); 41 | private const string RELATIVE_README_FILE_PATH = "../../ReadMe.md"; 42 | private static string _readMeFilePath = string.Empty; 43 | 44 | 45 | protected override void InitializeCore() 46 | { 47 | // https://docs.microsoft.com/en-sg/visualstudio/releases/2019/release-notes-preview#--visual-studio-2019-version-1610-preview-2- 48 | //Debugger.Launch(); 49 | } 50 | 51 | protected override void ExecuteCore(ref GeneratorExecutionContext context) 52 | { 53 | CancellationToken cancellationToken = context.CancellationToken; 54 | if (cancellationToken.IsCancellationRequested) return; 55 | 56 | // Get syntax receiver 57 | if (context.SyntaxReceiver is not ApiDocumentationGeneratorSyntaxReceiver apiDocumentationGeneratorSyntaxReceiver) 58 | { 59 | return; 60 | } 61 | 62 | // Parse readme 63 | if (_readMeFilePath == string.Empty) 64 | { 65 | _readMeFilePath = Path.Combine(_projectDirectory, RELATIVE_README_FILE_PATH); 66 | } 67 | 68 | string readMeContents; 69 | lock (this) 70 | { 71 | if (!File.Exists(_readMeFilePath)) 72 | { 73 | return; // Project has no readme 74 | } 75 | 76 | readMeContents = File.ReadAllText(_readMeFilePath); 77 | } 78 | MatchCollection matches = Regex.Matches(readMeContents, @"", RegexOptions.IgnoreCase | RegexOptions.Singleline); 79 | if (matches.Count == 0) 80 | { 81 | return; // No types to generate docs for 82 | } 83 | List<(int indexBeforeDocs, int indexAfterDocs, string typeName)> generatedDocsTypes = new(); 84 | foreach (Match match in matches) 85 | { 86 | generatedDocsTypes.Add((match.Groups[2].Index, match.Groups[3].Index, match.Groups[1].Value)); 87 | } 88 | 89 | // Generate docs 90 | StringBuilder stringBuilder = new(); 91 | int nextStartIndex = 0; 92 | foreach ((int indexBeforeDocs, int indexAfterDocs, string typeName) in generatedDocsTypes) 93 | { 94 | if (cancellationToken.IsCancellationRequested) 95 | { 96 | return; 97 | } 98 | 99 | stringBuilder. 100 | Append(readMeContents, nextStartIndex, indexBeforeDocs - nextStartIndex + 1). 101 | Append("\n\n"); 102 | 103 | if (typeName.StartsWith("I")) 104 | { 105 | if (!apiDocumentationGeneratorSyntaxReceiver.PublicInterfaceDeclarations.TryGetValue(typeName, out InterfaceDeclarationSyntax interfaceDeclarationSyntax)) 106 | { 107 | context.ReportDiagnostic(Diagnostic.Create(_missingInterfaceDeclaration, null, typeName)); 108 | continue; 109 | } 110 | 111 | stringBuilder.AppendInterfaceDocumentation(interfaceDeclarationSyntax, ref context); 112 | } 113 | else 114 | { 115 | if (!apiDocumentationGeneratorSyntaxReceiver.PublicClassDeclarations.TryGetValue(typeName, out ClassDeclarationSyntax classDeclarationSyntax)) 116 | { 117 | context.ReportDiagnostic(Diagnostic.Create(_missingClassDeclaration, null, typeName)); 118 | continue; 119 | } 120 | 121 | stringBuilder.AppendClassDocumentation(classDeclarationSyntax, ref context); 122 | } 123 | 124 | nextStartIndex = indexAfterDocs; 125 | } 126 | stringBuilder.Append(readMeContents, nextStartIndex, readMeContents.Length - nextStartIndex); 127 | 128 | // Update file 129 | string newReadMeContents = stringBuilder.ToString(); 130 | if (cancellationToken.IsCancellationRequested || newReadMeContents == readMeContents) 131 | { 132 | return; 133 | } 134 | lock (this) 135 | { 136 | File.WriteAllText(_readMeFilePath, newReadMeContents); 137 | } 138 | } 139 | } 140 | 141 | public static class StringBuilderExtensions 142 | { 143 | private static readonly DiagnosticDescriptor _missingTypeSymbol = new("G0015", 144 | "Missing type symbol", 145 | "Missing type symbol for: \"{0}\"", 146 | "Code generation", 147 | DiagnosticSeverity.Error, 148 | true); 149 | private static readonly DiagnosticDescriptor _missingMemberSymbol = new("G0016", 150 | "Missing member symbol", 151 | "Missing member symbol for: \"{0}\"", 152 | "Code generation", 153 | DiagnosticSeverity.Error, 154 | true); 155 | private static readonly DiagnosticDescriptor _crefValueWithUnexpectedPrefix = new("G0017", 156 | "Cref value with unexpected prefix", 157 | "Cref value with unexpected prefix: \"{0}\"", 158 | "Code generation", 159 | DiagnosticSeverity.Error, 160 | true); 161 | 162 | public static StringBuilder AppendInterfaceDocumentation(this StringBuilder stringBuilder, InterfaceDeclarationSyntax interfaceDeclarationSyntax, ref GeneratorExecutionContext context) 163 | { 164 | Compilation compilation = context.Compilation; 165 | SemanticModel semanticModel = compilation.GetSemanticModel(interfaceDeclarationSyntax.SyntaxTree); 166 | 167 | // Interface title 168 | INamedTypeSymbol? interfaceSymbol = semanticModel.GetDeclaredSymbol(interfaceDeclarationSyntax); 169 | if (interfaceSymbol == null) 170 | { 171 | return stringBuilder; 172 | } 173 | stringBuilder. 174 | Append("### "). 175 | Append(ToHtmlEncodedDisplayString(interfaceSymbol, DisplayFormats.TypeTitleDisplayFormat)). 176 | AppendLine(" Interface"); 177 | 178 | // Members 179 | IEnumerable publicMemberSymbols = interfaceSymbol.GetMembers().Where(memberSymbol => memberSymbol.DeclaredAccessibility == Accessibility.Public); 180 | if (publicMemberSymbols.Count() == 0) 181 | { 182 | return stringBuilder; 183 | } 184 | 185 | return stringBuilder. 186 | AppendProperties(publicMemberSymbols, compilation, ref context). 187 | AppendOrdinaryMethods(publicMemberSymbols, compilation, ref context); 188 | } 189 | 190 | public static StringBuilder AppendClassDocumentation(this StringBuilder stringBuilder, ClassDeclarationSyntax classDeclarationSyntax, ref GeneratorExecutionContext context) 191 | { 192 | Compilation compilation = context.Compilation; 193 | SemanticModel semanticModel = compilation.GetSemanticModel(classDeclarationSyntax.SyntaxTree); 194 | 195 | // Class title 196 | INamedTypeSymbol? classSymbol = semanticModel.GetDeclaredSymbol(classDeclarationSyntax); 197 | if (classSymbol == null) 198 | { 199 | return stringBuilder; 200 | } 201 | stringBuilder. 202 | Append("### "). 203 | Append(ToHtmlEncodedDisplayString(classSymbol, DisplayFormats.TypeTitleDisplayFormat)). 204 | AppendLine(" Class"); 205 | 206 | // Members 207 | IEnumerable publicMemberSymbols = classSymbol.GetMembers().Where(memberSymbol => memberSymbol.DeclaredAccessibility == Accessibility.Public); 208 | if (publicMemberSymbols.Count() == 0) 209 | { 210 | return stringBuilder; 211 | } 212 | 213 | // Members - Constructors 214 | IEnumerable publicMethodSymbols = publicMemberSymbols.OfType(); 215 | IEnumerable publicConstructorSymbols = publicMethodSymbols.Where(methodSymbol => methodSymbol.MethodKind == MethodKind.Constructor); 216 | if (publicConstructorSymbols.Count() > 0) 217 | { 218 | stringBuilder.AppendLine(@"#### Constructors"); 219 | 220 | foreach (IMethodSymbol constructorSymbol in publicConstructorSymbols) 221 | { 222 | XElement? rootXmlElement = TryGetXmlDocumentationRootElement(constructorSymbol); 223 | 224 | stringBuilder. 225 | AppendMemberTitle(constructorSymbol, DisplayFormats.ConstructorTitleDisplayFormat). 226 | AppendSummary(rootXmlElement, compilation, ref context). 227 | AppendSignature(constructorSymbol). 228 | AppendParameters(constructorSymbol, rootXmlElement, compilation, ref context). 229 | AppendRemarks(rootXmlElement, compilation, ref context); 230 | } 231 | } 232 | 233 | return stringBuilder. 234 | AppendProperties(publicMemberSymbols, compilation, ref context). 235 | AppendOrdinaryMethods(publicMethodSymbols, compilation, ref context); 236 | } 237 | 238 | public static StringBuilder AppendProperties(this StringBuilder stringBuilder, IEnumerable symbols, Compilation compilation, ref GeneratorExecutionContext context) 239 | { 240 | IEnumerable propertySymbols = symbols.OfType(); 241 | if (propertySymbols.Count() == 0) 242 | { 243 | return stringBuilder; 244 | } 245 | 246 | stringBuilder.AppendLine(@"#### Properties"); 247 | 248 | foreach (IPropertySymbol propertySymbol in propertySymbols) 249 | { 250 | XElement? rootXmlElement = TryGetXmlDocumentationRootElement(propertySymbol); 251 | 252 | stringBuilder. 253 | AppendMemberTitle(propertySymbol, DisplayFormats.propertyTitleDisplayFormat). 254 | AppendSummary(rootXmlElement, compilation, ref context). 255 | AppendSignature(propertySymbol). 256 | AppendRemarks(rootXmlElement, compilation, ref context); 257 | } 258 | 259 | return stringBuilder; 260 | } 261 | 262 | public static StringBuilder AppendOrdinaryMethods(this StringBuilder stringBuilder, IEnumerable symbols, Compilation compilation, ref GeneratorExecutionContext context) 263 | { 264 | IEnumerable ordinaryMethodSymbols = symbols.OfType().Where(methodSymbol => methodSymbol.MethodKind == MethodKind.Ordinary); 265 | if (ordinaryMethodSymbols.Count() == 0) 266 | { 267 | return stringBuilder; 268 | } 269 | 270 | stringBuilder.AppendLine(@"#### Methods"); 271 | 272 | foreach (IMethodSymbol ordinaryMethodSymbol in ordinaryMethodSymbols) 273 | { 274 | XElement? rootXmlElement = TryGetXmlDocumentationRootElement(ordinaryMethodSymbol); 275 | 276 | stringBuilder. 277 | AppendMemberTitle(ordinaryMethodSymbol, DisplayFormats.ordinaryMethodTitleDisplayFormat). 278 | AppendSummary(rootXmlElement, compilation, ref context). 279 | AppendSignature(ordinaryMethodSymbol). 280 | AppendTypeParameters(ordinaryMethodSymbol, rootXmlElement, compilation, ref context). 281 | AppendParameters(ordinaryMethodSymbol, rootXmlElement, compilation, ref context). 282 | AppendReturns(rootXmlElement, compilation, ref context). 283 | AppendExceptions(rootXmlElement, compilation, ref context). 284 | AppendRemarks(rootXmlElement, compilation, ref context). 285 | AppendExample(rootXmlElement, compilation, ref context); 286 | } 287 | 288 | return stringBuilder; 289 | } 290 | 291 | public static StringBuilder AppendMemberTitle(this StringBuilder stringBuilder, ISymbol symbol, SymbolDisplayFormat displayFormat) 292 | { 293 | return stringBuilder. 294 | Append("##### "). 295 | AppendLine(ToHtmlEncodedDisplayString(symbol, displayFormat)); 296 | } 297 | 298 | public static StringBuilder AppendSummary(this StringBuilder stringBuilder, XElement? rootXmlElement, Compilation compilation, ref GeneratorExecutionContext context) 299 | { 300 | if (rootXmlElement == null) 301 | { 302 | return stringBuilder; 303 | } 304 | 305 | return stringBuilder.AppendXmlDocumentation(rootXmlElement.Element("summary"), compilation, ref context); 306 | } 307 | 308 | public static StringBuilder AppendExceptions(this StringBuilder stringBuilder, XElement? rootXmlElement, Compilation compilation, ref GeneratorExecutionContext context) 309 | { 310 | if (rootXmlElement == null) 311 | { 312 | return stringBuilder; 313 | } 314 | 315 | IEnumerable exceptionXmlElements = rootXmlElement.Elements("exception"); 316 | 317 | if (exceptionXmlElements.Count() == 0) 318 | { 319 | return stringBuilder; 320 | } 321 | stringBuilder.AppendLine("###### Exceptions"); 322 | 323 | foreach (XElement exceptionXmlElement in exceptionXmlElements) 324 | { 325 | string? crefValue = exceptionXmlElement.Attribute("cref")?.Value; 326 | if (crefValue == null) 327 | { 328 | continue; 329 | } 330 | 331 | int indexOfLastSeparator = crefValue.LastIndexOf('.'); 332 | string exceptionName = crefValue.Substring(indexOfLastSeparator + 1); 333 | 334 | stringBuilder. 335 | Append('`'). 336 | Append(exceptionName). 337 | AppendLine("` "). 338 | AppendXmlDocumentation(exceptionXmlElement, compilation, ref context). 339 | AppendLine(); 340 | } 341 | 342 | return stringBuilder; 343 | } 344 | 345 | public static StringBuilder AppendReturns(this StringBuilder stringBuilder, XElement? rootXmlElement, Compilation compilation, ref GeneratorExecutionContext context) 346 | { 347 | if (rootXmlElement == null) 348 | { 349 | return stringBuilder; 350 | } 351 | 352 | XElement returnsXmlElement = rootXmlElement.Element("returns"); 353 | 354 | if (returnsXmlElement == null) 355 | { 356 | return stringBuilder; 357 | } 358 | 359 | return stringBuilder. 360 | AppendLine("###### Returns"). 361 | AppendXmlDocumentation(returnsXmlElement, compilation, ref context); 362 | } 363 | 364 | public static StringBuilder AppendRemarks(this StringBuilder stringBuilder, XElement? rootXmlElement, Compilation compilation, ref GeneratorExecutionContext context) 365 | { 366 | if (rootXmlElement == null) 367 | { 368 | return stringBuilder; 369 | } 370 | 371 | XElement remarksXmlElement = rootXmlElement.Element("remarks"); 372 | 373 | if (remarksXmlElement == null) 374 | { 375 | return stringBuilder; 376 | } 377 | 378 | return stringBuilder. 379 | AppendLine("###### Remarks"). 380 | AppendXmlDocumentation(remarksXmlElement, compilation, ref context); 381 | } 382 | 383 | public static StringBuilder AppendExample(this StringBuilder stringBuilder, XElement? rootXmlElement, Compilation compilation, ref GeneratorExecutionContext context) 384 | { 385 | if (rootXmlElement == null) 386 | { 387 | return stringBuilder; 388 | } 389 | 390 | XElement exampleXmlElement = rootXmlElement.Element("example"); 391 | 392 | if (exampleXmlElement == null) 393 | { 394 | return stringBuilder; 395 | } 396 | 397 | return stringBuilder. 398 | AppendLine("###### Example"). 399 | AppendXmlDocumentation(exampleXmlElement, compilation, ref context); 400 | } 401 | 402 | public static StringBuilder AppendTypeParameters(this StringBuilder stringBuilder, IMethodSymbol methodSymbol, XElement? rootXmlElement, Compilation compilation, ref GeneratorExecutionContext context) 403 | { 404 | ImmutableArray typeParameterSymbols = methodSymbol.TypeParameters; 405 | if (typeParameterSymbols.Length == 0) 406 | { 407 | return stringBuilder; 408 | } 409 | 410 | IEnumerable? typeparamXmlElements = null; 411 | if (rootXmlElement != null) 412 | { 413 | typeparamXmlElements = rootXmlElement.Elements("typeparam"); 414 | } 415 | 416 | stringBuilder.AppendLine("###### Type Parameters"); 417 | foreach (ITypeParameterSymbol typeParameterSymbol in typeParameterSymbols) 418 | { 419 | string typeParameterName = typeParameterSymbol.Name; 420 | 421 | stringBuilder. 422 | Append('`'). 423 | Append(typeParameterName). 424 | AppendLine("` "); 425 | 426 | if (typeparamXmlElements != null) 427 | { 428 | XElement? paramXmlElement = typeparamXmlElements.FirstOrDefault(paramXmlElement => paramXmlElement.Attribute("name").Value == typeParameterName); 429 | 430 | if (paramXmlElement != null) 431 | { 432 | stringBuilder.AppendXmlDocumentation(paramXmlElement, compilation, ref context); 433 | } 434 | } 435 | 436 | stringBuilder.Append('\n'); // New paragraph for each parameter 437 | } 438 | 439 | stringBuilder.Length -= 1; // Remove last \n 440 | 441 | return stringBuilder; 442 | } 443 | 444 | public static StringBuilder AppendParameters(this StringBuilder stringBuilder, IMethodSymbol methodSymbol, XElement? rootXmlElement, Compilation compilation, ref GeneratorExecutionContext context) 445 | { 446 | ImmutableArray parameterSymbols = methodSymbol.Parameters; 447 | if (parameterSymbols.Length == 0) 448 | { 449 | return stringBuilder; 450 | } 451 | 452 | IEnumerable? paramXmlElements = null; 453 | if (rootXmlElement != null) 454 | { 455 | paramXmlElements = rootXmlElement.Elements("param"); 456 | } 457 | 458 | stringBuilder.AppendLine("###### Parameters"); 459 | foreach (IParameterSymbol parameterSymbol in parameterSymbols) 460 | { 461 | string parameterName = parameterSymbol.Name; 462 | 463 | stringBuilder. 464 | Append(parameterName). 465 | Append(" `"). 466 | Append(parameterSymbol.Type.ToDisplayString(DisplayFormats.TypeInlineDisplayFormat)). 467 | AppendLine("` "); 468 | 469 | if (paramXmlElements != null) 470 | { 471 | XElement? paramXmlElement = paramXmlElements.FirstOrDefault(paramXmlElement => paramXmlElement.Attribute("name").Value == parameterName); 472 | 473 | if (paramXmlElement != null) 474 | { 475 | stringBuilder.AppendXmlDocumentation(paramXmlElement, compilation, ref context); 476 | } 477 | } 478 | 479 | stringBuilder.Append('\n'); // New paragraph for each parameter 480 | } 481 | 482 | stringBuilder.Length -= 1; // Remove last \n 483 | 484 | return stringBuilder; 485 | } 486 | 487 | public static StringBuilder AppendSignature(this StringBuilder stringBuilder, ISymbol symbol) 488 | { 489 | return stringBuilder. 490 | AppendLine("```csharp"). 491 | AppendLine(symbol.ToDisplayString(DisplayFormats.SignatureDisplayFormat)). 492 | AppendLine("```"); 493 | } 494 | 495 | public static StringBuilder AppendXmlDocumentation(this StringBuilder stringBuilder, XNode xNode, Compilation compilation, ref GeneratorExecutionContext context) 496 | { 497 | if (xNode == null) 498 | { 499 | return stringBuilder; 500 | } 501 | 502 | stringBuilder.AppendXmlNodeContents(xNode, compilation, false, ref context); 503 | stringBuilder.TrimEnd(); 504 | stringBuilder.AppendLine(" "); 505 | 506 | return stringBuilder; 507 | } 508 | 509 | public static void AppendXmlNodeContents(this StringBuilder stringBuilder, XNode xNode, Compilation compilation, bool decodeHtml, ref GeneratorExecutionContext context) 510 | { 511 | if (xNode.NodeType == XmlNodeType.Text) 512 | { 513 | string nodeText = xNode.ToString(); 514 | 515 | if (decodeHtml) 516 | { 517 | nodeText = HttpUtility.HtmlDecode(nodeText); 518 | } 519 | 520 | stringBuilder.Append(nodeText); 521 | return; 522 | } 523 | 524 | if (xNode.NodeType != XmlNodeType.Element) 525 | { 526 | return; 527 | } 528 | 529 | var xElement = (XElement)xNode; 530 | XName elementName = xElement.Name; 531 | 532 | if (elementName == "see") 533 | { 534 | string? crefValue = xElement.Attribute("cref")?.Value; 535 | 536 | if (crefValue == null) 537 | { 538 | return; 539 | } 540 | 541 | ISymbol? seeSymbol; 542 | SymbolDisplayFormat? displayFormat; 543 | if (crefValue.StartsWith("T:")) 544 | { 545 | string typeFullyQualifiedName = crefValue.Substring(2); // Drop "T:" prefix 546 | seeSymbol = compilation.GetTypeByMetadataName(typeFullyQualifiedName); 547 | 548 | if (seeSymbol == null) 549 | { 550 | context.ReportDiagnostic(Diagnostic.Create(_missingTypeSymbol, null, typeFullyQualifiedName)); 551 | return; 552 | } 553 | 554 | displayFormat = DisplayFormats.TypeInlineDisplayFormat; 555 | } 556 | else if (crefValue.StartsWith("M:") || crefValue.StartsWith("P:") || crefValue.StartsWith("F:")) // Method, field or property 557 | { 558 | int indexOfLastSeparator = GetLastIndexOfFullyQualifiedTypeName(crefValue) + 1; 559 | string typeFullyQualifiedName = crefValue.Substring(2, indexOfLastSeparator - 2); 560 | INamedTypeSymbol? typeSymbol = compilation.GetTypeByMetadataName(typeFullyQualifiedName); 561 | 562 | if (typeSymbol == null) 563 | { 564 | context.ReportDiagnostic(Diagnostic.Create(_missingTypeSymbol, null, typeFullyQualifiedName)); 565 | return; 566 | } 567 | 568 | string methodFullyQualifiedName = crefValue.Substring(indexOfLastSeparator + 1); 569 | seeSymbol = typeSymbol.GetMembers().FirstOrDefault(memberSymbol => methodFullyQualifiedName.StartsWith(memberSymbol.Name)); // We can't know which overload, so just take first 570 | 571 | if (seeSymbol == null) 572 | { 573 | context.ReportDiagnostic(Diagnostic.Create(_missingMemberSymbol, null, methodFullyQualifiedName)); 574 | return; 575 | } 576 | 577 | displayFormat = DisplayFormats.MethodInlineDisplayFormat; 578 | } 579 | else 580 | { 581 | context.ReportDiagnostic(Diagnostic.Create(_crefValueWithUnexpectedPrefix, null, crefValue)); 582 | return; 583 | } 584 | 585 | stringBuilder. 586 | Append('`'). 587 | Append(seeSymbol.ToDisplayString(displayFormat)). 588 | Append('`'); 589 | 590 | return; 591 | } 592 | else if (elementName == "paramref") 593 | { 594 | string? name = xElement.Attribute("name")?.Value; 595 | 596 | if (name == null) 597 | { 598 | return; 599 | } 600 | 601 | stringBuilder. 602 | Append('`'). 603 | Append(name). 604 | Append('`'); 605 | 606 | return; 607 | } 608 | 609 | // Text in code blocks are displayed as is, so we have to pre-decode. Otherwise say angle brackets used for generics are rendered 610 | // as > and <. Note that this means we can't include HTML entities in code blocks cause they'll get decoded. 611 | // TODO consider adding attribute flags to work around the issue (decodeHTML flag) 612 | bool decodeHTML = false; 613 | if (elementName == "c") 614 | { 615 | stringBuilder.Append('`'); 616 | decodeHTML = true; 617 | } 618 | else if (elementName == "a") 619 | { 620 | stringBuilder.Append('['); 621 | } 622 | else if (elementName == "code") 623 | { 624 | string? language = xElement.Attribute("language")?.Value; 625 | 626 | if (language != null) 627 | { 628 | stringBuilder. 629 | Append("```"). 630 | AppendLine(language); 631 | } 632 | else 633 | { 634 | stringBuilder.AppendLine("```"); 635 | } 636 | decodeHTML = true; 637 | } 638 | 639 | // Iterate over child nodes 640 | foreach (XNode descendantXNode in xElement.Nodes()) 641 | { 642 | stringBuilder.AppendXmlNodeContents(descendantXNode, compilation, decodeHTML, ref context); 643 | } 644 | 645 | if (elementName == "c") 646 | { 647 | stringBuilder.Append('`'); 648 | } 649 | else if (elementName == "a") 650 | { 651 | stringBuilder. 652 | Append("]("). 653 | Append(xElement.Attribute("href")?.Value ?? string.Empty). 654 | Append(')'); 655 | } 656 | else if (elementName == "para") 657 | { 658 | stringBuilder.Append(" \n\n"); 659 | } 660 | else if (elementName == "code") 661 | { 662 | stringBuilder. 663 | AppendLine("\n```"); 664 | } 665 | } 666 | 667 | // https://stackoverflow.com/questions/24769701/trim-whitespace-from-the-end-of-a-stringbuilder-without-calling-tostring-trim 668 | public static StringBuilder TrimEnd(this StringBuilder stringBuilder) 669 | { 670 | if (stringBuilder.Length == 0) return stringBuilder; 671 | 672 | int i = stringBuilder.Length - 1; 673 | 674 | for (; i >= 0; i--) 675 | if (!char.IsWhiteSpace(stringBuilder[i])) 676 | break; 677 | 678 | if (i < stringBuilder.Length - 1) 679 | stringBuilder.Length = i + 1; 680 | 681 | return stringBuilder; 682 | } 683 | 684 | 685 | private static XElement? TryGetXmlDocumentationRootElement(ISymbol symbol) 686 | { 687 | string? xmlComment = symbol.GetDocumentationCommentXml(); // Note: this method indents our XML, can mess up text indentation 688 | 689 | if (string.IsNullOrWhiteSpace(xmlComment)) 690 | { 691 | return null; 692 | } 693 | 694 | xmlComment = Regex.Replace(xmlComment, "^ ", "", RegexOptions.Multiline); // Get rid of indents 695 | 696 | XElement rootElement; 697 | try 698 | { 699 | rootElement = XDocument.Parse(xmlComment).Root; 700 | } 701 | catch 702 | { 703 | // Do nothing if xml is malformed 704 | return null; 705 | } 706 | 707 | XElement inheritDocElement = rootElement.Element("inheritdoc"); 708 | 709 | if (inheritDocElement == null) 710 | { 711 | return rootElement; 712 | } 713 | 714 | ImmutableArray containingTypeInterfaceSymbols = symbol.ContainingType.AllInterfaces; 715 | foreach (INamedTypeSymbol containingTypeInterfaceSymbol in containingTypeInterfaceSymbols) 716 | { 717 | ImmutableArray memberSymbols = containingTypeInterfaceSymbol.GetMembers(); 718 | foreach (ISymbol memberSymbol in memberSymbols) 719 | { 720 | // TODO 721 | // - More stringent checks to determine whether symbol is the implementation of memberSymbol, 722 | // in particular, methods could be overloaded. Add checks when we have code to test it on. 723 | if (symbol.Kind != memberSymbol.Kind || 724 | symbol.Name != memberSymbol.Name) 725 | { 726 | continue; 727 | } 728 | 729 | return TryGetXmlDocumentationRootElement(memberSymbol); 730 | } 731 | } 732 | 733 | return null; 734 | } 735 | 736 | private static string ToHtmlEncodedDisplayString(ISymbol symbol, SymbolDisplayFormat displayFormat) 737 | { 738 | return HttpUtility.HtmlEncode(symbol.ToDisplayString(displayFormat)); 739 | } 740 | 741 | // We can't simply use the last . since fullyQualifiedMemberName could be something like M:Jering.Javascript.NodeJS.INodeJSService.TryInvokeFromCacheAsync``1(System.String,System.String,System.Object[],System.Threading.CancellationToken) 742 | private static int GetLastIndexOfFullyQualifiedTypeName(string fullyQualifiedMemberName) 743 | { 744 | int length = fullyQualifiedMemberName.Length; 745 | int lastNamespaceDotIndex = 0; 746 | for (int i = 0; i < length; i++) 747 | { 748 | char c = fullyQualifiedMemberName[i]; 749 | if (c == '.') 750 | { 751 | lastNamespaceDotIndex = i; 752 | } 753 | else if (c == '`' || c == '(') 754 | { 755 | break; 756 | } 757 | } 758 | 759 | return lastNamespaceDotIndex - 1; 760 | } 761 | } 762 | 763 | public static class DisplayFormats 764 | { 765 | public static readonly SymbolDisplayFormat TypeTitleDisplayFormat = new(genericsOptions: SymbolDisplayGenericsOptions.IncludeTypeParameters); 766 | public static readonly SymbolDisplayFormat ConstructorTitleDisplayFormat = new( 767 | genericsOptions: SymbolDisplayGenericsOptions.IncludeTypeParameters, 768 | memberOptions: SymbolDisplayMemberOptions.IncludeParameters, 769 | parameterOptions: SymbolDisplayParameterOptions.IncludeParamsRefOut | SymbolDisplayParameterOptions.IncludeType, 770 | miscellaneousOptions: SymbolDisplayMiscellaneousOptions.UseSpecialTypes | SymbolDisplayMiscellaneousOptions.EscapeKeywordIdentifiers | SymbolDisplayMiscellaneousOptions.UseErrorTypeSymbolName); 771 | public static readonly SymbolDisplayFormat ordinaryMethodTitleDisplayFormat = new(genericsOptions: SymbolDisplayGenericsOptions.IncludeTypeParameters, 772 | memberOptions: SymbolDisplayMemberOptions.IncludeExplicitInterface | SymbolDisplayMemberOptions.IncludeParameters | SymbolDisplayMemberOptions.IncludeContainingType, 773 | parameterOptions: SymbolDisplayParameterOptions.IncludeParamsRefOut | SymbolDisplayParameterOptions.IncludeType, 774 | miscellaneousOptions: SymbolDisplayMiscellaneousOptions.UseSpecialTypes | SymbolDisplayMiscellaneousOptions.EscapeKeywordIdentifiers | SymbolDisplayMiscellaneousOptions.UseErrorTypeSymbolName); 775 | public static readonly SymbolDisplayFormat propertyTitleDisplayFormat = ordinaryMethodTitleDisplayFormat; 776 | public static readonly SymbolDisplayFormat SignatureDisplayFormat = new(genericsOptions: SymbolDisplayGenericsOptions.IncludeTypeParameters, 777 | memberOptions: SymbolDisplayMemberOptions.IncludeAccessibility | SymbolDisplayMemberOptions.IncludeModifiers | SymbolDisplayMemberOptions.IncludeParameters | SymbolDisplayMemberOptions.IncludeType, 778 | parameterOptions: SymbolDisplayParameterOptions.IncludeParamsRefOut | SymbolDisplayParameterOptions.IncludeType | SymbolDisplayParameterOptions.IncludeName | SymbolDisplayParameterOptions.IncludeDefaultValue | SymbolDisplayParameterOptions.IncludeExtensionThis | SymbolDisplayParameterOptions.IncludeOptionalBrackets, 779 | propertyStyle: SymbolDisplayPropertyStyle.ShowReadWriteDescriptor, 780 | miscellaneousOptions: SymbolDisplayMiscellaneousOptions.UseSpecialTypes | SymbolDisplayMiscellaneousOptions.EscapeKeywordIdentifiers | SymbolDisplayMiscellaneousOptions.UseErrorTypeSymbolName | SymbolDisplayMiscellaneousOptions.IncludeNullableReferenceTypeModifier); 781 | public static readonly SymbolDisplayFormat TypeInlineDisplayFormat = new(genericsOptions: SymbolDisplayGenericsOptions.IncludeTypeParameters, 782 | miscellaneousOptions: SymbolDisplayMiscellaneousOptions.UseSpecialTypes | SymbolDisplayMiscellaneousOptions.EscapeKeywordIdentifiers | SymbolDisplayMiscellaneousOptions.UseErrorTypeSymbolName); 783 | public static readonly SymbolDisplayFormat MethodInlineDisplayFormat = new(genericsOptions: SymbolDisplayGenericsOptions.IncludeTypeParameters, 784 | memberOptions: SymbolDisplayMemberOptions.IncludeContainingType, 785 | miscellaneousOptions: SymbolDisplayMiscellaneousOptions.UseSpecialTypes | SymbolDisplayMiscellaneousOptions.EscapeKeywordIdentifiers | SymbolDisplayMiscellaneousOptions.UseErrorTypeSymbolName); 786 | } 787 | 788 | public class ApiDocumentationGeneratorSyntaxReceiver : ISyntaxReceiver 789 | { 790 | public Dictionary PublicClassDeclarations = new(); 791 | public Dictionary PublicInterfaceDeclarations = new(); 792 | 793 | public void OnVisitSyntaxNode(SyntaxNode syntaxNode) 794 | { 795 | if (syntaxNode is ClassDeclarationSyntax classDeclarationSyntax) 796 | { 797 | if (classDeclarationSyntax.Modifiers.Any(SyntaxKind.PublicKeyword)) 798 | { 799 | PublicClassDeclarations.Add(classDeclarationSyntax.Identifier.ValueText, classDeclarationSyntax); 800 | } 801 | 802 | return; 803 | } 804 | 805 | if (syntaxNode is InterfaceDeclarationSyntax interfaceDeclarationSyntax && 806 | interfaceDeclarationSyntax.Modifiers.Any(SyntaxKind.PublicKeyword)) 807 | { 808 | PublicInterfaceDeclarations.Add(interfaceDeclarationSyntax.Identifier.ValueText, interfaceDeclarationSyntax); 809 | } 810 | } 811 | } 812 | } 813 | -------------------------------------------------------------------------------- /generators/Jering.KeyValueStore.Generators/Jering.KeyValueStore.Generators.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | preview 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /generators/Jering.KeyValueStore.Generators/SourceGenerator.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.CodeAnalysis; 2 | using System; 3 | using System.IO; 4 | using System.Linq; 5 | 6 | namespace Jering.KeyValueStore.Generators 7 | { 8 | public abstract class SourceGenerator : ISourceGenerator where T : ISyntaxReceiver, new() 9 | { 10 | protected static readonly DiagnosticDescriptor _unexpectedException = new("G0006", 11 | "UnexpectedException", 12 | "UnexpectedException: {0}", 13 | "Code generation", 14 | DiagnosticSeverity.Error, 15 | true); 16 | 17 | private volatile string _logFilePath = string.Empty; 18 | 19 | protected string _projectDirectory; 20 | protected string _solutionDirectory; 21 | 22 | protected abstract void ExecuteCore(ref GeneratorExecutionContext context); 23 | 24 | protected virtual void InitializeCore() { } 25 | 26 | public void Execute(GeneratorExecutionContext context) 27 | { 28 | try 29 | { 30 | // Initialize directories and file paths. This can only be done when we receive the first syntax tree. 31 | // If these directories and paths change, VS has to be restarted, so this only needs to be done once. 32 | if (_logFilePath == string.Empty) 33 | { 34 | lock (this) 35 | { 36 | if (_logFilePath == string.Empty) 37 | { 38 | _projectDirectory = Path.GetDirectoryName(context.Compilation.SyntaxTrees.First(tree => tree.FilePath.EndsWith("AssemblyInfo.cs")).FilePath); 39 | _solutionDirectory = Path.Combine(_projectDirectory, "../.."); 40 | _logFilePath = Path.Combine(_projectDirectory, $"{GetType().Name}.txt"); 41 | } 42 | } 43 | } 44 | 45 | ExecuteCore(ref context); 46 | } 47 | catch (Exception exception) 48 | { 49 | context.ReportDiagnostic(Diagnostic.Create(_unexpectedException, null, exception.Message)); 50 | } 51 | } 52 | 53 | public void Initialize(GeneratorInitializationContext context) 54 | { 55 | // Register a factory that can create our custom syntax receiver 56 | context.RegisterForSyntaxNotifications(() => new T()); 57 | InitializeCore(); 58 | } 59 | 60 | #pragma warning disable IDE0060 // Unused when logging is off 61 | protected void LogLine(string message) 62 | #pragma warning restore IDE0060 63 | { 64 | //File.AppendAllText(_logFilePath, message + "\n"); 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /keypair.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JeringTech/KeyValueStore/59a898dbc0a314a609ed00807e4c0e09b8ee017e/keypair.snk -------------------------------------------------------------------------------- /nuget_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JeringTech/KeyValueStore/59a898dbc0a314a609ed00807e4c0e09b8ee017e/nuget_icon.png -------------------------------------------------------------------------------- /perf/KeyValueStore/Jering.KeyValueStore.Performance.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Exe 5 | net5.0 6 | true 7 | enable 8 | 9 | ../../keypair.snk 10 | true 11 | true 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /perf/KeyValueStore/LowMemoryUsageBenchmarks.cs: -------------------------------------------------------------------------------- 1 | using BenchmarkDotNet.Attributes; 2 | using FASTER.core; 3 | using MessagePack; 4 | using System.Collections.Concurrent; 5 | using System.Threading.Tasks; 6 | 7 | namespace Jering.KeyValueStore.Performance 8 | { 9 | // TODO 10 | // - Benchmark different types of values 11 | // - Primitive 12 | // - Fixed length struct 13 | // - Variable length struct 14 | // - Class 15 | // - Benchmark different storage ratios 16 | // - Half in memory, half on disk 17 | // - All in memory 18 | // - Benchmark delete operation 19 | [MemoryDiagnoser] 20 | public class LowMemoryUsageBenchmarks 21 | { 22 | private MixedStorageKVStore? _mixedStorageKVStore; 23 | private MixedStorageKVStoreOptions? _mixedStorageKVStoreOptions; 24 | private const int NUM_INSERT_OPERATIONS = 350_000; 25 | private const int NUM_READ_OPERATIONS = 75_000; 26 | private readonly ConcurrentQueue> _readTasks = new(); 27 | private readonly ConcurrentQueue _upsertTasks = new(); 28 | private readonly DummyClass _dummyClassInstance = new() 29 | { 30 | // Populate with dummy values 31 | DummyString = "dummyString", 32 | DummyStringArray = new[] { "dummyString1", "dummyString2", "dummyString3", "dummyString4", "dummyString5" }, 33 | DummyInt = 10, 34 | DummyIntArray = new[] { 10, 100, 1000, 10000, 100000, 1000000, 10000000 } 35 | }; 36 | 37 | // Concurrent inserts without compression 38 | [GlobalSetup(Target = nameof(Inserts_WithoutCompression))] 39 | public void Inserts_WithoutCompression_GlobalSetup() 40 | { 41 | _mixedStorageKVStoreOptions = new() 42 | { 43 | PageSizeBits = 12, // 4 KB 44 | MemorySizeBits = 13, // 2 pages 45 | TimeBetweenLogCompactionsMS = -1, // Disable log compactions 46 | MessagePackSerializerOptions = MessagePackSerializerOptions.Standard 47 | }; 48 | } 49 | 50 | [IterationSetup(Target = nameof(Inserts_WithoutCompression))] 51 | public void Inserts_WithoutCompression_IterationSetup() 52 | { 53 | _mixedStorageKVStore = new MixedStorageKVStore(_mixedStorageKVStoreOptions); 54 | _upsertTasks.Clear(); 55 | } 56 | 57 | [Benchmark] 58 | public async Task Inserts_WithoutCompression() 59 | { 60 | Parallel.For(0, NUM_INSERT_OPERATIONS, key => _upsertTasks.Enqueue(_mixedStorageKVStore!.UpsertAsync(key, _dummyClassInstance))); 61 | await Task.WhenAll(_upsertTasks).ConfigureAwait(false); 62 | } 63 | 64 | [IterationCleanup(Target = nameof(Inserts_WithoutCompression))] 65 | public void Inserts_WithoutCompression_IterationCleanup() 66 | { 67 | _mixedStorageKVStore?.Dispose(); 68 | } 69 | 70 | // Concurrent reads without compression 71 | [GlobalSetup(Target = nameof(Reads_WithoutCompression))] 72 | public async Task Reads_WithoutCompression_GlobalSetup() 73 | { 74 | _mixedStorageKVStoreOptions = new() 75 | { 76 | PageSizeBits = 12, // 4 KB 77 | MemorySizeBits = 13, // 2 pages 78 | TimeBetweenLogCompactionsMS = -1, // Disable log compactions 79 | MessagePackSerializerOptions = MessagePackSerializerOptions.Standard 80 | }; 81 | _mixedStorageKVStore = new MixedStorageKVStore(_mixedStorageKVStoreOptions); 82 | Parallel.For(0, NUM_READ_OPERATIONS, key => _upsertTasks.Enqueue(_mixedStorageKVStore.UpsertAsync(key, _dummyClassInstance))); 83 | await Task.WhenAll(_upsertTasks).ConfigureAwait(false); 84 | } 85 | 86 | [IterationSetup(Target = nameof(Reads_WithoutCompression))] 87 | public void Reads_WithoutCompression_IterationSetup() 88 | { 89 | _readTasks.Clear(); 90 | } 91 | 92 | [Benchmark] 93 | public async Task Reads_WithoutCompression() 94 | { 95 | #pragma warning disable CA2012 // Use ValueTasks correctly 96 | Parallel.For(0, NUM_READ_OPERATIONS, key => _readTasks.Enqueue(_mixedStorageKVStore!.ReadAsync(key))); 97 | #pragma warning restore CA2012 // Use ValueTasks correctly 98 | foreach (ValueTask<(Status, DummyClass?)> task in _readTasks) 99 | { 100 | if (task.IsCompleted) 101 | { 102 | continue; 103 | } 104 | await task.ConfigureAwait(false); 105 | } 106 | } 107 | 108 | [MessagePackObject] 109 | public class DummyClass 110 | { 111 | [Key(0)] 112 | public string? DummyString { get; set; } 113 | 114 | [Key(1)] 115 | public string[]? DummyStringArray { get; set; } 116 | 117 | [Key(2)] 118 | public int DummyInt { get; set; } 119 | 120 | [Key(3)] 121 | public int[]? DummyIntArray { get; set; } 122 | } 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /perf/KeyValueStore/Program.cs: -------------------------------------------------------------------------------- 1 | using BenchmarkDotNet.Running; 2 | 3 | namespace Jering.KeyValueStore.Performance 4 | { 5 | class Program 6 | { 7 | static void Main() 8 | { 9 | BenchmarkRunner.Run(); 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/KeyValueStore/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.CompilerServices; 2 | 3 | [assembly: InternalsVisibleTo("Jering.KeyValueStore.Performance,PublicKey=0024000004800000940000000602000000240000525341310004000001000100ed30e6f1d3e00f6800aec83aee2311af5c6a2a9046f57fcc1618a4e38bcb2e9464ec505e4fe71061fb261b27287fcf1a3582472e8dbb6a145eeee0a0685ea5b4ef147de7c70cc823af6ad573ab12f6c292f43c6980419859d3a365befe2b1f7a043dc95025c51cd428bc86fe43a3fcfcc71cdf2252a08d684659289eb0de37bc")] 4 | [assembly: InternalsVisibleTo("Jering.KeyValueStore.Tests,PublicKey=0024000004800000940000000602000000240000525341310004000001000100ed30e6f1d3e00f6800aec83aee2311af5c6a2a9046f57fcc1618a4e38bcb2e9464ec505e4fe71061fb261b27287fcf1a3582472e8dbb6a145eeee0a0685ea5b4ef147de7c70cc823af6ad573ab12f6c292f43c6980419859d3a365befe2b1f7a043dc95025c51cd428bc86fe43a3fcfcc71cdf2252a08d684659289eb0de37bc")] 5 | -------------------------------------------------------------------------------- /src/KeyValueStore/IMixedStorageKVStore.cs: -------------------------------------------------------------------------------- 1 | using FASTER.core; 2 | using System; 3 | using System.Threading.Tasks; 4 | 5 | namespace Jering.KeyValueStore 6 | { 7 | /// 8 | /// An abstraction for a key-value store that spans memory and disk. 9 | /// 10 | /// The type of the key-value store's key. 11 | /// The type of the key-value store's values. 12 | public interface IMixedStorageKVStore 13 | { 14 | /// 15 | /// Gets the underlying instance. 16 | /// 17 | FasterKV FasterKV { get; } 18 | 19 | /// 20 | /// Updates or inserts a record asynchronously. 21 | /// 22 | /// The record's key. 23 | /// The record's new value. 24 | /// The task representing the asynchronous operation. 25 | /// Thrown if the instance or a dependency is disposed. 26 | /// This method is thread-safe. 27 | Task UpsertAsync(TKey key, TValue obj); 28 | 29 | /// 30 | /// Deletes a record asynchronously. 31 | /// 32 | /// The record's key. 33 | /// The task representing the asynchronous operation. 34 | /// Thrown if the instance or a dependency is disposed. 35 | /// This method is thread-safe. 36 | ValueTask DeleteAsync(TKey key); 37 | 38 | /// 39 | /// Reads a record asynchronously. 40 | /// 41 | /// The record's key. 42 | /// The task representing the asynchronous operation. 43 | /// Thrown if the instance or a dependency is disposed. 44 | /// This method is thread-safe. 45 | ValueTask<(Status, TValue?)> ReadAsync(TKey key); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/KeyValueStore/Jering.KeyValueStore.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.1 5 | Jering.KeyValueStore 6 | JeremyTCD 7 | Embeddable Mixed-Storage Key-Value Store for C# 8 | Jering.KeyValueStore enables you to store key-value data across memory and disk. This library is a Microsoft.Faster wrapper. 9 | © 2021 Jering. All rights reserved. 10 | https://github.com/JeringTech/KeyValueStore 11 | https://github.com/JeringTech/KeyValueStore 12 | $(RepositoryUrl)/blob/master/License.md 13 | $(RepositoryUrl)/blob/master/Changelog.md 14 | key-value store concurrent memory disk cache faster 15 | true 16 | true 17 | $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb 18 | git 19 | enable 20 | preview 21 | true 22 | https://raw.githubusercontent.com/JeringTech/KeyValueStore/master/nuget_icon.png 23 | true 24 | true 25 | ../../keypair.snk 26 | true 27 | true 28 | true 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 41 | 42 | 43 | 44 | 45 | True 46 | True 47 | Strings.resx 48 | 49 | 50 | 51 | 52 | 53 | ResXFileCodeGenerator 54 | Strings.Designer.cs 55 | 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /src/KeyValueStore/MixedStorageKVStore.cs: -------------------------------------------------------------------------------- 1 | using FASTER.core; 2 | using MessagePack; 3 | using Microsoft.Extensions.Logging; 4 | using System; 5 | using System.Buffers; 6 | using System.Collections.Concurrent; 7 | using System.IO; 8 | using System.Threading; 9 | using System.Threading.Tasks; 10 | 11 | namespace Jering.KeyValueStore 12 | { 13 | // TODO 14 | // - Fast paths for fixed size keys and values. We need an equivalent of FASTER.core.Utility.IsBlittableType to check 15 | // if a key/value type is blittable. If it is, we can either use a fast path or create a FasterKV instance with blittable key/value type. 16 | /// 17 | /// The default implementation of . 18 | /// 19 | public class MixedStorageKVStore : IMixedStorageKVStore, IDisposable 20 | { 21 | private static readonly MixedStorageKVStoreOptions _defaultMixedStorageKVStoreOptions = new(); 22 | 23 | // Faster store 24 | private readonly FasterKV _fasterKV; 25 | private readonly SpanByteFunctions _spanByteFunctions = new(); 26 | private readonly FasterKV.ClientSessionBuilder _clientSessionBuilder; 27 | 28 | // Sessions 29 | private readonly ConcurrentQueue>> _sessionPool = new(); 30 | 31 | // Log 32 | private readonly LogAccessor _logAccessor; 33 | private readonly CancellationTokenSource? _logCompactionCancellationTokenSource; 34 | private readonly int _timeBetweenLogCompactionsMS; 35 | private long _logCompactionThresholdBytes = 0; 36 | private byte _numConsecutiveLogCompactions = 0; 37 | private const byte NUM_CONSECUTIVE_COMPACTIONS_BEFORE_THRESHOLD_INCREASE = 5; 38 | 39 | // Serialization 40 | private readonly MessagePackSerializerOptions _messagePackSerializerOptions; 41 | private readonly ConcurrentQueue> _arrayBufferWriterPool = new(); 42 | 43 | // Logging 44 | private readonly ILogger>? _logger; 45 | private readonly bool _logTrace; 46 | private readonly bool _logWarning; 47 | 48 | // Disposal 49 | private bool _disposed; 50 | private readonly IDevice? _logDevice; 51 | 52 | /// 53 | public FasterKV FasterKV => _fasterKV; 54 | 55 | /// 56 | /// Creates a . 57 | /// 58 | /// The options for the . 59 | /// The logger for log compaction events. 60 | /// The underlying for the . 61 | /// This parameter allows you to use a manually configured Faster instance. 62 | /// 63 | public MixedStorageKVStore(MixedStorageKVStoreOptions? mixedStorageKVStoreOptions = null, 64 | ILogger>? logger = null, 65 | FasterKV? fasterKVStore = null) 66 | { 67 | mixedStorageKVStoreOptions ??= _defaultMixedStorageKVStoreOptions; 68 | 69 | // Store 70 | if (fasterKVStore == null) 71 | { 72 | LogSettings logSettings = CreateLogSettings(mixedStorageKVStoreOptions); 73 | _logDevice = logSettings.LogDevice; // _fasterKVStore.dispose doesn't dispose the underlying log device, so hold a reference for immediate manual disposal 74 | _fasterKV = new(mixedStorageKVStoreOptions.IndexNumBuckets, logSettings); 75 | } 76 | else 77 | { 78 | _fasterKV = fasterKVStore; 79 | } 80 | 81 | // Session 82 | _clientSessionBuilder = _fasterKV.For(_spanByteFunctions); 83 | 84 | // Log 85 | _logAccessor = _fasterKV.Log; 86 | _timeBetweenLogCompactionsMS = mixedStorageKVStoreOptions.TimeBetweenLogCompactionsMS; 87 | if (_timeBetweenLogCompactionsMS > -1) 88 | { 89 | _logCompactionThresholdBytes = mixedStorageKVStoreOptions.InitialLogCompactionThresholdBytes; 90 | _logCompactionThresholdBytes = _logCompactionThresholdBytes <= 0 ? (long)Math.Pow(2, mixedStorageKVStoreOptions.MemorySizeBits) * 2 : _logCompactionThresholdBytes; 91 | _logCompactionCancellationTokenSource = new CancellationTokenSource(); 92 | Task.Run(LogCompactionLoop); 93 | } 94 | 95 | // Serialization 96 | _messagePackSerializerOptions = mixedStorageKVStoreOptions.MessagePackSerializerOptions; 97 | 98 | // Logging 99 | _logger = logger; 100 | _logTrace = _logger?.IsEnabled(LogLevel.Trace) ?? false; 101 | _logWarning = _logger?.IsEnabled(LogLevel.Warning) ?? false; 102 | } 103 | 104 | /// 105 | public async Task UpsertAsync(TKey key, TValue obj) 106 | { 107 | if (_disposed) 108 | { 109 | throw new ObjectDisposedException(nameof(MixedStorageKVStore)); 110 | } 111 | 112 | // Session 113 | ClientSession> session = GetPooledSession(); 114 | 115 | // Serialize 116 | ArrayBufferWriter arrayBufferWriter = GetPooledArrayBufferWriter(); // If we use a ThreadLocal ArrayBufferWriter, we might call Clear on the wrong instance if the continuation is on a different thread 117 | MessagePackSerializer.Serialize(arrayBufferWriter, key, _messagePackSerializerOptions); 118 | int keyLength = arrayBufferWriter.WrittenCount; 119 | MessagePackSerializer.Serialize(arrayBufferWriter, obj, _messagePackSerializerOptions); 120 | int valueLength = arrayBufferWriter.WrittenCount - keyLength; 121 | 122 | // Upsert 123 | FasterKV.UpsertAsyncResult result; 124 | using (MemoryHandle memoryHandle = arrayBufferWriter.WrittenMemory.Pin()) 125 | { 126 | SpanByte keySpanByte; 127 | SpanByte objSpanByte; 128 | unsafe 129 | { 130 | byte* pointer = (byte*)memoryHandle.Pointer; 131 | keySpanByte = SpanByte.FromPointer(pointer, keyLength); 132 | objSpanByte = SpanByte.FromPointer(pointer + keyLength, valueLength); 133 | } 134 | 135 | result = await session.UpsertAsync(ref keySpanByte, ref objSpanByte).ConfigureAwait(false); 136 | } 137 | 138 | // No longer needed 139 | arrayBufferWriter.Clear(); 140 | _arrayBufferWriterPool.Enqueue(arrayBufferWriter); 141 | 142 | // Wait for completion 143 | while (result.Status == Status.PENDING) 144 | { 145 | result = await result.CompleteAsync().ConfigureAwait(false); 146 | } 147 | 148 | // Clean up 149 | _sessionPool.Enqueue(session); 150 | } 151 | 152 | /// 153 | public async ValueTask DeleteAsync(TKey key) 154 | { 155 | if (_disposed) 156 | { 157 | throw new ObjectDisposedException(nameof(MixedStorageKVStore)); 158 | } 159 | 160 | // Session 161 | ClientSession> session = GetPooledSession(); 162 | 163 | // Serialize 164 | ArrayBufferWriter arrayBufferWriter = GetPooledArrayBufferWriter(); // If we use a ThreadLocal ArrayBufferWriter, we might call Clear on the wrong instance if the continuation is on a different thread 165 | MessagePackSerializer.Serialize(arrayBufferWriter, key, _messagePackSerializerOptions); 166 | int keyLength = arrayBufferWriter.WrittenCount; 167 | 168 | // Delete 169 | FasterKV.DeleteAsyncResult result; 170 | using (MemoryHandle memoryHandle = arrayBufferWriter.WrittenMemory.Pin()) 171 | { 172 | SpanByte keySpanByte; 173 | unsafe 174 | { 175 | keySpanByte = SpanByte.FromPointer((byte*)memoryHandle.Pointer, keyLength); 176 | } 177 | 178 | result = await session.DeleteAsync(ref keySpanByte).ConfigureAwait(false); 179 | } 180 | 181 | // No longer needed 182 | arrayBufferWriter.Clear(); 183 | _arrayBufferWriterPool.Enqueue(arrayBufferWriter); 184 | 185 | // Wait for completion 186 | while (result.Status == Status.PENDING) 187 | { 188 | result = await result.CompleteAsync().ConfigureAwait(false); 189 | } 190 | 191 | // Clean up 192 | _sessionPool.Enqueue(session); 193 | 194 | return result.Status; 195 | } 196 | 197 | /// 198 | public async ValueTask<(Status, TValue?)> ReadAsync(TKey key) 199 | { 200 | if (_disposed) 201 | { 202 | throw new ObjectDisposedException(nameof(MixedStorageKVStore)); 203 | } 204 | 205 | // Session 206 | ClientSession> session = GetPooledSession(); 207 | 208 | // Serialize 209 | ArrayBufferWriter arrayBufferWriter = GetPooledArrayBufferWriter(); // If we use a ThreadLocal ArrayBufferWriter, we might call Clear on the wrong instance if the continuation is on a different thread 210 | MessagePackSerializer.Serialize(arrayBufferWriter, key, _messagePackSerializerOptions); 211 | int keyLength = arrayBufferWriter.WrittenCount; 212 | 213 | // Read 214 | FasterKV.ReadAsyncResult result; 215 | using (MemoryHandle memoryHandle = arrayBufferWriter.WrittenMemory.Pin()) 216 | { 217 | SpanByte keySpanByte; 218 | unsafe 219 | { 220 | keySpanByte = SpanByte.FromPointer((byte*)memoryHandle.Pointer, keyLength); 221 | } 222 | 223 | result = await session.ReadAsync(ref keySpanByte).ConfigureAwait(false); 224 | } 225 | 226 | // No longer needed 227 | arrayBufferWriter.Clear(); 228 | _arrayBufferWriterPool.Enqueue(arrayBufferWriter); 229 | 230 | // Wait for completion 231 | (Status status, SpanByteAndMemory spanByteAndMemory) = result.Complete(); 232 | 233 | // Clean up 234 | _sessionPool.Enqueue(session); 235 | 236 | // Deserialize 237 | using IMemoryOwner memoryOwner = spanByteAndMemory.Memory; 238 | 239 | return (status, status != Status.OK ? default : MessagePackSerializer.Deserialize(memoryOwner.Memory, _messagePackSerializerOptions)); 240 | } 241 | 242 | private async Task LogCompactionLoop() 243 | { 244 | CancellationToken cancellationToken = _logCompactionCancellationTokenSource!.Token; // If compaction loop is running, cts is not null (see constructor) 245 | 246 | while (!_disposed && !_logCompactionCancellationTokenSource.IsCancellationRequested) 247 | { 248 | try 249 | { 250 | await Task.Delay(_timeBetweenLogCompactionsMS, cancellationToken).ConfigureAwait(false); 251 | 252 | // (oldest entries here) BeginAddress <= HeadAddress (where the in-memory region begins) <= SafeReadOnlyAddress (entries between here and tail updated in-place) < TailAddress (entries added here) 253 | long safeReadOnlyRegionByteSize = _logAccessor.SafeReadOnlyAddress - _logAccessor.BeginAddress; 254 | if (safeReadOnlyRegionByteSize < _logCompactionThresholdBytes) 255 | { 256 | if (_logTrace) 257 | { 258 | _logger.LogTrace(string.Format(Strings.LogTrace_SkippingLogCompaction, safeReadOnlyRegionByteSize, _logCompactionThresholdBytes)); 259 | } 260 | _numConsecutiveLogCompactions = 0; 261 | continue; 262 | } 263 | 264 | // Compact 265 | long compactUntilAddress = (long)(_logAccessor.BeginAddress + 0.2 * (_logAccessor.SafeReadOnlyAddress - _logAccessor.BeginAddress)); 266 | ClientSession> session = GetPooledSession(); 267 | session.Compact(compactUntilAddress, true); 268 | _sessionPool.Enqueue(session); 269 | _numConsecutiveLogCompactions++; 270 | 271 | if (_logTrace) 272 | { 273 | _logger.LogTrace(string.Format(Strings.LogTrace_LogCompacted, 274 | safeReadOnlyRegionByteSize, 275 | _logAccessor.SafeReadOnlyAddress - _logAccessor.BeginAddress, 276 | _numConsecutiveLogCompactions)); 277 | } 278 | 279 | // Update threshold 280 | // Note that we can't simply check whether safeReadOnlyRegionByteSize has changed - when the log is compact, safeReadOnlyRegionByteSize may change (increase or decrease) 281 | // by small amounts every compaction. This is because records are shifted from head to tail, i.e. the set of records in the safe-readonly region changes. 282 | if (_numConsecutiveLogCompactions >= NUM_CONSECUTIVE_COMPACTIONS_BEFORE_THRESHOLD_INCREASE) 283 | { 284 | _logCompactionThresholdBytes *= 2; // Max long is ~9200 petabytes, overflow is not an issue for now 285 | if (_logTrace) 286 | { 287 | _logger.LogTrace(string.Format(Strings.LogTrace_LogCompactionThresholdIncreased, _logCompactionThresholdBytes / 2, _logCompactionThresholdBytes)); 288 | } 289 | _numConsecutiveLogCompactions = 0; 290 | } 291 | } 292 | catch (OperationCanceledException) 293 | { 294 | return; 295 | } 296 | catch (Exception exception) 297 | { 298 | // Compaction failed or we disposed of the instance. If we disposed of the instance, the next while loop boolean expression evaluation returns false. 299 | // If compaction failed, we try again after a delay. 300 | if (_logWarning) 301 | { 302 | _logger.LogWarning(string.Format(Strings.LogWarning_Exception, exception.Message)); 303 | } 304 | } 305 | } 306 | } 307 | 308 | private ClientSession> GetPooledSession() 309 | { 310 | if (_sessionPool.TryDequeue(out ClientSession>? result)) 311 | { 312 | return result; 313 | } 314 | 315 | return CreateSession(); 316 | } 317 | 318 | private ClientSession> CreateSession() 319 | { 320 | return _clientSessionBuilder.NewSession>(); 321 | } 322 | 323 | private ArrayBufferWriter GetPooledArrayBufferWriter() 324 | { 325 | if (_arrayBufferWriterPool.TryDequeue(out ArrayBufferWriter? result)) 326 | { 327 | return result; 328 | } 329 | 330 | return new(); 331 | } 332 | 333 | private LogSettings CreateLogSettings(MixedStorageKVStoreOptions options) 334 | { 335 | // Log settings 336 | string logDirectory = string.IsNullOrWhiteSpace(options.LogDirectory) ? Path.Combine(Path.GetTempPath(), "FasterLogs") : options.LogDirectory; 337 | string logFileName = string.IsNullOrWhiteSpace(options.LogFileNamePrefix) ? Guid.NewGuid().ToString() : options.LogFileNamePrefix; 338 | 339 | var logSettings = new LogSettings 340 | { 341 | LogDevice = Devices.CreateLogDevice(Path.Combine(logDirectory, $"{logFileName}.log"), 342 | preallocateFile: true, 343 | deleteOnClose: options.DeleteLogOnClose), 344 | PageSizeBits = options.PageSizeBits, 345 | MemorySizeBits = options.MemorySizeBits, 346 | SegmentSizeBits = options.SegmentSizeBits 347 | }; 348 | 349 | return logSettings; 350 | } 351 | 352 | /// 353 | /// Disposes this instance. 354 | /// 355 | public void Dispose() 356 | { 357 | Dispose(true); 358 | GC.SuppressFinalize(this); 359 | } 360 | 361 | /// 362 | /// Disposes the instance. 363 | /// 364 | /// True if the object is disposing or false if it is finalizing. 365 | protected virtual void Dispose(bool disposing) 366 | { 367 | if (!_disposed) 368 | { 369 | if (disposing) 370 | { 371 | _logCompactionCancellationTokenSource?.Cancel(); 372 | _logCompactionCancellationTokenSource?.Dispose(); // Should not be necessary to call Dispose if we call Cancel, but no harm 373 | 374 | foreach (ClientSession> session in _sessionPool) 375 | { 376 | session.Dispose(); 377 | } 378 | 379 | _fasterKV.Dispose(); // Only safe to call after disposing all sessions 380 | _logDevice?.Dispose(); 381 | } 382 | 383 | _disposed = true; 384 | } 385 | } 386 | } 387 | } 388 | -------------------------------------------------------------------------------- /src/KeyValueStore/MixedStorageKVStoreOptions.cs: -------------------------------------------------------------------------------- 1 | using FASTER.core; 2 | using MessagePack; 3 | using System; 4 | using System.IO; 5 | 6 | namespace Jering.KeyValueStore 7 | { 8 | /// Options for a . 9 | public class MixedStorageKVStoreOptions 10 | { 11 | /// The number of buckets in Faster's index. 12 | /// 13 | /// Each bucket is 64 bits. 14 | /// This value is ignored if a instance is supplied to the constructor. 15 | /// Defaults to 1048576 (64 MB index). 16 | /// 17 | public long IndexNumBuckets { get; set; } = 1L << 20; 18 | 19 | /// The size of a page in Faster's log. 20 | /// 21 | /// A page is a contiguous block of in-memory or on-disk storage. 22 | /// This value is ignored if a instance is supplied to the constructor. 23 | /// Defaults to 25 (2^25 = 33.5 MB). 24 | /// 25 | public int PageSizeBits { get; set; } = 25; 26 | 27 | /// The size of the in-memory region of Faster's log. 28 | /// 29 | /// If the log outgrows this region, overflow is moved to its on-disk region. 30 | /// This value is ignored if a instance is supplied to the constructor. 31 | /// Defaults to 26 (2^26 = 67 MB). 32 | /// 33 | public int MemorySizeBits { get; set; } = 26; // 67 MB 34 | 35 | /// The size of a segment of the on-disk region of Faster's log. 36 | /// 37 | /// What is a segment? Records on disk are split into groups called segments. Each segment corresponds to a file. 38 | /// For performance reasons, segments are "pre-allocated". This means they are not created empty and left to grow gradually, instead they are created at the size specified by this value and populated gradually. 39 | /// This value is ignored if a instance is supplied to the constructor. 40 | /// Defaults to 28 (268 MB). 41 | /// 42 | public int SegmentSizeBits { get; set; } = 28; 43 | 44 | /// The directory containing the on-disk region of Faster's log. 45 | /// 46 | /// If this value is null or an empty string, log files are placed in "<temporary path>/FasterLogs" where 47 | /// "<temporary path>" is the value returned by . 48 | /// Note that nothing is written to disk while your log fits in-memory. 49 | /// This value is ignored if a instance is supplied to the constructor. 50 | /// Defaults to null. 51 | /// 52 | public string? LogDirectory { get; set; } = null; 53 | 54 | /// The Faster log filename prefix. 55 | /// 56 | /// The on-disk region of the log is stored across multiple files. Each file is referred to as a segment. 57 | /// Each segment has file name "<log file name prefix>.log.<segment number>". 58 | /// 59 | /// If this value is null or an empty string, a random is used as the prefix. 60 | /// This value is ignored if a instance is supplied to the constructor. 61 | /// Defaults to null. 62 | /// 63 | public string? LogFileNamePrefix { get; set; } = null; 64 | 65 | /// The time between Faster log compaction attempts. 66 | /// 67 | /// If this value is negative, log compaction is disabled. 68 | /// Defaults to 60000. 69 | /// 70 | public int TimeBetweenLogCompactionsMS { get; set; } = 60_000; 71 | 72 | // TODO what if a FasterKV instance is supplied and MemorySizeBits is 0 or negative? 73 | /// The initial log compaction threshold. 74 | /// 75 | /// Initially, log compactions only run when the Faster log's safe-readonly region's size is larger than or equal to this value. 76 | /// If log compaction runs 5 times in a row, this value is doubled. Why? Consider the situation where the safe-readonly region is already 77 | /// compact, but still larger than the threshold. Not increasing the threshold would result in redundant compaction runs. 78 | /// If this value is less than or equal to 0, the initial log compaction threshold is 2 * memory size in bytes (). 79 | /// Defaults to 0. 80 | /// 81 | public long InitialLogCompactionThresholdBytes { get; internal set; } = 0; 82 | 83 | /// The value specifying whether log files are deleted when the is disposed or finalized (at which points underlying log files are closed). 84 | /// 85 | /// This value is ignored if a instance is supplied to the constructor. 86 | /// Defaults to true. 87 | /// 88 | public bool DeleteLogOnClose { get; set; } = true; 89 | 90 | /// The options for serializing data using MessagePack C#. 91 | /// 92 | /// MessagePack C# is a performant binary serialization library. Refer to MessagePack C# documentation 93 | /// for details. 94 | /// Defaults to with compression using . 95 | /// 96 | public MessagePackSerializerOptions MessagePackSerializerOptions { get; set; } = MessagePackSerializerOptions. 97 | Standard. 98 | WithCompression(MessagePackCompression.Lz4BlockArray); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/KeyValueStore/Strings.Designer.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // This code was generated by a tool. 4 | // Runtime Version:4.0.30319.42000 5 | // 6 | // Changes to this file may cause incorrect behavior and will be lost if 7 | // the code is regenerated. 8 | // 9 | //------------------------------------------------------------------------------ 10 | 11 | namespace Jering.KeyValueStore { 12 | using System; 13 | 14 | 15 | /// 16 | /// A strongly-typed resource class, for looking up localized strings, etc. 17 | /// 18 | // This class was auto-generated by the StronglyTypedResourceBuilder 19 | // class via a tool like ResGen or Visual Studio. 20 | // To add or remove a member, edit your .ResX file then rerun ResGen 21 | // with the /str option, or rebuild your VS project. 22 | [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "16.0.0.0")] 23 | [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] 24 | [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] 25 | internal class Strings { 26 | 27 | private static global::System.Resources.ResourceManager resourceMan; 28 | 29 | private static global::System.Globalization.CultureInfo resourceCulture; 30 | 31 | [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] 32 | internal Strings() { 33 | } 34 | 35 | /// 36 | /// Returns the cached ResourceManager instance used by this class. 37 | /// 38 | [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] 39 | internal static global::System.Resources.ResourceManager ResourceManager { 40 | get { 41 | if (object.ReferenceEquals(resourceMan, null)) { 42 | global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("Jering.KeyValueStore.Strings", typeof(Strings).Assembly); 43 | resourceMan = temp; 44 | } 45 | return resourceMan; 46 | } 47 | } 48 | 49 | /// 50 | /// Overrides the current thread's CurrentUICulture property for all 51 | /// resource lookups using this strongly typed resource class. 52 | /// 53 | [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] 54 | internal static global::System.Globalization.CultureInfo Culture { 55 | get { 56 | return resourceCulture; 57 | } 58 | set { 59 | resourceCulture = value; 60 | } 61 | } 62 | 63 | /// 64 | /// Looks up a localized string similar to LogCompacted. safe-readonly region byte size before compaction = {0}, after compaction = {1}. Num consecutive compactions = {2}.. 65 | /// 66 | internal static string LogTrace_LogCompacted { 67 | get { 68 | return ResourceManager.GetString("LogTrace_LogCompacted", resourceCulture); 69 | } 70 | } 71 | 72 | /// 73 | /// Looks up a localized string similar to Log compaction threshold increased. Previous = {0}, current = {1}.. 74 | /// 75 | internal static string LogTrace_LogCompactionThresholdIncreased { 76 | get { 77 | return ResourceManager.GetString("LogTrace_LogCompactionThresholdIncreased", resourceCulture); 78 | } 79 | } 80 | 81 | /// 82 | /// Looks up a localized string similar to Skipping log compaction. Current safe-readonly region byte size = {0}, log compaction threshold = {1}.. 83 | /// 84 | internal static string LogTrace_SkippingLogCompaction { 85 | get { 86 | return ResourceManager.GetString("LogTrace_SkippingLogCompaction", resourceCulture); 87 | } 88 | } 89 | 90 | /// 91 | /// Looks up a localized string similar to Exception thrown in log compaction loop: 92 | ///{0}. 93 | /// 94 | internal static string LogWarning_Exception { 95 | get { 96 | return ResourceManager.GetString("LogWarning_Exception", resourceCulture); 97 | } 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/KeyValueStore/Strings.resx: -------------------------------------------------------------------------------- 1 |  2 | 3 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | text/microsoft-resx 110 | 111 | 112 | 2.0 113 | 114 | 115 | System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 116 | 117 | 118 | System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 119 | 120 | 121 | LogCompacted. safe-readonly region byte size before compaction = {0}, after compaction = {1}. Num consecutive compactions = {2}. 122 | 123 | 124 | Log compaction threshold increased. Previous = {0}, current = {1}. 125 | 126 | 127 | Skipping log compaction. Current safe-readonly region byte size = {0}, log compaction threshold = {1}. 128 | 129 | 130 | Exception thrown in log compaction loop: 131 | {0} 132 | 133 | -------------------------------------------------------------------------------- /src/KeyValueStore/packages.lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "dependencies": { 4 | ".NETStandard,Version=v2.1": { 5 | "MessagePack": { 6 | "type": "Direct", 7 | "requested": "[2.2.85, )", 8 | "resolved": "2.2.85", 9 | "contentHash": "3SqAgwNV5LOf+ZapHmjQMUc7WDy/1ur9CfFNjgnfMZKCB5CxkVVbyHa06fObjGTEHZI7mcDathYjkI+ncr92ZQ==", 10 | "dependencies": { 11 | "MessagePack.Annotations": "2.2.85", 12 | "Microsoft.Bcl.AsyncInterfaces": "1.0.0", 13 | "System.Collections.Immutable": "1.5.0", 14 | "System.Memory": "4.5.3", 15 | "System.Reflection.Emit": "4.6.0", 16 | "System.Reflection.Emit.Lightweight": "4.6.0", 17 | "System.Runtime.CompilerServices.Unsafe": "4.5.2", 18 | "System.Threading.Tasks.Extensions": "4.5.3" 19 | } 20 | }, 21 | "Microsoft.Extensions.Logging": { 22 | "type": "Direct", 23 | "requested": "[5.0.0, )", 24 | "resolved": "5.0.0", 25 | "contentHash": "MgOwK6tPzB6YNH21wssJcw/2MKwee8b2gI7SllYfn6rvTpIrVvVS5HAjSU2vqSku1fwqRvWP0MdIi14qjd93Aw==", 26 | "dependencies": { 27 | "Microsoft.Extensions.DependencyInjection": "5.0.0", 28 | "Microsoft.Extensions.DependencyInjection.Abstractions": "5.0.0", 29 | "Microsoft.Extensions.Logging.Abstractions": "5.0.0", 30 | "Microsoft.Extensions.Options": "5.0.0", 31 | "System.Diagnostics.DiagnosticSource": "5.0.0" 32 | } 33 | }, 34 | "Microsoft.FASTER.Core": { 35 | "type": "Direct", 36 | "requested": "[1.9.3, )", 37 | "resolved": "1.9.3", 38 | "contentHash": "u3y0M6manowuKCu4kdfkzHcrQJQmUmseglZ/3rahQY9PA9Ef+VC9X3yIlb2D01msgqGE8kbj8kepwyUDIurNDg==", 39 | "dependencies": { 40 | "System.Interactive.Async": "5.0.0", 41 | "System.Runtime.CompilerServices.Unsafe": "5.0.0" 42 | } 43 | }, 44 | "MessagePack.Annotations": { 45 | "type": "Transitive", 46 | "resolved": "2.2.85", 47 | "contentHash": "YptRsDCQK35K5FhmZ0LojW4t8I6DpetLfK5KG8PVY2f6h7/gdyr8f4++xdSEK/xS6XX7/GPvEpqszKVPksCsiQ==" 48 | }, 49 | "Microsoft.Bcl.AsyncInterfaces": { 50 | "type": "Transitive", 51 | "resolved": "1.0.0", 52 | "contentHash": "K63Y4hORbBcKLWH5wnKgzyn7TOfYzevIEwIedQHBIkmkEBA9SCqgvom+XTuE+fAFGvINGkhFItaZ2dvMGdT5iw==", 53 | "dependencies": { 54 | "System.Threading.Tasks.Extensions": "4.5.2" 55 | } 56 | }, 57 | "Microsoft.Extensions.DependencyInjection": { 58 | "type": "Transitive", 59 | "resolved": "5.0.0", 60 | "contentHash": "Rc2kb/p3Ze6cP6rhFC3PJRdWGbLvSHZc0ev7YlyeU6FmHciDMLrhoVoTUEzKPhN5ZjFgKF1Cf5fOz8mCMIkvpA==", 61 | "dependencies": { 62 | "Microsoft.Extensions.DependencyInjection.Abstractions": "5.0.0" 63 | } 64 | }, 65 | "Microsoft.Extensions.DependencyInjection.Abstractions": { 66 | "type": "Transitive", 67 | "resolved": "5.0.0", 68 | "contentHash": "ORj7Zh81gC69TyvmcUm9tSzytcy8AVousi+IVRAI8nLieQjOFryRusSFh7+aLk16FN9pQNqJAiMd7BTKINK0kA==" 69 | }, 70 | "Microsoft.Extensions.Logging.Abstractions": { 71 | "type": "Transitive", 72 | "resolved": "5.0.0", 73 | "contentHash": "NxP6ahFcBnnSfwNBi2KH2Oz8Xl5Sm2krjId/jRR3I7teFphwiUoUeZPwTNA21EX+5PtjqmyAvKaOeBXcJjcH/w==" 74 | }, 75 | "Microsoft.Extensions.Options": { 76 | "type": "Transitive", 77 | "resolved": "5.0.0", 78 | "contentHash": "CBvR92TCJ5uBIdd9/HzDSrxYak+0W/3+yxrNg8Qm6Bmrkh5L+nu6m3WeazQehcZ5q1/6dDA7J5YdQjim0165zg==", 79 | "dependencies": { 80 | "Microsoft.Extensions.DependencyInjection.Abstractions": "5.0.0", 81 | "Microsoft.Extensions.Primitives": "5.0.0" 82 | } 83 | }, 84 | "Microsoft.Extensions.Primitives": { 85 | "type": "Transitive", 86 | "resolved": "5.0.0", 87 | "contentHash": "cI/VWn9G1fghXrNDagX9nYaaB/nokkZn0HYAawGaELQrl8InSezfe9OnfPZLcJq3esXxygh3hkq2c3qoV3SDyQ==", 88 | "dependencies": { 89 | "System.Buffers": "4.5.1", 90 | "System.Memory": "4.5.4", 91 | "System.Runtime.CompilerServices.Unsafe": "5.0.0" 92 | } 93 | }, 94 | "System.Buffers": { 95 | "type": "Transitive", 96 | "resolved": "4.5.1", 97 | "contentHash": "Rw7ijyl1qqRS0YQD/WycNst8hUUMgrMH4FCn1nNm27M4VxchZ1js3fVjQaANHO5f3sN4isvP4a+Met9Y4YomAg==" 98 | }, 99 | "System.Collections.Immutable": { 100 | "type": "Transitive", 101 | "resolved": "1.5.0", 102 | "contentHash": "EXKiDFsChZW0RjrZ4FYHu9aW6+P4MCgEDCklsVseRfhoO0F+dXeMSsMRAlVXIo06kGJ/zv+2w1a2uc2+kxxSaQ==" 103 | }, 104 | "System.Diagnostics.DiagnosticSource": { 105 | "type": "Transitive", 106 | "resolved": "5.0.0", 107 | "contentHash": "tCQTzPsGZh/A9LhhA6zrqCRV4hOHsK90/G7q3Khxmn6tnB1PuNU0cRaKANP2AWcF9bn0zsuOoZOSrHuJk6oNBA==", 108 | "dependencies": { 109 | "System.Memory": "4.5.4", 110 | "System.Runtime.CompilerServices.Unsafe": "5.0.0" 111 | } 112 | }, 113 | "System.Interactive.Async": { 114 | "type": "Transitive", 115 | "resolved": "5.0.0", 116 | "contentHash": "QaqhQVDiULcu4vm6o89+iP329HcK44cETHOYgy/jfEjtzeFy0ZxmuM7nel9ocjnKxEM4yh1mli7hgh8Q9o+/Iw==", 117 | "dependencies": { 118 | "System.Linq.Async": "5.0.0" 119 | } 120 | }, 121 | "System.Linq.Async": { 122 | "type": "Transitive", 123 | "resolved": "5.0.0", 124 | "contentHash": "cPtIuuH8TIjVHSi2ewwReWGW1PfChPE0LxPIDlfwVcLuTM9GANFTXiMB7k3aC4sk3f0cQU25LNKzx+jZMxijqw==" 125 | }, 126 | "System.Memory": { 127 | "type": "Transitive", 128 | "resolved": "4.5.4", 129 | "contentHash": "1MbJTHS1lZ4bS4FmsJjnuGJOu88ZzTT2rLvrhW7Ygic+pC0NWA+3hgAen0HRdsocuQXCkUTdFn9yHJJhsijDXw==", 130 | "dependencies": { 131 | "System.Buffers": "4.5.1", 132 | "System.Numerics.Vectors": "4.4.0", 133 | "System.Runtime.CompilerServices.Unsafe": "4.5.3" 134 | } 135 | }, 136 | "System.Numerics.Vectors": { 137 | "type": "Transitive", 138 | "resolved": "4.4.0", 139 | "contentHash": "UiLzLW+Lw6HLed1Hcg+8jSRttrbuXv7DANVj0DkL9g6EnnzbL75EB7EWsw5uRbhxd/4YdG8li5XizGWepmG3PQ==" 140 | }, 141 | "System.Reflection.Emit": { 142 | "type": "Transitive", 143 | "resolved": "4.6.0", 144 | "contentHash": "qAo4jyXtC9i71iElngX7P2r+zLaiHzxKwf66sc3X91tL5Ks6fnQ1vxL04o7ZSm3sYfLExySL7GN8aTpNYpU1qw==" 145 | }, 146 | "System.Reflection.Emit.Lightweight": { 147 | "type": "Transitive", 148 | "resolved": "4.6.0", 149 | "contentHash": "j/V5HVvxvBQ7uubYD0PptQW2KGsi1Pc2kZ9yfwLixv3ADdjL/4M78KyC5e+ymW612DY8ZE4PFoZmWpoNmN2mqg==" 150 | }, 151 | "System.Runtime.CompilerServices.Unsafe": { 152 | "type": "Transitive", 153 | "resolved": "5.0.0", 154 | "contentHash": "ZD9TMpsmYJLrxbbmdvhwt9YEgG5WntEnZ/d1eH8JBX9LBp+Ju8BSBhUGbZMNVHHomWo2KVImJhTDl2hIgw/6MA==" 155 | }, 156 | "System.Threading.Tasks.Extensions": { 157 | "type": "Transitive", 158 | "resolved": "4.5.3", 159 | "contentHash": "+MvhNtcvIbqmhANyKu91jQnvIRVSTiaOiFNfKWwXGHG48YAb4I/TyH8spsySiPYla7gKal5ZnF3teJqZAximyQ==", 160 | "dependencies": { 161 | "System.Runtime.CompilerServices.Unsafe": "4.5.2" 162 | } 163 | } 164 | } 165 | } 166 | } -------------------------------------------------------------------------------- /test/KeyValueStore/Helpers/StringBuilderLogger.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Logging; 2 | using System; 3 | using System.Text; 4 | 5 | namespace Jering.KeyValueStore.Tests 6 | { 7 | public class StringBuilderLogger : ILogger 8 | { 9 | private readonly object _lock = new(); 10 | private readonly StringBuilder _stringBuilder; 11 | 12 | public StringBuilderLogger(StringBuilder stringBuilder) 13 | { 14 | _stringBuilder = stringBuilder; 15 | } 16 | 17 | public IDisposable? BeginScope(TState state) 18 | { 19 | return null; 20 | } 21 | 22 | public bool IsEnabled(LogLevel logLevel) 23 | { 24 | return true; 25 | } 26 | 27 | public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) 28 | { 29 | lock (_lock) 30 | { 31 | _stringBuilder.Append(logLevel.ToString()).Append(": ").AppendLine(formatter(state, exception)); 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /test/KeyValueStore/Helpers/StringBuilderProvider.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Logging; 2 | using System; 3 | using System.Text; 4 | 5 | namespace Jering.KeyValueStore.Tests 6 | { 7 | public class StringBuilderProvider : ILoggerProvider 8 | { 9 | private readonly StringBuilder _stringBuilder; 10 | 11 | public StringBuilderProvider(StringBuilder stringBuilder) 12 | { 13 | _stringBuilder = stringBuilder; 14 | } 15 | 16 | public ILogger CreateLogger(string categoryName) 17 | { 18 | return new StringBuilderLogger(_stringBuilder); 19 | } 20 | 21 | public void Dispose() 22 | { 23 | GC.SuppressFinalize(this); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /test/KeyValueStore/Jering.KeyValueStore.Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net5.0 5 | enable 6 | false 7 | 8 | ../../keypair.snk 9 | true 10 | true 11 | true 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | runtime; build; native; contentfiles; analyzers; buildtransitive 20 | all 21 | 22 | 23 | all 24 | runtime; build; native; contentfiles; analyzers 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /test/KeyValueStore/MixedStorageKVStoreIntegrationTests.cs: -------------------------------------------------------------------------------- 1 | using FASTER.core; 2 | using MessagePack; 3 | using Microsoft.Extensions.DependencyInjection; 4 | using Microsoft.Extensions.Logging; 5 | using System; 6 | using System.Collections.Concurrent; 7 | using System.Collections.Generic; 8 | using System.IO; 9 | using System.Linq; 10 | using System.Text; 11 | using System.Text.RegularExpressions; 12 | using System.Threading; 13 | using System.Threading.Tasks; 14 | using Xunit; 15 | 16 | namespace Jering.KeyValueStore.Tests 17 | { 18 | /// 19 | /// Verifies behaviour of and its underlying instance. They: 20 | /// 21 | /// Handle concurrent insert, update, read and delete operations 22 | /// Handle reference-type and value-type keys and values 23 | /// Delete log files on dispose 24 | /// Compact logs periodically 25 | /// 26 | /// 27 | public class MixedStorageKVStoreIntegrationTests : IClassFixture 28 | { 29 | private const int TIMEOUT_MS = 60000; 30 | private readonly MixedStorageKVStoreIntegrationTestsFixture _fixture; 31 | private readonly MessagePackSerializerOptions _messagePackSerializerOptions = MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray); 32 | 33 | public MixedStorageKVStoreIntegrationTests(MixedStorageKVStoreIntegrationTestsFixture fixture) 34 | { 35 | _fixture = fixture; 36 | } 37 | 38 | // TODO Interleave operations 39 | [Fact] 40 | public async Task UpsertReadAsyncDelete_AreThreadSafe() 41 | { 42 | // Arrange 43 | var dummyOptions = new MixedStorageKVStoreOptions() 44 | { 45 | LogDirectory = _fixture.TempDirectory, 46 | LogFileNamePrefix = nameof(UpsertReadAsyncDelete_AreThreadSafe), 47 | PageSizeBits = 12, 48 | MemorySizeBits = 13 // Limit to 8KB so we're testing both in-memory and disk-based operations 49 | }; 50 | DummyClass dummyClassInstance = CreatePopulatedDummyClassInstance(); 51 | int numRecords = 10000; 52 | using var testSubject = new MixedStorageKVStore(dummyOptions); 53 | 54 | // Act and assert 55 | 56 | // Insert 57 | ConcurrentQueue upsertTasks = new(); 58 | Parallel.For(0, numRecords, key => upsertTasks.Enqueue(testSubject.UpsertAsync(key, dummyClassInstance))); 59 | await Task.WhenAll(upsertTasks).ConfigureAwait(false); 60 | 61 | // Read and verify inserts 62 | ConcurrentQueue> readTasks = new(); 63 | #pragma warning disable CA2012 // Can't await in Parallel.For actions 64 | Parallel.For(0, numRecords, key => readTasks.Enqueue(testSubject.ReadAsync(key))); 65 | foreach (ValueTask<(Status, DummyClass?)> task in readTasks) 66 | { 67 | (Status status, DummyClass? result) = await task.ConfigureAwait(false); 68 | Assert.Equal(Status.OK, status); 69 | Assert.Equal(dummyClassInstance, result); 70 | } 71 | 72 | // Update 73 | dummyClassInstance.DummyInt = 20; 74 | dummyClassInstance.DummyString = "anotherDummyString"; 75 | upsertTasks.Clear(); 76 | Parallel.For(0, numRecords, key => upsertTasks.Enqueue(testSubject.UpsertAsync(key, dummyClassInstance))); 77 | await Task.WhenAll(upsertTasks).ConfigureAwait(false); 78 | 79 | // Read and verify updates 80 | readTasks.Clear(); 81 | Parallel.For(0, numRecords, key => readTasks.Enqueue(testSubject.ReadAsync(key))); 82 | foreach (ValueTask<(Status, DummyClass?)> task in readTasks) 83 | { 84 | (Status status, DummyClass? result) = await task.ConfigureAwait(false); 85 | Assert.Equal(Status.OK, status); 86 | Assert.Equal(dummyClassInstance, result); 87 | } 88 | 89 | // Delete 90 | ConcurrentQueue> deleteTasks = new(); 91 | Parallel.For(0, numRecords, key => deleteTasks.Enqueue(testSubject.DeleteAsync(key))); 92 | foreach (ValueTask task in deleteTasks) 93 | { 94 | Status status = await task.ConfigureAwait(false); 95 | Assert.Equal(Status.OK, status); 96 | } 97 | 98 | // Verify deletes 99 | readTasks.Clear(); 100 | Parallel.For(0, numRecords, key => readTasks.Enqueue(testSubject.ReadAsync(key))); 101 | foreach (ValueTask<(Status, DummyClass?)> task in readTasks) 102 | { 103 | (Status status, DummyClass? result) = await task.ConfigureAwait(false); 104 | Assert.Equal(Status.NOTFOUND, status); 105 | Assert.Null(result); 106 | } 107 | } 108 | 109 | [Fact] 110 | public async Task KeysAndValues_SupportsObjects() 111 | { 112 | // Arrange 113 | var dummyOptions = new MixedStorageKVStoreOptions() 114 | { 115 | LogDirectory = _fixture.TempDirectory, 116 | LogFileNamePrefix = nameof(KeysAndValues_SupportsObjects), 117 | PageSizeBits = 12, 118 | MemorySizeBits = 13 // Limit to 8KB so we're testing both in-memory and disk-based operations 119 | }; 120 | int numRecords = 10000; 121 | using var testSubject = new MixedStorageKVStore(dummyOptions); 122 | 123 | // Act and assert 124 | 125 | // Insert 126 | ConcurrentQueue upsertTasks = new(); 127 | Parallel.For(0, numRecords, key => 128 | { 129 | string keyAsString = key.ToString(); 130 | upsertTasks.Enqueue(testSubject.UpsertAsync(keyAsString, keyAsString)); 131 | }); 132 | await Task.WhenAll(upsertTasks).ConfigureAwait(false); 133 | 134 | // Read and verify inserts 135 | ConcurrentDictionary> readTasks = new(); 136 | Parallel.For(0, numRecords, key => readTasks.TryAdd(key, testSubject.ReadAsync(key.ToString()))); 137 | foreach (KeyValuePair> keyValuePair in readTasks) 138 | { 139 | (Status status, string? result) = await keyValuePair.Value.ConfigureAwait(false); 140 | Assert.Equal(Status.OK, status); 141 | Assert.Equal(keyValuePair.Key.ToString(), result); 142 | } 143 | } 144 | 145 | [Fact] 146 | public async Task KeysAndValues_SupportsVariableLengthStructs() 147 | { 148 | // Arrange 149 | var dummyOptions = new MixedStorageKVStoreOptions() 150 | { 151 | LogDirectory = _fixture.TempDirectory, 152 | LogFileNamePrefix = nameof(KeysAndValues_SupportsVariableLengthStructs), 153 | PageSizeBits = 12, 154 | MemorySizeBits = 13 // Limit to 8KB so we're testing both in-memory and disk-based operations 155 | }; 156 | var dummyStructInstance = new DummyVariableLengthStruct() 157 | { 158 | // Populate with dummy values 159 | DummyString = "dummyString", 160 | DummyStringArray = new[] { "dummyString1", "dummyString2", "dummyString3", "dummyString4", "dummyString5" }, 161 | DummyIntArray = new[] { 10, 100, 1000, 10000, 100000, 1000000, 10000000 } 162 | }; 163 | int numRecords = 10000; 164 | using var testSubject = new MixedStorageKVStore(dummyOptions); 165 | 166 | // Act and assert 167 | 168 | // Insert 169 | ConcurrentQueue upsertTasks = new(); 170 | Parallel.For(0, numRecords, key => 171 | { 172 | DummyVariableLengthStruct localDummyStructInstance = dummyStructInstance; 173 | localDummyStructInstance.DummyInt = key; 174 | upsertTasks.Enqueue(testSubject.UpsertAsync(localDummyStructInstance, localDummyStructInstance)); 175 | }); 176 | await Task.WhenAll(upsertTasks).ConfigureAwait(false); 177 | 178 | // Read and verify 179 | ConcurrentDictionary> readTasks = new(); 180 | Parallel.For(0, numRecords, key => 181 | { 182 | DummyVariableLengthStruct localDummyStructInstance = dummyStructInstance; 183 | localDummyStructInstance.DummyInt = key; 184 | readTasks.TryAdd(key, testSubject.ReadAsync(localDummyStructInstance)); 185 | }); 186 | foreach (KeyValuePair> keyValuePair in readTasks) 187 | { 188 | (Status status, DummyVariableLengthStruct result) = await keyValuePair.Value.ConfigureAwait(false); 189 | Assert.Equal(Status.OK, status); 190 | DummyVariableLengthStruct localDummyStructInstance = dummyStructInstance; 191 | localDummyStructInstance.DummyInt = keyValuePair.Key; 192 | Assert.Equal(localDummyStructInstance, result); 193 | } 194 | } 195 | 196 | [Fact] 197 | public async Task KeysAndValues_SupportsFixedLengthStructs() 198 | { 199 | // Arrange 200 | var dummyOptions = new MixedStorageKVStoreOptions() 201 | { 202 | LogDirectory = _fixture.TempDirectory, 203 | LogFileNamePrefix = nameof(KeysAndValues_SupportsFixedLengthStructs), 204 | PageSizeBits = 12, 205 | MemorySizeBits = 13 // Limit to 8KB so we're testing both in-memory and disk-based operations 206 | }; 207 | var dummyStructInstance = new DummyFixedLengthStruct() 208 | { 209 | // Populate with dummy values 210 | DummyByte = byte.MaxValue, 211 | DummyShort = short.MaxValue, 212 | DummyLong = long.MaxValue 213 | }; 214 | int numRecords = 10000; 215 | using var testSubject = new MixedStorageKVStore(dummyOptions); 216 | 217 | // Act and assert 218 | 219 | // Insert 220 | ConcurrentQueue upsertTasks = new(); 221 | Parallel.For(0, numRecords, key => 222 | { 223 | DummyFixedLengthStruct localDummyStructInstance = dummyStructInstance; 224 | localDummyStructInstance.DummyInt = key; 225 | upsertTasks.Enqueue(testSubject.UpsertAsync(localDummyStructInstance, localDummyStructInstance)); 226 | }); 227 | await Task.WhenAll(upsertTasks).ConfigureAwait(false); 228 | 229 | // Read and verify 230 | ConcurrentDictionary> readTasks = new(); 231 | Parallel.For(0, numRecords, key => 232 | { 233 | DummyFixedLengthStruct localDummyStructInstance = dummyStructInstance; 234 | localDummyStructInstance.DummyInt = key; 235 | readTasks.TryAdd(key, testSubject.ReadAsync(localDummyStructInstance)); 236 | }); 237 | foreach (KeyValuePair> keyValuePair in readTasks) 238 | { 239 | (Status status, DummyFixedLengthStruct result) = await keyValuePair.Value.ConfigureAwait(false); 240 | Assert.Equal(Status.OK, status); 241 | DummyFixedLengthStruct localDummyStructInstance = dummyStructInstance; 242 | localDummyStructInstance.DummyInt = keyValuePair.Key; 243 | Assert.Equal(localDummyStructInstance, result); 244 | } 245 | } 246 | 247 | [Fact] 248 | public async Task KeysAndValues_SupportsPrimitives() 249 | { 250 | // Arrange 251 | var dummyOptions = new MixedStorageKVStoreOptions() 252 | { 253 | LogDirectory = _fixture.TempDirectory, 254 | LogFileNamePrefix = nameof(KeysAndValues_SupportsPrimitives), 255 | PageSizeBits = 12, 256 | MemorySizeBits = 13 // Limit to 8KB so we're testing both in-memory and disk-based operations 257 | }; 258 | const int dummyValue = 12345; 259 | const int numRecords = 10000; 260 | using var testSubject = new MixedStorageKVStore(dummyOptions); 261 | 262 | // Act and assert 263 | 264 | // Insert 265 | ConcurrentQueue upsertTasks = new(); 266 | Parallel.For(0, numRecords, key => upsertTasks.Enqueue(testSubject.UpsertAsync(key, dummyValue))); 267 | await Task.WhenAll(upsertTasks).ConfigureAwait(false); 268 | 269 | // Read and verify 270 | ConcurrentQueue> readTasks = new(); 271 | Parallel.For(0, numRecords, key => readTasks.Enqueue(testSubject.ReadAsync(key))); 272 | foreach (ValueTask<(Status, int)> task in readTasks) 273 | { 274 | (Status status, int result) = await task.ConfigureAwait(false); 275 | Assert.Equal(Status.OK, status); 276 | Assert.Equal(dummyValue, result); 277 | } 278 | } 279 | 280 | [Fact] 281 | public async Task LogFiles_DeletedOnClose() 282 | { 283 | // Arrange 284 | string directory = Path.Combine(_fixture.TempDirectory, nameof(LogFiles_DeletedOnClose)); // Use a separate directory so the test is never affected by other tests 285 | var dummyOptions = new MixedStorageKVStoreOptions() 286 | { 287 | LogDirectory = directory, 288 | LogFileNamePrefix = nameof(LogFiles_DeletedOnClose), 289 | PageSizeBits = 9, // Minimum 290 | MemorySizeBits = 10 // Minimum 291 | }; 292 | DummyClass dummyClassInstance = CreatePopulatedDummyClassInstance(); 293 | int numRecords = 50; // Just enough to make sure log files are created. Segment size isn't exceeded (only 1 of each log file). 294 | var testSubject = new MixedStorageKVStore(dummyOptions); 295 | ConcurrentQueue upsertTasks = new(); 296 | Parallel.For(0, numRecords, key => upsertTasks.Enqueue(testSubject.UpsertAsync(key, dummyClassInstance))); // Creates log 297 | await Task.WhenAll(upsertTasks).ConfigureAwait(false); 298 | Assert.Single(Directory.EnumerateFiles(directory, $"{nameof(LogFiles_DeletedOnClose)}*")); // Log and object log 299 | 300 | // Act 301 | testSubject.Dispose(); 302 | 303 | // Assert 304 | Assert.Empty(Directory.EnumerateFiles(directory)); // Logs deleted 305 | } 306 | 307 | [Fact(Timeout = TIMEOUT_MS)] 308 | public async Task LogCompaction_SkippedIfSafeReadOnlyRegionIsLessThanThreshold() 309 | { 310 | // Arrange 311 | var resultStringBuilder = new StringBuilder(); 312 | ILogger> dummyLogger = CreateLogger(resultStringBuilder, LogLevel.Trace); 313 | var logSettings = new LogSettings 314 | { 315 | LogDevice = Devices.CreateLogDevice(Path.Combine(_fixture.TempDirectory, $"{nameof(LogCompaction_SkippedIfSafeReadOnlyRegionIsLessThanThreshold)}.log"), 316 | deleteOnClose: true), 317 | PageSizeBits = 12, 318 | MemorySizeBits = 13 319 | }; 320 | DummyClass dummyClassInstance = CreatePopulatedDummyClassInstance(); 321 | int dummyThreshold = 100_000; 322 | var dummyOptions = new MixedStorageKVStoreOptions() 323 | { 324 | TimeBetweenLogCompactionsMS = 1, 325 | InitialLogCompactionThresholdBytes = 100_000 326 | }; 327 | int expectedResultMinLength = string.Format(Strings.LogTrace_SkippingLogCompaction, 0, dummyThreshold).Length; 328 | 329 | // Act 330 | using (var testSubject = new MixedStorageKVStore(dummyOptions, dummyLogger)) // Start log compaction 331 | { 332 | while (resultStringBuilder.Length <= expectedResultMinLength) 333 | { 334 | await Task.Delay(10).ConfigureAwait(false); 335 | } 336 | } 337 | 338 | // Assert 339 | string result = resultStringBuilder.ToString(); 340 | int numLines = result.Split("\n", StringSplitOptions.RemoveEmptyEntries).Length; 341 | string regexPattern = string.Format(Strings.LogTrace_SkippingLogCompaction, "0", dummyThreshold); // MixedStorageKVStore is empty 342 | Assert.Equal(numLines, Regex.Matches(result, regexPattern).Count); 343 | } 344 | 345 | [Fact(Timeout = TIMEOUT_MS)] 346 | public async Task LogCompaction_OccursIfSafeReadOnlyRegionIsLargerThanThreshold() 347 | { 348 | // Arrange 349 | var resultStringBuilder = new StringBuilder(); 350 | ILogger> dummyLogger = CreateLogger(resultStringBuilder, LogLevel.Trace); 351 | var logSettings = new LogSettings 352 | { 353 | LogDevice = Devices.CreateLogDevice(Path.Combine(_fixture.TempDirectory, $"{nameof(LogCompaction_OccursIfSafeReadOnlyRegionIsLargerThanThreshold)}.log"), 354 | deleteOnClose: true), 355 | PageSizeBits = 12, 356 | MemorySizeBits = 13 357 | }; 358 | DummyClass dummyClassInstance = CreatePopulatedDummyClassInstance(); 359 | var dummyOptions = new MixedStorageKVStoreOptions() 360 | { 361 | TimeBetweenLogCompactionsMS = 1, 362 | InitialLogCompactionThresholdBytes = 80_000 363 | }; 364 | // Create and populate faster KV store before passing it to MixedStorageKVStore, at which point the compaction loop starts. 365 | // For quicker tests, use thread local sessions. 366 | FasterKV? dummyFasterKVStore = null; 367 | ThreadLocal>>? dummyThreadLocalSession = null; 368 | 369 | try 370 | { 371 | dummyFasterKVStore = new FasterKV(1L << 20, logSettings); 372 | FasterKV.ClientSessionBuilder dummyClientSessionBuilder = dummyFasterKVStore.For(new SpanByteFunctions()); 373 | 374 | // Record size estimate: 375 | // - dummyClassInstance serialized and compressed = ~73 bytes 376 | // - int key serialized and compressed = ~3 bytes 377 | // - key length metadata = 4 bytes 378 | // - value length metadata = 4 bytes 379 | // - record header = 8 bytes 380 | // Total = ~92 bytes 381 | // 382 | // n * 92 - 8192 > InitialLogCompactionThresholdBytes 383 | 384 | // Insert 385 | byte[] dummyValueBytes = MessagePackSerializer.Serialize(dummyClassInstance, _messagePackSerializerOptions); 386 | await UpsertRangeAsync(0, 500, dummyValueBytes, dummyClientSessionBuilder).ConfigureAwait(false); 387 | 388 | // Update so compaction does something. Can't update in insert loop or we'll get a bunch of in-place updates. 389 | dummyClassInstance.DummyInt++; 390 | dummyValueBytes = MessagePackSerializer.Serialize(dummyClassInstance, _messagePackSerializerOptions); 391 | await UpsertRangeAsync(0, 500, dummyValueBytes, dummyClientSessionBuilder).ConfigureAwait(false); 392 | } 393 | finally 394 | { 395 | if (dummyThreadLocalSession != null) 396 | { 397 | foreach (ClientSession> session in dummyThreadLocalSession.Values) 398 | { 399 | session.Dispose(); // Faster synchronously completes all pending operations, so we should not get exceptions if we're in the middle of log compaction 400 | } 401 | } 402 | } 403 | 404 | // We compact 20% of the safe-readonly region of the log. Since we inserted then updated, compaction here means removal. 405 | // 90048 * 0.8 = 72038, ~72000 (missing 38 bytes likely has to do with the fact that we can't remove only part of a record). 406 | string expectedResultStart = $"{LogLevel.Trace}: {string.Format(Strings.LogTrace_LogCompacted, 94144, 75232, 1)}"; 407 | int expectedResultStartLength = expectedResultStart.Length; 408 | 409 | // Act 410 | using (var testSubject = new MixedStorageKVStore(dummyOptions, dummyLogger, dummyFasterKVStore)) // Start log compaction 411 | { 412 | while (resultStringBuilder.Length <= expectedResultStartLength) 413 | { 414 | await Task.Delay(10).ConfigureAwait(false); 415 | } 416 | } 417 | 418 | // Assert 419 | Assert.StartsWith(expectedResultStart, resultStringBuilder.ToString()); 420 | // If compaction runs more than once, should be skipped after first compaction (< threshold behaviour verified in previous test) 421 | } 422 | 423 | [Fact(Timeout = TIMEOUT_MS)] 424 | public async Task LogCompaction_IncreasesThresholdAfterFiveConsecutiveCompactions() 425 | { 426 | // Arrange 427 | var resultStringBuilder = new StringBuilder(); 428 | ILogger> dummyLogger = CreateLogger(resultStringBuilder, LogLevel.Trace); 429 | var logSettings = new LogSettings 430 | { 431 | LogDevice = Devices.CreateLogDevice(Path.Combine(_fixture.TempDirectory, $"{nameof(LogCompaction_IncreasesThresholdAfterFiveConsecutiveCompactions)}.log"), 432 | deleteOnClose: true), 433 | PageSizeBits = 12, 434 | MemorySizeBits = 13 435 | }; 436 | DummyClass dummyClassInstance = CreatePopulatedDummyClassInstance(); 437 | var dummyOptions = new MixedStorageKVStoreOptions() 438 | { 439 | TimeBetweenLogCompactionsMS = 1, 440 | InitialLogCompactionThresholdBytes = 20_000 // So we compact 5 times in a row 441 | }; 442 | // Create and populate faster KV store before passing it to MixedStorageKVStore, at which point the compaction loop starts. 443 | // For quicker tests, use thread local sessions. 444 | FasterKV? dummyFasterKVStore = null; 445 | ThreadLocal>>? dummyThreadLocalSession = null; 446 | try 447 | { 448 | dummyFasterKVStore = new FasterKV(1L << 20, logSettings); 449 | FasterKV.ClientSessionBuilder dummyClientSessionBuilder = dummyFasterKVStore.For(new SpanByteFunctions()); 450 | dummyThreadLocalSession = new(() => dummyClientSessionBuilder.NewSession>(), true); 451 | MessagePackSerializerOptions dummyMessagePackSerializerOptions = MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray); 452 | 453 | // Record size estimate: 454 | // - dummyClassInstance serialized and compressed = ~73 bytes 455 | // - int key serialized and compressed = ~3 bytes 456 | // - key length metadata = 4 bytes 457 | // - value length metadata = 4 bytes 458 | // - record header = 8 bytes 459 | // Total = ~92 bytes 460 | // 461 | // n * 92 - 8192 > InitialLogCompactionThresholdBytes 462 | 463 | // Insert 464 | byte[] dummyValueBytes = MessagePackSerializer.Serialize(dummyClassInstance, dummyMessagePackSerializerOptions); 465 | await UpsertRangeAsync(0, 500, dummyValueBytes, dummyClientSessionBuilder).ConfigureAwait(false); 466 | 467 | // Update so compaction does something. Can't update in insert loop or we'll get a bunch of in-place updates. 468 | dummyClassInstance.DummyInt++; 469 | dummyValueBytes = MessagePackSerializer.Serialize(dummyClassInstance, dummyMessagePackSerializerOptions); 470 | await UpsertRangeAsync(0, 500, dummyValueBytes, dummyClientSessionBuilder).ConfigureAwait(false); 471 | } 472 | finally 473 | { 474 | if (dummyThreadLocalSession != null) 475 | { 476 | foreach (ClientSession> session in dummyThreadLocalSession.Values) 477 | { 478 | session.Dispose(); // Faster synchronously completes all pending operations, so we should not get exceptions if we're in the middle of log compaction 479 | } 480 | } 481 | } 482 | 483 | // Runs 5 consecutive compactions, increases threshold, runs 5 more consecutive compactions (all redundant), increases threshold above 484 | // safe-readonly region size, skips compactions thereafter. 485 | string expectedResultStart = @$"{LogLevel.Trace}: {string.Format(Strings.LogTrace_LogCompacted, 94144, 75232, 1)} 486 | {LogLevel.Trace}: {string.Format(Strings.LogTrace_LogCompacted, 75232, 60096, 2)} 487 | {LogLevel.Trace}: {string.Format(Strings.LogTrace_LogCompacted, 60096, 48000, 3)} 488 | {LogLevel.Trace}: {string.Format(Strings.LogTrace_LogCompacted, 48000, 46560, 4)} 489 | {LogLevel.Trace}: {string.Format(Strings.LogTrace_LogCompacted, 46560, 45408, 5)} 490 | {LogLevel.Trace}: {string.Format(Strings.LogTrace_LogCompactionThresholdIncreased, 20000, 40000)} 491 | {LogLevel.Trace}: {string.Format(Strings.LogTrace_LogCompacted, 45408, 48576, 1)} 492 | {LogLevel.Trace}: {string.Format(Strings.LogTrace_LogCompacted, 48576, 47040, 2)} 493 | {LogLevel.Trace}: {string.Format(Strings.LogTrace_LogCompacted, 47040, 45792, 3)} 494 | {LogLevel.Trace}: {string.Format(Strings.LogTrace_LogCompacted, 45792, 44768, 4)} 495 | {LogLevel.Trace}: {string.Format(Strings.LogTrace_LogCompacted, 44768, 48096, 5)} 496 | {LogLevel.Trace}: {string.Format(Strings.LogTrace_LogCompactionThresholdIncreased, 40000, 80000)} 497 | {LogLevel.Trace}: {string.Format(Strings.LogTrace_SkippingLogCompaction, 48096, 80000)}"; 498 | int expectedResultStartLength = expectedResultStart.Length; 499 | 500 | // Act 501 | using (var testSubject = new MixedStorageKVStore(dummyOptions, dummyLogger, dummyFasterKVStore)) // Start log compaction 502 | { 503 | while (resultStringBuilder.Length <= expectedResultStartLength) 504 | { 505 | await Task.Delay(10).ConfigureAwait(false); 506 | } 507 | } 508 | 509 | // Assert 510 | Assert.StartsWith(expectedResultStart.Replace("\r\n", "\n"), resultStringBuilder.ToString().Replace("\r\n", "\n")); 511 | } 512 | 513 | #region Helpers 514 | // Upserts the same value to a range of keys 515 | private async Task UpsertRangeAsync(int startKey, 516 | int endKey, 517 | byte[] valueBytes, 518 | FasterKV.ClientSessionBuilder clientSessionBuilder) 519 | { 520 | ConcurrentQueue.UpsertAsyncResult>> upsertTasks = new(); 521 | Parallel.For(startKey, endKey, key => 522 | { 523 | byte[] dummyKeyBytes = MessagePackSerializer.Serialize(key, _messagePackSerializerOptions); 524 | unsafe 525 | { 526 | // Upsert 527 | fixed (byte* keyPointer = dummyKeyBytes) 528 | fixed (byte* valuePointer = valueBytes) 529 | { 530 | var keySpanByte = SpanByte.FromPointer(keyPointer, dummyKeyBytes.Length); 531 | var objSpanByte = SpanByte.FromPointer(valuePointer, valueBytes.Length); 532 | upsertTasks.Enqueue(clientSessionBuilder.NewSession>().UpsertAsync(ref keySpanByte, ref objSpanByte)); 533 | } 534 | } 535 | }); 536 | foreach (ValueTask.UpsertAsyncResult> task in upsertTasks) 537 | { 538 | FasterKV.UpsertAsyncResult result = await task.ConfigureAwait(false); 539 | 540 | while (result.Status == Status.PENDING) 541 | { 542 | result = await result.CompleteAsync().ConfigureAwait(false); 543 | } 544 | } 545 | } 546 | 547 | private static DummyClass CreatePopulatedDummyClassInstance() 548 | { 549 | return new DummyClass() 550 | { 551 | // Populate with dummy values 552 | DummyString = "dummyString", 553 | DummyStringArray = new[] { "dummyString1", "dummyString2", "dummyString3", "dummyString4", "dummyString5" }, 554 | DummyInt = 10, 555 | DummyIntArray = new[] { 10, 100, 1000, 10000, 100000, 1000000, 10000000 } 556 | }; 557 | } 558 | 559 | private static ILogger> CreateLogger(StringBuilder stringBuilder, LogLevel minLogLevel) 560 | { 561 | var services = new ServiceCollection(); 562 | services.AddLogging(lb => lb. 563 | AddProvider(new StringBuilderProvider(stringBuilder)). 564 | AddFilter(logLevel => logLevel >= minLogLevel)); 565 | 566 | ServiceProvider serviceProvider = services.BuildServiceProvider(); 567 | 568 | return serviceProvider.GetRequiredService>>(); 569 | } 570 | #endregion 571 | 572 | #region Types 573 | [MessagePackObject] 574 | public struct DummyFixedLengthStruct 575 | { 576 | [Key(0)] 577 | public byte DummyByte { get; set; } 578 | 579 | [Key(1)] 580 | public short DummyShort { get; set; } 581 | 582 | [Key(2)] 583 | public int DummyInt { get; set; } 584 | 585 | [Key(3)] 586 | public long DummyLong { get; set; } 587 | 588 | public override bool Equals(object? obj) 589 | { 590 | if (obj is not DummyFixedLengthStruct castObject) 591 | { 592 | return false; 593 | } 594 | 595 | return castObject.DummyByte == DummyByte && 596 | castObject.DummyShort == DummyShort && 597 | castObject.DummyInt == DummyInt && 598 | castObject.DummyLong == DummyLong; 599 | } 600 | 601 | public override int GetHashCode() 602 | { 603 | return HashCode.Combine(DummyByte, DummyShort, DummyInt, DummyLong); 604 | } 605 | 606 | public static bool operator ==(DummyFixedLengthStruct left, DummyFixedLengthStruct right) 607 | { 608 | return left.Equals(right); 609 | } 610 | 611 | public static bool operator !=(DummyFixedLengthStruct left, DummyFixedLengthStruct right) 612 | { 613 | return !(left == right); 614 | } 615 | } 616 | 617 | [MessagePackObject] 618 | public struct DummyVariableLengthStruct 619 | { 620 | [Key(0)] 621 | public string? DummyString { get; set; } 622 | 623 | [Key(1)] 624 | public string[]? DummyStringArray { get; set; } 625 | 626 | [Key(2)] 627 | public int DummyInt { get; set; } 628 | 629 | [Key(3)] 630 | public int[]? DummyIntArray { get; set; } 631 | 632 | public override bool Equals(object? obj) 633 | { 634 | if (obj is not DummyVariableLengthStruct castObject) 635 | { 636 | return false; 637 | } 638 | 639 | return castObject.DummyString == DummyString && 640 | castObject.DummyInt == DummyInt && 641 | #pragma warning disable CS8604 // Arrays should never be null in these tests, throw if they are 642 | castObject.DummyStringArray.SequenceEqual(DummyStringArray) && 643 | castObject.DummyIntArray.SequenceEqual(DummyIntArray); 644 | #pragma warning restore CS8604 645 | } 646 | 647 | public override int GetHashCode() 648 | { 649 | return HashCode.Combine(DummyString, DummyStringArray, DummyInt, DummyIntArray); 650 | } 651 | 652 | public static bool operator ==(DummyVariableLengthStruct left, DummyVariableLengthStruct right) 653 | { 654 | return left.Equals(right); 655 | } 656 | 657 | public static bool operator !=(DummyVariableLengthStruct left, DummyVariableLengthStruct right) 658 | { 659 | return !(left == right); 660 | } 661 | } 662 | 663 | [MessagePackObject] 664 | public class DummyClass 665 | { 666 | [Key(0)] 667 | public string? DummyString { get; set; } 668 | 669 | [Key(1)] 670 | public string[]? DummyStringArray { get; set; } 671 | 672 | [Key(2)] 673 | public int DummyInt { get; set; } 674 | 675 | [Key(3)] 676 | public int[]? DummyIntArray { get; set; } 677 | 678 | public override bool Equals(object? obj) 679 | { 680 | if (obj is not DummyClass castObject) 681 | { 682 | return false; 683 | } 684 | 685 | return castObject.DummyString == DummyString && 686 | castObject.DummyInt == DummyInt && 687 | #pragma warning disable CS8604 // Arrays should never be null in these tests, throw if they are 688 | castObject.DummyStringArray.SequenceEqual(DummyStringArray) && 689 | castObject.DummyIntArray.SequenceEqual(DummyIntArray); 690 | #pragma warning restore CS8604 691 | } 692 | 693 | public override int GetHashCode() 694 | { 695 | return HashCode.Combine(DummyString, DummyStringArray, DummyInt, DummyIntArray); 696 | } 697 | } 698 | #endregion 699 | } 700 | 701 | public class MixedStorageKVStoreIntegrationTestsFixture : IDisposable 702 | { 703 | public string TempDirectory { get; } = Path.Combine(Path.GetTempPath(), nameof(MixedStorageKVStoreIntegrationTests)); 704 | 705 | public MixedStorageKVStoreIntegrationTestsFixture() 706 | { 707 | TryDeleteDirectory(); 708 | Directory.CreateDirectory(TempDirectory); 709 | } 710 | 711 | private void TryDeleteDirectory() 712 | { 713 | try 714 | { 715 | Directory.Delete(TempDirectory, true); 716 | } 717 | catch 718 | { 719 | // Do nothing 720 | } 721 | } 722 | 723 | public void Dispose() 724 | { 725 | Dispose(true); 726 | GC.SuppressFinalize(this); 727 | } 728 | 729 | protected virtual void Dispose(bool _) 730 | { 731 | TryDeleteDirectory(); 732 | } 733 | } 734 | } 735 | --------------------------------------------------------------------------------