├── .github ├── CODEOWNERS ├── dependabot.yml ├── pull_request_template.md └── workflows │ ├── build_test.yml │ └── publish.yml ├── .gitignore ├── .vscode └── tasks.json ├── LICENSE.md ├── README.md ├── SemanticKernel.Connectors.Memory.SqlServer.sln ├── nuget └── nuget-package.props └── src ├── Connectors.Memory.SqlServer.Tests ├── .editorconfig ├── Extensions │ └── AsyncEnumerable.cs ├── IntegrationTests.csproj ├── SqlServerMemoryDbTests.cs ├── SqlServerMemoryStoreTests.cs └── testsettings.json ├── Connectors.Memory.SqlServer ├── Connectors.Memory.SqlServer.csproj ├── ISqlServerClient.cs ├── NUGET.md ├── SqlServerClient.cs ├── SqlServerConfig.cs ├── SqlServerMemoryEntry.cs └── SqlServerMemoryStore.cs ├── Directory.Build.props ├── Directory.Build.targets ├── Directory.Packages.props ├── KernelMemory.MemoryStorage.SqlServer ├── KernelMemory.MemoryStorage.SqlServer.csproj ├── KernelMemoryBuilderExtensions.cs ├── NUGET.md ├── SqlServerConfig.cs ├── SqlServerMemory.cs └── SqlServerMemoryException.cs └── nuget.config /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Each line is a file pattern followed by one or more owners. 2 | 3 | # These owners will be the default owners for everything in 4 | # the repo. Unless a later match takes precedence. 5 | 6 | * @kbeaugrand -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # Maintain dependencies for GitHub Actions 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | interval: "daily" 8 | # Maintain dependencies for NuGet packages 9 | - package-ecosystem: "nuget" 10 | directory: "/" 11 | schedule: 12 | interval: "daily" 13 | ignore: 14 | - dependency-name: "System.*" 15 | update-types: ["version-update:semver-major"] 16 | - dependency-name: "Microsoft.Extensions.*" 17 | update-types: ["version-update:semver-major"] 18 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | What's new? 4 | 5 | - 6 | 7 | ## What kind of change does this PR introduce? 8 | 9 | - [ ] Bugfix 10 | - [ ] Feature 11 | - [ ] Code style update (formatting, local variables) 12 | - [ ] Refactoring (no functional changes, no api changes) 13 | - [ ] Build related changes 14 | - [ ] CI related changes 15 | - [ ] Documentation content changes 16 | - [ ] Tests 17 | - [ ] Other -------------------------------------------------------------------------------- /.github/workflows/build_test.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a .NET project 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-net 3 | 4 | name: Build & Test 5 | 6 | on: 7 | push: 8 | branches: [ "main" ] 9 | pull_request: 10 | branches: [ "main" ] 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | 18 | - uses: actions/checkout@v4 19 | 20 | - name: Setup .NET 21 | uses: actions/setup-dotnet@v4 22 | with: 23 | dotnet-version: 6.0.x 24 | 25 | - name: Restore dependencies 26 | run: dotnet restore 27 | working-directory: ./src/Connectors.Memory.SqlServer.Tests 28 | 29 | - name: Build 30 | run: dotnet build --no-restore 31 | working-directory: ./src/Connectors.Memory.SqlServer.Tests 32 | 33 | - name: Launch SQL Server with docker 34 | run: | 35 | docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=P@ssw0rd" \ 36 | -p 1433:1433 --name sql \ 37 | -d \ 38 | mcr.microsoft.com/mssql/server:2022-latest; 39 | sleep 30 40 | 41 | - name: Test 42 | run: dotnet test --no-build --verbosity normal 43 | working-directory: ./src/Connectors.Memory.SqlServer.Tests 44 | 45 | - name: Stop SQL Server 46 | run: docker stop sql -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a .NET project 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-net 3 | 4 | name: Create Release 5 | 6 | on: 7 | release: 8 | types: [published] 9 | 10 | jobs: 11 | Publish: 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | package: 16 | - Connectors.Memory.SqlServer 17 | - KernelMemory.MemoryStorage.SqlServer 18 | 19 | steps: 20 | - uses: actions/checkout@v4 21 | - name: Setup .NET 22 | uses: actions/setup-dotnet@v4 23 | with: 24 | dotnet-version: 6.0.x 25 | - name: Restore dependencies 26 | run: dotnet restore 27 | working-directory: ./src/${{ matrix.package }} 28 | - name: Build 29 | run: dotnet build --no-restore --configuration Release 30 | working-directory: ./src/${{ matrix.package }} 31 | - name: Pack 32 | run: dotnet pack --configuration Release /p:Version=${{ github.event.release.tag_name }} 33 | working-directory: ./src/${{ matrix.package }} 34 | - name: Push to NuGet 35 | run: | 36 | dotnet nuget push **/*.nupkg --source nuget.org --api-key ${{ secrets.NUGET_API_KEY }} --skip-duplicate 37 | working-directory: ./src/${{ matrix.package }} 38 | 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dotnet/.config 2 | 3 | ## Ignore Visual Studio temporary files, build results, and 4 | ## files generated by popular Visual Studio add-ons. 5 | ## 6 | ## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore 7 | 8 | # User-specific files 9 | *.rsuser 10 | *.suo 11 | *.user 12 | *.userosscache 13 | *.sln.docstates 14 | 15 | # User-specific files (MonoDevelop/Xamarin Studio) 16 | *.userprefs 17 | 18 | # Mono auto generated files 19 | mono_crash.* 20 | 21 | # Build results 22 | [Dd]ebug/ 23 | [Dd]ebugPublic/ 24 | [Rr]elease/ 25 | [Rr]eleases/ 26 | x64/ 27 | x86/ 28 | [Ww][Ii][Nn]32/ 29 | [Aa][Rr][Mm]/ 30 | [Aa][Rr][Mm]64/ 31 | bld/ 32 | [Bb]in/ 33 | [Oo]bj/ 34 | [Ll]og/ 35 | [Ll]ogs/ 36 | 37 | # Visual Studio 2015/2017 cache/options directory 38 | .vs/ 39 | # Uncomment if you have tasks that create the project's static files in wwwroot 40 | #wwwroot/ 41 | 42 | # Visual Studio 2017 auto generated files 43 | Generated\ Files/ 44 | 45 | # MSTest test Results 46 | [Tt]est[Rr]esult*/ 47 | [Bb]uild[Ll]og.* 48 | 49 | # NUnit 50 | *.VisualState.xml 51 | TestResult.xml 52 | nunit-*.xml 53 | 54 | # Build Results of an ATL Project 55 | [Dd]ebugPS/ 56 | [Rr]eleasePS/ 57 | dlldata.c 58 | 59 | # Benchmark Results 60 | BenchmarkDotNet.Artifacts/ 61 | 62 | # .NET Core 63 | project.lock.json 64 | project.fragment.lock.json 65 | artifacts/ 66 | 67 | # ASP.NET Scaffolding 68 | ScaffoldingReadMe.txt 69 | 70 | # StyleCop 71 | StyleCopReport.xml 72 | 73 | # Files built by Visual Studio 74 | *_i.c 75 | *_p.c 76 | *_h.h 77 | *.ilk 78 | *.meta 79 | *.obj 80 | *.iobj 81 | *.pch 82 | *.pdb 83 | *.ipdb 84 | *.pgc 85 | *.pgd 86 | *.rsp 87 | *.sbr 88 | *.tlb 89 | *.tli 90 | *.tlh 91 | *.tmp 92 | *.tmp_proj 93 | *_wpftmp.csproj 94 | *.log 95 | *.tlog 96 | *.vspscc 97 | *.vssscc 98 | .builds 99 | *.pidb 100 | *.svclog 101 | *.scc 102 | 103 | # Chutzpah Test files 104 | _Chutzpah* 105 | 106 | # Visual C++ cache files 107 | ipch/ 108 | *.aps 109 | *.ncb 110 | *.opendb 111 | *.opensdf 112 | *.sdf 113 | *.cachefile 114 | *.VC.db 115 | *.VC.VC.opendb 116 | 117 | # Visual Studio profiler 118 | *.psess 119 | *.vsp 120 | *.vspx 121 | *.sap 122 | 123 | # Visual Studio Trace Files 124 | *.e2e 125 | 126 | # TFS 2012 Local Workspace 127 | $tf/ 128 | 129 | # Guidance Automation Toolkit 130 | *.gpState 131 | 132 | # ReSharper is a .NET coding add-in 133 | _ReSharper*/ 134 | *.[Rr]e[Ss]harper 135 | *.DotSettings.user 136 | 137 | # TeamCity is a build add-in 138 | _TeamCity* 139 | 140 | # DotCover is a Code Coverage Tool 141 | *.dotCover 142 | 143 | # AxoCover is a Code Coverage Tool 144 | .axoCover/* 145 | !.axoCover/settings.json 146 | 147 | # Coverlet is a free, cross platform Code Coverage Tool 148 | coverage*.json 149 | coverage*.xml 150 | coverage*.info 151 | 152 | # Visual Studio code coverage results 153 | *.coverage 154 | *.coveragexml 155 | 156 | # NCrunch 157 | _NCrunch_* 158 | .*crunch*.local.xml 159 | nCrunchTemp_* 160 | 161 | # MightyMoose 162 | *.mm.* 163 | AutoTest.Net/ 164 | 165 | # Web workbench (sass) 166 | .sass-cache/ 167 | 168 | # Installshield output folder 169 | [Ee]xpress/ 170 | 171 | # DocProject is a documentation generator add-in 172 | DocProject/buildhelp/ 173 | DocProject/Help/*.HxT 174 | DocProject/Help/*.HxC 175 | DocProject/Help/*.hhc 176 | DocProject/Help/*.hhk 177 | DocProject/Help/*.hhp 178 | DocProject/Help/Html2 179 | DocProject/Help/html 180 | 181 | # Click-Once directory 182 | publish/ 183 | 184 | # Publish Web Output 185 | *.[Pp]ublish.xml 186 | *.azurePubxml 187 | # Note: Comment the next line if you want to checkin your web deploy settings, 188 | # but database connection strings (with potential passwords) will be unencrypted 189 | *.pubxml 190 | *.publishproj 191 | 192 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 193 | # checkin your Azure Web App publish settings, but sensitive information contained 194 | # in these scripts will be unencrypted 195 | PublishScripts/ 196 | 197 | # NuGet Packages 198 | *.nupkg 199 | # NuGet Symbol Packages 200 | *.snupkg 201 | # The packages folder can be ignored because of Package Restore 202 | **/[Pp]ackages/* 203 | # except build/, which is used as an MSBuild target. 204 | !**/[Pp]ackages/build/ 205 | # Uncomment if necessary however generally it will be regenerated when needed 206 | #!**/[Pp]ackages/repositories.config 207 | # NuGet v3's project.json files produces more ignorable files 208 | *.nuget.props 209 | *.nuget.targets 210 | 211 | # Microsoft Azure Build Output 212 | csx/ 213 | *.build.csdef 214 | 215 | # Microsoft Azure Emulator 216 | ecf/ 217 | rcf/ 218 | 219 | # Windows Store app package directories and files 220 | AppPackages/ 221 | BundleArtifacts/ 222 | Package.StoreAssociation.xml 223 | _pkginfo.txt 224 | *.appx 225 | *.appxbundle 226 | *.appxupload 227 | 228 | # Visual Studio cache files 229 | # files ending in .cache can be ignored 230 | *.[Cc]ache 231 | # but keep track of directories ending in .cache 232 | !?*.[Cc]ache/ 233 | 234 | # Others 235 | ClientBin/ 236 | ~$* 237 | *~ 238 | *.dbmdl 239 | *.dbproj.schemaview 240 | *.jfm 241 | *.pfx 242 | *.publishsettings 243 | orleans.codegen.cs 244 | 245 | # Including strong name files can present a security risk 246 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 247 | #*.snk 248 | 249 | # Since there are multiple workflows, uncomment next line to ignore bower_components 250 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 251 | #bower_components/ 252 | 253 | # RIA/Silverlight projects 254 | Generated_Code/ 255 | 256 | # Backup & report files from converting an old project file 257 | # to a newer Visual Studio version. Backup files are not needed, 258 | # because we have git ;-) 259 | _UpgradeReport_Files/ 260 | Backup*/ 261 | UpgradeLog*.XML 262 | UpgradeLog*.htm 263 | ServiceFabricBackup/ 264 | *.rptproj.bak 265 | 266 | # SQL Server files 267 | *.mdf 268 | *.ldf 269 | *.ndf 270 | 271 | # Business Intelligence projects 272 | *.rdl.data 273 | *.bim.layout 274 | *.bim_*.settings 275 | *.rptproj.rsuser 276 | *- [Bb]ackup.rdl 277 | *- [Bb]ackup ([0-9]).rdl 278 | *- [Bb]ackup ([0-9][0-9]).rdl 279 | 280 | # Microsoft Fakes 281 | FakesAssemblies/ 282 | 283 | # GhostDoc plugin setting file 284 | *.GhostDoc.xml 285 | 286 | # Node.js Tools for Visual Studio 287 | .ntvs_analysis.dat 288 | node_modules/ 289 | 290 | # Visual Studio 6 build log 291 | *.plg 292 | 293 | # Visual Studio 6 workspace options file 294 | *.opt 295 | 296 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 297 | *.vbw 298 | 299 | # Visual Studio 6 auto-generated project file (contains which files were open etc.) 300 | *.vbp 301 | 302 | # Visual Studio 6 workspace and project file (working project files containing files to include in project) 303 | *.dsw 304 | *.dsp 305 | 306 | # Visual Studio 6 technical files 307 | *.ncb 308 | *.aps 309 | 310 | # Visual Studio LightSwitch build output 311 | **/*.HTMLClient/GeneratedArtifacts 312 | **/*.DesktopClient/GeneratedArtifacts 313 | **/*.DesktopClient/ModelManifest.xml 314 | **/*.Server/GeneratedArtifacts 315 | **/*.Server/ModelManifest.xml 316 | _Pvt_Extensions 317 | 318 | # Paket dependency manager 319 | .paket/paket.exe 320 | paket-files/ 321 | 322 | # FAKE - F# Make 323 | .fake/ 324 | 325 | # CodeRush personal settings 326 | .cr/personal 327 | 328 | # Python Tools for Visual Studio (PTVS) 329 | __pycache__/ 330 | *.pyc 331 | 332 | # Cake - Uncomment if you are using it 333 | # tools/** 334 | # !tools/packages.config 335 | 336 | # Tabs Studio 337 | *.tss 338 | 339 | # Telerik's JustMock configuration file 340 | *.jmconfig 341 | 342 | # BizTalk build output 343 | *.btp.cs 344 | *.btm.cs 345 | *.odx.cs 346 | *.xsd.cs 347 | 348 | # OpenCover UI analysis results 349 | OpenCover/ 350 | 351 | # Azure Stream Analytics local run output 352 | ASALocalRun/ 353 | 354 | # MSBuild Binary and Structured Log 355 | *.binlog 356 | 357 | # NVidia Nsight GPU debugger configuration file 358 | *.nvuser 359 | 360 | # MFractors (Xamarin productivity tool) working folder 361 | .mfractor/ 362 | 363 | # Local History for Visual Studio 364 | .localhistory/ 365 | 366 | # Visual Studio History (VSHistory) files 367 | .vshistory/ 368 | 369 | # BeatPulse healthcheck temp database 370 | healthchecksdb 371 | 372 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 373 | MigrationBackup/ 374 | 375 | # Ionide (cross platform F# VS Code tools) working folder 376 | .ionide/ 377 | 378 | # Fody - auto-generated XML schema 379 | FodyWeavers.xsd 380 | 381 | # VS Code files for those working on multiple tools 382 | .vscode/* 383 | !.vscode/settings.json 384 | !.vscode/tasks.json 385 | !.vscode/launch.json 386 | !.vscode/extensions.json 387 | *.code-workspace 388 | 389 | # Local History for Visual Studio Code 390 | .history/ 391 | 392 | # Windows Installer files from build outputs 393 | *.cab 394 | *.msi 395 | *.msix 396 | *.msm 397 | *.msp 398 | 399 | # JetBrains Rider 400 | *.sln.iml 401 | *.tmp 402 | *.log 403 | *.bck 404 | *.tgz 405 | *.tar 406 | *.zip 407 | *.cer 408 | *.crt 409 | *.key 410 | *.pem 411 | 412 | .env 413 | certs/ 414 | launchSettings.json 415 | config.development.yaml 416 | *.development.config 417 | *.development.json 418 | .DS_Store 419 | .idea/ 420 | node_modules/ 421 | obj/ 422 | bin/ 423 | _dev/ 424 | .dev/ 425 | *.devis.* 426 | .vs/ 427 | *.user 428 | **/.vscode/chrome 429 | **/.vscode/.ropeproject/objectdb 430 | *.pyc 431 | .ipynb_checkpoints 432 | .jython_cache/ 433 | __pycache__/ 434 | .mypy_cache/ 435 | __pypackages__/ 436 | .pdm.toml 437 | global.json 438 | 439 | # doxfx 440 | **/DROP/ 441 | **/TEMP/ 442 | **/packages/ 443 | **/bin/ 444 | **/obj/ 445 | _site 446 | 447 | # Yarn 448 | .yarn 449 | .yarnrc.yml 450 | 451 | # Python Environments 452 | .env 453 | .venv 454 | .myenv 455 | env/ 456 | venv/ 457 | myvenv/ 458 | ENV/ 459 | 460 | # Python dist 461 | dist/ 462 | 463 | # Peristant storage 464 | data/qdrant 465 | data/chatstore* 466 | 467 | # Java build 468 | java/**/target 469 | java/.mvn/wrapper/maven-wrapper.jar 470 | 471 | # Java settings 472 | conf.properties 473 | 474 | # Playwright 475 | playwright-report/ 476 | 477 | # Static Web App deployment config 478 | swa-cli.config.json 479 | **/copilot-chat-app/webapp/build 480 | **/copilot-chat-app/webapp/node_modules 481 | **/copilot-chat-app/webapi/data/eng.traineddata 482 | 483 | # Semantic Kernel Tools 484 | /.semantic-kernel 485 | -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=733558 3 | // for the documentation about the tasks.json format 4 | "version": "2.0.0", 5 | "tasks": [ 6 | { 7 | "label": "build", 8 | "command": "dotnet", 9 | "type": "shell", 10 | "args": [ 11 | "build", 12 | // Ask dotnet build to generate full paths for file names. 13 | "/property:GenerateFullPaths=true", 14 | // Do not generate summary otherwise it leads to duplicate errors in Problems panel 15 | "/consoleloggerparameters:NoSummary", 16 | // Build the test project 17 | "${workspaceFolder}/src/Connectors.Memory.SqlServer.Tests/IntegrationTests.csproj" 18 | ], 19 | "group": "build", 20 | "presentation": { 21 | "reveal": "silent" 22 | }, 23 | "problemMatcher": "$msCompile" 24 | }, 25 | { 26 | "label": "test", 27 | "command": "dotnet", 28 | "args": [ 29 | "test", 30 | // Ask dotnet test to not build the project again and use the build output. 31 | "/p:SkipCompilerExecution=true", 32 | "/consoleloggerparameters:NoSummary", 33 | "${workspaceFolder}/src/Connectors.Memory.SqlServer.Tests/IntegrationTests.csproj" 34 | ], 35 | "type": "shell", 36 | "group": "test", 37 | "presentation": { 38 | "reveal": "silent" 39 | }, 40 | "problemMatcher": "$msCompile" 41 | }, 42 | { 43 | "label": "clean", 44 | "command": "dotnet", 45 | "type": "shell", 46 | "args": [ 47 | "clean", 48 | "/property:GenerateFullPaths=true", 49 | "/consoleloggerparameters:NoSummary", 50 | "${workspaceFolder}/src/Connectors.Memory.SqlServer.Tests/IntegrationTests.csproj" 51 | ], 52 | "group": "build", 53 | "presentation": { 54 | "reveal": "silent" 55 | }, 56 | "problemMatcher": "$msCompile" 57 | }, 58 | { 59 | "label": "build & test", 60 | "dependsOrder": "sequence", 61 | "dependsOn": [ 62 | "build", 63 | "test" 64 | ], 65 | "group": "build", 66 | "presentation": { 67 | "reveal": "silent" 68 | } 69 | } 70 | ] 71 | } -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Kevin BEAUGRAND 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Semantic Kernel & Kernel Memory - SQL Connector 2 | 3 | [![Build & Test](https://github.com/kbeaugrand/SemanticKernel.Connectors.Memory.SqlServer/actions/workflows/build_test.yml/badge.svg)](https://github.com/kbeaugrand/SemanticKernel.Connectors.Memory.SqlServer/actions/workflows/build_test.yml) 4 | [![Create Release](https://github.com/kbeaugrand/SemanticKernel.Connectors.Memory.SqlServer/actions/workflows/publish.yml/badge.svg)](https://github.com/kbeaugrand/SemanticKernel.Connectors.Memory.SqlServer/actions/workflows/publish.yml) 5 | [![Version](https://img.shields.io/github/v/release/kbeaugrand/SemanticKernel.Connectors.Memory.SqlServer)](https://img.shields.io/github/v/release/kbeaugrand/SemanticKernel.Connectors.Memory.SqlServer) 6 | [![License](https://img.shields.io/github/license/kbeaugrand/SemanticKernel.Connectors.Memory.SqlServer)](https://img.shields.io/github/v/release/kbeaugrand/SemanticKernel.Connectors.Memory.SqlServer) 7 | 8 | This is a connector for the [Semantic Kernel](https://aka.ms/semantic-kernel). 9 | 10 | > This package has been deprecated as it is legacy and is no longer maintained. 11 | 12 | It provides a connection to a SQL database for the Semantic Kernel for the memories. 13 | 14 | ## About Semantic Kernel 15 | 16 | **Semantic Kernel (SK)** is a lightweight SDK enabling integration of AI Large 17 | Language Models (LLMs) with conventional programming languages. The SK 18 | extensible programming model combines natural language **semantic functions**, 19 | traditional code **native functions**, and **embeddings-based memory** unlocking 20 | new potential and adding value to applications with AI. 21 | 22 | Please take a look at [Semantic Kernel](https://aka.ms/semantic-kernel) for more information. 23 | 24 | ## About Kernel Memory 25 | 26 | **Kernel Memory** (KM) is a multi-modal **AI Service** specialized in the efficient indexing of datasets through custom continuous data hybrid pipelines, with support for **Retrieval Augmented Generation (RAG)**, synthetic memory, prompt engineering, and custom semantic memory processing. 27 | 28 | 29 | ## Semantic Kernel Plugin 30 | 31 | To install this memory store, you need to add the required nuget package to your project: 32 | 33 | ```dotnetcli 34 | dotnet add package SemanticKernel.Connectors.Memory.SqlServer 35 | ``` 36 | 37 | ## Usage 38 | 39 | To add your SQL Server memory connector, add the following statements to your kernel initialization code: 40 | 41 | ```csharp 42 | using SemanticKernel.Connectors.Memory.SqlServer; 43 | 44 | var kernel = Kernel.CreateBuilder() 45 | .Build(); 46 | 47 | var sqlMemoryStore = await SqlServerMemoryStore.ConnectAsync(connectionString: "Server=.;Database=SK;Trusted_Connection=True;"); 48 | var semanticTextMemory = new SemanticTextMemory(sqlMemoryStore, kernel.GetRequiredService()); 49 | 50 | kernel.ImportPluginFromObject(new TextMemoryPlugin(semanticTextMemory)); 51 | 52 | ``` 53 | 54 | The memory store will populate all the needed tables during startup and let you focus on the development of your plugin. 55 | 56 | ## Kernel Memory Plugin 57 | 58 | ```bash 59 | dotnet add package KernelMemory.MemoryStorage.SqlServer 60 | ``` 61 | 62 | ## Usage 63 | 64 | To add your SQL Server memory connector, add the following statements to your kernel memory initialization code: 65 | 66 | ```csharp 67 | using SemanticKernel.Connectors.Memory.SqlServer; 68 | ... 69 | var memory = new KernelMemoryBuilder() 70 | .WithOpenAIDefaults(Env.Var("OPENAI_API_KEY")) 71 | .WithSqlServerMemoryDb("YouSqlConnectionString") 72 | .Build(); 73 | ``` 74 | 75 | Then you can use the memory store to import documents and ask questions: 76 | 77 | ```csharp 78 | // Import a file 79 | await memory.ImportDocumentAsync("meeting-transcript.docx", tags: new() { { "user", "Blake" } }); 80 | 81 | // Import multiple files and apply multiple tags 82 | await memory.ImportDocumentAsync(new Document("file001") 83 | .AddFile("business-plan.docx") 84 | .AddFile("project-timeline.pdf") 85 | .AddTag("user", "Blake") 86 | .AddTag("collection", "business") 87 | .AddTag("collection", "plans") 88 | .AddTag("fiscalYear", "2023")); 89 | 90 | var answer1 = await memory.AskAsync("How many people attended the meeting?"); 91 | 92 | var answer2 = await memory.AskAsync("what's the project timeline?", filter: new MemoryFilter().ByTag("user", "Blake")); 93 | ``` 94 | 95 | The memory store will populate all the needed tables during startup and let you focus on the development of your plugin. 96 | 97 | ## License 98 | 99 | This project is licensed under the [MIT License](LICENSE). 100 | -------------------------------------------------------------------------------- /SemanticKernel.Connectors.Memory.SqlServer.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 17 4 | VisualStudioVersion = 17.5.002.0 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{5569512D-4D33-4DD8-A027-6AF4236546CB}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.SqlServer", "src\Connectors.Memory.SqlServer\Connectors.Memory.SqlServer.csproj", "{A669B285-4F66-469B-B79D-FDCA0E81BC2A}" 9 | EndProject 10 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "IntegrationTests", "src\Connectors.Memory.SqlServer.Tests\IntegrationTests.csproj", "{8CD5A6C2-0BF0-4A14-9F6F-0BDCD986D94F}" 11 | EndProject 12 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KernelMemory.MemoryStorage.SqlServer", "src\KernelMemory.MemoryStorage.SqlServer\KernelMemory.MemoryStorage.SqlServer.csproj", "{F59B1392-1175-4178-B652-3A1CDD37D3EF}" 13 | EndProject 14 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{0B7F0129-06C6-4408-90CF-16D9BA9C8CEC}" 15 | ProjectSection(SolutionItems) = preProject 16 | .gitignore = .gitignore 17 | .github\workflows\build_test.yml = .github\workflows\build_test.yml 18 | LICENSE.md = LICENSE.md 19 | nuget\nuget-package.props = nuget\nuget-package.props 20 | .github\workflows\publish.yml = .github\workflows\publish.yml 21 | README.md = README.md 22 | EndProjectSection 23 | EndProject 24 | Global 25 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 26 | Debug|Any CPU = Debug|Any CPU 27 | Release|Any CPU = Release|Any CPU 28 | EndGlobalSection 29 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 30 | {A669B285-4F66-469B-B79D-FDCA0E81BC2A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 31 | {A669B285-4F66-469B-B79D-FDCA0E81BC2A}.Debug|Any CPU.Build.0 = Debug|Any CPU 32 | {A669B285-4F66-469B-B79D-FDCA0E81BC2A}.Release|Any CPU.ActiveCfg = Release|Any CPU 33 | {A669B285-4F66-469B-B79D-FDCA0E81BC2A}.Release|Any CPU.Build.0 = Release|Any CPU 34 | {8CD5A6C2-0BF0-4A14-9F6F-0BDCD986D94F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 35 | {8CD5A6C2-0BF0-4A14-9F6F-0BDCD986D94F}.Debug|Any CPU.Build.0 = Debug|Any CPU 36 | {8CD5A6C2-0BF0-4A14-9F6F-0BDCD986D94F}.Release|Any CPU.ActiveCfg = Release|Any CPU 37 | {8CD5A6C2-0BF0-4A14-9F6F-0BDCD986D94F}.Release|Any CPU.Build.0 = Release|Any CPU 38 | {F59B1392-1175-4178-B652-3A1CDD37D3EF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 39 | {F59B1392-1175-4178-B652-3A1CDD37D3EF}.Debug|Any CPU.Build.0 = Debug|Any CPU 40 | {F59B1392-1175-4178-B652-3A1CDD37D3EF}.Release|Any CPU.ActiveCfg = Release|Any CPU 41 | {F59B1392-1175-4178-B652-3A1CDD37D3EF}.Release|Any CPU.Build.0 = Release|Any CPU 42 | EndGlobalSection 43 | GlobalSection(SolutionProperties) = preSolution 44 | HideSolutionNode = FALSE 45 | EndGlobalSection 46 | GlobalSection(NestedProjects) = preSolution 47 | {A669B285-4F66-469B-B79D-FDCA0E81BC2A} = {5569512D-4D33-4DD8-A027-6AF4236546CB} 48 | {8CD5A6C2-0BF0-4A14-9F6F-0BDCD986D94F} = {5569512D-4D33-4DD8-A027-6AF4236546CB} 49 | {F59B1392-1175-4178-B652-3A1CDD37D3EF} = {5569512D-4D33-4DD8-A027-6AF4236546CB} 50 | EndGlobalSection 51 | GlobalSection(ExtensibilityGlobals) = postSolution 52 | SolutionGuid = {A32F435C-6A25-4E24-8FFC-FB4F8A235B8C} 53 | EndGlobalSection 54 | EndGlobal 55 | -------------------------------------------------------------------------------- /nuget/nuget-package.props: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 0.0.1-alpha 5 | 6 | Debug;Release;Publish 7 | true 8 | 9 | 10 | Kevin BEAUGRAND 11 | 12 | Semantic Kernel 13 | Empowers app owners to integrate cutting-edge LLM technology quickly and easily into their apps. 14 | AI, Artificial Intelligence, SDK 15 | $(AssemblyName) 16 | 17 | 18 | MIT 19 | © Kevin BEAUGRAND. All rights reserved. 20 | https://github.com/kbeaugrand/SemanticKernel.Connectors.Memory.SqlServer 21 | https://github.com/kbeaugrand/SemanticKernel.Connectors.Memory.SqlServer 22 | true 23 | 24 | 25 | NUGET.md 26 | 27 | 28 | true 29 | snupkg 30 | 31 | 32 | bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | true 48 | 49 | 50 | -------------------------------------------------------------------------------- /src/Connectors.Memory.SqlServer.Tests/.editorconfig: -------------------------------------------------------------------------------- 1 | # Suppressing errors for Test projects under dotnet folder 2 | [*.cs] 3 | dotnet_diagnostic.CA2007.severity = none # Do not directly await a Task 4 | dotnet_diagnostic.VSTHRD111.severity = none # Use .ConfigureAwait(bool) is hidden by default, set to none to prevent IDE from changing on autosave 5 | 6 | -------------------------------------------------------------------------------- /src/Connectors.Memory.SqlServer.Tests/Extensions/AsyncEnumerable.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | using System.Collections.Generic; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | 7 | #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously 8 | #pragma warning disable CA1510 // Use 'ArgumentNullException.ThrowIfNull' (.NET 8) 9 | 10 | // Used for compatibility with System.Linq.Async Nuget pkg 11 | namespace System.Linq; 12 | 13 | internal static class AsyncEnumerable 14 | { 15 | public static IAsyncEnumerable Empty() => EmptyAsyncEnumerable.Instance; 16 | 17 | public static IEnumerable ToEnumerable(this IAsyncEnumerable source, CancellationToken cancellationToken = default) 18 | { 19 | var enumerator = source.GetAsyncEnumerator(cancellationToken); 20 | try 21 | { 22 | while (enumerator.MoveNextAsync().AsTask().GetAwaiter().GetResult()) 23 | { 24 | yield return enumerator.Current; 25 | } 26 | } 27 | finally 28 | { 29 | enumerator.DisposeAsync().AsTask().GetAwaiter().GetResult(); 30 | } 31 | } 32 | 33 | public static async IAsyncEnumerable ToAsyncEnumerable(this IEnumerable source) 34 | { 35 | foreach (var item in source) 36 | { 37 | yield return item; 38 | } 39 | } 40 | 41 | public static async ValueTask FirstOrDefaultAsync(this IAsyncEnumerable source, CancellationToken cancellationToken = default) 42 | { 43 | await foreach (var item in source.WithCancellation(cancellationToken).ConfigureAwait(false)) 44 | { 45 | return item; 46 | } 47 | 48 | return default; 49 | } 50 | 51 | public static async ValueTask LastOrDefaultAsync(this IAsyncEnumerable source, CancellationToken cancellationToken = default) 52 | { 53 | var last = default(T)!; // NB: Only matters when hasLast is set to true. 54 | var hasLast = false; 55 | 56 | await foreach (var item in source.WithCancellation(cancellationToken).ConfigureAwait(false)) 57 | { 58 | hasLast = true; 59 | last = item; 60 | } 61 | 62 | return hasLast ? last! : default; 63 | } 64 | 65 | public static async ValueTask> ToListAsync(this IAsyncEnumerable source, CancellationToken cancellationToken = default) 66 | { 67 | var result = new List(); 68 | 69 | await foreach (var item in source.WithCancellation(cancellationToken).ConfigureAwait(false)) 70 | { 71 | result.Add(item); 72 | } 73 | 74 | return result; 75 | } 76 | 77 | public static async ValueTask ContainsAsync(this IAsyncEnumerable source, T value) 78 | { 79 | await foreach (var item in source.ConfigureAwait(false)) 80 | { 81 | if (EqualityComparer.Default.Equals(item, value)) 82 | { 83 | return true; 84 | } 85 | } 86 | 87 | return false; 88 | } 89 | 90 | public static async ValueTask CountAsync(this IAsyncEnumerable source, CancellationToken cancellationToken = default) 91 | { 92 | int count = 0; 93 | await foreach (var _ in source.WithCancellation(cancellationToken).ConfigureAwait(false)) 94 | { 95 | checked { count++; } 96 | } 97 | 98 | return count; 99 | } 100 | 101 | /// 102 | /// Determines whether any element of an async-enumerable sequence satisfies a condition. 103 | /// 104 | /// The type of the elements in the source sequence. 105 | /// An async-enumerable sequence whose elements to apply the predicate to. 106 | /// A function to test each element for a condition. 107 | /// The optional cancellation token to be used for cancelling the sequence at any time. 108 | /// An async-enumerable sequence containing a single element determining whether any elements in the source sequence pass the test in the specified predicate. 109 | /// or is null. 110 | /// The return type of this operator differs from the corresponding operator on IEnumerable in order to retain asynchronous behavior. 111 | public static ValueTask AnyAsync(this IAsyncEnumerable source, Func predicate, CancellationToken cancellationToken = default) 112 | { 113 | if (source == null) 114 | { 115 | throw new ArgumentNullException(nameof(source)); 116 | } 117 | 118 | if (predicate == null) 119 | { 120 | throw new ArgumentNullException(nameof(predicate)); 121 | } 122 | 123 | return Core(source, predicate, cancellationToken); 124 | 125 | static async ValueTask Core(IAsyncEnumerable source, Func predicate, CancellationToken cancellationToken) 126 | { 127 | await foreach (var item in source.WithCancellation(cancellationToken).ConfigureAwait(false)) 128 | { 129 | if (predicate(item)) 130 | { 131 | return true; 132 | } 133 | } 134 | 135 | return false; 136 | } 137 | } 138 | 139 | private sealed class EmptyAsyncEnumerable : IAsyncEnumerable, IAsyncEnumerator 140 | { 141 | public static readonly EmptyAsyncEnumerable Instance = new(); 142 | public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) => this; 143 | public ValueTask MoveNextAsync() => new(false); 144 | public T Current => default!; 145 | public ValueTask DisposeAsync() => default; 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /src/Connectors.Memory.SqlServer.Tests/IntegrationTests.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Connectors.Memory.SqlServer.Tests 5 | Connectors.Memory.SqlServer.Tests 6 | net6.0 7 | LatestMajor 8 | true 9 | false 10 | CA2007,VSTHRD111;SKEXP0003 11 | b7762d10-e29b-4bb1-8b74-b6d69a667dd4 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | runtime; build; native; contentfiles; analyzers; buildtransitive 26 | all 27 | 28 | 29 | runtime; build; native; contentfiles; analyzers; buildtransitive 30 | all 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | Always 46 | 47 | 48 | Always 49 | 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /src/Connectors.Memory.SqlServer.Tests/SqlServerMemoryDbTests.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Kevin BEAUGRAND. All rights reserved. 2 | 3 | using KernelMemory.MemoryStorage.SqlServer; 4 | using Microsoft.Extensions.Configuration; 5 | using Microsoft.KernelMemory; 6 | using Microsoft.KernelMemory.AI; 7 | using Microsoft.KernelMemory.MemoryStorage; 8 | using Moq; 9 | using SemanticKernel.IntegrationTests.Connectors.Memory.SqlServer; 10 | using System; 11 | using System.Collections; 12 | using System.Linq; 13 | using System.Threading; 14 | using System.Threading.Tasks; 15 | using Xunit; 16 | 17 | namespace Connectors.Memory.SqlServer.Tests; 18 | 19 | /// 20 | /// Represents a SQL Server memory store test class. 21 | /// 22 | public class SqlServerMemoryDbTests : IAsyncLifetime 23 | { 24 | /// 25 | /// The SQL Server configuration. 26 | /// 27 | private SqlServerConfig _config; 28 | 29 | /// 30 | /// The text embedding generator mock. 31 | /// 32 | private Mock _textEmbeddingGeneratorMock = new Mock(); 33 | 34 | /// 35 | /// Creates a new instance of the class. 36 | /// 37 | /// 38 | private SqlServerMemory CreateMemoryDb() 39 | { 40 | return new SqlServerMemory(_config, _textEmbeddingGeneratorMock.Object); 41 | } 42 | 43 | /// 44 | public Task InitializeAsync() 45 | { 46 | // Load configuration 47 | var configuration = new ConfigurationBuilder() 48 | .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) 49 | .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) 50 | .AddEnvironmentVariables() 51 | .AddUserSecrets() 52 | .Build(); 53 | 54 | var connectionString = configuration["SqlServer:ConnectionString"]; 55 | 56 | if (string.IsNullOrWhiteSpace(connectionString)) 57 | { 58 | throw new ArgumentNullException("SqlServer memory connection string is not configured"); 59 | } 60 | 61 | this._config = new SqlServerConfig 62 | { 63 | ConnectionString = connectionString, 64 | Schema = "ai" 65 | }; 66 | 67 | return Task.CompletedTask; 68 | } 69 | 70 | /// 71 | public async Task DisposeAsync() 72 | { 73 | var memoryDb = this.CreateMemoryDb(); 74 | 75 | foreach (var item in await memoryDb.GetIndexesAsync(CancellationToken.None).ConfigureAwait(false)) 76 | { 77 | await memoryDb.DeleteIndexAsync(item, CancellationToken.None).ConfigureAwait(false); 78 | } 79 | } 80 | 81 | /// 82 | /// Verify that get indexes should return newly created index. 83 | /// 84 | /// 85 | [Fact] 86 | public async Task GetIndexesShouldReturnNewlyCreatedIndexTestAsync() 87 | { 88 | var memoryDb = this.CreateMemoryDb(); 89 | var collectionName = Guid.NewGuid().ToString(); 90 | 91 | await memoryDb.CreateIndexAsync(collectionName, 1536, CancellationToken.None).ConfigureAwait(true); 92 | 93 | var collections = await memoryDb.GetIndexesAsync(CancellationToken.None).ConfigureAwait(true); 94 | 95 | Assert.NotNull(collections); 96 | Assert.Single(collections); 97 | Assert.Equal(collectionName, collections.First()); 98 | } 99 | 100 | 101 | /// 102 | /// Verify that get indexes should not return newly deleted index. 103 | /// 104 | /// 105 | [Fact] 106 | public async Task GetIndexesShouldNotReturnNewlyDeletedIndexTestAsync() 107 | { 108 | var memoryDb = this.CreateMemoryDb(); 109 | var collectionName = Guid.NewGuid().ToString(); 110 | 111 | await memoryDb.CreateIndexAsync(collectionName, 1536, CancellationToken.None).ConfigureAwait(true); 112 | 113 | var collections = await memoryDb.GetIndexesAsync(CancellationToken.None).ConfigureAwait(true); 114 | 115 | Assert.NotNull(collections); 116 | Assert.Single(collections); 117 | Assert.Equal(collectionName, collections.First()); 118 | 119 | await memoryDb.DeleteIndexAsync(collectionName, CancellationToken.None).ConfigureAwait(true); 120 | 121 | collections = await memoryDb.GetIndexesAsync(CancellationToken.None).ConfigureAwait(true); 122 | 123 | Assert.NotNull(collections); 124 | Assert.Empty(collections); 125 | } 126 | 127 | /// 128 | /// Verify that create index should not throw exception if index already exists. 129 | /// 130 | /// 131 | [Fact] 132 | public async Task CreateIndexShouldNotThrowExceptionIfIndexAlreadyExistsTestAsync() 133 | { 134 | var memoryDb = this.CreateMemoryDb(); 135 | var collectionName = Guid.NewGuid().ToString(); 136 | 137 | await memoryDb.CreateIndexAsync(collectionName, 1536, CancellationToken.None).ConfigureAwait(true); 138 | 139 | var collections = await memoryDb.GetIndexesAsync(CancellationToken.None).ConfigureAwait(true); 140 | 141 | Assert.NotNull(collections); 142 | Assert.Single(collections); 143 | Assert.Equal(collectionName, collections.First()); 144 | 145 | await memoryDb.CreateIndexAsync(collectionName, 1536, CancellationToken.None).ConfigureAwait(true); 146 | 147 | collections = await memoryDb.GetIndexesAsync(CancellationToken.None).ConfigureAwait(true); 148 | 149 | Assert.NotNull(collections); 150 | Assert.Single(collections); 151 | Assert.Equal(collectionName, collections.First()); 152 | } 153 | 154 | /// 155 | /// Test that upsert record should pass. 156 | /// 157 | /// 158 | [Fact] 159 | public async Task UpsertRecordShouldPassAsync() 160 | { 161 | var memoryDb = this.CreateMemoryDb(); 162 | var collectionName = Guid.NewGuid().ToString(); 163 | 164 | await memoryDb.CreateIndexAsync(collectionName, 1536, CancellationToken.None).ConfigureAwait(true); 165 | 166 | var record = new MemoryRecord() 167 | { 168 | Id = Guid.NewGuid().ToString(), 169 | Vector = new float[] { 1, 2, 3 } 170 | }; 171 | 172 | await memoryDb.UpsertAsync(collectionName, record, CancellationToken.None).ConfigureAwait(true); 173 | } 174 | 175 | /// 176 | /// Test that upsert record should pass. 177 | /// 178 | /// 179 | [Fact] 180 | public async Task GetListAsyncShouldReturnTheStoredRecordsAsync() 181 | { 182 | var memoryDb = this.CreateMemoryDb(); 183 | var collectionName = Guid.NewGuid().ToString(); 184 | 185 | await memoryDb.CreateIndexAsync(collectionName, 1536, CancellationToken.None).ConfigureAwait(true); 186 | 187 | var record = new MemoryRecord() 188 | { 189 | Id = "test", 190 | Vector = new float[] { 1, 2, 3 } 191 | }; 192 | 193 | await memoryDb.UpsertAsync(collectionName, record, CancellationToken.None).ConfigureAwait(true); 194 | 195 | var results = await memoryDb.GetListAsync(collectionName, null, limit: 10, withEmbeddings: true, CancellationToken.None).ToListAsync().ConfigureAwait(true); 196 | 197 | Assert.NotNull(results); 198 | Assert.Single(results); 199 | Assert.Equal("test", results.First().Id); 200 | } 201 | 202 | /// 203 | /// Test that existing record can be removed. 204 | /// 205 | /// 206 | [Fact] 207 | public async Task ExistingRecordCanBeRemovedAsync() 208 | { 209 | var memoryDb = this.CreateMemoryDb(); 210 | var collectionName = Guid.NewGuid().ToString(); 211 | 212 | await memoryDb.CreateIndexAsync(collectionName, 1536, CancellationToken.None).ConfigureAwait(true); 213 | 214 | var record = new MemoryRecord() 215 | { 216 | Id = "test", 217 | Vector = new float[] { 1, 2, 3 } 218 | }; 219 | 220 | await memoryDb.UpsertAsync(collectionName, record, CancellationToken.None).ConfigureAwait(true); 221 | 222 | var results = await memoryDb.GetListAsync(collectionName, null, limit: 10, withEmbeddings: true, CancellationToken.None).ToListAsync().ConfigureAwait(true); 223 | 224 | Assert.NotNull(results); 225 | Assert.Single(results); 226 | Assert.Equal("test", results.First().Id); 227 | 228 | await memoryDb.DeleteAsync(collectionName, record, CancellationToken.None).ConfigureAwait(true); 229 | 230 | results = await memoryDb.GetListAsync(collectionName, null, limit: 10, withEmbeddings: true, CancellationToken.None).ToListAsync().ConfigureAwait(true); 231 | 232 | Assert.NotNull(results); 233 | Assert.Empty(results); 234 | } 235 | 236 | /// 237 | /// Test that existing record can be updated 238 | /// 239 | /// 240 | [Fact] 241 | public async Task UpsertShouldUpdateRecordTestAsync() 242 | { 243 | var memoryDb = this.CreateMemoryDb(); 244 | var collectionName = Guid.NewGuid().ToString(); 245 | 246 | await memoryDb.CreateIndexAsync(collectionName, 1536, CancellationToken.None).ConfigureAwait(true); 247 | 248 | var record = new MemoryRecord() 249 | { 250 | Id = "test", 251 | Vector = new float[] { 1, 2, 3 } 252 | }; 253 | 254 | await memoryDb.UpsertAsync(collectionName, record, CancellationToken.None).ConfigureAwait(true); 255 | 256 | var results = await memoryDb.GetListAsync(collectionName, null, limit: 10, withEmbeddings: true, CancellationToken.None).ToListAsync().ConfigureAwait(true); 257 | 258 | Assert.NotNull(results); 259 | Assert.Single(results); 260 | Assert.Equal("test", results.First().Id); 261 | 262 | record.Vector = new float[] { 4, 5, 6 }; 263 | 264 | await memoryDb.UpsertAsync(collectionName, record, CancellationToken.None).ConfigureAwait(true); 265 | 266 | results = await memoryDb.GetListAsync(collectionName, null, limit: 10, withEmbeddings: true, CancellationToken.None).ToListAsync().ConfigureAwait(true); 267 | 268 | Assert.NotNull(results); 269 | Assert.Single(results); 270 | Assert.Equal("test", results.First().Id); 271 | Assert.Equal(4, results.First().Vector.Data.ToArray()[0]); 272 | Assert.Equal(5, results.First().Vector.Data.ToArray()[1]); 273 | Assert.Equal(6, results.First().Vector.Data.ToArray()[2]); 274 | } 275 | 276 | [Fact] 277 | public async Task GetListShouldReturnFilteredRecordsTestAsync() 278 | { 279 | var memoryDb = this.CreateMemoryDb(); 280 | var collectionName = Guid.NewGuid().ToString(); 281 | 282 | await memoryDb.CreateIndexAsync(collectionName, 1536, CancellationToken.None).ConfigureAwait(true); 283 | 284 | var record1 = new MemoryRecord() 285 | { 286 | Id = "record1", 287 | Vector = new float[] { 1, 2, 3 }, 288 | Tags = new TagCollection() 289 | { 290 | { "test", "value1" } 291 | } 292 | }; 293 | 294 | await memoryDb.UpsertAsync(collectionName, record1, CancellationToken.None).ConfigureAwait(true); 295 | 296 | var record2 = new MemoryRecord() 297 | { 298 | Id = "record2", 299 | Vector = new float[] { 1, 2, 3 }, 300 | Tags = new TagCollection() 301 | { 302 | { "test", "value2" } 303 | } 304 | }; 305 | 306 | await memoryDb.UpsertAsync(collectionName, record2, CancellationToken.None).ConfigureAwait(true); 307 | 308 | var results = await memoryDb.GetListAsync(collectionName, new[] { new MemoryFilter().ByTag("test", "value1") }, limit: 10, withEmbeddings: true, CancellationToken.None).ToListAsync().ConfigureAwait(true); 309 | 310 | Assert.NotNull(results); 311 | Assert.Single(results); 312 | Assert.Equal("record1", results.First().Id); 313 | } 314 | 315 | /// 316 | /// Test that get similar list should return expected results. 317 | /// 318 | /// 319 | [Fact] 320 | public async Task GetSimilarListShouldReturnExpectedAsync() 321 | { 322 | // Arrange 323 | var memoryDb = this.CreateMemoryDb(); 324 | 325 | var compareEmbedding = new ReadOnlyMemory(new float[] { 1, 1, 1 }); 326 | string collection = Guid.NewGuid().ToString(); 327 | await memoryDb.CreateIndexAsync(collection, 1536); 328 | int i = 0; 329 | 330 | MemoryRecord testRecord = new MemoryRecord() 331 | { 332 | Id = "test" + i, 333 | Vector = new float[] { 1, 1, 1 } 334 | }; 335 | 336 | _ = await memoryDb.UpsertAsync(collection, testRecord); 337 | 338 | i++; 339 | testRecord = new MemoryRecord() 340 | { 341 | Id = "test" + i, 342 | Vector = new float[] { -1, -1, -1 } 343 | }; 344 | _ = await memoryDb.UpsertAsync(collection, testRecord); 345 | 346 | i++; 347 | testRecord = new MemoryRecord() 348 | { 349 | Id = "test" + i, 350 | Vector = new float[] { 1, 2, 3 } 351 | }; 352 | _ = await memoryDb.UpsertAsync(collection, testRecord); 353 | 354 | i++; 355 | testRecord = new MemoryRecord() 356 | { 357 | Id = "test" + i, 358 | Vector = new float[] { -1, -2, -3 } 359 | }; 360 | _ = await memoryDb.UpsertAsync(collection, testRecord); 361 | 362 | i++; 363 | testRecord = new MemoryRecord() 364 | { 365 | Id = "test" + i, 366 | Vector = new float[] { 1, -1, -2 } 367 | }; 368 | _ = await memoryDb.UpsertAsync(collection, testRecord); 369 | 370 | _ = this._textEmbeddingGeneratorMock.Setup(x => x.GenerateEmbeddingAsync(It.IsAny(), It.IsAny())) 371 | .ReturnsAsync(compareEmbedding); 372 | 373 | // Act 374 | double threshold = .75; 375 | var topNResults = memoryDb.GetSimilarListAsync(collection, "Sample", limit: 1, minRelevance: threshold) 376 | .ToEnumerable() 377 | .ToArray(); 378 | 379 | // Assert 380 | Assert.NotNull(topNResults); 381 | Assert.Single(topNResults); 382 | 383 | Assert.Equal("test0", topNResults[0].Item1.Id); 384 | Assert.True(topNResults[0].Item2 >= threshold); 385 | } 386 | 387 | /// 388 | /// Test that get similar list should return expected results. 389 | /// 390 | /// 391 | [Fact] 392 | public async Task GetSimilarListShouldReturnExpectedWithFiltersAsync() 393 | { 394 | // Arrange 395 | var memoryDb = this.CreateMemoryDb(); 396 | 397 | var compareEmbedding = new ReadOnlyMemory(new float[] { 1, 1, 1 }); 398 | string collection = Guid.NewGuid().ToString(); 399 | await memoryDb.CreateIndexAsync(collection, 1536); 400 | int i = 0; 401 | 402 | MemoryRecord testRecord = new MemoryRecord() 403 | { 404 | Id = "test" + i, 405 | Vector = new float[] { 1, 1, 1 } 406 | }; 407 | 408 | testRecord.Tags.Add("test", "record0"); 409 | 410 | _ = await memoryDb.UpsertAsync(collection, testRecord); 411 | 412 | i++; 413 | testRecord = new MemoryRecord() 414 | { 415 | Id = "test" + i, 416 | Vector = new float[] { -1, -1, -1 } 417 | }; 418 | testRecord.Tags.Add("test", "record1"); 419 | _ = await memoryDb.UpsertAsync(collection, testRecord); 420 | 421 | i++; 422 | testRecord = new MemoryRecord() 423 | { 424 | Id = "test" + i, 425 | Vector = new float[] { 1, 2, 3 } 426 | }; 427 | testRecord.Tags.Add("test", "record2"); 428 | _ = await memoryDb.UpsertAsync(collection, testRecord); 429 | 430 | i++; 431 | testRecord = new MemoryRecord() 432 | { 433 | Id = "test" + i, 434 | Vector = new float[] { -1, -2, -3 } 435 | }; 436 | testRecord.Tags.Add("test", "record3"); 437 | _ = await memoryDb.UpsertAsync(collection, testRecord); 438 | 439 | i++; 440 | testRecord = new MemoryRecord() 441 | { 442 | Id = "test" + i, 443 | Vector = new float[] { 1, -1, -2 } 444 | }; 445 | testRecord.Tags.Add("test", "record4"); 446 | _ = await memoryDb.UpsertAsync(collection, testRecord); 447 | 448 | _ = this._textEmbeddingGeneratorMock.Setup(x => x.GenerateEmbeddingAsync(It.IsAny(), It.IsAny())) 449 | .ReturnsAsync(compareEmbedding); 450 | 451 | var filter = new MemoryFilter().ByTag("test", "record0"); 452 | 453 | // Act 454 | double threshold = .75; 455 | var topNResults = memoryDb.GetSimilarListAsync( 456 | index: collection, 457 | text: "Sample", 458 | limit: 4, 459 | minRelevance: threshold, 460 | filters: new[] { filter }) 461 | .ToEnumerable() 462 | .ToArray(); 463 | 464 | // Assert 465 | Assert.NotNull(topNResults); 466 | Assert.Single(topNResults); 467 | 468 | Assert.Equal("test0", topNResults[0].Item1.Id); 469 | Assert.True(topNResults[0].Item2 >= threshold); 470 | } 471 | 472 | /// 473 | /// Test that get similar list should return expected results. 474 | /// 475 | /// 476 | [Fact] 477 | public async Task GetSimilarListShouldNotReturnExpectedWithFiltersAsync() 478 | { 479 | // Arrange 480 | var memoryDb = this.CreateMemoryDb(); 481 | 482 | var compareEmbedding = new ReadOnlyMemory(new float[] { 1, 1, 1 }); 483 | string collection = Guid.NewGuid().ToString(); 484 | await memoryDb.CreateIndexAsync(collection, 1536); 485 | int i = 0; 486 | 487 | MemoryRecord testRecord = new MemoryRecord() 488 | { 489 | Id = "test" + i, 490 | Vector = new float[] { 1, 1, 1 } 491 | }; 492 | 493 | testRecord.Tags.Add("test", "record0"); 494 | 495 | _ = await memoryDb.UpsertAsync(collection, testRecord); 496 | 497 | var filter = new MemoryFilter().ByTag("test", "record1"); 498 | 499 | _ = this._textEmbeddingGeneratorMock.Setup(x => x.GenerateEmbeddingAsync(It.IsAny(), It.IsAny())) 500 | .ReturnsAsync(compareEmbedding); 501 | 502 | // Act 503 | double threshold = -1; 504 | var topNResults = memoryDb.GetSimilarListAsync( 505 | index: collection, 506 | text: "Sample", 507 | limit: 4, 508 | minRelevance: threshold, 509 | filters: new[] { filter }) 510 | .ToEnumerable() 511 | .ToArray(); 512 | 513 | // Assert 514 | Assert.NotNull(topNResults); 515 | Assert.Empty(topNResults); 516 | } 517 | 518 | /// 519 | /// Test that get similar list should return expected results. 520 | /// 521 | /// 522 | [Fact] 523 | public async Task GetSimilarListShouldNotReturnExpectedWithFiltersWithANDClauseAsync() 524 | { 525 | // Arrange 526 | var memoryDb = this.CreateMemoryDb(); 527 | 528 | var compareEmbedding = new ReadOnlyMemory(new float[] { 1, 1, 1 }); 529 | string collection = Guid.NewGuid().ToString(); 530 | await memoryDb.CreateIndexAsync(collection, 1536); 531 | int i = 0; 532 | 533 | MemoryRecord testRecord = new MemoryRecord() 534 | { 535 | Id = "test" + i++, 536 | Vector = new float[] { 1, 1, 1 } 537 | }; 538 | 539 | testRecord.Tags.Add("test", "record0"); 540 | testRecord.Tags.Add("test", "test"); 541 | 542 | _ = await memoryDb.UpsertAsync(collection, testRecord); 543 | 544 | testRecord = new MemoryRecord() 545 | { 546 | Id = "test" + i, 547 | Vector = new float[] { 1, 1, 1 } 548 | }; 549 | 550 | testRecord.Tags.Add("test", "record1"); 551 | testRecord.Tags.Add("test", "test"); 552 | 553 | _ = await memoryDb.UpsertAsync(collection, testRecord); 554 | 555 | _ = this._textEmbeddingGeneratorMock.Setup(x => x.GenerateEmbeddingAsync(It.IsAny(), It.IsAny())) 556 | .ReturnsAsync(compareEmbedding); 557 | 558 | // Act 559 | double threshold = -1; 560 | var topNResults = memoryDb.GetSimilarListAsync( 561 | index: collection, 562 | text: "Sample", 563 | limit: 4, 564 | minRelevance: threshold, 565 | filters: new[] { 566 | new MemoryFilter() 567 | .ByTag("test", "record0") 568 | .ByTag("test", "test") 569 | }) 570 | .ToEnumerable() 571 | .ToArray(); 572 | 573 | // Assert 574 | Assert.NotNull(topNResults); 575 | Assert.Single(topNResults); 576 | } 577 | 578 | /// 579 | /// Test that get similar list should return expected results. 580 | /// 581 | /// 582 | [Fact] 583 | public async Task GetSimilarListShouldNotReturnExpectedWithFiltersWithORClauseAsync() 584 | { 585 | // Arrange 586 | var memoryDb = this.CreateMemoryDb(); 587 | var compareEmbedding = new ReadOnlyMemory(new float[] { 1, 1, 1 }); 588 | 589 | string collection = Guid.NewGuid().ToString(); 590 | await memoryDb.CreateIndexAsync(collection, 1536); 591 | int i = 0; 592 | 593 | MemoryRecord testRecord = new MemoryRecord() 594 | { 595 | Id = "test" + i++, 596 | Vector = new float[] { 1, 1, 1 } 597 | }; 598 | 599 | testRecord.Tags.Add("test", "record0"); 600 | testRecord.Tags.Add("test", "test"); 601 | 602 | _ = await memoryDb.UpsertAsync(collection, testRecord); 603 | 604 | testRecord = new MemoryRecord() 605 | { 606 | Id = "test" + i, 607 | Vector = new float[] { 1, 1, 1 } 608 | }; 609 | 610 | testRecord.Tags.Add("test", "record1"); 611 | testRecord.Tags.Add("test", "test"); 612 | 613 | _ = await memoryDb.UpsertAsync(collection, testRecord); 614 | 615 | _ = this._textEmbeddingGeneratorMock.Setup(x => x.GenerateEmbeddingAsync(It.IsAny(), It.IsAny())) 616 | .ReturnsAsync(compareEmbedding); 617 | 618 | // Act 619 | double threshold = -1; 620 | var topNResults = memoryDb.GetSimilarListAsync( 621 | index: collection, 622 | text: "Sample", 623 | limit: 4, 624 | minRelevance: threshold, 625 | filters: new[] { 626 | new MemoryFilter() 627 | .ByTag("test", "record0"), 628 | new MemoryFilter() 629 | .ByTag("test", "record1") 630 | }) 631 | .ToEnumerable() 632 | .ToArray(); 633 | 634 | // Assert 635 | Assert.NotNull(topNResults); 636 | Assert.Equal(2, topNResults.Length); 637 | } 638 | } 639 | -------------------------------------------------------------------------------- /src/Connectors.Memory.SqlServer.Tests/SqlServerMemoryStoreTests.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Kevin BEAUGRAND. All rights reserved. 2 | 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Collections.Immutable; 6 | using System.Linq; 7 | using System.Threading; 8 | using System.Threading.Tasks; 9 | using Microsoft.Extensions.Configuration; 10 | using Microsoft.SemanticKernel.Embeddings; 11 | using Microsoft.SemanticKernel.Memory; 12 | using SemanticKernel.Connectors.Memory.SqlServer; 13 | using Xunit; 14 | 15 | namespace SemanticKernel.IntegrationTests.Connectors.Memory.SqlServer; 16 | 17 | #pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. 18 | 19 | /// 20 | /// Integration tests of . 21 | /// 22 | public class SqlServerMemoryStoreTests : IAsyncLifetime 23 | { 24 | // If null, all tests will be enabled 25 | private const string? SkipReason = "Requires SqlServer server up and running"; 26 | 27 | public async Task InitializeAsync() 28 | { 29 | // Load configuration 30 | var configuration = new ConfigurationBuilder() 31 | .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) 32 | .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) 33 | .AddEnvironmentVariables() 34 | .AddUserSecrets() 35 | .Build(); 36 | 37 | var connectionString = configuration["SqlServer:ConnectionString"]; 38 | 39 | if (string.IsNullOrWhiteSpace(connectionString)) 40 | { 41 | throw new ArgumentNullException("SqlServer memory connection string is not configured"); 42 | } 43 | 44 | this._connectionString = connectionString; 45 | 46 | this._dataSource = new SqlServerClient(connectionString, new SqlServerConfig 47 | { 48 | Schema = "ai" 49 | }); 50 | 51 | await this._dataSource.CreateTablesAsync(CancellationToken.None).ConfigureAwait(false); 52 | } 53 | 54 | [Fact(Skip = SkipReason)] 55 | public void InitializeDbConnectionSucceeds() 56 | { 57 | SqlServerMemoryStore memoryStore = this.CreateMemoryStore(); 58 | // Assert 59 | Assert.NotNull(memoryStore); 60 | } 61 | 62 | [Fact(Skip = SkipReason)] 63 | public async Task ItCanCreateAndGetCollectionAsync() 64 | { 65 | // Arrange 66 | SqlServerMemoryStore memoryStore = this.CreateMemoryStore(); 67 | string collection = "test_collection"; 68 | 69 | // Act 70 | await memoryStore.CreateCollectionAsync(collection); 71 | var collections = memoryStore.GetCollectionsAsync(); 72 | 73 | // Assert 74 | Assert.NotEmpty(collections.ToEnumerable()); 75 | Assert.True(await collections.ContainsAsync(collection)); 76 | } 77 | 78 | [Fact(Skip = SkipReason)] 79 | public async Task ItCanCheckIfCollectionExistsAsync() 80 | { 81 | // Arrange 82 | SqlServerMemoryStore memoryStore = this.CreateMemoryStore(); 83 | string collection = "my_collection"; 84 | 85 | // Act 86 | await memoryStore.CreateCollectionAsync(collection); 87 | 88 | // Assert 89 | Assert.True(await memoryStore.DoesCollectionExistAsync("my_collection")); 90 | Assert.False(await memoryStore.DoesCollectionExistAsync("my_collection2")); 91 | } 92 | 93 | [Fact(Skip = SkipReason)] 94 | public async Task CollectionsCanBeDeletedAsync() 95 | { 96 | // Arrange 97 | SqlServerMemoryStore memoryStore = this.CreateMemoryStore(); 98 | string collection = "test_collection"; 99 | await memoryStore.CreateCollectionAsync(collection); 100 | Assert.True(await memoryStore.DoesCollectionExistAsync(collection)); 101 | 102 | // Act 103 | await memoryStore.DeleteCollectionAsync(collection); 104 | 105 | // Assert 106 | Assert.False(await memoryStore.DoesCollectionExistAsync(collection)); 107 | } 108 | 109 | [Fact(Skip = SkipReason)] 110 | public async Task GetAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync() 111 | { 112 | // Arrange 113 | SqlServerMemoryStore memoryStore = this.CreateMemoryStore(); 114 | MemoryRecord testRecord = MemoryRecord.LocalRecord( 115 | id: "test", 116 | text: "text", 117 | description: "description", 118 | embedding: new ReadOnlyMemory(new float[] { 1, 2, 3 }), 119 | key: null, 120 | timestamp: null); 121 | string collection = "test_collection"; 122 | 123 | // Act 124 | await memoryStore.CreateCollectionAsync(collection); 125 | var key = await memoryStore.UpsertAsync(collection, testRecord); 126 | var actualDefault = await memoryStore.GetAsync(collection, key); 127 | var actualWithEmbedding = await memoryStore.GetAsync(collection, key, true); 128 | 129 | // Assert 130 | Assert.NotNull(actualDefault); 131 | Assert.NotNull(actualWithEmbedding); 132 | Assert.Empty(actualDefault.Embedding.ToArray()); 133 | Assert.NotEmpty(actualWithEmbedding.Embedding.ToArray()); 134 | } 135 | 136 | [Fact(Skip = SkipReason)] 137 | public async Task ItCanUpsertAndRetrieveARecordWithNoTimestampAsync() 138 | { 139 | // Arrange 140 | SqlServerMemoryStore memoryStore = this.CreateMemoryStore(); 141 | MemoryRecord testRecord = MemoryRecord.LocalRecord( 142 | id: "test", 143 | text: "text", 144 | description: "description", 145 | embedding: new ReadOnlyMemory(new float[] { 1, 2, 3 }), 146 | key: null, 147 | timestamp: null); 148 | string collection = "test_collection"; 149 | 150 | // Act 151 | await memoryStore.CreateCollectionAsync(collection); 152 | var key = await memoryStore.UpsertAsync(collection, testRecord); 153 | var actual = await memoryStore.GetAsync(collection, key, true); 154 | 155 | // Assert 156 | Assert.NotNull(actual); 157 | Assert.Equal(testRecord.Metadata.Id, key); 158 | Assert.Equal(testRecord.Metadata.Id, actual.Key); 159 | Assert.Equal(testRecord.Embedding, actual.Embedding); 160 | Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text); 161 | Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description); 162 | Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName); 163 | Assert.Equal(testRecord.Metadata.Id, actual.Metadata.Id); 164 | } 165 | 166 | [Fact(Skip = SkipReason)] 167 | public async Task ItCanUpsertAndRetrieveARecordWithTimestampAsync() 168 | { 169 | // Arrange 170 | SqlServerMemoryStore memoryStore = this.CreateMemoryStore(); 171 | MemoryRecord testRecord = MemoryRecord.LocalRecord( 172 | id: "test", 173 | text: "text", 174 | description: "description", 175 | embedding: new ReadOnlyMemory(new float[] { 1, 2, 3 }), 176 | key: null, 177 | timestamp: DateTimeOffset.FromUnixTimeMilliseconds(DateTimeOffset.UtcNow.ToUnixTimeMilliseconds())); 178 | string collection = "test_collection"; 179 | 180 | // Act 181 | await memoryStore.CreateCollectionAsync(collection); 182 | var key = await memoryStore.UpsertAsync(collection, testRecord); 183 | var actual = await memoryStore.GetAsync(collection, key, true); 184 | 185 | // Assert 186 | Assert.NotNull(actual); 187 | Assert.Equal(testRecord.Metadata.Id, key); 188 | Assert.Equal(testRecord.Metadata.Id, actual.Key); 189 | Assert.Equal(testRecord.Embedding, actual.Embedding); 190 | Assert.Equal(testRecord.Metadata.Text, actual.Metadata.Text); 191 | Assert.Equal(testRecord.Metadata.Description, actual.Metadata.Description); 192 | Assert.Equal(testRecord.Metadata.ExternalSourceName, actual.Metadata.ExternalSourceName); 193 | Assert.Equal(testRecord.Metadata.Id, actual.Metadata.Id); 194 | Assert.Equal(testRecord.Timestamp, actual.Timestamp); 195 | } 196 | 197 | [Fact(Skip = SkipReason)] 198 | public async Task UpsertReplacesExistingRecordWithSameIdAsync() 199 | { 200 | // Arrange 201 | SqlServerMemoryStore memoryStore = this.CreateMemoryStore(); 202 | string commonId = "test"; 203 | MemoryRecord testRecord = MemoryRecord.LocalRecord( 204 | id: commonId, 205 | text: "text", 206 | description: "description", 207 | embedding: new ReadOnlyMemory(new float[] { 1, 2, 3 })); 208 | MemoryRecord testRecord2 = MemoryRecord.LocalRecord( 209 | id: commonId, 210 | text: "text2", 211 | description: "description2", 212 | embedding: new ReadOnlyMemory(new float[] { 1, 2, 4 })); 213 | string collection = "test_collection"; 214 | 215 | // Act 216 | await memoryStore.CreateCollectionAsync(collection); 217 | var key = await memoryStore.UpsertAsync(collection, testRecord); 218 | var key2 = await memoryStore.UpsertAsync(collection, testRecord2); 219 | var actual = await memoryStore.GetAsync(collection, key, true); 220 | 221 | // Assert 222 | Assert.NotNull(actual); 223 | Assert.Equal(testRecord.Metadata.Id, key); 224 | Assert.Equal(testRecord2.Metadata.Id, actual.Key); 225 | Assert.NotEqual(testRecord.Embedding, actual.Embedding); 226 | Assert.Equal(testRecord2.Embedding, actual.Embedding); 227 | Assert.NotEqual(testRecord.Metadata.Text, actual.Metadata.Text); 228 | Assert.Equal(testRecord2.Metadata.Description, actual.Metadata.Description); 229 | } 230 | 231 | [Fact(Skip = SkipReason)] 232 | public async Task ExistingRecordCanBeRemovedAsync() 233 | { 234 | // Arrange 235 | SqlServerMemoryStore memoryStore = this.CreateMemoryStore(); 236 | MemoryRecord testRecord = MemoryRecord.LocalRecord( 237 | id: "test", 238 | text: "text", 239 | description: "description", 240 | embedding: new ReadOnlyMemory(new float[] { 1, 2, 3 })); 241 | string collection = "test_collection"; 242 | 243 | // Act 244 | await memoryStore.CreateCollectionAsync(collection); 245 | var key = await memoryStore.UpsertAsync(collection, testRecord); 246 | var upsertedRecord = await memoryStore.GetAsync(collection, key); 247 | await memoryStore.RemoveAsync(collection, key); 248 | var actual = await memoryStore.GetAsync(collection, key); 249 | 250 | // Assert 251 | Assert.NotNull(upsertedRecord); 252 | Assert.Null(actual); 253 | } 254 | 255 | [Fact(Skip = SkipReason)] 256 | public async Task RemovingNonExistingRecordDoesNothingAsync() 257 | { 258 | // Arrange 259 | SqlServerMemoryStore memoryStore = this.CreateMemoryStore(); 260 | string collection = "test_collection"; 261 | 262 | // Act 263 | await memoryStore.CreateCollectionAsync(collection); 264 | await memoryStore.RemoveAsync(collection, "key"); 265 | var actual = await memoryStore.GetAsync(collection, "key"); 266 | 267 | // Assert 268 | Assert.Null(actual); 269 | } 270 | 271 | [Fact(Skip = SkipReason)] 272 | public async Task ItCanListAllDatabaseCollectionsAsync() 273 | { 274 | // Arrange 275 | SqlServerMemoryStore memoryStore = this.CreateMemoryStore(); 276 | string[] testCollections = { "random_collection1", "random_collection2", "random_collection3" }; 277 | await memoryStore.CreateCollectionAsync(testCollections[0]); 278 | await memoryStore.CreateCollectionAsync(testCollections[1]); 279 | await memoryStore.CreateCollectionAsync(testCollections[2]); 280 | 281 | // Act 282 | var collections = await memoryStore.GetCollectionsAsync().ToListAsync(); 283 | 284 | // Assert 285 | foreach (var collection in testCollections) 286 | { 287 | Assert.True(await memoryStore.DoesCollectionExistAsync(collection)); 288 | } 289 | 290 | Assert.NotNull(collections); 291 | Assert.NotEmpty(collections); 292 | Assert.Equal(testCollections.Length, collections.Count); 293 | Assert.True(collections.Contains(testCollections[0]), 294 | $"Collections does not contain the newly-created collection {testCollections[0]}"); 295 | Assert.True(collections.Contains(testCollections[1]), 296 | $"Collections does not contain the newly-created collection {testCollections[1]}"); 297 | Assert.True(collections.Contains(testCollections[2]), 298 | $"Collections does not contain the newly-created collection {testCollections[2]}"); 299 | } 300 | 301 | [Fact(Skip = SkipReason)] 302 | public async Task GetNearestMatchesReturnsAllResultsWithNoMinScoreAsync() 303 | { 304 | // Arrange 305 | SqlServerMemoryStore memoryStore = this.CreateMemoryStore(); 306 | var compareEmbedding = new ReadOnlyMemory(new float[] { 1, 1, 1 }); 307 | int topN = 4; 308 | string collection = "test_collection"; 309 | await memoryStore.CreateCollectionAsync(collection); 310 | int i = 0; 311 | MemoryRecord testRecord = MemoryRecord.LocalRecord( 312 | id: "test" + i, 313 | text: "text" + i, 314 | description: "description" + i, 315 | embedding: new ReadOnlyMemory(new float[] { 1, 1, 1 })); 316 | _ = await memoryStore.UpsertAsync(collection, testRecord); 317 | 318 | i++; 319 | testRecord = MemoryRecord.LocalRecord( 320 | id: "test" + i, 321 | text: "text" + i, 322 | description: "description" + i, 323 | embedding: new ReadOnlyMemory(new float[] { -1, -1, -1 })); 324 | _ = await memoryStore.UpsertAsync(collection, testRecord); 325 | 326 | i++; 327 | testRecord = MemoryRecord.LocalRecord( 328 | id: "test" + i, 329 | text: "text" + i, 330 | description: "description" + i, 331 | embedding: new ReadOnlyMemory(new float[] { 1, 2, 3 })); 332 | _ = await memoryStore.UpsertAsync(collection, testRecord); 333 | 334 | i++; 335 | testRecord = MemoryRecord.LocalRecord( 336 | id: "test" + i, 337 | text: "text" + i, 338 | description: "description" + i, 339 | embedding: new ReadOnlyMemory(new float[] { -1, -2, -3 })); 340 | _ = await memoryStore.UpsertAsync(collection, testRecord); 341 | 342 | i++; 343 | testRecord = MemoryRecord.LocalRecord( 344 | id: "test" + i, 345 | text: "text" + i, 346 | description: "description" + i, 347 | embedding: new ReadOnlyMemory(new float[] { 1, -1, -2 })); 348 | _ = await memoryStore.UpsertAsync(collection, testRecord); 349 | 350 | // Act 351 | double threshold = -1; 352 | var topNResults = memoryStore.GetNearestMatchesAsync(collection, compareEmbedding, limit: topN, minRelevanceScore: threshold).ToEnumerable().ToArray(); 353 | 354 | // Assert 355 | Assert.Equal(topN, topNResults.Length); 356 | for (int j = 0; j < topN - 1; j++) 357 | { 358 | int compare = topNResults[j].Item2.CompareTo(topNResults[j + 1].Item2); 359 | Assert.True(compare >= 0); 360 | } 361 | } 362 | 363 | [Fact(Skip = SkipReason)] 364 | public async Task GetNearestMatchAsyncReturnsEmptyEmbeddingUnlessSpecifiedAsync() 365 | { 366 | // Arrange 367 | SqlServerMemoryStore memoryStore = this.CreateMemoryStore(); 368 | var compareEmbedding = new ReadOnlyMemory(new float[] { 1, 1, 1 }); 369 | string collection = "test_collection"; 370 | await memoryStore.CreateCollectionAsync(collection); 371 | int i = 0; 372 | MemoryRecord testRecord = MemoryRecord.LocalRecord( 373 | id: "test" + i, 374 | text: "text" + i, 375 | description: "description" + i, 376 | embedding: new ReadOnlyMemory(new float[] { 1, 1, 1 })); 377 | _ = await memoryStore.UpsertAsync(collection, testRecord); 378 | 379 | i++; 380 | testRecord = MemoryRecord.LocalRecord( 381 | id: "test" + i, 382 | text: "text" + i, 383 | description: "description" + i, 384 | embedding: new ReadOnlyMemory(new float[] { -1, -1, -1 })); 385 | _ = await memoryStore.UpsertAsync(collection, testRecord); 386 | 387 | i++; 388 | testRecord = MemoryRecord.LocalRecord( 389 | id: "test" + i, 390 | text: "text" + i, 391 | description: "description" + i, 392 | embedding: new ReadOnlyMemory(new float[] { 1, 2, 3 })); 393 | _ = await memoryStore.UpsertAsync(collection, testRecord); 394 | 395 | i++; 396 | testRecord = MemoryRecord.LocalRecord( 397 | id: "test" + i, 398 | text: "text" + i, 399 | description: "description" + i, 400 | embedding: new ReadOnlyMemory(new float[] { -1, -2, -3 })); 401 | _ = await memoryStore.UpsertAsync(collection, testRecord); 402 | 403 | i++; 404 | testRecord = MemoryRecord.LocalRecord( 405 | id: "test" + i, 406 | text: "text" + i, 407 | description: "description" + i, 408 | embedding: new ReadOnlyMemory(new float[] { 1, -1, -2 })); 409 | _ = await memoryStore.UpsertAsync(collection, testRecord); 410 | 411 | // Act 412 | double threshold = 0.75; 413 | var topNResultDefault = await memoryStore.GetNearestMatchAsync(collection, compareEmbedding, minRelevanceScore: threshold); 414 | var topNResultWithEmbedding = await memoryStore.GetNearestMatchAsync(collection, compareEmbedding, minRelevanceScore: threshold, withEmbedding: true); 415 | 416 | // Assert 417 | Assert.NotNull(topNResultDefault); 418 | Assert.NotNull(topNResultWithEmbedding); 419 | Assert.Empty(topNResultDefault.Value.Item1.Embedding.ToArray()); 420 | Assert.NotEmpty(topNResultWithEmbedding.Value.Item1.Embedding.ToArray()); 421 | } 422 | 423 | [Fact(Skip = SkipReason)] 424 | public async Task GetNearestMatchAsyncReturnsExpectedAsync() 425 | { 426 | // Arrange 427 | SqlServerMemoryStore memoryStore = this.CreateMemoryStore(); 428 | var compareEmbedding = new ReadOnlyMemory(new float[] { 1, 1, 1 }); 429 | string collection = "test_collection"; 430 | await memoryStore.CreateCollectionAsync(collection); 431 | int i = 0; 432 | MemoryRecord testRecord = MemoryRecord.LocalRecord( 433 | id: "test" + i, 434 | text: "text" + i, 435 | description: "description" + i, 436 | embedding: new ReadOnlyMemory(new float[] { 1, 1, 1 })); 437 | _ = await memoryStore.UpsertAsync(collection, testRecord); 438 | 439 | i++; 440 | testRecord = MemoryRecord.LocalRecord( 441 | id: "test" + i, 442 | text: "text" + i, 443 | description: "description" + i, 444 | embedding: new ReadOnlyMemory(new float[] { -1, -1, -1 })); 445 | _ = await memoryStore.UpsertAsync(collection, testRecord); 446 | 447 | i++; 448 | testRecord = MemoryRecord.LocalRecord( 449 | id: "test" + i, 450 | text: "text" + i, 451 | description: "description" + i, 452 | embedding: new ReadOnlyMemory(new float[] { 1, 2, 3 })); 453 | _ = await memoryStore.UpsertAsync(collection, testRecord); 454 | 455 | i++; 456 | testRecord = MemoryRecord.LocalRecord( 457 | id: "test" + i, 458 | text: "text" + i, 459 | description: "description" + i, 460 | embedding: new ReadOnlyMemory(new float[] { -1, -2, -3 })); 461 | _ = await memoryStore.UpsertAsync(collection, testRecord); 462 | 463 | i++; 464 | testRecord = MemoryRecord.LocalRecord( 465 | id: "test" + i, 466 | text: "text" + i, 467 | description: "description" + i, 468 | embedding: new ReadOnlyMemory(new float[] { 1, -1, -2 })); 469 | _ = await memoryStore.UpsertAsync(collection, testRecord); 470 | 471 | // Act 472 | double threshold = 0.75; 473 | var topNResult = await memoryStore.GetNearestMatchAsync(collection, compareEmbedding, minRelevanceScore: threshold); 474 | 475 | // Assert 476 | Assert.NotNull(topNResult); 477 | Assert.Equal("test0", topNResult.Value.Item1.Metadata.Id); 478 | Assert.True(topNResult.Value.Item2 >= threshold); 479 | } 480 | 481 | [Fact(Skip = SkipReason)] 482 | public async Task GetNearestMatchesDifferentiatesIdenticalVectorsByKeyAsync() 483 | { 484 | // Arrange 485 | SqlServerMemoryStore memoryStore = this.CreateMemoryStore(); 486 | var compareEmbedding = new ReadOnlyMemory(new float[] { 1, 1, 1 }); 487 | int topN = 4; 488 | string collection = "test_collection"; 489 | await memoryStore.CreateCollectionAsync(collection); 490 | 491 | for (int i = 0; i < 10; i++) 492 | { 493 | MemoryRecord testRecord = MemoryRecord.LocalRecord( 494 | id: "test" + i, 495 | text: "text" + i, 496 | description: "description" + i, 497 | embedding: new ReadOnlyMemory(new float[] { 1, 1, 1 })); 498 | _ = await memoryStore.UpsertAsync(collection, testRecord); 499 | } 500 | 501 | // Act 502 | var topNResults = memoryStore.GetNearestMatchesAsync(collection, compareEmbedding, limit: topN, minRelevanceScore: 0.75).ToEnumerable().ToArray(); 503 | IEnumerable topNKeys = topNResults.Select(x => x.Item1.Key).ToImmutableSortedSet(); 504 | 505 | // Assert 506 | Assert.Equal(topN, topNResults.Length); 507 | Assert.Equal(topN, topNKeys.Count()); 508 | 509 | for (int i = 0; i < topNResults.Length; i++) 510 | { 511 | int compare = topNResults[i].Item2.CompareTo(0.75); 512 | Assert.True(compare >= 0); 513 | } 514 | } 515 | 516 | [Fact(Skip = SkipReason)] 517 | public async Task ItCanBatchUpsertRecordsAsync() 518 | { 519 | // Arrange 520 | SqlServerMemoryStore memoryStore = this.CreateMemoryStore(); 521 | int numRecords = 10; 522 | string collection = "test_collection"; 523 | IEnumerable records = this.CreateBatchRecords(numRecords); 524 | 525 | // Act 526 | await memoryStore.CreateCollectionAsync(collection); 527 | var keys = memoryStore.UpsertBatchAsync(collection, records); 528 | var resultRecords = memoryStore.GetBatchAsync(collection, keys.ToEnumerable()); 529 | 530 | // Assert 531 | Assert.NotNull(keys); 532 | Assert.Equal(numRecords, keys.ToEnumerable().Count()); 533 | Assert.Equal(numRecords, resultRecords.ToEnumerable().Count()); 534 | } 535 | 536 | [Fact(Skip = SkipReason)] 537 | public async Task ItCanBatchGetRecordsAsync() 538 | { 539 | // Arrange 540 | SqlServerMemoryStore memoryStore = this.CreateMemoryStore(); 541 | int numRecords = 10; 542 | string collection = "test_collection"; 543 | IEnumerable records = this.CreateBatchRecords(numRecords); 544 | var keys = memoryStore.UpsertBatchAsync(collection, records); 545 | 546 | // Act 547 | await memoryStore.CreateCollectionAsync(collection); 548 | var results = memoryStore.GetBatchAsync(collection, keys.ToEnumerable()); 549 | 550 | // Assert 551 | Assert.NotNull(keys); 552 | Assert.NotNull(results); 553 | Assert.Equal(numRecords, results.ToEnumerable().Count()); 554 | } 555 | 556 | [Fact(Skip = SkipReason)] 557 | public async Task ItCanBatchRemoveRecordsAsync() 558 | { 559 | // Arrange 560 | SqlServerMemoryStore memoryStore = this.CreateMemoryStore(); 561 | int numRecords = 10; 562 | string collection = "test_collection"; 563 | IEnumerable records = this.CreateBatchRecords(numRecords); 564 | await memoryStore.CreateCollectionAsync(collection); 565 | 566 | List keys = new(); 567 | 568 | // Act 569 | await foreach (var key in memoryStore.UpsertBatchAsync(collection, records)) 570 | { 571 | keys.Add(key); 572 | } 573 | 574 | await memoryStore.RemoveBatchAsync(collection, keys); 575 | 576 | // Assert 577 | await foreach (var result in memoryStore.GetBatchAsync(collection, keys)) 578 | { 579 | Assert.Null(result); 580 | } 581 | } 582 | 583 | [Fact(Skip = SkipReason)] 584 | public async Task DeletingNonExistentCollectionDoesNothingAsync() 585 | { 586 | // Arrange 587 | SqlServerMemoryStore memoryStore = this.CreateMemoryStore(); 588 | string collection = "test_collection"; 589 | 590 | // Act 591 | await memoryStore.DeleteCollectionAsync(collection); 592 | } 593 | 594 | [Fact(Skip = SkipReason)] 595 | public async Task ItCanBatchGetRecordsAndSkipIfKeysDoNotExistAsync() 596 | { 597 | // Arrange 598 | SqlServerMemoryStore memoryStore = this.CreateMemoryStore(); 599 | int numRecords = 10; 600 | string collection = "test_collection"; 601 | IEnumerable records = this.CreateBatchRecords(numRecords); 602 | 603 | // Act 604 | await memoryStore.CreateCollectionAsync(collection); 605 | var keys = await memoryStore.UpsertBatchAsync(collection, records).ToListAsync(); 606 | keys.Insert(0, "not-exist-key-0"); 607 | keys.Insert(5, "not-exist-key-5"); 608 | keys.Add("not-exist-key-n"); 609 | var resultRecords = memoryStore.GetBatchAsync(collection, keys); 610 | 611 | // Assert 612 | Assert.NotNull(keys); 613 | Assert.Equal(numRecords, keys.Count - 3); 614 | Assert.Equal(numRecords, resultRecords.ToEnumerable().Count()); 615 | } 616 | 617 | #region private ================================================================================ 618 | 619 | private string _connectionString = null!; 620 | private SqlServerClient _dataSource = null!; 621 | 622 | private SqlServerMemoryStore CreateMemoryStore() 623 | { 624 | return new SqlServerMemoryStore(this._dataSource); 625 | } 626 | 627 | private IEnumerable CreateBatchRecords(int numRecords) 628 | { 629 | Assert.True(numRecords % 2 == 0, "Number of records must be even"); 630 | Assert.True(numRecords > 0, "Number of records must be greater than 0"); 631 | 632 | IEnumerable records = new List(numRecords); 633 | for (int i = 0; i < numRecords / 2; i++) 634 | { 635 | var testRecord = MemoryRecord.LocalRecord( 636 | id: "test" + i, 637 | text: "text" + i, 638 | description: "description" + i, 639 | embedding: new ReadOnlyMemory(new float[] { 1, 1, 1 })); 640 | records = records.Append(testRecord); 641 | } 642 | 643 | for (int i = numRecords / 2; i < numRecords; i++) 644 | { 645 | var testRecord = MemoryRecord.ReferenceRecord( 646 | externalId: "test" + i, 647 | sourceName: "sourceName" + i, 648 | description: "description" + i, 649 | embedding: new ReadOnlyMemory(new float[] { 1, 2, 3 })); 650 | records = records.Append(testRecord); 651 | } 652 | 653 | return records; 654 | } 655 | 656 | public async Task DisposeAsync() 657 | { 658 | await foreach (var item in this._dataSource.GetCollectionsAsync(CancellationToken.None).ConfigureAwait(false)) 659 | { 660 | await this._dataSource.DeleteCollectionAsync(item, CancellationToken.None).ConfigureAwait(false); 661 | } 662 | } 663 | 664 | #endregion 665 | } 666 | 667 | #pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. -------------------------------------------------------------------------------- /src/Connectors.Memory.SqlServer.Tests/testsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "OpenAI": { 3 | "ServiceId": "text-davinci-003", 4 | "ModelId": "text-davinci-003", 5 | "ApiKey": "" 6 | }, 7 | "AzureOpenAI": { 8 | "ServiceId": "azure-text-davinci-003", 9 | "DeploymentName": "text-davinci-003", 10 | "ChatDeploymentName": "gpt-4", 11 | "Endpoint": "", 12 | "ApiKey": "" 13 | }, 14 | "OpenAIEmbeddings": { 15 | "ServiceId": "text-embedding-ada-002", 16 | "ModelId": "text-embedding-ada-002", 17 | "ApiKey": "" 18 | }, 19 | "AzureOpenAIEmbeddings": { 20 | "ServiceId": "azure-text-embedding-ada-002", 21 | "DeploymentName": "text-embedding-ada-002", 22 | "Endpoint": "", 23 | "ApiKey": "" 24 | }, 25 | "HuggingFace": { 26 | "ApiKey": "" 27 | }, 28 | "Bing": { 29 | "ApiKey": "" 30 | }, 31 | "Postgres": { 32 | "ConnectionString": "" 33 | }, 34 | "SqlServer": { 35 | "ConnectionString": "Server=tcp:localhost,1433;Initial Catalog=master;Persist Security Info=False;User ID=SA;Password=P@ssw0rd;MultipleActiveResultSets=False;TrustServerCertificate=True;Connection Timeout=30;" 36 | } 37 | } -------------------------------------------------------------------------------- /src/Connectors.Memory.SqlServer/Connectors.Memory.SqlServer.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | SemanticKernel.Connectors.Memory.SqlServer 6 | $(AssemblyName) 7 | netstandard2.0 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | Semantic Kernel - SQL Connector 16 | MSSQL connector for Semantic Kernel skills and semantic memory 17 | 18 | 19 | 20 | 1701;1702;SKEXP0003 21 | 22 | 23 | 24 | 1701;1702;SKEXP0003 25 | 26 | 27 | 28 | 1701;1702;SKEXP0003 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /src/Connectors.Memory.SqlServer/ISqlServerClient.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Kevin BEAUGRAND. All rights reserved. 2 | 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Runtime.CompilerServices; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | 9 | namespace SemanticKernel.Connectors.Memory.SqlServer; 10 | 11 | /// 12 | /// Interface for a client that interacts with a SQL Server database to store and retrieve data. 13 | /// 14 | public interface ISqlServerClient : IDisposable 15 | { 16 | /// 17 | /// Creates a new collection with the specified name in the SQL Server database. 18 | /// 19 | /// The name of the collection to create. 20 | /// A cancellation token that can be used to cancel the operation. 21 | /// A task that represents the asynchronous operation. 22 | Task CreateCollectionAsync(string collectionName, CancellationToken cancellationToken = default); 23 | 24 | /// 25 | /// Creates the necessary tables in the SQL Server database for the client to function properly. 26 | /// 27 | /// A cancellation token that can be used to cancel the operation. 28 | /// A task that represents the asynchronous operation. 29 | Task CreateTablesAsync(CancellationToken cancellationToken); 30 | 31 | /// 32 | /// Deletes a document from the specified collection in the SQL Server database. 33 | /// 34 | /// The name of the collection to delete the document from. 35 | /// The key of the document to delete. 36 | /// A cancellation token that can be used to cancel the operation. 37 | /// A task that represents the asynchronous delete operation. 38 | Task DeleteAsync(string collectionName, string key, CancellationToken cancellationToken = default); 39 | /// 40 | /// Deletes a batch of documents from the specified collection in the SQL Server database. 41 | /// 42 | /// The name of the collection to delete documents from. 43 | /// The keys of the documents to delete. 44 | /// A cancellation token that can be used to cancel the operation. 45 | /// A task that represents the asynchronous operation. 46 | Task DeleteBatchAsync(string collectionName, IEnumerable keys, CancellationToken cancellationToken = default); 47 | /// 48 | /// Deletes a collection with the specified name from the SQL Server database asynchronously. 49 | /// 50 | /// The name of the collection to delete. 51 | /// A cancellation token to cancel the asynchronous operation. 52 | /// A task that represents the asynchronous operation. 53 | Task DeleteCollectionAsync(string collectionName, CancellationToken cancellationToken = default); 54 | /// 55 | /// Determines whether a collection with the specified name exists in the SQL Server database. 56 | /// 57 | /// The name of the collection to check for existence. 58 | /// A cancellation token to observe while waiting for the task to complete. 59 | /// A task that represents the asynchronous operation. The task result contains a value indicating whether the collection exists. 60 | Task DoesCollectionExistsAsync(string collectionName, CancellationToken cancellationToken = default); 61 | /// 62 | /// Gets a list of all collections in the SQL Server database. 63 | /// 64 | /// A cancellation token to observe while waiting for the task to complete. 65 | /// An asynchronous enumerable of collection names. 66 | IAsyncEnumerable GetCollectionsAsync(CancellationToken cancellationToken = default); 67 | /// 68 | /// Retrieves the nearest matches to the specified embedding in the specified collection. 69 | /// 70 | /// The name of the collection to search in. 71 | /// The embedding to search for. 72 | /// The maximum number of matches to retrieve. 73 | /// The minimum relevance score for a match to be considered relevant. 74 | /// Whether to include embeddings in the results. 75 | /// A cancellation token that can be used to cancel the operation. 76 | /// An asynchronous enumerable of tuples containing the matching entries and their relevance scores. 77 | IAsyncEnumerable<(SqlServerMemoryEntry, double)> GetNearestMatchesAsync(string collectionName, string embedding, int limit, double minRelevanceScore = 0, bool withEmbeddings = false, [EnumeratorCancellation] CancellationToken cancellationToken = default); 78 | /// 79 | /// Reads a from the specified collection with the given key. 80 | /// 81 | /// The name of the collection to read from. 82 | /// The key of the entry to read. 83 | /// Whether to include embeddings in the returned entry. 84 | /// A to observe while waiting for the task to complete. 85 | /// A representing the asynchronous operation. The task result contains the read from the collection, or null if no entry was found with the given key. 86 | Task ReadAsync(string collectionName, string key, bool withEmbeddings = false, CancellationToken cancellationToken = default); 87 | /// 88 | /// Asynchronously reads a batch of objects from the specified collection in the SQL Server database. 89 | /// 90 | /// The name of the collection to read from. 91 | /// The keys of the objects to read. 92 | /// Whether to include embeddings in the objects. 93 | /// A to observe while waiting for the task to complete. 94 | /// An of objects. 95 | IAsyncEnumerable ReadBatchAsync(string collectionName, IEnumerable keys, bool withEmbeddings = false, CancellationToken cancellationToken = default); 96 | /// 97 | /// Upserts a document with the specified key and embedding into the specified collection. 98 | /// 99 | /// The name of the collection to upsert the document into. 100 | /// The key of the document to upsert. 101 | /// The metadata of the document to upsert. 102 | /// The embedding of the document to upsert. 103 | /// The timestamp of the document to upsert. 104 | /// The cancellation token. 105 | /// A task that represents the asynchronous upsert operation. 106 | Task UpsertAsync(string collectionName, string key, string? metadata, string embedding, DateTimeOffset? timestamp, CancellationToken cancellationToken = default); 107 | } 108 | -------------------------------------------------------------------------------- /src/Connectors.Memory.SqlServer/NUGET.md: -------------------------------------------------------------------------------- 1 | # SQL Server Connector for Semantic Kernel 2 | 3 | This is a connector for the Semantic Kernel. 4 | 5 | It provides a connection to a SQL database for the Semantic Kernel for the memories. 6 | 7 | ## About Semantic Kernel 8 | 9 | **Semantic Kernel (SK)** is a lightweight SDK enabling integration of AI Large 10 | Language Models (LLMs) with conventional programming languages. The SK 11 | extensible programming model combines natural language **semantic functions**, 12 | traditional code **native functions**, and **embeddings-based memory** unlocking 13 | new potential and adding value to applications with AI. 14 | 15 | Semantic Kernel incorporates cutting-edge design patterns from the latest in AI 16 | research. This enables developers to augment their applications with advanced 17 | capabilities, such as prompt engineering, prompt chaining, retrieval-augmented 18 | generation, contextual and long-term vectorized memory, embeddings, 19 | summarization, zero or few-shot learning, semantic indexing, recursive 20 | reasoning, intelligent planning, and access to external knowledge stores and 21 | proprietary data. -------------------------------------------------------------------------------- /src/Connectors.Memory.SqlServer/SqlServerClient.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Kevin BEAUGRAND. All rights reserved. 2 | 3 | using Microsoft.Data.SqlClient; 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Linq; 7 | using System.Runtime.CompilerServices; 8 | using System.Text.Json; 9 | using System.Text.RegularExpressions; 10 | using System.Threading; 11 | using System.Threading.Tasks; 12 | 13 | namespace SemanticKernel.Connectors.Memory.SqlServer; 14 | 15 | /// 16 | /// Represents a client for interacting with a SQL Server database for storing semantic memories and embeddings. 17 | /// 18 | [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities", 19 | Justification = "We need to build the full table name using schema and collection, it does not support parameterized passing.")] 20 | public sealed class SqlServerClient : ISqlServerClient 21 | { 22 | private readonly string _connectionString; 23 | private readonly SqlServerConfig _configuration; 24 | private SqlConnection _connection; 25 | 26 | /// 27 | /// Initializes a new instance of the class with the specified connection string and schema. 28 | /// 29 | /// The connection string to use for connecting to the SQL Server database. 30 | /// The configuration to use for the SQL Server database. 31 | public SqlServerClient(string connectionString, SqlServerConfig configuration) 32 | { 33 | this._connectionString = connectionString; 34 | this._configuration = configuration; 35 | 36 | this._connection = new SqlConnection(this._connectionString); 37 | this._connection.Open(); 38 | } 39 | 40 | /// 41 | public async Task CreateTablesAsync(CancellationToken cancellationToken) 42 | { 43 | var sql = $@"IF NOT EXISTS (SELECT * 44 | FROM sys.schemas 45 | WHERE name = N'{this._configuration.Schema}' ) 46 | EXEC('CREATE SCHEMA [{this._configuration.Schema}]'); 47 | IF OBJECT_ID(N'{this.GetFullTableName(this._configuration.MemoryCollectionTableName)}', N'U') IS NULL 48 | CREATE TABLE {this.GetFullTableName(this._configuration.MemoryCollectionTableName)} 49 | ( [id] NVARCHAR(256) NOT NULL, 50 | PRIMARY KEY ([id]) 51 | ); 52 | 53 | IF OBJECT_ID(N'{this.GetFullTableName(this._configuration.MemoryTableName)}', N'U') IS NULL 54 | CREATE TABLE {this.GetFullTableName(this._configuration.MemoryTableName)} 55 | ( [id] UNIQUEIDENTIFIER NOT NULL, 56 | [key] NVARCHAR(256) NOT NULL, 57 | [collection] NVARCHAR(256) NOT NULL, 58 | [metadata] TEXT, 59 | [embedding] TEXT, 60 | [timestamp] DATETIMEOFFSET, 61 | PRIMARY KEY ([id]), 62 | FOREIGN KEY ([collection]) REFERENCES {this.GetFullTableName(this._configuration.MemoryCollectionTableName)}([id]) ON DELETE CASCADE, 63 | CONSTRAINT UK_{this._configuration.MemoryTableName} UNIQUE([collection], [key]) 64 | );"; 65 | 66 | using (SqlCommand command = this._connection.CreateCommand()) 67 | { 68 | command.CommandText = sql; 69 | await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); 70 | } 71 | } 72 | 73 | /// 74 | public async Task CreateCollectionAsync(string collectionName, CancellationToken cancellationToken = default) 75 | { 76 | collectionName = NormalizeIndexName(collectionName); 77 | 78 | if (await this.DoesCollectionExistsAsync(collectionName, cancellationToken).ConfigureAwait(false)) 79 | { 80 | // Collection already exists 81 | return; 82 | } 83 | 84 | using (SqlCommand command = this._connection.CreateCommand()) 85 | { 86 | command.CommandText = $@" 87 | INSERT INTO {this.GetFullTableName(this._configuration.MemoryCollectionTableName)}([id]) 88 | VALUES (@collectionName); 89 | 90 | IF OBJECT_ID(N'{this.GetFullTableName($"{this._configuration.EmbeddingsTableName}_{collectionName}")}', N'U') IS NULL 91 | CREATE TABLE {this.GetFullTableName($"{this._configuration.EmbeddingsTableName}_{collectionName}")} 92 | ( 93 | [memory_id] UNIQUEIDENTIFIER NOT NULL, 94 | [vector_value_id] [int] NOT NULL, 95 | [vector_value] [float] NOT NULL 96 | FOREIGN KEY ([memory_id]) REFERENCES {this.GetFullTableName(this._configuration.MemoryTableName)}([id]) ON DELETE CASCADE 97 | ); 98 | 99 | IF OBJECT_ID(N'{this._configuration.Schema}.IXC_{$"{this._configuration.EmbeddingsTableName}_{collectionName}"}', N'U') IS NULL 100 | CREATE CLUSTERED COLUMNSTORE INDEX [IXC_{$"{this._configuration.EmbeddingsTableName}_{collectionName}"}] 101 | ON {this.GetFullTableName($"{this._configuration.EmbeddingsTableName}_{collectionName}")};"; 102 | 103 | command.Parameters.AddWithValue("@collectionName", collectionName); 104 | 105 | await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); 106 | }; 107 | } 108 | 109 | /// 110 | public async Task DoesCollectionExistsAsync(string collectionName, 111 | CancellationToken cancellationToken = default) 112 | { 113 | collectionName = NormalizeIndexName(collectionName); 114 | 115 | var collections = this.GetCollectionsAsync(cancellationToken) 116 | .WithCancellation(cancellationToken) 117 | .ConfigureAwait(false); 118 | 119 | await foreach (var item in collections) 120 | { 121 | if (item.Equals(collectionName, StringComparison.OrdinalIgnoreCase)) 122 | return true; 123 | } 124 | 125 | return false; 126 | } 127 | 128 | /// 129 | public async IAsyncEnumerable GetCollectionsAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) 130 | { 131 | using (SqlCommand command = this._connection.CreateCommand()) 132 | { 133 | command.CommandText = $"SELECT [id] FROM {this.GetFullTableName(this._configuration.MemoryCollectionTableName)}"; 134 | 135 | using var dataReader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); 136 | 137 | while (await dataReader.ReadAsync(cancellationToken).ConfigureAwait(false)) 138 | { 139 | yield return dataReader.GetString(dataReader.GetOrdinal("id")); 140 | } 141 | }; 142 | } 143 | 144 | /// 145 | public async Task DeleteCollectionAsync(string collectionName, CancellationToken cancellationToken = default) 146 | { 147 | collectionName = NormalizeIndexName(collectionName); 148 | 149 | if (!(await this.DoesCollectionExistsAsync(collectionName, cancellationToken).ConfigureAwait(false))) 150 | { 151 | // Collection does not exist 152 | return; 153 | } 154 | 155 | using (SqlCommand command = this._connection.CreateCommand()) 156 | { 157 | command.CommandText = $@"DELETE FROM {this.GetFullTableName(this._configuration.MemoryCollectionTableName)} 158 | WHERE [id] = @collectionName; 159 | 160 | DROP TABLE {this.GetFullTableName($"{this._configuration.EmbeddingsTableName}_{collectionName}")};"; 161 | 162 | command.Parameters.AddWithValue("@collectionName", collectionName); 163 | 164 | await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); 165 | }; 166 | } 167 | 168 | /// 169 | public async Task ReadAsync(string collectionName, string key, bool withEmbeddings = false, CancellationToken cancellationToken = default) 170 | { 171 | collectionName = NormalizeIndexName(collectionName); 172 | 173 | string queryColumns = "[id], [key], [metadata], [timestamp]"; 174 | 175 | if (withEmbeddings) 176 | { 177 | queryColumns += ", [embedding]"; 178 | } 179 | 180 | using SqlCommand entryCommand = this._connection.CreateCommand(); 181 | 182 | entryCommand.CommandText = $@" 183 | SELECT {queryColumns} 184 | FROM {this.GetFullTableName(this._configuration.MemoryTableName)} 185 | WHERE [collection] = @collection 186 | AND [key]=@key"; 187 | 188 | entryCommand.Parameters.AddWithValue("@key", key); 189 | entryCommand.Parameters.AddWithValue("@collection", collectionName); 190 | 191 | using var dataReader = await entryCommand.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); 192 | 193 | if (await dataReader.ReadAsync(cancellationToken).ConfigureAwait(false)) 194 | { 195 | return await this.ReadEntryAsync(dataReader, withEmbeddings, cancellationToken).ConfigureAwait(false); 196 | } 197 | 198 | return null; 199 | } 200 | 201 | /// 202 | public async IAsyncEnumerable ReadBatchAsync(string collectionName, IEnumerable keys, bool withEmbeddings = false, 203 | [EnumeratorCancellation] CancellationToken cancellationToken = default) 204 | { 205 | collectionName = NormalizeIndexName(collectionName); 206 | 207 | string[] keysArray = keys.ToArray(); 208 | 209 | if (keysArray.Length == 0) 210 | { 211 | yield break; 212 | } 213 | 214 | string queryColumns = "[id], [key], [metadata], [timestamp]"; 215 | 216 | if (withEmbeddings) 217 | { 218 | queryColumns += ", [embedding]"; 219 | } 220 | 221 | using SqlCommand cmd = this._connection.CreateCommand(); 222 | 223 | cmd.CommandText = $@" 224 | SELECT {queryColumns} 225 | FROM {this.GetFullTableName(this._configuration.MemoryTableName)} 226 | WHERE [key] IN ({string.Join(",", Enumerable.Range(0, keysArray.Length).Select(c => $"@key{c}"))})"; 227 | 228 | cmd.Parameters.AddWithValue("@collectionName", collectionName); 229 | 230 | for (int i = 0; i < keysArray.Length; i++) 231 | { 232 | cmd.Parameters.AddWithValue($"@key{i}", keysArray[i]); 233 | } 234 | 235 | using var dataReader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); 236 | 237 | while (await dataReader.ReadAsync(cancellationToken).ConfigureAwait(false)) 238 | { 239 | yield return await this.ReadEntryAsync(dataReader, withEmbeddings, cancellationToken).ConfigureAwait(false); 240 | } 241 | } 242 | 243 | /// 244 | public async Task DeleteAsync(string collectionName, string key, CancellationToken cancellationToken = default) 245 | { 246 | collectionName = NormalizeIndexName(collectionName); 247 | 248 | using SqlCommand cmd = this._connection.CreateCommand(); 249 | 250 | cmd.CommandText = $"DELETE FROM {this.GetFullTableName(this._configuration.MemoryTableName)} WHERE [collection] = @collectionName AND [key]=@key"; 251 | cmd.Parameters.AddWithValue("@collectionName", collectionName); 252 | cmd.Parameters.AddWithValue("@key", key); 253 | 254 | await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); 255 | } 256 | 257 | /// 258 | public async Task DeleteBatchAsync(string collectionName, IEnumerable keys, CancellationToken cancellationToken = default) 259 | { 260 | collectionName = NormalizeIndexName(collectionName); 261 | 262 | string[] keysArray = keys.ToArray(); 263 | 264 | if (keysArray.Length == 0) 265 | { 266 | return; 267 | } 268 | 269 | using SqlCommand cmd = this._connection.CreateCommand(); 270 | 271 | cmd.CommandText = $@" 272 | DELETE 273 | FROM {this.GetFullTableName(this._configuration.MemoryTableName)} 274 | WHERE [collection] = @collectionName 275 | AND [key] IN ({string.Join(",", Enumerable.Range(0, keysArray.Length).Select(c => $"@key{c}"))})"; 276 | 277 | 278 | cmd.Parameters.AddWithValue("@collectionName", collectionName); 279 | 280 | for (int i = 0; i < keysArray.Length; i++) 281 | { 282 | cmd.Parameters.AddWithValue($"@key{i}", keysArray[i]); 283 | } 284 | 285 | await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); 286 | 287 | } 288 | 289 | /// 290 | public async IAsyncEnumerable<(SqlServerMemoryEntry, double)> GetNearestMatchesAsync( 291 | string collectionName, string embedding, int limit, double minRelevanceScore = 0, bool withEmbeddings = false, 292 | [EnumeratorCancellation] CancellationToken cancellationToken = default) 293 | { 294 | collectionName = NormalizeIndexName(collectionName); 295 | 296 | string queryColumns = "[id], [key], [metadata], [timestamp]"; 297 | 298 | if (withEmbeddings) 299 | { 300 | queryColumns += ", [embedding]"; 301 | } 302 | 303 | using SqlCommand cmd = this._connection.CreateCommand(); 304 | 305 | cmd.CommandText = $@"WITH [embedding] as 306 | ( 307 | SELECT 308 | cast([key] AS INT) AS [vector_value_id], 309 | cast([value] AS FLOAT) AS [vector_value] 310 | FROM 311 | openjson(@vector) 312 | ), 313 | [similarity] AS 314 | ( 315 | SELECT TOP (@limit) 316 | {this.GetFullTableName($"{this._configuration.EmbeddingsTableName}_{collectionName}")}.[memory_id], 317 | SUM([embedding].[vector_value] * {this.GetFullTableName($"{this._configuration.EmbeddingsTableName}_{collectionName}")}.[vector_value]) / 318 | ( 319 | SQRT(SUM([embedding].[vector_value] * [embedding].[vector_value])) 320 | * 321 | SQRT(SUM({this.GetFullTableName($"{this._configuration.EmbeddingsTableName}_{collectionName}")}.[vector_value] * {this.GetFullTableName($"{this._configuration.EmbeddingsTableName}_{collectionName}")}.[vector_value])) 322 | ) AS cosine_similarity 323 | -- sum([embedding].[vector_value] * {this.GetFullTableName($"{this._configuration.EmbeddingsTableName}_{collectionName}")}.[vector_value]) as cosine_distance -- Optimized as per https://platform.openai.com/docs/guides/embeddings/which-distance-function-should-i-use 324 | FROM 325 | [embedding] 326 | INNER JOIN 327 | {this.GetFullTableName($"{this._configuration.EmbeddingsTableName}_{collectionName}")} ON [embedding].vector_value_id = {this.GetFullTableName($"{this._configuration.EmbeddingsTableName}_{collectionName}")}.vector_value_id 328 | GROUP BY 329 | {this.GetFullTableName($"{this._configuration.EmbeddingsTableName}_{collectionName}")}.[memory_id] 330 | ORDER BY 331 | cosine_similarity DESC 332 | ) 333 | SELECT 334 | {this.GetFullTableName(this._configuration.MemoryTableName)}.[id], 335 | {this.GetFullTableName(this._configuration.MemoryTableName)}.[key], 336 | {this.GetFullTableName(this._configuration.MemoryTableName)}.[metadata], 337 | {this.GetFullTableName(this._configuration.MemoryTableName)}.[timestamp], 338 | {this.GetFullTableName(this._configuration.MemoryTableName)}.[embedding], 339 | ( 340 | SELECT 341 | [vector_value] 342 | FROM {this.GetFullTableName($"{this._configuration.EmbeddingsTableName}_{collectionName}")} 343 | WHERE {this.GetFullTableName(this._configuration.MemoryTableName)}.[id] = {this.GetFullTableName($"{this._configuration.EmbeddingsTableName}_{collectionName}")}.[memory_id] 344 | ORDER BY vector_value_id 345 | FOR JSON AUTO 346 | ) AS [embeddings], 347 | [similarity].[cosine_similarity] 348 | FROM 349 | [similarity] 350 | INNER JOIN 351 | {this.GetFullTableName(this._configuration.MemoryTableName)} ON [similarity].[memory_id] = {this.GetFullTableName(this._configuration.MemoryTableName)}.[id] 352 | WHERE [cosine_similarity] >= @min_relevance_score 353 | ORDER BY [cosine_similarity] desc"; 354 | 355 | cmd.Parameters.AddWithValue("@vector", embedding); 356 | cmd.Parameters.AddWithValue("@collection", collectionName); 357 | cmd.Parameters.AddWithValue("@min_relevance_score", minRelevanceScore); 358 | cmd.Parameters.AddWithValue("@limit", limit); 359 | 360 | using var dataReader = await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); 361 | 362 | while (await dataReader.ReadAsync(cancellationToken).ConfigureAwait(false)) 363 | { 364 | double cosineSimilarity = dataReader.GetDouble(dataReader.GetOrdinal("cosine_similarity")); 365 | yield return (await this.ReadEntryAsync(dataReader, withEmbeddings, cancellationToken).ConfigureAwait(false), cosineSimilarity); 366 | } 367 | } 368 | 369 | /// 370 | public async Task UpsertAsync(string collectionName, 371 | string key, 372 | string? metadata, 373 | string embedding, 374 | DateTimeOffset? timestamp, 375 | CancellationToken cancellationToken = default) 376 | { 377 | collectionName = NormalizeIndexName(collectionName); 378 | 379 | using SqlCommand cmd = this._connection.CreateCommand(); 380 | 381 | cmd.CommandText = $@" 382 | MERGE INTO {this.GetFullTableName(this._configuration.MemoryTableName)} 383 | USING (SELECT @key) as [src]([key]) 384 | ON {this.GetFullTableName(this._configuration.MemoryTableName)}.[key] = [src].[key] 385 | WHEN MATCHED THEN 386 | UPDATE SET metadata=@metadata, embedding=@embedding, timestamp=@timestamp 387 | WHEN NOT MATCHED THEN 388 | INSERT ([id], [collection], [key], [metadata], [timestamp], [embedding]) 389 | VALUES (NEWID(), @collection, @key, @metadata, @timestamp, @embedding); 390 | 391 | 392 | MERGE {this.GetFullTableName($"{this._configuration.EmbeddingsTableName}_{collectionName}")} AS [tgt] 393 | USING ( 394 | SELECT 395 | {this.GetFullTableName(this._configuration.MemoryTableName)}.[id], 396 | cast([vector].[key] AS INT) AS [vector_value_id], 397 | cast([vector].[value] AS FLOAT) AS [vector_value] 398 | FROM {this.GetFullTableName(this._configuration.MemoryTableName)} 399 | CROSS APPLY 400 | openjson(@embedding) [vector] 401 | WHERE {this.GetFullTableName(this._configuration.MemoryTableName)}.[key] = @key 402 | AND {this.GetFullTableName(this._configuration.MemoryTableName)}.[collection] = @collection 403 | ) AS [src] 404 | ON [tgt].[memory_id] = [src].[id] AND [tgt].[vector_value_id] = [src].[vector_value_id] 405 | WHEN MATCHED THEN 406 | UPDATE SET [tgt].[vector_value] = [src].[vector_value] 407 | WHEN NOT MATCHED THEN 408 | INSERT ([memory_id], [vector_value_id], [vector_value]) 409 | VALUES ([src].[id], 410 | [src].[vector_value_id], 411 | [src].[vector_value] ) 412 | ;"; 413 | 414 | cmd.Parameters.AddWithValue("@collection", collectionName); 415 | cmd.Parameters.AddWithValue("@key", key); 416 | cmd.Parameters.AddWithValue("@metadata", metadata ?? (object)DBNull.Value); 417 | cmd.Parameters.AddWithValue("@embedding", embedding); 418 | cmd.Parameters.AddWithValue("@timestamp", timestamp ?? (object)DBNull.Value); 419 | 420 | await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); 421 | } 422 | 423 | private string GetFullTableName(string tableName) 424 | { 425 | return $"[{this._configuration.Schema}].[{tableName}]"; 426 | } 427 | 428 | 429 | /// 430 | private async Task ReadEntryAsync(SqlDataReader dataReader, bool withEmbedding, CancellationToken cancellationToken = default) 431 | { 432 | var entry = new SqlServerMemoryEntry(); 433 | 434 | entry.Id = dataReader.GetGuid(dataReader.GetOrdinal("id")); 435 | entry.Key = dataReader.GetString(dataReader.GetOrdinal("key")); 436 | 437 | if (!(await dataReader.IsDBNullAsync(dataReader.GetOrdinal("metadata"), cancellationToken).ConfigureAwait(false))) 438 | { 439 | entry.MetadataString = dataReader.GetString(dataReader.GetOrdinal("metadata")); 440 | } 441 | 442 | if (!(await dataReader.IsDBNullAsync(dataReader.GetOrdinal("timestamp"), cancellationToken).ConfigureAwait(false))) 443 | { 444 | entry.Timestamp = await dataReader.GetFieldValueAsync(dataReader.GetOrdinal("timestamp"), cancellationToken).ConfigureAwait(false); 445 | } 446 | 447 | if (withEmbedding) 448 | { 449 | entry.Embedding = new ReadOnlyMemory(JsonSerializer.Deserialize>(dataReader.GetString(dataReader.GetOrdinal("embedding")))!.ToArray()); 450 | } 451 | 452 | return entry; 453 | } 454 | 455 | // Note: "_" is allowed in Postgres, but we normalize it to "-" for consistency with other DBs 456 | private static readonly Regex s_replaceIndexNameCharsRegex = new(@"[\s|\\|/|.|_|:]"); 457 | private const string ValidSeparator = "-"; 458 | 459 | private static string NormalizeIndexName(string index) 460 | { 461 | index = s_replaceIndexNameCharsRegex.Replace(index.Trim().ToLowerInvariant(), ValidSeparator); 462 | 463 | return index; 464 | } 465 | 466 | public void Dispose() 467 | { 468 | if (this._connection != null) 469 | { 470 | this._connection.Dispose(); 471 | this._connection = null!; 472 | } 473 | } 474 | } 475 | -------------------------------------------------------------------------------- /src/Connectors.Memory.SqlServer/SqlServerConfig.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Kevin BEAUGRAND. All rights reserved. 2 | 3 | namespace SemanticKernel.Connectors.Memory.SqlServer; 4 | 5 | /// 6 | /// Configuration for the SQL Server memory store. 7 | /// 8 | public class SqlServerConfig 9 | { 10 | /// 11 | /// The default SQL Server collections table name. 12 | /// 13 | internal const string DefaultMemoryCollectionTableName = "SKMemoryCollections"; 14 | 15 | /// 16 | /// The default SQL Server memories table name. 17 | /// 18 | internal const string DefaultMemoryTableName = "SKMemories"; 19 | 20 | /// 21 | /// The default SQL Server embeddings table name. 22 | /// 23 | internal const string DefaultEmbeddingsTableName = "SKEmbeddings"; 24 | 25 | /// 26 | /// The default schema used by the SQL Server memory store. 27 | /// 28 | public const string DefaultSchema = "dbo"; 29 | 30 | /// 31 | /// The connection string to the SQL Server database. 32 | /// 33 | public string ConnectionString { get; set; } = null!; 34 | 35 | /// 36 | /// The schema used by the SQL Server memory store. 37 | /// 38 | public string Schema { get; set; } = DefaultSchema; 39 | 40 | /// 41 | /// The SQL Server collections table name. 42 | /// 43 | public string MemoryCollectionTableName { get; set; } = DefaultMemoryCollectionTableName; 44 | 45 | /// 46 | /// The SQL Server memories table name. 47 | /// 48 | public string MemoryTableName { get; set; } = DefaultMemoryTableName; 49 | 50 | /// 51 | /// The SQL Server embeddings table name. 52 | /// 53 | public string EmbeddingsTableName { get; set; } = DefaultEmbeddingsTableName; 54 | } 55 | -------------------------------------------------------------------------------- /src/Connectors.Memory.SqlServer/SqlServerMemoryEntry.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Kevin BEAUGRAND. All rights reserved. 2 | 3 | using System; 4 | 5 | namespace SemanticKernel.Connectors.Memory.SqlServer; 6 | 7 | /// 8 | /// A sql memory entry. 9 | /// 10 | public record struct SqlServerMemoryEntry 11 | { 12 | /// 13 | /// The unique identitfier of the memory entry. 14 | /// 15 | public Guid Id { get; set; } 16 | 17 | /// 18 | /// The entry collection name. 19 | /// 20 | public string Collection { get; set; } 21 | 22 | /// 23 | /// Unique identifier of the memory entry in the collection. 24 | /// 25 | public string Key { get; set; } 26 | 27 | /// 28 | /// Metadata as a string. 29 | /// 30 | public string MetadataString { get; set; } 31 | 32 | /// 33 | /// The embedding. 34 | /// 35 | public ReadOnlyMemory? Embedding { get; set; } 36 | 37 | /// 38 | /// Optional timestamp. Its 'DateTimeKind' is 39 | /// 40 | public DateTimeOffset? Timestamp { get; set; } 41 | } 42 | -------------------------------------------------------------------------------- /src/Connectors.Memory.SqlServer/SqlServerMemoryStore.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Kevin BEAUGRAND. All rights reserved. 2 | 3 | using Microsoft.SemanticKernel.Memory; 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Linq; 7 | using System.Runtime.CompilerServices; 8 | using System.Text.Json; 9 | using System.Threading; 10 | using System.Threading.Tasks; 11 | 12 | namespace SemanticKernel.Connectors.Memory.SqlServer; 13 | 14 | /// 15 | /// An implementation of backed by a SQL server database. 16 | /// 17 | /// The data is saved to a MSSQL server, specified in the connection string of the factory method. 18 | /// The data persists between subsequent instances. 19 | /// 20 | #pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. 21 | public sealed class SqlServerMemoryStore : IMemoryStore, IDisposable 22 | { 23 | private ISqlServerClient _dbClient; 24 | 25 | /// 26 | /// Connects to a SQL Server database using the provided connection string and schema, and returns a new instance of . 27 | /// 28 | /// The connection string to use for connecting to the SQL Server database. 29 | /// The SQL server configuration. 30 | /// A cancellation token that can be used to cancel the asynchronous operation. 31 | /// A new instance of connected to the specified SQL Server database. 32 | public static async Task ConnectAsync(string connectionString, SqlServerConfig config = default, CancellationToken cancellationToken = default) 33 | { 34 | var client = new SqlServerClient(connectionString, config ?? new()); 35 | 36 | await client.CreateTablesAsync(cancellationToken).ConfigureAwait(false); 37 | 38 | return new SqlServerMemoryStore(client); 39 | } 40 | 41 | /// 42 | /// Represents a memory store implementation that uses a SQL Server database as its backing store. 43 | /// 44 | public SqlServerMemoryStore(ISqlServerClient dbClient) 45 | { 46 | this._dbClient = dbClient; 47 | } 48 | 49 | /// 50 | public async Task CreateCollectionAsync(string collectionName, CancellationToken cancellationToken = default) 51 | { 52 | await this._dbClient.CreateCollectionAsync(collectionName, cancellationToken).ConfigureAwait(false); 53 | } 54 | 55 | /// 56 | public async Task DoesCollectionExistAsync(string collectionName, CancellationToken cancellationToken = default) 57 | { 58 | return await this._dbClient.DoesCollectionExistsAsync(collectionName, cancellationToken).ConfigureAwait(false); 59 | } 60 | 61 | /// 62 | public async IAsyncEnumerable GetCollectionsAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) 63 | { 64 | await foreach (var collection in this._dbClient.GetCollectionsAsync(cancellationToken)) 65 | { 66 | yield return collection; 67 | } 68 | } 69 | 70 | /// 71 | public async Task DeleteCollectionAsync(string collectionName, CancellationToken cancellationToken = default) 72 | { 73 | await this._dbClient.DeleteCollectionAsync(collectionName, cancellationToken).ConfigureAwait(false); 74 | } 75 | 76 | /// 77 | public async Task GetAsync(string collectionName, string key, bool withEmbedding = false, CancellationToken cancellationToken = default) 78 | { 79 | if (string.IsNullOrWhiteSpace(collectionName)) 80 | { 81 | throw new ArgumentNullException(nameof(collectionName)); 82 | } 83 | 84 | SqlServerMemoryEntry? entry = await this._dbClient.ReadAsync(collectionName, key, withEmbedding, cancellationToken).ConfigureAwait(false); 85 | 86 | if (!entry.HasValue) { return null; } 87 | 88 | return this.GetMemoryRecordFromEntry(entry.Value); 89 | } 90 | 91 | /// 92 | public async IAsyncEnumerable GetBatchAsync(string collectionName, IEnumerable keys, bool withEmbeddings = false, 93 | [EnumeratorCancellation] CancellationToken cancellationToken = default) 94 | { 95 | if (string.IsNullOrWhiteSpace(collectionName)) 96 | { 97 | throw new ArgumentNullException(nameof(collectionName)); 98 | } 99 | 100 | await foreach (SqlServerMemoryEntry entry in this._dbClient.ReadBatchAsync(collectionName, keys, withEmbeddings, cancellationToken).ConfigureAwait(false)) 101 | { 102 | yield return this.GetMemoryRecordFromEntry(entry); 103 | } 104 | } 105 | 106 | /// 107 | public async Task<(MemoryRecord, double)?> GetNearestMatchAsync(string collectionName, ReadOnlyMemory embedding, double minRelevanceScore = 0, bool withEmbedding = false, CancellationToken cancellationToken = default) 108 | { 109 | var nearest = this.GetNearestMatchesAsync( 110 | collectionName: collectionName, 111 | embedding: embedding, 112 | limit: 1, 113 | minRelevanceScore: minRelevanceScore, 114 | withEmbeddings: withEmbedding, 115 | cancellationToken: cancellationToken) 116 | .WithCancellation(cancellationToken); 117 | 118 | await foreach (var item in nearest) 119 | { 120 | return item; 121 | } 122 | 123 | return null; 124 | } 125 | 126 | /// 127 | public async IAsyncEnumerable<(MemoryRecord, double)> GetNearestMatchesAsync(string collectionName, ReadOnlyMemory embedding, int limit, double minRelevanceScore = 0, bool withEmbeddings = false, [EnumeratorCancellation] CancellationToken cancellationToken = default) 128 | { 129 | if (string.IsNullOrWhiteSpace(collectionName)) 130 | { 131 | throw new ArgumentNullException(nameof(collectionName)); 132 | } 133 | 134 | if (limit <= 0) 135 | { 136 | yield break; 137 | } 138 | 139 | IAsyncEnumerable<(SqlServerMemoryEntry, double)> results = this._dbClient.GetNearestMatchesAsync( 140 | collectionName: collectionName, 141 | embedding: JsonSerializer.Serialize(embedding.ToArray()), 142 | limit: limit, 143 | minRelevanceScore: minRelevanceScore, 144 | withEmbeddings: withEmbeddings, 145 | cancellationToken: cancellationToken); 146 | 147 | await foreach ((SqlServerMemoryEntry entry, double cosineSimilarity) in results.ConfigureAwait(false)) 148 | { 149 | yield return (this.GetMemoryRecordFromEntry(entry), cosineSimilarity); 150 | } 151 | } 152 | 153 | /// 154 | public async Task RemoveAsync(string collectionName, string key, CancellationToken cancellationToken = default) 155 | { 156 | if (string.IsNullOrWhiteSpace(collectionName)) 157 | { 158 | throw new ArgumentNullException(nameof(collectionName)); 159 | } 160 | 161 | await this._dbClient.DeleteAsync(collectionName, key, cancellationToken).ConfigureAwait(false); 162 | } 163 | 164 | /// 165 | public async Task RemoveBatchAsync(string collectionName, IEnumerable keys, CancellationToken cancellationToken = default) 166 | { 167 | if (string.IsNullOrWhiteSpace(collectionName)) 168 | { 169 | throw new ArgumentNullException(nameof(collectionName)); 170 | } 171 | 172 | await this._dbClient.DeleteBatchAsync(collectionName, keys, cancellationToken).ConfigureAwait(false); 173 | } 174 | 175 | 176 | /// 177 | public async Task UpsertAsync(string collectionName, MemoryRecord record, CancellationToken cancellationToken = default) 178 | { 179 | if (string.IsNullOrWhiteSpace(collectionName)) 180 | { 181 | throw new ArgumentNullException(nameof(collectionName)); 182 | } 183 | 184 | return await this.InternalUpsertAsync(collectionName, record, cancellationToken).ConfigureAwait(false); 185 | } 186 | 187 | /// 188 | public async IAsyncEnumerable UpsertBatchAsync( 189 | string collectionName, 190 | IEnumerable records, 191 | [EnumeratorCancellation] CancellationToken cancellationToken = default) 192 | { 193 | if (string.IsNullOrWhiteSpace(collectionName)) 194 | { 195 | throw new ArgumentNullException(nameof(collectionName)); 196 | } 197 | 198 | foreach (MemoryRecord record in records) 199 | { 200 | yield return await this.InternalUpsertAsync(collectionName, record, cancellationToken).ConfigureAwait(false); 201 | } 202 | } 203 | 204 | /// 205 | private MemoryRecord GetMemoryRecordFromEntry(SqlServerMemoryEntry entry) 206 | { 207 | return MemoryRecord.FromJsonMetadata( 208 | json: entry.MetadataString, 209 | embedding: entry.Embedding ?? new ReadOnlyMemory(), 210 | key: entry.Key, 211 | timestamp: entry.Timestamp 212 | ); 213 | } 214 | 215 | /// 216 | private async Task InternalUpsertAsync(string collectionName, MemoryRecord record, CancellationToken cancellationToken) 217 | { 218 | record.Key = record.Metadata.Id; 219 | 220 | await this._dbClient.UpsertAsync( 221 | collectionName: collectionName, 222 | key: record.Key, 223 | metadata: record.GetSerializedMetadata(), 224 | embedding: JsonSerializer.Serialize(record.Embedding.ToArray()), 225 | timestamp: record.Timestamp, 226 | cancellationToken: cancellationToken).ConfigureAwait(false); 227 | 228 | return record.Key; 229 | } 230 | 231 | public void Dispose() 232 | { 233 | if (this._dbClient is not null) 234 | { 235 | this._dbClient.Dispose(); 236 | this._dbClient = null!; 237 | } 238 | } 239 | } 240 | #pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. 241 | -------------------------------------------------------------------------------- /src/Directory.Build.props: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | true 5 | true 6 | AllEnabledByDefault 7 | latest 8 | true 9 | 10 10 | enable 11 | disable 12 | 13 | 14 | 15 | 16 | disable 17 | 18 | 19 | 20 | true 21 | full 22 | 23 | 24 | 25 | portable 26 | 27 | 28 | 29 | $([System.IO.Path]::GetDirectoryName($([MSBuild]::GetPathOfFileAbove('.gitignore', '$(MSBuildThisFileDirectory)')))) 30 | 31 | 32 | 33 | 34 | 35 | <_Parameter1>false 36 | 37 | 38 | -------------------------------------------------------------------------------- /src/Directory.Build.targets: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /src/Directory.Packages.props: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | true 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 45 | 46 | 47 | all 48 | runtime; build; native; contentfiles; analyzers; buildtransitive 49 | 50 | 51 | 52 | all 53 | runtime; build; native; contentfiles; analyzers; buildtransitive 54 | 55 | 56 | 57 | all 58 | runtime; build; native; contentfiles; analyzers; buildtransitive 59 | 60 | 61 | 62 | all 63 | runtime; build; native; contentfiles; analyzers; buildtransitive 64 | 65 | 66 | 67 | all 68 | runtime; build; native; contentfiles; analyzers; buildtransitive 69 | 70 | 71 | 72 | all 73 | runtime; build; native; contentfiles; analyzers; buildtransitive 74 | 75 | 76 | 77 | all 78 | runtime; build; native; contentfiles; analyzers; buildtransitive 79 | 80 | 81 | 82 | all 83 | runtime; build; native; contentfiles; analyzers; buildtransitive 84 | 85 | 86 | -------------------------------------------------------------------------------- /src/KernelMemory.MemoryStorage.SqlServer/KernelMemory.MemoryStorage.SqlServer.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | $(AssemblyName) 5 | netstandard2.0 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | Semantic Kernel - SQL Connector 14 | MSSQL connector for Semantic Kernel skills and semantic memory 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /src/KernelMemory.MemoryStorage.SqlServer/KernelMemoryBuilderExtensions.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.DependencyInjection; 2 | using Microsoft.KernelMemory; 3 | using Microsoft.KernelMemory.MemoryStorage; 4 | 5 | namespace KernelMemory.MemoryStorage.SqlServer; 6 | 7 | /// 8 | /// Extensions for KernelMemoryBuilder 9 | /// 10 | public static partial class KernelMemoryBuilderExtensions 11 | { 12 | /// 13 | /// Kernel Memory Builder extension method to add SqlServer memory connector. 14 | /// 15 | /// KM builder instance 16 | /// SqlServer configuration 17 | public static IKernelMemoryBuilder WithSqlServerMemoryDb(this IKernelMemoryBuilder builder, SqlServerConfig config) 18 | { 19 | builder.Services.AddSqlServerAsMemoryDb(config); 20 | return builder; 21 | } 22 | 23 | /// 24 | /// Kernel Memory Builder extension method to add SqlServer memory connector. 25 | /// 26 | /// KM builder instance 27 | /// SqlServer connection string 28 | public static IKernelMemoryBuilder WithSqlServerMemoryDb(this IKernelMemoryBuilder builder, string connString) 29 | { 30 | builder.Services.AddSqlServerAsMemoryDb(connString); 31 | return builder; 32 | } 33 | } 34 | 35 | /// 36 | /// Extensions for KernelMemoryBuilder and generic DI 37 | /// 38 | public static partial class DependencyInjection 39 | { 40 | /// 41 | /// Inject SqlServer as the default implementation of IMemoryDb 42 | /// 43 | /// Service collection 44 | /// Postgres configuration 45 | public static IServiceCollection AddSqlServerAsMemoryDb(this IServiceCollection services, SqlServerConfig config) 46 | { 47 | return services 48 | .AddSingleton(config) 49 | .AddSingleton(); 50 | } 51 | 52 | /// 53 | /// Inject SqlServer as the default implementation of IMemoryDb 54 | /// 55 | /// Service collection 56 | /// Postgres connection string 57 | public static IServiceCollection AddSqlServerAsMemoryDb(this IServiceCollection services, string connString) 58 | { 59 | var config = new SqlServerConfig { ConnectionString = connString }; 60 | return services.AddSqlServerAsMemoryDb(config); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/KernelMemory.MemoryStorage.SqlServer/NUGET.md: -------------------------------------------------------------------------------- 1 | # SQL Server memory for Kernel Memory 2 | 3 | This is a connector for the Kernel Memory. 4 | 5 | It provides a connection to a SQL database for the Kernel Memory. 6 | 7 | ## About Kernel Memory 8 | 9 | **Kernel Memory** (KM) is a multi-modal **AI Service** specialized in the efficient indexing of datasets through custom continuous data hybrid pipelines, with support for **Retrieval Augmented Generation (RAG)**, synthetic memory, prompt engineering, and custom semantic memory processing. 10 | 11 | ## Installation 12 | 13 | ```bash 14 | dotnet add package KernelMemory.MemoryStorage.SqlServer 15 | ``` 16 | 17 | ## Usage 18 | 19 | To add your SQL Server memory connector, add the following statements to your kernel memory initialization code: 20 | 21 | ```csharp 22 | using SemanticKernel.Connectors.Memory.SqlServer; 23 | ... 24 | var memory = new KernelMemoryBuilder() 25 | .WithOpenAIDefaults(Env.Var("OPENAI_API_KEY")) 26 | .WithSqlServerMemoryDb("YouSqlConnectionString") 27 | ... 28 | .Build(); 29 | ``` 30 | 31 | The memory store will populate all the needed tables during startup and let you focus on the development of your plugin. 32 | 33 | ## License 34 | 35 | This project is licensed under the [MIT License](LICENSE). -------------------------------------------------------------------------------- /src/KernelMemory.MemoryStorage.SqlServer/SqlServerConfig.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Kevin BEAUGRAND. All rights reserved. 2 | 3 | namespace KernelMemory.MemoryStorage.SqlServer; 4 | 5 | /// 6 | /// Configuration for the SQL Server memory store. 7 | /// 8 | public class SqlServerConfig 9 | { 10 | /// 11 | /// The default SQL Server collections table name. 12 | /// 13 | internal const string DefaultMemoryCollectionTableName = "SKMemoryCollections"; 14 | 15 | /// 16 | /// The default SQL Server memories table name. 17 | /// 18 | internal const string DefaultMemoryTableName = "SKMemories"; 19 | 20 | /// 21 | /// The default SQL Server embeddings table name. 22 | /// 23 | internal const string DefaultEmbeddingsTableName = "SKEmbeddings"; 24 | 25 | /// 26 | /// The default SQL Server tags table name. 27 | /// 28 | internal const string DefaultTagsTableName = "SKMemoriesTags"; 29 | 30 | /// 31 | /// The default schema used by the SQL Server memory store. 32 | /// 33 | public const string DefaultSchema = "dbo"; 34 | 35 | /// 36 | /// The connection string to the SQL Server database. 37 | /// 38 | public string ConnectionString { get; set; } = null!; 39 | 40 | /// 41 | /// The schema used by the SQL Server memory store. 42 | /// 43 | public string Schema { get; set; } = DefaultSchema; 44 | 45 | /// 46 | /// The SQL Server collections table name. 47 | /// 48 | public string MemoryCollectionTableName { get; set; } = DefaultMemoryCollectionTableName; 49 | 50 | /// 51 | /// The SQL Server memories table name. 52 | /// 53 | public string MemoryTableName { get; set; } = DefaultMemoryTableName; 54 | 55 | /// 56 | /// The SQL Server embeddings table name. 57 | /// 58 | public string EmbeddingsTableName { get; set; } = DefaultEmbeddingsTableName; 59 | 60 | /// 61 | /// The SQL Server tags table name. 62 | /// 63 | public string TagsTableName { get; set; } = DefaultTagsTableName; 64 | } 65 | -------------------------------------------------------------------------------- /src/KernelMemory.MemoryStorage.SqlServer/SqlServerMemory.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Kevin BEAUGRAND. All rights reserved. 2 | 3 | using Microsoft.Data.SqlClient; 4 | using Microsoft.Extensions.Logging; 5 | using Microsoft.KernelMemory; 6 | using Microsoft.KernelMemory.AI; 7 | using Microsoft.KernelMemory.Diagnostics; 8 | using Microsoft.KernelMemory.MemoryStorage; 9 | using System; 10 | using System.Collections.Generic; 11 | using System.Globalization; 12 | using System.Linq; 13 | using System.Runtime.CompilerServices; 14 | using System.Text; 15 | using System.Text.Json; 16 | using System.Text.RegularExpressions; 17 | using System.Threading; 18 | using System.Threading.Tasks; 19 | 20 | namespace KernelMemory.MemoryStorage.SqlServer; 21 | 22 | /// 23 | /// Represents a memory store implementation that uses a SQL Server database as its backing store. 24 | /// 25 | [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities", 26 | Justification = "We need to build the full table name using schema and collection, it does not support parameterized passing.")] 27 | public class SqlServerMemory : IMemoryDb 28 | { 29 | /// 30 | /// The SQL Server configuration. 31 | /// 32 | private readonly SqlServerConfig _config; 33 | 34 | /// 35 | /// The text embedding generator. 36 | /// 37 | private readonly ITextEmbeddingGenerator _embeddingGenerator; 38 | 39 | /// 40 | /// The logger. 41 | /// 42 | private readonly ILogger _log; 43 | 44 | /// 45 | /// Initializes a new instance of the class. 46 | /// 47 | /// The SQL server instance configuration. 48 | /// The text embedding generator. 49 | /// The logger. 50 | public SqlServerMemory( 51 | SqlServerConfig config, 52 | ITextEmbeddingGenerator embeddingGenerator, 53 | ILogger? log = null) 54 | { 55 | this._embeddingGenerator = embeddingGenerator; 56 | this._log = log ?? DefaultLogger.Instance; 57 | 58 | this._config = config; 59 | 60 | if (this._embeddingGenerator == null) 61 | { 62 | throw new SqlServerMemoryException("Embedding generator not configured"); 63 | } 64 | 65 | this.CreateTablesIfNotExists(); 66 | } 67 | 68 | /// 69 | public async Task CreateIndexAsync(string index, int vectorSize, CancellationToken cancellationToken = default) 70 | { 71 | index = NormalizeIndexName(index); 72 | 73 | if (await this.DoesIndexExistsAsync(index, cancellationToken).ConfigureAwait(false)) 74 | { 75 | // Index already exists 76 | return; 77 | } 78 | 79 | using var connection = new SqlConnection(this._config.ConnectionString); 80 | await connection.OpenAsync(cancellationToken).ConfigureAwait(false); 81 | 82 | using (SqlCommand command = connection.CreateCommand()) 83 | { 84 | command.CommandText = $@" 85 | BEGIN TRANSACTION; 86 | 87 | INSERT INTO {this.GetFullTableName(this._config.MemoryCollectionTableName)}([id]) 88 | VALUES (@index); 89 | 90 | IF OBJECT_ID(N'{this.GetFullTableName($"{this._config.EmbeddingsTableName}_{index}")}', N'U') IS NULL 91 | CREATE TABLE {this.GetFullTableName($"{this._config.EmbeddingsTableName}_{index}")} 92 | ( 93 | [memory_id] UNIQUEIDENTIFIER NOT NULL, 94 | [vector_value_id] [int] NOT NULL, 95 | [vector_value] [float] NOT NULL 96 | FOREIGN KEY ([memory_id]) REFERENCES {this.GetFullTableName(this._config.MemoryTableName)}([id]) 97 | ); 98 | 99 | IF OBJECT_ID(N'[{this._config.Schema}.IXC_{$"{this._config.EmbeddingsTableName}_{index}"}]', N'U') IS NULL 100 | CREATE CLUSTERED COLUMNSTORE INDEX [IXC_{$"{this._config.EmbeddingsTableName}_{index}"}] 101 | ON {this.GetFullTableName($"{this._config.EmbeddingsTableName}_{index}")} 102 | { (this.GetSqlServerMajorVersionNumber() >= 16 ? "ORDER ([memory_id])" : "")}; 103 | 104 | IF OBJECT_ID(N'{this.GetFullTableName($"{this._config.TagsTableName}_{index}")}', N'U') IS NULL 105 | CREATE TABLE {this.GetFullTableName($"{this._config.TagsTableName}_{index}")} 106 | ( 107 | [memory_id] UNIQUEIDENTIFIER NOT NULL, 108 | [name] NVARCHAR(256) NOT NULL, 109 | [value] NVARCHAR(256) NOT NULL, 110 | FOREIGN KEY ([memory_id]) REFERENCES {this.GetFullTableName(this._config.MemoryTableName)}([id]) 111 | ); 112 | 113 | COMMIT; 114 | "; 115 | 116 | command.Parameters.AddWithValue("@index", index); 117 | 118 | await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); 119 | }; 120 | } 121 | 122 | /// 123 | public async Task DeleteAsync(string index, MemoryRecord record, CancellationToken cancellationToken = default) 124 | { 125 | index = NormalizeIndexName(index); 126 | 127 | if (!(await this.DoesIndexExistsAsync(index, cancellationToken).ConfigureAwait(false))) 128 | { 129 | // Index does not exist 130 | return; 131 | } 132 | 133 | using var connection = new SqlConnection(this._config.ConnectionString); 134 | await connection.OpenAsync(cancellationToken).ConfigureAwait(false); 135 | 136 | using (SqlCommand command = connection.CreateCommand()) 137 | { 138 | command.CommandText = $@" 139 | BEGIN TRANSACTION; 140 | 141 | DELETE [embeddings] 142 | FROM {this.GetFullTableName($"{this._config.EmbeddingsTableName}_{index}")} [embeddings] 143 | INNER JOIN {this.GetFullTableName(this._config.MemoryTableName)} ON [embeddings].[memory_id] = {this.GetFullTableName(this._config.MemoryTableName)}.[id] 144 | WHERE 145 | {this.GetFullTableName(this._config.MemoryTableName)}.[collection] = @index 146 | AND {this.GetFullTableName(this._config.MemoryTableName)}.[key]=@key; 147 | 148 | DELETE [tags] 149 | FROM {this.GetFullTableName($"{this._config.TagsTableName}_{index}")} [tags] 150 | INNER JOIN {this.GetFullTableName(this._config.MemoryTableName)} ON [tags].[memory_id] = {this.GetFullTableName(this._config.MemoryTableName)}.[id] 151 | WHERE 152 | {this.GetFullTableName(this._config.MemoryTableName)}.[collection] = @index 153 | AND {this.GetFullTableName(this._config.MemoryTableName)}.[key]=@key; 154 | 155 | DELETE FROM {this.GetFullTableName(this._config.MemoryTableName)} WHERE [collection] = @index AND [key]=@key; 156 | 157 | COMMIT;"; 158 | 159 | command.Parameters.AddWithValue("@index", index); 160 | command.Parameters.AddWithValue("@key", record.Id); 161 | 162 | await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); 163 | 164 | } 165 | } 166 | 167 | /// 168 | public async Task DeleteIndexAsync(string index, CancellationToken cancellationToken = default) 169 | { 170 | index = NormalizeIndexName(index); 171 | 172 | if (!(await this.DoesIndexExistsAsync(index, cancellationToken).ConfigureAwait(false))) 173 | { 174 | // Index does not exist 175 | return; 176 | } 177 | 178 | using var connection = new SqlConnection(this._config.ConnectionString); 179 | await connection.OpenAsync(cancellationToken).ConfigureAwait(false); 180 | 181 | using (SqlCommand command = connection.CreateCommand()) 182 | { 183 | command.CommandText = $@" 184 | BEGIN TRANSACTION; 185 | 186 | DROP TABLE {this.GetFullTableName($"{this._config.EmbeddingsTableName}_{index}")}; 187 | DROP TABLE {this.GetFullTableName($"{this._config.TagsTableName}_{index}")}; 188 | 189 | DELETE FROM {this.GetFullTableName(this._config.MemoryCollectionTableName)} 190 | WHERE [id] = @index; 191 | 192 | COMMIT;"; 193 | 194 | command.Parameters.AddWithValue("@index", index); 195 | 196 | await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); 197 | }; 198 | } 199 | 200 | /// 201 | public async Task> GetIndexesAsync(CancellationToken cancellationToken = default) 202 | { 203 | List indexes = new(); 204 | 205 | using var connection = new SqlConnection(this._config.ConnectionString); 206 | await connection.OpenAsync(cancellationToken).ConfigureAwait(false); 207 | 208 | using (SqlCommand command = connection.CreateCommand()) 209 | { 210 | command.CommandText = $"SELECT [id] FROM {this.GetFullTableName(this._config.MemoryCollectionTableName)}"; 211 | 212 | using var dataReader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); 213 | 214 | while (await dataReader.ReadAsync(cancellationToken).ConfigureAwait(false)) 215 | { 216 | indexes.Add(dataReader.GetString(dataReader.GetOrdinal("id"))); 217 | } 218 | }; 219 | 220 | return indexes; 221 | } 222 | 223 | /// 224 | public async IAsyncEnumerable GetListAsync(string index, ICollection? filters = null, int limit = 1, bool withEmbeddings = false, [EnumeratorCancellation] CancellationToken cancellationToken = default) 225 | { 226 | index = NormalizeIndexName(index); 227 | 228 | if (!(await this.DoesIndexExistsAsync(index, cancellationToken).ConfigureAwait(false))) 229 | { 230 | // Index does not exist 231 | yield break; 232 | } 233 | 234 | string queryColumns = "[key], [payload], [tags]"; 235 | 236 | if (withEmbeddings) 237 | { 238 | queryColumns += ", [embedding]"; 239 | } 240 | 241 | if (limit < 0) 242 | { 243 | limit = int.MaxValue; 244 | } 245 | 246 | using var connection = new SqlConnection(this._config.ConnectionString); 247 | await connection.OpenAsync(cancellationToken).ConfigureAwait(false); 248 | 249 | using (SqlCommand command = connection.CreateCommand()) 250 | { 251 | var tagFilters = new TagCollection(); 252 | 253 | command.CommandText = $@" 254 | WITH [filters] AS 255 | ( 256 | SELECT 257 | cast([filters].[key] AS NVARCHAR(256)) COLLATE SQL_Latin1_General_CP1_CI_AS AS [name], 258 | cast([filters].[value] AS NVARCHAR(256)) COLLATE SQL_Latin1_General_CP1_CI_AS AS [value] 259 | FROM openjson(@filters) [filters] 260 | ) 261 | SELECT TOP (@limit) 262 | {queryColumns} 263 | FROM 264 | {this.GetFullTableName(this._config.MemoryTableName)} 265 | WHERE 1=1 266 | AND {this.GetFullTableName(this._config.MemoryTableName)}.[collection] = @index 267 | {GenerateFilters(index, command.Parameters, filters)};"; 268 | 269 | 270 | command.Parameters.AddWithValue("@index", index); 271 | command.Parameters.AddWithValue("@limit", limit); 272 | command.Parameters.AddWithValue("@filters", JsonSerializer.Serialize(tagFilters)); 273 | 274 | using var dataReader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); 275 | 276 | while (await dataReader.ReadAsync(cancellationToken).ConfigureAwait(false)) 277 | { 278 | yield return await this.ReadEntryAsync(dataReader, withEmbeddings, cancellationToken).ConfigureAwait(false); 279 | } 280 | 281 | } 282 | } 283 | 284 | /// 285 | public async IAsyncEnumerable<(MemoryRecord, double)> GetSimilarListAsync(string index, string text, ICollection? filters = null, double minRelevance = 0, int limit = 1, bool withEmbeddings = false, [EnumeratorCancellation] CancellationToken cancellationToken = default) 286 | { 287 | index = NormalizeIndexName(index); 288 | 289 | if (!(await this.DoesIndexExistsAsync(index, cancellationToken).ConfigureAwait(false))) 290 | { 291 | // Index does not exist 292 | yield break; 293 | } 294 | 295 | Embedding embedding = await this._embeddingGenerator.GenerateEmbeddingAsync(text, cancellationToken).ConfigureAwait(false); 296 | 297 | string queryColumns = "[id], [payload], [tags]"; 298 | 299 | if (withEmbeddings) 300 | { 301 | queryColumns += ", [embedding]"; 302 | } 303 | 304 | using var connection = new SqlConnection(this._config.ConnectionString); 305 | await connection.OpenAsync(cancellationToken).ConfigureAwait(false); 306 | 307 | using (SqlCommand command = connection.CreateCommand()) 308 | { 309 | var generatedFilters = GenerateFilters(index, command.Parameters, filters); 310 | 311 | command.CommandText = $@" 312 | WITH 313 | [embedding] as 314 | ( 315 | SELECT 316 | cast([key] AS INT) AS [vector_value_id], 317 | cast([value] AS FLOAT) AS [vector_value] 318 | FROM 319 | openjson(@vector) 320 | ), 321 | [similarity] AS 322 | ( 323 | SELECT TOP (@limit) 324 | {this.GetFullTableName($"{this._config.EmbeddingsTableName}_{index}")}.[memory_id], 325 | SUM([embedding].[vector_value] * {this.GetFullTableName($"{this._config.EmbeddingsTableName}_{index}")}.[vector_value]) / 326 | ( 327 | SQRT(SUM([embedding].[vector_value] * [embedding].[vector_value])) 328 | * 329 | SQRT(SUM({this.GetFullTableName($"{this._config.EmbeddingsTableName}_{index}")}.[vector_value] * {this.GetFullTableName($"{this._config.EmbeddingsTableName}_{index}")}.[vector_value])) 330 | ) AS cosine_similarity 331 | -- sum([embedding].[vector_value] * {this.GetFullTableName($"{this._config.EmbeddingsTableName}_{index}")}.[vector_value]) as cosine_distance -- Optimized as per https://platform.openai.com/docs/guides/embeddings/which-distance-function-should-i-use 332 | FROM 333 | [embedding] 334 | INNER JOIN 335 | {this.GetFullTableName($"{this._config.EmbeddingsTableName}_{index}")} ON [embedding].vector_value_id = {this.GetFullTableName($"{this._config.EmbeddingsTableName}_{index}")}.vector_value_id 336 | INNER JOIN 337 | {this.GetFullTableName(this._config.MemoryTableName)} ON {this.GetFullTableName($"{this._config.EmbeddingsTableName}_{index}")}.[memory_id] = {this.GetFullTableName(this._config.MemoryTableName)}.[id] 338 | WHERE 1=1 339 | {generatedFilters} 340 | GROUP BY 341 | {this.GetFullTableName($"{this._config.EmbeddingsTableName}_{index}")}.[memory_id] 342 | ORDER BY 343 | cosine_similarity DESC 344 | ) 345 | SELECT DISTINCT 346 | {this.GetFullTableName(this._config.MemoryTableName)}.[id], 347 | {this.GetFullTableName(this._config.MemoryTableName)}.[key], 348 | {this.GetFullTableName(this._config.MemoryTableName)}.[payload], 349 | {this.GetFullTableName(this._config.MemoryTableName)}.[tags], 350 | [similarity].[cosine_similarity] 351 | FROM 352 | [similarity] 353 | INNER JOIN 354 | {this.GetFullTableName(this._config.MemoryTableName)} ON [similarity].[memory_id] = {this.GetFullTableName(this._config.MemoryTableName)}.[id] 355 | WHERE 1=1 356 | AND [cosine_similarity] >= @min_relevance_score 357 | {generatedFilters} 358 | ORDER BY [cosine_similarity] desc"; 359 | 360 | command.Parameters.AddWithValue("@vector", JsonSerializer.Serialize(embedding.Data.ToArray())); 361 | command.Parameters.AddWithValue("@index", index); 362 | command.Parameters.AddWithValue("@min_relevance_score", minRelevance); 363 | command.Parameters.AddWithValue("@limit", limit); 364 | 365 | using var dataReader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); 366 | 367 | while (await dataReader.ReadAsync(cancellationToken).ConfigureAwait(false)) 368 | { 369 | double cosineSimilarity = dataReader.GetDouble(dataReader.GetOrdinal("cosine_similarity")); 370 | yield return (await this.ReadEntryAsync(dataReader, withEmbeddings, cancellationToken).ConfigureAwait(false), cosineSimilarity); 371 | } 372 | } 373 | } 374 | 375 | /// 376 | public async Task UpsertAsync(string index, MemoryRecord record, CancellationToken cancellationToken = default) 377 | { 378 | index = NormalizeIndexName(index); 379 | 380 | if (!(await this.DoesIndexExistsAsync(index, cancellationToken).ConfigureAwait(false))) 381 | { 382 | // Index does not exist 383 | return string.Empty; 384 | } 385 | 386 | using var connection = new SqlConnection(this._config.ConnectionString); 387 | await connection.OpenAsync(cancellationToken).ConfigureAwait(false); 388 | 389 | using (SqlCommand command = connection.CreateCommand()) 390 | { 391 | command.CommandText = $@" 392 | BEGIN TRANSACTION; 393 | 394 | MERGE INTO {this.GetFullTableName(this._config.MemoryTableName)} 395 | USING (SELECT @key) as [src]([key]) 396 | ON {this.GetFullTableName(this._config.MemoryTableName)}.[key] = [src].[key] 397 | WHEN MATCHED THEN 398 | UPDATE SET payload=@payload, embedding=@embedding, tags=@tags 399 | WHEN NOT MATCHED THEN 400 | INSERT ([id], [key], [collection], [payload], [tags], [embedding]) 401 | VALUES (NEWID(), @key, @index, @payload, @tags, @embedding); 402 | 403 | MERGE {this.GetFullTableName($"{this._config.EmbeddingsTableName}_{index}")} AS [tgt] 404 | USING ( 405 | SELECT 406 | {this.GetFullTableName(this._config.MemoryTableName)}.[id], 407 | cast([vector].[key] AS INT) AS [vector_value_id], 408 | cast([vector].[value] AS FLOAT) AS [vector_value] 409 | FROM {this.GetFullTableName(this._config.MemoryTableName)} 410 | CROSS APPLY 411 | openjson(@embedding) [vector] 412 | WHERE {this.GetFullTableName(this._config.MemoryTableName)}.[key] = @key 413 | AND {this.GetFullTableName(this._config.MemoryTableName)}.[collection] = @index 414 | ) AS [src] 415 | ON [tgt].[memory_id] = [src].[id] AND [tgt].[vector_value_id] = [src].[vector_value_id] 416 | WHEN MATCHED THEN 417 | UPDATE SET [tgt].[vector_value] = [src].[vector_value] 418 | WHEN NOT MATCHED THEN 419 | INSERT ([memory_id], [vector_value_id], [vector_value]) 420 | VALUES ([src].[id], 421 | [src].[vector_value_id], 422 | [src].[vector_value] ); 423 | 424 | DELETE FROM [tgt] 425 | FROM {this.GetFullTableName($"{this._config.TagsTableName}_{index}")} AS [tgt] 426 | INNER JOIN {this.GetFullTableName(this._config.MemoryTableName)} ON [tgt].[memory_id] = {this.GetFullTableName(this._config.MemoryTableName)}.[id] 427 | WHERE {this.GetFullTableName(this._config.MemoryTableName)}.[key] = @key 428 | AND {this.GetFullTableName(this._config.MemoryTableName)}.[collection] = @index; 429 | 430 | MERGE {this.GetFullTableName($"{this._config.TagsTableName}_{index}")} AS [tgt] 431 | USING ( 432 | SELECT 433 | {this.GetFullTableName(this._config.MemoryTableName)}.[id], 434 | cast([tags].[key] AS NVARCHAR(256)) COLLATE SQL_Latin1_General_CP1_CI_AS AS [tag_name], 435 | [tag_value].[value] AS [value] 436 | FROM {this.GetFullTableName(this._config.MemoryTableName)} 437 | CROSS APPLY openjson(@tags) [tags] 438 | CROSS APPLY openjson(cast([tags].[value] AS NVARCHAR(256)) COLLATE SQL_Latin1_General_CP1_CI_AS) [tag_value] 439 | WHERE {this.GetFullTableName(this._config.MemoryTableName)}.[key] = @key 440 | AND {this.GetFullTableName(this._config.MemoryTableName)}.[collection] = @index 441 | ) AS [src] 442 | ON [tgt].[memory_id] = [src].[id] AND [tgt].[name] = [src].[tag_name] 443 | WHEN MATCHED THEN 444 | UPDATE SET [tgt].[value] = [src].[value] 445 | WHEN NOT MATCHED THEN 446 | INSERT ([memory_id], [name], [value]) 447 | VALUES ([src].[id], 448 | [src].[tag_name], 449 | [src].[value]); 450 | 451 | COMMIT;"; 452 | 453 | command.Parameters.AddWithValue("@index", index); 454 | command.Parameters.AddWithValue("@key", record.Id); 455 | command.Parameters.AddWithValue("@payload", JsonSerializer.Serialize(record.Payload) ?? (object)DBNull.Value); 456 | command.Parameters.AddWithValue("@tags", JsonSerializer.Serialize(record.Tags) ?? (object)DBNull.Value); 457 | command.Parameters.AddWithValue("@embedding", JsonSerializer.Serialize(record.Vector.Data.ToArray())); 458 | 459 | await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); 460 | 461 | return record.Id; 462 | } 463 | } 464 | 465 | /// 466 | /// Creates the SQL Server tables if they do not exist. 467 | /// 468 | /// 469 | private void CreateTablesIfNotExists() 470 | { 471 | var sql = $@"IF NOT EXISTS (SELECT * 472 | FROM sys.schemas 473 | WHERE name = N'{this._config.Schema}' ) 474 | EXEC('CREATE SCHEMA [{this._config.Schema}]'); 475 | IF OBJECT_ID(N'{this.GetFullTableName(this._config.MemoryCollectionTableName)}', N'U') IS NULL 476 | CREATE TABLE {this.GetFullTableName(this._config.MemoryCollectionTableName)} 477 | ( [id] NVARCHAR(256) NOT NULL, 478 | PRIMARY KEY ([id]) 479 | ); 480 | 481 | IF OBJECT_ID(N'{this.GetFullTableName(this._config.MemoryTableName)}', N'U') IS NULL 482 | CREATE TABLE {this.GetFullTableName(this._config.MemoryTableName)} 483 | ( [id] UNIQUEIDENTIFIER NOT NULL, 484 | [key] NVARCHAR(256) NOT NULL, 485 | [collection] NVARCHAR(256) NOT NULL, 486 | [payload] NVARCHAR(MAX), 487 | [tags] NVARCHAR(MAX), 488 | [embedding] NVARCHAR(MAX), 489 | PRIMARY KEY ([id]), 490 | FOREIGN KEY ([collection]) REFERENCES {this.GetFullTableName(this._config.MemoryCollectionTableName)}([id]) ON DELETE CASCADE, 491 | CONSTRAINT UK_{this._config.MemoryTableName} UNIQUE([collection], [key]) 492 | ); 493 | "; 494 | 495 | using var connection = new SqlConnection(this._config.ConnectionString); 496 | connection.Open(); 497 | 498 | using (SqlCommand command = connection.CreateCommand()) 499 | { 500 | command.CommandText = sql; 501 | command.ExecuteNonQuery(); 502 | } 503 | } 504 | 505 | /// 506 | /// Checks if the index exists. 507 | /// 508 | /// The index name. 509 | /// The cancellation token. 510 | /// 511 | private async Task DoesIndexExistsAsync(string indexName, 512 | CancellationToken cancellationToken = default) 513 | { 514 | var collections = await this.GetIndexesAsync(cancellationToken) 515 | .ConfigureAwait(false); 516 | 517 | foreach (var item in collections) 518 | { 519 | if (item.Equals(indexName, StringComparison.OrdinalIgnoreCase)) 520 | return true; 521 | } 522 | 523 | return false; 524 | } 525 | 526 | /// 527 | /// Gets the full table name with schema. 528 | /// 529 | /// The table name. 530 | /// 531 | private string GetFullTableName(string tableName) 532 | { 533 | return $"[{this._config.Schema}].[{tableName}]"; 534 | } 535 | 536 | /// 537 | /// Generates the filters as SQL commands and sets the SQL parameters 538 | /// 539 | /// The index name. 540 | /// The SQL parameters to populate. 541 | /// The filters to apply 542 | /// 543 | private string GenerateFilters(string index, SqlParameterCollection parameters, ICollection? filters = null) 544 | { 545 | var filterBuilder = new StringBuilder(); 546 | 547 | if (filters is null || filters.Count <= 0 || filters.All(f => f.Count <= 0)) 548 | return string.Empty; 549 | 550 | 551 | filterBuilder.Append($@"AND ( 552 | "); 553 | 554 | for (int i = 0; i < filters.Count; i++) 555 | { 556 | var filter = filters.ElementAt(i); 557 | 558 | if (i > 0) 559 | { 560 | filterBuilder.Append(" OR "); 561 | } 562 | 563 | for (int j = 0; j < filter.Pairs.Count(); j++) 564 | { 565 | var value = filter.Pairs.ElementAt(j); 566 | 567 | if (j > 0) 568 | { 569 | filterBuilder.Append(@" AND 570 | "); 571 | } 572 | 573 | filterBuilder.Append(" ( "); 574 | 575 | filterBuilder.Append($@"EXISTS ( 576 | SELECT 577 | 1 578 | FROM {this.GetFullTableName($"{this._config.TagsTableName}_{index}")} AS [tags] 579 | WHERE 580 | [tags].[memory_id] = {this.GetFullTableName(this._config.MemoryTableName)}.[id] 581 | AND [name] = @filter_{i}_{j}_name 582 | AND [value] = @filter_{i}_{j}_value 583 | ) 584 | "); 585 | 586 | filterBuilder.Append(" ) "); 587 | 588 | parameters.AddWithValue($"@filter_{i}_{j}_name", value.Key); 589 | parameters.AddWithValue($"@filter_{i}_{j}_value", value.Value); 590 | } 591 | } 592 | 593 | filterBuilder.Append(@" 594 | )"); 595 | 596 | return filterBuilder.ToString(); 597 | } 598 | 599 | private async Task ReadEntryAsync(SqlDataReader dataReader, bool withEmbedding, CancellationToken cancellationToken = default) 600 | { 601 | var entry = new MemoryRecord(); 602 | 603 | entry.Id = dataReader.GetString(dataReader.GetOrdinal("key")); 604 | 605 | if (!(await dataReader.IsDBNullAsync(dataReader.GetOrdinal("payload"), cancellationToken).ConfigureAwait(false))) 606 | { 607 | entry.Payload = JsonSerializer.Deserialize>(dataReader.GetString(dataReader.GetOrdinal("payload")))!; 608 | } 609 | 610 | if (!(await dataReader.IsDBNullAsync(dataReader.GetOrdinal("tags"), cancellationToken).ConfigureAwait(false))) 611 | { 612 | entry.Tags = JsonSerializer.Deserialize(dataReader.GetString(dataReader.GetOrdinal("tags")))!; 613 | } 614 | 615 | if (withEmbedding) 616 | { 617 | entry.Vector = new ReadOnlyMemory(JsonSerializer.Deserialize>(dataReader.GetString(dataReader.GetOrdinal("embedding")))!.ToArray()); 618 | } 619 | 620 | return entry; 621 | } 622 | 623 | // Note: "_" is allowed in Postgres, but we normalize it to "-" for consistency with other DBs 624 | private static readonly Regex s_replaceIndexNameCharsRegex = new(@"[\s|\\|/|.|_|:]"); 625 | private const string ValidSeparator = "-"; 626 | 627 | private static string NormalizeIndexName(string index) 628 | { 629 | if (string.IsNullOrWhiteSpace(index)) 630 | { 631 | index = Constants.DefaultIndex; 632 | } 633 | 634 | index = s_replaceIndexNameCharsRegex.Replace(index.Trim().ToLowerInvariant(), ValidSeparator); 635 | 636 | 637 | return index; 638 | } 639 | 640 | private int GetSqlServerMajorVersionNumber() 641 | { 642 | using var connection = new SqlConnection(this._config.ConnectionString); 643 | connection.Open(); 644 | 645 | using (SqlCommand command = connection.CreateCommand()) 646 | { 647 | command.CommandText = "SELECT SERVERPROPERTY('ProductMajorVersion')"; 648 | 649 | var result = command.ExecuteScalar(); 650 | 651 | return Convert.ToInt32(result, CultureInfo.InvariantCulture); 652 | } 653 | } 654 | } 655 | -------------------------------------------------------------------------------- /src/KernelMemory.MemoryStorage.SqlServer/SqlServerMemoryException.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Kevin BEAUGRAND. All rights reserved. 2 | 3 | using Microsoft.KernelMemory; 4 | using System; 5 | 6 | namespace KernelMemory.MemoryStorage.SqlServer; 7 | 8 | /// 9 | /// Represents a SQL Server memory store exception. 10 | /// 11 | public class SqlServerMemoryException : KernelMemoryException 12 | { 13 | /// 14 | public SqlServerMemoryException() 15 | { 16 | } 17 | 18 | /// 19 | public SqlServerMemoryException(string? message) : base(message) 20 | { 21 | } 22 | 23 | /// 24 | public SqlServerMemoryException(string? message, Exception? innerException) : base(message, innerException) 25 | { 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/nuget.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | --------------------------------------------------------------------------------