├── .aspire └── settings.json ├── .devcontainer └── devcontainer.json ├── .editorconfig ├── .github └── workflows │ └── aspire-publish.yml ├── .gitignore ├── AIChat.AppHost ├── .gitignore ├── AIChat.AppHost.csproj ├── DashboardExtensions.cs ├── ModelExtensions.cs ├── Program.cs ├── Properties │ └── launchSettings.json ├── ProxyExtensions.cs ├── appsettings.Development.json ├── appsettings.json └── docker-infra │ └── docker-compose.yaml ├── AIChat.ServiceDefaults ├── AIChat.ServiceDefaults.csproj └── Extensions.cs ├── AIChat.slnx ├── ChatApi ├── AppDbContext.cs ├── ChatApi.cs ├── ChatApi.csproj ├── ChatClientConnectionInfo.cs ├── ChatClientExtensions.cs ├── ChatHub.cs ├── Program.cs ├── Properties │ └── launchSettings.json ├── Services │ ├── ChatStreamingCoordinator.cs │ ├── EnsureDatabaseCreatedHostedService.cs │ ├── ICancellationManager.cs │ ├── IConversationState.cs │ ├── RedisCancellationManager.cs │ ├── RedisConversationState.MessageBuffer.cs │ └── RedisConversationState.cs ├── appsettings.Development.json └── appsettings.json ├── README.md ├── chatui ├── Caddyfile ├── Dockerfile ├── index.html ├── package-lock.json ├── package.json ├── src │ ├── AppRouter.tsx │ ├── components │ │ ├── App.css │ │ ├── App.tsx │ │ ├── ChatContainer.tsx │ │ ├── ChatList.tsx │ │ ├── LandingPage.tsx │ │ ├── Sidebar.tsx │ │ └── VirtualizedChatList.tsx │ ├── index.tsx │ ├── services │ │ └── ChatService.ts │ ├── types │ │ └── ChatTypes.ts │ └── utils │ │ └── UnboundedChannel.ts ├── tsconfig.json ├── tsconfig.node.json └── vite.config.ts ├── global.json └── nuget.config /.aspire/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "appHostPath": "../AIChat.AppHost/AIChat.AppHost.csproj" 3 | } -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Aspire AI Chat", 3 | "image": "mcr.microsoft.com/devcontainers/dotnet:dev-9.0-bookworm", 4 | "features": { 5 | "ghcr.io/devcontainers/features/azure-cli:1": {}, 6 | "ghcr.io/devcontainers/features/docker-in-docker:2": {}, 7 | "ghcr.io/devcontainers/features/powershell:1": {}, 8 | "ghcr.io/azure/azure-dev/azd:0": {} 9 | }, 10 | "customizations": { 11 | "vscode": { 12 | "extensions": [ 13 | "ms-dotnettools.csdevkit", 14 | "ms-azuretools.vscode-bicep", 15 | "GitHub.copilot-chat", 16 | "GitHub.copilot" 17 | ], 18 | "settings": { 19 | "remote.autoForwardPorts": true, 20 | "remote.autoForwardPortsSource": "hybrid", 21 | "remote.otherPortsAttributes": { 22 | "onAutoForward": "ignore" 23 | } 24 | } 25 | } 26 | }, 27 | "onCreateCommand": "dotnet new install Aspire.ProjectTemplates::*-*", 28 | "postStartCommand": "dotnet dev-certs https --trust", 29 | "forwardPorts": [] 30 | } -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | [*.cs] 2 | 3 | # ASPIRECOSMOSDB001: Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. 4 | dotnet_diagnostic.ASPIRECOSMOSDB001.severity = none 5 | 6 | # ASPIREPROXYENDPOINTS001: Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. 7 | dotnet_diagnostic.ASPIREPROXYENDPOINTS001.severity = none 8 | 9 | # ASPIREAZURE001: Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. 10 | dotnet_diagnostic.ASPIREAZURE001.severity = none 11 | -------------------------------------------------------------------------------- /.github/workflows/aspire-publish.yml: -------------------------------------------------------------------------------- 1 | name: Aspire Publish Pipeline 2 | 3 | on: 4 | push: 5 | branches: ['*'] # Trigger on any branch 6 | pull_request: 7 | branches: ['*'] # Trigger on any branch 8 | 9 | permissions: 10 | contents: read 11 | packages: write 12 | 13 | jobs: 14 | publish: 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - name: Checkout code 19 | uses: actions/checkout@v3 20 | 21 | - name: Set up .NET 22 | uses: actions/setup-dotnet@v3 23 | with: 24 | dotnet-version: '9.x' 25 | 26 | - name: Install Aspire CLI 27 | run: dotnet tool install --global aspire.cli --prerelease 28 | 29 | - name: Run Aspire Publish 30 | run: aspire publish -p docker-compose -o artifacts 31 | working-directory: AIChat.AppHost 32 | 33 | - name: Log in to GitHub Container Registry 34 | run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin 35 | 36 | - name: Tag and Push Container Images 37 | run: | 38 | BUILD_NUMBER=${{ github.run_number }} 39 | BRANCH_NAME=${{ github.ref_name }} 40 | SANITIZED_BRANCH_NAME=$(echo "$BRANCH_NAME" | sed 's#[^a-zA-Z0-9._-]#-#g') 41 | for image in chatui chatapi; do 42 | docker tag $image:latest ghcr.io/${{ github.repository_owner }}/$image:${SANITIZED_BRANCH_NAME}-${BUILD_NUMBER} 43 | docker push ghcr.io/${{ github.repository_owner }}/$image:${SANITIZED_BRANCH_NAME}-${BUILD_NUMBER} 44 | done 45 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from `dotnet new gitignore` 5 | 6 | # dotenv files 7 | .env 8 | 9 | # User-specific files 10 | *.rsuser 11 | *.suo 12 | *.user 13 | *.userosscache 14 | *.sln.docstates 15 | 16 | # User-specific files (MonoDevelop/Xamarin Studio) 17 | *.userprefs 18 | 19 | # Mono auto generated files 20 | mono_crash.* 21 | 22 | # Build results 23 | [Dd]ebug/ 24 | [Dd]ebugPublic/ 25 | [Rr]elease/ 26 | [Rr]eleases/ 27 | x64/ 28 | x86/ 29 | [Ww][Ii][Nn]32/ 30 | [Aa][Rr][Mm]/ 31 | [Aa][Rr][Mm]64/ 32 | bld/ 33 | [Bb]in/ 34 | [Oo]bj/ 35 | [Ll]og/ 36 | [Ll]ogs/ 37 | 38 | # Visual Studio 2015/2017 cache/options directory 39 | .vs/ 40 | # Uncomment if you have tasks that create the project's static files in wwwroot 41 | #wwwroot/ 42 | 43 | # Visual Studio 2017 auto generated files 44 | Generated\ Files/ 45 | 46 | # MSTest test Results 47 | [Tt]est[Rr]esult*/ 48 | [Bb]uild[Ll]og.* 49 | 50 | # NUnit 51 | *.VisualState.xml 52 | TestResult.xml 53 | nunit-*.xml 54 | 55 | # Build Results of an ATL Project 56 | [Dd]ebugPS/ 57 | [Rr]eleasePS/ 58 | dlldata.c 59 | 60 | # Benchmark Results 61 | BenchmarkDotNet.Artifacts/ 62 | 63 | # .NET 64 | project.lock.json 65 | project.fragment.lock.json 66 | artifacts/ 67 | 68 | # Tye 69 | .tye/ 70 | 71 | # ASP.NET Scaffolding 72 | ScaffoldingReadMe.txt 73 | 74 | # StyleCop 75 | StyleCopReport.xml 76 | 77 | # Files built by Visual Studio 78 | *_i.c 79 | *_p.c 80 | *_h.h 81 | *.ilk 82 | *.meta 83 | *.obj 84 | *.iobj 85 | *.pch 86 | *.pdb 87 | *.ipdb 88 | *.pgc 89 | *.pgd 90 | *.rsp 91 | *.sbr 92 | *.tlb 93 | *.tli 94 | *.tlh 95 | *.tmp 96 | *.tmp_proj 97 | *_wpftmp.csproj 98 | *.log 99 | *.tlog 100 | *.vspscc 101 | *.vssscc 102 | .builds 103 | *.pidb 104 | *.svclog 105 | *.scc 106 | 107 | # Chutzpah Test files 108 | _Chutzpah* 109 | 110 | # Visual C++ cache files 111 | ipch/ 112 | *.aps 113 | *.ncb 114 | *.opendb 115 | *.opensdf 116 | *.sdf 117 | *.cachefile 118 | *.VC.db 119 | *.VC.VC.opendb 120 | 121 | # Visual Studio profiler 122 | *.psess 123 | *.vsp 124 | *.vspx 125 | *.sap 126 | 127 | # Visual Studio Trace Files 128 | *.e2e 129 | 130 | # TFS 2012 Local Workspace 131 | $tf/ 132 | 133 | # Guidance Automation Toolkit 134 | *.gpState 135 | 136 | # ReSharper is a .NET coding add-in 137 | _ReSharper*/ 138 | *.[Rr]e[Ss]harper 139 | *.DotSettings.user 140 | 141 | # TeamCity is a build add-in 142 | _TeamCity* 143 | 144 | # DotCover is a Code Coverage Tool 145 | *.dotCover 146 | 147 | # AxoCover is a Code Coverage Tool 148 | .axoCover/* 149 | !.axoCover/settings.json 150 | 151 | # Coverlet is a free, cross platform Code Coverage Tool 152 | coverage*.json 153 | coverage*.xml 154 | coverage*.info 155 | 156 | # Visual Studio code coverage results 157 | *.coverage 158 | *.coveragexml 159 | 160 | # NCrunch 161 | _NCrunch_* 162 | .*crunch*.local.xml 163 | nCrunchTemp_* 164 | 165 | # MightyMoose 166 | *.mm.* 167 | AutoTest.Net/ 168 | 169 | # Web workbench (sass) 170 | .sass-cache/ 171 | 172 | # Installshield output folder 173 | [Ee]xpress/ 174 | 175 | # DocProject is a documentation generator add-in 176 | DocProject/buildhelp/ 177 | DocProject/Help/*.HxT 178 | DocProject/Help/*.HxC 179 | DocProject/Help/*.hhc 180 | DocProject/Help/*.hhk 181 | DocProject/Help/*.hhp 182 | DocProject/Help/Html2 183 | DocProject/Help/html 184 | 185 | # Click-Once directory 186 | publish/ 187 | 188 | # Publish Web Output 189 | *.[Pp]ublish.xml 190 | *.azurePubxml 191 | # Note: Comment the next line if you want to checkin your web deploy settings, 192 | # but database connection strings (with potential passwords) will be unencrypted 193 | *.pubxml 194 | *.publishproj 195 | 196 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 197 | # checkin your Azure Web App publish settings, but sensitive information contained 198 | # in these scripts will be unencrypted 199 | PublishScripts/ 200 | 201 | # NuGet Packages 202 | *.nupkg 203 | # NuGet Symbol Packages 204 | *.snupkg 205 | # The packages folder can be ignored because of Package Restore 206 | **/[Pp]ackages/* 207 | # except build/, which is used as an MSBuild target. 208 | !**/[Pp]ackages/build/ 209 | # Uncomment if necessary however generally it will be regenerated when needed 210 | #!**/[Pp]ackages/repositories.config 211 | # NuGet v3's project.json files produces more ignorable files 212 | *.nuget.props 213 | *.nuget.targets 214 | 215 | # Microsoft Azure Build Output 216 | csx/ 217 | *.build.csdef 218 | 219 | # Microsoft Azure Emulator 220 | ecf/ 221 | rcf/ 222 | 223 | # Windows Store app package directories and files 224 | AppPackages/ 225 | BundleArtifacts/ 226 | Package.StoreAssociation.xml 227 | _pkginfo.txt 228 | *.appx 229 | *.appxbundle 230 | *.appxupload 231 | 232 | # Visual Studio cache files 233 | # files ending in .cache can be ignored 234 | *.[Cc]ache 235 | # but keep track of directories ending in .cache 236 | !?*.[Cc]ache/ 237 | 238 | # Others 239 | ClientBin/ 240 | ~$* 241 | *~ 242 | *.dbmdl 243 | *.dbproj.schemaview 244 | *.jfm 245 | *.pfx 246 | *.publishsettings 247 | orleans.codegen.cs 248 | 249 | # Including strong name files can present a security risk 250 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 251 | #*.snk 252 | 253 | # Since there are multiple workflows, uncomment next line to ignore bower_components 254 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 255 | #bower_components/ 256 | 257 | # RIA/Silverlight projects 258 | Generated_Code/ 259 | 260 | # Backup & report files from converting an old project file 261 | # to a newer Visual Studio version. Backup files are not needed, 262 | # because we have git ;-) 263 | _UpgradeReport_Files/ 264 | Backup*/ 265 | UpgradeLog*.XML 266 | UpgradeLog*.htm 267 | ServiceFabricBackup/ 268 | *.rptproj.bak 269 | 270 | # SQL Server files 271 | *.mdf 272 | *.ldf 273 | *.ndf 274 | 275 | # Business Intelligence projects 276 | *.rdl.data 277 | *.bim.layout 278 | *.bim_*.settings 279 | *.rptproj.rsuser 280 | *- [Bb]ackup.rdl 281 | *- [Bb]ackup ([0-9]).rdl 282 | *- [Bb]ackup ([0-9][0-9]).rdl 283 | 284 | # Microsoft Fakes 285 | FakesAssemblies/ 286 | 287 | # GhostDoc plugin setting file 288 | *.GhostDoc.xml 289 | 290 | # Node.js Tools for Visual Studio 291 | .ntvs_analysis.dat 292 | node_modules/ 293 | 294 | # Visual Studio 6 build log 295 | *.plg 296 | 297 | # Visual Studio 6 workspace options file 298 | *.opt 299 | 300 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 301 | *.vbw 302 | 303 | # Visual Studio 6 auto-generated project file (contains which files were open etc.) 304 | *.vbp 305 | 306 | # Visual Studio 6 workspace and project file (working project files containing files to include in project) 307 | *.dsw 308 | *.dsp 309 | 310 | # Visual Studio 6 technical files 311 | *.ncb 312 | *.aps 313 | 314 | # Visual Studio LightSwitch build output 315 | **/*.HTMLClient/GeneratedArtifacts 316 | **/*.DesktopClient/GeneratedArtifacts 317 | **/*.DesktopClient/ModelManifest.xml 318 | **/*.Server/GeneratedArtifacts 319 | **/*.Server/ModelManifest.xml 320 | _Pvt_Extensions 321 | 322 | # Paket dependency manager 323 | .paket/paket.exe 324 | paket-files/ 325 | 326 | # FAKE - F# Make 327 | .fake/ 328 | 329 | # CodeRush personal settings 330 | .cr/personal 331 | 332 | # Python Tools for Visual Studio (PTVS) 333 | __pycache__/ 334 | *.pyc 335 | 336 | # Cake - Uncomment if you are using it 337 | # tools/** 338 | # !tools/packages.config 339 | 340 | # Tabs Studio 341 | *.tss 342 | 343 | # Telerik's JustMock configuration file 344 | *.jmconfig 345 | 346 | # BizTalk build output 347 | *.btp.cs 348 | *.btm.cs 349 | *.odx.cs 350 | *.xsd.cs 351 | 352 | # OpenCover UI analysis results 353 | OpenCover/ 354 | 355 | # Azure Stream Analytics local run output 356 | ASALocalRun/ 357 | 358 | # MSBuild Binary and Structured Log 359 | *.binlog 360 | 361 | # NVidia Nsight GPU debugger configuration file 362 | *.nvuser 363 | 364 | # MFractors (Xamarin productivity tool) working folder 365 | .mfractor/ 366 | 367 | # Local History for Visual Studio 368 | .localhistory/ 369 | 370 | # Visual Studio History (VSHistory) files 371 | .vshistory/ 372 | 373 | # BeatPulse healthcheck temp database 374 | healthchecksdb 375 | 376 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 377 | MigrationBackup/ 378 | 379 | # Ionide (cross platform F# VS Code tools) working folder 380 | .ionide/ 381 | 382 | # Fody - auto-generated XML schema 383 | FodyWeavers.xsd 384 | 385 | # VS Code files for those working on multiple tools 386 | .vscode/* 387 | !.vscode/settings.json 388 | !.vscode/tasks.json 389 | !.vscode/launch.json 390 | !.vscode/extensions.json 391 | *.code-workspace 392 | 393 | # Local History for Visual Studio Code 394 | .history/ 395 | 396 | # Windows Installer files from build outputs 397 | *.cab 398 | *.msi 399 | *.msix 400 | *.msm 401 | *.msp 402 | 403 | # JetBrains Rider 404 | *.sln.iml 405 | .idea/ 406 | 407 | ## 408 | ## Visual studio for Mac 409 | ## 410 | 411 | 412 | # globs 413 | Makefile.in 414 | *.userprefs 415 | *.usertasks 416 | config.make 417 | config.status 418 | aclocal.m4 419 | install-sh 420 | autom4te.cache/ 421 | *.tar.gz 422 | tarballs/ 423 | test-results/ 424 | 425 | # Mac bundle stuff 426 | *.dmg 427 | *.app 428 | 429 | # content below from: https://github.com/github/gitignore/blob/main/Global/macOS.gitignore 430 | # General 431 | .DS_Store 432 | .AppleDouble 433 | .LSOverride 434 | 435 | # Icon must end with two \r 436 | Icon 437 | 438 | 439 | # Thumbnails 440 | ._* 441 | 442 | # Files that might appear in the root of a volume 443 | .DocumentRevisions-V100 444 | .fseventsd 445 | .Spotlight-V100 446 | .TemporaryItems 447 | .Trashes 448 | .VolumeIcon.icns 449 | .com.apple.timemachine.donotpresent 450 | 451 | # Directories potentially created on remote AFP share 452 | .AppleDB 453 | .AppleDesktop 454 | Network Trash Folder 455 | Temporary Items 456 | .apdisk 457 | 458 | # content below from: https://github.com/github/gitignore/blob/main/Global/Windows.gitignore 459 | # Windows thumbnail cache files 460 | Thumbs.db 461 | ehthumbs.db 462 | ehthumbs_vista.db 463 | 464 | # Dump file 465 | *.stackdump 466 | 467 | # Folder config file 468 | [Dd]esktop.ini 469 | 470 | # Recycle Bin used on file shares 471 | $RECYCLE.BIN/ 472 | 473 | # Windows Installer files 474 | *.cab 475 | *.msi 476 | *.msix 477 | *.msm 478 | *.msp 479 | 480 | # Windows shortcuts 481 | *.lnk 482 | 483 | # Vim temporary swap files 484 | *.swp 485 | -------------------------------------------------------------------------------- /AIChat.AppHost/.gitignore: -------------------------------------------------------------------------------- 1 | .azure 2 | -------------------------------------------------------------------------------- /AIChat.AppHost/AIChat.AppHost.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Exe 7 | net9.0 8 | enable 9 | enable 10 | true 11 | 04350d74-4615-41b1-a3a6-37edf0e71727 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /AIChat.AppHost/DashboardExtensions.cs: -------------------------------------------------------------------------------- 1 | public static class DashboardExtensions 2 | { 3 | public static void AddDashboard(this IDistributedApplicationBuilder builder) 4 | { 5 | if (builder.ExecutionContext.IsPublishMode) 6 | { 7 | // The name aspire-dashboard is special cased and excluded from the default 8 | var dashboard = builder.AddContainer("dashboard", "mcr.microsoft.com/dotnet/nightly/aspire-dashboard") 9 | .WithHttpEndpoint(targetPort: 18888) 10 | .WithHttpEndpoint(name: "otlp", targetPort: 18889) 11 | .PublishAsDockerComposeService((_, service) => 12 | { 13 | service.Restart = "always"; 14 | }); 15 | 16 | builder.Eventing.Subscribe((e, ct) => 17 | { 18 | // We loop over all resources and set the OTLP endpoint to the dashboard 19 | // we should make WithOtlpExporter() add an annotation so we can detect this 20 | // automatically in the future. 21 | foreach (var r in e.Model.Resources.OfType()) 22 | { 23 | if (r == dashboard.Resource) 24 | { 25 | continue; 26 | } 27 | 28 | builder.CreateResourceBuilder(r).WithEnvironment(c => 29 | { 30 | c.EnvironmentVariables["OTEL_EXPORTER_OTLP_ENDPOINT"] = dashboard.GetEndpoint("otlp"); 31 | c.EnvironmentVariables["OTEL_EXPORTER_OTLP_PROTOCOL"] = "grpc"; 32 | c.EnvironmentVariables["OTEL_SERVICE_NAME"] = r.Name; 33 | }); 34 | } 35 | 36 | return Task.CompletedTask; 37 | }); 38 | } 39 | } 40 | } -------------------------------------------------------------------------------- /AIChat.AppHost/ModelExtensions.cs: -------------------------------------------------------------------------------- 1 | public static class ModelExtensions 2 | { 3 | public static IResourceBuilder AddAIModel(this IDistributedApplicationBuilder builder, string name) 4 | { 5 | var model = new AIModel(name); 6 | return builder.CreateResourceBuilder(model); 7 | } 8 | 9 | public static IResourceBuilder RunAsOllama(this IResourceBuilder builder, string model, Action>? configure = null) 10 | { 11 | if (builder.ApplicationBuilder.ExecutionContext.IsRunMode) 12 | { 13 | builder.Reset(); 14 | 15 | var ollama = builder.ApplicationBuilder.AddOllama("ollama") 16 | .WithDataVolume(); 17 | 18 | configure?.Invoke(ollama); 19 | 20 | var ollamaModel = ollama.AddModel(builder.Resource.Name, model); 21 | 22 | builder.Resource.UnderlyingResource = ollamaModel.Resource; 23 | builder.Resource.ConnectionString = ReferenceExpression.Create($"{ollamaModel};Provider=Ollama"); 24 | } 25 | 26 | return builder; 27 | } 28 | 29 | public static IResourceBuilder RunAsOpenAI(this IResourceBuilder builder, string modelName, Func> addApiKey) 30 | { 31 | if (builder.ApplicationBuilder.ExecutionContext.IsRunMode) 32 | { 33 | return builder.AsOpenAI(modelName, addApiKey(builder.ApplicationBuilder)); 34 | } 35 | 36 | return builder; 37 | } 38 | 39 | public static IResourceBuilder PublishAsOpenAI(this IResourceBuilder builder, string modelName, Func> addApiKey) 40 | { 41 | if (builder.ApplicationBuilder.ExecutionContext.IsPublishMode) 42 | { 43 | return builder.AsOpenAI(modelName, addApiKey(builder.ApplicationBuilder)); 44 | } 45 | 46 | return builder; 47 | } 48 | 49 | public static IResourceBuilder AsOpenAI(this IResourceBuilder builder, string modelName, Func> addApiKey) 50 | { 51 | return builder.AsOpenAI(modelName, addApiKey(builder.ApplicationBuilder)); 52 | } 53 | 54 | public static IResourceBuilder AsOpenAI(this IResourceBuilder builder, string modelName, IResourceBuilder apiKey) 55 | { 56 | builder.Reset(); 57 | 58 | var cs = builder.ApplicationBuilder.AddConnectionString(builder.Resource.Name, csb => 59 | { 60 | csb.Append($"AccessKey={apiKey};"); 61 | csb.Append($"Model={modelName};"); 62 | csb.AppendLiteral("Provider=OpenAI"); 63 | }); 64 | 65 | builder.Resource.UnderlyingResource = cs.Resource; 66 | builder.Resource.ConnectionString = cs.Resource.ConnectionStringExpression; 67 | 68 | return builder; 69 | } 70 | 71 | private static void Reset(this IResourceBuilder builder) 72 | { 73 | // Reset the properties of the AIModel resource 74 | IResource? underlyingResource = builder.Resource.UnderlyingResource; 75 | 76 | if (underlyingResource is not null) 77 | { 78 | builder.ApplicationBuilder.Resources.Remove(underlyingResource); 79 | 80 | while (underlyingResource is IResourceWithParent resourceWithParent) 81 | { 82 | builder.ApplicationBuilder.Resources.Remove(resourceWithParent.Parent); 83 | 84 | underlyingResource = resourceWithParent.Parent; 85 | } 86 | } 87 | 88 | builder.Resource.ConnectionString = null; 89 | } 90 | } 91 | 92 | // A resource representing an AI model. 93 | public class AIModel(string name) : Resource(name), IResourceWithConnectionString, IResourceWithoutLifetime 94 | { 95 | internal IResourceWithConnectionString? UnderlyingResource { get; set; } 96 | internal ReferenceExpression? ConnectionString { get; set; } 97 | 98 | public ReferenceExpression ConnectionStringExpression => 99 | ConnectionString ?? throw new InvalidOperationException("No connection string available."); 100 | } 101 | 102 | -------------------------------------------------------------------------------- /AIChat.AppHost/Program.cs: -------------------------------------------------------------------------------- 1 | var builder = DistributedApplication.CreateBuilder(args); 2 | 3 | // Publish this as a Docker Compose application 4 | builder.AddDockerComposeEnvironment("env") 5 | .ConfigureComposeFile(file => 6 | { 7 | file.Name = "aspire-ai-chat"; 8 | }); 9 | 10 | builder.AddDashboard(); 11 | 12 | // This is the AI model our application will use 13 | var model = builder.AddAIModel("llm") 14 | .RunAsOllama("phi4", c => 15 | { 16 | // Enable to enable GPU support (if your machine has a GPU) 17 | if (!OperatingSystem.IsMacOS()) 18 | { 19 | c.WithGPUSupport(); 20 | } 21 | c.WithLifetime(ContainerLifetime.Persistent); 22 | }) 23 | .PublishAsOpenAI("gpt-4o", b => b.AddParameter("openaikey", secret: true)); 24 | 25 | // We use Postgres for our conversation history 26 | var db = builder.AddPostgres("pg") 27 | .WithDataVolume(builder.ExecutionContext.IsPublishMode ? "pgvolume" : null) 28 | .WithPgAdmin() 29 | .AddDatabase("conversations"); 30 | 31 | // Redis is used to store and broadcast the live message stream 32 | // so that multiple clients can connect to the same conversation. 33 | var cache = builder.AddRedis("cache") 34 | .WithRedisInsight(); 35 | 36 | var chatapi = builder.AddProject("chatapi") 37 | .WithReference(model) 38 | .WaitFor(model) 39 | .WithReference(db) 40 | .WaitFor(db) 41 | .WithReference(cache) 42 | .WaitFor(cache); 43 | 44 | builder.AddNpmApp("chatui", "../chatui") 45 | .WithNpmPackageInstallation() 46 | .WithHttpEndpoint(env: "PORT") 47 | .WithReverseProxy(chatapi.GetEndpoint("http")) 48 | .WithExternalHttpEndpoints() 49 | .WithOtlpExporter() 50 | .WithEnvironment("BROWSER", "none"); 51 | 52 | builder.Build().Run(); 53 | 54 | -------------------------------------------------------------------------------- /AIChat.AppHost/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json.schemastore.org/launchsettings.json", 3 | "profiles": { 4 | "https": { 5 | "commandName": "Project", 6 | "dotnetRunMessages": true, 7 | "launchBrowser": true, 8 | "applicationUrl": "https://localhost:17255;http://localhost:15259", 9 | "environmentVariables": { 10 | "ASPNETCORE_ENVIRONMENT": "Development", 11 | "DOTNET_ENVIRONMENT": "Development", 12 | "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:21099", 13 | "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:22098", 14 | "ASPIRE_ALLOW_UNSECURED_TRANSPORT": "true" 15 | } 16 | }, 17 | "http": { 18 | "commandName": "Project", 19 | "dotnetRunMessages": true, 20 | "launchBrowser": true, 21 | "applicationUrl": "http://localhost:15259", 22 | "environmentVariables": { 23 | "ASPNETCORE_ENVIRONMENT": "Development", 24 | "DOTNET_ENVIRONMENT": "Development", 25 | "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:19168", 26 | "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:20166", 27 | "ALLOW_INSECURE_TRANSPORT": "true" 28 | } 29 | }, 30 | "generate-manifest": { 31 | "commandName": "Project", 32 | "launchBrowser": true, 33 | "dotnetRunMessages": true, 34 | "commandLineArgs": "--publisher manifest --output-path aspire-manifest.json", 35 | "applicationUrl": "http://localhost:15888", 36 | "environmentVariables": { 37 | "ASPNETCORE_ENVIRONMENT": "Development", 38 | "DOTNET_ENVIRONMENT": "Development", 39 | "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:16157" 40 | } 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /AIChat.AppHost/ProxyExtensions.cs: -------------------------------------------------------------------------------- 1 | public static class ProxyExtensions 2 | { 3 | public static IResourceBuilder WithReverseProxy(this IResourceBuilder builder, EndpointReference upstreamEndpoint) 4 | { 5 | if (builder.ApplicationBuilder.ExecutionContext.IsRunMode) 6 | { 7 | return builder.WithEnvironment("BACKEND_URL", upstreamEndpoint); 8 | } 9 | 10 | return builder.PublishAsDockerFile(c => c.WithReverseProxy(upstreamEndpoint)); 11 | } 12 | 13 | public static IResourceBuilder WithReverseProxy(this IResourceBuilder builder, EndpointReference upstreamEndpoint) 14 | { 15 | // Caddy listens on port 80 16 | builder.WithEndpoint("http", e => e.TargetPort = 80); 17 | 18 | return builder.WithEnvironment(context => 19 | { 20 | // In the docker file, caddy uses the host and port without the scheme 21 | context.EnvironmentVariables["BACKEND_URL"] = upstreamEndpoint.Property(EndpointProperty.HostAndPort); 22 | context.EnvironmentVariables["SPAN"] = builder.Resource.Name; 23 | }); 24 | } 25 | } -------------------------------------------------------------------------------- /AIChat.AppHost/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft.AspNetCore": "Warning" 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /AIChat.AppHost/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft.AspNetCore": "Warning", 6 | "Aspire.Hosting.Dcp": "Warning" 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /AIChat.AppHost/docker-infra/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | name: "aspire-ai-chat" 2 | services: 3 | dashboard: 4 | image: "mcr.microsoft.com/dotnet/nightly/aspire-dashboard:latest" 5 | ports: 6 | - "8000:18888" 7 | - "8001:18889" 8 | networks: 9 | - "aspire" 10 | pg: 11 | image: "docker.io/library/postgres:17.4" 12 | environment: 13 | POSTGRES_HOST_AUTH_METHOD: "scram-sha-256" 14 | POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256 --auth-local=scram-sha-256" 15 | POSTGRES_USER: "postgres" 16 | POSTGRES_PASSWORD: "${PG_PASSWORD}" 17 | OTEL_EXPORTER_OTLP_ENDPOINT: "http://dashboard:18889" 18 | OTEL_EXPORTER_OTLP_PROTOCOL: "grpc" 19 | OTEL_SERVICE_NAME: "pg" 20 | ports: 21 | - "8002:5432" 22 | volumes: 23 | - type: "volume" 24 | target: "/var/lib/postgresql/data" 25 | source: "pgvolume" 26 | read_only: false 27 | networks: 28 | - "aspire" 29 | cache: 30 | image: "docker.io/library/redis:7.4" 31 | command: 32 | - "-c" 33 | - "redis-server --requirepass $$REDIS_PASSWORD" 34 | entrypoint: 35 | - "/bin/sh" 36 | environment: 37 | REDIS_PASSWORD: "${CACHE_PASSWORD}" 38 | OTEL_EXPORTER_OTLP_ENDPOINT: "http://dashboard:18889" 39 | OTEL_EXPORTER_OTLP_PROTOCOL: "grpc" 40 | OTEL_SERVICE_NAME: "cache" 41 | ports: 42 | - "8003:6379" 43 | networks: 44 | - "aspire" 45 | chatapi: 46 | image: "${CHATAPI_IMAGE}" 47 | environment: 48 | OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EXCEPTION_LOG_ATTRIBUTES: "true" 49 | OTEL_DOTNET_EXPERIMENTAL_OTLP_EMIT_EVENT_LOG_ATTRIBUTES: "true" 50 | OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY: "in_memory" 51 | ASPNETCORE_FORWARDEDHEADERS_ENABLED: "true" 52 | HTTP_PORTS: "8004" 53 | ConnectionStrings__llm: "AccessKey=${OPENAIKEY};Model=gpt-4o;Provider=OpenAI" 54 | ConnectionStrings__conversations: "Host=pg;Port=5432;Username=postgres;Password=${PG_PASSWORD};Database=conversations" 55 | ConnectionStrings__cache: "cache:6379,password=${CACHE_PASSWORD}" 56 | OTEL_EXPORTER_OTLP_ENDPOINT: "http://dashboard:18889" 57 | OTEL_EXPORTER_OTLP_PROTOCOL: "grpc" 58 | OTEL_SERVICE_NAME: "chatapi" 59 | ports: 60 | - "8005:8004" 61 | - "8007:8006" 62 | depends_on: 63 | pg: 64 | condition: "service_started" 65 | cache: 66 | condition: "service_started" 67 | networks: 68 | - "aspire" 69 | chatui: 70 | image: "${CHATUI_IMAGE}" 71 | environment: 72 | NODE_ENV: "production" 73 | PORT: "80" 74 | BACKEND_URL: "chatapi:8004" 75 | SPAN: "chatui" 76 | OTEL_EXPORTER_OTLP_ENDPOINT: "http://dashboard:18889" 77 | OTEL_EXPORTER_OTLP_PROTOCOL: "grpc" 78 | OTEL_SERVICE_NAME: "chatui" 79 | ports: 80 | - "8008:80" 81 | networks: 82 | - "aspire" 83 | networks: 84 | aspire: 85 | driver: "bridge" 86 | volumes: 87 | pgvolume: 88 | driver: "local" 89 | -------------------------------------------------------------------------------- /AIChat.ServiceDefaults/AIChat.ServiceDefaults.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net9.0 5 | enable 6 | enable 7 | true 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /AIChat.ServiceDefaults/Extensions.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.AspNetCore.Builder; 2 | using Microsoft.AspNetCore.Diagnostics.HealthChecks; 3 | using Microsoft.Extensions.DependencyInjection; 4 | using Microsoft.Extensions.Diagnostics.HealthChecks; 5 | using Microsoft.Extensions.Logging; 6 | using Microsoft.Extensions.ServiceDiscovery; 7 | using OpenTelemetry; 8 | using OpenTelemetry.Metrics; 9 | using OpenTelemetry.Trace; 10 | 11 | namespace Microsoft.Extensions.Hosting; 12 | 13 | // Adds common .NET Aspire services: service discovery, resilience, health checks, and OpenTelemetry. 14 | // This project should be referenced by each service project in your solution. 15 | // To learn more about using this project, see https://aka.ms/dotnet/aspire/service-defaults 16 | public static class Extensions 17 | { 18 | public static TBuilder AddServiceDefaults(this TBuilder builder) where TBuilder : IHostApplicationBuilder 19 | { 20 | builder.ConfigureOpenTelemetry(); 21 | 22 | builder.AddDefaultHealthChecks(); 23 | 24 | builder.Services.AddServiceDiscovery(); 25 | 26 | builder.Services.ConfigureHttpClientDefaults(http => 27 | { 28 | // Turn on resilience by default 29 | http.AddStandardResilienceHandler(); 30 | 31 | // Turn on service discovery by default 32 | http.AddServiceDiscovery(); 33 | }); 34 | 35 | // Uncomment the following to restrict the allowed schemes for service discovery. 36 | // builder.Services.Configure(options => 37 | // { 38 | // options.AllowedSchemes = ["https"]; 39 | // }); 40 | 41 | return builder; 42 | } 43 | 44 | public static TBuilder ConfigureOpenTelemetry(this TBuilder builder) where TBuilder : IHostApplicationBuilder 45 | { 46 | builder.Logging.AddOpenTelemetry(logging => 47 | { 48 | logging.IncludeFormattedMessage = true; 49 | logging.IncludeScopes = true; 50 | }); 51 | 52 | builder.Services.AddOpenTelemetry() 53 | .WithMetrics(metrics => 54 | { 55 | metrics.AddAspNetCoreInstrumentation() 56 | .AddHttpClientInstrumentation() 57 | .AddRuntimeInstrumentation(); 58 | }) 59 | .WithTracing(tracing => 60 | { 61 | tracing.AddSource(builder.Environment.ApplicationName) 62 | .AddAspNetCoreInstrumentation() 63 | // Uncomment the following line to enable gRPC instrumentation (requires the OpenTelemetry.Instrumentation.GrpcNetClient package) 64 | //.AddGrpcClientInstrumentation() 65 | .AddHttpClientInstrumentation(); 66 | }); 67 | 68 | builder.AddOpenTelemetryExporters(); 69 | 70 | return builder; 71 | } 72 | 73 | private static TBuilder AddOpenTelemetryExporters(this TBuilder builder) where TBuilder : IHostApplicationBuilder 74 | { 75 | var useOtlpExporter = !string.IsNullOrWhiteSpace(builder.Configuration["OTEL_EXPORTER_OTLP_ENDPOINT"]); 76 | 77 | if (useOtlpExporter) 78 | { 79 | builder.Services.AddOpenTelemetry().UseOtlpExporter(); 80 | } 81 | 82 | // Uncomment the following lines to enable the Azure Monitor exporter (requires the Azure.Monitor.OpenTelemetry.AspNetCore package) 83 | //if (!string.IsNullOrEmpty(builder.Configuration["APPLICATIONINSIGHTS_CONNECTION_STRING"])) 84 | //{ 85 | // builder.Services.AddOpenTelemetry() 86 | // .UseAzureMonitor(); 87 | //} 88 | 89 | return builder; 90 | } 91 | 92 | public static TBuilder AddDefaultHealthChecks(this TBuilder builder) where TBuilder : IHostApplicationBuilder 93 | { 94 | builder.Services.AddHealthChecks() 95 | // Add a default liveness check to ensure app is responsive 96 | .AddCheck("self", () => HealthCheckResult.Healthy(), ["live"]); 97 | 98 | return builder; 99 | } 100 | 101 | public static WebApplication MapDefaultEndpoints(this WebApplication app) 102 | { 103 | // Adding health checks endpoints to applications in non-development environments has security implications. 104 | // See https://aka.ms/dotnet/aspire/healthchecks for details before enabling these endpoints in non-development environments. 105 | if (app.Environment.IsDevelopment()) 106 | { 107 | // All health checks must pass for app to be considered ready to accept traffic after starting 108 | app.MapHealthChecks("/health"); 109 | 110 | // Only health checks tagged with the "live" tag must pass for app to be considered alive 111 | app.MapHealthChecks("/alive", new HealthCheckOptions 112 | { 113 | Predicate = r => r.Tags.Contains("live") 114 | }); 115 | } 116 | 117 | return app; 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /AIChat.slnx: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /ChatApi/AppDbContext.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore; 2 | 3 | public class AppDbContext(DbContextOptions options) : DbContext(options) 4 | { 5 | public required DbSet Conversations { get; set; } 6 | 7 | protected override void OnModelCreating(ModelBuilder modelBuilder) 8 | { 9 | modelBuilder.Entity().OwnsMany(c => c.Messages, c => c.ToJson()); 10 | } 11 | } 12 | 13 | public class ConversationChatMessage 14 | { 15 | public required Guid Id { get; set; } 16 | public required string Role { get; set; } 17 | public required string Text { get; set; } 18 | } 19 | 20 | public class Conversation 21 | { 22 | public required Guid Id { get; set; } 23 | public required string Name { get; set; } 24 | public required List Messages { get; set; } = []; 25 | } 26 | -------------------------------------------------------------------------------- /ChatApi/ChatApi.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore; 2 | 3 | public static class ChatExtensions 4 | { 5 | public static void MapChatApi(this WebApplication app) 6 | { 7 | var group = app.MapGroup("/api/chat"); 8 | 9 | group.MapGet("/", (AppDbContext db) => 10 | { 11 | return db.Conversations.ToListAsync(); 12 | }); 13 | 14 | group.MapGet("/{id}", async (Guid id, AppDbContext db) => 15 | { 16 | var conversation = await db.Conversations.FindAsync(id); 17 | 18 | if (conversation is null) 19 | { 20 | return Results.NotFound(); 21 | } 22 | 23 | var clientMessages = from m in conversation.Messages 24 | select new ClientMessage(m.Id, m.Role, m.Text); 25 | 26 | return Results.Ok(clientMessages); 27 | }); 28 | 29 | group.MapHub("/stream", o => o.AllowStatefulReconnects = true); 30 | 31 | group.MapPost("/", async (NewConversation newConversation, AppDbContext db) => 32 | { 33 | if (string.IsNullOrWhiteSpace(newConversation.Name)) 34 | { 35 | return Results.BadRequest(); 36 | } 37 | 38 | var conversation = new Conversation 39 | { 40 | Id = Guid.CreateVersion7(), 41 | Name = newConversation.Name, 42 | Messages = [] 43 | }; 44 | 45 | db.Conversations.Add(conversation); 46 | await db.SaveChangesAsync(); 47 | 48 | return Results.Created($"/api/chats/{conversation.Id}", conversation); 49 | }); 50 | 51 | group.MapPost("/{id}", async (Guid id, AppDbContext db, Prompt prompt, CancellationToken token, ChatStreamingCoordinator streaming) => 52 | { 53 | // Fire and forget 54 | await streaming.AddStreamingMessage(id, prompt.Text); 55 | 56 | return Results.Ok(); 57 | }); 58 | 59 | group.MapPost("/{id}/cancel", async (Guid id, ICancellationManager cancellationManager) => 60 | { 61 | await cancellationManager.CancelAsync(id); 62 | 63 | return Results.Ok(); 64 | }); 65 | 66 | group.MapDelete("/{id}", async (Guid id, AppDbContext db) => 67 | { 68 | var conversation = await db.Conversations.FindAsync(id); 69 | 70 | if (conversation is null) 71 | { 72 | return Results.NotFound(); 73 | } 74 | 75 | db.Conversations.Remove(conversation); 76 | await db.SaveChangesAsync(); 77 | 78 | return Results.Ok(); 79 | }); 80 | } 81 | } 82 | 83 | public record Prompt(string Text); 84 | 85 | public record NewConversation(string Name); 86 | 87 | public record ClientMessage(Guid Id, string Sender, string Text); 88 | 89 | public record ClientMessageFragment(Guid Id, string Sender, string Text, Guid FragmentId, bool IsFinal = false); 90 | 91 | public record StreamContext(Guid? LastMessageId, Guid? LastFragmentId); -------------------------------------------------------------------------------- /ChatApi/ChatApi.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | net9.0 5 | enable 6 | enable 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /ChatApi/ChatClientConnectionInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Data.Common; 2 | using System.Diagnostics.CodeAnalysis; 3 | 4 | public class ChatClientConnectionInfo 5 | { 6 | public Uri? Endpoint { get; init; } 7 | public required string SelectedModel { get; init; } 8 | 9 | public ClientChatProvider Provider { get; init; } 10 | public string? AccessKey { get; init; } 11 | 12 | // Example connection string: 13 | // Endpoint=https://localhost:4523;Model=phi3.5;AccessKey=1234;Provider=ollama; 14 | public static bool TryParse(string? connectionString, [NotNullWhen(true)] out ChatClientConnectionInfo? settings) 15 | { 16 | if (string.IsNullOrEmpty(connectionString)) 17 | { 18 | settings = null; 19 | return false; 20 | } 21 | 22 | var connectionBuilder = new DbConnectionStringBuilder 23 | { 24 | ConnectionString = connectionString 25 | }; 26 | 27 | Uri? endpoint = null; 28 | if (connectionBuilder.ContainsKey("Endpoint") && Uri.TryCreate(connectionBuilder["Endpoint"].ToString(), UriKind.Absolute, out endpoint)) 29 | { 30 | } 31 | 32 | string? model = null; 33 | if (connectionBuilder.ContainsKey("Model")) 34 | { 35 | model = (string)connectionBuilder["Model"]; 36 | } 37 | 38 | string? accessKey = null; 39 | if (connectionBuilder.ContainsKey("AccessKey")) 40 | { 41 | accessKey = (string)connectionBuilder["AccessKey"]; 42 | } 43 | 44 | var provider = ClientChatProvider.Unknown; 45 | if (connectionBuilder.ContainsKey("Provider")) 46 | { 47 | var providerValue = (string)connectionBuilder["Provider"]; 48 | Enum.TryParse(providerValue, ignoreCase: true, out provider); 49 | } 50 | 51 | if ((endpoint is null && provider != ClientChatProvider.OpenAI) || model is null || provider == ClientChatProvider.Unknown) 52 | { 53 | settings = null; 54 | return false; 55 | } 56 | 57 | settings = new ChatClientConnectionInfo 58 | { 59 | Endpoint = endpoint, 60 | SelectedModel = model, 61 | AccessKey = accessKey, 62 | Provider = provider 63 | }; 64 | 65 | return true; 66 | } 67 | } 68 | 69 | public enum ClientChatProvider 70 | { 71 | Unknown, 72 | Ollama, 73 | OpenAI 74 | } 75 | -------------------------------------------------------------------------------- /ChatApi/ChatClientExtensions.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.AI; 2 | 3 | public static class ChatClientExtensions 4 | { 5 | public static IHostApplicationBuilder AddChatClient(this IHostApplicationBuilder builder, string connectionName) 6 | { 7 | var cs = builder.Configuration.GetConnectionString(connectionName); 8 | 9 | if (!ChatClientConnectionInfo.TryParse(cs, out var connectionInfo)) 10 | { 11 | throw new InvalidOperationException($"Invalid connection string: {cs}. Expected format: 'Endpoint=endpoint;AccessKey=your_access_key;Model=model_name;Provider=ollama/openai/azureopenai;'."); 12 | } 13 | 14 | var chatClientBuilder = connectionInfo.Provider switch 15 | { 16 | ClientChatProvider.Ollama => builder.AddOllamaClient(connectionName, connectionInfo), 17 | ClientChatProvider.OpenAI => builder.AddOpenAIClient(connectionName, connectionInfo), 18 | _ => throw new NotSupportedException($"Unsupported provider: {connectionInfo.Provider}") 19 | }; 20 | 21 | // Add OpenTelemetry tracing for the ChatClient activity source 22 | chatClientBuilder.UseOpenTelemetry().UseLogging(); 23 | 24 | // This is the default name of the trace source and meter 25 | var telemetryName = "Experimental.Microsoft.Extensions.AI"; 26 | 27 | builder.Services.AddOpenTelemetry() 28 | .WithTracing(t => t.AddSource(telemetryName)) 29 | .WithMetrics(m => m.AddMeter(telemetryName)); 30 | 31 | return builder; 32 | } 33 | 34 | private static ChatClientBuilder AddOpenAIClient(this IHostApplicationBuilder builder, string connectionName, ChatClientConnectionInfo connectionInfo) 35 | { 36 | return builder.AddOpenAIClient(connectionName, settings => 37 | { 38 | settings.Endpoint = connectionInfo.Endpoint; 39 | settings.Key = connectionInfo.AccessKey; 40 | }) 41 | .AddChatClient(connectionInfo.SelectedModel); 42 | } 43 | 44 | private static ChatClientBuilder AddOllamaClient(this IHostApplicationBuilder builder, string connectionName, ChatClientConnectionInfo connectionInfo) 45 | { 46 | return builder.AddOllamaApiClient(connectionName, settings => 47 | { 48 | settings.SelectedModel = connectionInfo.SelectedModel; 49 | }) 50 | .AddChatClient(); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /ChatApi/ChatHub.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.AspNetCore.SignalR; 2 | 3 | public class ChatHub : Hub 4 | { 5 | public IAsyncEnumerable Stream(Guid id, StreamContext streamContext, ChatStreamingCoordinator streaming, CancellationToken token) 6 | { 7 | async IAsyncEnumerable Stream() 8 | { 9 | await foreach (var message in streaming.GetMessageStream(id, streamContext.LastMessageId, streamContext.LastFragmentId).WithCancellation(token)) 10 | { 11 | yield return message; 12 | } 13 | } 14 | 15 | return Stream(); 16 | } 17 | } -------------------------------------------------------------------------------- /ChatApi/Program.cs: -------------------------------------------------------------------------------- 1 | var builder = WebApplication.CreateBuilder(args); 2 | 3 | builder.AddServiceDefaults(); 4 | 5 | builder.AddChatClient("llm"); 6 | builder.AddRedisClient("cache"); 7 | builder.AddNpgsqlDbContext("conversations"); 8 | 9 | builder.Services.AddSignalR(); 10 | builder.Services.AddSingleton(); 11 | builder.Services.AddHostedService(); 12 | 13 | builder.Services.AddSingleton(); 14 | builder.Services.AddSingleton(); 15 | 16 | var app = builder.Build(); 17 | 18 | app.MapDefaultEndpoints(); 19 | 20 | app.MapChatApi(); 21 | 22 | app.Run(); 23 | -------------------------------------------------------------------------------- /ChatApi/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json.schemastore.org/launchsettings.json", 3 | "profiles": { 4 | "http": { 5 | "commandName": "Project", 6 | "dotnetRunMessages": true, 7 | "launchBrowser": true, 8 | "applicationUrl": "http://localhost:5191", 9 | "environmentVariables": { 10 | "ASPNETCORE_ENVIRONMENT": "Development" 11 | } 12 | }, 13 | "https": { 14 | "commandName": "Project", 15 | "dotnetRunMessages": true, 16 | "launchBrowser": true, 17 | "applicationUrl": "https://localhost:7272;http://localhost:5191", 18 | "environmentVariables": { 19 | "ASPNETCORE_ENVIRONMENT": "Development" 20 | } 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /ChatApi/Services/ChatStreamingCoordinator.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.CompilerServices; 2 | using Microsoft.Extensions.AI; 3 | 4 | public class ChatStreamingCoordinator( 5 | IChatClient chatClient, 6 | IServiceScopeFactory scopeFactory, 7 | ILogger logger, 8 | IConversationState conversationState, 9 | ICancellationManager cancellationManager) 10 | { 11 | // TODO: Read this from configuration 12 | private TimeSpan DefaultStreamItemTimeout = TimeSpan.FromMinutes(1); 13 | 14 | public async Task AddStreamingMessage(Guid conversationId, string text) 15 | { 16 | var messages = await SavePromptAndGetMessageHistoryAsync(conversationId, text); 17 | 18 | // Explicitly start the task to avoid blocking the caller. 19 | _ = Task.Run(StreamReplyAsync); 20 | 21 | async Task StreamReplyAsync() 22 | { 23 | Guid assistantReplyId = Guid.CreateVersion7(); 24 | 25 | logger.LogInformation("Adding streaming message for conversation {ConversationId} {MessageId}", conversationId, assistantReplyId); 26 | 27 | var allChunks = new List(); 28 | 29 | // Combine the provided cancellationToken with the distributed cancellation token. 30 | var token = cancellationManager.GetCancellationToken(assistantReplyId); 31 | 32 | var fragment = new ClientMessageFragment(assistantReplyId, ChatRole.Assistant.Value, "Generating reply...", Guid.CreateVersion7()); 33 | await conversationState.PublishFragmentAsync(conversationId, fragment); 34 | 35 | try 36 | { 37 | using var tokenSource = CancellationTokenSource.CreateLinkedTokenSource(token); 38 | tokenSource.CancelAfter(DefaultStreamItemTimeout); 39 | 40 | await foreach (var update in chatClient.GetStreamingResponseAsync(messages).WithCancellation(tokenSource.Token)) 41 | { 42 | // Extend the cancellation token's timeout for each update. 43 | tokenSource.CancelAfter(DefaultStreamItemTimeout); 44 | 45 | if (update.Text is not null) 46 | { 47 | allChunks.Add(update); 48 | fragment = new ClientMessageFragment(assistantReplyId, ChatRole.Assistant.Value, update.Text, Guid.CreateVersion7()); 49 | await conversationState.PublishFragmentAsync(conversationId, fragment); 50 | } 51 | } 52 | 53 | logger.LogInformation("Full message received for conversation {ConversationId} {MessageId}", conversationId, assistantReplyId); 54 | 55 | if (allChunks.Count > 0) 56 | { 57 | var fullMessage = allChunks.ToChatResponse().Text; 58 | await SaveAssistantMessageToDatabase(conversationId, assistantReplyId, fullMessage); 59 | } 60 | } 61 | catch (OperationCanceledException) 62 | { 63 | logger.LogInformation("Streaming message cancelled for conversation {ConversationId} {MessageId}", conversationId, assistantReplyId); 64 | 65 | if (allChunks.Count > 0) 66 | { 67 | var fullMessage = allChunks.ToChatResponse().Text; 68 | await SaveAssistantMessageToDatabase(conversationId, assistantReplyId, fullMessage); 69 | } 70 | } 71 | catch (Exception ex) 72 | { 73 | fragment = new ClientMessageFragment(assistantReplyId, ChatRole.Assistant.Value, "Error streaming message", Guid.CreateVersion7()); 74 | await conversationState.PublishFragmentAsync(conversationId, fragment); 75 | logger.LogError(ex, "Error streaming message for conversation {ConversationId} {MessageId}", conversationId, assistantReplyId); 76 | 77 | await SaveAssistantMessageToDatabase(conversationId, assistantReplyId, "Error streaming message"); 78 | } 79 | finally 80 | { 81 | // Publish a final fragment to indicate the end of the message. 82 | fragment = new ClientMessageFragment(assistantReplyId, ChatRole.Assistant.Value, "", Guid.CreateVersion7(), IsFinal: true); 83 | await conversationState.PublishFragmentAsync(conversationId, fragment); 84 | 85 | // Clean up the cancellation token. 86 | await cancellationManager.CancelAsync(assistantReplyId); 87 | } 88 | } 89 | } 90 | 91 | private async Task> SavePromptAndGetMessageHistoryAsync(Guid id, string text) 92 | { 93 | await using var scope = scopeFactory.CreateAsyncScope(); 94 | var db = scope.ServiceProvider.GetRequiredService(); 95 | 96 | var conversation = await db.Conversations.FindAsync(id) ?? throw new InvalidOperationException($"Conversation {id} not found"); 97 | 98 | var messageId = Guid.CreateVersion7(); 99 | 100 | conversation.Messages.Add(new() 101 | { 102 | Id = messageId, 103 | Role = ChatRole.User.Value, 104 | Text = text 105 | }); 106 | 107 | // Actually save conversation history 108 | await db.SaveChangesAsync(); 109 | 110 | // This is inefficient 111 | var messages = conversation.Messages 112 | .Select(m => new ChatMessage(new(m.Role), m.Text)) 113 | .ToList(); 114 | 115 | // Publish the initial fragment with the prompt text. 116 | var fragment = new ClientMessageFragment(messageId, ChatRole.User.Value, text, Guid.CreateVersion7(), IsFinal: true); 117 | await conversationState.PublishFragmentAsync(id, fragment); 118 | 119 | return messages; 120 | } 121 | 122 | private async Task SaveAssistantMessageToDatabase(Guid conversationId, Guid messageId, string text) 123 | { 124 | await using var scope = scopeFactory.CreateAsyncScope(); 125 | var db = scope.ServiceProvider.GetRequiredService(); 126 | 127 | var conversation = await db.Conversations.FindAsync(conversationId); 128 | if (conversation is not null) 129 | { 130 | conversation.Messages.Add(new ConversationChatMessage 131 | { 132 | Id = messageId, 133 | Role = ChatRole.Assistant.Value, 134 | Text = text 135 | }); 136 | 137 | await db.SaveChangesAsync(); 138 | } 139 | 140 | await conversationState.CompleteAsync(conversationId, messageId); 141 | } 142 | 143 | public async IAsyncEnumerable GetMessageStream( 144 | Guid conversationId, 145 | Guid? lastMessageId, 146 | Guid? lastDeliveredFragment, 147 | [EnumeratorCancellation] CancellationToken cancellationToken = default) 148 | { 149 | logger.LogInformation("Getting message stream for conversation {ConversationId}, {LastMessageId}", conversationId, lastMessageId); 150 | var stream = conversationState.Subscribe(conversationId, lastMessageId, cancellationToken); 151 | 152 | await foreach (var fragment in stream.WithCancellation(cancellationToken)) 153 | { 154 | // Use lastMessageId to filter out fragments from an already delivered message, 155 | // while using lastDeliveredFragment (a sortable GUID) for ordering and de-duping. 156 | if (lastDeliveredFragment is null || fragment.FragmentId > lastDeliveredFragment) 157 | { 158 | lastDeliveredFragment = fragment.FragmentId; 159 | } 160 | else 161 | { 162 | continue; 163 | } 164 | 165 | yield return fragment; 166 | } 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /ChatApi/Services/EnsureDatabaseCreatedHostedService.cs: -------------------------------------------------------------------------------- 1 | public class EnsureDatabaseCreatedHostedService(IServiceProvider serviceProvider) : BackgroundService 2 | { 3 | protected override async Task ExecuteAsync(CancellationToken stoppingToken) 4 | { 5 | await Task.Yield(); 6 | 7 | using var scope = serviceProvider.CreateScope(); 8 | var dbContext = scope.ServiceProvider.GetRequiredService(); 9 | await dbContext.Database.EnsureCreatedAsync(stoppingToken); 10 | } 11 | } -------------------------------------------------------------------------------- /ChatApi/Services/ICancellationManager.cs: -------------------------------------------------------------------------------- 1 | public interface ICancellationManager 2 | { 3 | CancellationToken GetCancellationToken(Guid id); 4 | 5 | Task CancelAsync(Guid id); 6 | } 7 | -------------------------------------------------------------------------------- /ChatApi/Services/IConversationState.cs: -------------------------------------------------------------------------------- 1 | public interface IConversationState 2 | { 3 | Task CompleteAsync(Guid conversationId, Guid messageId); 4 | Task PublishFragmentAsync(Guid conversationId, ClientMessageFragment fragment); 5 | IAsyncEnumerable Subscribe(Guid conversationId, Guid? lastMessageId, CancellationToken cancellationToken = default); 6 | Task> GetUnpublishedMessagesAsync(Guid conversationId); 7 | } 8 | -------------------------------------------------------------------------------- /ChatApi/Services/RedisCancellationManager.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Concurrent; 2 | using StackExchange.Redis; 3 | 4 | public class RedisCancellationManager : ICancellationManager, IDisposable 5 | { 6 | private readonly ISubscriber _subscriber; 7 | private readonly ILogger _logger; 8 | private readonly ConcurrentDictionary _tokens = []; 9 | private readonly RedisChannel _channelName = RedisChannel.Literal("chatapp-cancellation"); 10 | 11 | public RedisCancellationManager(IConnectionMultiplexer redis, ILogger logger) 12 | { 13 | _subscriber = redis.GetSubscriber(); 14 | _logger = logger; 15 | _subscriber.Subscribe(_channelName, OnCancellationMessage); 16 | _logger.LogInformation("Subscribed to cancellation channel {Channel}", _channelName); 17 | } 18 | 19 | private void OnCancellationMessage(RedisChannel channel, RedisValue message) 20 | { 21 | if (Guid.TryParse(message, out Guid replyId)) 22 | { 23 | _logger.LogInformation("Received cancellation message for reply {ReplyId}", replyId); 24 | 25 | if (_tokens.TryRemove(replyId, out var cts)) 26 | { 27 | cts.Cancel(); 28 | _logger.LogInformation("Cancelled token for reply {ReplyId}", replyId); 29 | } 30 | } 31 | else 32 | { 33 | _logger.LogWarning("Received invalid cancellation message: {Message}", message); 34 | } 35 | } 36 | 37 | public CancellationToken GetCancellationToken(Guid id) 38 | { 39 | var cts = new CancellationTokenSource(); 40 | _tokens[id] = cts; 41 | 42 | _logger.LogDebug("Created cancellation token for reply {ReplyId}", id); 43 | 44 | return cts.Token; 45 | } 46 | 47 | public async Task CancelAsync(Guid id) 48 | { 49 | _logger.LogDebug("Publishing cancellation message for reply {ReplyId}", id); 50 | 51 | await _subscriber.PublishAsync(_channelName, id.ToString()); 52 | } 53 | 54 | public void Dispose() 55 | { 56 | try 57 | { 58 | _subscriber.Unsubscribe(_channelName); 59 | 60 | _logger.LogInformation("Unsubscribed from cancellation channel {Channel}", _channelName); 61 | 62 | foreach (var kvp in _tokens) 63 | { 64 | kvp.Value.Dispose(); 65 | } 66 | } 67 | catch (Exception ex) 68 | { 69 | _logger.LogError(ex, "Error during RedisCancellationManager disposal"); 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /ChatApi/Services/RedisConversationState.MessageBuffer.cs: -------------------------------------------------------------------------------- 1 | using System.Text.Json; 2 | using StackExchange.Redis; 3 | using System.Collections.Concurrent; 4 | 5 | public partial class RedisConversationState 6 | { 7 | // Aggregates message fragments into a single coalesced message. 8 | // It uses a timer and size threshold to trigger a flush to Redis, 9 | // reducing network round-trips. Once flushing is triggered (or on disposal), 10 | // the buffer is drained and the fragments are sent to Redis. 11 | private class MessageBuffer : IAsyncDisposable 12 | { 13 | private readonly IDatabase _db; 14 | private readonly ISubscriber _subscriber; 15 | private readonly ILogger _logger; 16 | private readonly Guid _conversationId; 17 | private readonly ConcurrentQueue _buffer = []; 18 | // Invariant: Once _draining is set, no more fragments can be enqueued. 19 | private volatile bool _draining; 20 | // SemaphoreSlim ensures only one FlushAsync runs at a time. 21 | private readonly SemaphoreSlim _flushLock = new(1, 1); 22 | private readonly TimeSpan _flushInterval = TimeSpan.FromMilliseconds(MaxBufferTimeMs); 23 | private readonly Timer _flushTimer; 24 | private const int MaxBufferSize = 20; 25 | private const int MaxBufferTimeMs = 500; 26 | 27 | // Track how many fragments are currently in the queue, to avoid expensive .Count calls. 28 | private int _count; 29 | 30 | public MessageBuffer(IDatabase db, ISubscriber subscriber, ILogger logger, Guid conversationId) 31 | { 32 | _db = db; 33 | _subscriber = subscriber; 34 | _logger = logger; 35 | _conversationId = conversationId; 36 | _flushTimer = new Timer(_ => TriggerFlush(), null, _flushInterval, _flushInterval); 37 | } 38 | 39 | public async Task AddFragmentAsync(ClientMessageFragment fragment) 40 | { 41 | if (_draining) throw new InvalidOperationException("Buffer is draining"); 42 | 43 | _buffer.Enqueue(fragment); 44 | // Capture the new count from Increment 45 | var newCount = Interlocked.Increment(ref _count); 46 | // Invariant: newCount reflects the number of enqueued fragments. 47 | if (fragment.IsFinal || newCount >= MaxBufferSize) 48 | { 49 | await TriggerFlushAsync().ConfigureAwait(false); 50 | } 51 | } 52 | 53 | // Flush is triggered either by the timer or reaching thresholds. 54 | private async Task TriggerFlushAsync() 55 | { 56 | // Attempt to acquire the flush lock without blocking. 57 | if (!await _flushLock.WaitAsync(0).ConfigureAwait(false)) 58 | { 59 | return; // Another flush is in progress. 60 | } 61 | 62 | try 63 | { 64 | await FlushAsync().ConfigureAwait(false); 65 | } 66 | finally 67 | { 68 | // Invariant: _flushLock is always released. 69 | _flushLock.Release(); 70 | } 71 | } 72 | 73 | // Timer callback for flush. 74 | private void TriggerFlush() 75 | { 76 | if (_draining) 77 | { 78 | return; 79 | } 80 | _ = TriggerFlushAsync(); 81 | } 82 | 83 | private async Task FlushAsync() 84 | { 85 | var fragmentsToFlush = new List(); 86 | try 87 | { 88 | // Dequeue until empty and decrement the counter for each item. 89 | while (_buffer.TryDequeue(out var fragment)) 90 | { 91 | fragmentsToFlush.Add(fragment); 92 | Interlocked.Decrement(ref _count); 93 | } 94 | 95 | if (fragmentsToFlush.Count > 0) 96 | { 97 | // Log before initiating IO operations. 98 | _logger.LogInformation("Flushing {Count} fragments for conversation {ConversationId} {MessageId}", fragmentsToFlush.Count, _conversationId, fragmentsToFlush[0].Id); 99 | 100 | var key = GetBacklogKey(_conversationId); 101 | var channel = GetRedisChannelName(_conversationId); 102 | var coalescedFragment = CoalesceFragments(fragmentsToFlush); 103 | string serialized = JsonSerializer.Serialize(coalescedFragment); 104 | 105 | // Use WhenAll to send to Redis in parallel. 106 | await Task.WhenAll( 107 | _db.ListRightPushAsync(key, serialized), 108 | _subscriber.PublishAsync(channel, serialized) 109 | ).ConfigureAwait(false); 110 | 111 | // Log after successful IO. 112 | _logger.LogInformation("Successfully flushed fragments for conversation {ConversationId} {MessageId}", _conversationId, fragmentsToFlush[0].Id); 113 | } 114 | } 115 | catch (Exception ex) 116 | { 117 | _logger.LogError(ex, "Error flushing fragments for conversation {ConversationId}. Re-enqueueing fragments.", _conversationId); 118 | 119 | // On failure, re-enqueue fragments and re-adjust the counter. 120 | foreach (var fragment in fragmentsToFlush) 121 | { 122 | _buffer.Enqueue(fragment); 123 | Interlocked.Increment(ref _count); 124 | } 125 | throw; 126 | } 127 | } 128 | 129 | // Called once draining is triggered or on disposal. 130 | // Invariant: By the time DisposeAsync completes, no further flushes are pending. 131 | public async ValueTask DisposeAsync() 132 | { 133 | if (_draining) 134 | { 135 | return; 136 | } 137 | 138 | _draining = true; 139 | 140 | _flushTimer.Change(Timeout.Infinite, Timeout.Infinite); 141 | await Task.Yield(); // Let any pending timer callback complete 142 | 143 | try 144 | { 145 | await TriggerFlushAsync().ConfigureAwait(false); 146 | } 147 | finally 148 | { 149 | _flushTimer.Dispose(); 150 | _flushLock.Dispose(); 151 | } 152 | } 153 | } 154 | 155 | internal static ClientMessageFragment CoalesceFragments(List fragments) 156 | { 157 | var lastFragment = fragments[^1]; 158 | int count = fragments.Count; 159 | int totalLength = 0; 160 | for (int i = 0; i < count; i++) 161 | { 162 | totalLength += fragments[i].Text.Length; 163 | } 164 | string combined = string.Create(totalLength, fragments, (span, frags) => 165 | { 166 | int pos = 0; 167 | for (int i = 0; i < frags.Count; i++) 168 | { 169 | ReadOnlySpan text = frags[i].Text; 170 | text.CopyTo(span.Slice(pos)); 171 | pos += text.Length; 172 | } 173 | }); 174 | return new ClientMessageFragment(lastFragment.Id, lastFragment.Sender, combined, lastFragment.FragmentId, lastFragment.IsFinal); 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /ChatApi/Services/RedisConversationState.cs: -------------------------------------------------------------------------------- 1 | using System.Text.Json; 2 | using StackExchange.Redis; 3 | using System.Threading.Channels; 4 | using System.Runtime.CompilerServices; 5 | using System.Collections.Concurrent; 6 | 7 | public partial class RedisConversationState : IConversationState, IDisposable 8 | { 9 | private readonly IDatabase _db; 10 | private readonly ISubscriber _subscriber; 11 | private readonly ILogger _logger; 12 | 13 | // Global registry mapping conversationId to list of local callbacks. 14 | private static readonly ConcurrentDictionary>> GlobalSubscribers = new(); 15 | 16 | // Store the pattern subscription so we can unsubscribe. 17 | private readonly RedisChannel _patternChannel; 18 | 19 | // Nagle algorithm: Buffer messages to Redis to reduce network round trips. 20 | private readonly ConcurrentDictionary _messageBuffers = []; 21 | 22 | public RedisConversationState(IConnectionMultiplexer redis, ILogger logger) 23 | { 24 | _db = redis.GetDatabase(); 25 | _subscriber = redis.GetSubscriber(); 26 | _logger = logger; 27 | _patternChannel = RedisChannel.Pattern("conversation:*:channel"); 28 | _subscriber.Subscribe(_patternChannel, OnRedisMessage); 29 | _logger.LogInformation("Subscribed to pattern {Pattern}", _patternChannel); 30 | } 31 | 32 | private void OnRedisMessage(RedisChannel channel, RedisValue value) 33 | { 34 | // Parse conversationId from channel name, assuming format "conversation:{id}:channel". 35 | var channelStr = channel.ToString(); 36 | var parts = channelStr.Split(':'); 37 | if (parts.Length < 3 || !Guid.TryParse(parts[1], out var conversationId)) 38 | { 39 | return; 40 | } 41 | 42 | ClientMessageFragment? fragment = null; 43 | 44 | try 45 | { 46 | fragment = JsonSerializer.Deserialize((byte[])value!); 47 | } 48 | catch (Exception ex) 49 | { 50 | _logger.LogError(ex, "Error deserializing message from channel {Channel}", channel); 51 | return; 52 | } 53 | 54 | if (fragment is null) 55 | { 56 | return; 57 | } 58 | 59 | _logger.LogDebug("Redis message received for conversation {ConversationId} with fragment {FragmentId}", conversationId, fragment.Id); 60 | 61 | if (GlobalSubscribers.TryGetValue(conversationId, out var subscribers)) 62 | { 63 | lock (subscribers) 64 | { 65 | foreach (var sub in subscribers) 66 | { 67 | sub(fragment); 68 | } 69 | } 70 | } 71 | } 72 | 73 | public async Task PublishFragmentAsync(Guid conversationId, ClientMessageFragment fragment) 74 | { 75 | _logger.LogDebug("Publishing fragment {FragmentId} for conversation {ConversationId} to Redis", fragment.Id, conversationId); 76 | 77 | var buffer = _messageBuffers.GetOrAdd(fragment.Id, _ => new(_db, _subscriber, _logger, conversationId)); 78 | await buffer.AddFragmentAsync(fragment); 79 | } 80 | 81 | private static void AddLocalSubscriber(Guid conversationId, Action callback) 82 | { 83 | var list = GlobalSubscribers.GetOrAdd(conversationId, _ => []); 84 | lock (list) 85 | { 86 | list.Add(callback); 87 | } 88 | } 89 | 90 | private static void RemoveLocalSubscriber(Guid conversationId, Action callback) 91 | { 92 | if (GlobalSubscribers.TryGetValue(conversationId, out var list)) 93 | { 94 | lock (list) 95 | { 96 | list.Remove(callback); 97 | if (list.Count == 0) 98 | { 99 | GlobalSubscribers.TryRemove(conversationId, out _); 100 | } 101 | } 102 | } 103 | } 104 | 105 | public async IAsyncEnumerable Subscribe(Guid conversationId, Guid? lastMessageId, [EnumeratorCancellation] CancellationToken cancellationToken = default) 106 | { 107 | _logger.LogInformation("New Redis subscription for conversation {ConversationId} with lastMessageId {LastMessageId}", conversationId, lastMessageId); 108 | var channel = Channel.CreateUnbounded(); 109 | 110 | // Register a local callback BEFORE retrieving the backlog. 111 | void LocalCallback(ClientMessageFragment fragment) 112 | { 113 | if (lastMessageId != null && fragment.Id <= lastMessageId) 114 | { 115 | return; 116 | } 117 | 118 | _logger.LogDebug("Fanning out fragment {FragmentId} for conversation {ConversationId}", fragment.Id, conversationId); 119 | 120 | channel.Writer.TryWrite(fragment); 121 | } 122 | 123 | AddLocalSubscriber(conversationId, LocalCallback); 124 | 125 | // Inline backlog logic: iterate over the fetched RedisValues and yield fragments directly. 126 | 127 | var key = GetBacklogKey(conversationId); 128 | var values = await _db.ListRangeAsync(key); 129 | for (int i = 0; i < values.Length; i++) 130 | { 131 | var fragment = JsonSerializer.Deserialize(values[i]!); 132 | if (fragment is not null && (lastMessageId is null || fragment.Id > lastMessageId)) 133 | { 134 | yield return fragment; 135 | } 136 | } 137 | 138 | try 139 | { 140 | using var reg = cancellationToken.Register(() => channel.Writer.TryComplete()); 141 | await foreach (var fragment in channel.Reader.ReadAllAsync()) 142 | { 143 | yield return fragment; 144 | } 145 | } 146 | finally 147 | { 148 | RemoveLocalSubscriber(conversationId, LocalCallback); 149 | _logger.LogInformation("Redis subscription for conversation {ConversationId} ended", conversationId); 150 | } 151 | } 152 | 153 | public async Task CompleteAsync(Guid conversationId, Guid messageId) 154 | { 155 | if (_messageBuffers.TryRemove(messageId, out var buffer)) 156 | { 157 | await buffer.DisposeAsync(); 158 | } 159 | 160 | // Remove the backlog after 5 minutes of inactivity. 161 | await _db.KeyExpireAsync(GetBacklogKey(conversationId), TimeSpan.FromMinutes(5)); 162 | } 163 | 164 | public async Task> GetUnpublishedMessagesAsync(Guid conversationId) 165 | { 166 | var key = GetBacklogKey(conversationId); 167 | var values = await _db.ListRangeAsync(key); 168 | var fragments = new List(values.Length); 169 | 170 | for (int i = 0; i < values.Length; i++) 171 | { 172 | var fragment = JsonSerializer.Deserialize(values[i]!); 173 | if (fragment is not null) 174 | { 175 | fragments.Add(fragment); 176 | } 177 | } 178 | 179 | var messages = new List(); 180 | 181 | foreach (var g in fragments.GroupBy(f => f.Id)) 182 | { 183 | // The first fragment has the "Generating reply..." text. 184 | var coalescedFragment = CoalesceFragments([.. g.Skip(1)]); 185 | 186 | var message = new ClientMessage(coalescedFragment.Id, coalescedFragment.Sender, coalescedFragment.Text); 187 | 188 | messages.Add(message); 189 | } 190 | 191 | return messages; 192 | } 193 | 194 | public void Dispose() 195 | { 196 | try 197 | { 198 | _subscriber.Unsubscribe(_patternChannel); 199 | _logger.LogInformation("Unsubscribed from pattern {Pattern}", _patternChannel); 200 | } 201 | catch (Exception ex) 202 | { 203 | _logger.LogError(ex, "Error during unsubscription for pattern {Pattern}", _patternChannel); 204 | } 205 | } 206 | 207 | private static RedisKey GetBacklogKey(Guid conversationId) => $"conversation:{conversationId}:backlog"; 208 | private static RedisChannel GetRedisChannelName(Guid conversationId) => RedisChannel.Literal($"conversation:{conversationId}:channel"); 209 | } 210 | -------------------------------------------------------------------------------- /ChatApi/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft.AspNetCore": "Information" 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /ChatApi/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft.AspNetCore": "Information" 6 | } 7 | }, 8 | "AllowedHosts": "*" 9 | } 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Aspire AI Chat 2 | 3 | Aspire AI Chat is a full-stack chat sample that combines modern technologies to deliver a ChatGPT-like experience. 4 | 5 | ## High-Level Overview 6 | 7 | - **Backend API:** 8 | The backend is built with **ASP.NET Core** and interacts with an LLM using **Microsoft.Extensions.AI**. It leverages `IChatClient` to abstract the interaction between the API and the model. Chat responses are streamed back to the client using stream JSON array responses. 9 | 10 | - **Data & Persistence:** 11 | Uses **Entity Framework Core** with **PostgreSQL** for reliable relational data storage. The project includes database migrations for easy setup and version control of the schema. 12 | 13 | - **AI & Chat Capabilities:** 14 | - Uses **Ollama** (via OllamaSharp) for local inference, enabling context-aware responses. 15 | - In production, the application switches to [**OpenAI**](https://openai.com/) for LLM capabilities. 16 | 17 | - **Frontend UI:** 18 | Built with **React**, the user interface offers a modern and interactive chat experience. The React application is built and hosted using [**Caddy**](https://caddyserver.com/). 19 | 20 | ## Getting Started 21 | 22 | ### Prerequisites 23 | 24 | - [.NET 9.0](https://dotnet.microsoft.com/en-us/download/dotnet/9.0) 25 | - [Docker](https://www.docker.com/get-started) or [Podman](https://podman-desktop.io/) 26 | - [Node.js](https://nodejs.org/) (LTS version recommended) 27 | 28 | ### Running the Application 29 | 30 | Run the [AIChat.AppHost](AIChat.AppHost) project. This project uses 31 | [.NET Aspire](https://learn.microsoft.com/en-us/dotnet/aspire/get-started/aspire-overview) 32 | to run the application in a container. 33 | 34 | ### Configuration 35 | 36 | - By default, the application uses **Ollama** for local inference. 37 | - To use **OpenAI**, set the appropriate configuration values (e.g., API keys, endpoints). 38 | - The PostgreSQL database will be automatically created and migrated when running with Aspire. 39 | 40 | -------------------------------------------------------------------------------- /chatui/Caddyfile: -------------------------------------------------------------------------------- 1 | { 2 | # Enable debug mode for detailed logs 3 | debug 4 | } 5 | 6 | :80 { 7 | # Enable OpenTelemetry tracing at the site level. 8 | # This will instrument all requests with the span name provided by the SPAN environment variable. 9 | tracing { 10 | span {env.SPAN} 11 | } 12 | 13 | # Set the root directory for serving static files 14 | root * /usr/share/caddy 15 | 16 | # Handle API requests separately from the SPA 17 | handle /api/* { 18 | # Reverse proxy any request starting with /api/ to the backend service 19 | # The BACKEND_URL environment variable should contain the host:port of the backend service. 20 | reverse_proxy {env.BACKEND_URL} { 21 | # Override the Host header so that the backend service receives the expected hostname. 22 | header_up Host {http.reverse_proxy.upstream.hostport} 23 | } 24 | } 25 | 26 | # Handle all other requests for the React single-page application (SPA) 27 | handle { 28 | # Attempt to serve the requested file. If the file does not exist, fallback to index.html. 29 | # This allows client-side routing (e.g., React Router) to work properly. 30 | try_files {path} /index.html 31 | # Serve the static files from the root directory specified above. 32 | file_server 33 | } 34 | 35 | # Enable access logging to console for debugging and monitoring purposes. 36 | log { 37 | output stdout 38 | format console 39 | level info 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /chatui/Dockerfile: -------------------------------------------------------------------------------- 1 | # Build Stage: Build the React app 2 | FROM node:22 AS builder 3 | WORKDIR /app 4 | COPY package.json package-lock.json ./ 5 | RUN npm install 6 | COPY . . 7 | RUN npm run build 8 | 9 | # Production Stage: Serve the app with Caddy 10 | FROM caddy:2.9-alpine 11 | WORKDIR /usr/share/caddy 12 | 13 | # Copy built static files including index.html, etc. 14 | COPY --from=builder /app/build/ . 15 | 16 | # Copy the Caddyfile with reverse proxy config 17 | COPY Caddyfile /etc/caddy/Caddyfile 18 | 19 | EXPOSE 80 20 | 21 | # Run Caddy 22 | CMD ["caddy", "run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"] -------------------------------------------------------------------------------- /chatui/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Aspire AI Chat 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /chatui/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "my-react-app", 3 | "version": "1.0.0", 4 | "private": true, 5 | "scripts": { 6 | "start": "vite", 7 | "build": "vite build", 8 | "test": "vite test", 9 | "eject": "react-scripts eject" 10 | }, 11 | "dependencies": { 12 | "@microsoft/signalr": "^8.0.7", 13 | "denque": "^2.1.0", 14 | "react": "^19.0.0", 15 | "react-dom": "^19.0.0", 16 | "react-markdown": "^9.0.3", 17 | "react-router-dom": "^7.2.0", 18 | "react-window": "^1.8.11" 19 | }, 20 | "eslintConfig": { 21 | "extends": [ 22 | "react-app" 23 | ] 24 | }, 25 | "browserslist": { 26 | "production": [ 27 | ">0.2%", 28 | "not dead", 29 | "not op_mini all" 30 | ], 31 | "development": [ 32 | "last 1 chrome version", 33 | "last 1 firefox version", 34 | "last 1 safari version" 35 | ] 36 | }, 37 | "devDependencies": { 38 | "@types/node": "^22.13.8", 39 | "@types/react": "^19.0.10", 40 | "@types/react-dom": "^19.0.4", 41 | "@types/react-router-dom": "^5.3.3", 42 | "@types/react-window": "^1.8.8", 43 | "@vitejs/plugin-react": "^4.3.4", 44 | "typescript": "^5.8.2", 45 | "vite": "^6.1.1" 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /chatui/src/AppRouter.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { BrowserRouter, Routes, Route } from 'react-router-dom'; 3 | import App from './components/App'; 4 | 5 | const AppRouter: React.FC = () => ( 6 | 7 | 8 | } /> 9 | } /> 10 | 11 | 12 | ); 13 | 14 | export default AppRouter; 15 | -------------------------------------------------------------------------------- /chatui/src/components/App.css: -------------------------------------------------------------------------------- 1 | /* Add these at the very top of the file */ 2 | :root { 3 | background: #202123; 4 | } 5 | 6 | body { 7 | margin: 0; 8 | padding: 0; 9 | background: #202123; 10 | min-height: 100vh; 11 | } 12 | 13 | .app-container { 14 | display: flex; 15 | height: 100vh; 16 | margin: 0; 17 | padding: 0; 18 | font-family: 'Inter', 'Segoe UI', Arial, sans-serif; 19 | background: #343541; 20 | color: #ececf1; 21 | overflow: hidden; 22 | } 23 | 24 | /* --- Sidebar --- */ 25 | .sidebar { 26 | width: 260px; 27 | flex-shrink: 0; 28 | margin-right: 0; 29 | padding: 8px; 30 | box-sizing: border-box; 31 | background-color: #202123; 32 | border-right: 1px solid #2a2b32; 33 | display: flex; 34 | flex-direction: column; 35 | overflow: hidden; 36 | } 37 | 38 | .sidebar-header { 39 | display: flex; 40 | align-items: center; 41 | justify-content: space-between; 42 | padding: 24px 20px 12px 20px; 43 | border-bottom: 1px solid #2a2b32; 44 | } 45 | 46 | .sidebar-header h2 { 47 | font-size: 1.2rem; 48 | color: #ececf1; 49 | margin: 0; 50 | font-weight: 600; 51 | } 52 | 53 | .chat-list { 54 | list-style: none; 55 | padding: 0 0 0 0; 56 | margin: 0; 57 | flex-grow: 1; 58 | overflow-y: auto; 59 | } 60 | 61 | .chat-item { 62 | padding: 12px 20px; 63 | cursor: pointer; 64 | background-color: transparent; 65 | color: #ececf1; 66 | border-radius: 8px; 67 | margin: 0 8px 4px 8px; 68 | display: flex; 69 | justify-content: space-between; 70 | align-items: center; 71 | transition: background 0.15s; 72 | font-size: 1rem; 73 | } 74 | 75 | .chat-item.selected, 76 | .chat-item:hover { 77 | background-color: #353740; 78 | } 79 | 80 | .chat-name { 81 | flex-grow: 1; 82 | margin-right: 8px; 83 | white-space: nowrap; 84 | overflow: hidden; 85 | text-overflow: ellipsis; 86 | } 87 | 88 | .delete-chat-button { 89 | opacity: 0; 90 | background: none; 91 | border: none; 92 | color: #888; 93 | font-size: 20px; 94 | cursor: pointer; 95 | padding: 0 4px; 96 | line-height: 1; 97 | transition: opacity 0.2s, color 0.2s; 98 | } 99 | 100 | .chat-item:hover .delete-chat-button { 101 | opacity: 1; 102 | } 103 | 104 | .delete-chat-button:hover { 105 | color: #ff5c5c; 106 | } 107 | 108 | .new-chat-form { 109 | padding: 16px 20px 20px 20px; 110 | border-top: 1px solid #2a2b32; 111 | background: #202123; 112 | } 113 | 114 | .new-chat-input { 115 | width: 100%; 116 | padding: 10px 14px; 117 | border-radius: 8px; 118 | border: 1px solid #353740; 119 | margin-bottom: 10px; 120 | box-sizing: border-box; 121 | background: #353740; 122 | color: #ececf1; 123 | font-size: 1rem; 124 | } 125 | 126 | .new-chat-input::placeholder { 127 | color: #888; 128 | } 129 | 130 | .new-chat-button { 131 | width: 100%; 132 | padding: 10px; 133 | border-radius: 8px; 134 | border: none; 135 | background: #444654; 136 | color: #ececf1; 137 | cursor: pointer; 138 | font-weight: 500; 139 | font-size: 1rem; 140 | transition: background 0.15s; 141 | } 142 | 143 | .new-chat-button:hover { 144 | background: #565869; 145 | } 146 | 147 | .sidebar-new-chat { 148 | width: calc(100% - 16px); 149 | margin: 8px; 150 | padding: 12px 16px; 151 | background: transparent; 152 | border: 1px solid #565869; 153 | border-radius: 6px; 154 | color: #ececf1; 155 | display: flex; 156 | align-items: center; 157 | font-size: 14px; 158 | } 159 | 160 | /* --- Chat Container --- */ 161 | .chat-container { 162 | flex: 1; 163 | display: flex; 164 | flex-direction: column; 165 | height: 100vh; 166 | position: relative; 167 | overflow: hidden; 168 | } 169 | 170 | .chat-header { 171 | text-align: center; 172 | color: #ececf1; 173 | margin-bottom: 1rem; 174 | } 175 | 176 | .messages-container { 177 | flex: 1; 178 | overflow-y: auto; 179 | padding-bottom: 120px; /* Space for input form */ 180 | background: #343541; 181 | display: flex; 182 | flex-direction: column; 183 | gap: 12px; 184 | } 185 | 186 | .message { 187 | width: 100%; 188 | border-bottom: 1px solid rgba(32,33,35,0.5); 189 | padding: 24px 0; 190 | background: transparent; 191 | } 192 | 193 | .message.user { 194 | background: transparent; 195 | } 196 | 197 | .message.assistant { 198 | background: #444654; 199 | } 200 | 201 | .message-content { 202 | max-width: 768px; 203 | margin: 0 auto; 204 | padding: 24px 200px; 205 | color: #ececf1; 206 | font-size: 16px; 207 | line-height: 1.5; 208 | } 209 | 210 | /* --- Message Form --- */ 211 | .message-form { 212 | position: fixed; 213 | bottom: 0; 214 | left: 260px; /* sidebar width */ 215 | right: 0; 216 | padding: 24px 0; 217 | background: linear-gradient(180deg, rgba(53,53,65,0) 0%, #353541 30%); 218 | border-top: 1px solid #2a2b32; 219 | display: flex; 220 | justify-content: center; 221 | } 222 | 223 | .message-form form { 224 | width: 100%; 225 | max-width: 768px; 226 | padding: 0 200px; 227 | display: flex; 228 | gap: 12px; 229 | } 230 | 231 | .message-input { 232 | flex-grow: 1; 233 | background: #40414f; 234 | border: 1px solid rgba(32,33,35,0.5); 235 | border-radius: 6px; 236 | color: #ececf1; 237 | font-size: 16px; 238 | line-height: 1.5; 239 | padding: 12px 16px; 240 | margin: 0; 241 | box-shadow: 0 0 15px rgba(0,0,0,0.1); 242 | } 243 | 244 | .message-button { 245 | padding: 12px 16px; 246 | background: #40414f; 247 | border: 1px solid rgba(32,33,35,0.5); 248 | border-radius: 6px; 249 | color: #ececf1; 250 | font-size: 14px; 251 | transition: background 0.2s; 252 | } 253 | 254 | .message-button:hover:not(:disabled) { 255 | background: #2a2b32; 256 | } 257 | 258 | /* --- Smooth scrolling & thin scrollbars --- */ 259 | .messages-container, 260 | .chat-list { 261 | scroll-behavior: smooth; 262 | } 263 | 264 | .messages-container::-webkit-scrollbar, 265 | .chat-list::-webkit-scrollbar { 266 | width: 6px; 267 | } 268 | 269 | .messages-container::-webkit-scrollbar-thumb, 270 | .chat-list::-webkit-scrollbar-thumb { 271 | background: #565869; 272 | border-radius: 3px; 273 | } 274 | 275 | .messages-container::-webkit-scrollbar-thumb:hover, 276 | .chat-list::-webkit-scrollbar-thumb:hover { 277 | background: #6b6d7a; 278 | } 279 | 280 | /* Firefox */ 281 | .messages-container, 282 | .chat-list { 283 | scrollbar-width: thin; 284 | scrollbar-color: #565869 transparent; 285 | } 286 | 287 | /* --- Focus & hover polish --- */ 288 | .new-chat-input:focus, 289 | .message-input:focus { 290 | border: 1.5px solid #007aff; 291 | box-shadow: 0 0 0 2px rgba(0,122,255,0.25); 292 | } 293 | 294 | .new-chat-button:hover, 295 | .message-button:hover:not(:disabled) { 296 | filter: brightness(1.1); 297 | } 298 | 299 | .message-button:focus-visible, 300 | .new-chat-button:focus-visible { 301 | outline: 2px solid #007aff; 302 | outline-offset: 2px; 303 | } 304 | 305 | /* Subtle fade for message container edges (ChatGPT-like) */ 306 | .messages-container::after { 307 | content: ''; 308 | position: sticky; 309 | bottom: 0; 310 | height: 48px; 311 | pointer-events: none; 312 | background: linear-gradient(to bottom, rgba(52,53,65,0) 0%, #343541 100%); 313 | } 314 | 315 | /* --- Landing Page --- */ 316 | .landing-container { 317 | flex-grow: 1; 318 | display: flex; 319 | flex-direction: column; 320 | align-items: center; 321 | justify-content: center; 322 | text-align: center; 323 | padding: 20px; 324 | background: #343541; 325 | } 326 | 327 | .landing-content { 328 | padding: 24px 200px; 329 | max-width: 768px; 330 | margin: 0 auto; 331 | width: 100%; 332 | } 333 | 334 | .landing-content h1 { 335 | font-size: 2rem; 336 | margin-bottom: 2rem; 337 | color: #ececf1; 338 | font-weight: 600; 339 | } 340 | 341 | .examples-grid { 342 | display: grid; 343 | grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); 344 | gap: 16px; 345 | margin-top: 40px; 346 | padding: 0 20px; 347 | } 348 | 349 | .example-card { 350 | background: #40414f; 351 | border: 1px solid #565869; 352 | border-radius: 8px; 353 | padding: 16px; 354 | text-align: left; 355 | cursor: pointer; 356 | transition: all 0.2s ease; 357 | display: flex; 358 | align-items: flex-start; 359 | gap: 12px; 360 | color: #ececf1; 361 | width: 100%; 362 | } 363 | 364 | .example-card:hover { 365 | background: #2a2b32; 366 | } 367 | 368 | .example-emoji { 369 | font-size: 1.2rem; 370 | flex-shrink: 0; 371 | } 372 | 373 | .example-text { 374 | font-size: 0.95rem; 375 | line-height: 1.4; 376 | } 377 | 378 | /* Refine existing message styles */ 379 | .message.assistant { 380 | padding: 24px 48px; 381 | background: #444654; 382 | margin: 0; 383 | border-bottom: 1px solid #2a2b32; 384 | } 385 | 386 | .message.user { 387 | padding: 24px 48px; 388 | background: #343541; 389 | margin: 0; 390 | border-bottom: 1px solid #2a2b32; 391 | } 392 | 393 | .message-content { 394 | background: transparent; 395 | box-shadow: none; 396 | padding: 0; 397 | max-width: 768px; 398 | margin: 0 auto; 399 | } 400 | 401 | .message.assistant .message-content, 402 | .message.user .message-content { 403 | border-radius: 0; 404 | background: transparent; 405 | } 406 | -------------------------------------------------------------------------------- /chatui/src/components/App.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect, useRef, useCallback, useMemo } from 'react'; 2 | import { useNavigate, useParams } from 'react-router-dom'; 3 | import ChatService from '../services/ChatService'; 4 | import { Message, Chat } from '../types/ChatTypes'; 5 | import Sidebar from './Sidebar'; 6 | import ChatContainer from './ChatContainer'; 7 | import VirtualizedChatList from './VirtualizedChatList'; 8 | import './App.css'; 9 | import { nanoid } from 'nanoid'; // lightweight id helper (already in many React projects; falls back to simple Date.now() if not installed) 10 | 11 | const loadingIndicatorId = 'loading-indicator'; 12 | 13 | interface ChatParams { 14 | chatId?: string; 15 | [key: string]: string | undefined; 16 | } 17 | 18 | const App: React.FC = () => { 19 | const [messages, setMessages] = useState([]); 20 | const [prompt, setPrompt] = useState(''); 21 | const [chats, setChats] = useState([]); 22 | const [selectedChatId, setSelectedChatId] = useState(null); 23 | const selectedChatIdRef = useRef(null); 24 | const [loadingChats, setLoadingChats] = useState(true); 25 | const messagesEndRef = useRef(null); 26 | const [newChatName, setNewChatName] = useState(''); 27 | const abortControllerRef = useRef(null); 28 | const [shouldAutoScroll, setShouldAutoScroll] = useState(true); 29 | const [streamingMessageId, setStreamingMessageId] = useState(null); 30 | const { chatId } = useParams(); 31 | const navigate = useNavigate(); 32 | 33 | const chatService = useMemo(() => ChatService.getInstance('/api/chat'), []); 34 | 35 | useEffect(() => { 36 | const fetchChats = async () => { 37 | try { 38 | const data = await chatService.getChats(); 39 | setChats(data); 40 | } catch (error) { 41 | console.error('Error fetching chats:', error); 42 | } finally { 43 | setLoadingChats(false); 44 | } 45 | }; 46 | 47 | fetchChats(); 48 | 49 | if (chatId) { 50 | handleChatSelect(chatId); 51 | } 52 | }, [chatId, chatService]); 53 | 54 | const onSelectChat = useCallback((id: string) => { 55 | navigate(`/chat/${id}`); 56 | }, [navigate]); 57 | 58 | const scrollToBottom = useCallback((behavior: ScrollBehavior = 'smooth') => { 59 | if (messagesEndRef.current && shouldAutoScroll) { 60 | messagesEndRef.current.scrollTo({ 61 | top: messagesEndRef.current.scrollHeight, 62 | behavior 63 | }); 64 | } 65 | }, [shouldAutoScroll]); 66 | 67 | const handleChatSelect = useCallback((id: string) => { 68 | setSelectedChatId(id); 69 | selectedChatIdRef.current = id; 70 | // message fetching and streaming now handled in useEffect below 71 | }, []); 72 | 73 | // Fetch messages and start streaming when selectedChatId changes 74 | useEffect(() => { 75 | if (!selectedChatId) return; 76 | 77 | let isActive = true; 78 | let abortController: AbortController | null = null; 79 | 80 | const fetchAndStream = async () => { 81 | setLoadingChats(true); 82 | // 1. Fetch messages 83 | let lastMessageId: string | null = null; 84 | try { 85 | const chatMessages = await chatService.getChatMessages(selectedChatId); 86 | setMessages(chatMessages); 87 | const filteredMessages = chatMessages.filter(msg => msg.text); 88 | lastMessageId = filteredMessages.length > 0 ? filteredMessages[filteredMessages.length - 1].id : null; 89 | } catch (error) { 90 | console.error('Error fetching chat messages:', error); 91 | } finally { 92 | setLoadingChats(false); 93 | } 94 | 95 | // 2. Start streaming 96 | if (abortControllerRef.current) { 97 | abortControllerRef.current.abort(); 98 | } 99 | abortController = new AbortController(); 100 | abortControllerRef.current = abortController; 101 | 102 | try { 103 | const stream = chatService.stream(selectedChatId, lastMessageId, abortController); 104 | for await (const { id, sender, text, isFinal } of stream) { 105 | if (!isActive || selectedChatIdRef.current !== selectedChatId) break; 106 | if (isFinal) { 107 | setStreamingMessageId(null); 108 | if (!text) continue; 109 | } else { 110 | setStreamingMessageId(current => current ? current : id); 111 | } 112 | updateMessageById(id, text, sender, isFinal); 113 | } 114 | } catch (error) { 115 | console.error('Stream error:', error); 116 | } finally { 117 | setStreamingMessageId(null); 118 | } 119 | }; 120 | 121 | fetchAndStream(); 122 | 123 | return () => { 124 | isActive = false; 125 | if (abortControllerRef.current) { 126 | abortControllerRef.current.abort(); 127 | abortControllerRef.current = null; 128 | } 129 | }; 130 | // Only restart when selectedChatId changes 131 | // eslint-disable-next-line react-hooks/exhaustive-deps 132 | }, [selectedChatId]); 133 | 134 | const updateMessageById = (id: string, newText: string, sender: string, isFinal: boolean = false) => { 135 | const generatingReplyMessageText = 'Generating reply...'; 136 | 137 | function getMessageText(existingText: string, newText: string): string { 138 | // If the existing text is the same as the generating reply message text, replace it with the new text 139 | if (existingText === generatingReplyMessageText) { 140 | return newText; 141 | } 142 | 143 | // If the existing text starts with the generating reply message text, replace it with the new text 144 | if (existingText.startsWith(generatingReplyMessageText)) { 145 | return existingText.replace(generatingReplyMessageText, '') + newText; 146 | } 147 | 148 | // Otherwise, append the new text to the existing text 149 | return existingText + newText; 150 | } 151 | 152 | setMessages(prevMessages => { 153 | const lastUserMessage = prevMessages.filter(m => m.sender === 'user').slice(-1)[0]; 154 | if (isFinal && lastUserMessage && lastUserMessage.text === newText) { 155 | return prevMessages; 156 | } 157 | const existingMessage = prevMessages.find(msg => msg.id === id); 158 | if (existingMessage) { 159 | return prevMessages.map(msg => 160 | msg.id === id 161 | ? { 162 | ...msg, 163 | text: getMessageText(msg.text, newText), 164 | isLoading: false, 165 | sender: sender || msg.sender 166 | } 167 | : msg 168 | ); 169 | } else { 170 | return [...prevMessages.filter(msg => msg.id !== loadingIndicatorId), 171 | { id, sender, text: newText, isLoading: false }, 172 | ]; 173 | } 174 | }); 175 | }; 176 | 177 | const handleScroll = useCallback((e: React.UIEvent) => { 178 | const container = e.target as HTMLDivElement; 179 | const isNearBottom = container.scrollHeight - container.scrollTop - container.clientHeight < 100; 180 | setShouldAutoScroll(isNearBottom); 181 | }, []); 182 | 183 | useEffect(() => { 184 | const container = messagesEndRef.current; 185 | if (container) { 186 | container.addEventListener('scroll', handleScroll as unknown as EventListener); 187 | return () => container.removeEventListener('scroll', handleScroll as unknown as EventListener); 188 | } 189 | }, [handleScroll]); 190 | 191 | useEffect(() => { 192 | if (shouldAutoScroll) { 193 | scrollToBottom(); 194 | } 195 | }, [messages, scrollToBottom]); 196 | 197 | const handleNewChat = useCallback(() => { 198 | // Cancel any ongoing stream 199 | if (streamingMessageId) { 200 | chatService.cancelChat(streamingMessageId); 201 | setStreamingMessageId(null); 202 | } 203 | if (abortControllerRef.current) { 204 | abortControllerRef.current.abort(); 205 | abortControllerRef.current = null; 206 | } 207 | setSelectedChatId(null); 208 | setMessages([]); 209 | setPrompt(''); 210 | selectedChatIdRef.current = null; 211 | navigate('/'); 212 | }, [navigate, chatService, streamingMessageId]); 213 | 214 | const generateChatTitle = (text: string): string => { 215 | // Take first 40 chars of first line, or "New Chat" if empty 216 | const firstLine = text.split('\n')[0].trim(); 217 | return firstLine.length > 40 ? firstLine.substring(0, 37) + '...' : firstLine || 'New Chat'; 218 | }; 219 | 220 | const handleSubmit = useCallback(async (e: React.FormEvent) => { 221 | e.preventDefault(); 222 | if (!prompt.trim() || streamingMessageId) return; 223 | 224 | const userMessage = { id: nanoid(), sender: 'user', text: prompt } as Message; 225 | setMessages(prev => [...prev, userMessage]); 226 | 227 | setMessages(prev => [ 228 | ...prev, 229 | { id: loadingIndicatorId, sender: 'assistant', text: 'Generating reply...' } 230 | ]); 231 | 232 | try { 233 | let activeChatId = selectedChatId; 234 | if (!activeChatId) { 235 | const title = generateChatTitle(prompt); 236 | const newChat = await chatService.createChat(title); 237 | setChats(prev => [...prev, newChat]); 238 | activeChatId = newChat.id; 239 | setSelectedChatId(activeChatId); 240 | navigate(`/chat/${activeChatId}`); 241 | } 242 | 243 | await chatService.sendPrompt(activeChatId!, prompt); 244 | setPrompt(''); 245 | } catch (error) { 246 | console.error('handleSubmit error:', error); 247 | setMessages(prev => 248 | prev.map(msg => 249 | msg.id === loadingIndicatorId 250 | ? { ...msg, text: '[Error in receiving response]' } 251 | : msg 252 | ) 253 | ); 254 | } 255 | }, [prompt, selectedChatId, streamingMessageId, chatService, navigate]); 256 | 257 | const handleNewChatSubmit = useCallback(async (e: React.FormEvent) => { 258 | e.preventDefault(); 259 | if (!newChatName.trim()) return; 260 | 261 | try { 262 | const newChat = await chatService.createChat(newChatName); 263 | setChats(prevChats => [...prevChats, newChat]); 264 | setNewChatName(''); 265 | onSelectChat(newChat.id); 266 | } catch (error) { 267 | console.error('handleNewChatSubmit error:', error); 268 | } 269 | }, [newChatName, chatService, onSelectChat]); 270 | 271 | const handleDeleteChat = async (e: React.MouseEvent, chatId: string) => { 272 | e.stopPropagation(); 273 | try { 274 | await chatService.deleteChat(chatId); 275 | setChats(prevChats => prevChats.filter(chat => chat.id !== chatId)); 276 | if (selectedChatId === chatId) { 277 | setSelectedChatId(null); 278 | setMessages([]); 279 | } 280 | } catch (error) { 281 | console.error('handleDeleteChat error:', chatId, error); 282 | } 283 | }; 284 | 285 | const cancelChat = () => { 286 | if (!streamingMessageId) return; 287 | chatService.cancelChat(streamingMessageId); 288 | }; 289 | 290 | return ( 291 |
292 | 299 | ( 309 | 310 | )} 311 | /> 312 |
313 | ); 314 | }; 315 | 316 | export default App; 317 | -------------------------------------------------------------------------------- /chatui/src/components/ChatContainer.tsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect, ReactNode, RefObject } from 'react'; 2 | import ReactMarkdown from 'react-markdown'; 3 | import { Message } from '../types/ChatTypes'; 4 | import LandingPage from './LandingPage'; 5 | 6 | interface ChatContainerProps { 7 | messages: Message[]; 8 | prompt: string; 9 | setPrompt: (prompt: string) => void; 10 | handleSubmit: (e: React.FormEvent) => void; 11 | cancelChat: () => void; 12 | streamingMessageId: string | null; 13 | messagesEndRef: RefObject; 14 | shouldAutoScroll: boolean; 15 | renderMessages: () => ReactNode; 16 | onExampleClick?: (text: string) => void; 17 | } 18 | 19 | const ChatContainer: React.FC = ({ 20 | messages, 21 | prompt, 22 | setPrompt, 23 | handleSubmit, 24 | cancelChat, 25 | streamingMessageId, 26 | messagesEndRef, 27 | shouldAutoScroll, 28 | renderMessages, 29 | onExampleClick 30 | }: ChatContainerProps) => { 31 | // Scroll only if near the bottom 32 | useEffect(() => { 33 | if (shouldAutoScroll && messagesEndRef.current) { 34 | messagesEndRef.current.scrollIntoView({ behavior: 'smooth' }); 35 | } 36 | }, [messages, shouldAutoScroll, messagesEndRef]); 37 | 38 | return ( 39 |
40 |
41 | {messages.length === 0 && ( 42 | { 43 | setPrompt(text); 44 | handleSubmit(new Event('submit') as any); 45 | }} /> 46 | )} 47 | {messages.map(msg => ( 48 |
49 |
50 | {msg.text} 51 |
52 |
53 | ))} 54 |
55 |
56 |
57 | setPrompt(e.target.value)} 61 | placeholder="Send a message..." 62 | disabled={streamingMessageId ? true : false} 63 | className="message-input" 64 | /> 65 | {streamingMessageId ? ( 66 | 69 | ) : ( 70 | 73 | )} 74 |
75 |
76 |
77 | ); 78 | }; 79 | 80 | export default ChatContainer; 81 | -------------------------------------------------------------------------------- /chatui/src/components/ChatList.tsx: -------------------------------------------------------------------------------- 1 | import { useNavigate, useParams } from 'react-router-dom'; 2 | import { Chat } from '../types/ChatTypes'; 3 | 4 | interface ChatListProps { 5 | chats: Chat[]; 6 | selectedChat: Chat | null; 7 | setSelectedChat: (chat: Chat) => void; 8 | } 9 | 10 | function ChatList({ chats, selectedChat, setSelectedChat }: ChatListProps) { 11 | const navigate = useNavigate(); 12 | const params = useParams(); 13 | 14 | return ( 15 |
16 | { 17 | chats.map(chat => ( 18 | 29 | )) 30 | } 31 |
32 | ); 33 | } 34 | 35 | export default ChatList; 36 | -------------------------------------------------------------------------------- /chatui/src/components/LandingPage.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | interface LandingPageProps { 4 | onExampleClick: (text: string) => void; 5 | } 6 | 7 | const examples = [ 8 | "Explain quantum computing in simple terms", 9 | "Got any creative ideas for a 10 year old's birthday?", 10 | "How do I make an HTTP request in Javascript?" 11 | ]; 12 | 13 | const LandingPage: React.FC = ({ onExampleClick }) => { 14 | return ( 15 |
16 |
17 |

Chat with AI

18 |
19 | {examples.map((example, index) => ( 20 | 28 | ))} 29 |
30 |
31 |
32 | ); 33 | }; 34 | 35 | export default LandingPage; 36 | -------------------------------------------------------------------------------- /chatui/src/components/Sidebar.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { useNavigate } from 'react-router-dom'; 3 | import { Chat } from '../types/ChatTypes'; 4 | 5 | interface SidebarProps { 6 | chats: Chat[]; 7 | selectedChatId: string | null; 8 | loadingChats: boolean; 9 | handleDeleteChat: (e: React.MouseEvent, chatId: string) => void; 10 | onNewChat: () => void; 11 | } 12 | 13 | const Sidebar: React.FC = ({ 14 | chats, 15 | selectedChatId, 16 | loadingChats, 17 | handleDeleteChat, 18 | onNewChat 19 | }) => { 20 | const navigate = useNavigate(); 21 | 22 | return ( 23 |
24 |
25 |

Chats

26 | {loadingChats &&

Loading...

} 27 |
28 | 31 |
    32 | {chats.map(chat => ( 33 |
  • navigate(`/chat/${chat.id}`)} 36 | className={`chat-item ${selectedChatId === chat.id ? 'selected' : ''}`} 37 | > 38 | {chat.name} 39 | 46 |
    • 47 | ))} 48 |
49 |
50 | ); 51 | }; 52 | 53 | export default Sidebar; 54 | -------------------------------------------------------------------------------- /chatui/src/components/VirtualizedChatList.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { FixedSizeList as List } from 'react-window'; 3 | import { Message } from '../types/ChatTypes'; 4 | 5 | interface VirtualizedChatListProps { 6 | messages: Message[]; 7 | } 8 | 9 | function VirtualizedChatList({ messages }: VirtualizedChatListProps) { 10 | return ( 11 | 17 | {({ index, style }) => { 18 | const msg = messages[index]; 19 | return ( 20 |
21 | {msg.sender}: {msg.text} 22 |
23 | ); 24 | }} 25 | 26 | ); 27 | } 28 | 29 | export default VirtualizedChatList; 30 | -------------------------------------------------------------------------------- /chatui/src/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom/client'; 3 | import AppRouter from './AppRouter'; 4 | 5 | const rootElement = document.getElementById('root'); 6 | if (!rootElement) throw new Error('Failed to find the root element'); 7 | 8 | ReactDOM.createRoot(rootElement).render( 9 | 10 | 11 | 12 | ); 13 | -------------------------------------------------------------------------------- /chatui/src/services/ChatService.ts: -------------------------------------------------------------------------------- 1 | import { Chat, Message, MessageFragment } from '../types/ChatTypes'; 2 | import * as signalR from '@microsoft/signalr'; 3 | import { UnboundedChannel } from '../utils/UnboundedChannel'; 4 | 5 | class ChatService { 6 | private static instance: ChatService; 7 | private hubConnection?: signalR.HubConnection; 8 | private initialized = false; 9 | private backendUrl: string; 10 | private activeStreams = new Map>(); 11 | 12 | private constructor(backendUrl: string) { 13 | this.backendUrl = backendUrl; 14 | } 15 | 16 | static getInstance(backendUrl: string): ChatService { 17 | if (!ChatService.instance) { 18 | ChatService.instance = new ChatService(backendUrl); 19 | } 20 | return ChatService.instance; 21 | } 22 | 23 | async ensureInitialized(): Promise { 24 | if (!this.hubConnection && !this.initialized) { 25 | console.debug('Initializing SignalR connection...'); 26 | this.hubConnection = new signalR.HubConnectionBuilder() 27 | .withUrl(`${this.backendUrl}/stream`, { 28 | skipNegotiation: true, 29 | transport: signalR.HttpTransportType.WebSockets 30 | }) 31 | .withStatefulReconnect() 32 | .withAutomaticReconnect({ 33 | nextRetryDelayInMilliseconds: retryContext => { 34 | // We want to retry forever 35 | if (retryContext.elapsedMilliseconds > 15 * 1000) { 36 | // Max 15 seconds delay 37 | return 15000; 38 | } 39 | 40 | // Otherwise, we want to retry every second 41 | return (retryContext.previousRetryCount + 1) * 1000; 42 | } 43 | }) 44 | .build(); 45 | 46 | this.hubConnection.onreconnected(async () => { 47 | console.debug('Reconnected to SignalR hub'); 48 | 49 | for (const channel of this.activeStreams.values()) { 50 | channel?.close(); 51 | } 52 | this.activeStreams.clear(); 53 | }); 54 | 55 | await this.hubConnection.start(); 56 | this.initialized = true; 57 | } 58 | } 59 | 60 | async getChats(): Promise { 61 | const response = await fetch(`${this.backendUrl}`); 62 | if (!response.ok) { 63 | throw new Error('Error fetching chats'); 64 | } 65 | return await response.json(); 66 | } 67 | 68 | async getChatMessages(chatId: string): Promise { 69 | const response = await fetch(`${this.backendUrl}/${chatId}`); 70 | if (!response.ok) { 71 | throw new Error('Error fetching chat messages'); 72 | } 73 | return await response.json(); 74 | } 75 | 76 | async createChat(name: string): Promise { 77 | const response = await fetch(`${this.backendUrl}`, { 78 | method: 'POST', 79 | headers: { 80 | 'Content-Type': 'application/json' 81 | }, 82 | body: JSON.stringify({ name }) 83 | }); 84 | if (!response.ok) { 85 | throw new Error('Failed to create chat'); 86 | } 87 | return await response.json(); 88 | } 89 | 90 | async *stream( 91 | id: string, 92 | initialLastMessageId: string | null, 93 | abortController: AbortController 94 | ): AsyncGenerator { 95 | await this.ensureInitialized(); 96 | 97 | if (!this.hubConnection) { 98 | throw new Error('ChatService not initialized'); 99 | } 100 | 101 | let lastFragmentId: string | undefined; 102 | let lastMessageId = initialLastMessageId; 103 | 104 | // Set up and store the abort event handler 105 | const abortHandler = () => { 106 | console.log(`Aborting stream for chat: ${id}`); 107 | this.activeStreams.get(id)?.close(); 108 | }; 109 | 110 | abortController.signal.addEventListener('abort', abortHandler); 111 | 112 | try { 113 | while (!abortController.signal.aborted) { 114 | let channel = new UnboundedChannel(); 115 | this.activeStreams.set(id, channel); 116 | 117 | let subscription = this.hubConnection.stream("Stream", id, { lastMessageId, lastFragmentId }) 118 | .subscribe({ 119 | next: (value) => { 120 | const fragment: MessageFragment = { 121 | id: value.id, 122 | sender: value.sender, 123 | text: value.text, 124 | isFinal: value.isFinal ?? false, 125 | fragmentId: value.fragmentId 126 | }; 127 | lastFragmentId = fragment.fragmentId; 128 | if (fragment.isFinal) { 129 | lastMessageId = fragment.id; 130 | } 131 | channel.write(fragment); 132 | }, 133 | complete: () => { 134 | console.debug(`Stream completed for chat: ${id}`); 135 | channel.close(); 136 | }, 137 | error: (err) => { 138 | // Don't close the channel on error, if the connection breaks, signalr will reconnect 139 | // and we'll close the channel 140 | } 141 | }); 142 | 143 | try { 144 | for await (const fragment of channel) { 145 | yield fragment; 146 | } 147 | } catch (error) { 148 | console.error('Stream error:', error); 149 | // Only break the loop if we're aborting 150 | if (abortController.signal.aborted) { 151 | break; 152 | } 153 | } finally { 154 | subscription?.dispose(); 155 | this.activeStreams.delete(id); 156 | } 157 | 158 | // If we're not aborting, wait a second before retrying 159 | if (!abortController.signal.aborted) { 160 | await new Promise(resolve => setTimeout(resolve, 1000)); 161 | } 162 | } 163 | } finally { 164 | abortController.signal.removeEventListener('abort', abortHandler); 165 | } 166 | } 167 | 168 | async sendPrompt(id: string, prompt: string): Promise { 169 | const response = await fetch(`${this.backendUrl}/${id}`, { 170 | method: 'POST', 171 | headers: { 'Content-Type': 'application/json' }, 172 | body: JSON.stringify({ text: prompt }) 173 | }); 174 | 175 | if (!response.ok) { 176 | let errorMessage; 177 | try { 178 | errorMessage = await response.text(); 179 | } catch (e) { 180 | errorMessage = response.statusText; 181 | } 182 | throw new Error(`Error sending prompt: ${errorMessage}`); 183 | } 184 | } 185 | 186 | async deleteChat(id: string): Promise { 187 | const response = await fetch(`${this.backendUrl}/${id}`, { 188 | method: 'DELETE' 189 | }); 190 | 191 | if (!response.ok) { 192 | throw new Error('Failed to delete chat'); 193 | } 194 | } 195 | 196 | async cancelChat(id: string): Promise { 197 | const response = await fetch(`${this.backendUrl}/${id}/cancel`, { 198 | method: 'POST' 199 | }); 200 | if (!response.ok) { 201 | throw new Error('Failed to cancel chat'); 202 | } 203 | } 204 | } 205 | 206 | export default ChatService; 207 | -------------------------------------------------------------------------------- /chatui/src/types/ChatTypes.ts: -------------------------------------------------------------------------------- 1 | export interface Chat { 2 | id: string; 3 | name: string; 4 | } 5 | 6 | export interface Message { 7 | id: string; 8 | sender: string; 9 | text: string; 10 | } 11 | 12 | export interface MessageFragment extends Message { 13 | fragmentId: string; 14 | isFinal: boolean; 15 | } 16 | -------------------------------------------------------------------------------- /chatui/src/utils/UnboundedChannel.ts: -------------------------------------------------------------------------------- 1 | import Denque from "denque"; 2 | 3 | /** 4 | * Unbounded channel that supports: 5 | * - Multiple producers writing items 6 | * - Consumers reading items or using for-await-of 7 | * - Signaling channel closed (no more items) 8 | * - Signaling an error (consumers see a rejected Promise) 9 | * - Performance-friendly, O(1) enqueue/dequeue via denque 10 | */ 11 | export class UnboundedChannel { 12 | private _queue = new Denque(); 13 | // Store an array of [resolve, reject] for pending reads 14 | private _waitingConsumers: Array< 15 | [(value: IteratorResult) => void, (reason?: any) => void] 16 | > = []; 17 | 18 | private _closed = false; 19 | private _error: Error | null = null; 20 | 21 | /** 22 | * Writes an item into the channel. 23 | * Throws an error if the channel is closed or errored. 24 | */ 25 | public write(item: T): void { 26 | if (this._closed) { 27 | throw new Error("Cannot write to a closed channel."); 28 | } 29 | if (this._error) { 30 | throw new Error("Cannot write to an errored channel: " + this._error.message); 31 | } 32 | 33 | // If there's a waiting consumer, resolve it immediately with the new item 34 | if (this._waitingConsumers.length > 0) { 35 | const [resolve] = this._waitingConsumers.shift()!; 36 | resolve({ value: item, done: false }); 37 | } else { 38 | // Otherwise, enqueue the item 39 | this._queue.push(item); 40 | } 41 | } 42 | 43 | /** 44 | * Reads an item from the channel as an IteratorResult: 45 | * { value: T, done: false } if item is available 46 | * { value: undefined, done: true } if channel is closed and no items left 47 | * If an error has been signaled, this Promise rejects with that Error. 48 | */ 49 | public read(): Promise> { 50 | // If there's an error, reject immediately 51 | if (this._error) { 52 | return Promise.reject(this._error); 53 | } 54 | 55 | // If we have items in the queue, return the next one 56 | if (this._queue.length > 0) { 57 | const value = this._queue.shift()!; 58 | return Promise.resolve({ value, done: false }); 59 | } 60 | 61 | // If channel is closed and no items, return done 62 | if (this._closed) { 63 | return Promise.resolve({ value: undefined, done: true }); 64 | } 65 | 66 | // Otherwise, wait until an item is written or an error/close occurs 67 | return new Promise>((resolve, reject) => { 68 | this._waitingConsumers.push([resolve, reject]); 69 | }); 70 | } 71 | 72 | /** 73 | * Closes the channel. No further writes allowed. 74 | * All pending consumers receive { value: undefined, done: true }. 75 | */ 76 | public close(): void { 77 | // If already closed or errored, do nothing 78 | if (this._closed || this._error) { 79 | return; 80 | } 81 | 82 | this._closed = true; 83 | 84 | // Resolve all waiting consumers with 'done = true' 85 | while (this._waitingConsumers.length > 0) { 86 | const [resolve] = this._waitingConsumers.shift()!; 87 | resolve({ value: undefined, done: true }); 88 | } 89 | } 90 | 91 | /** 92 | * Signals an error. No further writes or reads. 93 | * All pending consumers get a rejected Promise with the given Error. 94 | */ 95 | public throwError(err: Error): void { 96 | if (this._error || this._closed) { 97 | // If already in an error/closed state, do nothing or rethrow as needed 98 | return; 99 | } 100 | 101 | this._error = err; 102 | 103 | // Reject all waiting consumers 104 | while (this._waitingConsumers.length > 0) { 105 | const [, reject] = this._waitingConsumers.shift()!; 106 | reject(err); 107 | } 108 | } 109 | 110 | /** 111 | * Makes the channel compatible with `for await...of`. 112 | */ 113 | public [Symbol.asyncIterator](): AsyncIterator { 114 | return { 115 | next: () => this.read() 116 | }; 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /chatui/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "useDefineForClassFields": true, 5 | "lib": ["ES2020", "DOM", "DOM.Iterable"], 6 | "module": "ESNext", 7 | "skipLibCheck": true, 8 | "esModuleInterop": true, 9 | 10 | /* Bundler mode */ 11 | "moduleResolution": "bundler", 12 | "allowImportingTsExtensions": true, 13 | "resolveJsonModule": true, 14 | "isolatedModules": true, 15 | "noEmit": true, 16 | "jsx": "react-jsx", 17 | 18 | /* Linting */ 19 | "strict": true, 20 | "noUnusedLocals": false, 21 | "noUnusedParameters": false, 22 | "noFallthroughCasesInSwitch": true 23 | }, 24 | "include": ["src"], 25 | "references": [{ "path": "./tsconfig.node.json" }] 26 | } 27 | -------------------------------------------------------------------------------- /chatui/tsconfig.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "composite": true, 4 | "skipLibCheck": true, 5 | "module": "ESNext", 6 | "moduleResolution": "bundler", 7 | "allowSyntheticDefaultImports": true 8 | }, 9 | "include": ["vite.config.ts"] 10 | } 11 | -------------------------------------------------------------------------------- /chatui/vite.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vite'; 2 | import react from '@vitejs/plugin-react'; 3 | import type { UserConfig, ServerOptions } from 'vite'; 4 | 5 | const port = process.env.PORT ? parseInt(process.env.PORT) : undefined; 6 | 7 | export default defineConfig({ 8 | plugins: [react()], 9 | server: { 10 | open: true, 11 | port, 12 | proxy: { 13 | '/api': { 14 | target: process.env.BACKEND_URL, 15 | changeOrigin: true, 16 | }, 17 | '/api/chat/stream': { 18 | target: process.env.BACKEND_URL, 19 | ws: true, 20 | changeOrigin: true, 21 | } 22 | } 23 | }, 24 | build: { 25 | outDir: 'build' 26 | } 27 | } as UserConfig); 28 | -------------------------------------------------------------------------------- /global.json: -------------------------------------------------------------------------------- 1 | { 2 | "sdk": { 3 | "version": "9.0.100", 4 | "rollForward": "latestMajor", 5 | "allowPrerelease": true 6 | } 7 | } -------------------------------------------------------------------------------- /nuget.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | --------------------------------------------------------------------------------