├── .vscode ├── ltex.dictionary.en-US.txt └── ltex.hiddenFalsePositives.en-US.txt ├── assets ├── backend.png ├── namedValues.png ├── api_url_suffix.png ├── product_policy.png ├── gpt_35_post_settings.png ├── product_subscription.png ├── subscription_key_name.png └── aoai_apim.drawio ├── LogParserFunction ├── Properties │ └── launchSettings.json ├── host.json ├── Models │ ├── Request.cs │ ├── Embedding │ │ ├── EmbeddingRequest.cs │ │ └── EmbeddingResponse.cs │ ├── ChatCompletion │ │ ├── Vision │ │ │ ├── ImageUrl.cs │ │ │ └── VisionContent.cs │ │ ├── Tool.cs │ │ ├── Stream │ │ │ ├── Choice.cs │ │ │ └── ChatCompletionStreamResponse.cs │ │ ├── ChatCompletionResponse.cs │ │ ├── Function.cs │ │ ├── Choice.cs │ │ ├── Message.cs │ │ └── ChatCompletionRequest.cs │ ├── ContentSafety │ │ ├── FilterResult.cs │ │ ├── PromptFilterResult.cs │ │ └── ContentFilterResult.cs │ ├── Usage.cs │ ├── Completion │ │ ├── CompletionResponse.cs │ │ ├── Choice.cs │ │ └── CompletionRequest.cs │ ├── TempLog.cs │ ├── TempResponseLog.cs │ ├── TempStreamResponseLog.cs │ ├── TempRequestLog.cs │ ├── Response.cs │ └── AOAILog.cs ├── __local.settings.json ├── Usings.cs ├── Program.cs ├── LogParserFunction.csproj ├── LogParser.cs ├── Services │ ├── ContentSafetyService.cs │ ├── TikTokenService.cs │ └── DataProcessService.cs ├── README.md ├── .gitignore └── Extensions │ └── StringExtensions.cs ├── LoggingWebApi ├── Models │ ├── Containers.cs │ ├── TempResponseLog.cs │ ├── TempStreamResponseLog.cs │ ├── TempRequestLog.cs │ └── TempLog.cs ├── __appsettings.json ├── Properties │ └── launchSettings.json ├── Usings.cs ├── LoggingWebApi.csproj ├── Program.cs ├── README.md └── Controllers │ └── OpenAI.cs ├── infra ├── network │ ├── dns.bicep │ ├── publicIp.bicep │ ├── privateEndpoint.bicep │ └── vnet.bicep ├── main.parameters.json ├── apim │ ├── apimPolicyFragment.bicep │ ├── apimApplicationInsights.bicep │ ├── apimBackend.bicep │ ├── apim.bicep │ └── apimApis.bicep ├── security │ ├── keyVault.bicep │ └── roles.bicep ├── ai │ ├── aoai.bicep │ └── contentSafety.bicep ├── README.md ├── monitoring │ └── applicationInsights.bicep ├── webapp │ ├── webApi.bicep │ └── function.bicep ├── db │ └── cosmosDb.bicep ├── abbreviations.json └── main.bicep ├── CODE_OF_CONDUCT.md ├── policies ├── operation-policy.xml ├── api-policy.xml └── inbound-logging.xml ├── azure.yaml ├── .dockerignore ├── LICENSE ├── SUPPORT.md ├── apim_log.sln ├── queries └── query.kql ├── .gitattributes ├── SECURITY.md ├── .gitignore ├── README.md └── APIM.md /.vscode/ltex.dictionary.en-US.txt: -------------------------------------------------------------------------------- 1 | APIM 2 | AOAI 3 | IaC 4 | Serilog 5 | dotnet 6 | VNet 7 | -------------------------------------------------------------------------------- /assets/backend.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/aoai-logging-with-apim/HEAD/assets/backend.png -------------------------------------------------------------------------------- /assets/namedValues.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/aoai-logging-with-apim/HEAD/assets/namedValues.png -------------------------------------------------------------------------------- /assets/api_url_suffix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/aoai-logging-with-apim/HEAD/assets/api_url_suffix.png -------------------------------------------------------------------------------- /assets/product_policy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/aoai-logging-with-apim/HEAD/assets/product_policy.png -------------------------------------------------------------------------------- /assets/gpt_35_post_settings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/aoai-logging-with-apim/HEAD/assets/gpt_35_post_settings.png -------------------------------------------------------------------------------- /assets/product_subscription.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/aoai-logging-with-apim/HEAD/assets/product_subscription.png -------------------------------------------------------------------------------- /assets/subscription_key_name.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/aoai-logging-with-apim/HEAD/assets/subscription_key_name.png -------------------------------------------------------------------------------- /LogParserFunction/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "profiles": { 3 | "LogParserFunction": { 4 | "commandName": "Project", 5 | "commandLineArgs": "--port 7014", 6 | "launchBrowser": false 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /LoggingWebApi/Models/Containers.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LoggingWebApi.Models; 4 | 5 | /// 6 | /// Cosmo DB Containers 7 | /// 8 | public class Containers : Dictionary 9 | { 10 | } 11 | -------------------------------------------------------------------------------- /LogParserFunction/host.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0", 3 | "logging": { 4 | "applicationInsights": { 5 | "samplingSettings": { 6 | "isEnabled": true, 7 | "excludedTypes": "Request" 8 | } 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /infra/network/dns.bicep: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | param privateDnsZoneNames array 4 | 5 | resource privateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = [for privateDnsZoneName in privateDnsZoneNames: { 6 | name: privateDnsZoneName 7 | location: 'global' 8 | }] 9 | -------------------------------------------------------------------------------- /LogParserFunction/Models/Request.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models; 4 | 5 | /// 6 | /// Request Message 7 | /// 8 | public class Request 9 | { 10 | [JsonProperty("id")] 11 | public string Id { get; set; } = string.Empty; 12 | } -------------------------------------------------------------------------------- /LoggingWebApi/__appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft.AspNetCore": "Warning" 6 | } 7 | }, 8 | "CosmosDbConnectionString": "", 9 | "CosmosDbDatabaseName": "", 10 | "CosmosDbLogContainerName": "", 11 | "CosmosDbTriggerContainerName": "" 12 | } 13 | -------------------------------------------------------------------------------- /LogParserFunction/Models/Embedding/EmbeddingRequest.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models.Embedding; 4 | 5 | /// 6 | /// Request Message 7 | /// 8 | public class EmbeddingRequest : Request 9 | { 10 | [JsonProperty("input")] 11 | public string Input { get; set; } = string.Empty; 12 | } -------------------------------------------------------------------------------- /LoggingWebApi/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "profiles": { 3 | "LoggingWebApi": { 4 | "commandName": "Project", 5 | "launchBrowser": true, 6 | "environmentVariables": { 7 | "ASPNETCORE_ENVIRONMENT": "Development" 8 | }, 9 | "applicationUrl": "https://localhost:56128;http://localhost:56129" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /infra/main.parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "environmentName": { 6 | "value": "${AZURE_ENV_NAME}" 7 | }, 8 | "location": { 9 | "value": "${AZURE_LOCATION}" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /LogParserFunction/Models/ChatCompletion/Vision/ImageUrl.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models.ChatCompletion.Vision; 4 | 5 | /// 6 | /// Image Url 7 | /// 8 | public class ImageUrl 9 | { 10 | [JsonProperty("url")] 11 | public string Url { get; set; } = string.Empty; 12 | 13 | [JsonProperty("detail")] 14 | public string? Details { get; set; } 15 | } -------------------------------------------------------------------------------- /LogParserFunction/Models/Embedding/EmbeddingResponse.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models.Embedding; 4 | 5 | /// 6 | /// EmbeddingResponse Response 7 | /// 8 | public class EmbeddingResponse : Response 9 | { 10 | // We are not logging embbedded data on purpose 11 | //[JsonProperty("data")] 12 | //public string Data { get; set; } = new(); 13 | } 14 | -------------------------------------------------------------------------------- /LogParserFunction/Models/ContentSafety/FilterResult.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models.ContentSafety; 4 | 5 | /// 6 | /// Content Filter Result 7 | /// 8 | public class FilterResult 9 | { 10 | [JsonProperty("filtered")] 11 | public bool Filtered { get; set; } 12 | 13 | [JsonProperty("severity")] 14 | public string Severity { get; set; } = "safe"; 15 | } 16 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /LogParserFunction/Models/Usage.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models; 4 | 5 | /// 6 | /// Token Usage 7 | /// 8 | public class Usage 9 | { 10 | [JsonProperty("prompt_tokens")] 11 | public int PromptTokens { get; set; } 12 | 13 | [JsonProperty("completion_tokens")] 14 | public int CompletionTokens { get; set; } 15 | 16 | [JsonProperty("total_tokens")] 17 | public int TotalTokens { get; set; } 18 | } -------------------------------------------------------------------------------- /LogParserFunction/Models/ContentSafety/PromptFilterResult.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models.ContentSafety; 4 | 5 | /// 6 | /// Prompt Filter Result 7 | /// 8 | public class PromptFilterResult 9 | { 10 | [JsonProperty("prompt_index")] 11 | public int PromptIndex { get; set; } 12 | 13 | [JsonProperty("content_filter_results")] 14 | public ContentFilterResults ContentFilterResults { get; set; } = new(); 15 | } -------------------------------------------------------------------------------- /infra/network/publicIp.bicep: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | param location string 4 | param publicIpName string 5 | 6 | resource publicIP 'Microsoft.Network/publicIPAddresses@2023-04-01' = { 7 | name: publicIpName 8 | location: location 9 | sku: { 10 | name: 'Standard' 11 | tier: 'Regional' 12 | } 13 | properties: { 14 | dnsSettings: { 15 | domainNameLabel: publicIpName 16 | } 17 | publicIPAllocationMethod: 'Static' 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /policies/operation-policy.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {{backend-url}} 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /LogParserFunction/Models/ChatCompletion/Tool.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | using System.Reflection.Metadata.Ecma335; 4 | 5 | namespace LogParser.Models.ChatCompletion; 6 | 7 | /// 8 | /// ChatCompletion Request Message 9 | /// 10 | public class Tool 11 | { 12 | [JsonProperty("type")] 13 | public string Type { get; set; } = string.Empty; 14 | 15 | [JsonProperty("function")] 16 | public Function Function { get; set; } = new(); 17 | } -------------------------------------------------------------------------------- /azure.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json 2 | 3 | name: aoai-logging-with-apim 4 | metadata: 5 | template: aoai-logging-with-apim@0.0.1-beta 6 | infra: 7 | provider: bicep 8 | services: 9 | loggingWebApi: 10 | project: ./LoggingWebApi/ 11 | language: dotnet 12 | host: appservice 13 | logParserFunction: 14 | project: ./LogParserFunction/ 15 | language: dotnet 16 | host: function 17 | 18 | -------------------------------------------------------------------------------- /LogParserFunction/Models/Completion/CompletionResponse.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models.Completion; 4 | 5 | /// 6 | /// Completion Response 7 | /// 8 | public class CompletionResponse : Response 9 | { 10 | [JsonProperty("choices")] 11 | public List Choices { get; set; } = new(); 12 | 13 | [JsonProperty("prompt_filter_results")] 14 | public List PromptFilterResults { get; set; } = new(); 15 | } 16 | -------------------------------------------------------------------------------- /LogParserFunction/__local.settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "IsEncrypted": false, 3 | "Values": { 4 | "AzureWebJobsStorage": "UseDevelopmentStorage=true", 5 | "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated", 6 | "CosmosDbConnectionString": "", 7 | "CosmosDbDatabaseName": "Logs", 8 | "CosmosDbLogContainerName": "TempLogs", 9 | "CosmosDbTriggerContainerName": "LogTriggers", 10 | "ContentSafetyUrl": "", 11 | "ContentSafetyKey": "", 12 | "ApplicationInsightsConnectionString": "" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | **/.classpath 2 | **/.dockerignore 3 | **/.env 4 | **/.git 5 | **/.gitignore 6 | **/.project 7 | **/.settings 8 | **/.toolstarget 9 | **/.vs 10 | **/.vscode 11 | **/*.*proj.user 12 | **/*.dbmdl 13 | **/*.jfm 14 | **/azds.yaml 15 | **/bin 16 | **/charts 17 | **/docker-compose* 18 | **/Dockerfile* 19 | **/node_modules 20 | **/npm-debug.log 21 | **/obj 22 | **/secrets.dev.yaml 23 | **/values.dev.yaml 24 | LICENSE 25 | README.md 26 | !**/.gitignore 27 | !.git/HEAD 28 | !.git/config 29 | !.git/packed-refs 30 | !.git/refs/heads/** -------------------------------------------------------------------------------- /LogParserFunction/Models/ChatCompletion/Vision/VisionContent.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models.ChatCompletion.Vision; 4 | 5 | /// 6 | /// Vision Content 7 | /// 8 | public class VisionContent 9 | { 10 | [JsonProperty("type")] 11 | public string Type { get; set; } = string.Empty; 12 | 13 | [JsonProperty("text")] 14 | public string? Text { get; set; } 15 | 16 | [JsonProperty("image_url")] 17 | public ImageUrl? ImageUrl { get; set; } 18 | } -------------------------------------------------------------------------------- /LogParserFunction/Models/ChatCompletion/Stream/Choice.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models.ChatCompletion.Stream; 4 | 5 | /// 6 | /// Streaming Choice 7 | /// 8 | public class Choice 9 | { 10 | [JsonProperty("finish_reason")] 11 | public string FinishReason { get; set; } = string.Empty; 12 | 13 | [JsonProperty("index")] 14 | public int Index { get; set; } 15 | 16 | [JsonProperty("delta")] 17 | public Message Delta { get; set; } = new(); 18 | } 19 | -------------------------------------------------------------------------------- /LogParserFunction/Models/ChatCompletion/ChatCompletionResponse.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models.ChatCompletion; 4 | 5 | /// 6 | /// ChatCompletion Response Message 7 | /// 8 | public class ChatCompletionResponse : Response 9 | { 10 | [JsonProperty("choices")] 11 | public List Choices { get; set; } = new(); 12 | 13 | [JsonProperty("prompt_filter_results")] 14 | public List PromptFilterResults { get; set; } = new(); 15 | 16 | } 17 | -------------------------------------------------------------------------------- /LoggingWebApi/Usings.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | global using Azure.Core; 4 | global using Azure.Identity; 5 | global using LoggingWebApi.Models; 6 | global using Microsoft.AspNetCore.Mvc; 7 | global using Microsoft.Azure.Cosmos; 8 | global using Microsoft.Azure.Cosmos.Fluent; 9 | global using Microsoft.Extensions.Primitives; 10 | global using Microsoft.Net.Http.Headers; 11 | global using Newtonsoft.Json; 12 | global using Newtonsoft.Json.Linq; 13 | global using System.Diagnostics; 14 | global using System.Text; 15 | -------------------------------------------------------------------------------- /LogParserFunction/Models/TempLog.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models; 4 | 5 | /// 6 | /// Temporary Log 7 | /// 8 | public class TempLog 9 | { 10 | [JsonProperty("id")] 11 | public Guid Id { get; set; } 12 | 13 | [JsonProperty("type")] 14 | public virtual string Type { get; } = string.Empty; 15 | 16 | [JsonProperty("requestId")] 17 | public string RequestId { get; set; } = string.Empty; 18 | 19 | [JsonProperty("headers")] 20 | public dynamic? Headers { get; set; } 21 | } -------------------------------------------------------------------------------- /LogParserFunction/Models/ChatCompletion/Stream/ChatCompletionStreamResponse.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models.ChatCompletion.Stream; 4 | 5 | /// 6 | /// Chat Completion Streaming Response Message 7 | /// 8 | public class ChatCompletionStreamResponse : Response 9 | { 10 | [JsonProperty("choices")] 11 | public List Choices { get; set; } = new(); 12 | 13 | [JsonProperty("prompt_filter_results")] 14 | public List PromptFilterResults { get; set; } = new(); 15 | } 16 | -------------------------------------------------------------------------------- /LogParserFunction/Models/TempResponseLog.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models; 4 | 5 | /// 6 | /// Temporary Response Log 7 | /// 8 | public class TempResponseLog : TempLog 9 | { 10 | /// 11 | /// Log Type 12 | /// 13 | [JsonProperty("type")] 14 | public override string Type { get; } = "Response"; 15 | 16 | /// 17 | /// Response 18 | /// 19 | [JsonProperty("response")] 20 | public JObject Response { get; set; } = new(); 21 | } 22 | -------------------------------------------------------------------------------- /LoggingWebApi/Models/TempResponseLog.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LoggingWebApi.Models; 4 | 5 | /// 6 | /// Temporary Response Log 7 | /// 8 | public class TempResponseLog : TempLog 9 | { 10 | /// 11 | /// Log Type 12 | /// 13 | [JsonProperty("type")] 14 | public override string Type { get; } = "Response"; 15 | 16 | /// 17 | /// Response 18 | /// 19 | [JsonProperty("response")] 20 | public JObject Response { get; set; } = new(); 21 | } 22 | -------------------------------------------------------------------------------- /LogParserFunction/Models/TempStreamResponseLog.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models; 4 | 5 | /// 6 | /// Temporary Stream Response Log 7 | /// 8 | public class TempStreamResponseLog : TempLog 9 | { 10 | /// 11 | /// Log Type 12 | /// 13 | [JsonProperty("type")] 14 | public override string Type { get; } = "StreamResponse"; 15 | 16 | /// 17 | /// Response 18 | /// 19 | [JsonProperty("response")] 20 | public string Response { get; set; } = string.Empty; 21 | } 22 | -------------------------------------------------------------------------------- /LoggingWebApi/Models/TempStreamResponseLog.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LoggingWebApi.Models; 4 | 5 | /// 6 | /// Temporary Stream Response Log 7 | /// 8 | public class TempStreamResponseLog : TempLog 9 | { 10 | /// 11 | /// Log Type 12 | /// 13 | [JsonProperty("type")] 14 | public override string Type { get; } = "StreamResponse"; 15 | 16 | /// 17 | /// Response 18 | /// 19 | [JsonProperty("response")] 20 | public string Response { get; set; } = string.Empty; 21 | } 22 | -------------------------------------------------------------------------------- /LogParserFunction/Models/ChatCompletion/Function.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models.ChatCompletion; 4 | 5 | /// 6 | /// Function 7 | /// 8 | public class Function 9 | { 10 | [JsonProperty("name")] 11 | public string Name { get; set; } = string.Empty; 12 | 13 | [JsonProperty("description")] 14 | public string Description { get; set; } = string.Empty; 15 | 16 | [JsonProperty("parameters")] 17 | public dynamic? Parameters { get; set; } 18 | 19 | [JsonProperty("required")] 20 | public List Required { get; set; } = new(); 21 | } -------------------------------------------------------------------------------- /infra/apim/apimPolicyFragment.bicep: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | param apiManagementServiceName string 4 | 5 | var inboundLoggingFragment = loadTextContent('../../policies/inbound-logging.xml') 6 | 7 | resource apiManagementService 'Microsoft.ApiManagement/service@2023-03-01-preview' existing = { 8 | name: apiManagementServiceName 9 | } 10 | 11 | resource inboundLogging 'Microsoft.ApiManagement/service/policyFragments@2023-03-01-preview' = { 12 | name: 'inbound-logging' 13 | parent: apiManagementService 14 | properties: { 15 | description: 'string' 16 | format: 'rawxml' 17 | value: inboundLoggingFragment 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /policies/api-policy.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /LogParserFunction/Models/ContentSafety/ContentFilterResult.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models.ContentSafety; 4 | 5 | /// 6 | /// Content Filter Results 7 | /// 8 | public class ContentFilterResults 9 | { 10 | [JsonProperty("hate")] 11 | public FilterResult Hate { get; set; } = new(); 12 | 13 | [JsonProperty("self_harm")] 14 | public FilterResult SelfHarm { get; set; } = new(); 15 | 16 | [JsonProperty("sexual")] 17 | public FilterResult Sexual { get; set; } = new(); 18 | 19 | [JsonProperty("violence")] 20 | public FilterResult Violence { get; set; } = new(); 21 | } 22 | -------------------------------------------------------------------------------- /LogParserFunction/Models/ChatCompletion/Choice.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | using LogParser.Models.ContentSafety; 4 | 5 | namespace LogParser.Models.ChatCompletion; 6 | 7 | /// 8 | /// ChatCompletion Response Choice 9 | /// 10 | public class Choice 11 | { 12 | [JsonProperty("finish_reason")] 13 | public string FinishReason { get; set; } = string.Empty; 14 | 15 | [JsonProperty("index")] 16 | public int Index { get; set; } 17 | 18 | [JsonProperty("message")] 19 | public Message Message { get; set; } = new(); 20 | 21 | [JsonProperty("content_filter_results")] 22 | public ContentFilterResults ContentFilterResults { get; set; } = new(); 23 | } 24 | -------------------------------------------------------------------------------- /LogParserFunction/Models/ChatCompletion/Message.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models.ChatCompletion; 4 | 5 | /// 6 | /// ChatCompletion Request Message 7 | /// 8 | public class Message 9 | { 10 | [JsonProperty("content")] 11 | public dynamic? Content { get; set; } 12 | 13 | [JsonProperty("role")] 14 | public string Role { get; set; } = string.Empty; 15 | 16 | [JsonProperty("tool_calls")] 17 | public dynamic? ToolCalls { get; set; } 18 | 19 | [JsonProperty("contentPart")] 20 | public string? ContentPart { get; set; } 21 | 22 | [JsonProperty("contentPartImage")] 23 | public string? ContentPartImage { get; set; } 24 | } -------------------------------------------------------------------------------- /LogParserFunction/Models/Completion/Choice.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models.Completion; 4 | 5 | /// 6 | /// Completion Choice 7 | /// 8 | public class Choice 9 | { 10 | [JsonProperty("finish_reason")] 11 | public string FinishReason { get; set; } = string.Empty; 12 | 13 | [JsonProperty("index")] 14 | public int Index { get; set; } 15 | 16 | [JsonProperty("content_filter_results")] 17 | public ContentFilterResults ContentFilterResults { get; set; } = new(); 18 | 19 | [JsonProperty("text")] 20 | public string Text { get; set; } = string.Empty; 21 | 22 | [JsonProperty("logprobs")] 23 | public int? Logprobs { get; set; } 24 | } 25 | -------------------------------------------------------------------------------- /LogParserFunction/Models/TempRequestLog.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models; 4 | 5 | /// 6 | /// Temporary Request Log 7 | /// 8 | public class TempRequestLog : TempLog 9 | { 10 | /// 11 | /// Log Type 12 | /// 13 | [JsonProperty("type")] 14 | public override string Type { get; } = "Request"; 15 | 16 | /// 17 | /// Request 18 | /// 19 | [JsonProperty("request")] 20 | public JObject Request { get; set; } = new(); 21 | 22 | /// 23 | /// Request URL 24 | /// 25 | [JsonProperty("requestUrl")] 26 | public string RequestUrl { get; set; } = string.Empty; 27 | } -------------------------------------------------------------------------------- /LoggingWebApi/Models/TempRequestLog.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LoggingWebApi.Models; 4 | 5 | /// 6 | /// Temporary Request Log 7 | /// 8 | public class TempRequestLog : TempLog 9 | { 10 | /// 11 | /// Log Type 12 | /// 13 | [JsonProperty("type")] 14 | public override string Type { get; } = "Request"; 15 | 16 | /// 17 | /// Request 18 | /// 19 | [JsonProperty("request")] 20 | public JObject Request { get; set; } = new(); 21 | 22 | /// 23 | /// Request URL 24 | /// 25 | [JsonProperty("requestUrl")] 26 | public string RequestUrl { get; set; } = string.Empty; 27 | } -------------------------------------------------------------------------------- /LogParserFunction/Models/Response.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models; 4 | 5 | /// 6 | /// ChatCompletion Response Message 7 | /// 8 | public class Response 9 | { 10 | [JsonProperty("id")] 11 | public string Id { get; set; } = string.Empty; 12 | 13 | [JsonProperty("object")] 14 | public string Object { get; set; } = string.Empty; 15 | 16 | [JsonProperty("created")] 17 | public double Created { get; set; } 18 | 19 | [JsonProperty("model")] 20 | public string Model { get; set; } = string.Empty; 21 | 22 | [JsonProperty("usage")] 23 | public Usage Usage { get; set; } = new(); 24 | 25 | [JsonProperty("error")] 26 | public dynamic? Error { get; set; } 27 | } 28 | -------------------------------------------------------------------------------- /.vscode/ltex.hiddenFalsePositives.en-US.txt: -------------------------------------------------------------------------------- 1 | {"rule":"UPPERCASE_SENTENCE_START","sentence":"^\\Qgpt-35-turbo, gpt-35-turbo-instruct, and text-ada-embedding-002.\\E$"} 2 | {"rule":"MORFOLOGIK_RULE_EN_US","sentence":"^\\QFor example, if you want to change the model and deployment names for the AOAI, change the \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q variable in aoai.bicep.\\E$"} 3 | {"rule":"MORFOLOGIK_RULE_EN_US","sentence":"^\\QWhen you change the deployment, you also need to update the \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q variable in apimApis.bicep as they are closely related.\\E$"} 4 | {"rule":"LC_AFTER_PERIOD","sentence":"^\\QWhen you change the deployment, you also need to update the \\E(?:Dummy|Ina|Jimmy-)[0-9]+\\Q variable in apimApis.bicep as they are closely related.\\E$"} 5 | {"rule":"LC_AFTER_PERIOD","sentence":"^\\QSee main.bicep for parameters information.\\E$"} 6 | -------------------------------------------------------------------------------- /LoggingWebApi/Models/TempLog.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LoggingWebApi.Models; 4 | 5 | /// 6 | /// Temporary Log 7 | /// 8 | public class TempLog 9 | { 10 | /// 11 | /// Id for Cosmos DB 12 | /// 13 | [JsonProperty("id")] 14 | public Guid Id { get; } = Guid.NewGuid(); 15 | 16 | /// 17 | /// Log Type 18 | /// 19 | [JsonProperty("type")] 20 | public virtual string Type { get; } = string.Empty; 21 | 22 | /// 23 | /// Request Id 24 | /// 25 | [JsonProperty("requestId")] 26 | public string RequestId { get; set; } = string.Empty; 27 | 28 | /// 29 | /// Headers 30 | /// 31 | [JsonProperty("headers")] 32 | public dynamic? Headers { get; set; } 33 | } -------------------------------------------------------------------------------- /infra/apim/apimApplicationInsights.bicep: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | param apiManagementServiceName string 4 | param applicationInsightsName string 5 | 6 | resource apiManagementService 'Microsoft.ApiManagement/service@2023-03-01-preview' existing = { 7 | name: apiManagementServiceName 8 | } 9 | 10 | resource applicationInsights 'Microsoft.Insights/components@2020-02-02' existing = { 11 | name: applicationInsightsName 12 | } 13 | 14 | resource logger 'Microsoft.ApiManagement/service/loggers@2023-03-01-preview' = { 15 | name: applicationInsightsName 16 | parent: apiManagementService 17 | properties: { 18 | loggerType: 'applicationInsights' 19 | description: 'Application Insights logger with connection string' 20 | credentials: { 21 | connectionString: applicationInsights.properties.ConnectionString 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /LoggingWebApi/LoggingWebApi.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net8.0 5 | enable 6 | enable 7 | true 8 | 3b58c477-3bf1-4dba-bd67-ebbbcb1f5ad4 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /LogParserFunction/Usings.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | global using Azure.AI.ContentSafety; 4 | global using LogParser.Extensions; 5 | global using LogParser.Models; 6 | global using LogParser.Models.ChatCompletion; 7 | global using LogParser.Models.ChatCompletion.Stream; 8 | global using LogParser.Models.ChatCompletion.Vision; 9 | global using LogParser.Models.Completion; 10 | global using LogParser.Models.ContentSafety; 11 | global using LogParser.Models.Embedding; 12 | global using LogParser.Services; 13 | global using Microsoft.ApplicationInsights; 14 | global using Microsoft.Azure.Cosmos; 15 | global using Microsoft.Azure.Cosmos.Fluent; 16 | global using Microsoft.Azure.Functions.Worker; 17 | global using Microsoft.Extensions.DependencyInjection; 18 | global using Microsoft.Extensions.Hosting; 19 | global using Microsoft.Extensions.Logging; 20 | global using Newtonsoft.Json; 21 | global using Newtonsoft.Json.Linq; 22 | global using SixLabors.ImageSharp; 23 | global using System.Text; 24 | global using TiktokenSharp; 25 | -------------------------------------------------------------------------------- /infra/apim/apimBackend.bicep: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | param apiManagementServiceName string 4 | param aoaiName string 5 | param loggingWebApiName string 6 | 7 | resource apiManagementService 'Microsoft.ApiManagement/service@2023-03-01-preview' existing = { 8 | name: apiManagementServiceName 9 | } 10 | 11 | resource aoai 'Microsoft.CognitiveServices/accounts@2023-10-01-preview' existing = { 12 | name: aoaiName 13 | } 14 | 15 | resource webApp 'Microsoft.Web/sites@2022-03-01' existing = { 16 | name: loggingWebApiName 17 | } 18 | 19 | resource backendNamedValue 'Microsoft.ApiManagement/service/namedValues@2023-03-01-preview' = { 20 | name: 'backend-${aoaiName}' 21 | parent: apiManagementService 22 | properties: { 23 | displayName: 'backend-${aoaiName}' 24 | value: '${aoai.properties.endpoint}openai/' 25 | secret: false 26 | } 27 | } 28 | 29 | resource backend 'Microsoft.ApiManagement/service/backends@2023-03-01-preview' = { 30 | name: loggingWebApiName 31 | parent: apiManagementService 32 | properties: { 33 | protocol: 'http' 34 | url: 'https://${webApp.properties.defaultHostName}/openai/' 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /infra/security/keyVault.bicep: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | param location string 4 | param keyVaultName string 5 | param skuName string 6 | param vnetName string 7 | param pepSubnetName string 8 | param keyVaultPrivateEndpointName string 9 | 10 | resource vault 'Microsoft.KeyVault/vaults@2022-07-01' = { 11 | name: keyVaultName 12 | location: location 13 | properties: { 14 | accessPolicies:[] 15 | enableRbacAuthorization: true 16 | enableSoftDelete: true 17 | softDeleteRetentionInDays: 90 18 | tenantId: subscription().tenantId 19 | sku: { 20 | name: skuName 21 | family: 'A' 22 | } 23 | publicNetworkAccess: 'Disabled' 24 | networkAcls: { 25 | defaultAction: 'Deny' 26 | } 27 | } 28 | } 29 | 30 | module privateEndpoint '../network/privateEndpoint.bicep' = { 31 | name: '${keyVaultName}-privateEndpoint' 32 | params: { 33 | groupIds: [ 34 | 'vault' 35 | ] 36 | dnsZoneName: 'privatelink.vaultcore.azure.net' 37 | name: keyVaultPrivateEndpointName 38 | pepSubnetName: pepSubnetName 39 | privateLinkServiceId: vault.id 40 | vnetName: vnetName 41 | location: location 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /infra/apim/apim.bicep: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | param location string 4 | param apiManagementServiceName string 5 | param publisherEmail string 6 | param publisherName string 7 | param sku string 8 | param skuCount int 9 | param publicIpName string 10 | param vnetName string 11 | param apimSubnetName string 12 | 13 | resource apimSubnet 'Microsoft.Network/virtualNetworks/subnets@2022-09-01' existing = { 14 | name: '${vnetName}/${apimSubnetName}' 15 | } 16 | 17 | resource publicIP 'Microsoft.Network/publicIPAddresses@2023-04-01' existing = { 18 | name: publicIpName 19 | } 20 | 21 | resource apiManagementService 'Microsoft.ApiManagement/service@2023-05-01-preview' = { 22 | name: apiManagementServiceName 23 | location: location 24 | sku: { 25 | name: sku 26 | capacity: skuCount 27 | } 28 | properties: { 29 | publisherEmail: publisherEmail 30 | publisherName: publisherName 31 | virtualNetworkType: 'External' 32 | publicIpAddressId: publicIP.id 33 | virtualNetworkConfiguration: { 34 | subnetResourceId: apimSubnet.id 35 | } 36 | } 37 | identity: { 38 | type: 'SystemAssigned' 39 | } 40 | } 41 | 42 | output apimIdentityId string = apiManagementService.identity.principalId 43 | -------------------------------------------------------------------------------- /LogParserFunction/Program.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | CosmosClient cosmosClient = new CosmosClientBuilder( 4 | Environment.GetEnvironmentVariable("CosmosDbConnectionString")) 5 | .WithConnectionModeGateway() 6 | .Build(); 7 | 8 | Container container = cosmosClient.GetContainer( 9 | Environment.GetEnvironmentVariable("CosmosDbDatabaseName"), 10 | Environment.GetEnvironmentVariable("CosmosDbLogContainerName")); 11 | 12 | ContentSafetyClient contentSafetyClient = new( 13 | new Uri(Environment.GetEnvironmentVariable("ContentSafetyUrl")!), 14 | new Azure.AzureKeyCredential(Environment.GetEnvironmentVariable("ContentSafetyKey")!)); 15 | 16 | TikTokenService tikTokenService = new(); 17 | ContentSafetyService contentSafetyService = new(contentSafetyClient); 18 | DataProcessService dataProcessService = new( 19 | tikTokenService, 20 | contentSafetyService, 21 | container); 22 | 23 | IHost host = new HostBuilder() 24 | .ConfigureFunctionsWebApplication() 25 | .ConfigureServices(services => 26 | { 27 | services.AddApplicationInsightsTelemetryWorkerService(); 28 | services.ConfigureFunctionsApplicationInsights(); 29 | services.AddTransient(sp => dataProcessService); 30 | }) 31 | .Build(); 32 | 33 | host.Run(); 34 | -------------------------------------------------------------------------------- /SUPPORT.md: -------------------------------------------------------------------------------- 1 | # TODO: The maintainer of this repo has not yet edited this file 2 | 3 | **REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project? 4 | 5 | - **No CSS support:** Fill out this template with information about how to file issues and get help. 6 | - **Yes CSS support:** Fill out an intake form at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). CSS will work with/help you to determine next steps. 7 | - **Not sure?** Fill out an intake as though the answer were "Yes". CSS will help you decide. 8 | 9 | *Then remove this first heading from this SUPPORT.MD file before publishing your repo.* 10 | 11 | # Support 12 | 13 | ## How to file issues and get help 14 | 15 | This project uses GitHub Issues to track bugs and feature requests. Please search the existing 16 | issues before filing new issues to avoid duplicates. For new issues, file your bug or 17 | feature request as a new Issue. 18 | 19 | For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE 20 | FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER 21 | CHANNEL. WHERE WILL YOU HELP PEOPLE?**. 22 | 23 | ## Microsoft Support Policy 24 | 25 | Support for this **PROJECT or PRODUCT** is limited to the resources listed above. 26 | -------------------------------------------------------------------------------- /infra/network/privateEndpoint.bicep: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | param location string 4 | param name string 5 | param vnetName string 6 | param pepSubnetName string 7 | param privateLinkServiceId string 8 | param groupIds array 9 | param dnsZoneName string 10 | 11 | resource privateEndpointSubnet 'Microsoft.Network/virtualNetworks/subnets@2022-09-01' existing = { 12 | name: '${vnetName}/${pepSubnetName}' 13 | } 14 | 15 | resource privateEndpointDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { 16 | name: dnsZoneName 17 | } 18 | 19 | resource privateEndpoint 'Microsoft.Network/privateEndpoints@2022-09-01' = { 20 | name: name 21 | location: location 22 | properties: { 23 | subnet: { 24 | id: privateEndpointSubnet.id 25 | } 26 | privateLinkServiceConnections: [ 27 | { 28 | name: name 29 | properties: { 30 | privateLinkServiceId: privateLinkServiceId 31 | groupIds: groupIds 32 | } 33 | } 34 | ] 35 | } 36 | } 37 | 38 | resource privateEndpointDnsGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2022-09-01' = { 39 | parent: privateEndpoint 40 | name: 'privateDnsZoneGroup' 41 | properties: { 42 | privateDnsZoneConfigs: [ 43 | { 44 | name: 'default' 45 | properties: { 46 | privateDnsZoneId: privateEndpointDnsZone.id 47 | } 48 | } 49 | ] 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /LoggingWebApi/Program.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | WebApplicationBuilder builder = WebApplication.CreateBuilder(args); 4 | 5 | CosmosClient cosmosClient = new CosmosClientBuilder( 6 | builder.Configuration["CosmosDbConnectionString"]) 7 | .WithConnectionModeGateway() 8 | .Build(); 9 | 10 | Container logContainer = cosmosClient.GetContainer( 11 | builder.Configuration["CosmosDbDatabaseName"], 12 | builder.Configuration["CosmosDbLogContainerName"]); 13 | 14 | Container triggerContainer = cosmosClient.GetContainer( 15 | builder.Configuration["CosmosDbDatabaseName"], 16 | builder.Configuration["CosmosDbTriggerContainerName"]); 17 | 18 | Containers containers = new() 19 | { 20 | { "logContainer", logContainer }, 21 | { "triggerContainer", triggerContainer }, 22 | }; 23 | 24 | // Add services to the container. 25 | 26 | builder.Services.AddControllers().AddNewtonsoftJson(); 27 | builder.Services.AddEndpointsApiExplorer(); 28 | builder.Services.AddSwaggerGen(); 29 | builder.Services.AddHttpClient(); 30 | builder.Services.AddHttpContextAccessor(); 31 | builder.Services.AddSingleton(containers); 32 | WebApplication app = builder.Build(); 33 | 34 | // Configure the HTTP request pipeline. 35 | if (app.Environment.IsDevelopment()) 36 | { 37 | app.UseSwagger(); 38 | app.UseSwaggerUI(); 39 | } 40 | 41 | app.UseHttpsRedirection(); 42 | 43 | app.UseAuthorization(); 44 | 45 | app.MapControllers(); 46 | 47 | app.Run(); 48 | -------------------------------------------------------------------------------- /infra/ai/aoai.bicep: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | param location string 4 | param aoaiName string 5 | param vnetName string 6 | param pepSubnetName string 7 | param privateEndpointName string 8 | param deployments array 9 | 10 | resource aoai 'Microsoft.CognitiveServices/accounts@2023-10-01-preview' = { 11 | name: aoaiName 12 | location: location 13 | sku: { 14 | name: 'S0' 15 | } 16 | kind: 'OpenAI' 17 | properties: { 18 | customSubDomainName: toLower(aoaiName) 19 | publicNetworkAccess: 'Disabled' 20 | } 21 | } 22 | 23 | @batchSize(1) 24 | resource models 'Microsoft.CognitiveServices/accounts/deployments@2023-10-01-preview' = [for deployment in deployments: { 25 | name: deployment.deploymentName 26 | parent: aoai 27 | sku: { 28 | name: deployment.skuName 29 | capacity: deployment.capacity 30 | } 31 | properties: { 32 | model: { 33 | format: 'OpenAI' 34 | name: deployment.modelName 35 | version: deployment.version 36 | } 37 | raiPolicyName: 'Microsoft.Default' 38 | versionUpgradeOption: 'OnceNewDefaultVersionAvailable' 39 | } 40 | }] 41 | 42 | module privateEndpoint '../network/privateEndpoint.bicep' = { 43 | name: '${aoaiName}-privateEndpoint' 44 | params: { 45 | groupIds: [ 46 | 'account' 47 | ] 48 | dnsZoneName: 'privatelink.openai.azure.com' 49 | name: privateEndpointName 50 | pepSubnetName: pepSubnetName 51 | privateLinkServiceId: aoai.id 52 | vnetName: vnetName 53 | location: location 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /infra/ai/contentSafety.bicep: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | param location string 4 | param contentSafetyAccountName string 5 | param keyVaultName string 6 | param vnetName string 7 | param pepSubnetName string 8 | param privateEndpointName string 9 | 10 | resource vault 'Microsoft.KeyVault/vaults@2022-07-01' existing = { 11 | name: keyVaultName 12 | } 13 | 14 | resource contentSafety 'Microsoft.CognitiveServices/accounts@2023-05-01' = { 15 | name: contentSafetyAccountName 16 | location: location 17 | sku: { 18 | name: 'S0' 19 | } 20 | kind: 'ContentSafety' 21 | identity: { 22 | type: 'SystemAssigned' 23 | } 24 | properties: { 25 | customSubDomainName: contentSafetyAccountName 26 | publicNetworkAccess: 'Disabled' 27 | } 28 | } 29 | 30 | module privateEndpoint '../network/privateEndpoint.bicep' = { 31 | name: '${contentSafetyAccountName}-privateEndpoint' 32 | params: { 33 | groupIds: [ 34 | 'account' 35 | ] 36 | dnsZoneName: 'privatelink.cognitiveservices.azure.com' 37 | name: privateEndpointName 38 | pepSubnetName: pepSubnetName 39 | privateLinkServiceId: contentSafety.id 40 | vnetName: vnetName 41 | location: location 42 | } 43 | } 44 | 45 | resource contentSafetyKey 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = { 46 | name: contentSafetyAccountName 47 | parent: vault 48 | properties: { 49 | attributes: { 50 | enabled: true 51 | } 52 | contentType: 'string' 53 | value: contentSafety.listKeys().key1 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /LogParserFunction/Models/Completion/CompletionRequest.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models.Completion; 4 | 5 | /// 6 | /// ChatCompletion Request Message 7 | /// 8 | public class CompletionRequest : Request 9 | { 10 | [JsonProperty("prompt")] 11 | public string? Prompt { get; set; } = "<\\|endoftext\\|>"; 12 | 13 | [JsonProperty("logprobs")] 14 | public int? LogProbs { get; set; } 15 | 16 | [JsonProperty("best_of")] 17 | public int BestOf { get; set; } = 1; 18 | 19 | [JsonProperty("max_tokens")] 20 | public int? MaxTokens { get; set; } 21 | 22 | [JsonProperty("temperature")] 23 | public double? Temperature { get; set; } 24 | 25 | [JsonProperty("top_p")] 26 | public double? TopP { get; set; } 27 | 28 | [JsonProperty("logit_bias")] 29 | public string? LogitBias { get; set; } 30 | 31 | [JsonProperty("user")] 32 | public string? User { get; set; } 33 | 34 | [JsonProperty("n")] 35 | public int? N { get; set; } 36 | 37 | [JsonProperty("stream")] 38 | public bool? Stream { get; set; } 39 | 40 | [JsonProperty("suffix")] 41 | public string? Suffix { get; set; } 42 | 43 | [JsonProperty("echo")] 44 | public bool? Echo { get; set; } 45 | 46 | [JsonProperty("stop")] 47 | public string? Stop { get; set; } 48 | 49 | [JsonProperty("presence_penalty")] 50 | public double? PresencePenalty { get; set; } 51 | 52 | [JsonProperty("frequency_penalty")] 53 | public double? FrequencyPenalty { get; set; } 54 | } -------------------------------------------------------------------------------- /LoggingWebApi/README.md: -------------------------------------------------------------------------------- 1 | # Logging Web API 2 | 3 | C# sample Web API that works as a proxy. 4 | 5 | ![Architecture](/assets/aoai_apim.svg) 6 | 7 | 1. The Web API receive the request from AOAI. 8 | 1. Forward the request to AOAI specified in headers. 9 | 1. Create request log by setting ``request id`` and store in a cache. 10 | 1. Once AOAI respond, it returns the results back to APIM. 11 | - If it's streaming mode, then return SSE. 12 | - If it's non-streaming mode, then return HttpResponseMessage 13 | - If it's not succeeded, return the error as it is. 14 | 1. While replying the response, it also stores the response log by settings ``request id`` into the cache. 15 | 1. After it returns the response back to APIM, then send the logs to Cosmos DB. 16 | 1. Then send the ``request id`` to the Cosmos DB Container for trigger. 17 | 18 | ## How to run in local environment 19 | 20 | If you want to run and debug the application, follow the steps below. 21 | 22 | 1. Rename ``__appsettings.json`` into ``appsettings,json`` 23 | 1. Fill all the variables. 24 | 1. Run by using any IDE or ``dotnet run`` 25 | 26 | Please note that when you provision the solution by using [bicep files](/infra), Azure resources blocks external access. If the [Log Parser Function](/LogParserFunction/) is up and running in the cloud, it's triggered by the Cosmos DB change feed, so that you may end up have duplicate logs in the Application Insights when you run the function locally at the same time. 27 | 28 | If you want to test from APIM, then you can change the backend address to any proxy address that routes the request to your local server. -------------------------------------------------------------------------------- /policies/inbound-logging.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | @(context.Timestamp.ToString()) 4 | 5 | 6 | @(context.Subscription.Id.ToString()) 7 | 8 | 9 | @(context.Subscription.Name) 10 | 11 | 12 | @(context.Operation.Id.ToString()) 13 | 14 | 15 | @(context.Deployment.ServiceName) 16 | 17 | 18 | @(context.RequestId.ToString()) 19 | 20 | 21 | @(context.Request.IpAddress) 22 | 23 | 24 | @(context.Operation.Name) 25 | 26 | 27 | @(context.Deployment.Region) 28 | 29 | 30 | @(context.Api.Name) 31 | 32 | 33 | @(context.Api.Revision) 34 | 35 | 36 | @(context.Operation.Method) 37 | 38 | -------------------------------------------------------------------------------- /apim_log.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 17 4 | VisualStudioVersion = 17.5.002.0 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LoggingWebApi", "LoggingWebApi\LoggingWebApi.csproj", "{33B0EC9A-466F-44B3-823B-0A9B09ED46C8}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LogParserFunction", "LogParserFunction\LogParserFunction.csproj", "{9E53ADC6-469B-49BB-806A-03574493894A}" 9 | EndProject 10 | Global 11 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 12 | Debug|Any CPU = Debug|Any CPU 13 | Release|Any CPU = Release|Any CPU 14 | EndGlobalSection 15 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 16 | {33B0EC9A-466F-44B3-823B-0A9B09ED46C8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 17 | {33B0EC9A-466F-44B3-823B-0A9B09ED46C8}.Debug|Any CPU.Build.0 = Debug|Any CPU 18 | {33B0EC9A-466F-44B3-823B-0A9B09ED46C8}.Release|Any CPU.ActiveCfg = Release|Any CPU 19 | {33B0EC9A-466F-44B3-823B-0A9B09ED46C8}.Release|Any CPU.Build.0 = Release|Any CPU 20 | {9E53ADC6-469B-49BB-806A-03574493894A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 21 | {9E53ADC6-469B-49BB-806A-03574493894A}.Debug|Any CPU.Build.0 = Debug|Any CPU 22 | {9E53ADC6-469B-49BB-806A-03574493894A}.Release|Any CPU.ActiveCfg = Release|Any CPU 23 | {9E53ADC6-469B-49BB-806A-03574493894A}.Release|Any CPU.Build.0 = Release|Any CPU 24 | EndGlobalSection 25 | GlobalSection(SolutionProperties) = preSolution 26 | HideSolutionNode = FALSE 27 | EndGlobalSection 28 | GlobalSection(ExtensibilityGlobals) = postSolution 29 | SolutionGuid = {1C6498F5-8C18-4F18-BA74-56476C7A8A4A} 30 | EndGlobalSection 31 | EndGlobal 32 | -------------------------------------------------------------------------------- /queries/query.kql: -------------------------------------------------------------------------------- 1 | traces 2 | | where message contains "Azure Open AI" 3 | | order by timestamp desc 4 | | extend responseBody = todynamic(tostring(message)) 5 | | extend 6 | requestId = responseBody.requestId, 7 | subscriptionId = responseBody.subscriptionId, 8 | subscriptionName = responseBody.subscriptionName, 9 | operationId = responseBody.operationId, 10 | apiName = responseBody.apiName, 11 | apiRevision = responseBody.apiRevision, 12 | method = responseBody.method, 13 | headers = responseBody.headers, 14 | elapsed = responseBody.elapsed, 15 | serviceName = responseBody.serviceName, 16 | requestIp = responseBody.requestIp, 17 | url = responseBody.url, 18 | operationName = responseBody.operationName, 19 | region = responseBody.region, 20 | statusCode = responseBody.statusCode, 21 | statusReason = responseBody.statusReason, 22 | request = responseBody.request, 23 | response = responseBody.response, 24 | promptTokens = responseBody.response.usage.prompt_tokens, 25 | completionTokens = responseBody.response.usage.completion_tokens, 26 | totalTokens = responseBody.response.usage.total_tokens, 27 | model = responseBody.response.model, 28 | stream = responseBody.request.stream 29 | |project 30 | timestamp, 31 | requestId, 32 | subscriptionId, 33 | subscriptionName, 34 | operationId, 35 | apiName, 36 | apiRevision, 37 | method, 38 | headers, 39 | elapsed, 40 | serviceName, 41 | requestIp, 42 | url, 43 | operationName, 44 | region, 45 | statusCode, 46 | statusReason, 47 | request, 48 | response, 49 | promptTokens, 50 | completionTokens, 51 | totalTokens, 52 | model, 53 | stream 54 | -------------------------------------------------------------------------------- /LogParserFunction/Models/ChatCompletion/ChatCompletionRequest.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models.ChatCompletion; 4 | 5 | /// 6 | /// ChatCompletion Request Message 7 | /// 8 | public class ChatCompletionRequest : Request 9 | { 10 | [JsonProperty("messages")] 11 | public List Messages { get; set; } = new(); 12 | 13 | [JsonProperty("max_tokens")] 14 | public int? MaxTokens { get; set; } 15 | 16 | [JsonProperty("temperature")] 17 | public double? Temperature { get; set; } 18 | 19 | [JsonProperty("top_p")] 20 | public double? TopP { get; set; } 21 | 22 | [JsonProperty("logit_bias")] 23 | public string? LogitBias { get; set; } 24 | 25 | [JsonProperty("user")] 26 | public string? User { get; set; } 27 | 28 | [JsonProperty("n")] 29 | public int? N { get; set; } 30 | 31 | [JsonProperty("stream")] 32 | public bool? Stream { get; set; } 33 | 34 | [JsonProperty("suffix")] 35 | public string? Suffix { get; set; } 36 | 37 | [JsonProperty("echo")] 38 | public bool? Echo { get; set; } 39 | 40 | [JsonProperty("stop")] 41 | public string? Stop { get; set; } 42 | 43 | [JsonProperty("presence_penalty")] 44 | public double? PresencePenalty { get; set; } 45 | 46 | [JsonProperty("frequency_penalty")] 47 | public double? FrequencyPenalty { get; set; } 48 | 49 | [JsonProperty("function_call")] 50 | public dynamic? FunctionCall { get; set; } 51 | 52 | [JsonProperty("functions")] 53 | public dynamic? Functions { get; set; } 54 | 55 | [JsonProperty("tools")] 56 | public dynamic? Tools { get; set; } 57 | 58 | [JsonProperty("tool_choice")] 59 | public dynamic? ToolChoice { get; set; } 60 | 61 | [JsonProperty("enhancements")] 62 | public dynamic? Enhancements { get; set; } 63 | 64 | [JsonProperty("dataSources")] 65 | public dynamic? DataSources { get; set; } 66 | } -------------------------------------------------------------------------------- /infra/security/roles.bicep: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | param keyVaultName string 4 | param aoaiName string 5 | param loggingWebApiIdentityId string 6 | param logParserFunctionIdentityId string 7 | 8 | resource vault 'Microsoft.KeyVault/vaults@2022-07-01' existing = { 9 | name: keyVaultName 10 | } 11 | 12 | resource aoai 'Microsoft.CognitiveServices/accounts@2023-10-01-preview' existing = { 13 | name: aoaiName 14 | } 15 | 16 | //Cognitive Services OpenAI User 17 | resource openAIRoleDefinition 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { 18 | name:'5e0bd9bd-7b93-4f28-af87-19fc36ad61bd' 19 | scope: aoai 20 | } 21 | 22 | resource vaultRoleDefinition 'Microsoft.Authorization/roleDefinitions@2022-04-01' existing = { 23 | name:'4633458b-17de-408a-b874-0445c86b69e6' 24 | scope: vault 25 | } 26 | 27 | resource webAppopenAIRoleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { 28 | scope: aoai 29 | name: guid(aoai.id, loggingWebApiIdentityId, openAIRoleDefinition.id) 30 | properties: { 31 | roleDefinitionId: openAIRoleDefinition.id 32 | principalId: loggingWebApiIdentityId 33 | principalType: 'ServicePrincipal' 34 | } 35 | } 36 | 37 | 38 | resource webAppVaultRoleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { 39 | scope: vault 40 | name: guid(vault.id, loggingWebApiIdentityId, vaultRoleDefinition.id) 41 | properties: { 42 | roleDefinitionId: vaultRoleDefinition.id 43 | principalId: loggingWebApiIdentityId 44 | principalType: 'ServicePrincipal' 45 | } 46 | } 47 | 48 | 49 | resource functionAppVaultRoleAssignment 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = { 50 | scope: vault 51 | name: guid(vault.id, logParserFunctionIdentityId, vaultRoleDefinition.id) 52 | properties: { 53 | roleDefinitionId: vaultRoleDefinition.id 54 | principalId: logParserFunctionIdentityId 55 | principalType: 'ServicePrincipal' 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /LogParserFunction/LogParserFunction.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | net8.0 4 | v4 5 | Exe 6 | enable 7 | enable 8 | cdca9380-4770-40f4-b735-6dab5b94319d 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | PreserveNewest 26 | 27 | 28 | PreserveNewest 29 | Never 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /LogParserFunction/Models/AOAILog.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Models; 4 | 5 | /// 6 | /// Final log format to logging to Application Insights 7 | /// 8 | public class AOAILog 9 | { 10 | [JsonProperty("apiName")] 11 | public string ApiName { get; set; } = string.Empty; 12 | 13 | [JsonProperty("apiRevision")] 14 | public string ApiRevision { get; set; } = string.Empty; 15 | 16 | [JsonProperty("elapsed")] 17 | public double Elapsed { get; set; } 18 | 19 | [JsonProperty("headers")] 20 | public dynamic? Headers { get; set; } 21 | 22 | [JsonProperty("method")] 23 | public string Method { get; set; } = string.Empty; 24 | 25 | [JsonProperty("operationId")] 26 | public string OperationId { get; set; } = string.Empty; 27 | 28 | [JsonProperty("operationName")] 29 | public string OperationName { get; set; } = string.Empty; 30 | 31 | [JsonProperty("region")] 32 | public string Region { get; set; } = string.Empty; 33 | 34 | [JsonProperty("request")] 35 | public Request Request { get; set; } = new(); 36 | 37 | [JsonProperty("requestId")] 38 | public string RequestId { get; set; } = string.Empty; 39 | 40 | [JsonProperty("requestIp")] 41 | public string RequestIp { get; set; } = string.Empty; 42 | 43 | [JsonProperty("response")] 44 | public Response Response { get; set; } = new(); 45 | 46 | [JsonProperty("serviceName")] 47 | public string ServiceName { get; set; } = string.Empty; 48 | 49 | [JsonProperty("statusCode")] 50 | public int StatusCode { get; set; } 51 | 52 | [JsonProperty("statusReason")] 53 | public string StatusReason { get; set; } = string.Empty; 54 | 55 | [JsonProperty("subscriptionId")] 56 | public string SubscriptionId { get; set; } = string.Empty; 57 | 58 | [JsonProperty("subscriptionName")] 59 | public string SubscriptionName { get; set; } = string.Empty; 60 | 61 | [JsonProperty("timestamp")] 62 | public string Timestamp { get; set; } = string.Empty; 63 | 64 | [JsonProperty("url")] 65 | public string Url { get; set; } = string.Empty; 66 | } -------------------------------------------------------------------------------- /LogParserFunction/LogParser.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParserFunction; 4 | 5 | public class LogParser( 6 | DataProcessService dataProcessService, 7 | TelemetryClient telemetryClient, 8 | ILogger logger) 9 | { 10 | private readonly DataProcessService dataProcessService = dataProcessService; 11 | private readonly TelemetryClient telemetryClient = telemetryClient; 12 | private readonly ILogger logger = logger; 13 | 14 | /// 15 | /// Main function to parse the log. 16 | /// 17 | /// 18 | /// 19 | [Function("LogParser")] 20 | public async Task Run([CosmosDBTrigger( 21 | databaseName: "%CosmosDbDatabaseName%", 22 | containerName: "%CosmosDbTriggerContainerName%", 23 | Connection = "CosmosDbConnectionString", 24 | LeaseContainerName = "leases", 25 | CreateLeaseContainerIfNotExists = true)]IReadOnlyList logs) 26 | { 27 | List exceptions = new(); 28 | foreach (TempLog log in logs) 29 | { 30 | try 31 | { 32 | logger.LogInformation($"Parse {log.RequestId}"); 33 | AOAILog? aoaiLog = await dataProcessService.ProcessLogsAsync(log.RequestId); 34 | if (aoaiLog is not null) 35 | { 36 | telemetryClient.TrackTrace( 37 | JsonConvert.SerializeObject( 38 | aoaiLog, 39 | new JsonSerializerSettings() { NullValueHandling = NullValueHandling.Ignore })); 40 | } 41 | } 42 | catch (Exception ex) 43 | { 44 | exceptions.Add(ex); 45 | } 46 | } 47 | 48 | if (exceptions.Count is 1) 49 | { 50 | throw exceptions.First(); 51 | } 52 | else if (exceptions.Count > 1) 53 | { 54 | throw new AggregateException(exceptions); 55 | } 56 | else 57 | { 58 | return; 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /infra/README.md: -------------------------------------------------------------------------------- 1 | # Infrastructure as Code (bicep) 2 | 3 | We can use the bicep files to provision a sample environment. [Bicep](https://learn.microsoft.com/azure/azure-resource-manager/bicep/) is an Infrastructure as Code (IaC) solution for Azure that let us write YAML to define the Azure resources. 4 | 5 | ## Run the bicep 6 | 7 | 1. Create a resource group in your subscription 8 | 1. Run the ``az`` command. 9 | ```shell 10 | az deployment group create -g -f .\main.bicep --parameters projectName= 11 | ``` 12 | 1. Wait until all the resources are deployed. 13 | 14 | ## Resources 15 | 16 | The bicep files deploys following resources. 17 | 18 | - Network: Virtual Network, Subnet, DNS, Public IP address and private endpoint to secure Azure Resources. 19 | - Application Insights: Logging metrics from APIM. 20 | - Key Vault: Stores the keys and connection strings. 21 | - Cosmos Db: The temporary log store. 22 | - AOAI and Deployments: The AOAI account and three deployments. gpt-35-turbo, gpt-35-turbo-instruct, and text-ada-embedding-002. 23 | - APIM: The APIM instance 24 | - Named Value: Linked to the Key Vault for AOAI key. 25 | - Backend: Stores the AOAI endpoint information with ``api-key`` header. 26 | - Policy Fragments: The inbound and outbound logging policy. 27 | - API and Operations: The API and the operations of Azure Open AI deployments. 28 | - Web App and Function App: The logging related components. 29 | 30 | ## How to test 31 | 32 | Once the deployment has been completed, we should be able to consume the endpoint. 33 | 34 | 1. Get the subscription key that has access to the deployed API. 35 | 1. Use any HTTP client tool to call the Chat Completion, Completion or Embedding endpoint. 36 | 37 | The header and body formats are exactly same as AOAI endpoints. Only the differences are the endpoint address and the key, which is APIM subscription key. 38 | 39 | As other resources are protected from external access, you cannot access to AOAI endpoint directly. 40 | 41 | # Use existing resources 42 | 43 | You can pass parameters to the ``main.bicep`` if you want to use existing Key Vault, Application Insights, etc. See [main.bicep](./main.bicep) for parameters information. 44 | 45 | # How to customize 46 | 47 | Each bicep contains each resource definitions. For example, if you want to change the model and deployment names for the AOAI, change the ``deployments`` variable in [aoai.bicep](./aoai.bicep). 48 | 49 | When you change the deployment, you also need to update the ``deployments`` variable in [apimApis.bicep](./apimApis.bicep) as they are closely related. -------------------------------------------------------------------------------- /LogParserFunction/Services/ContentSafetyService.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Services; 4 | 5 | /// 6 | /// Content Safety Service 7 | /// 8 | /// ContentSafetyClient 9 | public class ContentSafetyService(ContentSafetyClient contentSafetyClient) 10 | { 11 | private readonly ContentSafetyClient contentSafetyClient = contentSafetyClient; 12 | 13 | /// 14 | /// Get Content Filter Results 15 | /// 16 | /// input text 17 | /// ContentFilterResults 18 | public async Task GetContentFilterResultsAsync(string text) 19 | { 20 | Azure.Response response = 21 | await contentSafetyClient.AnalyzeTextAsync(new AnalyzeTextOptions(text)); 22 | 23 | if (response is null) 24 | { 25 | return new(); 26 | } 27 | 28 | return new() 29 | { 30 | Hate = new() 31 | { 32 | Severity = GetSeverity(response.Value.CategoriesAnalysis.First(x => x.Category == TextCategory.Hate).Severity), 33 | Filtered = false 34 | }, 35 | SelfHarm = new() 36 | { 37 | Severity = GetSeverity(response.Value.CategoriesAnalysis.First(x => x.Category == TextCategory.SelfHarm).Severity), 38 | Filtered = false 39 | }, 40 | Sexual = new() 41 | { 42 | Severity = GetSeverity(response.Value.CategoriesAnalysis.First(x => x.Category == TextCategory.Sexual).Severity), 43 | Filtered = false 44 | }, 45 | Violence = new() 46 | { 47 | Severity = GetSeverity(response.Value.CategoriesAnalysis.First(x => x.Category == TextCategory.Violence).Severity), 48 | Filtered = false 49 | }, 50 | }; 51 | } 52 | 53 | /// 54 | /// Convert severity to match the severiy of AOAI content safety. 55 | /// 56 | /// 57 | /// 58 | private string GetSeverity(int? severityLevel) 59 | { 60 | switch (severityLevel) 61 | { 62 | case 0: 63 | case 1: 64 | return "safe"; 65 | case 2: 66 | case 3: 67 | return "low"; 68 | case 4: 69 | case 5: 70 | return "medium"; 71 | default: 72 | return "high"; 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /LogParserFunction/README.md: -------------------------------------------------------------------------------- 1 | # Azure API Management(APIM) Log parser for Azure Open AI(AOAI) 2 | 3 | As described in [the challenges](../README.md#challenges-of-azure-open-ai-in-production), one of the issue when monitoring the AOAI accounts is the inconsistency of response format depending on the models, features to use, and ``stream`` or ``non-stream`` mode for Completion endpoints. 4 | 5 | For example, the stream mode responses do not contain, not only token usage information, but also it's difficult to read the result as each response contains just a snippet of the generated answer. 6 | 7 | This C# sample application demonstrate how to parse these logs to make them identical so that it is easier to create reports and dashboards. 8 | 9 | # Architecture 10 | 11 | ![Architecture](../assets/aoai_apim.svg) 12 | 13 | 1. The log parser is triggered from the Cosmos DB Change Feed that has ``request id``. 14 | 1. It retrieves all the logs for the ``request id`` from Cosmos DB. 15 | 1. Depending on the response type, it converts the logs into uniform format. 16 | 1. Then it sends the converted logs to Application Insights. 17 | 18 | ## Features 19 | 20 | The log parser does: 21 | 22 | - Combine a request and the corresponding response into single log. 23 | - Combine multiple stream responses into a single response by joining the content, then calculate token usage and content safeness. 24 | 25 | ### Limitation 26 | 27 | We are not creating the function calling response when using streaming mode at the moment. 28 | 29 | # How to run the log parser 30 | 31 | The log parser is a C# Azure Functions application. We can run it locally by using any IDE that supports Azure Functions, such as Visual Studio and Visual Studio Code. 32 | 33 | ## local.settings.json 34 | 35 | Rename the ``__local.settings.json`` into ``local.settings.json`` and fill the necessary information. 36 | 37 | - __CosmosDbConnectionString__: The connection string to the Cosmos DB. 38 | - __CosmosDbDatabaseName__: The database name of the cosmos db. 39 | - __CosmosDbLogContainerName__ and __CosmosDbTriggerContainerName__: The container names for temporary log store and for trigger the function app. 40 | - __ContentSafetyUrl__ and __ContentSafetyKey__: The content safety service endpoint and key. 41 | - __ApplicationInsightsConnectionString__: The application insights connection string to store the log trace. 42 | 43 | ## Run locally 44 | 45 | You can run by dotnet runtime or use any IDE to run the program. 46 | 47 | # Stream Log Parser 48 | 49 | GPT models returns responses token by token when we use stream mode, so that we can return the results little by little to consumer. See [How to stream completion](https://cookbook.openai.com/examples/how_to_stream_completions) for more detail. 50 | 51 | As described in the article, one of [the downsides](https://cookbook.openai.com/examples/how_to_stream_completions#downsides) is that it doesn't contain ``usage`` field that tells us how many tokens were consumed. 52 | 53 | To solve this challenge, the log parser accumulates all the responses and concatenates the content, then calculates the token usage by using [TiktokenSharp](https://github.com/aiqinxuancai/TiktokenSharp). 54 | 55 | Another challenge is that the response content safety check may not be accurate as it returns token by token. The log parser sends the concatenated result to [Azure Content Safety](https://learn.microsoft.com/azure/ai-services/content-safety/overview) service to analyze it. -------------------------------------------------------------------------------- /infra/apim/apimApis.bicep: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | param apiManagementServiceName string 4 | param aoaiName string 5 | param loggingWebApiName string 6 | param applicationInsightsName string 7 | param deployments array 8 | param tokenLimitTPM int 9 | 10 | resource apiManagementService 'Microsoft.ApiManagement/service@2023-03-01-preview' existing = { 11 | name: apiManagementServiceName 12 | } 13 | 14 | resource aoai 'Microsoft.CognitiveServices/accounts@2023-10-01-preview' existing = { 15 | name: aoaiName 16 | } 17 | 18 | resource api 'Microsoft.ApiManagement/service/apis@2023-03-01-preview' = { 19 | name: 'AzureOpenAI' 20 | parent: apiManagementService 21 | properties: { 22 | apiType: 'http' 23 | description: 'Azure Open AI API' 24 | displayName: 'Azure Open AI' 25 | path: 'openai' 26 | protocols: [ 27 | 'https' 28 | ] 29 | serviceUrl: '${aoai.properties.endpoint}openai' 30 | subscriptionKeyParameterNames: { 31 | header: 'api-key' 32 | query: 'api-key' 33 | } 34 | subscriptionRequired: true 35 | type: 'http' 36 | } 37 | } 38 | 39 | var originalApiPolicy = loadTextContent('../../policies/api-policy.xml') 40 | var apiPolicy = replace(originalApiPolicy, '{{tokenLimitTPM}}', string(tokenLimitTPM)) 41 | 42 | resource topLevelPolicy 'Microsoft.ApiManagement/service/apis/policies@2023-03-01-preview' = { 43 | name: 'policy' 44 | parent: api 45 | properties: { 46 | format: 'rawxml' 47 | value: apiPolicy 48 | } 49 | } 50 | 51 | resource diag 'Microsoft.ApiManagement/service/apis/diagnostics@2023-03-01-preview' = { 52 | name: 'applicationinsights' 53 | parent: api 54 | properties: { 55 | alwaysLog: 'allErrors' 56 | metrics: true 57 | operationNameFormat: 'Name' 58 | verbosity: 'information' 59 | logClientIp: true 60 | httpCorrelationProtocol: 'W3C' 61 | loggerId: resourceId('Microsoft.ApiManagement/service/loggers', applicationInsightsName, applicationInsightsName) 62 | } 63 | } 64 | 65 | var originalOperationPolicy = loadTextContent('../../policies/operation-policy.xml') 66 | var operationPolicy1 = replace(originalOperationPolicy, '{{api-key}}', '{{${aoaiName}}}') 67 | var operationPolicy2 = replace(operationPolicy1, '{{backend-url}}', '{{backend-${aoaiName}}}') 68 | var operationPolicy = replace(operationPolicy2, '{backend-id}', loggingWebApiName) 69 | 70 | 71 | resource operations 'Microsoft.ApiManagement/service/apis/operations@2023-03-01-preview' = [for i in range(0, length(deployments)): { 72 | name: deployments[i].name 73 | parent: api 74 | properties: { 75 | description: deployments[i].description 76 | displayName: deployments[i].displayName 77 | method: deployments[i].method 78 | templateParameters: [ 79 | { 80 | name: 'deployment-id' 81 | required: true 82 | type: 'string' 83 | } 84 | { 85 | name: 'api-version' 86 | required: true 87 | type: 'string' 88 | } 89 | ] 90 | urlTemplate: deployments[i].urlTemplate 91 | } 92 | }] 93 | 94 | resource operationLevelpolicy 'Microsoft.ApiManagement/service/apis/operations/policies@2023-03-01-preview' = [for i in range(0, length(deployments)): { 95 | name: 'policy' 96 | parent: operations[i] 97 | properties: { 98 | format: 'rawxml' 99 | value: operationPolicy 100 | } 101 | }] 102 | -------------------------------------------------------------------------------- /infra/network/vnet.bicep: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | param location string 4 | param vnetName string 5 | param pepSubnetName string 6 | param apimSubnetName string 7 | param webAppSubnetName string 8 | param apimNsgName string 9 | param privateDnsZoneNames array 10 | 11 | resource apimNsg 'Microsoft.Network/networkSecurityGroups@2020-07-01' = { 12 | name: apimNsgName 13 | location: location 14 | properties: { 15 | securityRules: [ 16 | { 17 | name: 'AllowClientToGateway' 18 | properties: { 19 | protocol: 'Tcp' 20 | sourcePortRange: '*' 21 | destinationPortRange: '443' 22 | sourceAddressPrefix: 'Internet' 23 | destinationAddressPrefix: 'VirtualNetwork' 24 | access: 'Allow' 25 | priority: 2721 26 | direction: 'Inbound' 27 | } 28 | } 29 | { 30 | name: 'AllowAPIMPortal' 31 | properties: { 32 | protocol: 'Tcp' 33 | sourcePortRange: '*' 34 | destinationPortRange: '3443' 35 | sourceAddressPrefix: 'ApiManagement' 36 | destinationAddressPrefix: 'VirtualNetwork' 37 | access: 'Allow' 38 | priority: 2731 39 | direction: 'Inbound' 40 | } 41 | } 42 | { 43 | name: 'AllowAPIMLoadBalancer' 44 | properties: { 45 | protocol: '*' 46 | sourcePortRange: '*' 47 | destinationPortRange: '6390' 48 | sourceAddressPrefix: 'AzureLoadBalancer' 49 | destinationAddressPrefix: 'VirtualNetwork' 50 | access: 'Allow' 51 | priority: 2741 52 | direction: 'Inbound' 53 | } 54 | } 55 | ] 56 | } 57 | } 58 | 59 | resource vnet 'Microsoft.Network/virtualNetworks@2023-04-01' = { 60 | name: vnetName 61 | location: location 62 | properties: { 63 | addressSpace: { 64 | addressPrefixes: [ '10.0.0.0/16' ] 65 | } 66 | subnets: [ 67 | { 68 | name: apimSubnetName 69 | properties: { 70 | addressPrefix: '10.0.0.0/24' 71 | networkSecurityGroup: { 72 | id: apimNsg.id 73 | } 74 | delegations: [ 75 | { 76 | name: 'Microsoft.Web/serverFarms' 77 | properties: { 78 | serviceName: 'Microsoft.Web/serverFarms' 79 | } 80 | } 81 | ] 82 | } 83 | } 84 | { 85 | name: pepSubnetName 86 | properties: { 87 | addressPrefix: '10.0.1.0/24' 88 | } 89 | } 90 | { 91 | name: webAppSubnetName 92 | properties: { 93 | addressPrefix: '10.0.2.0/24' 94 | delegations: [ 95 | { 96 | name: 'Microsoft.Web/serverFarms' 97 | properties: { 98 | serviceName: 'Microsoft.Web/serverFarms' 99 | } 100 | } 101 | ] 102 | } 103 | } 104 | ] 105 | } 106 | } 107 | 108 | resource privateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = [for privateDnsZoneName in privateDnsZoneNames: { 109 | name: '${privateDnsZoneName}/privateDnsZoneLink' 110 | location: 'global' 111 | properties: { 112 | virtualNetwork: { 113 | id: vnet.id 114 | } 115 | registrationEnabled: false 116 | } 117 | }] 118 | -------------------------------------------------------------------------------- /infra/monitoring/applicationInsights.bicep: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | param location string 4 | param applicationInsightsName string 5 | param workspaceName string 6 | param keyVaultName string 7 | param vnetName string 8 | param pepSubnetName string 9 | param privateEndpointName string 10 | 11 | resource vault 'Microsoft.KeyVault/vaults@2022-07-01' existing = { 12 | name: keyVaultName 13 | } 14 | 15 | resource privateLinkScope 'microsoft.insights/privateLinkScopes@2021-07-01-preview' = { 16 | name: 'private-link-scope' 17 | location: 'global' 18 | properties: { 19 | accessModeSettings: { 20 | ingestionAccessMode: 'Open' 21 | queryAccessMode: 'Open' 22 | } 23 | } 24 | } 25 | 26 | resource workspace 'Microsoft.OperationalInsights/workspaces@2022-10-01' = { 27 | name: workspaceName 28 | location: location 29 | properties: { 30 | sku: { 31 | name: 'PerGB2018' 32 | } 33 | retentionInDays: 30 34 | workspaceCapping: {} 35 | publicNetworkAccessForIngestion: 'Disabled' 36 | publicNetworkAccessForQuery: 'Enabled' 37 | } 38 | } 39 | 40 | resource workspaceScopedResource 'Microsoft.Insights/privateLinkScopes/scopedResources@2021-07-01-preview' = { 41 | parent: privateLinkScope 42 | name: '${workspaceName}-connection' 43 | properties: { 44 | linkedResourceId: workspace.id 45 | } 46 | } 47 | 48 | resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = { 49 | name: applicationInsightsName 50 | location: location 51 | kind: 'other' 52 | properties: { 53 | Application_Type: 'web' 54 | Flow_Type: 'Bluefield' 55 | WorkspaceResourceId: workspace.id 56 | RetentionInDays: 90 57 | IngestionMode: 'LogAnalytics' 58 | publicNetworkAccessForIngestion: 'Disabled' 59 | publicNetworkAccessForQuery: 'Enabled' 60 | } 61 | } 62 | 63 | module privateEndpoint '../network/privateEndpoint.bicep' = { 64 | name: '${applicationInsightsName}-privateEndpoint' 65 | params: { 66 | groupIds: [ 67 | 'azuremonitor' 68 | ] 69 | dnsZoneName: 'privatelink.monitor.azure.com' 70 | name: privateEndpointName 71 | pepSubnetName: pepSubnetName 72 | privateLinkServiceId: privateLinkScope.id 73 | vnetName: vnetName 74 | location: location 75 | } 76 | dependsOn: [ 77 | applicationInsights 78 | ] 79 | } 80 | 81 | resource appInsightsScopedResource 'Microsoft.Insights/privateLinkScopes/scopedResources@2021-07-01-preview' = { 82 | parent: privateLinkScope 83 | name: '${applicationInsightsName}-connection' 84 | properties: { 85 | linkedResourceId: applicationInsights.id 86 | } 87 | dependsOn: [ 88 | privateEndpoint 89 | ] 90 | } 91 | 92 | resource applicationInsightsConnectionString 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = { 93 | name: '${applicationInsightsName}-ConnectionString' 94 | parent: vault 95 | properties: { 96 | attributes: { 97 | enabled: true 98 | } 99 | contentType: 'string' 100 | value: applicationInsights.properties.ConnectionString 101 | } 102 | } 103 | 104 | resource applicationInsightsKey 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = { 105 | name: applicationInsightsName 106 | parent: vault 107 | properties: { 108 | attributes: { 109 | enabled: true 110 | } 111 | contentType: 'string' 112 | value: applicationInsights.properties.InstrumentationKey 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /infra/webapp/webApi.bicep: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | param location string 4 | param loggingWebApiName string 5 | param appServicePlanName string 6 | param sku string 7 | param keyVaultName string 8 | param applicationInsightsName string 9 | param cosmosDbAccountName string 10 | param cosmosDbDatabaseName string 11 | param cosmosDbLogContainerName string 12 | param cosmosDbTriggerContainerName string 13 | param vnetName string 14 | param webAppSubnetName string 15 | param pepSubnetName string 16 | param loggingWebApiPrivateEndpointName string 17 | 18 | resource vault 'Microsoft.KeyVault/vaults@2022-07-01' existing = { 19 | name: keyVaultName 20 | } 21 | 22 | resource cosmosDbConnectionString 'Microsoft.KeyVault/vaults/secrets@2022-07-01' existing = { 23 | name: '${toLower(cosmosDbAccountName)}-ConnectionString' 24 | parent: vault 25 | } 26 | 27 | resource applicationInsightsConnectionString 'Microsoft.KeyVault/vaults/secrets@2022-07-01' existing = { 28 | name: '${applicationInsightsName}-ConnectionString' 29 | parent: vault 30 | } 31 | 32 | resource vnet 'Microsoft.Network/virtualNetworks@2023-04-01' existing = { 33 | name: vnetName 34 | } 35 | 36 | resource webAppSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-04-01' existing = { 37 | name: webAppSubnetName 38 | parent: vnet 39 | } 40 | 41 | resource asp 'Microsoft.Web/serverfarms@2022-03-01' = { 42 | name: appServicePlanName 43 | location: location 44 | sku: { 45 | name: sku 46 | } 47 | } 48 | 49 | resource loggingWebApi 'Microsoft.Web/sites@2022-03-01' = { 50 | name: loggingWebApiName 51 | location: location 52 | tags: { 53 | 'azd-service-name': 'loggingWebApi' 54 | } 55 | identity: { 56 | type: 'SystemAssigned' 57 | } 58 | properties: { 59 | serverFarmId: asp.id 60 | httpsOnly: true 61 | virtualNetworkSubnetId: webAppSubnet.id 62 | publicNetworkAccess: 'Enabled' 63 | siteConfig: { 64 | appSettings: [ 65 | { 66 | name: 'APPLICATIONINSIGHTS_CONNECTION_STRING' 67 | value: '@Microsoft.KeyVault(SecretUri=${applicationInsightsConnectionString.properties.secretUri})' 68 | } 69 | { 70 | name: 'ApplicationInsightsAgent_EXTENSION_VERSION' 71 | value: '~2' 72 | } 73 | { 74 | name: 'XDT_MicrosoftApplicationInsights_Mode' 75 | value: 'default' 76 | } 77 | { 78 | name: 'CosmosDbConnectionString' 79 | value: '@Microsoft.KeyVault(SecretUri=${cosmosDbConnectionString.properties.secretUri})' 80 | } 81 | { 82 | name: 'CosmosDbDatabaseName' 83 | value: cosmosDbDatabaseName 84 | } 85 | { 86 | name: 'CosmosDbLogContainerName' 87 | value: cosmosDbLogContainerName 88 | } 89 | { 90 | name: 'CosmosDbTriggerContainerName' 91 | value: cosmosDbTriggerContainerName 92 | } 93 | ] 94 | phpVersion: 'OFF' 95 | netFrameworkVersion: 'v8.0' 96 | ftpsState: 'FtpsOnly' 97 | minTlsVersion: '1.2' 98 | use32BitWorkerProcess: false 99 | alwaysOn: true 100 | } 101 | } 102 | } 103 | 104 | module webAppPrivateEndpoint '../network/privateEndpoint.bicep' = { 105 | name: '${loggingWebApiName}-privateEndpoint' 106 | params: { 107 | location: location 108 | name: loggingWebApiPrivateEndpointName 109 | groupIds: [ 110 | 'sites' 111 | ] 112 | dnsZoneName: 'privatelink.azurewebsites.net' 113 | vnetName: vnetName 114 | pepSubnetName: pepSubnetName 115 | privateLinkServiceId: loggingWebApi.id 116 | } 117 | } 118 | 119 | output loggingWebApiIdentityId string = loggingWebApi.identity.principalId 120 | -------------------------------------------------------------------------------- /infra/db/cosmosDb.bicep: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | param location string 4 | param accountName string 5 | param databaseName string 6 | param logContainerName string 7 | param triggerContainerName string 8 | param primaryRegion string 9 | param keyVaultName string 10 | param vnetName string 11 | param pepSubnetName string 12 | param privateEndpointName string 13 | 14 | var locations = [ 15 | { 16 | locationName: primaryRegion 17 | failoverPriority: 0 18 | isZoneRedundant: false 19 | } 20 | ] 21 | 22 | resource vault 'Microsoft.KeyVault/vaults@2022-07-01' existing = { 23 | name: keyVaultName 24 | } 25 | 26 | resource account 'Microsoft.DocumentDB/databaseAccounts@2022-05-15' = { 27 | name: toLower(accountName) 28 | kind: 'GlobalDocumentDB' 29 | location: location 30 | properties: { 31 | consistencyPolicy: { 32 | defaultConsistencyLevel: 'Session' 33 | } 34 | locations: locations 35 | databaseAccountOfferType: 'Standard' 36 | enableMultipleWriteLocations: false 37 | enableAutomaticFailover: false 38 | publicNetworkAccess: 'Disabled' 39 | capabilities: [ 40 | { 41 | name: 'DeleteAllItemsByPartitionKey' 42 | } 43 | ] 44 | } 45 | } 46 | 47 | resource database 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases@2022-05-15' = { 48 | parent: account 49 | name: databaseName 50 | properties: { 51 | resource: { 52 | id: databaseName 53 | } 54 | } 55 | } 56 | 57 | resource logContainer 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers@2022-05-15' = { 58 | parent: database 59 | name: logContainerName 60 | properties: { 61 | resource: { 62 | id: logContainerName 63 | partitionKey: { 64 | paths: [ 65 | '/requestId' 66 | ] 67 | kind: 'Hash' 68 | } 69 | indexingPolicy: { 70 | indexingMode: 'consistent' 71 | includedPaths: [ 72 | { 73 | path: '/*' 74 | } 75 | ] 76 | excludedPaths: [ 77 | { 78 | path: '/_etag/?' 79 | } 80 | ] 81 | } 82 | defaultTtl: 86400 83 | } 84 | } 85 | } 86 | 87 | resource triggerContainer 'Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers@2022-05-15' = { 88 | parent: database 89 | name: triggerContainerName 90 | properties: { 91 | resource: { 92 | id: triggerContainerName 93 | partitionKey: { 94 | paths: [ 95 | '/id' 96 | ] 97 | kind: 'Hash' 98 | } 99 | indexingPolicy: { 100 | indexingMode: 'consistent' 101 | includedPaths: [ 102 | { 103 | path: '/*' 104 | } 105 | ] 106 | excludedPaths: [ 107 | { 108 | path: '/_etag/?' 109 | } 110 | ] 111 | } 112 | defaultTtl: 86400 113 | } 114 | } 115 | } 116 | 117 | module privateEndpoint '../network/privateEndpoint.bicep' = { 118 | name: '${accountName}-privateEndpoint' 119 | params: { 120 | groupIds: [ 121 | 'Sql' 122 | ] 123 | dnsZoneName: 'privatelink.documents.azure.com' 124 | name: privateEndpointName 125 | pepSubnetName: pepSubnetName 126 | privateLinkServiceId: account.id 127 | vnetName: vnetName 128 | location: location 129 | } 130 | } 131 | 132 | resource cosmosDbConnectionString 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = { 133 | name: '${toLower(accountName)}-ConnectionString' 134 | parent: vault 135 | properties: { 136 | attributes: { 137 | enabled: true 138 | } 139 | contentType: 'string' 140 | value: account.listConnectionStrings().connectionStrings[0].connectionString 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /LogParserFunction/Services/TikTokenService.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Services; 4 | 5 | /// 6 | /// TikToken Service 7 | /// 8 | public class TikTokenService 9 | { 10 | HttpClient client = new(); 11 | TikToken tikToken = TikToken.EncodingForModel("gpt-3.5-turbo"); 12 | 13 | /// 14 | /// Count the number of tokens for the input 15 | /// 16 | /// TikTokenService 17 | /// Token count 18 | public async Task CountToken(List messages) 19 | { 20 | int total = 0; 21 | foreach(Message message in messages) 22 | { 23 | if (message.Content is string) 24 | { 25 | total += tikToken.Encode(message.Content).Count; 26 | } 27 | else if (message.Content is JArray) 28 | { 29 | List contents = 30 | JsonConvert.DeserializeObject>(message.Content.ToString()); 31 | 32 | foreach(VisionContent content in contents) 33 | { 34 | if (!string.IsNullOrEmpty(content.Text)) 35 | { 36 | total += tikToken.Encode(content.Text).Count; 37 | } 38 | else 39 | { 40 | if (content.ImageUrl is null) 41 | { 42 | continue; 43 | } 44 | if (content.ImageUrl.Details == "low") 45 | { 46 | total += 85; 47 | } 48 | else if (content.ImageUrl.Url.StartsWith("data")) 49 | { 50 | string base64Image = content.ImageUrl.Url.Split(",")[1]; 51 | byte[] imageBytes = Convert.FromBase64String(base64Image); 52 | MemoryStream stream = new(imageBytes); 53 | Image img = await Image.LoadAsync(stream); 54 | total += CalculateFromImage(img); 55 | } 56 | else 57 | { 58 | Stream imageStream = await client.GetStreamAsync(content.ImageUrl.Url); 59 | Image img = await Image.LoadAsync(imageStream); 60 | total += CalculateFromImage(img); 61 | } 62 | } 63 | } 64 | } 65 | } 66 | 67 | return total; 68 | } 69 | 70 | /// 71 | /// Calcualte token for image input. 72 | /// https://platform.openai.com/docs/guides/vision/calculating-costs 73 | /// 74 | /// 75 | /// 76 | private int CalculateFromImage(Image img) 77 | { 78 | if (Math.Max(img.Width, img.Height) < 512) 79 | { 80 | return 85; 81 | } 82 | else if (Math.Max(img.Width, img.Height) < 2048) 83 | { 84 | if (Math.Min(img.Width, img.Height) > 768) 85 | { 86 | int largeSide = (int)Math.Ceiling( 87 | (double)768 / 88 | Math.Min(img.Width, img.Height) * 89 | Math.Max(img.Width, img.Height) / 90 | 512); 91 | return largeSide * 2 * 170 + 85; 92 | } 93 | else 94 | { 95 | int largeSide = (int)Math.Ceiling( 96 | (double)(Math.Max(img.Width, img.Height) / 97 | 512)); 98 | return largeSide * 2 * 170 + 85; 99 | } 100 | } 101 | else 102 | { 103 | int largeSide = (int)Math.Ceiling( 104 | (double)768 / 105 | (Math.Min(img.Width, img.Height) / (Math.Max(img.Width, img.Height) / 2048)) * 106 | 2048 / 107 | 512); 108 | return largeSide * 2 * 170 + 85; 109 | } 110 | } 111 | 112 | /// 113 | /// Count the number of tokens for the input 114 | /// 115 | /// TikTokenService 116 | /// Token count 117 | public int CountToken(string input) 118 | { 119 | return tikToken.Encode(input).Count; 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /LogParserFunction/Services/DataProcessService.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Services; 4 | 5 | /// 6 | /// DataProcess Service that process the data from Cosmos DB 7 | /// 8 | /// TikTokenService 9 | /// ResultCacheService 10 | public class DataProcessService( 11 | TikTokenService tikTokenService, 12 | ContentSafetyService contentSafetyService, 13 | Container container) 14 | { 15 | private readonly TikTokenService tikTokenService = tikTokenService; 16 | private readonly ContentSafetyService contentSafetyService = contentSafetyService; 17 | private readonly Container container = container; 18 | 19 | /// 20 | /// Process the log data from Cosmos DB 21 | /// 22 | /// 23 | /// AOAILog 24 | public async Task ProcessLogsAsync(string requestId) 25 | { 26 | if (string.IsNullOrEmpty(requestId) || requestId.Count() < 5) 27 | { 28 | return default; 29 | } 30 | 31 | QueryDefinition query = new(query: $"SELECT * FROM c WHERE c.requestId = '{requestId}'"); 32 | using FeedIterator feed = container.GetItemQueryIterator( 33 | queryDefinition: query 34 | ); 35 | List items = new(); 36 | while (feed.HasMoreResults) 37 | { 38 | FeedResponse docs = await feed.ReadNextAsync(); 39 | foreach (JToken doc in docs) 40 | { 41 | items.Add(doc); 42 | } 43 | } 44 | TempRequestLog? tempRequestLog = new(); 45 | long elapsed = 0; 46 | int statusCode = 0; 47 | string statusReason = string.Empty; 48 | Request? request = new(); 49 | Response? response = new(); 50 | StringBuilder sb = new(); 51 | 52 | if (!items.Any()) 53 | { 54 | return default; 55 | } 56 | 57 | foreach (JToken item in items) 58 | { 59 | if (item["type"].ToString() == "Request") 60 | { 61 | tempRequestLog = JsonConvert.DeserializeObject(item.ToString()); 62 | request = item.ToString().GetRequest(); 63 | } 64 | else if (item["type"].ToString() == "Response") 65 | { 66 | TempResponseLog tempResponseLog = JsonConvert.DeserializeObject(item.ToString()); 67 | elapsed = tempResponseLog.Headers!["Elapsed"]; 68 | statusCode = tempResponseLog.Headers!["StatusCode"]; 69 | statusReason = tempResponseLog.Headers!["StatusReason"]; 70 | response = await item.ToString().GetResponse(request!, tikTokenService, contentSafetyService); 71 | } 72 | else if (item["type"].ToString() == "StreamResponse") 73 | { 74 | TempStreamResponseLog tempStreamResponseLog = JsonConvert.DeserializeObject(item.ToString()); 75 | elapsed = tempStreamResponseLog.Headers!["Elapsed"]; 76 | statusCode = tempStreamResponseLog.Headers!["StatusCode"]; 77 | statusReason = tempStreamResponseLog.Headers!["StatusReason"]; 78 | sb.Append(item["response"].ToString()+ "\n\n"); 79 | } 80 | } 81 | 82 | Console.WriteLine($"records: {items.Count}"); 83 | if(sb.Length > 0) 84 | { 85 | JObject token = new(); 86 | token["response"] = sb.ToString(); 87 | response = await token.ToString().GetResponse(request!, tikTokenService, contentSafetyService); 88 | } 89 | 90 | AOAILog aoaiLog = new() 91 | { 92 | ApiName = tempRequestLog.Headers!["ApiName"], 93 | ApiRevision = tempRequestLog.Headers!["ApiRevision"], 94 | Elapsed = elapsed, 95 | Method = tempRequestLog.Headers!["Method"], 96 | Headers = tempRequestLog.Headers, 97 | OperationId = tempRequestLog.Headers!["OperationId"], 98 | OperationName = tempRequestLog.Headers!["OperationName"], 99 | Region = tempRequestLog.Headers!["Region"], 100 | RequestId = tempRequestLog.Headers!["RequestId"], 101 | RequestIp = tempRequestLog.Headers!["RequestIp"], 102 | Request = request!, 103 | Response = response!, 104 | ServiceName = tempRequestLog.Headers!["ServiceName"], 105 | SubscriptionId = tempRequestLog.Headers!["SubscriptionId"], 106 | SubscriptionName = tempRequestLog.Headers!["SubscriptionName"], 107 | StatusCode = statusCode, 108 | StatusReason = statusReason, 109 | Timestamp = tempRequestLog.Headers!["Timestamp"], 110 | Url = tempRequestLog.Headers!["RequestUrl"] 111 | }; 112 | 113 | //https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/how-to-delete-by-partition-key?tabs=dotnet-example 114 | // Cosmos DB needs to support delete by partition key feature. 115 | await container.DeleteAllItemsByPartitionKeyStreamAsync(new PartitionKey(requestId)); 116 | 117 | return aoaiLog; 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /LogParserFunction/.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # Azure Functions localsettings file 5 | local.settings.json 6 | 7 | # User-specific files 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Build results 17 | [Dd]ebug/ 18 | [Dd]ebugPublic/ 19 | [Rr]elease/ 20 | [Rr]eleases/ 21 | x64/ 22 | x86/ 23 | bld/ 24 | [Bb]in/ 25 | [Oo]bj/ 26 | [Ll]og/ 27 | 28 | # Visual Studio 2015 cache/options directory 29 | .vs/ 30 | # Uncomment if you have tasks that create the project's static files in wwwroot 31 | #wwwroot/ 32 | 33 | # MSTest test Results 34 | [Tt]est[Rr]esult*/ 35 | [Bb]uild[Ll]og.* 36 | 37 | # NUNIT 38 | *.VisualState.xml 39 | TestResult.xml 40 | 41 | # Build Results of an ATL Project 42 | [Dd]ebugPS/ 43 | [Rr]eleasePS/ 44 | dlldata.c 45 | 46 | # DNX 47 | project.lock.json 48 | project.fragment.lock.json 49 | artifacts/ 50 | 51 | *_i.c 52 | *_p.c 53 | *_i.h 54 | *.ilk 55 | *.meta 56 | *.obj 57 | *.pch 58 | *.pdb 59 | *.pgc 60 | *.pgd 61 | *.rsp 62 | *.sbr 63 | *.tlb 64 | *.tli 65 | *.tlh 66 | *.tmp 67 | *.tmp_proj 68 | *.log 69 | *.vspscc 70 | *.vssscc 71 | .builds 72 | *.pidb 73 | *.svclog 74 | *.scc 75 | 76 | # Chutzpah Test files 77 | _Chutzpah* 78 | 79 | # Visual C++ cache files 80 | ipch/ 81 | *.aps 82 | *.ncb 83 | *.opendb 84 | *.opensdf 85 | *.sdf 86 | *.cachefile 87 | *.VC.db 88 | *.VC.VC.opendb 89 | 90 | # Visual Studio profiler 91 | *.psess 92 | *.vsp 93 | *.vspx 94 | *.sap 95 | 96 | # TFS 2012 Local Workspace 97 | $tf/ 98 | 99 | # Guidance Automation Toolkit 100 | *.gpState 101 | 102 | # ReSharper is a .NET coding add-in 103 | _ReSharper*/ 104 | *.[Rr]e[Ss]harper 105 | *.DotSettings.user 106 | 107 | # JustCode is a .NET coding add-in 108 | .JustCode 109 | 110 | # TeamCity is a build add-in 111 | _TeamCity* 112 | 113 | # DotCover is a Code Coverage Tool 114 | *.dotCover 115 | 116 | # NCrunch 117 | _NCrunch_* 118 | .*crunch*.local.xml 119 | nCrunchTemp_* 120 | 121 | # MightyMoose 122 | *.mm.* 123 | AutoTest.Net/ 124 | 125 | # Web workbench (sass) 126 | .sass-cache/ 127 | 128 | # Installshield output folder 129 | [Ee]xpress/ 130 | 131 | # DocProject is a documentation generator add-in 132 | DocProject/buildhelp/ 133 | DocProject/Help/*.HxT 134 | DocProject/Help/*.HxC 135 | DocProject/Help/*.hhc 136 | DocProject/Help/*.hhk 137 | DocProject/Help/*.hhp 138 | DocProject/Help/Html2 139 | DocProject/Help/html 140 | 141 | # Click-Once directory 142 | publish/ 143 | 144 | # Publish Web Output 145 | *.[Pp]ublish.xml 146 | *.azurePubxml 147 | # TODO: Comment the next line if you want to checkin your web deploy settings 148 | # but database connection strings (with potential passwords) will be unencrypted 149 | #*.pubxml 150 | *.publishproj 151 | 152 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 153 | # checkin your Azure Web App publish settings, but sensitive information contained 154 | # in these scripts will be unencrypted 155 | PublishScripts/ 156 | 157 | # NuGet Packages 158 | *.nupkg 159 | # The packages folder can be ignored because of Package Restore 160 | **/packages/* 161 | # except build/, which is used as an MSBuild target. 162 | !**/packages/build/ 163 | # Uncomment if necessary however generally it will be regenerated when needed 164 | #!**/packages/repositories.config 165 | # NuGet v3's project.json files produces more ignoreable files 166 | *.nuget.props 167 | *.nuget.targets 168 | 169 | # Microsoft Azure Build Output 170 | csx/ 171 | *.build.csdef 172 | 173 | # Microsoft Azure Emulator 174 | ecf/ 175 | rcf/ 176 | 177 | # Windows Store app package directories and files 178 | AppPackages/ 179 | BundleArtifacts/ 180 | Package.StoreAssociation.xml 181 | _pkginfo.txt 182 | 183 | # Visual Studio cache files 184 | # files ending in .cache can be ignored 185 | *.[Cc]ache 186 | # but keep track of directories ending in .cache 187 | !*.[Cc]ache/ 188 | 189 | # Others 190 | ClientBin/ 191 | ~$* 192 | *~ 193 | *.dbmdl 194 | *.dbproj.schemaview 195 | *.jfm 196 | *.pfx 197 | *.publishsettings 198 | node_modules/ 199 | orleans.codegen.cs 200 | 201 | # Since there are multiple workflows, uncomment next line to ignore bower_components 202 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 203 | #bower_components/ 204 | 205 | # RIA/Silverlight projects 206 | Generated_Code/ 207 | 208 | # Backup & report files from converting an old project file 209 | # to a newer Visual Studio version. Backup files are not needed, 210 | # because we have git ;-) 211 | _UpgradeReport_Files/ 212 | Backup*/ 213 | UpgradeLog*.XML 214 | UpgradeLog*.htm 215 | 216 | # SQL Server files 217 | *.mdf 218 | *.ldf 219 | 220 | # Business Intelligence projects 221 | *.rdl.data 222 | *.bim.layout 223 | *.bim_*.settings 224 | 225 | # Microsoft Fakes 226 | FakesAssemblies/ 227 | 228 | # GhostDoc plugin setting file 229 | *.GhostDoc.xml 230 | 231 | # Node.js Tools for Visual Studio 232 | .ntvs_analysis.dat 233 | 234 | # Visual Studio 6 build log 235 | *.plg 236 | 237 | # Visual Studio 6 workspace options file 238 | *.opt 239 | 240 | # Visual Studio LightSwitch build output 241 | **/*.HTMLClient/GeneratedArtifacts 242 | **/*.DesktopClient/GeneratedArtifacts 243 | **/*.DesktopClient/ModelManifest.xml 244 | **/*.Server/GeneratedArtifacts 245 | **/*.Server/ModelManifest.xml 246 | _Pvt_Extensions 247 | 248 | # Paket dependency manager 249 | .paket/paket.exe 250 | paket-files/ 251 | 252 | # FAKE - F# Make 253 | .fake/ 254 | 255 | # JetBrains Rider 256 | .idea/ 257 | *.sln.iml 258 | 259 | # CodeRush 260 | .cr/ 261 | 262 | # Python Tools for Visual Studio (PTVS) 263 | __pycache__/ 264 | *.pyc -------------------------------------------------------------------------------- /infra/abbreviations.json: -------------------------------------------------------------------------------- 1 | { 2 | "analysisServicesServers": "as", 3 | "apiManagementService": "apim-", 4 | "appConfigurationConfigurationStores": "appcs-", 5 | "appManagedEnvironments": "cae-", 6 | "appContainerApps": "ca-", 7 | "authorizationPolicyDefinitions": "policy-", 8 | "automationAutomationAccounts": "aa-", 9 | "blueprintBlueprints": "bp-", 10 | "blueprintBlueprintsArtifacts": "bpa-", 11 | "cacheRedis": "redis-", 12 | "cdnProfiles": "cdnp-", 13 | "cdnProfilesEndpoints": "cdne-", 14 | "cognitiveServicesAccounts": "cog-", 15 | "cognitiveServicesContentSafety": "cog-cs-", 16 | "cognitiveServicesFormRecognizer": "cog-fr-", 17 | "cognitiveServicesOpenAi": "cog-aoai-", 18 | "cognitiveServicesTextAnalytics": "cog-ta-", 19 | "computeAvailabilitySets": "avail-", 20 | "computeCloudServices": "cld-", 21 | "computeDiskEncryptionSets": "des", 22 | "computeDisks": "disk", 23 | "computeDisksOs": "osdisk", 24 | "computeGalleries": "gal", 25 | "computeSnapshots": "snap-", 26 | "computeVirtualMachines": "vm", 27 | "computeVirtualMachineScaleSets": "vmss-", 28 | "containerInstanceContainerGroups": "ci", 29 | "containerRegistryRegistries": "cr", 30 | "containerServiceManagedClusters": "aks-", 31 | "databricksWorkspaces": "dbw-", 32 | "dataFactoryFactories": "adf-", 33 | "dataLakeAnalyticsAccounts": "dla", 34 | "dataLakeStoreAccounts": "dls", 35 | "dataMigrationServices": "dms-", 36 | "dBforMySQLServers": "mysql-", 37 | "dBforPostgreSQLServers": "psql-", 38 | "devicesIotHubs": "iot-", 39 | "devicesProvisioningServices": "provs-", 40 | "devicesProvisioningServicesCertificates": "pcert-", 41 | "documentDBDatabaseAccounts": "cosmos-", 42 | "eventGridDomains": "evgd-", 43 | "eventGridDomainsTopics": "evgt-", 44 | "eventGridEventSubscriptions": "evgs-", 45 | "eventHubNamespaces": "evhns-", 46 | "eventHubNamespacesEventHubs": "evh-", 47 | "hdInsightClustersHadoop": "hadoop-", 48 | "hdInsightClustersHbase": "hbase-", 49 | "hdInsightClustersKafka": "kafka-", 50 | "hdInsightClustersMl": "mls-", 51 | "hdInsightClustersSpark": "spark-", 52 | "hdInsightClustersStorm": "storm-", 53 | "hybridComputeMachines": "arcs-", 54 | "insightsActionGroups": "ag-", 55 | "insightsComponents": "appi-", 56 | "keyVaultVaults": "kv-", 57 | "kubernetesConnectedClusters": "arck", 58 | "kustoClusters": "dec", 59 | "kustoClustersDatabases": "dedb", 60 | "logicIntegrationAccounts": "ia-", 61 | "logicWorkflows": "logic-", 62 | "machineLearningServicesWorkspaces": "mlw-", 63 | "managedIdentityUserAssignedIdentities": "id-", 64 | "managementManagementGroups": "mg-", 65 | "migrateAssessmentProjects": "migr-", 66 | "networkApplicationGateways": "agw-", 67 | "networkApplicationSecurityGroups": "asg-", 68 | "networkAzureFirewalls": "afw-", 69 | "networkBastionHosts": "bas-", 70 | "networkConnections": "con-", 71 | "networkDnsZones": "dnsz-", 72 | "networkExpressRouteCircuits": "erc-", 73 | "networkFirewallPolicies": "afwp-", 74 | "networkFirewallPoliciesWebApplication": "waf", 75 | "networkFirewallPoliciesRuleGroups": "wafrg", 76 | "networkFrontDoors": "fd-", 77 | "networkFrontdoorWebApplicationFirewallPolicies": "fdfp-", 78 | "networkLoadBalancersExternal": "lbe-", 79 | "networkLoadBalancersInternal": "lbi-", 80 | "networkLoadBalancersInboundNatRules": "rule-", 81 | "networkLocalNetworkGateways": "lgw-", 82 | "networkNatGateways": "ng-", 83 | "networkNetworkInterfaces": "nic-", 84 | "networkNetworkSecurityGroups": "nsg-", 85 | "networkNetworkSecurityGroupsSecurityRules": "nsgsr-", 86 | "networkNetworkWatchers": "nw-", 87 | "networkPrivateDnsZones": "pdnsz-", 88 | "networkPrivateLinkServices": "pl-", 89 | "networkPublicIPAddresses": "pip-", 90 | "networkPublicIPPrefixes": "ippre-", 91 | "networkRouteFilters": "rf-", 92 | "networkRouteTables": "rt-", 93 | "networkRouteTablesRoutes": "udr-", 94 | "networkTrafficManagerProfiles": "traf-", 95 | "networkVirtualNetworkGateways": "vgw-", 96 | "networkVirtualNetworks": "vnet-", 97 | "networkVirtualNetworksSubnets": "snet-", 98 | "networkVirtualNetworksVirtualNetworkPeerings": "peer-", 99 | "networkVirtualWans": "vwan-", 100 | "networkVpnGateways": "vpng-", 101 | "networkVpnGatewaysVpnConnections": "vcn-", 102 | "networkVpnGatewaysVpnSites": "vst-", 103 | "notificationHubsNamespaces": "ntfns-", 104 | "notificationHubsNamespacesNotificationHubs": "ntf-", 105 | "operationalInsightsWorkspaces": "log-", 106 | "portalDashboards": "dash-", 107 | "powerBIDedicatedCapacities": "pbi-", 108 | "purviewAccounts": "pview-", 109 | "recoveryServicesVaults": "rsv-", 110 | "resourcesResourceGroups": "rg-", 111 | "searchSearchServices": "srch-", 112 | "serviceBusNamespaces": "sb-", 113 | "serviceBusNamespacesQueues": "sbq-", 114 | "serviceBusNamespacesTopics": "sbt-", 115 | "serviceEndPointPolicies": "se-", 116 | "serviceFabricClusters": "sf-", 117 | "signalRServiceSignalR": "sigr", 118 | "sqlManagedInstances": "sqlmi-", 119 | "sqlServers": "sql-", 120 | "sqlServersDataWarehouse": "sqldw-", 121 | "sqlServersDatabases": "sqldb-", 122 | "sqlServersDatabasesStretch": "sqlstrdb-", 123 | "storageStorageAccounts": "st", 124 | "storageStorageAccountsVm": "stvm", 125 | "storSimpleManagers": "ssimp", 126 | "streamAnalyticsCluster": "asa-", 127 | "synapseWorkspaces": "syn", 128 | "synapseWorkspacesAnalyticsWorkspaces": "synw", 129 | "synapseWorkspacesSqlPoolsDedicated": "syndp", 130 | "synapseWorkspacesSqlPoolsSpark": "synsp", 131 | "timeSeriesInsightsEnvironments": "tsi-", 132 | "webServerFarms": "plan-", 133 | "webSitesAppService": "app-", 134 | "webSitesAppServiceEnvironment": "ase-", 135 | "webSitesFunctions": "func-", 136 | "webStaticSites": "stapp-" 137 | } -------------------------------------------------------------------------------- /infra/webapp/function.bicep: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | param location string 4 | param logParserFunctionName string 5 | param appServicePlanName string 6 | param sku string 7 | param functionStorageAccountName string 8 | param functionStorageAccountType string 9 | param keyVaultName string 10 | param applicationInsightsName string 11 | param cosmosDbAccountName string 12 | param cosmosDbDatabaseName string 13 | param cosmosDbLogContainerName string 14 | param cosmosDbTriggerContainerName string 15 | param contentSafetyAccountName string 16 | param vnetName string 17 | param webAppSubnetName string 18 | param pepSubnetName string 19 | param logParserFunctionPrivateEndpointName string 20 | 21 | resource vault 'Microsoft.KeyVault/vaults@2022-07-01' existing = { 22 | name: keyVaultName 23 | } 24 | 25 | resource contentSafety 'Microsoft.CognitiveServices/accounts@2023-05-01' existing = { 26 | name: contentSafetyAccountName 27 | } 28 | 29 | resource cosmosDbConnectionString 'Microsoft.KeyVault/vaults/secrets@2022-07-01' existing = { 30 | name: '${toLower(cosmosDbAccountName)}-ConnectionString' 31 | parent: vault 32 | } 33 | 34 | resource contentSafetyKey 'Microsoft.KeyVault/vaults/secrets@2022-07-01' existing = { 35 | name: contentSafetyAccountName 36 | parent: vault 37 | } 38 | 39 | resource applicationInsights 'Microsoft.Insights/components@2020-02-02' existing = { 40 | name: applicationInsightsName 41 | } 42 | 43 | resource applicationInsightsConnectionString 'Microsoft.KeyVault/vaults/secrets@2022-07-01' existing = { 44 | name: '${applicationInsightsName}-ConnectionString' 45 | parent: vault 46 | } 47 | 48 | resource vnet 'Microsoft.Network/virtualNetworks@2023-04-01' existing = { 49 | name: vnetName 50 | } 51 | 52 | resource webAppSubne 'Microsoft.Network/virtualNetworks/subnets@2023-04-01' existing = { 53 | name: webAppSubnetName 54 | parent: vnet 55 | } 56 | 57 | resource asp 'Microsoft.Web/serverfarms@2022-03-01' = { 58 | name: appServicePlanName 59 | location: location 60 | sku: { 61 | name: sku 62 | } 63 | } 64 | 65 | resource storageAccount 'Microsoft.Storage/storageAccounts@2022-05-01' = { 66 | name: functionStorageAccountName 67 | location: location 68 | sku: { 69 | name: functionStorageAccountType 70 | } 71 | kind: 'Storage' 72 | properties: { 73 | supportsHttpsTrafficOnly: true 74 | defaultToOAuthAuthentication: true 75 | } 76 | } 77 | 78 | resource logParserFunction 'Microsoft.Web/sites@2022-03-01' = { 79 | name: logParserFunctionName 80 | location: location 81 | identity: { 82 | type: 'SystemAssigned' 83 | } 84 | tags: { 85 | 'azd-service-name': 'logParserFunction' 86 | } 87 | kind: 'functionapp' 88 | properties: { 89 | serverFarmId: asp.id 90 | httpsOnly: true 91 | virtualNetworkSubnetId: webAppSubne.id 92 | publicNetworkAccess: 'Enabled' 93 | siteConfig: { 94 | appSettings: [ 95 | { 96 | name: 'FUNCTIONS_EXTENSION_VERSION' 97 | value: '~4' 98 | } 99 | { 100 | name: 'WEBSITE_USE_PLACEHOLDER_DOTNETISOLATED' 101 | value: '1' 102 | } 103 | { 104 | name: 'APPINSIGHTS_INSTRUMENTATIONKEY' 105 | value: applicationInsights.properties.InstrumentationKey 106 | } 107 | { 108 | name: 'AzureWebJobsStorage' 109 | value: 'DefaultEndpointsProtocol=https;AccountName=${functionStorageAccountName};EndpointSuffix=${environment().suffixes.storage};AccountKey=${storageAccount.listKeys().keys[0].value}' 110 | } 111 | { 112 | name: 'WEBSITE_CONTENTAZUREFILECONNECTIONSTRING' 113 | value: 'DefaultEndpointsProtocol=https;AccountName=${functionStorageAccountName};EndpointSuffix=${environment().suffixes.storage};AccountKey=${storageAccount.listKeys().keys[0].value}' 114 | } 115 | { 116 | name: 'WEBSITE_CONTENTSHARE' 117 | value: toLower(logParserFunctionName) 118 | } 119 | { 120 | name: 'FUNCTIONS_WORKER_RUNTIME' 121 | value: 'dotnet-isolated' 122 | } 123 | { 124 | name: 'CosmosDbConnectionString' 125 | value: '@Microsoft.KeyVault(SecretUri=${cosmosDbConnectionString.properties.secretUri})' 126 | } 127 | { 128 | name: 'CosmosDbDatabaseName' 129 | value: cosmosDbDatabaseName 130 | } 131 | { 132 | name: 'CosmosDbLogContainerName' 133 | value: cosmosDbLogContainerName 134 | } 135 | { 136 | name: 'CosmosDbTriggerContainerName' 137 | value: cosmosDbTriggerContainerName 138 | } 139 | { 140 | name: 'ContentSafetyUrl' 141 | value: contentSafety.properties.endpoint 142 | } 143 | { 144 | name: 'ContentSafetyKey' 145 | value: '@Microsoft.KeyVault(SecretUri=${contentSafetyKey.properties.secretUri})' 146 | } 147 | { 148 | name: 'ApplicationInsightsConnectionString' 149 | value: '@Microsoft.KeyVault(SecretUri=${applicationInsightsConnectionString.properties.secretUri})' 150 | } 151 | ] 152 | phpVersion: 'OFF' 153 | netFrameworkVersion: 'v8.0' 154 | ftpsState: 'FtpsOnly' 155 | minTlsVersion: '1.2' 156 | alwaysOn: true 157 | use32BitWorkerProcess: false 158 | vnetRouteAllEnabled: true 159 | } 160 | } 161 | } 162 | 163 | module functionPrivateEndpoint '../network/privateEndpoint.bicep' = { 164 | name: '${logParserFunctionName}-privateEndpoint' 165 | params: { 166 | groupIds: [ 167 | 'sites' 168 | ] 169 | dnsZoneName: 'privatelink.azurewebsites.net' 170 | name: logParserFunctionPrivateEndpointName 171 | pepSubnetName: pepSubnetName 172 | privateLinkServiceId: logParserFunction.id 173 | vnetName: vnetName 174 | location: location 175 | } 176 | } 177 | 178 | output logParserFunctionIdentityId string = logParserFunction.identity.principalId 179 | -------------------------------------------------------------------------------- /LoggingWebApi/Controllers/OpenAI.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LoggingWebApi.Controllers; 4 | 5 | /// 6 | /// OpenAI controller that accepts any request and forwards it to the AOAI backend endpoint. 7 | /// It logs both request and response as temporary logs to Cosmos DB for monitoring. 8 | /// 9 | /// IHttpClientFactory 10 | /// IHttpContextAccessor 11 | /// Cosmos DB Container 12 | [ApiController] 13 | [Route("[controller]")] 14 | public class OpenAI( 15 | IHttpClientFactory factory, 16 | IHttpContextAccessor accessor, 17 | Containers containers) : ControllerBase 18 | { 19 | private readonly IHttpContextAccessor accessor = accessor; 20 | private readonly Containers containers = containers; 21 | private readonly HttpClient httpClient = factory.CreateClient(); 22 | private readonly string LINE_END = $"{Environment.NewLine}{Environment.NewLine}"; 23 | 24 | /// 25 | /// Accept Post method for any path and query parameters, then forward the request to AOAI endpoint. 26 | /// Log both request and response as temporary logs to Cosmos DB for monitoring. 27 | /// 28 | /// URL Path 29 | /// Request Body 30 | /// 31 | [HttpPost] 32 | [Route("{*path}")] 33 | public async Task Post(string path, [FromBody] JObject body) 34 | { 35 | Stopwatch sw = Stopwatch.StartNew(); 36 | sw.Start(); 37 | HttpResponse response = accessor.HttpContext!.Response; 38 | HttpRequest request = accessor.HttpContext!.Request; 39 | string requestId = request.Headers["RequestId"].ToString(); 40 | 41 | // Default action result is Empty for SSE. 42 | IActionResult actionResult = new EmptyResult(); 43 | List tempLogs = new(); 44 | 45 | // Log the request 46 | JObject headers = new(); 47 | foreach (KeyValuePair header in request.Headers) 48 | { 49 | if (header.Key is "BackendUrl") 50 | { 51 | httpClient.BaseAddress = new Uri(header.Value.ToString()); 52 | } 53 | else if (header.Key is "api-key") 54 | { 55 | continue; // Do not log key 56 | } 57 | headers[header.Key] = string.Join(",", header.Value!); 58 | } 59 | headers["RequestUrl"] = $"{request.Path}{request.QueryString}"; 60 | TempRequestLog tempRequestLog = new() 61 | { 62 | RequestId = requestId, 63 | Headers = headers, 64 | Request = body, 65 | RequestUrl = $"{request.Path}{request.QueryString}", 66 | }; 67 | 68 | tempLogs.Add(tempRequestLog); 69 | 70 | // Foward the request to AOAI endpoint 71 | ManagedIdentityCredential managedIdentityCredential = new(); 72 | AccessToken accessToken = await managedIdentityCredential.GetTokenAsync(new TokenRequestContext(new[] { "https://cognitiveservices.azure.com/" })).ConfigureAwait(false); 73 | httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", accessToken.Token); 74 | HttpRequestMessage requestMessage = new(HttpMethod.Post, path + Request.QueryString); 75 | requestMessage.Content = new StringContent(body.ToString(), Encoding.UTF8, "application/json"); 76 | HttpResponseMessage res = await httpClient.SendAsync(requestMessage, HttpCompletionOption.ResponseHeadersRead); 77 | 78 | // Log the response 79 | headers = new(); 80 | headers["StatusCode"] = (int)res.StatusCode; 81 | headers["StatusReason"] = res.ReasonPhrase; 82 | 83 | if (!res.IsSuccessStatusCode) 84 | { 85 | JObject responseContent = JObject.Parse(await res.Content.ReadAsStringAsync()); 86 | // Return the response as we don't need to log the AOAI level error. 87 | return StatusCode((int)res.StatusCode, responseContent); 88 | } 89 | // Reply SSE as stream results. 90 | else if (body["stream"] is not null && body["stream"]!.Value()) 91 | { 92 | response.Headers.TryAdd(HeaderNames.ContentType, "text/event-stream"); 93 | response.Headers.TryAdd(HeaderNames.CacheControl, "no-cache"); 94 | response.Headers.TryAdd(HeaderNames.Connection, "keep-alive"); 95 | 96 | using (StreamReader streamReader = new(await res.Content.ReadAsStreamAsync())) 97 | { 98 | while (!streamReader.EndOfStream) 99 | { 100 | string? message = await streamReader.ReadLineAsync(); 101 | if (string.IsNullOrEmpty(message)) 102 | { 103 | continue; 104 | } 105 | 106 | headers["Elapsed"] = sw.ElapsedMilliseconds; 107 | 108 | TempStreamResponseLog tempStreamResponseLog = new() 109 | { 110 | Headers = headers, 111 | RequestId = requestId, 112 | Response = message, 113 | }; 114 | 115 | // Send SSE with LINE_END 116 | await response.WriteAsync($"{message}{LINE_END}"); 117 | await response.Body.FlushAsync(); 118 | tempLogs.Add(tempStreamResponseLog); 119 | } 120 | } 121 | } 122 | // Handle non-streaming response 123 | else 124 | { 125 | JObject responseContent = JObject.Parse(await res.Content.ReadAsStringAsync()); 126 | headers["Elapsed"] = sw.ElapsedMilliseconds; 127 | TempResponseLog tempResponseLog = new() 128 | { 129 | Headers = headers, 130 | RequestId = requestId, 131 | Response = responseContent, 132 | }; 133 | tempLogs.Add(tempResponseLog); 134 | 135 | actionResult = this.Ok(responseContent); 136 | } 137 | 138 | // Return the response first, then do logging. 139 | try 140 | { 141 | return actionResult; 142 | } 143 | finally 144 | { 145 | Container logContainer = containers["logContainer"]; 146 | Container triggerContainer = containers["triggerContainer"]; 147 | Response.OnCompleted(async () => 148 | { 149 | foreach (TempLog tempLog in tempLogs) 150 | { 151 | await logContainer.CreateItemAsync(tempLog, 152 | new PartitionKey(requestId)); 153 | } 154 | 155 | // Once all logging complete for the request, create trigger item. 156 | await triggerContainer.CreateItemAsync(new TempLog() { RequestId = requestId }); 157 | }); 158 | } 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.rsuser 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Mono auto generated files 17 | mono_crash.* 18 | 19 | # Build results 20 | [Dd]ebug/ 21 | [Dd]ebugPublic/ 22 | [Rr]elease/ 23 | [Rr]eleases/ 24 | x64/ 25 | x86/ 26 | [Ww][Ii][Nn]32/ 27 | [Aa][Rr][Mm]/ 28 | [Aa][Rr][Mm]64/ 29 | bld/ 30 | [Bb]in/ 31 | [Oo]bj/ 32 | [Oo]ut/ 33 | [Ll]og/ 34 | [Ll]ogs/ 35 | 36 | # Visual Studio 2015/2017 cache/options directory 37 | .vs/ 38 | # Uncomment if you have tasks that create the project's static files in wwwroot 39 | #wwwroot/ 40 | 41 | # Visual Studio 2017 auto generated files 42 | Generated\ Files/ 43 | 44 | # MSTest test Results 45 | [Tt]est[Rr]esult*/ 46 | [Bb]uild[Ll]og.* 47 | 48 | # NUnit 49 | *.VisualState.xml 50 | TestResult.xml 51 | nunit-*.xml 52 | 53 | # Build Results of an ATL Project 54 | [Dd]ebugPS/ 55 | [Rr]eleasePS/ 56 | dlldata.c 57 | 58 | # Benchmark Results 59 | BenchmarkDotNet.Artifacts/ 60 | 61 | # .NET Core 62 | project.lock.json 63 | project.fragment.lock.json 64 | artifacts/ 65 | 66 | # ASP.NET Scaffolding 67 | ScaffoldingReadMe.txt 68 | 69 | # StyleCop 70 | StyleCopReport.xml 71 | 72 | # Files built by Visual Studio 73 | *_i.c 74 | *_p.c 75 | *_h.h 76 | *.ilk 77 | *.meta 78 | *.obj 79 | *.iobj 80 | *.pch 81 | *.pdb 82 | *.ipdb 83 | *.pgc 84 | *.pgd 85 | *.rsp 86 | *.sbr 87 | *.tlb 88 | *.tli 89 | *.tlh 90 | *.tmp 91 | *.tmp_proj 92 | *_wpftmp.csproj 93 | *.log 94 | *.vspscc 95 | *.vssscc 96 | .builds 97 | *.pidb 98 | *.svclog 99 | *.scc 100 | 101 | # Chutzpah Test files 102 | _Chutzpah* 103 | 104 | # Visual C++ cache files 105 | ipch/ 106 | *.aps 107 | *.ncb 108 | *.opendb 109 | *.opensdf 110 | *.sdf 111 | *.cachefile 112 | *.VC.db 113 | *.VC.VC.opendb 114 | 115 | # Visual Studio profiler 116 | *.psess 117 | *.vsp 118 | *.vspx 119 | *.sap 120 | 121 | # Visual Studio Trace Files 122 | *.e2e 123 | 124 | # TFS 2012 Local Workspace 125 | $tf/ 126 | 127 | # Guidance Automation Toolkit 128 | *.gpState 129 | 130 | # ReSharper is a .NET coding add-in 131 | _ReSharper*/ 132 | *.[Rr]e[Ss]harper 133 | *.DotSettings.user 134 | 135 | # TeamCity is a build add-in 136 | _TeamCity* 137 | 138 | # DotCover is a Code Coverage Tool 139 | *.dotCover 140 | 141 | # AxoCover is a Code Coverage Tool 142 | .axoCover/* 143 | !.axoCover/settings.json 144 | 145 | # Coverlet is a free, cross platform Code Coverage Tool 146 | coverage*.json 147 | coverage*.xml 148 | coverage*.info 149 | 150 | # Visual Studio code coverage results 151 | *.coverage 152 | *.coveragexml 153 | 154 | # NCrunch 155 | _NCrunch_* 156 | .*crunch*.local.xml 157 | nCrunchTemp_* 158 | 159 | # MightyMoose 160 | *.mm.* 161 | AutoTest.Net/ 162 | 163 | # Web workbench (sass) 164 | .sass-cache/ 165 | 166 | # Installshield output folder 167 | [Ee]xpress/ 168 | 169 | # DocProject is a documentation generator add-in 170 | DocProject/buildhelp/ 171 | DocProject/Help/*.HxT 172 | DocProject/Help/*.HxC 173 | DocProject/Help/*.hhc 174 | DocProject/Help/*.hhk 175 | DocProject/Help/*.hhp 176 | DocProject/Help/Html2 177 | DocProject/Help/html 178 | 179 | # Click-Once directory 180 | publish/ 181 | 182 | # Publish Web Output 183 | *.[Pp]ublish.xml 184 | *.azurePubxml 185 | # Note: Comment the next line if you want to checkin your web deploy settings, 186 | # but database connection strings (with potential passwords) will be unencrypted 187 | *.pubxml 188 | *.publishproj 189 | 190 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 191 | # checkin your Azure Web App publish settings, but sensitive information contained 192 | # in these scripts will be unencrypted 193 | PublishScripts/ 194 | 195 | # NuGet Packages 196 | *.nupkg 197 | # NuGet Symbol Packages 198 | *.snupkg 199 | # The packages folder can be ignored because of Package Restore 200 | **/[Pp]ackages/* 201 | # except build/, which is used as an MSBuild target. 202 | !**/[Pp]ackages/build/ 203 | # Uncomment if necessary however generally it will be regenerated when needed 204 | #!**/[Pp]ackages/repositories.config 205 | # NuGet v3's project.json files produces more ignorable files 206 | *.nuget.props 207 | *.nuget.targets 208 | 209 | # Microsoft Azure Build Output 210 | csx/ 211 | *.build.csdef 212 | 213 | # Microsoft Azure Emulator 214 | ecf/ 215 | rcf/ 216 | 217 | # Windows Store app package directories and files 218 | AppPackages/ 219 | BundleArtifacts/ 220 | Package.StoreAssociation.xml 221 | _pkginfo.txt 222 | *.appx 223 | *.appxbundle 224 | *.appxupload 225 | 226 | # Visual Studio cache files 227 | # files ending in .cache can be ignored 228 | *.[Cc]ache 229 | # but keep track of directories ending in .cache 230 | !?*.[Cc]ache/ 231 | 232 | # Others 233 | ClientBin/ 234 | ~$* 235 | *~ 236 | *.dbmdl 237 | *.dbproj.schemaview 238 | *.jfm 239 | *.pfx 240 | *.publishsettings 241 | orleans.codegen.cs 242 | 243 | # Including strong name files can present a security risk 244 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 245 | #*.snk 246 | 247 | # Since there are multiple workflows, uncomment next line to ignore bower_components 248 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 249 | #bower_components/ 250 | 251 | # RIA/Silverlight projects 252 | Generated_Code/ 253 | 254 | # Backup & report files from converting an old project file 255 | # to a newer Visual Studio version. Backup files are not needed, 256 | # because we have git ;-) 257 | _UpgradeReport_Files/ 258 | Backup*/ 259 | UpgradeLog*.XML 260 | UpgradeLog*.htm 261 | ServiceFabricBackup/ 262 | *.rptproj.bak 263 | 264 | # SQL Server files 265 | *.mdf 266 | *.ldf 267 | *.ndf 268 | 269 | # Business Intelligence projects 270 | *.rdl.data 271 | *.bim.layout 272 | *.bim_*.settings 273 | *.rptproj.rsuser 274 | *- [Bb]ackup.rdl 275 | *- [Bb]ackup ([0-9]).rdl 276 | *- [Bb]ackup ([0-9][0-9]).rdl 277 | 278 | # Microsoft Fakes 279 | FakesAssemblies/ 280 | 281 | # GhostDoc plugin setting file 282 | *.GhostDoc.xml 283 | 284 | # Node.js Tools for Visual Studio 285 | .ntvs_analysis.dat 286 | node_modules/ 287 | 288 | # Visual Studio 6 build log 289 | *.plg 290 | 291 | # Visual Studio 6 workspace options file 292 | *.opt 293 | 294 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 295 | *.vbw 296 | 297 | # Visual Studio LightSwitch build output 298 | **/*.HTMLClient/GeneratedArtifacts 299 | **/*.DesktopClient/GeneratedArtifacts 300 | **/*.DesktopClient/ModelManifest.xml 301 | **/*.Server/GeneratedArtifacts 302 | **/*.Server/ModelManifest.xml 303 | _Pvt_Extensions 304 | 305 | # Paket dependency manager 306 | .paket/paket.exe 307 | paket-files/ 308 | 309 | # FAKE - F# Make 310 | .fake/ 311 | 312 | # CodeRush personal settings 313 | .cr/personal 314 | 315 | # Python Tools for Visual Studio (PTVS) 316 | __pycache__/ 317 | *.pyc 318 | 319 | # Cake - Uncomment if you are using it 320 | # tools/** 321 | # !tools/packages.config 322 | 323 | # Tabs Studio 324 | *.tss 325 | 326 | # Telerik's JustMock configuration file 327 | *.jmconfig 328 | 329 | # BizTalk build output 330 | *.btp.cs 331 | *.btm.cs 332 | *.odx.cs 333 | *.xsd.cs 334 | 335 | # OpenCover UI analysis results 336 | OpenCover/ 337 | 338 | # Azure Stream Analytics local run output 339 | ASALocalRun/ 340 | 341 | # MSBuild Binary and Structured Log 342 | *.binlog 343 | 344 | # NVidia Nsight GPU debugger configuration file 345 | *.nvuser 346 | 347 | # MFractors (Xamarin productivity tool) working folder 348 | .mfractor/ 349 | 350 | # Local History for Visual Studio 351 | .localhistory/ 352 | 353 | # BeatPulse healthcheck temp database 354 | healthchecksdb 355 | 356 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 357 | MigrationBackup/ 358 | 359 | # Ionide (cross platform F# VS Code tools) working folder 360 | .ionide/ 361 | 362 | # Fody - auto-generated XML schema 363 | FodyWeavers.xsd 364 | 365 | appsettings.json 366 | appsettings.Development.json 367 | 368 | .azure 369 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | --- 2 | page_type: sample 3 | languages: 4 | - azdeveloper 5 | - bicep 6 | - csharp 7 | products: 8 | - azure 9 | - azure-api-management 10 | - azure-openai 11 | - azure-key-vault 12 | urlFragment: aoai-logging-with-apim 13 | name: Azure OpenAI Operation Management with Azure API Management 14 | description: The solution uses Azure APIM and other resources to to use Azure Open AI in production. 15 | --- 16 | 17 | 18 | # Azure OpenAI Operation Management with Azure API Management 19 | 20 | ## Challenges of Azure OpenAI in production 21 | 22 | We often see common challenges when we use Azure OpenAI(AOAI) in production environment. 23 | 24 | - __Key Management__: AOAI only has primary and secondary key per account, therefore we need to share the same key with users, teams and organizations. However, in most scenario, we need to manage each user separately for performance and monitoring purposes. The risk of sharing the same key is quite severe, for example, if someone reveals or lost the key, all the other users and applications are affected by rotating the existing keys. 25 | - __Different throttling settings__: Customers want to control how each audience consumes the service, but AOAI doesn't provide granular controls. 26 | - __Monitor Token Usage__: When using streaming mode, AOAI doesn't return consumed token count information. 27 | - __Monitor Request/Response body and headers__: Customers often needs actual request/response body and headers data to further analyze the usage, but AOAI doesn't provide it by default. 28 | - __Different Formats__: Each endpoint has slightly different request/response formats. Streaming mode also has quite different and hard to read response format that makes harder to generate reports. 29 | - __Content Safety for Stream Response__: As stream response returns the result token by token, the content safety results may not be accurate. 30 | - __Create Usage Dashboard__: Though AOAI integrates with Application Insights, they cannot create granular dashboard by using BI tool such as Power BI because the log doesn't contain enough information for enterprise scenario. 31 | - __Multiple Endpoints__: Not all models are available in a single AOAI account, so users have to manage endpoint and key combinations. 32 | 33 | ## How Azure API Management solves the challenges 34 | 35 | [Azure API Management (APIM)](https://learn.microsoft.com/azure/api-management/api-management-key-concepts) is a hybrid, multi-cloud management platform for APIs across all environments. As a platform-as-a-service, API Management supports the complete API lifecycle. 36 | 37 | We have more granular control to any APIs by using APIM. 38 | 39 | - Consolidate the endpoint access by hiding APIs behind the APIM instance. 40 | - Granular access control by issuing keys by using [subscriptions feature](https://learn.microsoft.com/azure/api-management/api-management-subscriptions). We can manage the access by API, APIs and/or by products. 41 | - Use [policies](https://learn.microsoft.com/azure/api-management/api-management-howto-policies) to manage APIs such as setting token per minutes throttling, use different backends, load-balancing them, set/remove headers, specify cache policies, etc. 42 | - Manage APIs by using [backends](https://learn.microsoft.com/azure/api-management/backends?tabs=bicep) and security store keys and connection strings by using [named values](https://learn.microsoft.com/azure/api-management/api-management-howto-properties?tabs=azure-portal). Azure resources including AOAI can be accessed via Managed Identity as well. 43 | - It provides out-of-box monitor capabilities and custom logger that can send log to any supported destination when we need more detailed logging. 44 | - Use custom logging to log the request and response body so calculate consumed token as well as analyze the content safety. 45 | 46 | ## Solution Architecture 47 | 48 | ### Application Level 49 | 50 | APIM Policy handles network traffic and logging. 51 | ![architecture](/assets/aoai_apim.svg) 52 | 53 | ### Network Level 54 | 55 | This solution uses VNet and Private Endpoints to secure Azure resources. 56 | 57 | - APIM: Use External VNet integration mode. 58 | - Azure Function and Web App: Use VNet integration mode so that they can access Azure resources via VNet and private endpoints. 59 | - Other resources: Use VNet and private endpoint. Block all external access via Firewall rule. 60 | 61 | # How to deploy the solution 62 | 63 | The repo support ``azd`` CLI. 64 | 65 | 1. Install the ``azd`` CLI. See [Install or update the Azure Developer CLI](https://learn.microsoft.com/azure/developer/azure-developer-cli/install-azd?tabs=winget-windows%2Cbrew-mac%2Cscript-linux&pivots=os-windows) for more detail. 66 | 1. Run ``azd up`` command from the terminal. 67 | 1. Select an Azure subscription and enter the environment name. 68 | 69 | The command deploys required Azure resources by following bicep files in [infra](/infra/) directory and deploy applications. 70 | 71 | # Repo structure 72 | 73 | ```shell 74 | ├─assets 75 | ├─infra 76 | ├─LoggingWebApi 77 | ├─LogParserFunction 78 | ├─policies 79 | ├─queries 80 | ├─azure.yaml 81 | └─README.md 82 | ``` 83 | 84 | - __infra__: The infrastructure as code (IaC) assets. 85 | - __LoggingWebApi__: C# sample Web API code to that works as proxy between APIM and AOAI, which send logs to Cosmos DB. Once logging completed, it sends the ``request id`` information to Cosmos DB container to trigger the Log Parser Function via change feed. 86 | - __LogParserFunction__: C# sample Azure Function code to parse the log in the Cosmos DB. It is triggered via Cosmos DB Change Feed, then retrieve all the logs for the ``request id``, transform them and store the final log to Application Insights. 87 | - __policies__: APIM policy fragments 88 | - __queries__: contains Kusto and Cosmos DB query that are used for creating report 89 | - __azure.yaml__: The main file for ``azd`` command 90 | 91 | See the following for more detail in each component. 92 | 93 | - [How to use Azure API Management with Azure OpenAI](APIM.md) 94 | - [Infrastructure as Code (bicep)](/infra/README.md) 95 | - [C# Logging Web App](/LoggingWebApi/README.md) 96 | - [C# Log Parser Function](/LogParserFunction/README.md) 97 | 98 | # Limitations 99 | 100 | Currently, there are several limitations. 101 | 102 | - Function Calling with stream mode: We are not consolidating the result for function calling in stream mode for now. 103 | - GPT 4 Vision with URL: If there is authentication/authorization for the image URL that the log parser cannot obtain, it fails to read the image. 104 | 105 | ## Contributing 106 | 107 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 108 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 109 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 110 | 111 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 112 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 113 | provided by the bot. You will only need to do this once across all repos using our CLA. 114 | 115 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 116 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 117 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 118 | 119 | ## Trademarks 120 | 121 | This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft 122 | trademarks or logos is subject to and must follow 123 | [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). 124 | Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. 125 | Any use of third-party trademarks or logos are subject to those third-party's policies. 126 | -------------------------------------------------------------------------------- /APIM.md: -------------------------------------------------------------------------------- 1 | # How to use Azure API Management (APIM) with Azure OpenAI (AOAI) 2 | 3 | To solve [the challenges](/README.md#challenges-of-azure-open-ai-in-production), we can use several APIM features. 4 | 5 | # APIs 6 | 7 | First, we need to add API definition for the AOAI. There are several features we recommend we to use while adding the API. 8 | 9 | ## Named values 10 | 11 | [Names values](https://learn.microsoft.com/azure/api-management/api-management-howto-properties?tabs=azure-portal) are the place where we can store secrets safely. It supports three value types. 12 | 13 | |Type |Description| 14 | |---|---| 15 | |Plain| Literal string or policy expression| 16 | |Secret| Literal string or policy expression that is encrypted by API Management| 17 | |Key vault| Identifier of a secret stored in an Azure key vault.| 18 | 19 | We use Managed Identity to access AOAI, so we don't need to store the AOAI keys in the Key Vault, but if you need to store the key, then store it in KeyVault and use Named Value to retrieve it. 20 | 21 | ## Backends 22 | 23 | We can use [backends](https://learn.microsoft.com/azure/api-management/backends?tabs=bicep) to manage the Logging Web App. Add ``aoai`` at the end as we use this part to create AOAI request. 24 | 25 | ![Backend](/assets//backend.png) 26 | 27 | ## APIs 28 | 29 | We can create API definition either by manual or import OpenAPI definition. We can find the Open API definition for each version of AOAI [here](https://github.com/Azure/azure-rest-api-specs/tree/main/specification/cognitiveservices/data-plane/AzureOpenAI/inference) 30 | 31 | Please note that we need to modify the endpoint address. 32 | 33 | See [Add an API manually](https://learn.microsoft.com/azure/api-management/add-api-manually) or [Import an OpenAPI specification](https://learn.microsoft.com/azure/api-management/import-api-from-oas?tabs=portal) for more detail how to add the API. 34 | 35 | Once we added APIs, we can edit the inbound policy to specify backend that we created. 36 | 37 | ```xml 38 | 39 | 40 | 41 | 42 | {{backend-url}} 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | ``` 57 | See [API Management policies overview](https://learn.microsoft.com/azure/api-management/api-management-howto-policies) for more detail about policies. 58 | 59 | We need to set API URL suffix as ``oepnai`` that makes the base address as ``https://.azure-api.net/openai`` 60 | 61 | ![API URL suffix](/assets/api_url_suffix.png) 62 | 63 | Then use the rest of the URL as the post method address. 64 | 65 | ![GPT-35 post settings](/assets/gpt_35_post_settings.png) 66 | 67 | With these settings, when the user access ``https://.azure-api.net/openai/deployments/gpt-35-turbo/chat/completions?api-version=2023-12-01-preview``, the request will be matched to this definition and will be redirected to ``/deployments/gpt-35-turbo/chat/completions?api-version=2023-12-01-previe`` because the backend contains ``openai`` as part of the base address. 68 | 69 | # Subscriptions 70 | 71 | We hide AOAI keys by using Managed Identity. We can give new key to end users, that is APIM subscription keys. 72 | 73 | [Subscriptions](https://learn.microsoft.com/azure/api-management/api-management-subscriptions) are the most common way for API consumers to access APIs published through an API Management instance. 74 | 75 | We can control the [scope of the subscriptions](https://learn.microsoft.com/azure/api-management/api-management-subscriptions#scope-of-subscriptions) so that each subscription may have different set of APIs to access. 76 | 77 | Name the subscription key so that it's easy to distinguish later. 78 | 79 | ## Subscription key name 80 | 81 | By default, we use ``Ocp-Apim-Subscription-Key`` as a header key name for the subscription keys. To make APIs compatible with AOAI, however, we need to change the key name to ``api-key`` that is same as AOAI. By doing this, we can simply replace the endpoint URL and the key of the existing applications to make them work. 82 | 83 | ![Subscription key name](/assets/subscription_key_name.png) 84 | 85 | ## Throttling by Token usage by subscription key 86 | 87 | If we want to set the throttling by token usage for each subscription key, we can use the [the GenAI gateway capability to throttle by token usage](https://learn.microsoft.com/en-us/azure/api-management/azure-openai-token-limit-policy). 88 | 89 | For example, we can add policy as below. 90 | 91 | ```xml 92 | 93 | ``` 94 | 95 | # Logging 96 | 97 | APIM provides out-of-box logging capabilities. See [How to integrate Azure API Management with Azure Application Insights](https://learn.microsoft.com/azure/api-management/api-management-howto-app-insights?tabs=rest) for detail setup. 98 | 99 | ## AOAI Out-of-box logging and its limitations 100 | 101 | AOAI provides basic logging, and we can use [Azure-OpenAI-Insights](https://github.com/dolevshor/Azure-OpenAI-Insights) and [Visualize data using Managed Grafana](https://learn.microsoft.com/azure/api-management/visualize-using-managed-grafana-dashboard) to visualize the log. 102 | 103 | However, these logging has some limitations. 104 | 105 | - It doesn't log actual request and response body. 106 | - When using streaming mode, it doesn't provide token usage information. See [How to stream completions](https://cookbook.openai.com/examples/how_to_stream_completions) for more detail. 107 | 108 | ## Custom Logging solution by using Web API proxy 109 | 110 | To solve these challenges, we can use Web API as proxy between APIM and AOAI that send logs to Cosmos DB. 111 | 112 | APIM has an Event Hub logger to send any information for request/response, then handles log information by ourselves, however it has several critical limitations. 113 | 114 | - The 200KB size limit for each log 115 | - When using AOAI streaming mode, APIM blocks it as it needs to capture all response before sending SSE to the client. 116 | 117 | See [Log events to Azure Event Hubs](https://learn.microsoft.com/azure/api-management/api-management-howto-log-event-hubs?tabs=PowerShell) and [Configure API for server-sent events](https://learn.microsoft.com/en-us/azure/api-management/how-to-server-sent-events) for more detail. 118 | 119 | This solution uses C# Web API as a proxy to overcome these limitations. We use following [policy fragment](https://learn.microsoft.com/azure/api-management/policy-fragments) to send the information to the proxy via headers. You can add/remove headers by yourself. 120 | 121 | [__inbound-logging__](/policies/inbound-logging.xml) 122 | ```xml 123 | 124 | 125 | @(context.Timestamp.ToString()) 126 | 127 | 128 | @(context.Subscription.Id.ToString()) 129 | 130 | 131 | @(context.Subscription.Name) 132 | 133 | 134 | @(context.Operation.Id.ToString()) 135 | 136 | 137 | @(context.Deployment.ServiceName) 138 | 139 | 140 | @(context.RequestId.ToString()) 141 | 142 | 143 | @(context.Request.IpAddress) 144 | 145 | 146 | @(context.Operation.Name) 147 | 148 | 149 | @(context.Deployment.Region) 150 | 151 | 152 | @(context.Api.Name) 153 | 154 | 155 | @(context.Api.Revision) 156 | 157 | 158 | @(context.Operation.Method) 159 | 160 | 161 | ``` 162 | Once the policy fragments are defined, we can use it in the API policy scope. We can set it on the top level of the API or in each operation. 163 | 164 | The ``forward-request`` is important to support SSE. 165 | 166 | ```xml 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | ``` 184 | 185 | When the [Logging Web API](/LoggingWebApi/) stores all the logs to Cosmos DB account, it sends the ``request id`` to Event Hub so that [Log Parser Function](/LogParserFunction/) is triggered. It transforms the various types of logs, such as Completion, Chat Completion, function callings, stream or non-stream, Embeddings results into identical format so that we can easily analyze the log. 186 | -------------------------------------------------------------------------------- /LogParserFunction/Extensions/StringExtensions.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | namespace LogParser.Extensions; 4 | 5 | /// 6 | /// String Extensions 7 | /// 8 | public static class StringExtensions 9 | { 10 | /// 11 | /// 12 | /// 13 | /// 14 | /// 15 | public static Request? GetRequest(this string body) 16 | { 17 | dynamic requestBodyNode = JsonConvert.DeserializeObject(body); 18 | 19 | if (requestBodyNode is null || 20 | requestBodyNode["request"] is null || 21 | string.IsNullOrEmpty(requestBodyNode["request"]!.ToString())) 22 | { 23 | return default; 24 | } 25 | 26 | if (body.Contains("/chat/completions?api-version")) 27 | { 28 | return JsonConvert.DeserializeObject( 29 | requestBodyNode!["request"].ToString()); 30 | } 31 | else if (body.Contains("/completions?api-version")) 32 | { 33 | return JsonConvert.DeserializeObject( 34 | requestBodyNode!["request"].ToString()); 35 | } 36 | else if (body.Contains("embeddings?api-version")) 37 | { 38 | return JsonConvert.DeserializeObject( 39 | requestBodyNode!["request"].ToString()); 40 | } 41 | 42 | return default; 43 | } 44 | 45 | public static async Task GetResponse( 46 | this string body, 47 | Request request, 48 | TikTokenService tikTokenService, 49 | ContentSafetyService contentSafetyService) 50 | { 51 | dynamic? responseBodyNode = JsonConvert.DeserializeObject(body); 52 | 53 | if (responseBodyNode is null || 54 | responseBodyNode["response"] is null || 55 | string.IsNullOrEmpty(responseBodyNode["response"]!.ToString())) 56 | { 57 | return default; 58 | } 59 | 60 | string responseBody = responseBodyNode["response"]!.ToString(); 61 | // Sream response consists of data list. 62 | if (request is ChatCompletionRequest) 63 | { 64 | if (responseBody.StartsWith("data")) 65 | { 66 | string responseBodyJson = responseBody 67 | .Replace("[DONE]\n\n", "]") 68 | .Replace("\n\ndata: ", ",") 69 | .Replace("data: ", "[") 70 | .Replace("},]", "}]"); 71 | 72 | ChatCompletionStreamResponse[]? streamResponses = 73 | JsonConvert.DeserializeObject(responseBodyJson); 74 | if (streamResponses is null || !streamResponses.Any()) 75 | { 76 | return default; 77 | } 78 | 79 | // Recreate the assistant response by combining all the stream delta contents. 80 | string concatenatedContent = string.Join("", 81 | streamResponses.Select(x => x.Choices.FirstOrDefault()?.Delta.Content).ToList()); 82 | string requestContent = string.Join(Environment.NewLine, 83 | ((ChatCompletionRequest)request)!.Messages.Select(x => x.Content)); 84 | // Check the content safety 85 | ContentFilterResults responseContentFilterResults = string.IsNullOrEmpty(concatenatedContent) ? 86 | new() : 87 | await contentSafetyService.GetContentFilterResultsAsync(concatenatedContent); 88 | ContentFilterResults requestContentFilterResults = 89 | await contentSafetyService.GetContentFilterResultsAsync(requestContent); 90 | await contentSafetyService.GetContentFilterResultsAsync(requestContent); 91 | // Count the tokens for request 92 | //TODO: Need additional calculation for vision https://platform.openai.com/docs/guides/vision 93 | // promtTokens adjusted by adding 11 tokens to match the token count of the non stream mode. 94 | int promptTokens = await tikTokenService.CountToken(((ChatCompletionRequest)request).Messages) + 11; 95 | // comletionTokens uses the token count of the concatenated content to match the token count of the non stream mode. 96 | int comletionTokens = tikTokenService.CountToken(concatenatedContent); 97 | ChatCompletionStreamResponse streamResponse = streamResponses.Where(x => !string.IsNullOrEmpty(x.Id)).First(); 98 | 99 | // Create idential response format with non-stream response. 100 | return new ChatCompletionResponse() 101 | { 102 | Id = streamResponse.Id, 103 | Object = streamResponse.Object, 104 | Model = streamResponse.Model, 105 | Created = streamResponse.Created, 106 | Choices = new() 107 | { 108 | new() 109 | { 110 | FinishReason = streamResponses.Last().Choices.First().FinishReason, 111 | Index = 0, 112 | Message = new() 113 | { 114 | Role = "assistant", 115 | Content = concatenatedContent 116 | }, 117 | ContentFilterResults = responseContentFilterResults 118 | } 119 | }, 120 | Usage = new() 121 | { 122 | PromptTokens = promptTokens, 123 | CompletionTokens = comletionTokens, 124 | TotalTokens = promptTokens + comletionTokens, 125 | }, 126 | PromptFilterResults = new() 127 | { 128 | new() 129 | { 130 | PromptIndex = 0, 131 | ContentFilterResults = requestContentFilterResults 132 | } 133 | } 134 | }; 135 | } 136 | else 137 | { 138 | return JsonConvert.DeserializeObject(responseBody)!; 139 | } 140 | } 141 | else if (request is CompletionRequest) 142 | { 143 | if (responseBody.StartsWith("data")) 144 | { 145 | string responseBodyJson = responseBody 146 | .Replace("[DONE]\n\n", "]") 147 | .Replace("\n\ndata: ", ",") 148 | .Replace("data: ", "[") 149 | .Replace("},]", "}]"); 150 | 151 | CompletionResponse[]? streamResponses = JsonConvert.DeserializeObject(responseBodyJson); 152 | if (streamResponses is null || !streamResponses.Any()) 153 | { 154 | return default; 155 | } 156 | 157 | // Recreate the assistant response by combining all the stream delta contents. 158 | string concatenatedContent = string.Join("", 159 | streamResponses.Select(x => x.Choices.FirstOrDefault()?.Text).ToList()); 160 | string requestContent = string.Join(Environment.NewLine, ((CompletionRequest)request).Prompt); 161 | // Check the content safety 162 | ContentFilterResults responseContentFilterResults = string.IsNullOrEmpty(concatenatedContent) ? 163 | new(): 164 | await contentSafetyService.GetContentFilterResultsAsync(concatenatedContent); 165 | ContentFilterResults requestContentFilterResults = 166 | await contentSafetyService.GetContentFilterResultsAsync(requestContent); 167 | // Count the tokens for request 168 | int promptTokens = tikTokenService.CountToken(requestContent) + 11; 169 | int comletionTokens = tikTokenService.CountToken(concatenatedContent); 170 | CompletionResponse streamResponse = streamResponses.Where(x => !string.IsNullOrEmpty(x.Id)).First(); 171 | 172 | // Create idential response format with non-stream response. 173 | return new CompletionResponse() 174 | { 175 | Id = streamResponse.Id, 176 | Object = streamResponse.Object, 177 | Model = streamResponse.Model, 178 | Created = streamResponse.Created, 179 | Choices = new() 180 | { 181 | new() 182 | { 183 | FinishReason = streamResponses.Last().Choices.First().FinishReason, 184 | Index = 0, 185 | Text = concatenatedContent, 186 | ContentFilterResults = responseContentFilterResults 187 | } 188 | }, 189 | Usage = new() 190 | { 191 | PromptTokens = promptTokens, 192 | CompletionTokens = comletionTokens, 193 | TotalTokens = promptTokens + comletionTokens, 194 | }, 195 | PromptFilterResults = new() 196 | { 197 | new() 198 | { 199 | PromptIndex = 0, 200 | ContentFilterResults = requestContentFilterResults 201 | } 202 | } 203 | }; 204 | } 205 | else 206 | { 207 | return JsonConvert.DeserializeObject(responseBody)!; 208 | } 209 | } 210 | else if (request is EmbeddingRequest) 211 | { 212 | return JsonConvert.DeserializeObject(responseBody)!; 213 | } 214 | 215 | return default; 216 | } 217 | } 218 | -------------------------------------------------------------------------------- /assets/aoai_apim.drawio: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | -------------------------------------------------------------------------------- /infra/main.bicep: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft. All rights reserved. 2 | 3 | targetScope = 'subscription' 4 | var abbrs = loadJsonContent('./abbreviations.json') 5 | 6 | @minLength(1) 7 | @maxLength(64) 8 | @description('Name of the the environment which is used to generate a short unique hash used in all resources.') 9 | param environmentName string 10 | 11 | @description('Azure API Management Publisher Email') 12 | param publisherEmail string = 'your_email_address@your.domain' 13 | 14 | @description('Azure API Management Publisher Name') 15 | param publisherName string = 'Your Name' 16 | 17 | @description('Azure location for all resources.') 18 | param location string 19 | 20 | @description('Resource Group Name') 21 | param resourceGroupName string = '' 22 | 23 | @description('Azure Open AI Name') 24 | param aoaiName string = '' 25 | 26 | @description('Azure Open AI Token Limit TPM. default is 1000') 27 | param tokenLimitTPM int = 10000 28 | 29 | @description('Application Insights Name') 30 | param applicationInsightsName string = '' 31 | 32 | @description('Application Insights Workspace Name') 33 | param workspaceName string = '' 34 | 35 | @description('Key Vault Name') 36 | param keyVaultName string = '' 37 | 38 | @description('Key Vault Sku') 39 | @allowed([ 40 | 'standard' 41 | 'premium' 42 | ]) 43 | param keyVaultSku string = 'standard' 44 | 45 | @description('Azure API Management Name') 46 | param apiManagementServiceName string = '' 47 | 48 | @description('Cosmos Db Account Name') 49 | param cosmosDbAccountName string = '' 50 | 51 | @description('Cosmos Db Database Name') 52 | param cosmosDbDatabaseName string = 'Logs' 53 | 54 | @description('Cosmos Db Log Container Name') 55 | param cosmosDbLogContainerName string = 'TempLogs' 56 | 57 | @description('Cosmos Db Trigger Container Name') 58 | param cosmosDbTriggerContainerName string = 'LogTriggers' 59 | 60 | @description('Content Safety Account Name') 61 | param contentSafetyAccountName string = '' 62 | 63 | @description('Logging Web App Name') 64 | param loggingWebApiName string = '' 65 | 66 | @description('Log Parser Function App Name') 67 | param logParserFunctionName string = '' 68 | 69 | @description('Log Parser Function Storage Account Name') 70 | param functionStorageAccountName string = '' 71 | 72 | @description('App Service Name for Logging and Log Parser') 73 | param appServiceName string = '' 74 | 75 | @description('Public IP name for Azure API Management') 76 | param publicIpName string = '' 77 | 78 | @description('Azure API Management Sku') 79 | @allowed([ 80 | 'Developer' 81 | 'BasicV2' 82 | 'StandardV2' 83 | 'Premium' 84 | ]) 85 | param apimSku string = 'StandardV2' 86 | 87 | @description('Azure API Management Sku Count') 88 | @allowed([ 89 | 0 90 | 1 91 | 2 92 | ]) 93 | param skuCount int = 1 94 | 95 | @description('Virtual Network Name') 96 | param vnetName string = '' 97 | 98 | @description('Virtual Network Subnet Name for APIM') 99 | param apimSubnetName string = '' 100 | 101 | @description('Virtual Network Subnet Name for Private Endpoints') 102 | param pepSubnetName string = '' 103 | 104 | @description('Virtual Network Subnet Name for Web App') 105 | param webAppSubnetName string = '' 106 | 107 | @description('Network Security Group Name for APIM') 108 | param apimNsgName string = '' 109 | 110 | @description('Private Endpoint Name for Application Insights') 111 | param applicationInsightsPrivateEndpointName string = '' 112 | 113 | @description('Private Endpoint Name for Key Vault') 114 | param keyVaultPrivateEndpointName string = '' 115 | 116 | @description('Private Endpoint Name for Azure Open AI') 117 | param aoaiPrivateEndpointName string = '' 118 | 119 | @description('Private Endpoint Name for Cosmos Db') 120 | param cosmosDbPrivateEndpointName string = '' 121 | 122 | @description('Private Endpoint Name for Logging Web App') 123 | param loggingWebApiPrivateEndpointName string = '' 124 | 125 | @description('Private Endpoint Name for Log Parser Function App') 126 | param logParserFunctionPrivateEndpointName string = '' 127 | 128 | @description('Private Endpoint Name for Content Safety') 129 | param contentSafetyPrivateEndpointName string = '' 130 | 131 | var privateDnsZoneNames = [ 132 | 'privatelink.openai.azure.com' 133 | 'privatelink.vaultcore.azure.net' 134 | 'privatelink.monitor.azure.com' 135 | 'privatelink.documents.azure.com' 136 | 'privatelink.azurewebsites.net' 137 | 'privatelink.cognitiveservices.azure.com' 138 | ] 139 | 140 | var deployments = [ 141 | { 142 | name: 'Embedding' 143 | displayName: 'Embedding' 144 | description: 'Embedding' 145 | method: 'POST' 146 | urlTemplate: '/deployments/{deployment-id}/embeddings?api-version={api-version}' 147 | backend: aoaiName 148 | modelName: 'text-embedding-ada-002' 149 | deploymentName: 'text-embedding-ada-002' 150 | version: '2' 151 | capacity: 10 152 | skuName:'Standard' 153 | } 154 | { 155 | name: 'Completon' 156 | displayName: 'GPT Completion' 157 | description: 'GPT Completion' 158 | method: 'POST' 159 | urlTemplate: '/deployments/{deployment-id}/completions?api-version={api-version}' 160 | backend: aoaiName 161 | modelName: 'gpt-35-turbo-instruct' 162 | deploymentName: 'gpt-35-turbo-instruct' 163 | version: '0914' 164 | capacity: 10 165 | skuName:'Standard' 166 | } 167 | { 168 | name: 'ChatCompleton' 169 | displayName: 'Chat Completion' 170 | description: 'Chat Completion' 171 | method: 'POST' 172 | urlTemplate: '/deployments/{deployment-id}/chat/completions?api-version={api-version}' 173 | backend: aoaiName 174 | modelName: 'gpt-4o' 175 | deploymentName: 'gpt-4o' 176 | version: '2024-05-13' 177 | capacity: 10 178 | skuName:'GlobalStandard' 179 | } 180 | ] 181 | 182 | // Organize resources in a resource group 183 | resource rg 'Microsoft.Resources/resourceGroups@2021-04-01' = { 184 | name: !empty(resourceGroupName) ? resourceGroupName : '${abbrs.resourcesResourceGroups}${environmentName}' 185 | location: location 186 | } 187 | 188 | //## Create Dns for Private Endpoint ## 189 | module dns './network/dns.bicep' = { 190 | scope: rg 191 | name: 'dnsDeployment' 192 | params: { 193 | privateDnsZoneNames:privateDnsZoneNames 194 | } 195 | } 196 | 197 | //## Create VNet ## 198 | module vnet './network/vnet.bicep' = { 199 | scope: rg 200 | name: 'vnetDeployment' 201 | params: { 202 | location: location 203 | vnetName: !empty(vnetName) ? vnetName : '${abbrs.networkVirtualNetworks}${environmentName}' 204 | apimSubnetName: !empty(apimSubnetName) ? apimSubnetName : '${abbrs.networkVirtualNetworksSubnets}${abbrs.apiManagementService}${environmentName}' 205 | pepSubnetName: !empty(pepSubnetName) ? pepSubnetName : '${abbrs.networkVirtualNetworksSubnets}${abbrs.networkPrivateLinkServices}${environmentName}' 206 | webAppSubnetName: !empty(webAppSubnetName) ? webAppSubnetName : '${abbrs.networkVirtualNetworksSubnets}${abbrs.webSitesAppService}${environmentName}' 207 | apimNsgName: !empty(apimNsgName) ? apimNsgName : '${abbrs.networkNetworkSecurityGroups}${abbrs.apiManagementService}${environmentName}' 208 | privateDnsZoneNames: privateDnsZoneNames 209 | } 210 | dependsOn: [ 211 | dns 212 | ] 213 | } 214 | 215 | //## Create Key Vault that stores Keys ## 216 | module keyVault './security/keyVault.bicep' = { 217 | scope: rg 218 | name: 'keyVaultDeployment' 219 | params: { 220 | location: location 221 | keyVaultName: !empty(keyVaultName) ? keyVaultName : '${abbrs.keyVaultVaults}${environmentName}' 222 | skuName: keyVaultSku 223 | vnetName: !empty(vnetName) ? vnetName : '${abbrs.networkVirtualNetworks}${environmentName}' 224 | pepSubnetName: !empty(pepSubnetName) ? pepSubnetName : '${abbrs.networkVirtualNetworksSubnets}${abbrs.networkPrivateLinkServices}${environmentName}' 225 | keyVaultPrivateEndpointName: !empty(keyVaultPrivateEndpointName) ? keyVaultPrivateEndpointName : '${abbrs.networkPrivateLinkServices}${abbrs.keyVaultVaults}${environmentName}' 226 | } 227 | dependsOn: [ 228 | vnet 229 | dns 230 | ] 231 | } 232 | 233 | //## Create Cosmos Db ## 234 | module cosmosDb './db/cosmosDb.bicep' = { 235 | scope: rg 236 | name: 'cosmosDbDeployment' 237 | params: { 238 | location: location 239 | accountName: !empty(cosmosDbAccountName) ? cosmosDbAccountName : '${abbrs.documentDBDatabaseAccounts}${environmentName}' 240 | databaseName: cosmosDbDatabaseName 241 | logContainerName: cosmosDbLogContainerName 242 | triggerContainerName: cosmosDbTriggerContainerName 243 | primaryRegion: location 244 | keyVaultName: !empty(keyVaultName) ? keyVaultName : '${abbrs.keyVaultVaults}${environmentName}' 245 | vnetName: !empty(vnetName) ? vnetName : '${abbrs.networkVirtualNetworks}${environmentName}' 246 | pepSubnetName: !empty(pepSubnetName) ? pepSubnetName : '${abbrs.networkVirtualNetworksSubnets}${abbrs.networkPrivateLinkServices}${environmentName}' 247 | privateEndpointName: !empty(cosmosDbPrivateEndpointName) ? cosmosDbPrivateEndpointName : '${abbrs.networkPrivateLinkServices}${abbrs.documentDBDatabaseAccounts}${environmentName}' 248 | } 249 | dependsOn:[ 250 | vnet 251 | dns 252 | keyVault 253 | ] 254 | } 255 | 256 | //## Create Application Insights ## 257 | module applicationInsights './monitoring/applicationInsights.bicep' = { 258 | scope: rg 259 | name: 'applicationInsightsDeployment' 260 | params: { 261 | location: location 262 | applicationInsightsName: !empty(applicationInsightsName) ? applicationInsightsName : '${abbrs.insightsComponents}${environmentName}' 263 | workspaceName: !empty(workspaceName) ? workspaceName : '${abbrs.operationalInsightsWorkspaces}${environmentName}' 264 | keyVaultName: !empty(keyVaultName) ? keyVaultName : '${abbrs.keyVaultVaults}${environmentName}' 265 | vnetName: !empty(vnetName) ? vnetName : '${abbrs.networkVirtualNetworks}${environmentName}' 266 | pepSubnetName: !empty(pepSubnetName) ? pepSubnetName : '${abbrs.networkVirtualNetworksSubnets}${abbrs.networkPrivateLinkServices}${environmentName}' 267 | privateEndpointName: !empty(applicationInsightsPrivateEndpointName) ? applicationInsightsPrivateEndpointName : '${abbrs.networkPrivateLinkServices}${abbrs.insightsComponents}${environmentName}' 268 | } 269 | dependsOn: [ 270 | vnet 271 | dns 272 | keyVault 273 | ] 274 | } 275 | 276 | //## Create Application Insights ## 277 | module contentsafety './ai/contentSafety.bicep' = { 278 | scope: rg 279 | name: 'contentsafetyDeployment' 280 | params: { 281 | location: location 282 | contentSafetyAccountName: !empty(contentSafetyAccountName) ? contentSafetyAccountName : '${abbrs.cognitiveServicesContentSafety}${environmentName}' 283 | keyVaultName: !empty(keyVaultName) ? keyVaultName : '${abbrs.keyVaultVaults}${environmentName}' 284 | vnetName: !empty(vnetName) ? vnetName : '${abbrs.networkVirtualNetworks}${environmentName}' 285 | pepSubnetName: !empty(pepSubnetName) ? pepSubnetName : '${abbrs.networkVirtualNetworksSubnets}${abbrs.networkPrivateLinkServices}${environmentName}' 286 | privateEndpointName: !empty(contentSafetyPrivateEndpointName) ? contentSafetyPrivateEndpointName : '${abbrs.networkPrivateLinkServices}${abbrs.cognitiveServicesContentSafety}${environmentName}' 287 | } 288 | dependsOn: [ 289 | vnet 290 | dns 291 | keyVault 292 | ] 293 | } 294 | 295 | //## Create Azure Open AI and stores the key to the Key Vault ## 296 | module aoai './ai/aoai.bicep' = { 297 | scope: rg 298 | name: 'aoaiDeployment' 299 | params: { 300 | location: location 301 | aoaiName: !empty(aoaiName) ? aoaiName : '${abbrs.cognitiveServicesOpenAi}${environmentName}' 302 | vnetName: !empty(vnetName) ? vnetName : '${abbrs.networkVirtualNetworks}${environmentName}' 303 | pepSubnetName: !empty(pepSubnetName) ? pepSubnetName : '${abbrs.networkVirtualNetworksSubnets}${abbrs.networkPrivateLinkServices}${environmentName}' 304 | privateEndpointName: !empty(aoaiPrivateEndpointName) ? aoaiPrivateEndpointName : '${abbrs.networkPrivateLinkServices}${abbrs.cognitiveServicesOpenAi}${environmentName}' 305 | deployments: deployments 306 | } 307 | dependsOn: [ 308 | keyVault 309 | vnet 310 | dns 311 | ] 312 | } 313 | 314 | module loggingWebApi './webapp/webApi.bicep' = { 315 | scope: rg 316 | name: 'webApiDeployment' 317 | params: { 318 | location: location 319 | loggingWebApiName: !empty(loggingWebApiName) ? loggingWebApiName : '${abbrs.webSitesAppService}loggingweb-${environmentName}' 320 | appServicePlanName: !empty(appServiceName) ? appServiceName : '${abbrs.webServerFarms}${environmentName}' 321 | sku: 'S1' 322 | keyVaultName: !empty(keyVaultName) ? keyVaultName : '${abbrs.keyVaultVaults}${environmentName}' 323 | applicationInsightsName: !empty(applicationInsightsName) ? applicationInsightsName : '${abbrs.insightsComponents}${environmentName}' 324 | cosmosDbAccountName: !empty(cosmosDbAccountName) ? cosmosDbAccountName : '${abbrs.documentDBDatabaseAccounts}${environmentName}' 325 | cosmosDbDatabaseName: cosmosDbDatabaseName 326 | cosmosDbLogContainerName: cosmosDbLogContainerName 327 | cosmosDbTriggerContainerName: cosmosDbTriggerContainerName 328 | vnetName: !empty(vnetName) ? vnetName : '${abbrs.networkVirtualNetworks}${environmentName}' 329 | webAppSubnetName: !empty(webAppSubnetName) ? webAppSubnetName : '${abbrs.networkVirtualNetworksSubnets}${abbrs.webSitesAppService}${environmentName}' 330 | pepSubnetName: !empty(pepSubnetName) ? pepSubnetName : '${abbrs.networkVirtualNetworksSubnets}${abbrs.networkPrivateLinkServices}${environmentName}' 331 | loggingWebApiPrivateEndpointName: !empty(loggingWebApiPrivateEndpointName) ? loggingWebApiPrivateEndpointName : '${abbrs.networkPrivateLinkServices}${abbrs.webSitesAppService}loggingweb-${environmentName}' 332 | } 333 | dependsOn: [ 334 | vnet 335 | dns 336 | applicationInsights 337 | cosmosDb 338 | keyVault 339 | contentsafety 340 | ] 341 | } 342 | 343 | module logParserFunction './webapp/function.bicep' = { 344 | scope: rg 345 | name: 'webAppDeployment' 346 | params: { 347 | location: location 348 | logParserFunctionName: !empty(logParserFunctionName) ? logParserFunctionName : '${abbrs.webSitesFunctions}logparser-${environmentName}' 349 | appServicePlanName: !empty(appServiceName) ? appServiceName : '${abbrs.webServerFarms}${environmentName}' 350 | sku: 'S1' 351 | functionStorageAccountName: !empty(functionStorageAccountName) ? functionStorageAccountName : '${abbrs.storageStorageAccounts}logparser${environmentName}' 352 | functionStorageAccountType: 'Standard_LRS' 353 | keyVaultName: !empty(keyVaultName) ? keyVaultName : '${abbrs.keyVaultVaults}${environmentName}' 354 | applicationInsightsName: !empty(applicationInsightsName) ? applicationInsightsName : '${abbrs.insightsComponents}${environmentName}' 355 | cosmosDbAccountName: !empty(cosmosDbAccountName) ? cosmosDbAccountName : '${abbrs.documentDBDatabaseAccounts}${environmentName}' 356 | cosmosDbDatabaseName: cosmosDbDatabaseName 357 | cosmosDbLogContainerName: cosmosDbLogContainerName 358 | cosmosDbTriggerContainerName: cosmosDbTriggerContainerName 359 | contentSafetyAccountName: !empty(contentSafetyAccountName) ? contentSafetyAccountName : '${abbrs.cognitiveServicesContentSafety}${environmentName}' 360 | vnetName: !empty(vnetName) ? vnetName : '${abbrs.networkVirtualNetworks}${environmentName}' 361 | webAppSubnetName: !empty(webAppSubnetName) ? webAppSubnetName : '${abbrs.networkVirtualNetworksSubnets}${abbrs.webSitesAppService}${environmentName}' 362 | pepSubnetName: !empty(pepSubnetName) ? pepSubnetName : '${abbrs.networkVirtualNetworksSubnets}${abbrs.networkPrivateLinkServices}${environmentName}' 363 | logParserFunctionPrivateEndpointName: !empty(logParserFunctionPrivateEndpointName) ? logParserFunctionPrivateEndpointName : '${abbrs.networkPrivateLinkServices}${abbrs.webSitesFunctions}logparser-${environmentName}' 364 | } 365 | dependsOn: [ 366 | vnet 367 | dns 368 | applicationInsights 369 | cosmosDb 370 | keyVault 371 | contentsafety 372 | ] 373 | } 374 | 375 | module publicIp './network/publicIp.bicep' = { 376 | scope: rg 377 | name: 'publicIpDeployment' 378 | params: { 379 | location: location 380 | publicIpName: !empty(publicIpName) ? publicIpName : '${abbrs.networkPublicIPAddresses}${environmentName}' 381 | } 382 | } 383 | 384 | //## Create API Management ## 385 | module apim './apim/apim.bicep' = { 386 | scope: rg 387 | name: 'apimDeployment' 388 | params: { 389 | location: location 390 | apiManagementServiceName: !empty(apiManagementServiceName) ? apiManagementServiceName : '${abbrs.apiManagementService}${environmentName}' 391 | publisherEmail: publisherEmail 392 | publisherName: publisherName 393 | sku: apimSku 394 | skuCount: skuCount 395 | publicIpName: !empty(publicIpName) ? publicIpName : '${abbrs.networkPublicIPAddresses}${environmentName}' 396 | vnetName: !empty(vnetName) ? vnetName : '${abbrs.networkVirtualNetworks}${environmentName}' 397 | apimSubnetName: !empty(apimSubnetName) ? apimSubnetName : '${abbrs.networkVirtualNetworksSubnets}${abbrs.apiManagementService}${environmentName}' 398 | } 399 | dependsOn: [ 400 | vnet 401 | dns 402 | loggingWebApi 403 | logParserFunction 404 | ] 405 | } 406 | 407 | //## Assign API Managemnet Managed Identity to appropriate roles ## 408 | module roles './security/roles.bicep' = { 409 | scope: rg 410 | name: 'rolesDeployment' 411 | params: { 412 | keyVaultName: !empty(keyVaultName) ? keyVaultName : '${abbrs.keyVaultVaults}${environmentName}' 413 | aoaiName: !empty(aoaiName) ? aoaiName : '${abbrs.cognitiveServicesOpenAi}${environmentName}' 414 | loggingWebApiIdentityId: loggingWebApi.outputs.loggingWebApiIdentityId 415 | logParserFunctionIdentityId: logParserFunction.outputs.logParserFunctionIdentityId 416 | } 417 | dependsOn: [ 418 | apim 419 | keyVault 420 | loggingWebApi 421 | logParserFunction 422 | ] 423 | } 424 | 425 | //## Link the Application Insights to API Management ## 426 | module apimApplicationInsights './apim/apimApplicationInsights.bicep' = { 427 | scope: rg 428 | name: 'apimApplicationInsightsDeployment' 429 | params: { 430 | apiManagementServiceName: !empty(apiManagementServiceName) ? apiManagementServiceName : '${abbrs.apiManagementService}${environmentName}' 431 | applicationInsightsName: !empty(applicationInsightsName) ? applicationInsightsName : '${abbrs.insightsComponents}${environmentName}' 432 | } 433 | dependsOn: [ 434 | apim 435 | applicationInsights 436 | ] 437 | } 438 | 439 | //## Create API Management Policy for APIs ## 440 | module apimPolicyFragment './apim/apimPolicyFragment.bicep' = { 441 | scope: rg 442 | name: 'apimPolicyFragmentDeployment' 443 | params: { 444 | apiManagementServiceName: !empty(apiManagementServiceName) ? apiManagementServiceName : '${abbrs.apiManagementService}${environmentName}' 445 | } 446 | dependsOn: [ 447 | apim 448 | ] 449 | } 450 | 451 | //## Create Named Value ad Backed to store Azure Open AI Inforamtion ## 452 | module apimBackend './apim/apimBackend.bicep' = { 453 | scope: rg 454 | name: 'apimBackendDeployment' 455 | params: { 456 | apiManagementServiceName: !empty(apiManagementServiceName) ? apiManagementServiceName : '${abbrs.apiManagementService}${environmentName}' 457 | aoaiName: !empty(aoaiName) ? aoaiName : '${abbrs.cognitiveServicesOpenAi}${environmentName}' 458 | loggingWebApiName: !empty(loggingWebApiName) ? loggingWebApiName : '${abbrs.webSitesAppService}loggingweb-${environmentName}' 459 | } 460 | dependsOn: [ 461 | aoai 462 | apim 463 | keyVault 464 | roles 465 | loggingWebApi 466 | logParserFunction 467 | ] 468 | } 469 | 470 | //## Create API Management API and Operations ## 471 | module apimApis './apim/apimApis.bicep' = { 472 | scope: rg 473 | name: 'apimApisDeployment' 474 | params: { 475 | apiManagementServiceName: !empty(apiManagementServiceName) ? apiManagementServiceName : '${abbrs.apiManagementService}${environmentName}' 476 | aoaiName: !empty(aoaiName) ? aoaiName : '${abbrs.cognitiveServicesOpenAi}${environmentName}' 477 | applicationInsightsName: !empty(applicationInsightsName) ? applicationInsightsName : '${abbrs.insightsComponents}${environmentName}' 478 | loggingWebApiName: !empty(loggingWebApiName) ? loggingWebApiName : '${abbrs.webSitesAppService}loggingweb-${environmentName}' 479 | deployments: deployments 480 | tokenLimitTPM: tokenLimitTPM 481 | } 482 | dependsOn: [ 483 | apim 484 | aoai 485 | apimBackend 486 | loggingWebApi 487 | logParserFunction 488 | ] 489 | } 490 | --------------------------------------------------------------------------------