├── Notes ├── Assets │ ├── TextPad.ico │ ├── StoreLogo.backup.png │ ├── LargeTile.scale-100.png │ ├── LargeTile.scale-125.png │ ├── LargeTile.scale-150.png │ ├── LargeTile.scale-200.png │ ├── LargeTile.scale-400.png │ ├── SmallTile.scale-100.png │ ├── SmallTile.scale-125.png │ ├── SmallTile.scale-150.png │ ├── SmallTile.scale-200.png │ ├── SmallTile.scale-400.png │ ├── StoreLogo.scale-100.png │ ├── StoreLogo.scale-125.png │ ├── StoreLogo.scale-150.png │ ├── StoreLogo.scale-200.png │ ├── StoreLogo.scale-400.png │ ├── LockScreenLogo.scale-200.png │ ├── SplashScreen.scale-100.png │ ├── SplashScreen.scale-125.png │ ├── SplashScreen.scale-150.png │ ├── SplashScreen.scale-200.png │ ├── SplashScreen.scale-400.png │ ├── Square44x44Logo.scale-100.png │ ├── Square44x44Logo.scale-125.png │ ├── Square44x44Logo.scale-150.png │ ├── Square44x44Logo.scale-200.png │ ├── Square44x44Logo.scale-400.png │ ├── Wide310x150Logo.scale-100.png │ ├── Wide310x150Logo.scale-125.png │ ├── Wide310x150Logo.scale-150.png │ ├── Wide310x150Logo.scale-200.png │ ├── Wide310x150Logo.scale-400.png │ ├── Square150x150Logo.scale-100.png │ ├── Square150x150Logo.scale-125.png │ ├── Square150x150Logo.scale-150.png │ ├── Square150x150Logo.scale-200.png │ ├── Square150x150Logo.scale-400.png │ ├── Square44x44Logo.targetsize-16.png │ ├── Square44x44Logo.targetsize-24.png │ ├── Square44x44Logo.targetsize-32.png │ ├── Square44x44Logo.targetsize-48.png │ ├── Square44x44Logo.targetsize-256.png │ ├── Square44x44Logo.altform-unplated_targetsize-16.png │ ├── Square44x44Logo.altform-unplated_targetsize-32.png │ ├── Square44x44Logo.altform-unplated_targetsize-48.png │ ├── Square44x44Logo.targetsize-24_altform-unplated.png │ ├── Square44x44Logo.altform-unplated_targetsize-256.png │ ├── Square44x44Logo.altform-lightunplated_targetsize-16.png │ ├── Square44x44Logo.altform-lightunplated_targetsize-24.png │ ├── Square44x44Logo.altform-lightunplated_targetsize-32.png │ ├── Square44x44Logo.altform-lightunplated_targetsize-48.png │ └── Square44x44Logo.altform-lightunplated_targetsize-256.png ├── AI │ ├── Embeddings │ │ ├── IVectorObject.cs │ │ ├── TextChunk.cs │ │ └── SemanticIndex.cs │ ├── IChatClient │ │ ├── LlmPromptTemplate.cs │ │ ├── IChatClientExtensions.cs │ │ ├── PhiSilicaClient.cs │ │ └── GenAIModel.cs │ ├── TextRecognition │ │ ├── ImageText.cs │ │ └── TextRecognition.cs │ └── Whisper │ │ ├── Whisper.cs │ │ ├── VoiceActivity │ │ ├── SlieroVadDetector.cs │ │ ├── SlieroVadOnnxModel.cs │ │ └── WhisperChunking.cs │ │ └── WhisperUtils.cs ├── Properties │ └── launchSettings.json ├── Models │ ├── Note.cs │ ├── TranscriptionBlock.cs │ └── Attachment.cs ├── App.xaml ├── Themes │ └── Generic.xaml ├── app.manifest ├── Utils │ ├── Utils.Rag.cs │ ├── AppDataContext.cs │ ├── Utils.DX.cs │ ├── WaveformRenderer.cs │ ├── Utils.Audio.cs │ ├── Utils.cs │ └── AttachmentProcessor.cs ├── ViewModels │ ├── AttachmentViewModel.cs │ ├── SearchViewModel.cs │ ├── ViewModel.cs │ └── NoteViewModel.cs ├── Package.appxmanifest ├── App.xaml.cs ├── Controls │ ├── SearchView.xaml.cs │ ├── SearchView.xaml │ ├── AttachmentView.xaml │ ├── Phi3View.xaml │ ├── Phi3View.xaml.cs │ └── AttachmentView.xaml.cs ├── MainWindow.xaml.cs ├── Notes.csproj └── MainWindow.xaml ├── SUPPORT.md ├── CODE_OF_CONDUCT.md ├── LICENSE ├── Notes.sln ├── SECURITY.md ├── README.md └── .gitignore /Notes/Assets/TextPad.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/TextPad.ico -------------------------------------------------------------------------------- /Notes/Assets/StoreLogo.backup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/StoreLogo.backup.png -------------------------------------------------------------------------------- /Notes/Assets/LargeTile.scale-100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/LargeTile.scale-100.png -------------------------------------------------------------------------------- /Notes/Assets/LargeTile.scale-125.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/LargeTile.scale-125.png -------------------------------------------------------------------------------- /Notes/Assets/LargeTile.scale-150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/LargeTile.scale-150.png -------------------------------------------------------------------------------- /Notes/Assets/LargeTile.scale-200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/LargeTile.scale-200.png -------------------------------------------------------------------------------- /Notes/Assets/LargeTile.scale-400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/LargeTile.scale-400.png -------------------------------------------------------------------------------- /Notes/Assets/SmallTile.scale-100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/SmallTile.scale-100.png -------------------------------------------------------------------------------- /Notes/Assets/SmallTile.scale-125.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/SmallTile.scale-125.png -------------------------------------------------------------------------------- /Notes/Assets/SmallTile.scale-150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/SmallTile.scale-150.png -------------------------------------------------------------------------------- /Notes/Assets/SmallTile.scale-200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/SmallTile.scale-200.png -------------------------------------------------------------------------------- /Notes/Assets/SmallTile.scale-400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/SmallTile.scale-400.png -------------------------------------------------------------------------------- /Notes/Assets/StoreLogo.scale-100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/StoreLogo.scale-100.png -------------------------------------------------------------------------------- /Notes/Assets/StoreLogo.scale-125.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/StoreLogo.scale-125.png -------------------------------------------------------------------------------- /Notes/Assets/StoreLogo.scale-150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/StoreLogo.scale-150.png -------------------------------------------------------------------------------- /Notes/Assets/StoreLogo.scale-200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/StoreLogo.scale-200.png -------------------------------------------------------------------------------- /Notes/Assets/StoreLogo.scale-400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/StoreLogo.scale-400.png -------------------------------------------------------------------------------- /Notes/Assets/LockScreenLogo.scale-200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/LockScreenLogo.scale-200.png -------------------------------------------------------------------------------- /Notes/Assets/SplashScreen.scale-100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/SplashScreen.scale-100.png -------------------------------------------------------------------------------- /Notes/Assets/SplashScreen.scale-125.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/SplashScreen.scale-125.png -------------------------------------------------------------------------------- /Notes/Assets/SplashScreen.scale-150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/SplashScreen.scale-150.png -------------------------------------------------------------------------------- /Notes/Assets/SplashScreen.scale-200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/SplashScreen.scale-200.png -------------------------------------------------------------------------------- /Notes/Assets/SplashScreen.scale-400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/SplashScreen.scale-400.png -------------------------------------------------------------------------------- /Notes/Assets/Square44x44Logo.scale-100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square44x44Logo.scale-100.png -------------------------------------------------------------------------------- /Notes/Assets/Square44x44Logo.scale-125.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square44x44Logo.scale-125.png -------------------------------------------------------------------------------- /Notes/Assets/Square44x44Logo.scale-150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square44x44Logo.scale-150.png -------------------------------------------------------------------------------- /Notes/Assets/Square44x44Logo.scale-200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square44x44Logo.scale-200.png -------------------------------------------------------------------------------- /Notes/Assets/Square44x44Logo.scale-400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square44x44Logo.scale-400.png -------------------------------------------------------------------------------- /Notes/Assets/Wide310x150Logo.scale-100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Wide310x150Logo.scale-100.png -------------------------------------------------------------------------------- /Notes/Assets/Wide310x150Logo.scale-125.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Wide310x150Logo.scale-125.png -------------------------------------------------------------------------------- /Notes/Assets/Wide310x150Logo.scale-150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Wide310x150Logo.scale-150.png -------------------------------------------------------------------------------- /Notes/Assets/Wide310x150Logo.scale-200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Wide310x150Logo.scale-200.png -------------------------------------------------------------------------------- /Notes/Assets/Wide310x150Logo.scale-400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Wide310x150Logo.scale-400.png -------------------------------------------------------------------------------- /Notes/Assets/Square150x150Logo.scale-100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square150x150Logo.scale-100.png -------------------------------------------------------------------------------- /Notes/Assets/Square150x150Logo.scale-125.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square150x150Logo.scale-125.png -------------------------------------------------------------------------------- /Notes/Assets/Square150x150Logo.scale-150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square150x150Logo.scale-150.png -------------------------------------------------------------------------------- /Notes/Assets/Square150x150Logo.scale-200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square150x150Logo.scale-200.png -------------------------------------------------------------------------------- /Notes/Assets/Square150x150Logo.scale-400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square150x150Logo.scale-400.png -------------------------------------------------------------------------------- /Notes/Assets/Square44x44Logo.targetsize-16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square44x44Logo.targetsize-16.png -------------------------------------------------------------------------------- /Notes/Assets/Square44x44Logo.targetsize-24.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square44x44Logo.targetsize-24.png -------------------------------------------------------------------------------- /Notes/Assets/Square44x44Logo.targetsize-32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square44x44Logo.targetsize-32.png -------------------------------------------------------------------------------- /Notes/Assets/Square44x44Logo.targetsize-48.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square44x44Logo.targetsize-48.png -------------------------------------------------------------------------------- /Notes/Assets/Square44x44Logo.targetsize-256.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square44x44Logo.targetsize-256.png -------------------------------------------------------------------------------- /Notes/AI/Embeddings/IVectorObject.cs: -------------------------------------------------------------------------------- 1 | namespace Notes.AI.Embeddings 2 | { 3 | public interface IVectorObject 4 | { 5 | float[] Vectors { get; } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /Notes/Assets/Square44x44Logo.altform-unplated_targetsize-16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square44x44Logo.altform-unplated_targetsize-16.png -------------------------------------------------------------------------------- /Notes/Assets/Square44x44Logo.altform-unplated_targetsize-32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square44x44Logo.altform-unplated_targetsize-32.png -------------------------------------------------------------------------------- /Notes/Assets/Square44x44Logo.altform-unplated_targetsize-48.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square44x44Logo.altform-unplated_targetsize-48.png -------------------------------------------------------------------------------- /Notes/Assets/Square44x44Logo.targetsize-24_altform-unplated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square44x44Logo.targetsize-24_altform-unplated.png -------------------------------------------------------------------------------- /Notes/Assets/Square44x44Logo.altform-unplated_targetsize-256.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square44x44Logo.altform-unplated_targetsize-256.png -------------------------------------------------------------------------------- /Notes/Assets/Square44x44Logo.altform-lightunplated_targetsize-16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square44x44Logo.altform-lightunplated_targetsize-16.png -------------------------------------------------------------------------------- /Notes/Assets/Square44x44Logo.altform-lightunplated_targetsize-24.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square44x44Logo.altform-lightunplated_targetsize-24.png -------------------------------------------------------------------------------- /Notes/Assets/Square44x44Logo.altform-lightunplated_targetsize-32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square44x44Logo.altform-lightunplated_targetsize-32.png -------------------------------------------------------------------------------- /Notes/Assets/Square44x44Logo.altform-lightunplated_targetsize-48.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square44x44Logo.altform-lightunplated_targetsize-48.png -------------------------------------------------------------------------------- /Notes/Assets/Square44x44Logo.altform-lightunplated_targetsize-256.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/ai-powered-notes-winui3-sample/HEAD/Notes/Assets/Square44x44Logo.altform-lightunplated_targetsize-256.png -------------------------------------------------------------------------------- /Notes/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "profiles": { 3 | "Notes (Package)": { 4 | "commandName": "MsixPackage" 5 | }, 6 | "Notes (Unpackaged)": { 7 | "commandName": "Project" 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /Notes/AI/IChatClient/LlmPromptTemplate.cs: -------------------------------------------------------------------------------- 1 | namespace Notes.AI; 2 | 3 | internal class LlmPromptTemplate 4 | { 5 | public string? System { get; init; } 6 | public string? User { get; init; } 7 | public string? Assistant { get; init; } 8 | public string[]? Stop { get; init; } 9 | } -------------------------------------------------------------------------------- /SUPPORT.md: -------------------------------------------------------------------------------- 1 | # Support 2 | 3 | ## How to file issues and get help 4 | 5 | This project uses GitHub Issues to track bugs and feature requests. Please search the existing 6 | issues before filing new issues to avoid duplicates. For new issues, file your bug or 7 | feature request as a new Issue. 8 | 9 | ## Microsoft Support Policy 10 | 11 | Support for this sample is limited to the resources listed above. 12 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /Notes/Models/Note.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Notes.Models 8 | { 9 | public class Note 10 | { 11 | public int Id { get; set; } 12 | public string Title { get; set; } 13 | public string Filename { get; set; } 14 | public DateTime Created { get; set; } 15 | public DateTime Modified { get; set; } 16 | public List Attachments { get; set; } = new(); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /Notes/App.xaml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /Notes/Themes/Generic.xaml: -------------------------------------------------------------------------------- 1 | 6 | 7 | 20 | 21 | -------------------------------------------------------------------------------- /Notes/AI/Embeddings/TextChunk.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Notes.AI.Embeddings 4 | { 5 | /// 6 | /// A chunk of text from a pdf document. Will also contain the page number and the source file. 7 | /// 8 | public class TextChunk : IVectorObject 9 | { 10 | public TextChunk() 11 | { 12 | Vectors = Array.Empty(); 13 | } 14 | 15 | public int Id { get; set; } 16 | public int SourceId { get; set; } 17 | public string ContentType { get; set; } 18 | public int ChunkIndexInSource { get; set; } 19 | public string? Text1 { get; set; } 20 | public string? Text2 { get; set; } 21 | public string? Text3 { get; set; } 22 | 23 | public string Text => string.Join(" ", new[] { Text1, Text2, Text3 }); 24 | public float[] Vectors { get; set; } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Notes/app.manifest: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | PerMonitorV2 17 | 18 | 19 | -------------------------------------------------------------------------------- /Notes/Utils/Utils.Rag.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using System.Runtime.CompilerServices; 4 | using System.Threading; 5 | using Notes.AI; 6 | 7 | namespace Notes 8 | { 9 | internal partial class Utils 10 | { 11 | public static async IAsyncEnumerable Rag(string question, [EnumeratorCancellation] CancellationToken ct = default) 12 | { 13 | if (App.ChatClient == null) 14 | { 15 | yield return string.Empty; 16 | } 17 | else 18 | { 19 | var searchResults = await SearchAsync(question, top: 2); 20 | 21 | var content = string.Join(" ", searchResults.Select(c => c.Content).ToList()); 22 | 23 | var systemMessage = "You are a helpful assistant answering questions about this content"; 24 | 25 | await foreach (var token in App.ChatClient.InferStreaming($"{systemMessage}: {content}", question, ct)) 26 | { 27 | yield return token; 28 | } 29 | } 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /Notes/Models/TranscriptionBlock.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace Notes.Models 8 | { 9 | public class TranscriptionBlock 10 | { 11 | public string Text { get; set; } 12 | public TimeSpan Start { get; set; } 13 | public TimeSpan End { get; set; } 14 | public string StartDisplayText { get; set; } 15 | public TranscriptionBlock(string text, double start, double end) 16 | { 17 | Text = text; 18 | Start = TimeSpan.FromSeconds(start); 19 | End = TimeSpan.FromSeconds(end); 20 | StartDisplayText = FormatTimeSpan(Start); 21 | } 22 | 23 | public static string FormatTimeSpan(TimeSpan span) 24 | { 25 | return FixTimeSegmentLength(span.Hours) + ":" + FixTimeSegmentLength(span.Minutes) + ":" + FixTimeSegmentLength(span.Seconds); 26 | } 27 | 28 | public static string FixTimeSegmentLength(int timeSegment) 29 | { 30 | string castedTimeSegment = timeSegment.ToString(); 31 | return timeSegment < 10 ? "0" + castedTimeSegment : castedTimeSegment; 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /Notes/Models/Attachment.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.ComponentModel; 3 | using System.Runtime.CompilerServices; 4 | 5 | namespace Notes.Models 6 | { 7 | public class Attachment : INotifyPropertyChanged 8 | { 9 | private bool isProcessed; 10 | public int Id { get; set; } 11 | public string Filename { get; set; } 12 | public string? FilenameForText { get; set; } 13 | public NoteAttachmentType Type { get; set; } 14 | public bool IsProcessed 15 | { 16 | get { return this.isProcessed; } 17 | set 18 | { 19 | if (value != this.isProcessed) 20 | { 21 | this.isProcessed = value; 22 | NotifyPropertyChanged(); 23 | } 24 | } 25 | } 26 | public int NoteId { get; set; } 27 | public Note Note { get; set; } 28 | 29 | private void NotifyPropertyChanged([CallerMemberName] String propertyName = "") 30 | { 31 | PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName)); 32 | } 33 | 34 | public event PropertyChangedEventHandler PropertyChanged; 35 | } 36 | 37 | public enum NoteAttachmentType 38 | { 39 | Image = 0, 40 | Audio = 1, 41 | Video = 2, 42 | Document = 3 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /Notes/Utils/AppDataContext.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.EntityFrameworkCore; 2 | using System.ComponentModel.DataAnnotations.Schema; 3 | using System.Threading.Tasks; 4 | using Notes.AI.Embeddings; 5 | using Notes.Models; 6 | 7 | namespace Notes 8 | { 9 | public class AppDataContext : DbContext 10 | { 11 | public DbSet TextChunks { get; set; } 12 | 13 | public DbSet Notes { get; set; } 14 | 15 | public DbSet Attachments { get; set; } 16 | 17 | private string _dbPath { get; set; } 18 | private static AppDataContext _current; 19 | public static async Task GetCurrentAsync() 20 | { 21 | if (_current == null) 22 | { 23 | _current = new AppDataContext($@"{(await Utils.GetStateFolderAsync()).Path}\state.db"); 24 | await _current.Database.EnsureCreatedAsync(); 25 | } 26 | 27 | return _current; 28 | } 29 | 30 | public static async Task SaveCurrentAsync() 31 | { 32 | var context = await GetCurrentAsync(); 33 | await context.SaveChangesAsync(); 34 | } 35 | 36 | private AppDataContext(string dbPath) 37 | { 38 | _dbPath = dbPath; 39 | } 40 | 41 | protected override void OnConfiguring(DbContextOptionsBuilder options) => options.UseSqlite($"Data Source={_dbPath}"); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /Notes/ViewModels/AttachmentViewModel.cs: -------------------------------------------------------------------------------- 1 | using CommunityToolkit.Mvvm.ComponentModel; 2 | using System.Diagnostics; 3 | using Notes.Models; 4 | 5 | namespace Notes.ViewModels 6 | { 7 | public partial class AttachmentViewModel : ObservableObject 8 | { 9 | public readonly Attachment Attachment; 10 | 11 | [ObservableProperty] 12 | private bool isProcessing; 13 | 14 | [ObservableProperty] 15 | private float processingProgress; 16 | 17 | public AttachmentViewModel(Attachment attachment) 18 | { 19 | Attachment = attachment; 20 | if (!attachment.IsProcessed) 21 | { 22 | AttachmentProcessor.AttachmentProcessed += AttachmentProcessor_AttachmentProcessed; 23 | ProcessingProgress = 0; 24 | } 25 | else 26 | { 27 | ProcessingProgress = 1; 28 | } 29 | IsProcessing = !attachment.IsProcessed; 30 | } 31 | 32 | private void AttachmentProcessor_AttachmentProcessed(object? sender, AttachmentProcessedEventArgs e) 33 | { 34 | if (e.AttachmentId == Attachment.Id) 35 | { 36 | MainWindow.Instance.DispatcherQueue.TryEnqueue(() => 37 | { 38 | IsProcessing = e.Progress < 1; 39 | ProcessingProgress = e.Progress * 100; 40 | }); 41 | 42 | Debug.WriteLine($"Attachment {Attachment.Id}, step: {e.ProcessingStep}, processed: {e.Progress * 100}%"); 43 | } 44 | } 45 | } 46 | 47 | 48 | } 49 | -------------------------------------------------------------------------------- /Notes/AI/TextRecognition/ImageText.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Windows.Vision; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | 8 | namespace Notes.AI.TextRecognition 9 | { 10 | internal class ImageText 11 | { 12 | public List Lines { get; set; } = new(); 13 | public double ImageAngle { get; set; } 14 | 15 | public static ImageText GetFromRecognizedText(RecognizedText? recognizedText) 16 | { 17 | ImageText attachmentRecognizedText = new(); 18 | 19 | if (recognizedText == null) 20 | { 21 | return attachmentRecognizedText; 22 | } 23 | 24 | attachmentRecognizedText.ImageAngle = recognizedText.ImageAngle; 25 | attachmentRecognizedText.Lines = recognizedText.Lines.Select(l => new RecognizedTextLine 26 | { 27 | Text = l.Text, 28 | X = l.BoundingBox.TopLeft.X, 29 | Y = l.BoundingBox.TopLeft.Y, 30 | Width = Math.Abs(l.BoundingBox.TopRight.X - l.BoundingBox.TopLeft.X), 31 | Height = Math.Abs(l.BoundingBox.BottomLeft.Y - l.BoundingBox.TopLeft.Y) 32 | }).ToList(); 33 | 34 | return attachmentRecognizedText; 35 | } 36 | } 37 | 38 | internal class RecognizedTextLine 39 | { 40 | public string Text { get; set; } 41 | public double X { get; set; } 42 | public double Y { get; set; } 43 | public double Width { get; set; } 44 | public double Height { get; set; } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /Notes/Utils/Utils.DX.cs: -------------------------------------------------------------------------------- 1 | using SharpDX.DXGI; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | 5 | namespace Notes 6 | { 7 | internal partial class Utils 8 | { 9 | public static List GetAdapters() 10 | { 11 | var factory1 = new Factory1(); 12 | var adapters = new List(); 13 | for (int i = 0; i < factory1.GetAdapterCount1(); i++) 14 | { 15 | adapters.Add(factory1.GetAdapter1(i)); 16 | } 17 | 18 | return adapters; 19 | } 20 | 21 | public static int GetBestDeviceId() 22 | { 23 | int deviceId = 0; 24 | Adapter1? selectedAdapter = null; 25 | List list = GetAdapters(); 26 | for (int i = 0; i < list.Count; i++) 27 | { 28 | Adapter1? adapter = list[i]; 29 | Debug.WriteLine($"Adapter {i}:"); 30 | Debug.WriteLine($"\tDescription: {adapter.Description1.Description}"); 31 | Debug.WriteLine($"\tDedicatedVideoMemory: {(long)adapter.Description1.DedicatedVideoMemory / 1000000000}GB"); 32 | Debug.WriteLine($"\tSharedSystemMemory: {(long)adapter.Description1.SharedSystemMemory / 1000000000}GB"); 33 | if (selectedAdapter == null || (long)adapter.Description1.DedicatedVideoMemory > (long)selectedAdapter.Description1.DedicatedVideoMemory) 34 | { 35 | selectedAdapter = adapter; 36 | deviceId = i; 37 | } 38 | } 39 | 40 | return deviceId; 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /Notes/AI/TextRecognition/TextRecognition.cs: -------------------------------------------------------------------------------- 1 | //using Microsoft.Windows.Imaging; 2 | //using Microsoft.Windows.Vision; 3 | using Microsoft.Graphics.Imaging; 4 | using Microsoft.Windows.Vision; 5 | using System; 6 | using System.Text.Json; 7 | using System.Threading.Tasks; 8 | using Windows.Graphics.Imaging; 9 | using Windows.Storage; 10 | 11 | namespace Notes.AI.TextRecognition 12 | { 13 | internal static class TextRecognition 14 | { 15 | public static async Task GetTextFromImage(SoftwareBitmap image) 16 | { 17 | if (!TextRecognizer.IsAvailable()) 18 | { 19 | var op = await TextRecognizer.MakeAvailableAsync(); 20 | if (op.Status != Microsoft.Windows.Management.Deployment.PackageDeploymentStatus.CompletedSuccess) 21 | { 22 | return null; 23 | } 24 | } 25 | 26 | var textRecognizer = await TextRecognizer.CreateAsync(); 27 | 28 | using var imageBuffer = ImageBuffer.CreateBufferAttachedToBitmap(image); 29 | RecognizedText? result = textRecognizer?.RecognizeTextFromImage(imageBuffer, new TextRecognizerOptions()); 30 | 31 | return ImageText.GetFromRecognizedText(result); 32 | } 33 | 34 | public static async Task GetSavedText(string filename) 35 | { 36 | var folder = await Utils.GetAttachmentsTranscriptsFolderAsync(); 37 | var file = await folder.GetFileAsync(filename); 38 | 39 | var text = await FileIO.ReadTextAsync(file); 40 | 41 | var lines = JsonSerializer.Deserialize(text); 42 | return lines ?? new ImageText(); 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /Notes/Package.appxmanifest: -------------------------------------------------------------------------------- 1 | 2 | 3 | 9 | 10 | 14 | 15 | 16 | 17 | 18 | Notes 19 | nikol 20 | Assets\StoreLogo.png 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 36 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /Notes/ViewModels/SearchViewModel.cs: -------------------------------------------------------------------------------- 1 | using CommunityToolkit.Mvvm.ComponentModel; 2 | using Microsoft.UI.Xaml; 3 | using System; 4 | using System.Collections.ObjectModel; 5 | using System.Diagnostics; 6 | using System.Linq; 7 | using System.Threading.Tasks; 8 | using Notes.AI.Embeddings; 9 | 10 | // To learn more about WinUI, the WinUI project structure, 11 | // and more about our project templates, see: http://aka.ms/winui-project-info. 12 | 13 | namespace Notes.ViewModels 14 | { 15 | public partial class SearchViewModel : ObservableObject 16 | { 17 | [ObservableProperty] 18 | public bool showResults = false; 19 | public ObservableCollection Results { get; set; } = new(); 20 | 21 | private DispatcherTimer _searchTimer = new DispatcherTimer(); 22 | private string _searchText = string.Empty; 23 | 24 | public SearchViewModel() 25 | { 26 | _searchTimer.Interval = TimeSpan.FromMilliseconds(1500); 27 | _searchTimer.Tick += SearchTimerTick; 28 | } 29 | public void HandleTextChanged(string text) 30 | { 31 | Debug.WriteLine("reseting timer"); 32 | _searchTimer.Stop(); 33 | _searchTimer.Start(); 34 | _searchText = text; 35 | } 36 | 37 | public void Reset() 38 | { 39 | Results.Clear(); 40 | ShowResults = false; 41 | } 42 | 43 | private async Task Search() 44 | { 45 | Debug.WriteLine("searching"); 46 | Reset(); 47 | if (string.IsNullOrWhiteSpace(_searchText)) 48 | { 49 | return; 50 | } 51 | 52 | // TODO: handle cancelation 53 | var results = await Utils.SearchAsync(_searchText); 54 | 55 | foreach (var result in results) 56 | { 57 | Results.Add(result); 58 | } 59 | 60 | ShowResults = true; 61 | } 62 | 63 | private async void SearchTimerTick(object? sender, object e) 64 | { 65 | ShowResults = false; 66 | _searchTimer.Stop(); 67 | Search(); 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /Notes/App.xaml.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.AI; 2 | using Microsoft.UI.Xaml; 3 | using Notes.AI; 4 | using System; 5 | using System.IO; 6 | using System.Threading.Tasks; 7 | 8 | // To learn more about WinUI, the WinUI project structure, 9 | // and more about our project templates, see: http://aka.ms/winui-project-info. 10 | 11 | namespace Notes 12 | { 13 | /// 14 | /// Provides application-specific behavior to supplement the default Application class. 15 | /// 16 | public partial class App : Application 17 | { 18 | public static IChatClient? ChatClient { get; private set; } 19 | /// 20 | /// Initializes the singleton application object. This is the first line of authored code 21 | /// executed, and as such is the logical equivalent of main() or WinMain(). 22 | /// 23 | public App() 24 | { 25 | this.InitializeComponent(); 26 | } 27 | 28 | /// 29 | /// Invoked when the application is launched. 30 | /// 31 | /// Details about the launch request and process. 32 | protected override void OnLaunched(Microsoft.UI.Xaml.LaunchActivatedEventArgs args) 33 | { 34 | m_window = new MainWindow(); 35 | m_window.Activate(); 36 | 37 | _ = InitializeIChatClient(); 38 | } 39 | 40 | private async Task InitializeIChatClient() 41 | { 42 | // use PhiSilica 43 | // ChatClient = await PhiSilicaClient.CreateAsync(); 44 | 45 | // use genai model 46 | ChatClient = await GenAIModel.CreateAsync( 47 | Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "onnx-models", "genai-model"), 48 | new LlmPromptTemplate 49 | { 50 | System = "<|system|>\n{{CONTENT}}<|end|>\n", 51 | User = "<|user|>\n{{CONTENT}}<|end|>\n", 52 | Assistant = "<|assistant|>\n{{CONTENT}}<|end|>\n", 53 | Stop = ["<|system|>", "<|user|>", "<|assistant|>", "<|end|>"] 54 | }); 55 | } 56 | 57 | private Window m_window; 58 | public Window Window => m_window; 59 | 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /Notes.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 17 4 | VisualStudioVersion = 17.9.34728.123 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Notes", "Notes\Notes.csproj", "{D9582F5B-7986-4342-994C-1DC17DE8D87D}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Debug|ARM64 = Debug|ARM64 12 | Debug|x64 = Debug|x64 13 | Release|Any CPU = Release|Any CPU 14 | Release|ARM64 = Release|ARM64 15 | Release|x64 = Release|x64 16 | EndGlobalSection 17 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 18 | {D9582F5B-7986-4342-994C-1DC17DE8D87D}.Debug|Any CPU.ActiveCfg = Debug|x64 19 | {D9582F5B-7986-4342-994C-1DC17DE8D87D}.Debug|Any CPU.Build.0 = Debug|x64 20 | {D9582F5B-7986-4342-994C-1DC17DE8D87D}.Debug|Any CPU.Deploy.0 = Debug|x64 21 | {D9582F5B-7986-4342-994C-1DC17DE8D87D}.Debug|ARM64.ActiveCfg = Debug|ARM64 22 | {D9582F5B-7986-4342-994C-1DC17DE8D87D}.Debug|ARM64.Build.0 = Debug|ARM64 23 | {D9582F5B-7986-4342-994C-1DC17DE8D87D}.Debug|ARM64.Deploy.0 = Debug|ARM64 24 | {D9582F5B-7986-4342-994C-1DC17DE8D87D}.Debug|x64.ActiveCfg = Debug|x64 25 | {D9582F5B-7986-4342-994C-1DC17DE8D87D}.Debug|x64.Build.0 = Debug|x64 26 | {D9582F5B-7986-4342-994C-1DC17DE8D87D}.Debug|x64.Deploy.0 = Debug|x64 27 | {D9582F5B-7986-4342-994C-1DC17DE8D87D}.Release|Any CPU.ActiveCfg = Release|x64 28 | {D9582F5B-7986-4342-994C-1DC17DE8D87D}.Release|Any CPU.Build.0 = Release|x64 29 | {D9582F5B-7986-4342-994C-1DC17DE8D87D}.Release|Any CPU.Deploy.0 = Release|x64 30 | {D9582F5B-7986-4342-994C-1DC17DE8D87D}.Release|ARM64.ActiveCfg = Release|ARM64 31 | {D9582F5B-7986-4342-994C-1DC17DE8D87D}.Release|ARM64.Build.0 = Release|ARM64 32 | {D9582F5B-7986-4342-994C-1DC17DE8D87D}.Release|ARM64.Deploy.0 = Release|ARM64 33 | {D9582F5B-7986-4342-994C-1DC17DE8D87D}.Release|x64.ActiveCfg = Release|x64 34 | {D9582F5B-7986-4342-994C-1DC17DE8D87D}.Release|x64.Build.0 = Release|x64 35 | {D9582F5B-7986-4342-994C-1DC17DE8D87D}.Release|x64.Deploy.0 = Release|x64 36 | EndGlobalSection 37 | GlobalSection(SolutionProperties) = preSolution 38 | HideSolutionNode = FALSE 39 | EndGlobalSection 40 | GlobalSection(ExtensibilityGlobals) = postSolution 41 | SolutionGuid = {C14D211A-EC35-4B9A-A233-C7BAC1E4CF29} 42 | EndGlobalSection 43 | EndGlobal 44 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /Notes/ViewModels/ViewModel.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Collections.ObjectModel; 4 | using System.Diagnostics; 5 | using System.Linq; 6 | using System.Text; 7 | using System.Text.Json; 8 | using System.Threading.Tasks; 9 | using Notes.Models; 10 | using Windows.Storage; 11 | 12 | namespace Notes.ViewModels 13 | { 14 | public class ViewModel 15 | { 16 | public ObservableCollection Notes { get; set; } = new(); 17 | 18 | public ViewModel() 19 | { 20 | LoadNotes(); 21 | } 22 | 23 | public async Task CreateNewNote() 24 | { 25 | var title = "New note"; 26 | 27 | var folder = await Utils.GetLocalFolderAsync(); 28 | var file = await folder.CreateFileAsync(title + Utils.FileExtension, CreationCollisionOption.GenerateUniqueName); 29 | 30 | var note = new Note() 31 | { 32 | Title = title, 33 | Created = DateTime.Now, 34 | Modified = DateTime.Now, 35 | Filename = file.Name 36 | }; 37 | 38 | var noteViewModel = new NoteViewModel(note); 39 | Notes.Insert(0, noteViewModel); 40 | var dataContext = await AppDataContext.GetCurrentAsync(); 41 | dataContext.Notes.Add(note); 42 | await dataContext.SaveChangesAsync(); 43 | 44 | return noteViewModel; 45 | } 46 | 47 | private async Task LoadNotes() 48 | { 49 | var dataContext = await AppDataContext.GetCurrentAsync(); 50 | var savedNotes = dataContext.Notes.Select(note => note).ToList(); 51 | 52 | StorageFolder notesFolder = await Utils.GetLocalFolderAsync(); 53 | var files = await notesFolder.GetFilesAsync(); 54 | var filenames = files.ToDictionary(f => f.Name, f=> f); 55 | 56 | foreach (var note in savedNotes) 57 | { 58 | if (filenames.ContainsKey(note.Filename)) 59 | { 60 | filenames.Remove(note.Filename); 61 | Notes.Add(new NoteViewModel(note)); 62 | } 63 | else 64 | { 65 | // delete note from db 66 | dataContext.Notes.Remove(note); 67 | } 68 | } 69 | 70 | foreach (var filename in filenames) 71 | { 72 | if (filename.Key.EndsWith(Utils.FileExtension)) 73 | { 74 | var file = filename.Value; 75 | var note = new Note() 76 | { 77 | Title = file.DisplayName, 78 | Created = file.DateCreated.DateTime, 79 | Filename = file.Name, 80 | Modified = DateTime.Now 81 | }; 82 | dataContext.Notes.Add(note); 83 | Notes.Add(new NoteViewModel(note)); 84 | } 85 | } 86 | 87 | await dataContext.SaveChangesAsync(); 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /Notes/Utils/WaveformRenderer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Drawing; 3 | using System.Drawing.Imaging; 4 | using System.IO; 5 | using Microsoft.UI.Xaml.Media.Imaging; 6 | using NAudio.Wave; 7 | using NAudio.WaveFormRenderer; 8 | using Windows.Storage; 9 | 10 | namespace Notes 11 | { 12 | public static class WaveformRenderer 13 | { 14 | public enum PeakProvider 15 | { 16 | Max, 17 | RMS, 18 | Sampling, 19 | Average 20 | } 21 | 22 | private static Image Render(StorageFile audioFile, int height, int width, PeakProvider peakProvider) 23 | { 24 | WaveFormRendererSettings settings = new StandardWaveFormRendererSettings(); 25 | settings.BackgroundColor = Color.Transparent; 26 | settings.SpacerPixels = 0; 27 | settings.TopHeight = height; 28 | settings.BottomHeight = height; 29 | settings.Width = width; 30 | settings.TopPeakPen = new Pen(Color.DarkGray); 31 | settings.BottomPeakPen = new Pen(Color.DarkGray); 32 | AudioFileReader audioFileReader = new AudioFileReader(audioFile.Path); 33 | 34 | IPeakProvider provider; 35 | switch (peakProvider) 36 | { 37 | case PeakProvider.Max: 38 | provider = new MaxPeakProvider(); 39 | break; 40 | case PeakProvider.RMS: 41 | provider = new RmsPeakProvider(200); 42 | break; 43 | case PeakProvider.Sampling: 44 | provider = new SamplingPeakProvider(1600); 45 | break; 46 | default: 47 | provider = new AveragePeakProvider(4); 48 | break; 49 | } 50 | 51 | WaveFormRenderer renderer = new WaveFormRenderer(); 52 | return renderer.Render(audioFileReader, provider, settings); 53 | } 54 | 55 | public static async System.Threading.Tasks.Task GetWaveformImage(StorageFile audioFile) 56 | { 57 | StorageFile imageFile; 58 | StorageFolder attachmentsFolder = await Utils.GetAttachmentsTranscriptsFolderAsync(); 59 | string waveformFileName = Path.ChangeExtension(Path.GetFileName(audioFile.Path) + "-waveform", ".png"); 60 | try 61 | { 62 | imageFile = await attachmentsFolder.CreateFileAsync(waveformFileName, CreationCollisionOption.FailIfExists); 63 | using (var stream = await imageFile.OpenStreamForWriteAsync()) 64 | { 65 | System.Drawing.Image image = Render(audioFile, 400, 800, PeakProvider.Average); 66 | image.Save(stream, ImageFormat.Png); 67 | } 68 | } 69 | catch 70 | { 71 | imageFile = await attachmentsFolder.GetFileAsync(waveformFileName); 72 | } 73 | 74 | 75 | Uri uri = new Uri(imageFile.Path); 76 | BitmapImage bi = new BitmapImage(uri); 77 | 78 | return bi; 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /Notes/Controls/SearchView.xaml.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.UI.Input; 2 | using Microsoft.UI.Xaml.Controls; 3 | using Microsoft.UI.Xaml.Controls.Primitives; 4 | using Microsoft.UI.Xaml.Data; 5 | using Microsoft.UI.Xaml.Input; 6 | using Microsoft.UI.Xaml.Media; 7 | using Microsoft.UI.Xaml.Navigation; 8 | using Microsoft.UI.Xaml; 9 | using System; 10 | using System.Collections.Generic; 11 | using System.IO; 12 | using System.Linq; 13 | using System.Runtime.InteropServices.WindowsRuntime; 14 | using Notes.ViewModels; 15 | using System.Threading.Tasks; 16 | using System.Diagnostics; 17 | 18 | // To learn more about WinUI, the WinUI project structure, 19 | // and more about our project templates, see: http://aka.ms/winui-project-info. 20 | 21 | namespace Notes.Controls 22 | { 23 | public sealed partial class SearchView : UserControl 24 | { 25 | public SearchViewModel ViewModel { get; } = new SearchViewModel(); 26 | 27 | 28 | public SearchView() 29 | { 30 | this.InitializeComponent(); 31 | this.KeyDown += SearchView_KeyDown; 32 | } 33 | 34 | private async void ListView_ItemClick(object sender, ItemClickEventArgs e) 35 | { 36 | var context = await AppDataContext.GetCurrentAsync(); 37 | 38 | var item = e.ClickedItem as SearchResult; 39 | if (item.ContentType == ContentType.Note) 40 | { 41 | MainWindow.Instance.SelectNoteById(item.SourceId); 42 | } 43 | else 44 | { 45 | var attachment = context.Attachments.Where(a => a.Id == item.SourceId).FirstOrDefault(); 46 | if (attachment != null) 47 | { 48 | var note = context.Notes.Where(n => n.Id == attachment.NoteId).FirstOrDefault(); 49 | MainWindow.Instance.SelectNoteById(note.Id, attachment.Id, item.MostRelevantSentence ?? null); 50 | } 51 | } 52 | 53 | this.Visibility = Visibility.Collapsed; 54 | } 55 | 56 | private void SearchView_KeyDown(object sender, KeyRoutedEventArgs e) 57 | { 58 | if (e.Key == Windows.System.VirtualKey.Escape) 59 | { 60 | this.Visibility = Visibility.Collapsed; 61 | } 62 | } 63 | 64 | public void Show(string? text = null) 65 | { 66 | if (!string.IsNullOrWhiteSpace(text)) 67 | { 68 | SearchBox.Text = text; 69 | ViewModel.Reset(); 70 | } 71 | 72 | this.Visibility = Visibility.Visible; 73 | Task.Run(async () => 74 | { 75 | await Task.Delay(100); 76 | DispatcherQueue.TryEnqueue(() => 77 | { 78 | SearchBox.Focus(FocusState.Programmatic); 79 | }); 80 | }); 81 | } 82 | 83 | 84 | 85 | private void BackgroundTapped(object sender, TappedRoutedEventArgs e) 86 | { 87 | // hide the search view only when the backround was tapped but not any of the content inside 88 | if (e.OriginalSource == Root) 89 | this.Visibility = Visibility.Collapsed; 90 | } 91 | 92 | private async void AutoSuggestBox_TextChanged(AutoSuggestBox sender, AutoSuggestBoxTextChangedEventArgs args) 93 | { 94 | Debug.WriteLine("Text changed"); 95 | ViewModel.HandleTextChanged(sender.Text); 96 | } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /Notes/AI/IChatClient/IChatClientExtensions.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.AI; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Runtime.CompilerServices; 5 | using System.Text.RegularExpressions; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | 9 | namespace Notes.AI 10 | { 11 | public static class IChatClientExtensions 12 | { 13 | public static async IAsyncEnumerable InferStreaming(this IChatClient client, string system, string prompt, [EnumeratorCancellation] CancellationToken ct = default) 14 | { 15 | await foreach (var messagePart in client.GetStreamingResponseAsync( 16 | [ 17 | new ChatMessage(ChatRole.System, system), 18 | new ChatMessage(ChatRole.User, prompt) 19 | ], 20 | null, 21 | ct)) 22 | { 23 | yield return messagePart.Text ?? string.Empty; 24 | } 25 | } 26 | 27 | public static IAsyncEnumerable SummarizeTextAsync(this IChatClient client, string userText, CancellationToken ct = default) 28 | { 29 | return client.InferStreaming("", $"Summarize this text in three to five bullet points:\r {userText}", ct); 30 | } 31 | 32 | public static IAsyncEnumerable FixAndCleanUpTextAsync(this IChatClient client, string userText, CancellationToken ct = default) 33 | { 34 | var systemMessage = "Your job is to fix spelling, and clean up the text from the user. Only respond with the updated text. Do not explain anything."; 35 | return client.InferStreaming(systemMessage, userText, ct); 36 | } 37 | 38 | public static IAsyncEnumerable AutocompleteSentenceAsync(this IChatClient client, string sentence, CancellationToken ct = default) 39 | { 40 | var systemMessage = "You are an assistant that helps the user complete sentences. Ignore spelling mistakes and just respond with the words that complete the sentence. Do not repeat the begining of the sentence."; 41 | return client.InferStreaming(systemMessage, sentence, ct); 42 | } 43 | 44 | public static Task> GetTodoItemsFromText(this IChatClient client, string text) 45 | { 46 | return Task.Run(async () => 47 | { 48 | 49 | var system = "Summarize the user text to 2-3 to-do items. Use the format [\"to-do 1\", \"to-do 2\"]. Respond only in one json array format"; 50 | string response = string.Empty; 51 | 52 | CancellationTokenSource cts = new CancellationTokenSource(); 53 | await foreach (var partialResult in client.InferStreaming(system, text, cts.Token)) 54 | { 55 | response += partialResult; 56 | if (partialResult.Contains("]")) 57 | { 58 | cts.Cancel(); 59 | break; 60 | } 61 | } 62 | 63 | var todos = Regex.Matches(response, @"""([^""]*)""", RegexOptions.IgnoreCase | RegexOptions.Multiline) 64 | .Select(m => m.Groups[1].Value) 65 | .Where(t => !t.Contains("todo")).ToList(); 66 | 67 | return todos; 68 | }); 69 | } 70 | 71 | public static IAsyncEnumerable AskForContentAsync(this IChatClient client, string content, string question, CancellationToken ct = default) 72 | { 73 | var systemMessage = "You are a helpful assistant answering questions about this content"; 74 | return client.InferStreaming($"{systemMessage}: {content}", question, ct); 75 | } 76 | 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /Notes/Utils/Utils.Audio.cs: -------------------------------------------------------------------------------- 1 | using NReco.VideoConverter; 2 | using System; 3 | using System.IO; 4 | using System.Threading.Tasks; 5 | using Windows.Storage; 6 | 7 | namespace Notes 8 | { 9 | internal partial class Utils 10 | { 11 | public static byte[] LoadAudioBytes(string file) 12 | { 13 | var ffmpeg = new FFMpegConverter(); 14 | var output = new MemoryStream(); 15 | 16 | var extension = Path.GetExtension(file).Substring(1); 17 | 18 | // Convert to PCM 19 | ffmpeg.ConvertMedia(inputFile: file, 20 | inputFormat: null, 21 | outputStream: output, 22 | // DE s16le PCM signed 16-bit little-endian 23 | outputFormat: "s16le", 24 | new ConvertSettings() 25 | { 26 | AudioCodec = "pcm_s16le", 27 | AudioSampleRate = 16000, 28 | // Convert to mono 29 | CustomOutputArgs = "-ac 1" 30 | }); 31 | return output.ToArray(); 32 | } 33 | 34 | public static float[] ExtractAudioSegment(string inPath, double startTimeInSeconds, double segmentDurationInSeconds) 35 | { 36 | try 37 | { 38 | var extension = Path.GetExtension(inPath).Substring(1); 39 | var output = new MemoryStream(); 40 | 41 | var convertSettings = new ConvertSettings 42 | { 43 | Seek = (float?)startTimeInSeconds, 44 | MaxDuration = (float?)segmentDurationInSeconds, 45 | //AudioCodec = "pcm_s16le", 46 | AudioSampleRate = 16000, 47 | CustomOutputArgs = "-vn -ac 1", 48 | }; 49 | 50 | var ffMpegConverter = new FFMpegConverter(); 51 | ffMpegConverter.ConvertMedia( 52 | inputFile: inPath, 53 | inputFormat: null, 54 | outputStream: output, 55 | outputFormat: "wav", 56 | convertSettings); 57 | 58 | //return output.ToArray(); 59 | var buffer = output.ToArray(); 60 | int bytesPerSample = 2; // Assuming 16-bit depth (2 bytes per sample) 61 | 62 | // Calculate total samples in the buffer 63 | int totalSamples = buffer.Length / bytesPerSample; 64 | float[] samples = new float[totalSamples]; 65 | 66 | for (int i = 0; i < totalSamples; i++) 67 | { 68 | int bufferIndex = i * bytesPerSample; 69 | short sample = (short)(buffer[bufferIndex + 1] << 8 | buffer[bufferIndex]); 70 | samples[i] = sample / 32768.0f; // Normalize to range [-1,1] for floating point samples 71 | } 72 | 73 | return samples; 74 | } 75 | catch (Exception ex) 76 | { 77 | Console.WriteLine("Error during the audio extraction: " + ex.Message); 78 | return []; // Return an empty array in case of exception 79 | } 80 | } 81 | 82 | public static async Task SaveAudioFileAsWav(StorageFile file, StorageFolder folderToSaveTo) 83 | { 84 | var ffmpeg = new FFMpegConverter(); 85 | var newFilePath = $"{folderToSaveTo.Path}\\{file.DisplayName}.wav"; 86 | ffmpeg.ConvertMedia(file.Path, newFilePath, "wav"); 87 | var newFile = await StorageFile.GetFileFromPathAsync(newFilePath); 88 | return newFile; 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /Notes/Controls/SearchView.xaml: -------------------------------------------------------------------------------- 1 | 2 | 12 | 13 | 14 | 21 | 22 | 23 | 24 | 25 | 26 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | ... 44 | 45 | ... 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | -------------------------------------------------------------------------------- /Notes/MainWindow.xaml.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.UI.Dispatching; 2 | using Microsoft.UI.Xaml; 3 | using Microsoft.UI.Xaml.Controls; 4 | using System.Collections.Specialized; 5 | using System.Linq; 6 | using System.Threading.Tasks; 7 | using Notes.Controls; 8 | using Notes.AI.Embeddings; 9 | using Notes.Models; 10 | using Notes.Pages; 11 | using Notes.ViewModels; 12 | 13 | namespace Notes 14 | { 15 | public sealed partial class MainWindow : Window 16 | { 17 | 18 | public static Phi3View Phi3View; 19 | public static SearchView SearchView; 20 | public static MainWindow Instance; 21 | public ViewModel VM; 22 | 23 | public MainWindow() 24 | { 25 | VM = new ViewModel(); 26 | this.InitializeComponent(); 27 | 28 | this.ExtendsContentIntoTitleBar = true; 29 | this.SetTitleBar(TitleBar); 30 | 31 | Instance = this; 32 | Phi3View = phi3View; 33 | SearchView = searchView; 34 | 35 | VM.Notes.CollectionChanged += Notes_CollectionChanged; 36 | } 37 | 38 | public async Task SelectNoteById(int id, int? attachmentId = null, string? attachmentText = null) 39 | { 40 | var note = VM.Notes.Where(n => n.Note.Id == id).FirstOrDefault(); 41 | if (note != null) 42 | { 43 | navView.SelectedItem = note; 44 | 45 | if (attachmentId.HasValue) 46 | { 47 | var attachmentViewModel = note.Attachments.Where(a => a.Attachment.Id == attachmentId).FirstOrDefault(); 48 | if (attachmentViewModel == null) 49 | { 50 | var context = await AppDataContext.GetCurrentAsync(); 51 | var attachment = context.Attachments.Where(a => a.Id == attachmentId.Value).FirstOrDefault(); 52 | if (attachment == null) 53 | { 54 | return; 55 | } 56 | 57 | attachmentViewModel = new AttachmentViewModel(attachment); 58 | } 59 | 60 | OpenAttachmentView(attachmentViewModel, attachmentText); 61 | } 62 | } 63 | } 64 | 65 | private void navView_Loaded(object sender, RoutedEventArgs e) 66 | { 67 | if (navView.MenuItems.Count > 0) 68 | navView.SelectedItem = navView.MenuItems[0]; 69 | } 70 | 71 | private void Notes_CollectionChanged(object? sender, NotifyCollectionChangedEventArgs e) 72 | { 73 | if (navView.SelectedItem == null && VM.Notes.Count > 0) 74 | navView.SelectedItem = VM.Notes[0]; 75 | } 76 | 77 | private async void NewButton_Click(object sender, RoutedEventArgs e) 78 | { 79 | var note = await VM.CreateNewNote(); 80 | navView.SelectedItem = note; 81 | } 82 | 83 | private void navView_SelectionChanged(NavigationView sender, Microsoft.UI.Xaml.Controls.NavigationViewSelectionChangedEventArgs args) 84 | { 85 | if (args.SelectedItem is NoteViewModel note) 86 | { 87 | navFrame.Navigate(typeof(NotesPage), note); 88 | } 89 | } 90 | 91 | private void Search_Click(object sender, RoutedEventArgs e) 92 | { 93 | searchView.Show(); 94 | } 95 | 96 | private void AskMyNotesClicked(object sender, RoutedEventArgs e) 97 | { 98 | phi3View.ShowForRag(); 99 | } 100 | 101 | public void OpenAttachmentView(AttachmentViewModel attachment, string? attachmentText = null) 102 | { 103 | attachmentView.UpdateAttachment(attachment, attachmentText); 104 | attachmentView.Show(); 105 | } 106 | } 107 | 108 | class MenuItemTemplateSelector : DataTemplateSelector 109 | { 110 | public DataTemplate NoteTemplate { get; set; } 111 | public DataTemplate DefaultTemplate { get; set; } 112 | protected override DataTemplate SelectTemplateCore(object item) 113 | { 114 | return item is NoteViewModel ? NoteTemplate : DefaultTemplate; 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /Notes/Notes.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | WinExe 4 | net8.0-windows10.0.22621.0 5 | 10.0.17763.0 6 | Notes 7 | app.manifest 8 | x64;ARM64 9 | win10-x64;win10-arm64 10 | win10-$(Platform).pubxml 11 | enable 12 | true 13 | true 14 | true 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | PreserveNewest 61 | 62 | 63 | 64 | 69 | 70 | 71 | 72 | 73 | 74 | MSBuild:Compile 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | MSBuild:Compile 83 | 84 | 85 | 86 | 87 | MSBuild:Compile 88 | 89 | 90 | 91 | 96 | 97 | true 98 | 99 | 100 | -------------------------------------------------------------------------------- /Notes/AI/Whisper/Whisper.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.ML.OnnxRuntime.Tensors; 2 | using Microsoft.ML.OnnxRuntime; 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Threading.Tasks; 6 | using Windows.Storage; 7 | using Notes.AI.VoiceRecognition.VoiceActivity; 8 | using System.Diagnostics; 9 | 10 | namespace Notes.AI.VoiceRecognition 11 | { 12 | public static class Whisper 13 | { 14 | private static InferenceSession? _inferenceSession; 15 | private static InferenceSession InitializeModel() 16 | { 17 | var modelPath = $@"{AppDomain.CurrentDomain.BaseDirectory}onnx-models\whisper\whisper_tiny.onnx"; 18 | 19 | SessionOptions options = new SessionOptions(); 20 | options.RegisterOrtExtensions(); 21 | options.GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL; 22 | options.EnableMemoryPattern = false; 23 | 24 | var session = new InferenceSession(modelPath, options); 25 | 26 | return session; 27 | } 28 | 29 | private static async Task> TranscribeChunkAsync(float[] pcmAudioData, string inputLanguage, WhisperTaskType taskType, int offsetSeconds = 30) 30 | { 31 | if (_inferenceSession == null) 32 | { 33 | _inferenceSession = InitializeModel(); 34 | } 35 | 36 | var audioTensor = new DenseTensor(pcmAudioData, [1, pcmAudioData.Length]); 37 | var timestampsEnableTensor = new DenseTensor(new[] { 1 }, [1]); 38 | 39 | int task = (int)taskType; 40 | int langCode = WhisperUtils.GetLangId(inputLanguage); 41 | var decoderInputIds = new int[] { 50258, langCode, task }; 42 | var langAndModeTensor = new DenseTensor(decoderInputIds, [1, 3]); 43 | 44 | var inputs = new List { 45 | NamedOnnxValue.CreateFromTensor("audio_pcm", audioTensor), 46 | NamedOnnxValue.CreateFromTensor("min_length", new DenseTensor(new int[] { 0 }, [1])), 47 | NamedOnnxValue.CreateFromTensor("max_length", new DenseTensor(new int[] { 448 }, [1])), 48 | NamedOnnxValue.CreateFromTensor("num_beams", new DenseTensor(new int[] {1}, [1])), 49 | NamedOnnxValue.CreateFromTensor("num_return_sequences", new DenseTensor(new int[] { 1 }, [1])), 50 | NamedOnnxValue.CreateFromTensor("length_penalty", new DenseTensor(new float[] { 1.0f }, [1])), 51 | NamedOnnxValue.CreateFromTensor("repetition_penalty", new DenseTensor(new float[] { 1.2f }, [1])), 52 | NamedOnnxValue.CreateFromTensor("logits_processor", timestampsEnableTensor), 53 | NamedOnnxValue.CreateFromTensor("decoder_input_ids", langAndModeTensor) 54 | }; 55 | 56 | try 57 | { 58 | using var results = _inferenceSession.Run(inputs); 59 | string result = results[0].AsTensor().GetValue(0); 60 | return WhisperUtils.ProcessTranscriptionWithTimestamps(result, offsetSeconds); 61 | } 62 | catch (Exception ex) 63 | { 64 | // return empty list in case of exception 65 | return new List(); 66 | } 67 | } 68 | 69 | public async static Task> TranscribeAsync(StorageFile audioFile, EventHandler? progress = null) 70 | { 71 | var transcribedChunks = new List(); 72 | 73 | var sw = Stopwatch.StartNew(); 74 | 75 | var audioBytes = Utils.LoadAudioBytes(audioFile.Path); 76 | 77 | sw.Stop(); 78 | Debug.WriteLine($"Loading took {sw.ElapsedMilliseconds} ms"); 79 | sw.Start(); 80 | 81 | var dynamicChunks = WhisperChunking.SmartChunking(audioBytes); 82 | 83 | sw.Stop(); 84 | Debug.WriteLine($"Chunking took {sw.ElapsedMilliseconds} ms"); 85 | 86 | for (var i = 0; i < dynamicChunks.Count; i++) 87 | { 88 | var chunk = dynamicChunks[i]; 89 | 90 | var audioSegment = Utils.ExtractAudioSegment(audioFile.Path, chunk.Start, chunk.End - chunk.Start); 91 | 92 | var transcription = await TranscribeChunkAsync(audioSegment, "en", WhisperTaskType.Transcribe, (int)chunk.Start); 93 | 94 | transcribedChunks.AddRange(transcription); 95 | 96 | progress?.Invoke(null, (float)i / dynamicChunks.Count); 97 | } 98 | 99 | return transcribedChunks; 100 | } 101 | } 102 | 103 | internal enum WhisperTaskType 104 | { 105 | Translate = 50358, 106 | Transcribe = 50359 107 | } 108 | 109 | public class WhisperTranscribedChunk 110 | { 111 | public string Text { get; set; } 112 | public double Start { get; set; } 113 | public double End { get; set; } 114 | public double Length => End - Start; 115 | 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /Notes/AI/Whisper/VoiceActivity/SlieroVadDetector.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace Notes.AI.VoiceRecognition.VoiceActivity 5 | { 6 | public class SlieroVadDetector 7 | { 8 | private readonly SlieroVadOnnxModel model; 9 | private readonly float startThreshold; 10 | private readonly float endThreshold; 11 | private readonly int samplingRate; 12 | private readonly float minSilenceSamples; 13 | private readonly float speechPadSamples; 14 | private bool triggered; 15 | private int tempEnd; 16 | private int currentSample; 17 | 18 | public SlieroVadDetector(float startThreshold, 19 | float endThreshold, 20 | int samplingRate, 21 | int minSilenceDurationMs, 22 | int speechPadMs) 23 | { 24 | if (samplingRate != 8000 && samplingRate != 16000) 25 | { 26 | throw new ArgumentException("does not support sampling rates other than [8000, 16000]"); 27 | } 28 | 29 | this.model = new SlieroVadOnnxModel(); 30 | this.startThreshold = startThreshold; 31 | this.endThreshold = endThreshold; 32 | this.samplingRate = samplingRate; 33 | this.minSilenceSamples = samplingRate * minSilenceDurationMs / 1000f; 34 | this.speechPadSamples = samplingRate * speechPadMs / 1000f; 35 | 36 | Reset(); 37 | } 38 | 39 | public void Reset() 40 | { 41 | model.ResetStates(); 42 | triggered = false; 43 | tempEnd = 0; 44 | currentSample = 0; 45 | } 46 | 47 | public Dictionary Apply(byte[] data, bool returnSeconds) 48 | { 49 | float[] audioData = new float[data.Length / 2]; 50 | for (int i = 0; i < audioData.Length; i++) 51 | { 52 | audioData[i] = ((data[i * 2] & 0xff) | (data[i * 2 + 1] << 8)) / 32767.0f; 53 | } 54 | 55 | int windowSizeSamples = audioData.Length; 56 | currentSample += windowSizeSamples; 57 | 58 | float speechProb = 0; 59 | try 60 | { 61 | speechProb = model.Call(new float[][] { audioData }, samplingRate)[0]; 62 | } 63 | catch (Exception ex) 64 | { 65 | throw new InvalidOperationException("An error occurred while calling the model", ex); 66 | } 67 | 68 | if (speechProb >= startThreshold && tempEnd != 0) 69 | { 70 | tempEnd = 0; 71 | } 72 | 73 | if (speechProb >= startThreshold && !triggered) 74 | { 75 | triggered = true; 76 | int speechStart = (int)(currentSample - speechPadSamples); 77 | speechStart = Math.Max(speechStart, 0); 78 | 79 | Dictionary result = new Dictionary(); 80 | if (returnSeconds) 81 | { 82 | double speechStartSeconds = speechStart / (double)samplingRate; 83 | double roundedSpeechStart = Math.Round(speechStartSeconds, 1, MidpointRounding.AwayFromZero); 84 | result["start"] = roundedSpeechStart; 85 | } 86 | else 87 | { 88 | result["start"] = speechStart; 89 | } 90 | 91 | return result; 92 | } 93 | 94 | if (speechProb < endThreshold && triggered) 95 | { 96 | if (tempEnd == 0) 97 | { 98 | tempEnd = currentSample; 99 | } 100 | 101 | if (currentSample - tempEnd < minSilenceSamples) 102 | { 103 | return new Dictionary(); 104 | } 105 | else 106 | { 107 | int speechEnd = (int)(tempEnd + speechPadSamples); 108 | tempEnd = 0; 109 | triggered = false; 110 | 111 | Dictionary result = new Dictionary(); 112 | 113 | if (returnSeconds) 114 | { 115 | double speechEndSeconds = speechEnd / (double)samplingRate; 116 | double roundedSpeechEnd = Math.Round(speechEndSeconds, 1, MidpointRounding.AwayFromZero); 117 | result["end"] = roundedSpeechEnd; 118 | } 119 | else 120 | { 121 | result["end"] = speechEnd; 122 | } 123 | return result; 124 | } 125 | } 126 | 127 | return new Dictionary(); 128 | } 129 | 130 | public void Close() 131 | { 132 | Reset(); 133 | model.Close(); 134 | } 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /Notes/AI/Whisper/VoiceActivity/SlieroVadOnnxModel.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.ML.OnnxRuntime; 2 | using Microsoft.ML.OnnxRuntime.Tensors; 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Diagnostics; 6 | using System.Linq; 7 | 8 | namespace Notes.AI.VoiceRecognition.VoiceActivity 9 | { 10 | public class SlieroVadOnnxModel : IDisposable 11 | { 12 | private readonly InferenceSession session; 13 | private Tensor h; 14 | private Tensor c; 15 | private int lastSr = 0; 16 | private int lastBatchSize = 0; 17 | private static readonly List SampleRates = new List { 8000, 16000 }; 18 | 19 | public SlieroVadOnnxModel() 20 | { 21 | var modelPath = $@"{AppDomain.CurrentDomain.BaseDirectory}onnx-models\whisper\silero_vad.onnx"; 22 | 23 | var options = new SessionOptions(); 24 | options.InterOpNumThreads = 1; 25 | options.IntraOpNumThreads = 1; 26 | options.GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_EXTENDED; 27 | session = new InferenceSession(modelPath, options); 28 | ResetStates(); 29 | } 30 | 31 | public void ResetStates() 32 | { 33 | try 34 | { 35 | h = new DenseTensor(new[] { 2, 1, 64 }); 36 | c = new DenseTensor(new[] { 2, 1, 64 }); 37 | lastSr = 0; 38 | lastBatchSize = 0; 39 | } 40 | catch (Exception ex) 41 | { 42 | 43 | } 44 | } 45 | 46 | public void Close() 47 | { 48 | session.Dispose(); 49 | } 50 | 51 | public class ValidationResult 52 | { 53 | public readonly float[][] X; 54 | public readonly int Sr; 55 | 56 | public ValidationResult(float[][] x, int sr) 57 | { 58 | X = x; 59 | Sr = sr; 60 | } 61 | } 62 | 63 | private ValidationResult ValidateInput(float[][] x, int sr) 64 | { 65 | if (x.Length == 1) 66 | { 67 | x = [x[0]]; 68 | } 69 | if (x.Length > 2) 70 | { 71 | throw new ArgumentException($"Incorrect audio data dimension: {x.Length}"); 72 | } 73 | 74 | if (sr != 16000 && sr % 16000 == 0) 75 | { 76 | int step = sr / 16000; 77 | float[][] reducedX = x.Select(row => row.Where((_, i) => i % step == 0).ToArray()).ToArray(); 78 | x = reducedX; 79 | sr = 16000; 80 | } 81 | 82 | if (!SampleRates.Contains(sr)) 83 | { 84 | throw new ArgumentException($"Only supports sample rates {String.Join(", ", SampleRates)} (or multiples of 16000)"); 85 | } 86 | 87 | if ((float)sr / x[0].Length > 31.25) 88 | { 89 | throw new ArgumentException("Input audio is too short"); 90 | } 91 | 92 | return new ValidationResult(x, sr); 93 | } 94 | 95 | public float[] Call(float[][] x, int sr) 96 | { 97 | 98 | var result = ValidateInput(x, sr); 99 | x = result.X; 100 | sr = result.Sr; 101 | 102 | int batchSize = x.Length; 103 | int sampleSize = x[0].Length; // Assuming all subarrays have identical length 104 | 105 | if (lastBatchSize == 0 || lastSr != sr || lastBatchSize != batchSize) 106 | { 107 | ResetStates(); 108 | } 109 | 110 | // Flatten the jagged array and create the tensor with the correct shape 111 | var flatArray = x.SelectMany(inner => inner).ToArray(); 112 | var inputTensor = new DenseTensor(flatArray, [batchSize, sampleSize]); 113 | 114 | // Convert sr to a tensor, if the model expects a scalar as a single-element tensor, ensure matching the expected dimensions 115 | var srTensor = new DenseTensor(new long[] { sr }, [1]); 116 | 117 | 118 | var inputs = new List 119 | { 120 | NamedOnnxValue.CreateFromTensor("input", inputTensor), 121 | NamedOnnxValue.CreateFromTensor("sr", srTensor), 122 | NamedOnnxValue.CreateFromTensor("h", h), 123 | NamedOnnxValue.CreateFromTensor("c", c) 124 | }; 125 | 126 | try 127 | { 128 | using (var results = session.Run(inputs)) 129 | { 130 | var output = results.First().AsEnumerable().ToArray(); 131 | h = results.ElementAt(1).AsTensor(); 132 | c = results.ElementAt(2).AsTensor(); 133 | 134 | lastSr = sr; 135 | lastBatchSize = batchSize; 136 | 137 | return output; 138 | } 139 | } 140 | catch (Exception ex) 141 | { 142 | throw new InvalidOperationException("An error occurred while calling the model", ex); 143 | } 144 | } 145 | 146 | public static int count = 0; 147 | 148 | public void Dispose() 149 | { 150 | session?.Dispose(); 151 | GC.SuppressFinalize(this); 152 | } 153 | } 154 | } -------------------------------------------------------------------------------- /Notes/Controls/AttachmentView.xaml: -------------------------------------------------------------------------------- 1 | 2 | 10 | 11 | 12 | 18 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 54 | 55 | 61 | 62 | 67 | 72 | 73 | 80 | 81 | 82 | 83 | 86 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 100 | 101 | 106 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AI Powered Notes app [Sample] 2 | 3 | This sample is a simple note taking app that uses local APIs and models to provide AI powered features. The app is built using WinUI3. 4 | 5 | ![image](https://github.com/microsoft/ai-powered-notes-winui3-sample/assets/711864/19839b9a-34fe-4330-94d4-d4e0baf6c94d) 6 | 7 | Watch the Build session: [Use AI for "Real Things" in your Windows apps](https://www.youtube.com/watch?v=st7aIx8B4Rk) 8 | 9 | ## Set Up 10 | 11 | You will need to have Visual Studio installed with the latest workloads for WinAppSDK and WinUI 3 development. You can find instructions on how to set up your environment [here.](https://learn.microsoft.com/en-us/windows/apps/windows-app-sdk/set-up-your-development-environment?tabs=cs-vs-community%2Ccpp-vs-community%2Cvs-2022-17-1-a%2Cvs-2022-17-1-b#install-visual-studio) 12 | 13 | Clone the repository and open the solution in Visual Studio. Before you can get started exploring the sample, you will need to download the ML model files required for the project and place them in the `onnx-models` folder. 14 | 15 | The final folder structure should look like this: 16 | 17 | ![image](https://github.com/user-attachments/assets/05436579-9bf9-4dc0-a30d-24b1c4006d19) 18 | 19 | 20 | > [!NOTE] 21 | > Many of these models can be downloaded quickly using the [AI Dev Gallery](https://github.com/microsoft/ai-dev-gallery). 22 | 23 |

24 | AI Dev Gallery 25 |

26 |

27 | 28 | Store badge 29 | 30 |

31 | 32 | ## Phi Silica 33 | Phi Silica is disabled by deault, but you can enable it in `App.xaml.cs` in the `InitializeIChatClient` method. To use Phi Silica for the generative tasks, see requirements here: https://learn.microsoft.com/en-us/windows/ai/apis/phi-silica 34 | 35 | ## Downloading Phi3.5 (or other GenAI model) 36 | 37 | The model can be downloaded from the following link: 38 | - [https://huggingface.co/microsoft/Phi-3.5-mini-instruct-onnx](https://huggingface.co/microsoft/Phi-3.5-mini-instruct-onnx) 39 | 40 | Use the AI Dev Gallery linked above to download the model files. Alternatively, Huggingface models are in repositories which you can clone to get the model files. Clone the model repository and copy the required files to this project. 41 | 42 | > [!NOTE] 43 | > You can use any GenAI model by downloading the right model files and droping them in the `genai-model` folder. If using a model other than phi, make sure to also update the prompt template in the `App.xaml.cs` 44 | 45 | ## Downloading all-MiniLM-L6-v2 46 | The model can be downloaded from the following link: 47 | - [https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) 48 | 49 | This is model we use for semantic search. The two files you will need are `model.onnx` and `vocab.txt`. Create a new folder under `onnx-models` called `embedding` and place the files there. 50 | 51 | ## Downloading Sliero VAD 52 | The Sliero Voice Activity Detection model can be downloaded from the following link: 53 | - [https://github.com/snakers4/silero-vad ](https://github.com/snakers4/silero-vad/tree/a9d2b591dea11451d23aa4b480eff8e55dbd9d99/files) 54 | 55 | This is the model we use for smart chunking of audio and the only file you will need is the `sliero_vad.onnx` file. 56 | 57 | This should also be placed under a new folder called `whisper` under the `onnx-models` folder. 58 | 59 | 60 | ## Downloading Whisper 61 | The Whisper model can be downloaded from the following link: 62 | - [https://huggingface.co/khmyznikov/whisper-int8-cpu-ort.onnx](https://huggingface.co/khmyznikov/whisper-int8-cpu-ort.onnx) 63 | 64 | Download any of the versions on the repo or from the AI Dev Gallery and place them in the `onnx-models\whisper` folder. Make sure the path in `AI\Whisper\Whisper.cs` in the `InitializeModel` method reflects the same name: 65 | image 66 | 67 | 68 | ## Troubleshooting 69 | 70 | ### Path name too long 71 | You might run into an issue if you clone the repo in a location that will make the path too long to some of the generated binaries. Recomendation is to place the repo closer to the root of the drive and rename the repo folder name to something shorter. Alternatively, you can change the settings in Windows to support long paths 72 | https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=registry#enable-long-paths-in-windows-10-version-1607-and-later . 73 | 74 | ## Contributing 75 | 76 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 77 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 78 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 79 | 80 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 81 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 82 | provided by the bot. You will only need to do this once across all repos using our CLA. 83 | 84 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 85 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 86 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 87 | 88 | ## Trademarks 89 | 90 | This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft 91 | trademarks or logos is subject to and must follow 92 | [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). 93 | Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. 94 | Any use of third-party trademarks or logos are subject to those third-party's policies. 95 | -------------------------------------------------------------------------------- /Notes/MainWindow.xaml: -------------------------------------------------------------------------------- 1 | 2 | 13 | 14 | 15 | 16 | 17 | 18 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 38 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 61 | 71 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | -------------------------------------------------------------------------------- /Notes/AI/Whisper/VoiceActivity/WhisperChunking.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Linq; 5 | 6 | namespace Notes.AI.VoiceRecognition.VoiceActivity 7 | { 8 | public class DetectionResult 9 | { 10 | public string Type { get; set; } 11 | public double Seconds { get; set; } 12 | } 13 | 14 | public class WhisperChunk 15 | { 16 | public double Start { get; set; } 17 | public double End { get; set; } 18 | 19 | public WhisperChunk(double start, double end) 20 | { 21 | this.Start = start; 22 | this.End = end; 23 | } 24 | 25 | public double Length => End - Start; 26 | 27 | } 28 | public static class WhisperChunking 29 | { 30 | private static int SAMPLE_RATE = 16000; 31 | private static float START_THRESHOLD = 0.25f; 32 | private static float END_THRESHOLD = 0.25f; 33 | private static int MIN_SILENCE_DURATION_MS = 1000; 34 | private static int SPEECH_PAD_MS = 400; 35 | private static int WINDOW_SIZE_SAMPLES = 3200; 36 | 37 | private static double MAX_CHUNK_S = 29; 38 | private static double MIN_CHUNK_S = 5; 39 | 40 | public static List SmartChunking(byte[] audioBytes) 41 | { 42 | SlieroVadDetector vadDetector; 43 | vadDetector = new SlieroVadDetector(START_THRESHOLD, END_THRESHOLD, SAMPLE_RATE, MIN_SILENCE_DURATION_MS, SPEECH_PAD_MS); 44 | 45 | int bytesPerSample = 2; 46 | int bytesPerWindow = WINDOW_SIZE_SAMPLES * bytesPerSample; 47 | 48 | float totalSeconds = audioBytes.Length / (SAMPLE_RATE * 2); 49 | var result = new List(); 50 | var sw = Stopwatch.StartNew(); 51 | for (int offset = 0; offset + bytesPerWindow <= audioBytes.Length; offset += bytesPerWindow) 52 | { 53 | byte[] data = new byte[bytesPerWindow]; 54 | Array.Copy(audioBytes, offset, data, 0, bytesPerWindow); 55 | 56 | // Simulating the process as if data was being read in chunks 57 | try 58 | { 59 | var detectResult = vadDetector.Apply(data, true); 60 | // iterate over detectResult and apply the data to result: 61 | foreach (var (key, value) in detectResult) 62 | { 63 | result.Add(new DetectionResult { Type = key, Seconds = value }); 64 | } 65 | } 66 | catch (Exception e) 67 | { 68 | Console.Error.WriteLine($"Error applying VAD detector: {e.Message}"); 69 | // Depending on the need, you might want to break out of the loop or just report the error 70 | } 71 | } 72 | sw.Stop(); 73 | Debug.WriteLine($"VAD detection took {sw.ElapsedMilliseconds} ms"); 74 | var stamps = GetTimeStamps(result, totalSeconds, MAX_CHUNK_S, MIN_CHUNK_S); 75 | return stamps; 76 | } 77 | private static List GetTimeStamps(List voiceAreas, double totalSeconds, double maxChunkLength, double minChunkLength) 78 | { 79 | //const int maxLength = 30; 80 | //List chunks = new(); 81 | //int currChunk = 1; 82 | //double startTime = 0; 83 | //for(int i=1;i startTime + maxLength && chunks.Count < currChunk) 86 | // { 87 | // chunks.Add(new Chunk(startTime, currChunk * maxLength)); 88 | // currChunk++; 89 | // startTime = currChunk * maxLength; 90 | // } 91 | // //TODO: This is a very basic check, we can check for a threshold of values instead, Amrutha will work on that 92 | // if (voiceAreas[i].Seconds <= startTime + maxLength && (i == voiceAreas.Count - 1 || voiceAreas[i + 1].Seconds > startTime + maxLength)) { 93 | // chunks.Add(new Chunk(startTime, voiceAreas[i].Seconds)); 94 | // currChunk++; 95 | // startTime = voiceAreas[i].Seconds; 96 | // } 97 | //} 98 | 99 | //double j; 100 | ////Sometimes the last chunk is really large 101 | //for(j=startTime; j { new WhisperChunk(0, totalSeconds) }; 110 | } 111 | 112 | voiceAreas = voiceAreas.OrderBy(va => va.Seconds).ToList(); 113 | 114 | List chunks = new List(); 115 | 116 | double nextChunkStart = 0.0; 117 | while (nextChunkStart < totalSeconds) 118 | { 119 | double idealChunkEnd = nextChunkStart + maxChunkLength; 120 | double chunkEnd = idealChunkEnd > totalSeconds ? totalSeconds : idealChunkEnd; 121 | 122 | var validVoiceAreas = voiceAreas.Where(va => va.Seconds > nextChunkStart && va.Seconds <= chunkEnd).ToList(); 123 | 124 | if (validVoiceAreas.Any()) 125 | { 126 | chunkEnd = validVoiceAreas.Last().Seconds; 127 | } 128 | 129 | chunks.Add(new WhisperChunk(nextChunkStart, chunkEnd)); 130 | nextChunkStart = chunkEnd + 0.1; 131 | } 132 | 133 | return MergeSmallChunks(chunks, maxChunkLength, minChunkLength); 134 | } 135 | 136 | private static List MergeSmallChunks(List chunks, double maxChunkLength, double minChunkLength) 137 | { 138 | for (int i = 1; i < chunks.Count; i++) 139 | { 140 | // Check if current chunk is small and can be merged with previous 141 | if (chunks[i].Length < minChunkLength) 142 | { 143 | double prevChunkLength = chunks[i - 1].Length; 144 | double combinedLength = prevChunkLength + chunks[i].Length; 145 | 146 | if (combinedLength <= maxChunkLength) 147 | { 148 | chunks[i - 1].End = chunks[i].End; // Merge with previous chunk 149 | chunks.RemoveAt(i); // Remove current chunk 150 | i--; // Adjust index to recheck current position now pointing to next chunk 151 | } 152 | } 153 | } 154 | 155 | return chunks; 156 | } 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /Notes/Utils/Utils.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.VisualBasic; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | using Notes.AI.Embeddings; 8 | using Windows.Storage; 9 | 10 | namespace Notes 11 | { 12 | internal partial class Utils 13 | { 14 | public static readonly string FolderName = "MyNotes"; 15 | public static readonly string FileExtension = ".txt"; 16 | public static readonly string StateFolderName = ".notes"; 17 | public static readonly string AttachmentsFolderName = "attachments"; 18 | 19 | private static string localFolderPath = string.Empty; 20 | 21 | public static async Task GetLocalFolderPathAsync() 22 | { 23 | if (string.IsNullOrWhiteSpace(localFolderPath)) 24 | { 25 | localFolderPath = (await GetLocalFolderAsync()).Path; 26 | } 27 | 28 | return localFolderPath; 29 | } 30 | 31 | public static async Task GetLocalFolderAsync() 32 | { 33 | return await KnownFolders.DocumentsLibrary.CreateFolderAsync(FolderName, CreationCollisionOption.OpenIfExists); 34 | } 35 | 36 | public static async Task GetStateFolderAsync() 37 | { 38 | var notesFolder = await GetLocalFolderAsync(); 39 | return await notesFolder.CreateFolderAsync(StateFolderName, CreationCollisionOption.OpenIfExists); 40 | } 41 | 42 | public static async Task GetAttachmentsFolderAsync() 43 | { 44 | var notesFolder = await GetLocalFolderAsync(); 45 | return await notesFolder.CreateFolderAsync(AttachmentsFolderName, CreationCollisionOption.OpenIfExists); 46 | } 47 | 48 | public static async Task GetAttachmentsTranscriptsFolderAsync() 49 | { 50 | var notesFolder = await GetStateFolderAsync(); 51 | return await notesFolder.CreateFolderAsync(AttachmentsFolderName, CreationCollisionOption.OpenIfExists); 52 | } 53 | 54 | public static async Task> SearchAsync(string query, int top = 5) 55 | { 56 | var results = new List(); 57 | if (string.IsNullOrWhiteSpace(query)) 58 | { 59 | return results; 60 | } 61 | 62 | // TODO: handle cancelation 63 | var searchVectors = await SemanticIndex.Instance.Search(query, top); 64 | var context = await AppDataContext.GetCurrentAsync(); 65 | 66 | while (searchVectors.Count > 0) 67 | { 68 | var searchVector = searchVectors[0]; 69 | 70 | var sameContent = searchVectors 71 | .Where(r => r.ContentType == searchVector.ContentType && r.SourceId == searchVector.SourceId) 72 | .OrderBy(r => r.ChunkIndexInSource) 73 | .ToList(); 74 | 75 | var content = new StringBuilder(); 76 | 77 | int previousSourceIndex = sameContent.First().ChunkIndexInSource; 78 | content.Append(sameContent.First().Text); 79 | searchVectors.Remove(sameContent.First()); 80 | sameContent.RemoveAt(0); 81 | 82 | while (sameContent.Count > 0) 83 | { 84 | var currentContent = sameContent.First(); 85 | 86 | if (currentContent.ChunkIndexInSource == previousSourceIndex + 1) 87 | { 88 | content.Append(currentContent.Text3 ?? ""); 89 | } 90 | else if (currentContent.ChunkIndexInSource == previousSourceIndex + 2) 91 | { 92 | content.Append(currentContent.Text2 ?? ""); 93 | content.Append(currentContent.Text3 ?? ""); 94 | } 95 | else 96 | { 97 | content.Append(currentContent.Text); 98 | } 99 | 100 | previousSourceIndex = currentContent.ChunkIndexInSource; 101 | searchVectors.Remove(currentContent); 102 | sameContent.RemoveAt(0); 103 | } 104 | 105 | var searchResult = new SearchResult(); 106 | searchResult.Content = content.ToString(); 107 | 108 | if (searchVector.ContentType == "note") 109 | { 110 | var note = await context.Notes.FindAsync(searchVector.SourceId); 111 | 112 | searchResult.ContentType = ContentType.Note; 113 | searchResult.SourceId = note.Id; 114 | searchResult.Title = note.Title; 115 | } 116 | else if (searchVector.ContentType == "attachment") 117 | { 118 | var attachment = await context.Attachments.FindAsync(searchVector.SourceId); 119 | 120 | searchResult.ContentType = (ContentType)attachment.Type; 121 | searchResult.SourceId = attachment.Id; 122 | searchResult.Title = attachment.Filename; 123 | } 124 | 125 | var topSentence = await SubSearchAsync(query, searchResult.Content); 126 | searchResult.MostRelevantSentence = topSentence; 127 | results.Add(searchResult); 128 | 129 | } 130 | 131 | return results; 132 | } 133 | 134 | public static async Task SubSearchAsync(string query, string text) 135 | { 136 | var sentences = text.Split(new string[] { "." }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); 137 | var vectors = await Embeddings.Instance.GetEmbeddingsAsync(sentences); 138 | var searchVector = await Embeddings.Instance.GetEmbeddingsAsync(new string[] { query }); 139 | 140 | var ranking = SemanticIndex.CalculateRanking(searchVector[0], vectors.ToList()); 141 | 142 | return sentences[ranking[0]]; 143 | } 144 | } 145 | public record SearchResult 146 | { 147 | public string Title { get; set; } 148 | public string? Content { get; set; } 149 | public string? MostRelevantSentence { get; set; } 150 | public int SourceId { get; set; } 151 | public ContentType ContentType { get; set; } 152 | 153 | public static string ContentTypeToGlyph(ContentType type) 154 | { 155 | return type switch 156 | { 157 | ContentType.Note => "📝", 158 | ContentType.Image => "🖼️", 159 | ContentType.Audio => "🎙️", 160 | ContentType.Video => "🎞️", 161 | ContentType.Document => "📄" 162 | }; 163 | } 164 | } 165 | 166 | public enum ContentType 167 | { 168 | Image = 0, 169 | Audio = 1, 170 | Video = 2, 171 | Document = 3, 172 | Note = 4, 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /Notes/AI/Whisper/WhisperUtils.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Text.RegularExpressions; 6 | using System.Threading.Tasks; 7 | 8 | namespace Notes.AI.VoiceRecognition 9 | { 10 | internal static class WhisperUtils 11 | { 12 | private static Dictionary languageCodes = new() 13 | { 14 | {"English", "en"}, 15 | {"Serbian", "sr"}, 16 | {"Hindi", "hi"}, 17 | {"Spanish", "es"}, 18 | {"Russian", "ru"}, 19 | {"Korean", "ko"}, 20 | {"French", "fr"}, 21 | {"Japanese", "ja"}, 22 | {"Portuguese", "pt"}, 23 | {"Turkish", "tr"}, 24 | {"Polish", "pl"}, 25 | {"Catalan", "ca"}, 26 | {"Dutch", "nl"}, 27 | {"Arabic", "ar"}, 28 | {"Swedish", "sv"}, 29 | {"Italian", "it"}, 30 | {"Indonesian", "id"}, 31 | {"Macedonian", "mk" }, 32 | {"Mandarin", "zh" } 33 | }; 34 | 35 | public static int GetLangId(string languageString) 36 | { 37 | int langId = 50259; 38 | Dictionary langToId = new Dictionary 39 | { 40 | {"af", 50327}, 41 | {"am", 50334}, 42 | {"ar", 50272}, 43 | {"as", 50350}, 44 | {"az", 50304}, 45 | {"ba", 50355}, 46 | {"be", 50330}, 47 | {"bg", 50292}, 48 | {"bn", 50302}, 49 | {"bo", 50347}, 50 | {"br", 50309}, 51 | {"bs", 50315}, 52 | {"ca", 50270}, 53 | {"cs", 50283}, 54 | {"cy", 50297}, 55 | {"da", 50285}, 56 | {"de", 50261}, 57 | {"el", 50281}, 58 | {"en", 50259}, 59 | {"es", 50262}, 60 | {"et", 50307}, 61 | {"eu", 50310}, 62 | {"fa", 50300}, 63 | {"fi", 50277}, 64 | {"fo", 50338}, 65 | {"fr", 50265}, 66 | {"gl", 50319}, 67 | {"gu", 50333}, 68 | {"haw", 50352}, 69 | {"ha", 50354}, 70 | {"he", 50279}, 71 | {"hi", 50276}, 72 | {"hr", 50291}, 73 | {"ht", 50339}, 74 | {"hu", 50286}, 75 | {"hy", 50312}, 76 | {"id", 50275}, 77 | {"is", 50311}, 78 | {"it", 50274}, 79 | {"ja", 50266}, 80 | {"jw", 50356}, 81 | {"ka", 50329}, 82 | {"kk", 50316}, 83 | {"km", 50323}, 84 | {"kn", 50306}, 85 | {"ko", 50264}, 86 | {"la", 50294}, 87 | {"lb", 50345}, 88 | {"ln", 50353}, 89 | {"lo", 50336}, 90 | {"lt", 50293}, 91 | {"lv", 50301}, 92 | {"mg", 50349}, 93 | {"mi", 50295}, 94 | {"mk", 50308}, 95 | {"ml", 50296}, 96 | {"mn", 50314}, 97 | {"mr", 50320}, 98 | {"ms", 50282}, 99 | {"mt", 50343}, 100 | {"my", 50346}, 101 | {"ne", 50313}, 102 | {"nl", 50271}, 103 | {"nn", 50342}, 104 | {"no", 50288}, 105 | {"oc", 50328}, 106 | {"pa", 50321}, 107 | {"pl", 50269}, 108 | {"ps", 50340}, 109 | {"pt", 50267}, 110 | {"ro", 50284}, 111 | {"ru", 50263}, 112 | {"sa", 50344}, 113 | {"sd", 50332}, 114 | {"si", 50322}, 115 | {"sk", 50298}, 116 | {"sl", 50305}, 117 | {"sn", 50324}, 118 | {"so", 50326}, 119 | {"sq", 50317}, 120 | {"sr", 50303}, 121 | {"su", 50357}, 122 | {"sv", 50273}, 123 | {"sw", 50318}, 124 | {"ta", 50287}, 125 | {"te", 50299}, 126 | {"tg", 50331}, 127 | {"th", 50289}, 128 | {"tk", 50341}, 129 | {"tl", 50325}, 130 | {"tr", 50268}, 131 | {"tt", 50335}, 132 | {"ug", 50348}, 133 | {"uk", 50260}, 134 | {"ur", 50337}, 135 | {"uz", 50351}, 136 | {"vi", 50278}, 137 | {"xh", 50322}, 138 | {"yi", 50305}, 139 | {"yo", 50324}, 140 | {"zh", 50258}, 141 | {"zu", 50321} 142 | }; 143 | 144 | if (languageCodes.TryGetValue(languageString, out string langCode)) 145 | { 146 | langId = langToId[langCode]; 147 | } 148 | 149 | return langId; 150 | } 151 | 152 | public static List ProcessTranscriptionWithTimestamps(string transcription, double offsetSeconds = 0) 153 | { 154 | Regex pattern = new Regex(@"<\|([\d.]+)\|>([^<]+)<\|([\d.]+)\|>"); 155 | MatchCollection matches = pattern.Matches(transcription); 156 | List list = new(); 157 | for (int i = 0; i < matches.Count; i++) 158 | { 159 | // Parse the original start and end times 160 | double start = double.Parse(matches[i].Groups[1].Value); 161 | double end = double.Parse(matches[i].Groups[3].Value); 162 | string subtitle = string.IsNullOrEmpty(matches[i].Groups[2].Value) ? "" : matches[i].Groups[2].Value.Trim(); 163 | WhisperTranscribedChunk chunk = new() 164 | { 165 | Text = subtitle, 166 | Start = start + offsetSeconds, 167 | End = end + offsetSeconds 168 | }; 169 | list.Add(chunk); 170 | } 171 | return list; 172 | } 173 | 174 | public static List MergeTranscribedChunks(List chunks) 175 | { 176 | List list = new(); 177 | WhisperTranscribedChunk transcribedChunk = chunks[0]; 178 | 179 | for (int i = 1; i < chunks.Count; i++) 180 | { 181 | char lastCharOfPrev = transcribedChunk.Text[transcribedChunk.Text.Length - 1]; 182 | char firstCharOfNext = chunks[i].Text[0]; 183 | //Approach 1: Get full sentences together //Approach 2: Sliding window of desired duration 184 | if (char.IsLower(firstCharOfNext) || (lastCharOfPrev != '.' && lastCharOfPrev != '?' && lastCharOfPrev != '!')) 185 | { 186 | transcribedChunk.End = chunks[i].End; 187 | transcribedChunk.Text += " " + chunks[i].Text; 188 | } 189 | else 190 | { 191 | list.Add(transcribedChunk); 192 | transcribedChunk = chunks[i]; 193 | } 194 | } 195 | list.Add(transcribedChunk); 196 | 197 | return list; 198 | } 199 | } 200 | } 201 | -------------------------------------------------------------------------------- /Notes/Utils/AttachmentProcessor.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Linq; 5 | using System.Text.RegularExpressions; 6 | using System.Threading.Tasks; 7 | using Notes.AI.Embeddings; 8 | using Notes.Models; 9 | using Notes.AI.VoiceRecognition; 10 | using Windows.Storage; 11 | using Windows.Storage.Streams; 12 | using Windows.Graphics.Imaging; 13 | using Notes.AI.TextRecognition; 14 | using System.Text.Json; 15 | 16 | namespace Notes 17 | { 18 | public static class AttachmentProcessor 19 | { 20 | private static List _toBeProcessed = new(); 21 | private static bool _isProcessing = false; 22 | 23 | public static EventHandler AttachmentProcessed; 24 | 25 | public async static Task AddAttachment(Attachment attachment) 26 | { 27 | _toBeProcessed.Add(attachment); 28 | 29 | if (!_isProcessing) 30 | { 31 | try 32 | { 33 | _isProcessing = true; 34 | await Process(); 35 | } 36 | catch (Exception ex) 37 | { 38 | Debug.WriteLine($"Error processing attachment: {ex.Message}"); 39 | } 40 | _isProcessing = false; 41 | } 42 | } 43 | 44 | private static async Task Process() 45 | { 46 | while (_toBeProcessed.Count > 0) 47 | { 48 | var attachment = _toBeProcessed[0]; 49 | _toBeProcessed.RemoveAt(0); 50 | 51 | if (attachment.IsProcessed) 52 | { 53 | continue; 54 | } 55 | 56 | if (attachment.Type == NoteAttachmentType.Image) 57 | { 58 | await ProcessImage(attachment); 59 | } 60 | else if (attachment.Type == NoteAttachmentType.Audio || attachment.Type == NoteAttachmentType.Video) 61 | { 62 | await ProcessAudio(attachment); 63 | } 64 | } 65 | } 66 | 67 | private static async Task ProcessImage(Models.Attachment attachment, EventHandler? progress = null) 68 | { 69 | // get softwarebitmap from file 70 | var attachmentsFolder = await Utils.GetAttachmentsFolderAsync(); 71 | var file = await attachmentsFolder.GetFileAsync(attachment.Filename); 72 | 73 | using (IRandomAccessStream stream = await file.OpenAsync(FileAccessMode.Read)) 74 | { 75 | BitmapDecoder decoder = await BitmapDecoder.CreateAsync(stream); 76 | SoftwareBitmap softwareBitmap = await decoder.GetSoftwareBitmapAsync(); 77 | 78 | var recognizedText = await TextRecognition.GetTextFromImage(softwareBitmap); 79 | if (recognizedText == null) 80 | { 81 | attachment.IsProcessed = true; 82 | InvokeAttachmentProcessedComplete(attachment); 83 | return; 84 | } 85 | 86 | var joinedText = string.Join("\n", recognizedText.Lines.Select(l => l.Text)); 87 | var serializedText = JsonSerializer.Serialize(recognizedText); 88 | 89 | var filename = await SaveTextToFileAsync(serializedText, file.DisplayName + ".txt"); 90 | attachment.FilenameForText = filename; 91 | 92 | await SemanticIndex.Instance.AddOrReplaceContent(joinedText, attachment.Id, "attachment", (o, p) => 93 | { 94 | if (progress != null) 95 | { 96 | progress.Invoke("Indexing image", 0.5f + (p / 2)); 97 | } 98 | }); 99 | attachment.IsProcessed = true; 100 | InvokeAttachmentProcessedComplete(attachment); 101 | 102 | var context = await AppDataContext.GetCurrentAsync(); 103 | context.Update(attachment); 104 | await context.SaveChangesAsync(); 105 | } 106 | } 107 | 108 | private static async Task ProcessAudio(Attachment attachment) 109 | { 110 | await Task.Run(async () => 111 | { 112 | var attachmentsFolder = await Utils.GetAttachmentsFolderAsync(); 113 | var file = await attachmentsFolder.GetFileAsync(attachment.Filename); 114 | 115 | var transcribedChunks = await Whisper.TranscribeAsync(file, (o, p) => 116 | { 117 | if (AttachmentProcessed != null) 118 | { 119 | AttachmentProcessed.Invoke(null, new AttachmentProcessedEventArgs 120 | { 121 | AttachmentId = attachment.Id, 122 | Progress = p / 2, 123 | ProcessingStep = "Transcribing audio" 124 | }); 125 | } 126 | }); 127 | 128 | var textToSave = string.Join("\n", transcribedChunks.Select(t => $@"<|{t.Start:0.00}|>{t.Text}<|{t.End:0.00}|>")); 129 | 130 | var filename = await SaveTextToFileAsync(textToSave, file.DisplayName + ".txt"); 131 | attachment.FilenameForText = filename; 132 | 133 | var textToIndex = string.Join(" ", transcribedChunks.Select(t => t.Text)); 134 | 135 | await SemanticIndex.Instance.AddOrReplaceContent(textToIndex, attachment.Id, "attachment", (o, p) => 136 | { 137 | if (AttachmentProcessed != null) 138 | { 139 | AttachmentProcessed.Invoke(null, new AttachmentProcessedEventArgs 140 | { 141 | AttachmentId = attachment.Id, 142 | Progress = 0.5f + p / 2, 143 | ProcessingStep = "Indexing audio transcript" 144 | }); 145 | } 146 | }); 147 | attachment.IsProcessed = true; 148 | InvokeAttachmentProcessedComplete(attachment); 149 | 150 | var context = await AppDataContext.GetCurrentAsync(); 151 | context.Update(attachment); 152 | await context.SaveChangesAsync(); 153 | }); 154 | } 155 | 156 | private async static Task SaveTextToFileAsync(string text, string filename) 157 | { 158 | var stateAttachmentsFolder = await Utils.GetAttachmentsTranscriptsFolderAsync(); 159 | 160 | var file = await stateAttachmentsFolder.CreateFileAsync(filename, CreationCollisionOption.GenerateUniqueName); 161 | await FileIO.WriteTextAsync(file, text); 162 | return file.Name; 163 | } 164 | 165 | private static void InvokeAttachmentProcessedComplete(Attachment attachment) 166 | { 167 | if (AttachmentProcessed != null) 168 | { 169 | AttachmentProcessed.Invoke(null, new AttachmentProcessedEventArgs 170 | { 171 | AttachmentId = attachment.Id, 172 | Progress = 1, 173 | ProcessingStep = "Complete" 174 | }); 175 | } 176 | } 177 | 178 | } 179 | 180 | public class AttachmentProcessedEventArgs 181 | { 182 | public int AttachmentId { get; set; } 183 | public float Progress { get; set; } 184 | public string? ProcessingStep { get; set; } 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.rsuser 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Mono auto generated files 17 | mono_crash.* 18 | 19 | # Build results 20 | [Dd]ebug/ 21 | [Dd]ebugPublic/ 22 | [Rr]elease/ 23 | [Rr]eleases/ 24 | x64/ 25 | x86/ 26 | [Ww][Ii][Nn]32/ 27 | [Aa][Rr][Mm]/ 28 | [Aa][Rr][Mm]64/ 29 | bld/ 30 | [Bb]in/ 31 | [Oo]bj/ 32 | [Oo]ut/ 33 | [Ll]og/ 34 | [Ll]ogs/ 35 | 36 | # Visual Studio 2015/2017 cache/options directory 37 | .vs/ 38 | # Uncomment if you have tasks that create the project's static files in wwwroot 39 | #wwwroot/ 40 | 41 | # Visual Studio 2017 auto generated files 42 | Generated\ Files/ 43 | 44 | # MSTest test Results 45 | [Tt]est[Rr]esult*/ 46 | [Bb]uild[Ll]og.* 47 | 48 | # NUnit 49 | *.VisualState.xml 50 | TestResult.xml 51 | nunit-*.xml 52 | 53 | # Build Results of an ATL Project 54 | [Dd]ebugPS/ 55 | [Rr]eleasePS/ 56 | dlldata.c 57 | 58 | # Benchmark Results 59 | BenchmarkDotNet.Artifacts/ 60 | 61 | # .NET Core 62 | project.lock.json 63 | project.fragment.lock.json 64 | artifacts/ 65 | 66 | # ASP.NET Scaffolding 67 | ScaffoldingReadMe.txt 68 | 69 | # StyleCop 70 | StyleCopReport.xml 71 | 72 | # Files built by Visual Studio 73 | *_i.c 74 | *_p.c 75 | *_h.h 76 | *.ilk 77 | *.meta 78 | *.obj 79 | *.iobj 80 | *.pch 81 | *.pdb 82 | *.ipdb 83 | *.pgc 84 | *.pgd 85 | *.rsp 86 | *.sbr 87 | *.tlb 88 | *.tli 89 | *.tlh 90 | *.tmp 91 | *.tmp_proj 92 | *_wpftmp.csproj 93 | *.log 94 | *.vspscc 95 | *.vssscc 96 | .builds 97 | *.pidb 98 | *.svclog 99 | *.scc 100 | 101 | # Chutzpah Test files 102 | _Chutzpah* 103 | 104 | # Visual C++ cache files 105 | ipch/ 106 | *.aps 107 | *.ncb 108 | *.opendb 109 | *.opensdf 110 | *.sdf 111 | *.cachefile 112 | *.VC.db 113 | *.VC.VC.opendb 114 | 115 | # Visual Studio profiler 116 | *.psess 117 | *.vsp 118 | *.vspx 119 | *.sap 120 | 121 | # Visual Studio Trace Files 122 | *.e2e 123 | 124 | # TFS 2012 Local Workspace 125 | $tf/ 126 | 127 | # Guidance Automation Toolkit 128 | *.gpState 129 | 130 | # ReSharper is a .NET coding add-in 131 | _ReSharper*/ 132 | *.[Rr]e[Ss]harper 133 | *.DotSettings.user 134 | 135 | # TeamCity is a build add-in 136 | _TeamCity* 137 | 138 | # DotCover is a Code Coverage Tool 139 | *.dotCover 140 | 141 | # AxoCover is a Code Coverage Tool 142 | .axoCover/* 143 | !.axoCover/settings.json 144 | 145 | # Coverlet is a free, cross platform Code Coverage Tool 146 | coverage*.json 147 | coverage*.xml 148 | coverage*.info 149 | 150 | # Visual Studio code coverage results 151 | *.coverage 152 | *.coveragexml 153 | 154 | # NCrunch 155 | _NCrunch_* 156 | .*crunch*.local.xml 157 | nCrunchTemp_* 158 | 159 | # MightyMoose 160 | *.mm.* 161 | AutoTest.Net/ 162 | 163 | # Web workbench (sass) 164 | .sass-cache/ 165 | 166 | # Installshield output folder 167 | [Ee]xpress/ 168 | 169 | # DocProject is a documentation generator add-in 170 | DocProject/buildhelp/ 171 | DocProject/Help/*.HxT 172 | DocProject/Help/*.HxC 173 | DocProject/Help/*.hhc 174 | DocProject/Help/*.hhk 175 | DocProject/Help/*.hhp 176 | DocProject/Help/Html2 177 | DocProject/Help/html 178 | 179 | # Click-Once directory 180 | publish/ 181 | 182 | # Publish Web Output 183 | *.[Pp]ublish.xml 184 | *.azurePubxml 185 | # Note: Comment the next line if you want to checkin your web deploy settings, 186 | # but database connection strings (with potential passwords) will be unencrypted 187 | *.pubxml 188 | *.publishproj 189 | 190 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 191 | # checkin your Azure Web App publish settings, but sensitive information contained 192 | # in these scripts will be unencrypted 193 | PublishScripts/ 194 | 195 | # NuGet Packages 196 | *.nupkg 197 | # NuGet Symbol Packages 198 | *.snupkg 199 | # The packages folder can be ignored because of Package Restore 200 | **/[Pp]ackages/* 201 | # except build/, which is used as an MSBuild target. 202 | !**/[Pp]ackages/build/ 203 | # Uncomment if necessary however generally it will be regenerated when needed 204 | #!**/[Pp]ackages/repositories.config 205 | # NuGet v3's project.json files produces more ignorable files 206 | *.nuget.props 207 | *.nuget.targets 208 | 209 | # Microsoft Azure Build Output 210 | csx/ 211 | *.build.csdef 212 | 213 | # Microsoft Azure Emulator 214 | ecf/ 215 | rcf/ 216 | 217 | # Windows Store app package directories and files 218 | AppPackages/ 219 | BundleArtifacts/ 220 | Package.StoreAssociation.xml 221 | _pkginfo.txt 222 | *.appx 223 | *.appxbundle 224 | *.appxupload 225 | 226 | # Visual Studio cache files 227 | # files ending in .cache can be ignored 228 | *.[Cc]ache 229 | # but keep track of directories ending in .cache 230 | !?*.[Cc]ache/ 231 | 232 | # Others 233 | ClientBin/ 234 | ~$* 235 | *~ 236 | *.dbmdl 237 | *.dbproj.schemaview 238 | *.jfm 239 | *.pfx 240 | *.publishsettings 241 | orleans.codegen.cs 242 | 243 | # Including strong name files can present a security risk 244 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 245 | #*.snk 246 | 247 | # Since there are multiple workflows, uncomment next line to ignore bower_components 248 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 249 | #bower_components/ 250 | 251 | # RIA/Silverlight projects 252 | Generated_Code/ 253 | 254 | # Backup & report files from converting an old project file 255 | # to a newer Visual Studio version. Backup files are not needed, 256 | # because we have git ;-) 257 | _UpgradeReport_Files/ 258 | Backup*/ 259 | UpgradeLog*.XML 260 | UpgradeLog*.htm 261 | ServiceFabricBackup/ 262 | *.rptproj.bak 263 | 264 | # SQL Server files 265 | *.mdf 266 | *.ldf 267 | *.ndf 268 | 269 | # Business Intelligence projects 270 | *.rdl.data 271 | *.bim.layout 272 | *.bim_*.settings 273 | *.rptproj.rsuser 274 | *- [Bb]ackup.rdl 275 | *- [Bb]ackup ([0-9]).rdl 276 | *- [Bb]ackup ([0-9][0-9]).rdl 277 | 278 | # Microsoft Fakes 279 | FakesAssemblies/ 280 | 281 | # GhostDoc plugin setting file 282 | *.GhostDoc.xml 283 | 284 | # Node.js Tools for Visual Studio 285 | .ntvs_analysis.dat 286 | node_modules/ 287 | 288 | # Visual Studio 6 build log 289 | *.plg 290 | 291 | # Visual Studio 6 workspace options file 292 | *.opt 293 | 294 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 295 | *.vbw 296 | 297 | # Visual Studio LightSwitch build output 298 | **/*.HTMLClient/GeneratedArtifacts 299 | **/*.DesktopClient/GeneratedArtifacts 300 | **/*.DesktopClient/ModelManifest.xml 301 | **/*.Server/GeneratedArtifacts 302 | **/*.Server/ModelManifest.xml 303 | _Pvt_Extensions 304 | 305 | # Paket dependency manager 306 | .paket/paket.exe 307 | paket-files/ 308 | 309 | # FAKE - F# Make 310 | .fake/ 311 | 312 | # CodeRush personal settings 313 | .cr/personal 314 | 315 | # Python Tools for Visual Studio (PTVS) 316 | __pycache__/ 317 | *.pyc 318 | 319 | # Cake - Uncomment if you are using it 320 | # tools/** 321 | # !tools/packages.config 322 | 323 | # Tabs Studio 324 | *.tss 325 | 326 | # Telerik's JustMock configuration file 327 | *.jmconfig 328 | 329 | # BizTalk build output 330 | *.btp.cs 331 | *.btm.cs 332 | *.odx.cs 333 | *.xsd.cs 334 | 335 | # OpenCover UI analysis results 336 | OpenCover/ 337 | 338 | # Azure Stream Analytics local run output 339 | ASALocalRun/ 340 | 341 | # MSBuild Binary and Structured Log 342 | *.binlog 343 | 344 | # NVidia Nsight GPU debugger configuration file 345 | *.nvuser 346 | 347 | # MFractors (Xamarin productivity tool) working folder 348 | .mfractor/ 349 | 350 | # Local History for Visual Studio 351 | .localhistory/ 352 | 353 | # BeatPulse healthcheck temp database 354 | healthchecksdb 355 | 356 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 357 | MigrationBackup/ 358 | 359 | # Ionide (cross platform F# VS Code tools) working folder 360 | .ionide/ 361 | 362 | # Fody - auto-generated XML schema 363 | FodyWeavers.xsd 364 | 365 | onnx-models -------------------------------------------------------------------------------- /Notes/Controls/Phi3View.xaml: -------------------------------------------------------------------------------- 1 | 2 | 11 | 12 | 13 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 38 | 39 | You 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 53 | 54 | AI 55 | 56 | 57 | 58 | Sources: 59 | 65 | 66 | 67 | 71 | 72 | 73 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 99 | 100 | 101 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 116 | 122 | 123 | 124 | 125 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | -------------------------------------------------------------------------------- /Notes/AI/Embeddings/SemanticIndex.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Diagnostics; 3 | using System.Threading.Tasks; 4 | using System; 5 | using System.Linq; 6 | using System.Numerics.Tensors; 7 | 8 | namespace Notes.AI.Embeddings 9 | { 10 | public partial class SemanticIndex 11 | { 12 | public static SemanticIndex Instance { get; } = new SemanticIndex(); 13 | 14 | public async Task> Search(string searchTerm, int top = 5) 15 | { 16 | List chunks = []; 17 | 18 | var dataContext = await AppDataContext.GetCurrentAsync(); 19 | var storedVectors = dataContext.TextChunks.Select(chunk => chunk).ToList(); 20 | 21 | var searchVectors = await Embeddings.Instance.GetEmbeddingsAsync(searchTerm).ConfigureAwait(false); 22 | var ranking = CalculateRanking(searchVectors.First(), storedVectors.Select(chunk => chunk.Vectors).ToList()); 23 | 24 | for (var i = 0; i < ranking.Length && i < top; i++) 25 | { 26 | chunks.Add(storedVectors[ranking[i]]); 27 | } 28 | 29 | return chunks; 30 | } 31 | 32 | public static int[] CalculateRanking(float[] searchVector, List vectors) 33 | { 34 | float[] scores = new float[vectors.Count]; 35 | int[] indexranks = new int[vectors.Count]; 36 | 37 | for (int i = 0; i < vectors.Count; i++) 38 | { 39 | var score = TensorPrimitives.CosineSimilarity(vectors[i], searchVector); 40 | scores[i] = (float)score; 41 | } 42 | 43 | var indexedFloats = scores.Select((value, index) => new { Value = value, Index = index }) 44 | .ToArray(); 45 | 46 | // Sort the indexed floats by value in descending order 47 | Array.Sort(indexedFloats, (a, b) => b.Value.CompareTo(a.Value)); 48 | 49 | // Extract the top k indices 50 | indexranks = indexedFloats.Select(item => item.Index).ToArray(); 51 | 52 | return indexranks; 53 | } 54 | 55 | private List SplitParagraphInChunks(string paragraph, int maxLength) 56 | { 57 | List textChunks = new(); 58 | 59 | var sentences = paragraph.Split(new[] { "." }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); 60 | var currentChunk = string.Empty; 61 | 62 | foreach (var sentence in sentences) 63 | { 64 | if (sentence.Length > maxLength) 65 | { 66 | if (currentChunk.Length > 0) 67 | { 68 | textChunks.Add(currentChunk); 69 | currentChunk = string.Empty; 70 | } 71 | 72 | sentence.Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToList().ForEach(word => 73 | { 74 | if (currentChunk.Length + word.Length > maxLength) 75 | { 76 | textChunks.Add(currentChunk); 77 | currentChunk = string.Empty; 78 | } 79 | 80 | currentChunk += word + " "; 81 | }); 82 | 83 | continue; 84 | } 85 | 86 | if (currentChunk.Length + sentence.Length > maxLength) 87 | { 88 | textChunks.Add(currentChunk); 89 | 90 | currentChunk = string.Empty; 91 | } 92 | 93 | currentChunk += sentence + ". "; 94 | } 95 | 96 | if (!string.IsNullOrWhiteSpace(currentChunk)) 97 | { 98 | textChunks.Add(currentChunk); 99 | } 100 | 101 | return textChunks; 102 | } 103 | 104 | private List SplitIntoOverlappingChunks(string content, int sourceId, string contentType) 105 | { 106 | // number of maximum characters in a chunk 107 | var maxLength = 500; 108 | var text = content; 109 | var paragraphs = text.Split(new[] { "\r", "\n", "\r\n" }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); 110 | 111 | List textChunks = new(); 112 | 113 | // make sure no paragraphs are longer than maxLength 114 | // if they are, split them into smaller chunks 115 | 116 | var currentChunk = string.Empty; 117 | 118 | foreach (var paragraph in paragraphs) 119 | { 120 | if (paragraph.Length > maxLength) 121 | { 122 | if (currentChunk.Length > 0) 123 | { 124 | textChunks.Add(currentChunk); 125 | currentChunk = string.Empty; 126 | } 127 | textChunks.AddRange(SplitParagraphInChunks(paragraph, maxLength)); 128 | continue; 129 | } 130 | 131 | if (currentChunk.Length + paragraph.Length >= maxLength) 132 | { 133 | textChunks.Add(currentChunk); 134 | 135 | currentChunk = string.Empty; 136 | } 137 | 138 | currentChunk += paragraph + "\n"; 139 | } 140 | 141 | if (!string.IsNullOrWhiteSpace(currentChunk)) 142 | { 143 | textChunks.Add(currentChunk); 144 | } 145 | 146 | List chunks = new(); 147 | 148 | // 3 at a time, with a sliding window of 1 149 | if (textChunks.Count <= 2) 150 | { 151 | chunks.Add(new TextChunk() 152 | { 153 | SourceId = sourceId, 154 | ContentType = contentType, 155 | Text1 = textChunks[0], 156 | Text2 = textChunks.Count > 1 ? textChunks[1] : null, 157 | ChunkIndexInSource = 0 158 | }); ; 159 | } 160 | else 161 | { 162 | for (int i = 0; i < textChunks.Count - 2; i++) 163 | { 164 | chunks.Add(new TextChunk() 165 | { 166 | SourceId = sourceId, 167 | ContentType = contentType, 168 | Text1 = textChunks[i], 169 | Text2 = textChunks[i + 1], 170 | Text3 = textChunks[i + 2], 171 | ChunkIndexInSource = i 172 | }); 173 | } 174 | } 175 | 176 | return chunks; 177 | 178 | } 179 | 180 | public async Task AddOrReplaceContent(string content, int sourceId, string contentType, EventHandler? progress = null) 181 | { 182 | var dataContext = await AppDataContext.GetCurrentAsync(); 183 | dataContext.TextChunks.RemoveRange(dataContext.TextChunks.Where(chunk => chunk.SourceId == sourceId && chunk.ContentType == contentType)); 184 | 185 | var stopwatch = Stopwatch.StartNew(); 186 | 187 | await Task.Run(async () => 188 | { 189 | List chunks = SplitIntoOverlappingChunks(content, sourceId, contentType); 190 | 191 | int chunkBatchSize = 32; 192 | for (int i = 0; i < chunks.Count; i += chunkBatchSize) 193 | { 194 | var chunkBatch = chunks.Skip(i).Take(chunkBatchSize).ToList(); 195 | var vectors = await Embeddings.Instance.GetEmbeddingsAsync(chunkBatch.Select(c => c.Text).ToArray()).ConfigureAwait(false); 196 | 197 | for (int j = 0; j < chunkBatch.Count; j++) 198 | { 199 | chunkBatch[j].Vectors = vectors[j]; 200 | dataContext.TextChunks.Add(chunkBatch[j]); 201 | } 202 | 203 | progress?.Invoke(this, (float)i / chunks.Count); 204 | } 205 | 206 | dataContext.SaveChanges(); 207 | 208 | }).ConfigureAwait(false); 209 | 210 | stopwatch.Stop(); 211 | Debug.WriteLine($"Indexing took {stopwatch.ElapsedMilliseconds} ms"); 212 | } 213 | } 214 | } 215 | -------------------------------------------------------------------------------- /Notes/ViewModels/NoteViewModel.cs: -------------------------------------------------------------------------------- 1 | using CommunityToolkit.Mvvm.ComponentModel; 2 | using Microsoft.UI.Dispatching; 3 | using Microsoft.UI.Xaml; 4 | using System; 5 | using System.Collections.ObjectModel; 6 | using System.Diagnostics; 7 | using System.Linq; 8 | using System.Threading.Tasks; 9 | using Notes.AI.Embeddings; 10 | using Notes.Models; 11 | using Windows.Graphics.Imaging; 12 | using Windows.Storage; 13 | using System.Collections.Generic; 14 | using Notes.AI; 15 | 16 | namespace Notes.ViewModels 17 | { 18 | public partial class NoteViewModel : ObservableObject 19 | { 20 | public readonly Note Note; 21 | 22 | [ObservableProperty] 23 | private ObservableCollection attachments = new(); 24 | 25 | [ObservableProperty] 26 | private ObservableCollection todos = new(); 27 | 28 | [ObservableProperty] 29 | private bool todosLoading = false; 30 | 31 | private DispatcherTimer _saveTimer; 32 | private bool _contentLoaded = false; 33 | 34 | public DispatcherQueue DispatcherQueue { get; set; } 35 | 36 | public NoteViewModel(Note note) 37 | { 38 | Note = note; 39 | _saveTimer = new DispatcherTimer(); 40 | _saveTimer.Interval = TimeSpan.FromSeconds(5); 41 | _saveTimer.Tick += SaveTimerTick; 42 | } 43 | 44 | public string Title 45 | { 46 | get => Note.Title; 47 | set => SetProperty(Note.Title, value, Note, (note, value) => 48 | { 49 | note.Title = value; 50 | HandleTitleChanged(value); 51 | }); 52 | } 53 | 54 | public DateTime Modified 55 | { 56 | get => Note.Modified; 57 | set => SetProperty(Note.Modified, value, Note, (note, value) => note.Modified = value); 58 | } 59 | 60 | [ObservableProperty] 61 | private string content; 62 | 63 | private async Task HandleTitleChanged(string value) 64 | { 65 | var folder = await Utils.GetLocalFolderAsync(); 66 | var file = await folder.GetFileAsync(Note.Filename); 67 | 68 | await file.RenameAsync(value.Trim() + Utils.FileExtension, NameCollisionOption.GenerateUniqueName); 69 | Note.Filename = file.Name; 70 | await AppDataContext.SaveCurrentAsync(); 71 | } 72 | 73 | private async Task SaveContentAsync() 74 | { 75 | var folder = await Utils.GetLocalFolderAsync(); 76 | var file = await folder.GetFileAsync(Note.Filename); 77 | await FileIO.WriteTextAsync(file, Content); 78 | } 79 | 80 | public async Task LoadContentAsync() 81 | { 82 | if (_contentLoaded) 83 | { 84 | return; 85 | } 86 | 87 | _contentLoaded = true; 88 | 89 | var folder = await Utils.GetLocalFolderAsync(); 90 | var file = await folder.GetFileAsync(Note.Filename); 91 | content = await FileIO.ReadTextAsync(file); 92 | 93 | var context = await AppDataContext.GetCurrentAsync(); 94 | var attachments = context.Attachments.Where(a => a.NoteId == Note.Id).ToList(); 95 | foreach (var attachment in attachments) 96 | { 97 | Attachments.Add(new AttachmentViewModel(attachment)); 98 | } 99 | } 100 | 101 | partial void OnContentChanged(string value) 102 | { 103 | _saveTimer.Stop(); 104 | _saveTimer.Start(); 105 | } 106 | 107 | public async Task AddAttachmentAsync(StorageFile file) 108 | { 109 | var attachmentsFolder = await Utils.GetAttachmentsFolderAsync(); 110 | bool shouldCopyFile = true; 111 | 112 | var attachment = new Attachment() 113 | { 114 | IsProcessed = false, 115 | Note = Note 116 | }; 117 | 118 | if (new string[] { ".png", ".jpg", ".jpeg"}.Contains(file.FileType)) 119 | { 120 | attachment.Type = NoteAttachmentType.Image; 121 | } 122 | else if (new string[] { ".mp3", ".wav", ".m4a", ".opus", ".waptt" }.Contains(file.FileType)) 123 | { 124 | attachment.Type = NoteAttachmentType.Audio; 125 | file = await Utils.SaveAudioFileAsWav(file, attachmentsFolder); 126 | shouldCopyFile = false; 127 | } 128 | else if (file.FileType == ".mp4") 129 | { 130 | attachment.Type = NoteAttachmentType.Video; 131 | } 132 | else 133 | { 134 | attachment.Type = NoteAttachmentType.Document; 135 | } 136 | 137 | if (shouldCopyFile && !file.Path.StartsWith(attachmentsFolder.Path)) 138 | { 139 | file = await file.CopyAsync(attachmentsFolder, file.Name, NameCollisionOption.GenerateUniqueName); 140 | } 141 | 142 | attachment.Filename = file.Name; 143 | 144 | Attachments.Add(new AttachmentViewModel(attachment)); 145 | 146 | var context = await AppDataContext.GetCurrentAsync(); 147 | await context.Attachments.AddAsync(attachment); 148 | 149 | await context.SaveChangesAsync(); 150 | 151 | AttachmentProcessor.AddAttachment(attachment); 152 | } 153 | 154 | public async Task RemoveAttachmentAsync(AttachmentViewModel attachmentViewModel) 155 | { 156 | Attachments.Remove(attachmentViewModel); 157 | 158 | var attachment = attachmentViewModel.Attachment; 159 | Note.Attachments.Remove(attachment); 160 | 161 | var attachmentsFolder = await Utils.GetAttachmentsFolderAsync(); 162 | var file = await attachmentsFolder.GetFileAsync(attachment.Filename); 163 | await file.DeleteAsync(); 164 | 165 | if (attachment.IsProcessed && !string.IsNullOrEmpty(attachment.FilenameForText)) 166 | { 167 | var attachmentsTranscriptFolder = await Utils.GetAttachmentsTranscriptsFolderAsync(); 168 | var transcriptFile = await attachmentsTranscriptFolder.GetFileAsync(attachment.FilenameForText); 169 | await transcriptFile.DeleteAsync(); 170 | } 171 | 172 | var context = await AppDataContext.GetCurrentAsync(); 173 | context.Attachments.Remove(attachment); 174 | context.TextChunks.RemoveRange(context.TextChunks.Where(tc => tc.SourceId == attachment.Id && tc.ContentType == "attachment")); 175 | 176 | await context.SaveChangesAsync(); 177 | } 178 | 179 | public async Task ShowTodos() 180 | { 181 | if (App.ChatClient == null) 182 | { 183 | return; 184 | } 185 | 186 | if (!TodosLoading && (Todos == null || Todos.Count == 0)) 187 | { 188 | DispatcherQueue.TryEnqueue(() => TodosLoading = true); 189 | var todos = await App.ChatClient.GetTodoItemsFromText(Content); 190 | if (todos != null && todos.Count > 0) 191 | { 192 | DispatcherQueue.TryEnqueue(() => Todos = new ObservableCollection(todos)); 193 | } 194 | } 195 | 196 | DispatcherQueue.TryEnqueue(() => TodosLoading = false); 197 | } 198 | 199 | public async Task AddAttachmentAsync(SoftwareBitmap bitmap) 200 | { 201 | // save bitmap to file 202 | var attachmentsFolder = await Utils.GetAttachmentsFolderAsync(); 203 | var file = await attachmentsFolder.CreateFileAsync(Guid.NewGuid().ToString() + ".png", CreationCollisionOption.GenerateUniqueName); 204 | using (var stream = await file.OpenAsync(FileAccessMode.ReadWrite)) 205 | { 206 | var encoder = await BitmapEncoder.CreateAsync(BitmapEncoder.PngEncoderId, stream); 207 | encoder.SetSoftwareBitmap(bitmap); 208 | await encoder.FlushAsync(); 209 | } 210 | 211 | await AddAttachmentAsync(file); 212 | } 213 | 214 | private void SaveTimerTick(object? sender, object e) 215 | { 216 | _saveTimer.Stop(); 217 | SaveContentToFileAndReIndex(); 218 | } 219 | 220 | private async Task SaveContentToFileAndReIndex() 221 | { 222 | var folder = await Utils.GetLocalFolderAsync(); 223 | var file = await folder.GetFileAsync(Note.Filename); 224 | 225 | Debug.WriteLine("Saving note " + Note.Title + " to filename " + Note.Filename); 226 | await FileIO.WriteTextAsync(file, Content); 227 | 228 | await SemanticIndex.Instance.AddOrReplaceContent(Content, Note.Id, "note", (o, p) => Debug.WriteLine($"Indexing note {Note.Title} {p * 100}%")); 229 | } 230 | } 231 | } 232 | -------------------------------------------------------------------------------- /Notes/Controls/Phi3View.xaml.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.UI.Xaml; 2 | using Microsoft.UI.Xaml.Controls; 3 | using Microsoft.UI.Xaml.Input; 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Collections.ObjectModel; 7 | using System.Linq; 8 | using System.Threading; 9 | using System.Threading.Tasks; 10 | using Notes.AI; 11 | 12 | // To learn more about WinUI, the WinUI project structure, 13 | // and more about our project templates, see: http://aka.ms/winui-project-info. 14 | 15 | namespace Notes.Controls 16 | { 17 | public sealed partial class Phi3View : UserControl 18 | { 19 | private CancellationTokenSource _cts; 20 | public ObservableCollection Sources { get; } = new ObservableCollection(); 21 | 22 | public Phi3View() 23 | { 24 | this.InitializeComponent(); 25 | } 26 | 27 | public async Task ShowAndSummarize(string text) 28 | { 29 | if (App.ChatClient == null) 30 | { 31 | return; 32 | } 33 | 34 | _cts = new CancellationTokenSource(); 35 | var token = _cts.Token; 36 | userPromptText.Text = $"Summarize \n \"{text.Substring(0, Math.Min(1000, text.Length))}...\""; 37 | aIResponseText.Text = "..."; 38 | userQuestionRoot.Visibility = Visibility.Visible; 39 | aIAnswerRoot.Visibility = Visibility.Visible; 40 | sourcesText.Visibility = Visibility.Collapsed; 41 | Sources.Clear(); 42 | 43 | this.Visibility = Visibility.Visible; 44 | 45 | await Task.Run(async () => 46 | { 47 | var firstPartial = true; 48 | 49 | await foreach (var partialResult in App.ChatClient.SummarizeTextAsync(text, token)) 50 | { 51 | if (token.IsCancellationRequested) 52 | { 53 | break; 54 | } 55 | 56 | DispatcherQueue.TryEnqueue(() => 57 | { 58 | if (firstPartial) 59 | { 60 | aIResponseText.Text = string.Empty; 61 | firstPartial = false; 62 | } 63 | 64 | aIResponseText.Text += partialResult; 65 | }); 66 | 67 | } 68 | }); 69 | } 70 | 71 | public async Task ShowForRag() 72 | { 73 | this.textBox.Text = string.Empty; 74 | userQuestionRoot.Visibility = Visibility.Collapsed; 75 | aIAnswerRoot.Visibility = Visibility.Collapsed; 76 | textBox.IsEnabled = true; 77 | textBoxRoot.Visibility = Visibility.Visible; 78 | Sources.Clear(); 79 | this.Visibility = Visibility.Visible; 80 | } 81 | 82 | internal async Task FixAndCleanUp(string text) 83 | { 84 | if (App.ChatClient == null) 85 | { 86 | return; 87 | } 88 | 89 | _cts = new CancellationTokenSource(); 90 | var token = _cts.Token; 91 | userPromptText.Text = $"Fix this text \n \"{text.Substring(0, Math.Min(1000, text.Length))}...\""; 92 | aIResponseText.Text = "..."; 93 | userQuestionRoot.Visibility = Visibility.Visible; 94 | aIAnswerRoot.Visibility = Visibility.Visible; 95 | Sources.Clear(); 96 | 97 | this.Visibility = Visibility.Visible; 98 | 99 | await Task.Run(async () => 100 | { 101 | var firstPartial = true; 102 | 103 | await foreach (var partialResult in App.ChatClient.FixAndCleanUpTextAsync(text, token)) 104 | { 105 | if (token.IsCancellationRequested) 106 | { 107 | break; 108 | } 109 | 110 | DispatcherQueue.TryEnqueue(() => 111 | { 112 | if (firstPartial) 113 | { 114 | aIResponseText.Text = string.Empty; 115 | firstPartial = false; 116 | } 117 | 118 | aIResponseText.Text += partialResult; 119 | }); 120 | 121 | } 122 | }); 123 | } 124 | 125 | private void Button_Click(object sender, RoutedEventArgs e) 126 | { 127 | Hide(); 128 | } 129 | 130 | public void Hide() 131 | { 132 | this.Visibility = Visibility.Collapsed; 133 | if (_cts != null && !_cts.IsCancellationRequested) 134 | { 135 | _cts.Cancel(); 136 | } 137 | 138 | textBoxRoot.Visibility = Visibility.Collapsed; 139 | } 140 | 141 | private void StopResponding_Clicked(object sender, RoutedEventArgs e) 142 | { 143 | _cts.Cancel(); 144 | } 145 | 146 | private void BackgroundTapped(object sender, TappedRoutedEventArgs e) 147 | { 148 | // hide the search view only when the backround was tapped but not any of the content inside 149 | if (e.OriginalSource == Root) 150 | this.Hide(); 151 | } 152 | 153 | private async void TextBox_KeyUp(object sender, KeyRoutedEventArgs e) 154 | { 155 | var textBox = sender as TextBox; 156 | if (e.Key == Windows.System.VirtualKey.Enter) 157 | { 158 | if (textBox.Text.Length > 0) 159 | { 160 | HandleRagQuestion(textBox.Text); 161 | 162 | } 163 | } 164 | } 165 | 166 | private async Task HandleRagQuestion(string question) 167 | { 168 | if (App.ChatClient == null) 169 | { 170 | return; 171 | } 172 | 173 | _cts = new CancellationTokenSource(); 174 | var token = _cts.Token; 175 | textBox.Text = string.Empty; 176 | textBox.IsEnabled = false; 177 | 178 | userPromptText.Text = question; 179 | userQuestionRoot.Visibility = Visibility.Visible; 180 | aIAnswerRoot.Visibility = Visibility.Visible; 181 | stopRespondingButton.Visibility = Visibility.Collapsed; 182 | sourcesText.Visibility = Visibility.Collapsed; 183 | aIResponseText.Text = "..."; 184 | Sources.Clear(); 185 | 186 | List foundSources = null; 187 | 188 | await Task.Run(async () => 189 | { 190 | var firstPartial = true; 191 | 192 | foundSources = await Utils.SearchAsync(question, top: 1); 193 | 194 | var information = string.Join(" ", foundSources.Select(chunk => chunk.Content).ToList()); 195 | var response = string.Empty; 196 | 197 | await foreach (var partialResult in App.ChatClient.AskForContentAsync(information, question, _cts.Token)) 198 | { 199 | if (token.IsCancellationRequested) 200 | { 201 | break; 202 | } 203 | 204 | response += partialResult; 205 | 206 | if (response.Contains("\n\n")) 207 | { 208 | _cts.Cancel(); 209 | break; 210 | } 211 | 212 | DispatcherQueue.TryEnqueue(() => 213 | { 214 | if (firstPartial) 215 | { 216 | stopRespondingButton.Visibility = Visibility.Visible; 217 | aIResponseText.Text = string.Empty; 218 | firstPartial = false; 219 | } 220 | 221 | aIResponseText.Text = response.Trim(); 222 | }); 223 | } 224 | }); 225 | 226 | if (foundSources != null && foundSources.Count > 0) 227 | { 228 | sourcesText.Visibility = Visibility.Visible; 229 | foreach (var result in foundSources) 230 | { 231 | if (Sources.Where(r => r.SourceId == result.SourceId).Count() == 0) 232 | { 233 | Sources.Add(result); 234 | } 235 | } 236 | } 237 | 238 | stopRespondingButton.Visibility = Visibility.Collapsed; 239 | textBox.IsEnabled = true; 240 | } 241 | 242 | private async void ListView_ItemClick(object sender, ItemClickEventArgs e) 243 | { 244 | var context = await AppDataContext.GetCurrentAsync(); 245 | 246 | var item = e.ClickedItem as SearchResult; 247 | if (item.ContentType == ContentType.Note) 248 | { 249 | MainWindow.Instance.SelectNoteById(item.SourceId); 250 | } 251 | else 252 | { 253 | var attachment = context.Attachments.Where(a => a.Id == item.SourceId).FirstOrDefault(); 254 | if (attachment != null) 255 | { 256 | var note = context.Notes.Where(n => n.Id == attachment.NoteId).FirstOrDefault(); 257 | MainWindow.Instance.SelectNoteById(note.Id, attachment.Id, item.MostRelevantSentence); 258 | } 259 | } 260 | 261 | this.Hide(); 262 | } 263 | 264 | } 265 | } 266 | -------------------------------------------------------------------------------- /Notes/AI/IChatClient/PhiSilicaClient.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.AI; 2 | using Microsoft.Windows.AI.ContentModeration; 3 | using Microsoft.Windows.AI.Generative; 4 | using System; 5 | using System.Collections.Generic; 6 | using System.Linq; 7 | using System.Runtime.CompilerServices; 8 | using System.Threading; 9 | using System.Threading.Tasks; 10 | using Windows.Foundation; 11 | 12 | namespace Notes.AI; 13 | 14 | internal class PhiSilicaClient : IChatClient 15 | { 16 | // Search Options 17 | private const LanguageModelSkill DefaultLanguageModelSkill = LanguageModelSkill.General; 18 | private const SeverityLevel DefaultInputModeration = SeverityLevel.None; 19 | private const SeverityLevel DefaultOutputModeration = SeverityLevel.None; 20 | private const int DefaultTopK = 50; 21 | private const float DefaultTopP = 0.9f; 22 | private const float DefaultTemperature = 1; 23 | 24 | private LanguageModel? _languageModel; 25 | private LanguageModelContext? _languageModelContext; 26 | 27 | public ChatClientMetadata Metadata { get; } 28 | 29 | private PhiSilicaClient() 30 | { 31 | Metadata = new ChatClientMetadata("PhiSilica", new Uri($"file:///PhiSilica")); 32 | } 33 | 34 | private static ChatOptions GetDefaultChatOptions() 35 | { 36 | return new ChatOptions 37 | { 38 | AdditionalProperties = new AdditionalPropertiesDictionary 39 | { 40 | { "skill", DefaultLanguageModelSkill }, 41 | { "input_moderation", DefaultInputModeration }, 42 | { "output_moderation", DefaultOutputModeration }, 43 | }, 44 | Temperature = DefaultTemperature, 45 | TopP = DefaultTopP, 46 | TopK = DefaultTopK, 47 | }; 48 | } 49 | 50 | public static async Task CreateAsync(CancellationToken cancellationToken = default) 51 | { 52 | #pragma warning disable CA2000 // Dispose objects before losing scope 53 | var phiSilicaClient = new PhiSilicaClient(); 54 | #pragma warning restore CA2000 // Dispose objects before losing scope 55 | 56 | try 57 | { 58 | await phiSilicaClient.InitializeAsync(cancellationToken); 59 | } 60 | catch 61 | { 62 | return null; 63 | } 64 | 65 | return phiSilicaClient; 66 | } 67 | 68 | public Task GetResponseAsync(IList chatMessages, ChatOptions? options = null, CancellationToken cancellationToken = default) => 69 | GetStreamingResponseAsync(chatMessages, options, cancellationToken).ToChatResponseAsync(cancellationToken: cancellationToken); 70 | 71 | public async IAsyncEnumerable GetStreamingResponseAsync(IList chatMessages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) 72 | { 73 | if (_languageModel == null) 74 | { 75 | throw new InvalidOperationException("Language model is not loaded."); 76 | } 77 | 78 | var prompt = GetPrompt(chatMessages); 79 | 80 | await foreach (var part in GenerateStreamResponseAsync(prompt, options, cancellationToken)) 81 | { 82 | yield return new ChatResponseUpdate 83 | { 84 | Role = ChatRole.Assistant, 85 | Text = part, 86 | }; 87 | } 88 | } 89 | 90 | private (LanguageModelOptions? ModelOptions, ContentFilterOptions? FilterOptions) GetModelOptions(ChatOptions options) 91 | { 92 | if (options == null) 93 | { 94 | return (null, null); 95 | } 96 | 97 | var languageModelOptions = new LanguageModelOptions 98 | { 99 | Skill = options.AdditionalProperties?.TryGetValue("skill", out LanguageModelSkill skill) == true ? skill : DefaultLanguageModelSkill, 100 | Temp = options.Temperature ?? DefaultTemperature, 101 | Top_k = (uint)(options.TopK ?? DefaultTopK), 102 | Top_p = (uint)(options.TopP ?? DefaultTopP), 103 | }; 104 | 105 | var contentFilterOptions = new ContentFilterOptions(); 106 | 107 | if (options?.AdditionalProperties?.TryGetValue("input_moderation", out SeverityLevel inputModeration) == true && inputModeration != SeverityLevel.None) 108 | { 109 | contentFilterOptions.PromptMinSeverityLevelToBlock = new TextContentFilterSeverity 110 | { 111 | HateContentSeverity = inputModeration, 112 | SexualContentSeverity = inputModeration, 113 | ViolentContentSeverity = inputModeration, 114 | SelfHarmContentSeverity = inputModeration 115 | }; 116 | } 117 | 118 | if (options?.AdditionalProperties?.TryGetValue("output_moderation", out SeverityLevel outputModeration) == true && outputModeration != SeverityLevel.None) 119 | { 120 | contentFilterOptions.ResponseMinSeverityLevelToBlock = new TextContentFilterSeverity 121 | { 122 | HateContentSeverity = outputModeration, 123 | SexualContentSeverity = outputModeration, 124 | ViolentContentSeverity = outputModeration, 125 | SelfHarmContentSeverity = outputModeration 126 | }; 127 | } 128 | 129 | return (languageModelOptions, contentFilterOptions); 130 | } 131 | 132 | private string GetPrompt(IEnumerable history) 133 | { 134 | if (!history.Any()) 135 | { 136 | return string.Empty; 137 | } 138 | 139 | string prompt = string.Empty; 140 | 141 | var firstMessage = history.FirstOrDefault(); 142 | 143 | _languageModelContext = firstMessage?.Role == ChatRole.System ? 144 | _languageModel?.CreateContext(firstMessage.Text, new ContentFilterOptions()) : 145 | _languageModel?.CreateContext(); 146 | 147 | for (var i = 0; i < history.Count(); i++) 148 | { 149 | var message = history.ElementAt(i); 150 | if (message.Role == ChatRole.System) 151 | { 152 | if (i > 0) 153 | { 154 | throw new ArgumentException("Only first message can be a system message"); 155 | } 156 | } 157 | else if (message.Role == ChatRole.User) 158 | { 159 | string msgText = message.Text ?? string.Empty; 160 | prompt += msgText; 161 | } 162 | else if (message.Role == ChatRole.Assistant) 163 | { 164 | prompt += message.Text; 165 | } 166 | } 167 | 168 | return prompt; 169 | } 170 | 171 | public void Dispose() 172 | { 173 | _languageModel?.Dispose(); 174 | _languageModel = null; 175 | } 176 | 177 | public object? GetService(Type serviceType, object? serviceKey = null) 178 | { 179 | return 180 | serviceKey is not null ? null : 181 | _languageModel is not null && serviceType?.IsInstanceOfType(_languageModel) is true ? _languageModel : 182 | serviceType?.IsInstanceOfType(this) is true ? this : 183 | serviceType?.IsInstanceOfType(typeof(ChatOptions)) is true ? GetDefaultChatOptions() : 184 | null; 185 | } 186 | 187 | public static bool IsAvailable() 188 | { 189 | try 190 | { 191 | return LanguageModel.IsAvailable(); 192 | } 193 | catch 194 | { 195 | return false; 196 | } 197 | } 198 | 199 | private async Task InitializeAsync(CancellationToken cancellationToken = default) 200 | { 201 | cancellationToken.ThrowIfCancellationRequested(); 202 | 203 | if (!IsAvailable()) 204 | { 205 | await LanguageModel.MakeAvailableAsync(); 206 | } 207 | 208 | cancellationToken.ThrowIfCancellationRequested(); 209 | 210 | _languageModel = await LanguageModel.CreateAsync(); 211 | } 212 | 213 | #pragma warning disable IDE0060 // Remove unused parameter 214 | public async IAsyncEnumerable GenerateStreamResponseAsync(string prompt, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) 215 | #pragma warning restore IDE0060 // Remove unused parameter 216 | { 217 | if (_languageModel == null) 218 | { 219 | throw new InvalidOperationException("Language model is not loaded."); 220 | } 221 | 222 | string currentResponse = string.Empty; 223 | using var newPartEvent = new ManualResetEventSlim(false); 224 | 225 | if (!_languageModel.IsPromptLargerThanContext(prompt)) 226 | { 227 | IAsyncOperationWithProgress? progress; 228 | if (options == null) 229 | { 230 | progress = _languageModel.GenerateResponseWithProgressAsync(new LanguageModelOptions(), prompt, new ContentFilterOptions(), _languageModelContext); 231 | } 232 | else 233 | { 234 | var (modelOptions, filterOptions) = GetModelOptions(options); 235 | progress = _languageModel.GenerateResponseWithProgressAsync(modelOptions, prompt, filterOptions, _languageModelContext); 236 | } 237 | 238 | progress.Progress = (result, value) => 239 | { 240 | currentResponse = value; 241 | newPartEvent.Set(); 242 | if (cancellationToken.IsCancellationRequested) 243 | { 244 | progress.Cancel(); 245 | } 246 | }; 247 | 248 | while (progress.Status != AsyncStatus.Completed) 249 | { 250 | await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); 251 | 252 | if (newPartEvent.Wait(10, cancellationToken)) 253 | { 254 | yield return currentResponse; 255 | newPartEvent.Reset(); 256 | } 257 | } 258 | 259 | var response = await progress; 260 | 261 | yield return response?.Status switch 262 | { 263 | LanguageModelResponseStatus.BlockedByPolicy => "\nBlocked by policy", 264 | LanguageModelResponseStatus.PromptBlockedByPolicy => "\nPrompt blocked by policy", 265 | LanguageModelResponseStatus.ResponseBlockedByPolicy => "\nResponse blocked by policy", 266 | _ => string.Empty, 267 | }; 268 | } 269 | else 270 | { 271 | yield return "Prompt is too large for this model. Please submit a smaller prompt"; 272 | } 273 | } 274 | } -------------------------------------------------------------------------------- /Notes/AI/IChatClient/GenAIModel.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.AI; 2 | using Microsoft.ML.OnnxRuntimeGenAI; 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Diagnostics.CodeAnalysis; 6 | using System.Linq; 7 | using System.Runtime.CompilerServices; 8 | using System.Text; 9 | using System.Threading; 10 | using System.Threading.Tasks; 11 | 12 | namespace Notes.AI; 13 | 14 | internal class GenAIModel : IChatClient 15 | { 16 | private const string TEMPLATE_PLACEHOLDER = "{{CONTENT}}"; 17 | 18 | private const int DefaultTopK = 50; 19 | private const float DefaultTopP = 0.9f; 20 | private const float DefaultTemperature = 1; 21 | private const int DefaultMinLength = 0; 22 | private const int DefaultMaxLength = 1024; 23 | private const bool DefaultDoSample = false; 24 | 25 | private readonly ChatClientMetadata _metadata; 26 | private Model? _model; 27 | private Tokenizer? _tokenizer; 28 | private LlmPromptTemplate? _template; 29 | private static readonly SemaphoreSlim _createSemaphore = new(1, 1); 30 | private static OgaHandle? _ogaHandle; 31 | 32 | private static ChatOptions GetDefaultChatOptions() 33 | { 34 | return new ChatOptions 35 | { 36 | AdditionalProperties = new AdditionalPropertiesDictionary 37 | { 38 | { "min_length", DefaultMinLength }, 39 | { "do_sample", DefaultDoSample }, 40 | }, 41 | MaxOutputTokens = DefaultMaxLength, 42 | Temperature = DefaultTemperature, 43 | TopP = DefaultTopP, 44 | TopK = DefaultTopK, 45 | }; 46 | } 47 | 48 | private GenAIModel(string modelDir) 49 | { 50 | _metadata = new ChatClientMetadata("GenAIChatClient", new Uri($"file:///{modelDir}")); 51 | } 52 | 53 | public static async Task CreateAsync(string modelDir, LlmPromptTemplate? template = null, CancellationToken cancellationToken = default) 54 | { 55 | #pragma warning disable CA2000 // Dispose objects before losing scope 56 | var model = new GenAIModel(modelDir); 57 | #pragma warning restore CA2000 // Dispose objects before losing scope 58 | 59 | var lockAcquired = false; 60 | try 61 | { 62 | // ensure we call CreateAsync one at a time to avoid fun issues 63 | await _createSemaphore.WaitAsync(cancellationToken); 64 | lockAcquired = true; 65 | cancellationToken.ThrowIfCancellationRequested(); 66 | await model.InitializeAsync(modelDir, cancellationToken); 67 | } 68 | catch 69 | { 70 | model?.Dispose(); 71 | return null; 72 | } 73 | finally 74 | { 75 | if (lockAcquired) 76 | { 77 | _createSemaphore.Release(); 78 | } 79 | } 80 | 81 | model._template = template; 82 | return model; 83 | } 84 | 85 | public static void InitializeGenAI() 86 | { 87 | _ogaHandle = new OgaHandle(); 88 | } 89 | 90 | [MemberNotNullWhen(true, nameof(_model), nameof(_tokenizer))] 91 | public bool IsReady => _model != null && _tokenizer != null; 92 | 93 | public void Dispose() 94 | { 95 | _model?.Dispose(); 96 | _tokenizer?.Dispose(); 97 | _ogaHandle?.Dispose(); 98 | } 99 | 100 | private string GetPrompt(IList history) 101 | { 102 | if (!history.Any()) 103 | { 104 | return string.Empty; 105 | } 106 | 107 | if (_template == null) 108 | { 109 | return string.Join(". ", history); 110 | } 111 | 112 | StringBuilder prompt = new(); 113 | 114 | string systemMsgWithoutSystemTemplate = string.Empty; 115 | 116 | for (var i = 0; i < history.Count; i++) 117 | { 118 | var message = history[i]; 119 | if (message.Role == ChatRole.System) 120 | { 121 | // ignore system prompts that aren't at the beginning 122 | if (i == 0) 123 | { 124 | if (string.IsNullOrWhiteSpace(_template.System)) 125 | { 126 | systemMsgWithoutSystemTemplate = message.Text ?? string.Empty; 127 | } 128 | else 129 | { 130 | prompt.Append(_template.System.Replace(TEMPLATE_PLACEHOLDER, message.Text)); 131 | } 132 | } 133 | } 134 | else if (message.Role == ChatRole.User) 135 | { 136 | string msgText = message.Text ?? string.Empty; 137 | if (i == 1 && !string.IsNullOrWhiteSpace(systemMsgWithoutSystemTemplate)) 138 | { 139 | msgText = $"{systemMsgWithoutSystemTemplate} {msgText}"; 140 | } 141 | 142 | prompt.Append(string.IsNullOrWhiteSpace(_template.User) ? 143 | msgText : 144 | _template.User.Replace(TEMPLATE_PLACEHOLDER, msgText)); 145 | } 146 | else if (message.Role == ChatRole.Assistant) 147 | { 148 | prompt.Append(string.IsNullOrWhiteSpace(_template.Assistant) ? 149 | message.Text : 150 | _template.Assistant.Replace(TEMPLATE_PLACEHOLDER, message.Text)); 151 | } 152 | } 153 | 154 | if (!string.IsNullOrWhiteSpace(_template.Assistant)) 155 | { 156 | var substringIndex = _template.Assistant.IndexOf(TEMPLATE_PLACEHOLDER, StringComparison.InvariantCulture); 157 | prompt.Append(_template.Assistant[..substringIndex]); 158 | } 159 | 160 | return prompt.ToString(); 161 | } 162 | 163 | public Task GetResponseAsync(IList chatMessages, ChatOptions? options = null, CancellationToken cancellationToken = default) => 164 | GetStreamingResponseAsync(chatMessages, options, cancellationToken).ToChatResponseAsync(cancellationToken: cancellationToken); 165 | 166 | public async IAsyncEnumerable GetStreamingResponseAsync( 167 | IList chatMessages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) 168 | { 169 | var prompt = GetPrompt(chatMessages); 170 | 171 | if (!IsReady) 172 | { 173 | throw new InvalidOperationException("Model is not ready"); 174 | } 175 | 176 | await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); 177 | 178 | using var generatorParams = new GeneratorParams(_model); 179 | 180 | using var sequences = _tokenizer.Encode(prompt); 181 | 182 | void TransferMetadataValue(string propertyName, object defaultValue) 183 | { 184 | object? val = null; 185 | options?.AdditionalProperties?.TryGetValue(propertyName, out val); 186 | 187 | val ??= defaultValue; 188 | 189 | if (val is int intVal) 190 | { 191 | generatorParams.SetSearchOption(propertyName, intVal); 192 | } 193 | else if (val is float floatVal) 194 | { 195 | generatorParams.SetSearchOption(propertyName, floatVal); 196 | } 197 | else if (val is bool boolVal) 198 | { 199 | generatorParams.SetSearchOption(propertyName, boolVal); 200 | } 201 | } 202 | 203 | if (options != null) 204 | { 205 | TransferMetadataValue("min_length", DefaultMinLength); 206 | TransferMetadataValue("do_sample", DefaultDoSample); 207 | generatorParams.SetSearchOption("temperature", (double)(options?.Temperature ?? DefaultTemperature)); 208 | generatorParams.SetSearchOption("top_p", (double)(options?.TopP ?? DefaultTopP)); 209 | generatorParams.SetSearchOption("top_k", options?.TopK ?? DefaultTopK); 210 | } 211 | 212 | generatorParams.SetSearchOption("max_length", (options?.MaxOutputTokens ?? DefaultMaxLength) + sequences[0].Length); 213 | generatorParams.TryGraphCaptureWithMaxBatchSize(1); 214 | 215 | using var tokenizerStream = _tokenizer.CreateStream(); 216 | using var generator = new Generator(_model, generatorParams); 217 | generator.AppendTokenSequences(sequences); 218 | StringBuilder stringBuilder = new(); 219 | bool stopTokensAvailable = _template != null && _template.Stop != null && _template.Stop.Length > 0; 220 | while (!generator.IsDone()) 221 | { 222 | string part; 223 | try 224 | { 225 | if (cancellationToken.IsCancellationRequested) 226 | { 227 | break; 228 | } 229 | 230 | generator.GenerateNextToken(); 231 | part = tokenizerStream.Decode(generator.GetSequence(0)[^1]); 232 | 233 | if (cancellationToken.IsCancellationRequested && stopTokensAvailable) 234 | { 235 | part = _template!.Stop!.Last(); 236 | } 237 | 238 | stringBuilder.Append(part); 239 | 240 | if (stopTokensAvailable) 241 | { 242 | var str = stringBuilder.ToString(); 243 | if (_template!.Stop!.Any(str.Contains)) 244 | { 245 | break; 246 | } 247 | } 248 | } 249 | catch (Exception) 250 | { 251 | break; 252 | } 253 | 254 | yield return new() 255 | { 256 | Role = ChatRole.Assistant, 257 | Text = part, 258 | }; 259 | } 260 | } 261 | 262 | private Task InitializeAsync(string modelDir, CancellationToken cancellationToken = default) 263 | { 264 | return Task.Run( 265 | () => 266 | { 267 | _model = new Model(modelDir); 268 | cancellationToken.ThrowIfCancellationRequested(); 269 | _tokenizer = new Tokenizer(_model); 270 | }, 271 | cancellationToken); 272 | } 273 | 274 | public object? GetService(Type serviceType, object? serviceKey = null) 275 | { 276 | return 277 | serviceKey is not null ? null : 278 | serviceType == typeof(ChatClientMetadata) ? _metadata : 279 | _model is not null && serviceType?.IsInstanceOfType(_model) is true ? _model : 280 | _tokenizer is not null && serviceType?.IsInstanceOfType(_tokenizer) is true ? _tokenizer : 281 | serviceType?.IsInstanceOfType(this) is true ? this : 282 | serviceType?.IsInstanceOfType(typeof(ChatOptions)) is true ? GetDefaultChatOptions() : 283 | null; 284 | } 285 | } -------------------------------------------------------------------------------- /Notes/Controls/AttachmentView.xaml.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Runtime.InteropServices.WindowsRuntime; 6 | using Windows.Foundation; 7 | using Windows.Foundation.Collections; 8 | using Microsoft.UI.Xaml; 9 | using Microsoft.UI.Xaml.Controls; 10 | using Microsoft.UI.Xaml.Input; 11 | using Microsoft.UI.Xaml.Media; 12 | using System.Collections.ObjectModel; 13 | using System.Threading.Tasks; 14 | using System.Threading; 15 | using Windows.Storage; 16 | using Windows.Media.Core; 17 | using Microsoft.UI.Dispatching; 18 | using Notes.Models; 19 | using Notes.AI.VoiceRecognition; 20 | using Notes.ViewModels; 21 | using Microsoft.UI.Xaml.Media.Imaging; 22 | using Windows.UI; 23 | using Notes.AI.TextRecognition; 24 | 25 | // To learn more about WinUI, the WinUI project structure, 26 | // and more about our project templates, see: http://aka.ms/winui-project-info. 27 | 28 | namespace Notes.Controls 29 | { 30 | public sealed partial class AttachmentView : UserControl 31 | { 32 | private CancellationTokenSource _cts; 33 | private DispatcherQueue _dispatcher; 34 | private Timer _timer; 35 | 36 | public ObservableCollection TranscriptionBlocks { get; set; } = new ObservableCollection(); 37 | public AttachmentViewModel AttachmentVM { get; set; } 38 | public bool AutoScrollEnabled { get; set; } = true; 39 | 40 | public AttachmentView() 41 | { 42 | this.InitializeComponent(); 43 | this.Visibility = Visibility.Collapsed; 44 | this._dispatcher = DispatcherQueue.GetForCurrentThread(); 45 | } 46 | 47 | public async Task Show() 48 | { 49 | this.Visibility = Visibility.Visible; 50 | } 51 | 52 | public void Hide() 53 | { 54 | TranscriptionBlocks.Clear(); 55 | transcriptLoadingProgressRing.IsActive = false; 56 | AttachmentImage.Source = null; 57 | WaveformImage.Source = null; 58 | this.Visibility = Visibility.Collapsed; 59 | 60 | if (AttachmentVM.Attachment.Type == NoteAttachmentType.Video || AttachmentVM.Attachment.Type == NoteAttachmentType.Audio) 61 | { 62 | ResetMediaPlayer(); 63 | } 64 | if (_cts != null && !_cts.IsCancellationRequested) 65 | { 66 | _cts.Cancel(); 67 | } 68 | } 69 | 70 | private void BackgroundTapped(object sender, TappedRoutedEventArgs e) 71 | { 72 | // hide the search view only when the backround was tapped but not any of the content inside 73 | if (e.OriginalSource == Root) 74 | this.Hide(); 75 | } 76 | 77 | public async Task UpdateAttachment(AttachmentViewModel attachment, string? attachmentText = null) 78 | { 79 | AttachmentImageTextCanvas.Children.Clear(); 80 | 81 | AttachmentVM = attachment; 82 | StorageFolder attachmentsFolder = await Utils.GetAttachmentsFolderAsync(); 83 | StorageFile attachmentFile = await attachmentsFolder.GetFileAsync(attachment.Attachment.Filename); 84 | switch(AttachmentVM.Attachment.Type) 85 | { 86 | case NoteAttachmentType.Audio: 87 | ImageGrid.Visibility = Visibility.Collapsed; 88 | MediaGrid.Visibility = Visibility.Visible; 89 | RunWaitForTranscriptionTask(attachmentText); 90 | WaveformImage.Source = await WaveformRenderer.GetWaveformImage(attachmentFile); 91 | SetMediaPlayerSource(attachmentFile); 92 | break; 93 | case NoteAttachmentType.Image: 94 | ImageGrid.Visibility = Visibility.Visible; 95 | MediaGrid.Visibility = Visibility.Collapsed; 96 | AttachmentImage.Source = new BitmapImage(new Uri(attachmentFile.Path)); 97 | LoadImageText(attachment.Attachment.Filename); 98 | break; 99 | case NoteAttachmentType.Video: 100 | ImageGrid.Visibility = Visibility.Collapsed; 101 | MediaGrid.Visibility = Visibility.Visible; 102 | RunWaitForTranscriptionTask(attachmentText); 103 | SetMediaPlayerSource(attachmentFile); 104 | break; 105 | } 106 | } 107 | 108 | private async Task LoadImageText(string fileName) 109 | { 110 | var text = await TextRecognition.GetSavedText(fileName.Split('.')[0] + ".txt"); 111 | foreach (var line in text.Lines) 112 | { 113 | var height = line.Height; 114 | var width = line.Width; 115 | AttachmentImageTextCanvas.Children.Add( 116 | new Border() 117 | { 118 | Child = new Viewbox() 119 | { 120 | Child = new TextBlock() 121 | { 122 | Text = line.Text, 123 | FontSize = 16, 124 | IsTextSelectionEnabled = true, 125 | Foreground = new SolidColorBrush(Microsoft.UI.Colors.Transparent), 126 | }, 127 | Stretch = Stretch.Fill, 128 | StretchDirection = StretchDirection.Both, 129 | VerticalAlignment = VerticalAlignment.Center, 130 | Margin = new Thickness(4), 131 | Height = height, 132 | Width = width, 133 | }, 134 | Height = height + 8, 135 | Width = width + 8, 136 | CornerRadius = new CornerRadius(8), 137 | Margin = new Thickness(line.X - 4, line.Y - 4, 0, 0), 138 | RenderTransform = new RotateTransform() { Angle = text.ImageAngle }, 139 | BorderThickness = new Thickness(0), 140 | Background = new LinearGradientBrush() 141 | { 142 | GradientStops = new GradientStopCollection() 143 | { 144 | new GradientStop() { Color = Color.FromArgb(20, 52, 185, 159), Offset = 0.1 }, 145 | new GradientStop() { Color = Color.FromArgb(20, 50, 181, 173), Offset = 0.5 }, 146 | new GradientStop() { Color = Color.FromArgb(20, 59, 177, 119), Offset = 0.9 } 147 | } 148 | }, 149 | //BorderBrush = new LinearGradientBrush() 150 | //{ 151 | // GradientStops = new GradientStopCollection() 152 | // { 153 | // new GradientStop() { Color = Color.FromArgb(255, 147, 89, 248), Offset = 0.1 }, 154 | // new GradientStop() { Color = Color.FromArgb(255, 203, 123, 190), Offset = 0.5 }, 155 | // new GradientStop() { Color = Color.FromArgb(255, 240, 184, 131), Offset = 0.9 }, 156 | // }, 157 | //}, 158 | } 159 | ); 160 | } 161 | } 162 | 163 | private void SetMediaPlayerSource(StorageFile file) 164 | { 165 | mediaPlayer.Source = MediaSource.CreateFromStorageFile(file); 166 | mediaPlayer.MediaPlayer.CurrentStateChanged += MediaPlayer_CurrentStateChanged; 167 | } 168 | 169 | private async void RunWaitForTranscriptionTask(string? transcriptionTextToTryToShow = null) 170 | { 171 | transcriptLoadingProgressRing.IsActive = true; 172 | _ = Task.Run(async () => 173 | { 174 | while (AttachmentVM.IsProcessing) 175 | { 176 | Thread.Sleep(500); 177 | } 178 | StorageFile transcriptFile = await (await Utils.GetAttachmentsTranscriptsFolderAsync()).GetFileAsync(AttachmentVM.Attachment.FilenameForText); 179 | string rawTranscription = File.ReadAllText(transcriptFile.Path); 180 | _dispatcher.TryEnqueue(() => 181 | { 182 | transcriptLoadingProgressRing.IsActive = false; 183 | var transcripts = WhisperUtils.ProcessTranscriptionWithTimestamps(rawTranscription); 184 | 185 | foreach (var t in transcripts) 186 | { 187 | TranscriptionBlocks.Add(new TranscriptionBlock(t.Text, t.Start, t.End)); 188 | } 189 | 190 | if (transcriptionTextToTryToShow != null) 191 | { 192 | var block = TranscriptionBlocks.Where(t => t.Text.Contains(transcriptionTextToTryToShow)).FirstOrDefault(); 193 | if (block != null) 194 | { 195 | transcriptBlocksListView.SelectedItem = block; 196 | ScrollTranscriptionToItem(block); 197 | } 198 | } 199 | }); 200 | }); 201 | } 202 | 203 | private void MediaPlayer_CurrentStateChanged(Windows.Media.Playback.MediaPlayer sender, object args) 204 | { 205 | if (sender.CurrentState.ToString() == "Playing") 206 | { 207 | _timer = new Timer(CheckTimestampAndSelectTranscription, null, 0, 250); 208 | } 209 | else if(_timer != null) 210 | { 211 | _timer.Dispose(); 212 | } 213 | } 214 | 215 | private void CheckTimestampAndSelectTranscription(object? state) 216 | { 217 | _dispatcher.TryEnqueue(() => { 218 | TimeSpan currentPos = mediaPlayer.MediaPlayer.Position; 219 | foreach (TranscriptionBlock block in TranscriptionBlocks) 220 | { 221 | if (block.Start < currentPos & block.End > currentPos) 222 | { 223 | transcriptBlocksListView.SelectionChanged -= TranscriptBlocksListView_SelectionChanged; 224 | transcriptBlocksListView.SelectedItem = block; 225 | transcriptBlocksListView.SelectionChanged += TranscriptBlocksListView_SelectionChanged; 226 | ScrollTranscriptionToItem(block); 227 | break; 228 | } 229 | } 230 | }); 231 | } 232 | 233 | private void ResetMediaPlayer() 234 | { 235 | if(_timer != null) 236 | { 237 | _timer.Dispose(); 238 | } 239 | mediaPlayer.MediaPlayer.CurrentStateChanged -= MediaPlayer_CurrentStateChanged; 240 | mediaPlayer.MediaPlayer.Pause(); 241 | mediaPlayer.Source = null; 242 | } 243 | 244 | private void TranscriptBlocksListView_SelectionChanged(object sender, SelectionChangedEventArgs e) 245 | { 246 | if (sender is ListView transcriptListView) 247 | { 248 | TranscriptionBlock selectedBlock = (TranscriptionBlock)transcriptListView.SelectedItem; 249 | if (selectedBlock != null) 250 | { 251 | mediaPlayer.MediaPlayer.Position = selectedBlock.Start; 252 | } 253 | } 254 | } 255 | 256 | private void ScrollTranscriptionToItem(TranscriptionBlock block) 257 | { 258 | if(AutoScrollEnabled) 259 | { 260 | transcriptBlocksListView.ScrollIntoView(block, ScrollIntoViewAlignment.Leading); 261 | } 262 | } 263 | } 264 | } 265 | --------------------------------------------------------------------------------