├── .gitattributes ├── .gitignore ├── DistanceFunctions.cs ├── Properties └── AssemblyInfo.cs ├── README.md ├── SQLRAG.application ├── AzureOpenaiFunctions.cs ├── OpenaiFunction.cs ├── Properties │ └── AssemblyInfo.cs ├── RagFunction.cs ├── SQLRAG.application.sqlproj ├── SqlRAGCertificate.sql ├── SqlRagStoredProcedure.cs ├── dbo.EncryptedKeys.sql ├── dbo.KnowledgeBase.sql ├── dbo.QueryIntentCache.sql ├── dbo.WebPilotLogs.sql └── postdeployscript.sql ├── SQLRAG.publish.xml ├── SQLRAG.sln ├── SQLRAG.sqlproj ├── SqlArray.cs ├── TextFunction.cs └── assets ├── ChatCompletion.png ├── ChatCompletion_demo.sql ├── QueryCache.png └── QueryIntentCache_demo.sql /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.rsuser 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Mono auto generated files 17 | mono_crash.* 18 | 19 | # Build results 20 | [Dd]ebug/ 21 | [Dd]ebugPublic/ 22 | [Rr]elease/ 23 | [Rr]eleases/ 24 | x64/ 25 | x86/ 26 | [Ww][Ii][Nn]32/ 27 | [Aa][Rr][Mm]/ 28 | [Aa][Rr][Mm]64/ 29 | bld/ 30 | [Bb]in/ 31 | [Oo]bj/ 32 | [Oo]ut/ 33 | [Ll]og/ 34 | [Ll]ogs/ 35 | 36 | # Visual Studio 2015/2017 cache/options directory 37 | .vs/ 38 | # Uncomment if you have tasks that create the project's static files in wwwroot 39 | #wwwroot/ 40 | 41 | # Visual Studio 2017 auto generated files 42 | Generated\ Files/ 43 | 44 | # MSTest test Results 45 | [Tt]est[Rr]esult*/ 46 | [Bb]uild[Ll]og.* 47 | 48 | # NUnit 49 | *.VisualState.xml 50 | TestResult.xml 51 | nunit-*.xml 52 | 53 | # Build Results of an ATL Project 54 | [Dd]ebugPS/ 55 | [Rr]eleasePS/ 56 | dlldata.c 57 | 58 | # Benchmark Results 59 | BenchmarkDotNet.Artifacts/ 60 | 61 | # .NET Core 62 | project.lock.json 63 | project.fragment.lock.json 64 | artifacts/ 65 | 66 | # ASP.NET Scaffolding 67 | ScaffoldingReadMe.txt 68 | 69 | # StyleCop 70 | StyleCopReport.xml 71 | 72 | # Files built by Visual Studio 73 | *_i.c 74 | *_p.c 75 | *_h.h 76 | *.ilk 77 | *.meta 78 | *.obj 79 | *.iobj 80 | *.pch 81 | *.pdb 82 | *.ipdb 83 | *.pgc 84 | *.pgd 85 | *.rsp 86 | *.sbr 87 | *.tlb 88 | *.tli 89 | *.tlh 90 | *.tmp 91 | *.tmp_proj 92 | *_wpftmp.csproj 93 | *.log 94 | *.vspscc 95 | *.vssscc 96 | .builds 97 | *.pidb 98 | *.svclog 99 | *.scc 100 | 101 | # Chutzpah Test files 102 | _Chutzpah* 103 | 104 | # Visual C++ cache files 105 | ipch/ 106 | *.aps 107 | *.ncb 108 | *.opendb 109 | *.opensdf 110 | *.sdf 111 | *.cachefile 112 | *.VC.db 113 | *.VC.VC.opendb 114 | 115 | # Visual Studio profiler 116 | *.psess 117 | *.vsp 118 | *.vspx 119 | *.sap 120 | 121 | # Visual Studio Trace Files 122 | *.e2e 123 | 124 | # TFS 2012 Local Workspace 125 | $tf/ 126 | 127 | # Guidance Automation Toolkit 128 | *.gpState 129 | 130 | # ReSharper is a .NET coding add-in 131 | _ReSharper*/ 132 | *.[Rr]e[Ss]harper 133 | *.DotSettings.user 134 | 135 | # TeamCity is a build add-in 136 | _TeamCity* 137 | 138 | # DotCover is a Code Coverage Tool 139 | *.dotCover 140 | 141 | # AxoCover is a Code Coverage Tool 142 | .axoCover/* 143 | !.axoCover/settings.json 144 | 145 | # Coverlet is a free, cross platform Code Coverage Tool 146 | coverage*.json 147 | coverage*.xml 148 | coverage*.info 149 | 150 | # Visual Studio code coverage results 151 | *.coverage 152 | *.coveragexml 153 | 154 | # NCrunch 155 | _NCrunch_* 156 | .*crunch*.local.xml 157 | nCrunchTemp_* 158 | 159 | # MightyMoose 160 | *.mm.* 161 | AutoTest.Net/ 162 | 163 | # Web workbench (sass) 164 | .sass-cache/ 165 | 166 | # Installshield output folder 167 | [Ee]xpress/ 168 | 169 | # DocProject is a documentation generator add-in 170 | DocProject/buildhelp/ 171 | DocProject/Help/*.HxT 172 | DocProject/Help/*.HxC 173 | DocProject/Help/*.hhc 174 | DocProject/Help/*.hhk 175 | DocProject/Help/*.hhp 176 | DocProject/Help/Html2 177 | DocProject/Help/html 178 | 179 | # Click-Once directory 180 | publish/ 181 | 182 | # Publish Web Output 183 | *.[Pp]ublish.xml 184 | *.azurePubxml 185 | # Note: Comment the next line if you want to checkin your web deploy settings, 186 | # but database connection strings (with potential passwords) will be unencrypted 187 | *.pubxml 188 | *.publishproj 189 | 190 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 191 | # checkin your Azure Web App publish settings, but sensitive information contained 192 | # in these scripts will be unencrypted 193 | PublishScripts/ 194 | 195 | # NuGet Packages 196 | *.nupkg 197 | # NuGet Symbol Packages 198 | *.snupkg 199 | # The packages folder can be ignored because of Package Restore 200 | **/[Pp]ackages/* 201 | # except build/, which is used as an MSBuild target. 202 | !**/[Pp]ackages/build/ 203 | # Uncomment if necessary however generally it will be regenerated when needed 204 | #!**/[Pp]ackages/repositories.config 205 | # NuGet v3's project.json files produces more ignorable files 206 | *.nuget.props 207 | *.nuget.targets 208 | 209 | # Microsoft Azure Build Output 210 | csx/ 211 | *.build.csdef 212 | 213 | # Microsoft Azure Emulator 214 | ecf/ 215 | rcf/ 216 | 217 | # Windows Store app package directories and files 218 | AppPackages/ 219 | BundleArtifacts/ 220 | Package.StoreAssociation.xml 221 | _pkginfo.txt 222 | *.appx 223 | *.appxbundle 224 | *.appxupload 225 | 226 | # Visual Studio cache files 227 | # files ending in .cache can be ignored 228 | *.[Cc]ache 229 | # but keep track of directories ending in .cache 230 | !?*.[Cc]ache/ 231 | 232 | # Others 233 | ClientBin/ 234 | ~$* 235 | *~ 236 | *.dbmdl 237 | *.dbproj.schemaview 238 | *.jfm 239 | *.pfx 240 | *.publishsettings 241 | orleans.codegen.cs 242 | 243 | # Including strong name files can present a security risk 244 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 245 | #*.snk 246 | 247 | # Since there are multiple workflows, uncomment next line to ignore bower_components 248 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 249 | #bower_components/ 250 | 251 | # RIA/Silverlight projects 252 | Generated_Code/ 253 | 254 | # Backup & report files from converting an old project file 255 | # to a newer Visual Studio version. Backup files are not needed, 256 | # because we have git ;-) 257 | _UpgradeReport_Files/ 258 | Backup*/ 259 | UpgradeLog*.XML 260 | UpgradeLog*.htm 261 | ServiceFabricBackup/ 262 | *.rptproj.bak 263 | 264 | # SQL Server files 265 | *.mdf 266 | *.ldf 267 | *.ndf 268 | 269 | # Business Intelligence projects 270 | *.rdl.data 271 | *.bim.layout 272 | *.bim_*.settings 273 | *.rptproj.rsuser 274 | *- [Bb]ackup.rdl 275 | *- [Bb]ackup ([0-9]).rdl 276 | *- [Bb]ackup ([0-9][0-9]).rdl 277 | 278 | # Microsoft Fakes 279 | FakesAssemblies/ 280 | 281 | # GhostDoc plugin setting file 282 | *.GhostDoc.xml 283 | 284 | # Node.js Tools for Visual Studio 285 | .ntvs_analysis.dat 286 | node_modules/ 287 | 288 | # Visual Studio 6 build log 289 | *.plg 290 | 291 | # Visual Studio 6 workspace options file 292 | *.opt 293 | 294 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 295 | *.vbw 296 | 297 | # Visual Studio LightSwitch build output 298 | **/*.HTMLClient/GeneratedArtifacts 299 | **/*.DesktopClient/GeneratedArtifacts 300 | **/*.DesktopClient/ModelManifest.xml 301 | **/*.Server/GeneratedArtifacts 302 | **/*.Server/ModelManifest.xml 303 | _Pvt_Extensions 304 | 305 | # Paket dependency manager 306 | .paket/paket.exe 307 | paket-files/ 308 | 309 | # FAKE - F# Make 310 | .fake/ 311 | 312 | # CodeRush personal settings 313 | .cr/personal 314 | 315 | # Python Tools for Visual Studio (PTVS) 316 | __pycache__/ 317 | *.pyc 318 | 319 | # Cake - Uncomment if you are using it 320 | # tools/** 321 | # !tools/packages.config 322 | 323 | # Tabs Studio 324 | *.tss 325 | 326 | # Telerik's JustMock configuration file 327 | *.jmconfig 328 | 329 | # BizTalk build output 330 | *.btp.cs 331 | *.btm.cs 332 | *.odx.cs 333 | *.xsd.cs 334 | 335 | # OpenCover UI analysis results 336 | OpenCover/ 337 | 338 | # Azure Stream Analytics local run output 339 | ASALocalRun/ 340 | 341 | # MSBuild Binary and Structured Log 342 | *.binlog 343 | 344 | # NVidia Nsight GPU debugger configuration file 345 | *.nvuser 346 | 347 | # MFractors (Xamarin productivity tool) working folder 348 | .mfractor/ 349 | 350 | # Local History for Visual Studio 351 | .localhistory/ 352 | 353 | # BeatPulse healthcheck temp database 354 | healthchecksdb 355 | 356 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 357 | MigrationBackup/ 358 | 359 | # Ionide (cross platform F# VS Code tools) working folder 360 | .ionide/ 361 | 362 | # Fody - auto-generated XML schema 363 | FodyWeavers.xsd 364 | /SQLQuery1.sql 365 | -------------------------------------------------------------------------------- /DistanceFunctions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Data; 3 | using System.Data.SqlClient; 4 | using System.Data.SqlTypes; 5 | using Microsoft.SqlServer.Server; 6 | 7 | public partial class DistanceFunctions 8 | { 9 | 10 | 11 | 12 | /// 13 | /// Calculates the cosine similarity between two vectors. 14 | /// 15 | /// The first vector. 16 | /// The second vector. 17 | /// The cosine similarity between the two vectors. 18 | [SqlFunction] 19 | public static SqlDouble CosineSimilarity(SqlArray vector1, SqlArray vector2) 20 | { 21 | if (vector1.IsNull || vector2.IsNull) 22 | { 23 | return SqlDouble.Null; 24 | } 25 | 26 | double[] vec1 = vector1.Data; 27 | double[] vec2 = vector2.Data; 28 | 29 | if (vec1.Length != vec2.Length) 30 | return SqlDouble.Null; 31 | 32 | double dotProduct = 0.0; 33 | double normVec1 = 0.0; 34 | double normVec2 = 0.0; 35 | 36 | for (int i = 0; i < vec1.Length; i++) 37 | { 38 | dotProduct += vec1[i] * vec2[i]; 39 | normVec1 += vec1[i] * vec1[i]; 40 | normVec2 += vec2[i] * vec2[i]; 41 | } 42 | 43 | if (normVec1 == 0.0 || normVec2 == 0.0) 44 | return 0; 45 | 46 | return new SqlDouble(dotProduct / (Math.Sqrt(normVec1) * Math.Sqrt(normVec2))); 47 | } 48 | 49 | 50 | [SqlFunction] 51 | public static SqlDouble EuclideanDistance(SqlArray vector1, SqlArray vector2) 52 | { 53 | if (vector1.IsNull || vector2.IsNull) 54 | { 55 | return SqlDouble.Null; 56 | } 57 | 58 | double[] vec1 = vector1.Data; 59 | double[] vec2 = vector2.Data; 60 | 61 | if (vec1.Length != vec2.Length) 62 | return SqlDouble.Null; 63 | 64 | double sum = 0.0; 65 | for (int i = 0; i < vec1.Length; i++) 66 | { 67 | sum += Math.Pow(vec1[i] - vec2[i], 2); 68 | } 69 | 70 | return Math.Sqrt(sum); 71 | } 72 | 73 | 74 | [SqlFunction] 75 | public static SqlDouble ManhattanDistance(SqlArray vector1, SqlArray vector2) 76 | { 77 | if (vector1.IsNull || vector2.IsNull) 78 | { 79 | return SqlDouble.Null; 80 | } 81 | 82 | double[] vec1 = vector1.Data; 83 | double[] vec2 = vector2.Data; 84 | 85 | if (vec1.Length != vec2.Length) 86 | return SqlDouble.Null; 87 | 88 | double sum = 0.0; 89 | for (int i = 0; i < vec1.Length; i++) 90 | { 91 | sum += Math.Abs(vec1[i] - vec2[i]); 92 | } 93 | 94 | return sum; 95 | } 96 | 97 | 98 | 99 | [SqlFunction] 100 | public static SqlDouble HammingDistance(SqlString InputString1, SqlString InputString2) 101 | { 102 | if (InputString1.IsNull || InputString2.IsNull) 103 | { 104 | return SqlDouble.Null; 105 | } 106 | 107 | char[] vec1 = InputString1.Value.ToCharArray(); 108 | char[] vec2 = InputString2.Value.ToCharArray(); 109 | 110 | if (vec1.Length != vec2.Length) 111 | return SqlDouble.Null; 112 | 113 | int distance = 0; 114 | for (int i = 0; i < vec1.Length; i++) 115 | { 116 | if (vec1[i] != vec2[i]) 117 | distance++; 118 | } 119 | 120 | return distance; 121 | } 122 | 123 | [SqlFunction] 124 | public static SqlDouble ChebyshevDistance(SqlArray vector1, SqlArray vector2) 125 | { 126 | if (vector1.IsNull || vector2.IsNull) 127 | { 128 | return SqlDouble.Null; 129 | } 130 | 131 | double[] vec1 = vector1.Data; 132 | double[] vec2 = vector2.Data; 133 | 134 | if (vec1.Length != vec2.Length) 135 | return SqlDouble.Null; 136 | 137 | double maxDifference = 0.0; 138 | for (int i = 0; i < vec1.Length; i++) 139 | { 140 | double difference = Math.Abs(vec1[i] - vec2[i]); 141 | if (difference > maxDifference) 142 | maxDifference = difference; 143 | } 144 | 145 | return maxDifference; 146 | } 147 | 148 | 149 | 150 | 151 | [SqlFunction] 152 | public static SqlDouble MinkowskiDistance(SqlArray vector1, SqlArray vector2, SqlDouble p) 153 | { 154 | if (vector1.IsNull || vector2.IsNull) 155 | { 156 | return SqlDouble.Null; 157 | } 158 | 159 | if (p.Value < 1) 160 | return SqlDouble.Null; // p 必須大於等於1 161 | 162 | double[] vec1 = vector1.Data; 163 | double[] vec2 = vector2.Data; 164 | 165 | 166 | if (vec1.Length != vec2.Length) 167 | return SqlDouble.Null; 168 | 169 | double sum = 0.0; 170 | for (int i = 0; i < vec1.Length; i++) 171 | { 172 | sum += Math.Pow(Math.Abs(vec1[i] - vec2[i]), p.Value); 173 | } 174 | 175 | return Math.Pow(sum, 1.0 / p.Value); 176 | } 177 | 178 | 179 | [SqlFunction] 180 | public static SqlInt32 LevenshteinDistance([SqlFacet(MaxSize = -1)] SqlString InputString1, [SqlFacet(MaxSize = -1)] SqlString InputString2) 181 | { 182 | if (InputString1.IsNull || InputString2.IsNull) 183 | { 184 | return SqlInt32.Null; 185 | } 186 | 187 | int n = InputString1.Value.Length; 188 | int m = InputString2.Value.Length; 189 | int[,] d = new int[n + 1, m + 1]; 190 | 191 | // Step 1 192 | if (n == 0) 193 | { 194 | return new SqlInt32(m); 195 | } 196 | 197 | if (m == 0) 198 | { 199 | return new SqlInt32(n); 200 | } 201 | 202 | // Step 2 203 | for (int i = 0; i <= n; d[i, 0] = i++) 204 | { 205 | } 206 | 207 | for (int j = 0; j <= m; d[0, j] = j++) 208 | { 209 | } 210 | 211 | // Step 3 212 | for (int i = 1; i <= n; i++) 213 | { 214 | //Step 4 215 | for (int j = 1; j <= m; j++) 216 | { 217 | // Step 5 218 | int cost = (InputString2.Value[j - 1] == InputString1.Value[i - 1]) ? 0 : 1; 219 | 220 | // Step 6 221 | d[i, j] = Math.Min( 222 | Math.Min(d[i - 1, j] + 1, d[i, j - 1] + 1), 223 | d[i - 1, j - 1] + cost); 224 | } 225 | } 226 | // Step 7 227 | return new SqlInt32(d[n, m]); 228 | } 229 | 230 | 231 | 232 | 233 | } 234 | 235 | 236 | -------------------------------------------------------------------------------- /Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | 3 | // 組件的一般資訊是由下列的屬性集控制。 4 | // 變更這些屬性的值即可修改 5 | // SQLCLR 組件的相關資訊。 6 | [assembly: AssemblyTitle("SQLRAG")] 7 | [assembly: AssemblyDescription("")] 8 | [assembly: AssemblyConfiguration("")] 9 | [assembly: AssemblyCompany("DataDecision.ai")] 10 | [assembly: AssemblyProduct("SQLRAG")] 11 | [assembly: AssemblyCopyright("Copyright © 2023")] 12 | [assembly: AssemblyTrademark("")] 13 | [assembly: AssemblyCulture("")] 14 | 15 | // 組件的版本資訊是由下列四項值構成: 16 | // 17 | // 主要版本 18 | // 次要版本 19 | // 組建編號 20 | // 修訂版本 21 | // 22 | [assembly: AssemblyVersion("0.1.0.0")] 23 | [assembly: AssemblyFileVersion("0.1.0.0")] 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SQLRAG 2 | ## 讓SQL Server也能執行向量相似性查詢 3 | 4 | 目前已經實現功能: 5 | - 各種相似性比較CLR函數: CosineSimilarity, EuclideanDistance... [參考 (https://chat.openai.com/share/40723def-afdd-4f83-adb3-fbcae34b617a)] 6 | - 透過CLR函數調用Openai API(GetEmbedding, ChatCompletion...) 7 | 8 | 9 | ## 安裝 10 | 最小安裝要求:SQL Server 2022, Visual Studio 2022(需安裝SSDT) 11 | 12 | 1. 首先要啟用SQL Server CLR: 13 | ```sql 14 | use SQLRAG 15 | sp_configure 'show advanced options', 1; 16 | RECONFIGURE; 17 | sp_configure 'clr enabled', 1; 18 | RECONFIGURE; 19 | ``` 20 | 21 | 2. 有以下幾種方式可以安裝: 22 | - 使用release中提供的SQLRAG_CREATE.sql直接執行即可完成安裝(適合首次安裝者) 23 | - 使用release中提供的dacpac檔，參考[網頁 (https://learn.microsoft.com/zh-tw/sql/relational-databases/data-tier-applications/upgrade-a-data-tier-application?view=sql-server-ver16)]內容安裝。 24 | - 使用release中提供的原始碼，使用Visual Studio 2022開啟後修改各專案的資料庫連線後直接部署方案 25 | 26 | 3. 如果要使用OpenaiFunction，之前的做法是在OpenaiFunction.cs中輸入你實際的OPENAI_API_KEY，這樣的做法是不能確保API KEY的安全的。在這一版中，我在SqlRAG資料庫中增加了dbo.EncryptedKeys資料表，其中KeyValue部分透過憑證加密。看起來複雜但我已經將自動調用的部分處理好，開發者只需要透過以下SQL 語法將API KEY加密後寫入資料表中即可。 27 | ```sql 28 | use SQLRAG 29 | Declare @cleartext varchar(512)='sk-輸入你的OPENAI_API_KEYAPI KEY' 30 | Declare @encrytext varbinary(4000)=EncryptByCert(Cert_ID('SqlRAGCertificate'), @cleartext) 31 | 32 | INSERT INTO [SQLRAG].[dbo].[EncryptedKeys] 33 | VALUES ( N'OPENAI_API_KEY', N'調用OPENAI所用之API KEY',@encrytext ); 34 | ``` 35 | 之後則需要透過顯式的聲明基於哪個憑證以及對應的密碼來進行解密: 36 | ```sql 37 | 38 | DecryptByCert(Cert_ID('SqlRAGCertificate'), EncryptByCert(Cert_ID('SqlRAGCertificate'), @cleartext) ,'P@ssw0rd') 39 | 40 | ``` 41 | 42 | 如果你是Azure Openai Service用戶則需要將api key以及endpoint加密後存入 43 | ```sql 44 | 45 | use SQLRAG 46 | Declare @cleartext varchar(512)='***' 47 | Declare @encrytext varbinary(4000)=EncryptByCert(Cert_ID('SqlRAGCertificate'), @cleartext) 48 | 49 | Declare @cleartext2 varchar(512)='https://***.openai.azure.com' 50 | Declare @encrytext2 varbinary(4000)=EncryptByCert(Cert_ID('SqlRAGCertificate'), @cleartext2) 51 | 52 | Declare @cleartext3 varchar(512)='***' 53 | Declare @encrytext3 varbinary(4000)=EncryptByCert(Cert_ID('SqlRAGCertificate'), @cleartext3) 54 | 55 | INSERT INTO [SQLRAG].[dbo].[EncryptedKeys] 56 | VALUES ( N'AZURE_OPENAI_API_KEY', N'調用Azure Openai Service 所用之API KEY',@encrytext ); 57 | INSERT INTO [SQLRAG].[dbo].[EncryptedKeys] 58 | VALUES ( N'AZURE_OPENAI_ENDPOINT', N'調用Azure Openai Service 所用之endpoint',@encrytext2 ); 59 | INSERT INTO [SQLRAG].[dbo].[EncryptedKeys] 60 | VALUES ( N'OPENAI_API_VERSION', N'調用Azure Openai Service 所用之API Version',@encrytext3 ); 61 | 62 | ``` 63 | 64 | 65 | 66 | 4. assets中的QueryIntentCache_demo.sql (語意快取範例)以及ChatCompletion_demo.sql(ChatGPT回答範例) 67 | 68 | ![ChatCompletion](assets/QueryCache.png) 69 | ![ChatCompletion](assets/ChatCompletion.png) 70 | 71 | 72 | -------------------------------------------------------------------------------- /SQLRAG.application/AzureOpenaiFunctions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Data; 3 | using System.Data.SqlClient; 4 | using System.Data.SqlTypes; 5 | using System.IO; 6 | using System.Net; 7 | using System.Reflection; 8 | using System.Security.Policy; 9 | using System.Text.RegularExpressions; 10 | using Microsoft.SqlServer.Server; 11 | 12 | public partial class AzureOpenaiFunction 13 | { 14 | 15 | private static string GetKey() 16 | { 17 | System.Data.SqlClient.SqlConnection.ColumnEncryptionKeyCacheTtl = TimeSpan.Zero; 18 | using (SqlConnection conn = new SqlConnection("context connection=true")) 19 | { 20 | conn.Open(); 21 | SqlCommand cmd = new SqlCommand( 22 | @"Declare @encrytext varbinary(4000)=(SELECT [KeyValue] FROM [SQLRAG].[dbo].[EncryptedKeys] WHERE [KeyName]='AZURE_OPENAI_API_KEY') 23 | Declare @decrytext varchar(512)=DecryptByCert(Cert_ID('SqlRAGCertificate'),@encrytext,N'SqlRAGP@ssw0rd') 24 | select @decrytext ", conn); 25 | 26 | using (SqlDataReader reader = cmd.ExecuteReader()) 27 | { 28 | while (reader.Read()) 29 | { 30 | return (string)reader[0]; 31 | } 32 | } 33 | return string.Empty; 34 | } 35 | } 36 | 37 | private static string GetEndpoint() 38 | { 39 | System.Data.SqlClient.SqlConnection.ColumnEncryptionKeyCacheTtl = TimeSpan.Zero; 40 | using (SqlConnection conn = new SqlConnection("context connection=true")) 41 | { 42 | conn.Open(); 43 | SqlCommand cmd = new SqlCommand( 44 | @"Declare @encrytext varbinary(4000)=(SELECT [KeyValue] FROM [SQLRAG].[dbo].[EncryptedKeys] WHERE [KeyName]='AZURE_OPENAI_ENDPOINT') 45 | Declare @decrytext varchar(512)=DecryptByCert(Cert_ID('SqlRAGCertificate'),@encrytext,N'SqlRAGP@ssw0rd') 46 | select @decrytext ", conn); 47 | 48 | using (SqlDataReader reader = cmd.ExecuteReader()) 49 | { 50 | while (reader.Read()) 51 | { 52 | return (string)reader[0]; 53 | } 54 | } 55 | return string.Empty; 56 | } 57 | } 58 | 59 | private static string GetVersion() 60 | { 61 | System.Data.SqlClient.SqlConnection.ColumnEncryptionKeyCacheTtl = TimeSpan.Zero; 62 | using (SqlConnection conn = new SqlConnection("context connection=true")) 63 | { 64 | conn.Open(); 65 | SqlCommand cmd = new SqlCommand( 66 | @"Declare @encrytext varbinary(4000)=(SELECT [KeyValue] FROM [SQLRAG].[dbo].[EncryptedKeys] WHERE [KeyName]='OPENAI_API_VERSION') 67 | Declare @decrytext varchar(512)=DecryptByCert(Cert_ID('SqlRAGCertificate'),@encrytext,N'SqlRAGP@ssw0rd') 68 | select @decrytext ", conn); 69 | 70 | using (SqlDataReader reader = cmd.ExecuteReader()) 71 | { 72 | while (reader.Read()) 73 | { 74 | return (string)reader[0]; 75 | } 76 | } 77 | return string.Empty; 78 | } 79 | } 80 | 81 | 82 | [SqlFunction(DataAccess = DataAccessKind.Read)] 83 | public static SqlArray GetAzureEmbedding([SqlFacet(MaxSize = -1)] SqlString inputText,SqlString deploymentName) 84 | { 85 | 86 | string apiKey = GetKey(); 87 | string apiUrl = ""; 88 | string endpoint = GetEndpoint(); 89 | string apiVersion = GetVersion(); 90 | if (!deploymentName.IsNull) 91 | { 92 | apiUrl = string.Format("{0}/openai/deployments/{1}/embeddings?api-version={2}", endpoint, deploymentName.Value, apiVersion); 93 | } 94 | else 95 | { 96 | throw new ArgumentException(); 97 | } 98 | 99 | string requestBody = $"{{\"input\": \"{inputText}\",\"encoding_format\":\"float\"}}"; 100 | ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; 101 | try 102 | { 103 | HttpWebRequest request = (HttpWebRequest)WebRequest.Create(apiUrl); 104 | request.UserAgent = 105 | "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; WOW64; " + 106 | "Trident/4.0; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; " + 107 | ".NET CLR 3.5.21022; .NET CLR 3.5.30729; .NET CLR 3.0.30618; " + 108 | "InfoPath.2; OfficeLiveConnector.1.3; OfficeLivePatch.0.0)"; 109 | request.Method = "POST"; 110 | request.Headers["api-key"] = apiKey; 111 | request.Headers["Accept-Language"] = "zh-TW"; 112 | request.ContentType = "application/json"; 113 | 114 | 115 | using (StreamWriter streamWriter = new StreamWriter(request.GetRequestStream())) 116 | { 117 | streamWriter.Write(requestBody); 118 | } 119 | 120 | HttpWebResponse response = (HttpWebResponse)request.GetResponse(); 121 | string result; 122 | using (StreamReader streamReader = new StreamReader(response.GetResponseStream())) 123 | { 124 | result = streamReader.ReadToEnd(); 125 | 126 | } 127 | return new SqlArray(System.Array.ConvertAll(ParseEmbedding(result).Split(','), Double.Parse)); // 返回結果 128 | } 129 | catch (Exception ex) 130 | { 131 | // 錯誤處理 132 | throw ex; 133 | } 134 | } 135 | 136 | 137 | 138 | 139 | 140 | [return: SqlFacet(MaxSize = -1)] 141 | [SqlFunction(DataAccess = DataAccessKind.Read)] 142 | public static SqlString AzureChatCompletion([SqlFacet(MaxSize = -1)] SqlString inputPrompt, [SqlFacet(MaxSize = -1)] SqlString systemProimpt = default(SqlString), SqlString deploymentName = default(SqlString)) 143 | { 144 | ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; 145 | string apiKey = GetKey(); 146 | string apiUrl = ""; 147 | string endpoint = GetEndpoint(); 148 | string apiVersion = GetVersion(); 149 | if (!deploymentName.IsNull) 150 | { 151 | apiUrl = string.Format("{0}/openai/deployments/{1}/chat/completions?api-version={2}", endpoint, deploymentName.Value, apiVersion); 152 | } 153 | else 154 | { 155 | throw new ArgumentException(); 156 | } 157 | 158 | 159 | string requestBody1 = $@"{{""messages"": [{{""role"": ""system"", ""content"": ""{systemProimpt.ToString()}""}},{{""role"": ""user"", ""content"": ""{inputPrompt.ToString()}""}}]}}"; 160 | string requestBody = systemProimpt.IsNull || String.IsNullOrWhiteSpace(systemProimpt.Value) ? $@"{{ ""messages"": [{{""role"": ""user"", ""content"": ""{inputPrompt.ToString()}""}}]}}" : requestBody1; 161 | 162 | try 163 | { 164 | HttpWebRequest request = (HttpWebRequest)WebRequest.Create(apiUrl); 165 | request.UserAgent = 166 | "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; WOW64; " + 167 | "Trident/4.0; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; " + 168 | ".NET CLR 3.5.21022; .NET CLR 3.5.30729; .NET CLR 3.0.30618; " + 169 | "InfoPath.2; OfficeLiveConnector.1.3; OfficeLivePatch.0.0)"; 170 | request.Method = "POST"; 171 | request.Headers["api-key"] = apiKey; 172 | request.Headers["Accept-Language"] = "zh-TW"; 173 | request.ContentType = "application/json"; 174 | 175 | using (StreamWriter streamWriter = new StreamWriter(request.GetRequestStream())) 176 | { 177 | streamWriter.Write(requestBody); 178 | } 179 | 180 | HttpWebResponse response = (HttpWebResponse)request.GetResponse(); 181 | using (StreamReader streamReader = new StreamReader(response.GetResponseStream())) 182 | { 183 | string result = streamReader.ReadToEnd(); 184 | 185 | return ParseChatting(result); // 返回結果 186 | } 187 | } 188 | catch (Exception ex) 189 | { 190 | // 錯誤處理 191 | return new SqlString($"Error: {ex.Message}"); 192 | } 193 | } 194 | 195 | 196 | 197 | private static string ParseEmbedding(string jsonResponse) 198 | { 199 | // 簡單的 JSON 解析來提取 embedding 200 | string startPattern = "\"embedding\": ["; 201 | string endPattern = "]"; 202 | int startIndex = jsonResponse.IndexOf(startPattern) + startPattern.Length; 203 | int endIndex = jsonResponse.IndexOf(endPattern, startIndex); 204 | string embeddingString = jsonResponse.Substring(startIndex, endIndex - startIndex); 205 | 206 | // 清理和格式化數據 207 | embeddingString = embeddingString.Replace("\n", "").Replace("\r", "").Replace(" ", ""); 208 | return embeddingString; 209 | } 210 | 211 | private const RegexOptions ExpressionOptions = RegexOptions.Singleline | RegexOptions.Multiline | RegexOptions.IgnoreCase; 212 | private static SqlString ParseChatting(string jsonResponse) 213 | { 214 | 215 | Match match = Regex.Match(jsonResponse, "\"content\":\"(.*?)\"", ExpressionOptions); 216 | // 判斷是否匹配成功 217 | if (match.Success) 218 | { 219 | // 獲取第一個子組的值 220 | string content = match.Groups[1].Value; 221 | return new SqlString(content); 222 | } 223 | else 224 | { 225 | return SqlString.Null; 226 | } 227 | 228 | } 229 | 230 | 231 | } 232 | -------------------------------------------------------------------------------- /SQLRAG.application/OpenaiFunction.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Data; 3 | using System.Data.SqlClient; 4 | using System.Net; 5 | using System.Data.SqlTypes; 6 | using System.Text; 7 | using System.Threading.Tasks; 8 | using Microsoft.SqlServer.Server; 9 | using System.IO; 10 | using System.Reflection; 11 | using System.Data.Common; 12 | using System.Collections.ObjectModel; 13 | using System.CodeDom.Compiler; 14 | using System.Text.RegularExpressions; 15 | 16 | 17 | 18 | public partial class OpenaiFunction 19 | { 20 | 21 | 22 | 23 | private static string GetKey() 24 | { 25 | //System.Data.SqlClient.SqlConnection.ColumnEncryptionKeyCacheTtl = TimeSpan.Zero; 26 | using (SqlConnection conn = new SqlConnection("context connection=true")) 27 | { 28 | conn.Open(); 29 | SqlCommand cmd = new SqlCommand( 30 | @"Declare @encrytext varbinary(4000)=(SELECT [KeyValue] FROM [SQLRAG].[dbo].[EncryptedKeys] WHERE [KeyName]='OPENAI_API_KEY') 31 | Declare @decrytext varchar(512)=DecryptByCert(Cert_ID('SqlRAGCertificate'),@encrytext,N'SqlRAGP@ssw0rd') 32 | select @decrytext ", conn); 33 | 34 | using (SqlDataReader reader = cmd.ExecuteReader()) 35 | { 36 | while (reader.Read()) 37 | { 38 | return (string)reader[0]; 39 | } 40 | } 41 | return string.Empty; 42 | } 43 | } 44 | 45 | 46 | [SqlFunction(DataAccess = DataAccessKind.Read)] 47 | public static SqlArray GetEmbedding([SqlFacet(MaxSize = -1)] SqlString inputText) 48 | { 49 | 50 | string apiKey = GetKey(); 51 | string apiUrl = "https://api.openai.com/v1/embeddings"; 52 | string requestBody = $"{{\"model\":\"text-embedding-ada-002\",\"input\": \"{inputText}\",\"encoding_format\":\"float\"}}"; 53 | ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; 54 | 55 | try 56 | { 57 | HttpWebRequest request = (HttpWebRequest)WebRequest.Create(apiUrl); 58 | request.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; WOW64; " + 59 | "Trident/4.0; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; " + 60 | ".NET CLR 3.5.21022; .NET CLR 3.5.30729; .NET CLR 3.0.30618; " + 61 | "InfoPath.2; OfficeLiveConnector.1.3; OfficeLivePatch.0.0)"; 62 | request.Method = "POST"; 63 | request.Headers["Authorization"] = $"Bearer {apiKey}"; 64 | request.Headers["Accept-Language"] = "zh-TW"; 65 | request.ContentType = "application/json"; 66 | 67 | using (StreamWriter streamWriter = new StreamWriter(request.GetRequestStream())) 68 | { 69 | streamWriter.Write(requestBody); 70 | } 71 | 72 | using (HttpWebResponse response = (HttpWebResponse)request.GetResponse()) 73 | { 74 | string result; 75 | using (StreamReader streamReader = new StreamReader(response.GetResponseStream())) 76 | { 77 | result = streamReader.ReadToEnd(); 78 | } 79 | 80 | return new SqlArray(System.Array.ConvertAll(ParseEmbedding(result).Split(','), Double.Parse)); // 返回結果 81 | } 82 | } 83 | catch (Exception) 84 | { 85 | return null; 86 | } 87 | } 88 | 89 | 90 | 91 | 92 | 93 | [return: SqlFacet(MaxSize = -1)] 94 | [SqlFunction(DataAccess = DataAccessKind.Read)] 95 | public static SqlString ChatCompletion([SqlFacet(MaxSize = -1)] SqlString inputPrompt, [SqlFacet(MaxSize = -1)] SqlString systemProimpt = default(SqlString), SqlString model = default(SqlString)) 96 | { 97 | ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; 98 | string apiKey = GetKey(); 99 | string apiUrl = "https://api.openai.com/v1/chat/completions"; 100 | string modelToUse = model.IsNull || String.IsNullOrWhiteSpace(model.Value) ? "gpt-3.5-turbo" : model.Value; 101 | 102 | string requestBody1 =$"{{\"model\": \"{modelToUse}\", \"messages\": [{{\"role\": \"system\", \"content\": \"{systemProimpt.Value}\"}},{{\"role\": \"user\", \"content\": \"{inputPrompt.Value}\"}}]}}"; 103 | string requestBody = systemProimpt.IsNull || String.IsNullOrWhiteSpace(systemProimpt.Value) ? $"{{\"model\": \"{modelToUse}\", \"messages\": [{{\"role\": \"user\", \"content\": \"{inputPrompt.ToString()}\"}}]}}" : requestBody1; 104 | 105 | try 106 | { 107 | HttpWebRequest request = (HttpWebRequest)WebRequest.Create(apiUrl); 108 | request.UserAgent = 109 | "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; WOW64; " + 110 | "Trident/4.0; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; " + 111 | ".NET CLR 3.5.21022; .NET CLR 3.5.30729; .NET CLR 3.0.30618; " + 112 | "InfoPath.2; OfficeLiveConnector.1.3; OfficeLivePatch.0.0)"; 113 | request.Method = "POST"; 114 | request.Headers["Authorization"] = $"Bearer {apiKey}"; 115 | request.Headers["Accept-Language"] = "zh-TW"; 116 | request.ContentType = "application/json"; 117 | 118 | using (StreamWriter streamWriter = new StreamWriter(request.GetRequestStream())) 119 | { 120 | streamWriter.Write(requestBody); 121 | } 122 | 123 | HttpWebResponse response = (HttpWebResponse)request.GetResponse(); 124 | using (StreamReader streamReader = new StreamReader(response.GetResponseStream())) 125 | { 126 | string result = streamReader.ReadToEnd(); 127 | return ParseChatting(result); // 返回結果 128 | } 129 | } 130 | catch (Exception ex) 131 | { 132 | // 錯誤處理 133 | return new SqlString($"Error: {ex.Message}"); 134 | } 135 | } 136 | 137 | 138 | 139 | private static string ParseEmbedding(string jsonResponse) 140 | { 141 | // 簡單的 JSON 解析來提取 embedding 142 | string startPattern = "\"embedding\": ["; 143 | string endPattern = "]"; 144 | int startIndex = jsonResponse.IndexOf(startPattern) + startPattern.Length; 145 | int endIndex = jsonResponse.IndexOf(endPattern, startIndex); 146 | string embeddingString = jsonResponse.Substring(startIndex, endIndex - startIndex); 147 | 148 | // 清理和格式化數據 149 | embeddingString = embeddingString.Replace("\n", "").Replace("\r", "").Replace(" ", ""); 150 | return embeddingString; 151 | } 152 | 153 | private const RegexOptions ExpressionOptions = RegexOptions.Singleline | RegexOptions.Multiline | RegexOptions.IgnoreCase; 154 | private static SqlString ParseChatting(string jsonResponse) 155 | { 156 | 157 | Match match = Regex.Match(jsonResponse, "\"content\": \"(.*?)\"", ExpressionOptions); 158 | // 判斷是否匹配成功 159 | if (match.Success) 160 | { 161 | // 獲取第一個子組的值 162 | string content = match.Groups[1].Value; 163 | return new SqlString(content); 164 | } 165 | else 166 | { 167 | return SqlString.Null; 168 | } 169 | 170 | } 171 | 172 | 173 | 174 | //[return: SqlFacet(MaxSize = -1)] 175 | //[SqlFunction(DataAccess = DataAccessKind.Read)] 176 | //public static SqlString Translate2zhtw([SqlFacet(MaxSize = -1)] SqlString inputPrompt, SqlString model = default(SqlString)) 177 | //{ 178 | 179 | // ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; 180 | // string apiKey = GetKey(); 181 | // string apiUrl = "https://api.openai.com/v1/chat/completions"; 182 | // string modelToUse = model.IsNull || String.IsNullOrWhiteSpace(model.Value) ? "gpt-3.5-turbo" : model.Value; 183 | 184 | // string requestBody = $"{{\"model\": \"{modelToUse}\", \"messages\": [{{\"role\": \"user\", \"content\": \"請直接將以下文字內容翻譯為繁體中文:\r\n{inputPrompt.Value}\"}}]}}"; 185 | 186 | // try 187 | // { 188 | // HttpWebRequest request = (HttpWebRequest)WebRequest.Create(apiUrl); 189 | // request.Method = "POST"; 190 | // request.Headers["Authorization"] = $"Bearer {apiKey}"; 191 | // request.Headers["Accept-Language"] = "zh-TW"; 192 | // request.ContentType = "application/json"; 193 | 194 | // using (StreamWriter streamWriter = new StreamWriter(request.GetRequestStream())) 195 | // { 196 | // streamWriter.Write(requestBody); 197 | // } 198 | 199 | // HttpWebResponse response = (HttpWebResponse)request.GetResponse(); 200 | // using (StreamReader streamReader = new StreamReader(response.GetResponseStream())) 201 | // { 202 | // string result = streamReader.ReadToEnd(); 203 | // return ParseChatting(result); // 返回結果 204 | // } 205 | // } 206 | // catch (Exception ex) 207 | // { 208 | // // 錯誤處理 209 | // return new SqlString($"Error: {ex.Message}"); 210 | // } 211 | 212 | 213 | //} 214 | 215 | 216 | 217 | } 218 | -------------------------------------------------------------------------------- /SQLRAG.application/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | 3 | // 組件的一般資訊是由下列的屬性集控制。 4 | // 變更這些屬性的值即可修改 5 | // SQLCLR 組件的相關資訊。 6 | [assembly: AssemblyTitle("SQLRAG.application")] 7 | [assembly: AssemblyDescription("")] 8 | [assembly: AssemblyConfiguration("")] 9 | [assembly: AssemblyCompany("DataDecision.ai")] 10 | [assembly: AssemblyProduct("SQLRAG.application")] 11 | [assembly: AssemblyCopyright("Copyright © 2023")] 12 | [assembly: AssemblyTrademark("")] 13 | [assembly: AssemblyCulture("")] 14 | 15 | // 組件的版本資訊是由下列四項值構成: 16 | // 17 | // 主要版本 18 | // 次要版本 19 | // 組建編號 20 | // 修訂版本 21 | // 22 | [assembly: AssemblyVersion("0.1.0.0")] 23 | [assembly: AssemblyFileVersion("0.1.0.0")] 24 | -------------------------------------------------------------------------------- /SQLRAG.application/RagFunction.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Data; 3 | using System.Data.SqlClient; 4 | using System.Data.SqlTypes; 5 | using Microsoft.SqlServer.Server; 6 | 7 | public partial class RagFunctions 8 | { 9 | 10 | [return: SqlFacet(MaxSize = -1)] 11 | [SqlFunction(DataAccess = DataAccessKind.Read)] 12 | public static SqlString GetCachedSQL([SqlFacet(MaxSize = -1)] SqlString question) 13 | { 14 | using (SqlConnection connection = new SqlConnection("context connection=true")) 15 | { 16 | SqlArray embedded = OpenaiFunction.GetEmbedding(question); 17 | connection.Open(); 18 | SqlCommand command = new SqlCommand("Declare @embedded SqlArray \r\nset @embedded=SqlArray::Parse(@embedded_string) \r\nselect top 1 [GeneratedTSQL]\r\nFROM\r\n(SELECT [QueryIntent] \r\n,[GeneratedTSQL]\r\n,[CreateDate]\r\n,SqlRAG.dbo.CosineSimilarity(@embedded,VectorizedQueryIntent) as cosine_similarity\r\n,1-SqlRAG.dbo.EuclideanDistance(@embedded,VectorizedQueryIntent) as euclidean_similarity\r\nFROM [SqlRAG].[dbo].[QueryIntentCache]\r\nwhere ExecStatus is null) A\r\nWHERE (cosine_similarity>=0.96) or (cosine_similarity>=0.90 and euclidean_similarity>0.70) or (cosine_similarity=-1 and euclidean_similarity>-1)\r\norder by cosine_similarity desc", connection); 19 | SqlParameter paraEmbedded = new SqlParameter("embedded_string", SqlDbType.NVarChar,-1); 20 | paraEmbedded.Value = embedded.ToString(); 21 | command.Parameters.Add(paraEmbedded); 22 | return new SqlString((string)command.ExecuteScalar()); 23 | 24 | 25 | 26 | 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /SQLRAG.application/SQLRAG.application.sqlproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Debug 5 | AnyCPU 6 | SQLRAG.application 7 | 2.0 8 | 4.1 9 | {60c728d1-4969-4138-8344-fb002ded1f59} 10 | Microsoft.Data.Tools.Schema.Sql.Sql160DatabaseSchemaProvider 11 | Database 12 | 13 | SQLRAG 14 | SQLRAG.application 15 | 1028,CI 16 | BySchemaAndSchemaType 17 | True 18 | v4.7.2 19 | CS 20 | Properties 21 | False 22 | True 23 | True 24 | UNSAFE 25 | dbo 26 | Chinese_Taiwan_Stroke_CI_AS 27 | True 28 | True 29 | False 30 | 0.1.0.0 31 | 32 | 33 | bin\Release\ 34 | $(MSBuildProjectName).sql 35 | False 36 | pdbonly 37 | true 38 | false 39 | true 40 | prompt 41 | 4 42 | 43 | 44 | bin\Debug\ 45 | $(MSBuildProjectName).sql 46 | false 47 | true 48 | full 49 | false 50 | true 51 | true 52 | prompt 53 | 4 54 | x64 55 | 56 | 57 | 11.0 58 | True 59 | 11.0 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | DoNotCopy 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | SQLRAG 95 | {7e86c45f-4d32-4ab0-af50-79b6270edaee} 96 | True 97 | True 98 | 99 | 100 | -------------------------------------------------------------------------------- /SQLRAG.application/SqlRAGCertificate.sql: -------------------------------------------------------------------------------- 1 | CREATE CERTIFICATE [SqlRAGCertificate] 2 | ENCRYPTION BY PASSWORD = 'SqlRAGP@ssw0rd' 3 | WITH SUBJECT = 'certificate_subject' 4 | -------------------------------------------------------------------------------- /SQLRAG.application/SqlRagStoredProcedure.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.CodeDom; 3 | using System.Data; 4 | using System.Data.SqlClient; 5 | using System.Data.SqlTypes; 6 | using Microsoft.SqlServer.Server; 7 | 8 | public partial class StoredProcedures 9 | { 10 | 11 | [Microsoft.SqlServer.Server.SqlProcedure] 12 | public static void CheckQueryIntentCache(SqlString question) 13 | { 14 | using (SqlConnection connection = new SqlConnection("context connection=true")) 15 | { 16 | SqlArray embedded = OpenaiFunction.GetEmbedding(question); 17 | connection.Open(); 18 | SqlCommand command = new SqlCommand("SELECT top 1 [GeneratedTSQL]\r\nFROM (\r\nSELECT [QueryIntent]\r\n ,[GeneratedTSQL]\r\n\t ,[CreateDate]\r\n\t ,SqlRAG.dbo.CosineSimilarity(@embedded,VectorizedQueryIntent) as cosine_similarity\r\n\t ,1-SqlRAG.dbo.EuclideanDistance(@embedded,VectorizedQueryIntent) as euclidean_similarity\r\n\t ,1-SqlRAG.dbo.MinkowskiDistance(@embedded,VectorizedQueryIntent,2.5) as minkowski_similarity\r\n FROM [SqlRAG].[dbo].[QueryIntentCache] \r\n where ExecStatus is null) A\r\n WHERE (cosine_similarity>=0.96) or (cosine_similarity>=0.90 and euclidean_similarity>0.70)\r\n order by cosine_similarity desc", connection); 19 | SqlParameter paraEmbedded = new SqlParameter("embedded", embedded); 20 | command.Parameters.Add(paraEmbedded); 21 | SqlContext.Pipe.ExecuteAndSend(command); 22 | SqlDataReader r = command.ExecuteReader(); 23 | 24 | } 25 | } 26 | 27 | 28 | [Microsoft.SqlServer.Server.SqlProcedure] 29 | public static void InsertQueryIntentCache(SqlString query_intent, [SqlFacet(MaxSize = -1)] SqlString embedded, [SqlFacet(MaxSize = -1)] SqlString tsql, [SqlFacet(MaxSize = -1)] SqlString data, SqlString exec_status) 30 | { 31 | using (SqlConnection connection = new SqlConnection("context connection=true")) 32 | { 33 | 34 | connection.Open(); 35 | SqlCommand command = new SqlCommand("INSERT INTO SQLRAG.[dbo].[QueryIntentCache]\r\n ([QueryIntent]\r\n ,[VectorizedQueryIntent]\r\n ,[GeneratedTSQL]\r\n ,[GeneratedData]\r\n ,[ExecStatus])\r\n VALUES\r\n (@query_intent\r\n ,SqlArray::Parse(@embedded)\r\n ,@tsql\r\n ,@data\r\n ,@exec_status)", connection); 36 | SqlParameter paraQuestion = new SqlParameter("query_intent", query_intent.Value); 37 | SqlParameter paraEmbedded = new SqlParameter("embedded", embedded.Value); 38 | SqlParameter paraTSql = new SqlParameter("tsql", tsql.Value); 39 | SqlParameter paraData = new SqlParameter("data", data.Value); 40 | SqlParameter paraExecStatus = new SqlParameter("exec_status", exec_status.Value); 41 | 42 | 43 | command.Parameters.Add(paraQuestion); 44 | command.Parameters.Add(paraEmbedded); 45 | command.Parameters.Add(paraTSql); 46 | command.Parameters.Add(paraData); 47 | command.Parameters.Add(paraExecStatus); 48 | command.ExecuteNonQuery(); 49 | 50 | } 51 | } 52 | 53 | [Microsoft.SqlServer.Server.SqlProcedure] 54 | public static void InsertKnowledgeBase(SqlGuid id, SqlGuid parent_id, SqlInt32 ordinal, SqlBoolean is_rewrite, SqlInt16 source_type, SqlString url, [SqlFacet(MaxSize = -1)] SqlString text_content, [SqlFacet(MaxSize = -1)] SqlString raw) 55 | { 56 | 57 | using (SqlConnection connection = new SqlConnection("context connection=true")) 58 | { 59 | try 60 | { 61 | SqlArray embedded=SqlArray.Null; 62 | try 63 | { 64 | embedded = OpenaiFunction.GetEmbedding(text_content); 65 | } 66 | catch 67 | { 68 | 69 | } 70 | connection.Open(); 71 | SqlCommand command = new SqlCommand("INSERT INTO SQLRAG.[dbo].[KnowledgeBase]\r\n ([PartId]\r\n ,[ParentId]\r\n ,[Ordinal]\r\n ,[IsRewrite]\r\n ,[SourceType]\r\n ,[Url]\r\n ,[TextContent]\r\n ,[Embeddings]\r\n ,[Raw])\r\n VALUES\r\n (@id ,@parent_id, @ordinal , @is_rewrite, @source_type,@url, @text_content,@embedded, @raw)", connection); 72 | SqlParameter para_id = new SqlParameter("id", id); 73 | SqlParameter para_parentid = new SqlParameter("parent_id", parent_id); 74 | SqlParameter para_ordinal = new SqlParameter("ordinal", ordinal); 75 | SqlParameter para_isrewrite = new SqlParameter("is_rewrite", is_rewrite); 76 | SqlParameter para_source_type = new SqlParameter("source_type", source_type); 77 | SqlParameter para_url = new SqlParameter("url", url); 78 | SqlParameter para_text_content = new SqlParameter("text_content", text_content); 79 | SqlParameter paraEmbedded = new SqlParameter("embedded", embedded); 80 | SqlParameter para_raw = new SqlParameter("raw", raw); 81 | 82 | para_url.SqlDbType = SqlDbType.NVarChar; 83 | para_text_content.SqlDbType = SqlDbType.NVarChar; 84 | para_raw.SqlDbType = SqlDbType.NVarChar; 85 | paraEmbedded.SqlDbType = SqlDbType.Udt; 86 | paraEmbedded.UdtTypeName = "SqlArray"; 87 | 88 | command.Parameters.Add(para_id); 89 | command.Parameters.Add(para_parentid); 90 | command.Parameters.Add(para_ordinal); 91 | command.Parameters.Add(para_isrewrite); 92 | command.Parameters.Add(para_source_type); 93 | command.Parameters.Add(para_url); 94 | command.Parameters.Add(para_text_content); 95 | command.Parameters.Add(paraEmbedded); 96 | command.Parameters.Add(para_raw); 97 | command.ExecuteNonQuery(); 98 | 99 | } 100 | catch (System.Net.WebException we) 101 | { 102 | throw new Exception(we.Message + "\r\n" + we.InnerException.Message + "\r\n" + we.Response + "\r\n" + we.StackTrace); 103 | } 104 | 105 | 106 | } 107 | } 108 | 109 | 110 | 111 | } 112 | -------------------------------------------------------------------------------- /SQLRAG.application/dbo.EncryptedKeys.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [dbo].[EncryptedKeys]( 2 | [KeyName] [nvarchar](50) NOT NULL, 3 | [KeyDesc] [nvarchar](512) NULL, 4 | [KeyValue] VARBINARY(4000) NOT NULL, 5 | CONSTRAINT [PK_EncryptedKeys] PRIMARY KEY CLUSTERED 6 | ( 7 | [KeyName] ASC 8 | )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] 9 | ) ON [PRIMARY] -------------------------------------------------------------------------------- /SQLRAG.application/dbo.KnowledgeBase.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [dbo].[KnowledgeBase]( 2 | [PartId] [uniqueidentifier] NOT NULL, 3 | [ParentId] [uniqueidentifier] NULL, 4 | [Ordinal] int NULL, 5 | [IsRewrite] [bit] NULL DEFAULT 0, 6 | [SourceType] [smallint] NULL, 7 | [Url] [nvarchar](512) NULL, 8 | [TextContent] [nvarchar](max) NOT NULL, 9 | [Embeddings] [dbo].[SqlArray] NULL, 10 | [Raw] [nvarchar](max) NULL, 11 | [CreateDate] [datetime] NOT NULL, 12 | CONSTRAINT [PK_TextContent] PRIMARY KEY CLUSTERED 13 | ( 14 | [PartId] ASC 15 | )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] 16 | ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] 17 | GO 18 | 19 | ALTER TABLE [dbo].[KnowledgeBase] ADD CONSTRAINT [DF_KnowledgeBase_CreateDate] DEFAULT (getdate()) FOR [CreateDate] 20 | GO 21 | 22 | -------------------------------------------------------------------------------- /SQLRAG.application/dbo.QueryIntentCache.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [dbo].[QueryIntentCache]( 2 | [id] [uniqueidentifier] NOT NULL, 3 | [QueryIntent] [nvarchar](max) NULL, 4 | [VectorizedQueryIntent] SqlArray NULL, 5 | [GeneratedTSQL] [nvarchar](max) NULL, 6 | [GeneratedData] [nvarchar](max) NULL, 7 | [CreateDate] [datetime] NULL, 8 | [ExecStatus] [nvarchar](256) NULL, 9 | CONSTRAINT [PK_QueryIntentCache] PRIMARY KEY CLUSTERED 10 | ( 11 | [id] ASC 12 | )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY] 13 | ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY] 14 | GO 15 | 16 | ALTER TABLE [dbo].[QueryIntentCache] ADD CONSTRAINT [DF_QueryIntentCache_id] DEFAULT (newid()) FOR [id] 17 | GO 18 | 19 | ALTER TABLE [dbo].[QueryIntentCache] ADD CONSTRAINT [DF_QueryIntentCache_CreateDate] DEFAULT (getdate()) FOR [CreateDate] 20 | GO 21 | 22 | -------------------------------------------------------------------------------- /SQLRAG.application/dbo.WebPilotLogs.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [dbo].[WebPilotLogs] 2 | ( 3 | [Id] UNIQUEIDENTIFIER NOT NULL PRIMARY KEY DEFAULT (newid()), 4 | [ParentId] UNIQUEIDENTIFIER NULL 5 | ) 6 | -------------------------------------------------------------------------------- /SQLRAG.application/postdeployscript.sql: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /SQLRAG.publish.xml: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | False 5 | SQLRAG 6 | SQLRAG.sql 7 | 1 8 | Data Source=.;Persist Security Info=True;User ID=sa;Pooling=False;Multiple Active Result Sets=False;Connect Timeout=60;Encrypt=True;Trust Server Certificate=True;Command Timeout=0 9 | False 10 | False 11 | False 12 | True 13 | False 14 | False 15 | True 16 | True 17 | False 18 | False 19 | True 20 | True 21 | True 22 | False 23 | False 24 | False 25 | False 26 | True 27 | True 28 | True 29 | True 30 | True 31 | True 32 | True 33 | True 34 | 35 | -------------------------------------------------------------------------------- /SQLRAG.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 17 4 | VisualStudioVersion = 17.8.34309.116 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{00D1A9C2-B5F0-4AF3-8072-F6C62B433612}") = "SQLRAG", "SQLRAG.sqlproj", "{7E86C45F-4D32-4AB0-AF50-79B6270EDAEE}" 7 | EndProject 8 | Project("{00D1A9C2-B5F0-4AF3-8072-F6C62B433612}") = "SQLRAG.application", "SQLRAG.application\SQLRAG.application.sqlproj", "{60C728D1-4969-4138-8344-FB002DED1F59}" 9 | EndProject 10 | Global 11 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 12 | Debug|Any CPU = Debug|Any CPU 13 | Release|Any CPU = Release|Any CPU 14 | EndGlobalSection 15 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 16 | {7E86C45F-4D32-4AB0-AF50-79B6270EDAEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 17 | {7E86C45F-4D32-4AB0-AF50-79B6270EDAEE}.Debug|Any CPU.Build.0 = Debug|Any CPU 18 | {7E86C45F-4D32-4AB0-AF50-79B6270EDAEE}.Debug|Any CPU.Deploy.0 = Debug|Any CPU 19 | {7E86C45F-4D32-4AB0-AF50-79B6270EDAEE}.Release|Any CPU.ActiveCfg = Release|Any CPU 20 | {7E86C45F-4D32-4AB0-AF50-79B6270EDAEE}.Release|Any CPU.Build.0 = Release|Any CPU 21 | {7E86C45F-4D32-4AB0-AF50-79B6270EDAEE}.Release|Any CPU.Deploy.0 = Release|Any CPU 22 | {60C728D1-4969-4138-8344-FB002DED1F59}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 23 | {60C728D1-4969-4138-8344-FB002DED1F59}.Debug|Any CPU.Build.0 = Debug|Any CPU 24 | {60C728D1-4969-4138-8344-FB002DED1F59}.Debug|Any CPU.Deploy.0 = Debug|Any CPU 25 | {60C728D1-4969-4138-8344-FB002DED1F59}.Release|Any CPU.ActiveCfg = Release|Any CPU 26 | {60C728D1-4969-4138-8344-FB002DED1F59}.Release|Any CPU.Build.0 = Release|Any CPU 27 | {60C728D1-4969-4138-8344-FB002DED1F59}.Release|Any CPU.Deploy.0 = Release|Any CPU 28 | EndGlobalSection 29 | GlobalSection(SolutionProperties) = preSolution 30 | HideSolutionNode = FALSE 31 | EndGlobalSection 32 | GlobalSection(ExtensibilityGlobals) = postSolution 33 | SolutionGuid = {7F450B2E-31E1-42AA-8C3C-5E32BA01E5E3} 34 | EndGlobalSection 35 | EndGlobal 36 | -------------------------------------------------------------------------------- /SQLRAG.sqlproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Debug 5 | AnyCPU 6 | SQLRAG 7 | 2.0 8 | 4.1 9 | {7e86c45f-4d32-4ab0-af50-79b6270edaee} 10 | Microsoft.Data.Tools.Schema.Sql.Sql160DatabaseSchemaProvider 11 | Database 12 | 13 | SQLRAG 14 | SQLRAG 15 | 1028,CI 16 | BySchemaAndSchemaType 17 | True 18 | v4.7.2 19 | CS 20 | Properties 21 | False 22 | True 23 | True 24 | True 25 | True 26 | Chinese_Taiwan_Stroke_CI_AS 27 | True 28 | dbo 29 | SAFE 30 | True 31 | 0.1.0.0 32 | 33 | 34 | bin\Release\ 35 | $(MSBuildProjectName).sql 36 | False 37 | pdbonly 38 | true 39 | false 40 | true 41 | prompt 42 | 4 43 | 44 | 45 | bin\Debug\ 46 | $(MSBuildProjectName).sql 47 | false 48 | true 49 | full 50 | true 51 | true 52 | true 53 | prompt 54 | 4 55 | x64 56 | True 57 | 58 | 59 | 11.0 60 | True 61 | 11.0 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | Always 90 | 91 | 92 | 93 | 94 | 95 | PreserveNewest 96 | 97 | 98 | PreserveNewest 99 | 100 | 101 | 102 | 103 | 104 | -------------------------------------------------------------------------------- /SqlArray.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Runtime; 3 | using System.Data; 4 | using System.Data.SqlClient; 5 | using System.Data.SqlTypes; 6 | using System.Security.Cryptography; 7 | using Microsoft.SqlServer.Server; 8 | using System.Collections.Generic; 9 | using System.IO; 10 | using System.Security.Policy; 11 | using System.Data.Linq; 12 | 13 | 14 | 15 | [Serializable] 16 | [Microsoft.SqlServer.Server.SqlUserDefinedType(Format.UserDefined, MaxByteSize = -1, IsByteOrdered = true, IsFixedLength = false, Name = "SqlArray")] 17 | public struct SqlArray : INullable, IBinarySerialize 18 | { 19 | // Private 成員 20 | private bool _null; 21 | private double[] _data; 22 | 23 | public static SqlArray Null 24 | { 25 | get 26 | { 27 | SqlArray h = new SqlArray(); 28 | h._null = true; 29 | return h; 30 | } 31 | } 32 | 33 | 34 | public double[] Data 35 | { 36 | get 37 | { 38 | // 將程式碼放在此處 39 | return _data; 40 | } 41 | private set { _data = value; } 42 | 43 | } 44 | 45 | internal SqlArray(bool isNull) { _null = isNull; _data = Array.Empty(); } 46 | 47 | //public SqlArray() 48 | //{ 49 | // _null = true; 50 | // _data = []; 51 | //} 52 | 53 | public SqlArray(double[] dataArray) 54 | { 55 | _null = false; 56 | _data = dataArray; 57 | } 58 | 59 | //public static SqlArray operator +(SqlArray a) => a; 60 | 61 | public static SqlArray operator -(SqlArray a) 62 | { 63 | // 檢查維度是否匹配，實際實現中應該添加更多錯誤處理 64 | var result = new double[a._data.Length]; 65 | for (int i = 0; i < a._data.Length; i++) 66 | { 67 | result[i] = -a._data[i]; 68 | } 69 | return new SqlArray(result); 70 | } 71 | 72 | public static SqlBoolean Equals(SqlArray a, SqlArray b) 73 | { 74 | if (a.Length != b.Length) 75 | { 76 | return new SqlBoolean(false); 77 | } 78 | else 79 | { 80 | for (int i = 0; i < a.Length; i++) 81 | { 82 | if (a._data[i] != b._data[i]) 83 | { 84 | return new SqlBoolean(false); 85 | } 86 | } 87 | return new SqlBoolean(true); 88 | } 89 | } 90 | 91 | 92 | public static SqlBoolean operator ==(SqlArray a, SqlArray b) 93 | { 94 | if (a.Length != b.Length) 95 | { 96 | return new SqlBoolean(false); 97 | } 98 | else 99 | { 100 | for (int i = 0; i < a.Length; i++) 101 | { 102 | if (a._data[i] != b._data[i]) 103 | { 104 | return new SqlBoolean(false); 105 | } 106 | } 107 | return new SqlBoolean(true); 108 | } 109 | } 110 | 111 | public static SqlBoolean operator !=(SqlArray a, SqlArray b) 112 | { 113 | if (a.Length != b.Length) 114 | { 115 | return new SqlBoolean(true); 116 | } 117 | else 118 | { 119 | for (int i = 0; i < a.Length; i++) 120 | { 121 | if (a._data[i] != b._data[i]) 122 | { 123 | return new SqlBoolean(true); 124 | } 125 | } 126 | return new SqlBoolean(false); 127 | } 128 | } 129 | 130 | public override int GetHashCode() 131 | => _data.GetHashCode(); 132 | 133 | public override bool Equals(object other) 134 | => other is SqlArray otherArray && _data == otherArray._data; 135 | 136 | //public bool Equals(SqlArray other) 137 | // => _data == other._data; 138 | 139 | 140 | [SqlMethod(IsDeterministic = true, IsPrecise = true,Name ="Add")] 141 | public static SqlArray Add(SqlArray a, SqlArray b) 142 | { 143 | 144 | var result = new double[a._data.Length]; 145 | for (int i = 0; i < a._data.Length; i++) 146 | { 147 | result[i] = a._data[i] + b._data[i]; 148 | } 149 | return new SqlArray(result); 150 | } 151 | 152 | public static SqlArray operator +(SqlArray a, SqlArray b) 153 | { 154 | // 檢查維度是否匹配，實際實現中應該添加更多錯誤處理 155 | var result = new double[a._data.Length]; 156 | for (int i = 0; i < a._data.Length; i++) 157 | { 158 | result[i] = a._data[i] + b._data[i]; 159 | } 160 | return new SqlArray(result); 161 | } 162 | 163 | [SqlMethod(IsDeterministic = true, IsPrecise = true)] 164 | public static SqlArray Subtract(SqlArray a, SqlArray b) 165 | { 166 | var result = new double[a._data.Length]; 167 | for (int i = 0; i < a._data.Length; i++) 168 | { 169 | result[i] = a._data[i] - b._data[i]; 170 | } 171 | return new SqlArray(result); 172 | } 173 | public static SqlArray operator -(SqlArray a, SqlArray b) 174 | { 175 | 176 | var result = new double[a._data.Length]; 177 | for (int i = 0; i < a._data.Length; i++) 178 | { 179 | result[i] = a._data[i] - b._data[i]; 180 | } 181 | return new SqlArray(result); 182 | } 183 | 184 | [SqlMethod(IsDeterministic = true, IsPrecise = true)] 185 | public static SqlArray Divide(SqlArray a, SqlArray b) 186 | { 187 | if (System.Array.Exists(b._data, x => x == 0)) 188 | { 189 | throw new DivideByZeroException(); 190 | } 191 | var result = new double[a._data.Length]; 192 | for (int i = 0; i < a._data.Length; i++) 193 | { 194 | result[i] = a._data[i] / b._data[i]; 195 | } 196 | return new SqlArray(result); 197 | } 198 | public static SqlArray operator /(SqlArray a, SqlArray b) 199 | { 200 | if (System.Array.Exists(b._data, x => x == 0)) 201 | { 202 | throw new DivideByZeroException(); 203 | } 204 | var result = new double[a._data.Length]; 205 | for (int i = 0; i < a._data.Length; i++) 206 | { 207 | result[i] = a._data[i] / b._data[i]; 208 | } 209 | return new SqlArray(result); 210 | } 211 | 212 | [SqlMethod(IsDeterministic = true, IsPrecise = true)] 213 | public static SqlArray Multiply(SqlArray a, SqlArray b) 214 | { 215 | // 檢查維度是否匹配，實際實現中應該添加更多錯誤處理 216 | var result = new double[a._data.Length]; 217 | for (int i = 0; i < a._data.Length; i++) 218 | { 219 | result[i] = a._data[i] * b._data[i]; 220 | } 221 | return new SqlArray(result); 222 | } 223 | [SqlMethod(IsDeterministic = true, IsPrecise = true)] 224 | public static SqlArray Dot(SqlArray a, SqlArray b) 225 | { 226 | // 檢查維度是否匹配，實際實現中應該添加更多錯誤處理 227 | var result = new double[a._data.Length]; 228 | for (int i = 0; i < a._data.Length; i++) 229 | { 230 | result[i] = a._data[i] * b._data[i]; 231 | } 232 | return new SqlArray(result); 233 | } 234 | public static SqlArray operator *(SqlArray a, SqlArray b) 235 | { 236 | // 檢查維度是否匹配，實際實現中應該添加更多錯誤處理 237 | var result = new double[a._data.Length]; 238 | for (int i = 0; i < a._data.Length; i++) 239 | { 240 | result[i] = a._data[i] * b._data[i]; 241 | } 242 | return new SqlArray(result); 243 | } 244 | 245 | 246 | public static explicit operator SqlArray(SqlString s) 247 | { 248 | if (s.IsNull) 249 | return Null; 250 | 251 | // 將程式碼放在此處 252 | return new SqlArray(System.Array.ConvertAll(s.Value.Split(','), Double.Parse)); 253 | } 254 | public static implicit operator SqlArray(string s) 255 | { 256 | if (string.IsNullOrEmpty(s)) 257 | return Null; 258 | 259 | // 將程式碼放在此處 260 | return new SqlArray(System.Array.ConvertAll(s.Split(','), Double.Parse)); 261 | } 262 | public static explicit operator string(SqlArray x) 263 | { 264 | return string.Join(",", x._data); 265 | } 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | [return: SqlFacet(MaxSize = -1)] 282 | [SqlMethod(IsDeterministic = true, IsPrecise = false)] 283 | public override string ToString() 284 | { 285 | 286 | string _str = string.Join(",", _data); 287 | return _str; 288 | 289 | } 290 | 291 | public bool IsNull 292 | { 293 | get 294 | { 295 | return _null; 296 | } 297 | } 298 | 299 | public int Rank 300 | { 301 | get 302 | { 303 | 304 | return _data.Rank; 305 | } 306 | } 307 | 308 | public int Length 309 | { 310 | get 311 | { 312 | return _data.Length; 313 | } 314 | } 315 | 316 | public long LongLength 317 | { 318 | get 319 | { 320 | return _data.LongLength; 321 | } 322 | } 323 | 324 | 325 | 326 | 327 | 328 | [SqlMethod(IsDeterministic = true, IsPrecise = true)] 329 | public static SqlArray Parse([SqlFacet(MaxSize = -1)] SqlString s) 330 | { 331 | if (s.IsNull) 332 | return Null; 333 | 334 | // 將程式碼放在此處 335 | return new SqlArray(System.Array.ConvertAll(s.Value.Split(','), Double.Parse)); 336 | } 337 | 338 | //[SqlMethod(IsDeterministic = true, IsPrecise = true)] 339 | //public static SqlArray add(SqlArray a, SqlArray b) 340 | //{ 341 | // if (a._data.Length != b._data.Length) 342 | // { 343 | // throw new ArgumentException("維度不匹配"); 344 | // } 345 | // var result = new double[a._data.Length]; 346 | // for (int i = 0; i < a._data.Length; i++) 347 | // { 348 | // result[i] = a._data[i] + b._data[i]; 349 | // } 350 | // return new SqlArray(result); 351 | //} 352 | 353 | //[SqlMethod(IsDeterministic = true, IsPrecise = true)] 354 | //public static SqlArray subtract(SqlArray a, SqlArray b) 355 | //{ 356 | // if (a._data.Length != b._data.Length) 357 | // { 358 | // throw new ArgumentException("維度不匹配"); 359 | // } 360 | // var result = new double[a._data.Length]; 361 | // for (int i = 0; i < a._data.Length; i++) 362 | // { 363 | // result[i] = a._data[i] - b._data[i]; 364 | // } 365 | // return new SqlArray(result); 366 | //} 367 | 368 | 369 | 370 | public SqlArray Concate(List arrays) 371 | { 372 | List data_list = new List(); 373 | foreach (SqlArray a in arrays) 374 | { 375 | data_list.AddRange(a.Data); 376 | } 377 | return new SqlArray(data_list.ToArray()); 378 | } 379 | [SqlMethodAttribute(IsDeterministic = true, IsPrecise = true)] 380 | public static SqlInt32 ArgMax(SqlArray arr) 381 | { 382 | int idx = 0; 383 | double max_value = arr._data[0]; 384 | for(int i=0;i< arr._data.Length;i++) 385 | { 386 | double d = arr._data[i]; 387 | if (d > max_value) 388 | { 389 | idx = i; 390 | max_value = d; 391 | } 392 | } 393 | return new SqlInt32(idx); 394 | } 395 | 396 | [SqlMethodAttribute(IsDeterministic = true, IsPrecise = true)] 397 | public static SqlInt32 ArgMin(SqlArray arr) 398 | { 399 | int idx = 0; 400 | double min_value = arr._data[0]; 401 | for (int i = 0; i < arr._data.Length; i++) 402 | { 403 | double d = arr._data[i]; 404 | if (d segAsSentence(string txt) 22 | { 23 | // 使用正則表達式處理字符串 24 | txt = Regex.Replace(txt, Prefixes, "$1"); 25 | txt = Regex.Replace(txt, Websites, "$1"); 26 | if (txt.Contains("Ph.D")) txt = txt.Replace("Ph.D.", "PhD"); 27 | txt = Regex.Replace(txt, "\\s" + Alphabets + "[.] ", " $1 "); 28 | txt = Regex.Replace(txt, Acronyms + " " + Starters, "$1 $2"); 29 | txt = Regex.Replace(txt, Alphabets + "[.]" + Alphabets + "[.]" + Alphabets + "[.]", "$1$2$3"); 30 | txt = Regex.Replace(txt, Alphabets + "[.]" + Alphabets + "[.]", "$1$2"); 31 | txt = Regex.Replace(txt, Numbers + "[.]" + Numbers, "$1$2"); 32 | txt = Regex.Replace(txt, " " + Suffixes + "[.] " + Starters, " $1 $2"); 33 | txt = Regex.Replace(txt, " " + Suffixes + "[.]", " $1"); 34 | txt = Regex.Replace(txt, " " + Alphabets + "[.]", " $1"); 35 | 36 | // 處理引號和句末標點的組合 37 | txt = txt.Replace(".”", "”.") 38 | .Replace(".\"", "\".") 39 | .Replace("!\"", "\"!") 40 | .Replace("?\"", "\"?"); 41 | 42 | // 替換省略號和接下來的字符 43 | txt = Regex.Replace(txt, "(\\.\\.\\.)([^”’])", ""); 44 | 45 | // 將句末的點、問號、驚嘆號替換為 46 | txt = txt.Replace(".", ".") 47 | .Replace("?", "?") 48 | .Replace("!", "!") 49 | .Replace("", "."); 50 | 51 | // 對中文的標點進行處理 52 | txt = Regex.Replace(txt, "([。！？\\?])([^”’])", "$1$2"); 53 | txt = Regex.Replace(txt, "(\\.{6})([^”’])", "$1$2"); 54 | txt = Regex.Replace(txt, "([。！？\\?][”’])([^。！？\\?])", "$1$2"); 55 | 56 | 57 | // 移除尾部空白並按分割 58 | txt = txt.Trim(); 59 | List sentences = new List(txt.Split(new string[] { "" }, StringSplitOptions.RemoveEmptyEntries)); 60 | return sentences; 61 | } 62 | 63 | [return: SqlFacet(MaxSize = -1)] 64 | [Microsoft.SqlServer.Server.SqlFunction(IsDeterministic = true)] 65 | public static SqlString SplitText([SqlFacet(MaxSize = -1)] SqlString inputText) 66 | { 67 | string returnstring = string.Join("\r\n", segAsSentence(inputText.Value)); 68 | 69 | return new SqlString (returnstring); 70 | } 71 | 72 | [return: SqlFacet(MaxSize = -1)] 73 | [Microsoft.SqlServer.Server.SqlFunction(IsDeterministic = true)] 74 | public static SqlString ChineseFull2Half([SqlFacet(MaxSize = -1)] SqlString inputText) 75 | { 76 | string returnstring = string.Join("\r\n", segAsSentence(inputText.Value)); 77 | 78 | return new SqlString(returnstring); 79 | } 80 | 81 | [return: SqlFacet(MaxSize = -1)] 82 | [Microsoft.SqlServer.Server.SqlFunction(IsDeterministic = true)] 83 | public static SqlString ChineseHalf2Full([SqlFacet(MaxSize = -1)] SqlString inputText) 84 | { 85 | string returnstring = string.Join("\r\n", segAsSentence(inputText.Value)); 86 | 87 | return new SqlString(returnstring); 88 | } 89 | 90 | 91 | [Microsoft.SqlServer.Server.SqlFunction] 92 | [return: SqlFacet(MaxSize = -1)] 93 | public static SqlString ToTraditionalChinese([SqlFacet(MaxSize = -1)] SqlString InputString) 94 | { 95 | // 將程式碼放在此處 96 | return new SqlString(Microsoft.VisualBasic.Strings.StrConv(InputString.Value, Microsoft.VisualBasic.VbStrConv.TraditionalChinese, 2052)); 97 | } 98 | 99 | [Microsoft.SqlServer.Server.SqlFunction] 100 | [return: SqlFacet(MaxSize = -1)] 101 | public static SqlString ToSimplifiedChinese([SqlFacet(MaxSize = -1)] SqlString InputString) 102 | { 103 | 104 | return new SqlString(Microsoft.VisualBasic.Strings.StrConv(InputString.Value, Microsoft.VisualBasic.VbStrConv.SimplifiedChinese, 2052)); 105 | } 106 | 107 | 108 | [Microsoft.SqlServer.Server.SqlFunction] 109 | [return: SqlFacet(MaxSize = -1)] 110 | public static SqlString RegexReplace([SqlFacet(MaxSize = -1)] string InputString, string PattermString, string ReplaceWord) 111 | { 112 | 113 | MatchCollection matchcollection = Regex.Matches(InputString, PattermString, ExpressionOptions); 114 | if (matchcollection != null) 115 | { 116 | 117 | foreach (Match item in matchcollection) 118 | { 119 | InputString = InputString.Replace(item.Value, ReplaceWord); 120 | } 121 | return new SqlString(InputString); 122 | 123 | } 124 | else 125 | { 126 | return SqlString.Null; 127 | } 128 | } 129 | 130 | private const RegexOptions ExpressionOptions = RegexOptions.Singleline | RegexOptions.Multiline | RegexOptions.IgnoreCase; 131 | 132 | [Microsoft.SqlServer.Server.SqlFunction] 133 | public static SqlString RegexMatch([SqlFacet(MaxSize = -1)] string InputString, string PattermString, int Index) 134 | { 135 | if (string.IsNullOrEmpty(InputString)) 136 | { 137 | return SqlString.Null; 138 | } 139 | else 140 | { 141 | 142 | MatchCollection matchcollection = Regex.Matches(InputString, PattermString, ExpressionOptions); 143 | if (matchcollection != null) 144 | { 145 | if (Index == 0) 146 | { 147 | string returnstring = ""; 148 | foreach (Match item in matchcollection) 149 | { 150 | returnstring += item.Value; 151 | } 152 | return new SqlString(returnstring); 153 | } 154 | else if (Index <= matchcollection.Count && Index >= 1) 155 | { 156 | 157 | return new SqlString(matchcollection[Index - 1].Value); 158 | } 159 | else 160 | { 161 | return SqlString.Null; 162 | } 163 | } 164 | else 165 | { 166 | return SqlString.Null; 167 | } 168 | } 169 | } 170 | 171 | } 172 | -------------------------------------------------------------------------------- /assets/ChatCompletion.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AllanYiin/SQLRAG/a72d1028b7f5f60e96a7f4f14052af752d8200c6/assets/ChatCompletion.png -------------------------------------------------------------------------------- /assets/ChatCompletion_demo.sql: -------------------------------------------------------------------------------- 1 | --dbo.ChatCompletion(@inputPrompt, @systemProimpt, @model) 2 | 3 | select dbo.ChatCompletion('什麼是搜索增強生成RAG?','','') 4 | 5 | select dbo.ChatCompletion('關於代理貴公司產品一事，不知道你們考慮的如何，是否可以進入簽約階段?' 6 | ,'你是一個20年以上的商業日文翻譯專家，接下來任何輸入給你的內容你都能將它翻譯成得體的日文' 7 | ,'gpt-4-1106-preview') 8 | -------------------------------------------------------------------------------- /assets/QueryCache.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AllanYiin/SQLRAG/a72d1028b7f5f60e96a7f4f14052af752d8200c6/assets/QueryCache.png -------------------------------------------------------------------------------- /assets/QueryIntentCache_demo.sql: -------------------------------------------------------------------------------- 1 | Declare @question varchar(max) 2 | set @question = SqlRAG.dbo.GetEmbedding('2013年度各產品大分類的毛利率') 3 | 4 | SELECT [QueryIntent] 5 | ,[GeneratedTSQL] 6 | ,SqlRAG.dbo.CosineSimilarity(@question,VectorizedQueryIntent) as cosine_similarity 7 | ,1-SqlRAG.dbo.EuclideanDistance(@question,VectorizedQueryIntent) as euclidean_similarity 8 | ,1-SqlRAG.dbo.MinkowskiDistance(@question,VectorizedQueryIntent,2.5) as minkowski_similarity 9 | FROM [SqlRAG].[dbo].[QueryIntentCache] 10 | order by 5 desc 11 | --------------------------------------------------------------------------------