├── .gitattributes ├── .gitignore ├── .vs └── SqlDirectory │ └── v14 │ └── .suo ├── LICENSE ├── LuceneNetSqlDirectory.Tests ├── App.config ├── DemoUsage.cs ├── LockCanBeReleasedTests.cs ├── LuceneNetSqlDirectory.Tests.csproj ├── LuceneTestCase.cs ├── Properties │ └── AssemblyInfo.cs ├── Tests_from_Lucene │ ├── DocHelper.cs │ ├── ReadMe.txt │ ├── TestCachingSpanFilter.cs │ ├── TestDemo.cs │ ├── TestDirectoryReader.cs │ ├── TestSegmentReader.cs │ ├── TestTermScorer.cs │ └── TestWildcard.cs └── packages.config ├── LuceneNetSqlDirectory ├── App.config ├── AutoStopWatch.cs ├── Database.cs ├── Helpers │ ├── AutoStopwatch.cs │ ├── HigherOrderFunctions.cs │ ├── SqlConnectionHelper.cs │ └── SqlHelper.cs ├── LuceneNetSqlDirectory.csproj ├── LuceneNetSqlDirectory.nuspec ├── NuGet.exe ├── Options.cs ├── Properties │ └── AssemblyInfo.cs ├── SqlServerDirectory.cs ├── SqlServerIndexInput.cs ├── SqlServerIndexOutput.cs ├── SqlServerLock.cs ├── SqlServerLockFactory.cs └── packages.config ├── README.md └── SqlDirectory.sln /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | obj/ 3 | packages/ -------------------------------------------------------------------------------- /.vs/SqlDirectory/v14/.suo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahyTim/LuceneNetSqlDirectory/a2cf88a554426237f3dddfd0095a30b801217713/.vs/SqlDirectory/v14/.suo -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /LuceneNetSqlDirectory.Tests/App.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /LuceneNetSqlDirectory.Tests/DemoUsage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Text; 3 | using System.Threading; 4 | using Lucene.Net.Analysis.Standard; 5 | using Lucene.Net.Documents; 6 | using Lucene.Net.Index; 7 | using Lucene.Net.Search; 8 | using Lucene.Net.Store; 9 | using LuceneNetSqlDirectory.Helpers; 10 | using Microsoft.VisualStudio.TestTools.UnitTesting; 11 | 12 | namespace LuceneNetSqlDirectory.Tests 13 | { 14 | [TestClass] 15 | public class DemoUsage : LuceneTestCase 16 | { 17 | [TestMethod] 18 | public void Demo_Optimize_Test() 19 | { 20 | for (int i = 0; i < 3; i++) 21 | { 22 | var directory = new SqlServerDirectory(Connection, new Options()); 23 | var indexWriter = new IndexWriter(directory, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30), !IndexReader.IndexExists(directory), new Lucene.Net.Index.IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH)); 24 | indexWriter.SetMergeScheduler(new ConcurrentMergeScheduler()); 25 | indexWriter.SetMaxBufferedDocs(1000); 26 | 27 | for (int iDoc = 0; iDoc < 1000 * 10; iDoc++) 28 | { 29 | Document doc = new Document(); 30 | doc.Add(new Field("id", DateTime.Now.ToFileTimeUtc().ToString(), Field.Store.YES, 31 | Field.Index.ANALYZED, Field.TermVector.NO)); 32 | doc.Add(new Field("Title", "dog " + " microsoft rules", Field.Store.NO, Field.Index.ANALYZED, 33 | Field.TermVector.NO)); 34 | doc.Add(new Field("Body", "dog " + " microsoft rules", Field.Store.NO, Field.Index.ANALYZED, 35 | Field.TermVector.NO)); 36 | indexWriter.AddDocument(doc); 37 | } 38 | indexWriter.Flush(true, true, true); 39 | indexWriter.Optimize(true); 40 | indexWriter.Dispose(true); 41 | 42 | var searcher = new IndexSearcher(directory); 43 | Console.WriteLine("Number of docs: {0}", searcher.IndexReader.NumDocs()); 44 | SearchForPhrase(searcher, "microsoft", 999); 45 | searcher.Dispose(); 46 | } 47 | } 48 | 49 | [TestMethod] 50 | public void Demo_Usage_Test() 51 | { 52 | var directory = new SqlServerDirectory(Connection, new Options()); 53 | 54 | for (int outer = 0; outer < 10; outer++) 55 | { 56 | IndexWriter indexWriter = null; 57 | while (indexWriter == null) 58 | { 59 | try 60 | { 61 | indexWriter = new IndexWriter(directory, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30), 62 | !IndexReader.IndexExists(directory), 63 | new Lucene.Net.Index.IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH)); 64 | } 65 | catch (LockObtainFailedException) 66 | { 67 | Console.WriteLine("Lock is taken, waiting for timeout..."); 68 | Thread.Sleep(1000); 69 | } 70 | } 71 | ; 72 | Console.WriteLine("IndexWriter lock obtained, this process has exclusive write access to index"); 73 | indexWriter.SetRAMBufferSizeMB(100.0); 74 | //indexWriter.SetInfoStream(new StreamWriter(Console.OpenStandardOutput())); 75 | indexWriter.SetMergeScheduler(new SerialMergeScheduler()); 76 | indexWriter.SetMaxBufferedDocs(500); 77 | 78 | for (int iDoc = 0; iDoc < 1000; iDoc++) 79 | { 80 | Document doc = new Document(); 81 | doc.Add(new Field("id", DateTime.Now.ToFileTimeUtc().ToString(), Field.Store.YES, 82 | Field.Index.ANALYZED, Field.TermVector.NO)); 83 | doc.Add(new Field("Title", "dog " + GeneratePhrase(50), Field.Store.NO, Field.Index.ANALYZED, 84 | Field.TermVector.NO)); 85 | doc.Add(new Field("Body", "dog " + GeneratePhrase(50), Field.Store.NO, Field.Index.ANALYZED, 86 | Field.TermVector.NO)); 87 | indexWriter.AddDocument(doc); 88 | } 89 | 90 | Console.WriteLine("Total docs is {0}", indexWriter.NumDocs()); 91 | 92 | Console.Write("Flushing and disposing writer..."); 93 | indexWriter.Flush(true, true, true); 94 | //indexWriter.Optimize(); 95 | indexWriter.Commit(); 96 | indexWriter.Dispose(); 97 | } 98 | 99 | var searcher = new IndexSearcher(directory); 100 | Console.WriteLine("Number of docs: {0}", searcher.IndexReader.NumDocs()); 101 | SearchForPhrase(searcher, "microsoft", 2); 102 | } 103 | 104 | static void SearchForPhrase(IndexSearcher searcher, string phrase, int minNumberOfHits) 105 | { 106 | using (new AutoStopWatch($"Search for {phrase}")) 107 | { 108 | Lucene.Net.QueryParsers.QueryParser parser = new Lucene.Net.QueryParsers.QueryParser(Lucene.Net.Util.Version.LUCENE_30, "Body", new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30)); 109 | Lucene.Net.Search.Query query = parser.Parse(phrase); 110 | 111 | var hits = searcher.Search(new TermQuery(new Term("Title", "find me")), 100); 112 | 113 | hits = searcher.Search(query, 100); 114 | Console.WriteLine("Found {0} results for {1}", hits.TotalHits, phrase); 115 | Assert.IsTrue(hits.TotalHits > minNumberOfHits); 116 | } 117 | } 118 | 119 | static readonly Random Random = new Random((int)DateTime.Now.Ticks); 120 | static readonly string[] SampleTerms = 121 | { 122 | "dog","cat","car","horse","door","tree","chair","microsoft","apple","adobe","google","golf","linux","windows","firefox","mouse","hornet","monkey","giraffe","computer","monitor", 123 | "steve","fred","lili","albert","tom","shane","gerald","chris", 124 | "love","hate","scared","fast","slow","new","old" 125 | }; 126 | 127 | private static string GeneratePhrase(int maxTerms) 128 | { 129 | StringBuilder phrase = new StringBuilder(); 130 | int nWords = 2 + Random.Next(maxTerms); 131 | for (int i = 0; i < nWords; i++) 132 | { 133 | phrase.AppendFormat(" {0} {1}", SampleTerms[Random.Next(SampleTerms.Length)], Random.Next(32768).ToString()); 134 | } 135 | return phrase.ToString(); 136 | } 137 | } 138 | } -------------------------------------------------------------------------------- /LuceneNetSqlDirectory.Tests/LockCanBeReleasedTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | using Lucene.Net.Analysis.Standard; 7 | using Lucene.Net.Index; 8 | using Lucene.Net.Store; 9 | using Microsoft.VisualStudio.TestTools.UnitTesting; 10 | 11 | namespace LuceneNetSqlDirectory.Tests 12 | { 13 | [TestClass] 14 | public class LockCanBeReleasedTests : LuceneTestCase 15 | { 16 | [TestMethod] 17 | public void Test_Lock_Is_Released() 18 | { 19 | var directory = new SqlServerDirectory(Connection, new Options() { LockTimeoutInSeconds = 3 }); 20 | 21 | new IndexWriter(directory, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30), 22 | !IndexReader.IndexExists(directory), 23 | new Lucene.Net.Index.IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH)); 24 | 25 | IndexWriter indexWriter = null; 26 | while (indexWriter == null) 27 | { 28 | try 29 | { 30 | indexWriter = new IndexWriter(directory, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30), !IndexReader.IndexExists(directory), 31 | new Lucene.Net.Index.IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH)); 32 | } 33 | catch (LockObtainFailedException) 34 | { 35 | Console.WriteLine("Lock is taken, waiting for timeout...{0}", DateTime.Now); 36 | Thread.Sleep(1000); 37 | } 38 | } 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /LuceneNetSqlDirectory.Tests/LuceneNetSqlDirectory.Tests.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Debug 5 | AnyCPU 6 | {0BA12033-05D5-4DA7-8E43-A89596CB5A4C} 7 | Library 8 | Properties 9 | LuceneNetSqlDirectory.Tests 10 | LuceneNetSqlDirectory.Tests 11 | v4.5.2 12 | 512 13 | {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} 14 | 10.0 15 | $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) 16 | $(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages 17 | False 18 | UnitTest 19 | 20 | 21 | true 22 | full 23 | false 24 | bin\Debug\ 25 | DEBUG;TRACE 26 | prompt 27 | 4 28 | 29 | 30 | pdbonly 31 | true 32 | bin\Release\ 33 | TRACE 34 | prompt 35 | 4 36 | 37 | 38 | 39 | ..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll 40 | True 41 | 42 | 43 | ..\packages\Lucene.Net.3.0.3\lib\NET40\Lucene.Net.dll 44 | True 45 | 46 | 47 | ..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Analyzers.dll 48 | True 49 | 50 | 51 | ..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Core.dll 52 | True 53 | 54 | 55 | ..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.FastVectorHighlighter.dll 56 | True 57 | 58 | 59 | ..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Highlighter.dll 60 | True 61 | 62 | 63 | ..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Memory.dll 64 | True 65 | 66 | 67 | ..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Queries.dll 68 | True 69 | 70 | 71 | ..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Regex.dll 72 | True 73 | 74 | 75 | ..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.SimpleFacetedSearch.dll 76 | True 77 | 78 | 79 | ..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Snowball.dll 80 | True 81 | 82 | 83 | ..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.SpellChecker.dll 84 | True 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | {931814dd-8065-41e8-8472-8641fd1cf110} 122 | LuceneNetSqlDirectory 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | False 133 | 134 | 135 | False 136 | 137 | 138 | False 139 | 140 | 141 | False 142 | 143 | 144 | 145 | 146 | 147 | 148 | 155 | -------------------------------------------------------------------------------- /LuceneNetSqlDirectory.Tests/LuceneTestCase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Data.SqlClient; 3 | using Lucene.Net.Store; 4 | using Microsoft.VisualStudio.TestTools.UnitTesting; 5 | using Directory = System.IO.Directory; 6 | 7 | namespace LuceneNetSqlDirectory.Tests 8 | { 9 | [TestClass] 10 | public class LuceneTestCase 11 | { 12 | [TestInitialize] 13 | public virtual void TestInitialize() 14 | { 15 | Connection = new SqlConnection(System.Configuration.ConfigurationManager.ConnectionStrings["databaseForTests"].ConnectionString); 16 | Connection.Open(); 17 | SqlServerDirectory.ProvisionDatabase(Connection, schemaName: new Options().SchemaName, dropExisting: true); 18 | } 19 | 20 | [TestCleanup] 21 | public void TestCleanup() 22 | { 23 | Connection.Dispose(); 24 | } 25 | 26 | protected SqlConnection Connection { get; private set; } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /LuceneNetSqlDirectory.Tests/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("LuceneNetSqlDirectory.Tests")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("LuceneNetSqlDirectory.Tests")] 13 | [assembly: AssemblyCopyright("Copyright © 2017")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("0ba12033-05d5-4da7-8e43-a89596cb5a4c")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /LuceneNetSqlDirectory.Tests/Tests_from_Lucene/DocHelper.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | using Lucene.Net.Documents; 19 | using Lucene.Net.Index; 20 | using Analyzer = Lucene.Net.Analysis.Analyzer; 21 | using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer; 22 | using Document = Lucene.Net.Documents.Document; 23 | using Field = Lucene.Net.Documents.Field; 24 | using Directory = Lucene.Net.Store.Directory; 25 | using Similarity = Lucene.Net.Search.Similarity; 26 | 27 | namespace LuceneNetSqlDirectory.Tests 28 | { 29 | 30 | class DocHelper 31 | { 32 | public const System.String Field1Text = "field one text"; 33 | public const System.String TextField1Key = "textField1"; 34 | public static Field TextField1; 35 | 36 | public const System.String Field2Text = "field field field two text"; 37 | //Fields will be lexicographically sorted. So, the order is: field, text, two 38 | public static readonly int[] Field2Freqs = new int[]{3, 1, 1}; 39 | public const System.String TextField2Key = "textField2"; 40 | public static Field TextField2; 41 | 42 | 43 | public const System.String Field3Text = "aaaNoNorms aaaNoNorms bbbNoNorms"; 44 | public const System.String TextField3Key = "textField3"; 45 | public static Field TextField3; 46 | 47 | public const System.String KeywordText = "Keyword"; 48 | public const System.String KeywordFieldKey = "keyField"; 49 | public static Field KeyField; 50 | 51 | public const System.String NoNormsText = "omitNormsText"; 52 | public const System.String NoNormsKey = "omitNorms"; 53 | public static Field NoNormsField; 54 | 55 | public const System.String NoTfText = "analyzed with no tf and positions"; 56 | public const System.String NoTfKey = "omitTermFreqAndPositions"; 57 | public static Field NoTfField; 58 | 59 | public const System.String UnindexedFieldText = "unindexed field text"; 60 | public const System.String UnindexedFieldKey = "unIndField"; 61 | public static Field UnIndField; 62 | 63 | 64 | public const System.String Unstored1FieldText = "unstored field text"; 65 | public const System.String UnstoredField1Key = "unStoredField1"; 66 | public static Field UnStoredField1; 67 | 68 | public const System.String Unstored2FieldText = "unstored field text"; 69 | public const System.String UnstoredField2Key = "unStoredField2"; 70 | public static Field UnStoredField2; 71 | 72 | public const System.String LazyFieldBinaryKey = "lazyFieldBinary"; 73 | public static byte[] LazyFieldBinaryBytes; 74 | public static Field LazyFieldBinary; 75 | 76 | public const System.String LazyFieldKey = "lazyField"; 77 | public const System.String LazyFieldText = "These are some field bytes"; 78 | public static Field LazyField; 79 | 80 | public const System.String LargeLazyFieldKey = "largeLazyField"; 81 | public static System.String LargeLazyFieldText; 82 | public static Field LargeLazyField; 83 | 84 | //From Issue 509 85 | public const System.String FieldUtf1Text = "field one \u4e00text"; 86 | public const System.String TextFieldUtf1Key = "textField1Utf8"; 87 | public static Field TextUtfField1; 88 | 89 | public const System.String FieldUtf2Text = "field field field \u4e00two text"; 90 | //Fields will be lexicographically sorted. So, the order is: field, text, two 91 | public static readonly int[] FieldUtf2Freqs = new int[]{3, 1, 1}; 92 | public const System.String TextFieldUtf2Key = "textField2Utf8"; 93 | public static Field TextUtfField2; 94 | 95 | 96 | 97 | 98 | public static System.Collections.IDictionary NameValues = null; 99 | 100 | // ordered list of all the fields... 101 | // could use LinkedHashMap for this purpose if Java1.4 is OK 102 | public static Field[] Fields = null; 103 | 104 | // Map 105 | public static System.Collections.IDictionary All = new System.Collections.Hashtable(); 106 | public static System.Collections.IDictionary Indexed = new System.Collections.Hashtable(); 107 | public static System.Collections.IDictionary Stored = new System.Collections.Hashtable(); 108 | public static System.Collections.IDictionary Unstored = new System.Collections.Hashtable(); 109 | public static System.Collections.IDictionary Unindexed = new System.Collections.Hashtable(); 110 | public static System.Collections.IDictionary Termvector = new System.Collections.Hashtable(); 111 | public static System.Collections.IDictionary Notermvector = new System.Collections.Hashtable(); 112 | public static System.Collections.IDictionary Lazy = new System.Collections.Hashtable(); 113 | public static System.Collections.IDictionary NoNorms = new System.Collections.Hashtable(); 114 | public static System.Collections.IDictionary NoTf = new System.Collections.Hashtable(); 115 | 116 | 117 | private static void Add(System.Collections.IDictionary map, IFieldable field) 118 | { 119 | map[field.Name] = field; 120 | } 121 | 122 | /// Adds the fields above to a document 123 | /// The document to write 124 | /// 125 | public static void SetupDoc(Document doc) 126 | { 127 | for (int i = 0; i < Fields.Length; i++) 128 | { 129 | doc.Add(Fields[i]); 130 | } 131 | } 132 | 133 | /// Writes the document to the directory using a segment 134 | /// named "test"; returns the SegmentInfo describing the new 135 | /// segment 136 | /// 137 | /// 138 | /// 139 | /// 140 | /// 141 | /// IOException 142 | public static SegmentInfo WriteDoc(Directory dir, Document doc) 143 | { 144 | return WriteDoc(dir, new WhitespaceAnalyzer(), Similarity.Default, doc); 145 | } 146 | 147 | /// Writes the document to the directory using the analyzer 148 | /// and the similarity score; returns the SegmentInfo 149 | /// describing the new segment 150 | /// 151 | /// 152 | /// 153 | /// 154 | /// 155 | /// 156 | /// 157 | /// 158 | /// 159 | /// IOException 160 | public static SegmentInfo WriteDoc(Directory dir, Analyzer analyzer, Similarity similarity, Document doc) 161 | { 162 | IndexWriter writer = new IndexWriter(dir, analyzer, IndexWriter.MaxFieldLength.LIMITED); 163 | writer.SetSimilarity(similarity); 164 | //writer.setUseCompoundFile(false); 165 | writer.AddDocument(doc); 166 | writer.Commit(); 167 | SegmentInfo info = writer.NewestSegment(); 168 | writer.Close(); 169 | return info; 170 | } 171 | 172 | public static int NumFields(Document doc) 173 | { 174 | return doc.GetFields().Count; 175 | } 176 | static DocHelper() 177 | { 178 | TextField1 = new Field(TextField1Key, Field1Text, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO); 179 | TextField2 = new Field(TextField2Key, Field2Text, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS); 180 | TextField3 = new Field(TextField3Key, Field3Text, Field.Store.YES, Field.Index.ANALYZED); 181 | { 182 | TextField3.OmitNorms = true; 183 | } 184 | KeyField = new Field(KeywordFieldKey, KeywordText, Field.Store.YES, Field.Index.NOT_ANALYZED); 185 | NoNormsField = new Field(NoNormsKey, NoNormsText, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS); 186 | NoTfField = new Field(NoTfKey, NoTfText, Field.Store.YES, Field.Index.ANALYZED); 187 | { 188 | NoTfField.OmitTermFreqAndPositions = true; 189 | } 190 | UnIndField = new Field(UnindexedFieldKey, UnindexedFieldText, Field.Store.YES, Field.Index.NO); 191 | UnStoredField1 = new Field(UnstoredField1Key, Unstored1FieldText, Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.NO); 192 | UnStoredField2 = new Field(UnstoredField2Key, Unstored2FieldText, Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.YES); 193 | LazyField = new Field(LazyFieldKey, LazyFieldText, Field.Store.YES, Field.Index.ANALYZED); 194 | TextUtfField1 = new Field(TextFieldUtf1Key, FieldUtf1Text, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO); 195 | TextUtfField2 = new Field(TextFieldUtf2Key, FieldUtf2Text, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS); 196 | Fields = new Field[] { TextField1, TextField2, TextField3, KeyField, NoNormsField, NoTfField, UnIndField, UnStoredField1, UnStoredField2, TextUtfField1, TextUtfField2, LazyField, LazyFieldBinary, LargeLazyField }; 197 | { 198 | //Initialize the large Lazy Field 199 | System.Text.StringBuilder buffer = new System.Text.StringBuilder(); 200 | for (int i = 0; i < 10000; i++) 201 | { 202 | buffer.Append("Lazily loading lengths of language in lieu of laughing "); 203 | } 204 | 205 | try 206 | { 207 | LazyFieldBinaryBytes = System.Text.Encoding.UTF8.GetBytes("These are some binary field bytes"); 208 | } 209 | catch (System.IO.IOException) 210 | { 211 | } 212 | LazyFieldBinary = new Field(LazyFieldBinaryKey, LazyFieldBinaryBytes, Field.Store.YES); 213 | Fields[Fields.Length - 2] = LazyFieldBinary; 214 | LargeLazyFieldText = buffer.ToString(); 215 | LargeLazyField = new Field(LargeLazyFieldKey, LargeLazyFieldText, Field.Store.YES, Field.Index.ANALYZED); 216 | 217 | Fields[Fields.Length - 1] = LargeLazyField; 218 | for (int i = 0; i < Fields.Length; i++) 219 | { 220 | IFieldable f = Fields[i]; 221 | Add(All, f); 222 | if (f.IsIndexed) 223 | Add(Indexed, f); 224 | else 225 | Add(Unindexed, f); 226 | if (f.IsTermVectorStored) 227 | Add(Termvector, f); 228 | if (f.IsIndexed && !f.IsTermVectorStored) 229 | Add(Notermvector, f); 230 | if (f.IsStored) 231 | Add(Stored, f); 232 | else 233 | Add(Unstored, f); 234 | if (f.OmitNorms) 235 | Add(NoNorms, f); 236 | if (f.OmitTermFreqAndPositions) 237 | Add(NoTf, f); 238 | if (f.IsLazy) 239 | Add(Lazy, f); 240 | } 241 | } 242 | { 243 | NameValues = new System.Collections.Hashtable(); 244 | NameValues[TextField1Key] = Field1Text; 245 | NameValues[TextField2Key] = Field2Text; 246 | NameValues[TextField3Key] = Field3Text; 247 | NameValues[KeywordFieldKey] = KeywordText; 248 | NameValues[NoNormsKey] = NoNormsText; 249 | NameValues[NoTfKey] = NoTfText; 250 | NameValues[UnindexedFieldKey] = UnindexedFieldText; 251 | NameValues[UnstoredField1Key] = Unstored1FieldText; 252 | NameValues[UnstoredField2Key] = Unstored2FieldText; 253 | NameValues[LazyFieldKey] = LazyFieldText; 254 | NameValues[LazyFieldBinaryKey] = LazyFieldBinaryBytes; 255 | NameValues[LargeLazyFieldKey] = LargeLazyFieldText; 256 | NameValues[TextFieldUtf1Key] = FieldUtf1Text; 257 | NameValues[TextFieldUtf2Key] = FieldUtf2Text; 258 | } 259 | } 260 | } 261 | } -------------------------------------------------------------------------------- /LuceneNetSqlDirectory.Tests/Tests_from_Lucene/ReadMe.txt: -------------------------------------------------------------------------------- 1 | These tests are taken over from the Lucene project verify some specific situations. -------------------------------------------------------------------------------- /LuceneNetSqlDirectory.Tests/Tests_from_Lucene/TestCachingSpanFilter.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | using Lucene.Net.Analysis; 19 | using Lucene.Net.Documents; 20 | using Lucene.Net.Index; 21 | using Lucene.Net.Search; 22 | using Lucene.Net.Search.Spans; 23 | using Lucene.Net.Store; 24 | using Microsoft.VisualStudio.TestTools.UnitTesting; 25 | 26 | namespace LuceneNetSqlDirectory.Tests 27 | { 28 | [TestClass] 29 | public class TestCachingSpanFilter : LuceneTestCase 30 | { 31 | [TestMethod] 32 | public void TestEnforceDeletions() 33 | { 34 | var dir = new SqlServerDirectory(Connection, new Options()); 35 | IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED); 36 | IndexReader reader = writer.GetReader(); 37 | IndexSearcher searcher = new IndexSearcher(reader); 38 | 39 | // add a doc, refresh the reader, and check that its there 40 | Document doc = new Document(); 41 | doc.Add(new Field("id", "1", Field.Store.YES, Field.Index.NOT_ANALYZED)); 42 | writer.AddDocument(doc); 43 | 44 | reader = RefreshReader(reader); 45 | searcher = new IndexSearcher(reader); 46 | 47 | TopDocs docs = searcher.Search(new MatchAllDocsQuery(), 1); 48 | Assert.AreEqual(1, docs.TotalHits, "Should find a hit..."); 49 | 50 | SpanFilter startFilter = new SpanQueryFilter(new SpanTermQuery(new Term("id", "1"))); 51 | 52 | // ignore deletions 53 | CachingSpanFilter filter = new CachingSpanFilter(startFilter, CachingWrapperFilter.DeletesMode.IGNORE); 54 | 55 | docs = searcher.Search(new MatchAllDocsQuery(), filter, 1); 56 | Assert.AreEqual(1, docs.TotalHits, "[query + filter] Should find a hit..."); 57 | ConstantScoreQuery constantScore = new ConstantScoreQuery(filter); 58 | docs = searcher.Search(constantScore, 1); 59 | Assert.AreEqual(1, docs.TotalHits, "[just filter] Should find a hit..."); 60 | 61 | // now delete the doc, refresh the reader, and see that it's not there 62 | writer.DeleteDocuments(new Term("id", "1")); 63 | 64 | reader = RefreshReader(reader); 65 | searcher = new IndexSearcher(reader); 66 | 67 | docs = searcher.Search(new MatchAllDocsQuery(), filter, 1); 68 | Assert.AreEqual(0, docs.TotalHits, "[query + filter] Should *not* find a hit..."); 69 | 70 | docs = searcher.Search(constantScore, 1); 71 | Assert.AreEqual(1, docs.TotalHits, "[just filter] Should find a hit..."); 72 | 73 | 74 | // force cache to regenerate: 75 | filter = new CachingSpanFilter(startFilter, CachingWrapperFilter.DeletesMode.RECACHE); 76 | 77 | writer.AddDocument(doc); 78 | reader = RefreshReader(reader); 79 | searcher = new IndexSearcher(reader); 80 | 81 | docs = searcher.Search(new MatchAllDocsQuery(), filter, 1); 82 | Assert.AreEqual(1, docs.TotalHits, "[query + filter] Should find a hit..."); 83 | 84 | constantScore = new ConstantScoreQuery(filter); 85 | docs = searcher.Search(constantScore, 1); 86 | Assert.AreEqual(1, docs.TotalHits, "[just filter] Should find a hit..."); 87 | 88 | // make sure we get a cache hit when we reopen readers 89 | // that had no new deletions 90 | IndexReader newReader = RefreshReader(reader); 91 | Assert.IsTrue(reader != newReader); 92 | reader = newReader; 93 | searcher = new IndexSearcher(reader); 94 | int missCount = filter.missCount; 95 | docs = searcher.Search(constantScore, 1); 96 | Assert.AreEqual(1, docs.TotalHits, "[just filter] Should find a hit..."); 97 | Assert.AreEqual(missCount, filter.missCount); 98 | 99 | // now delete the doc, refresh the reader, and see that it's not there 100 | writer.DeleteDocuments(new Term("id", "1")); 101 | 102 | reader = RefreshReader(reader); 103 | searcher = new IndexSearcher(reader); 104 | 105 | docs = searcher.Search(new MatchAllDocsQuery(), filter, 1); 106 | Assert.AreEqual(0, docs.TotalHits, "[query + filter] Should *not* find a hit..."); 107 | 108 | docs = searcher.Search(constantScore, 1); 109 | Assert.AreEqual(0, docs.TotalHits, "[just filter] Should *not* find a hit..."); 110 | } 111 | 112 | private static IndexReader RefreshReader(IndexReader reader) 113 | { 114 | IndexReader oldReader = reader; 115 | reader = reader.Reopen(); 116 | if (reader != oldReader) 117 | { 118 | oldReader.Dispose(); 119 | } 120 | return reader; 121 | } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /LuceneNetSqlDirectory.Tests/Tests_from_Lucene/TestDemo.cs: -------------------------------------------------------------------------------- 1 | using Lucene.Net.Analysis; 2 | using Lucene.Net.Analysis.Standard; 3 | using Lucene.Net.Documents; 4 | using Lucene.Net.Index; 5 | using Lucene.Net.QueryParsers; 6 | using Lucene.Net.Search; 7 | using Lucene.Net.Util; 8 | using Microsoft.VisualStudio.TestTools.UnitTesting; 9 | 10 | namespace LuceneNetSqlDirectory.Tests 11 | { 12 | [TestClass] 13 | public class TestDemo : LuceneTestCase 14 | { 15 | [TestMethod] 16 | public virtual void TestDemo_Renamed() 17 | { 18 | Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_30); 19 | // Store the index in memory: 20 | using (var directory = new SqlServerDirectory(Connection, new Options())) 21 | { 22 | // To store an index on disk, use this instead: 23 | //Directory directory = FSDirectory.open("/tmp/testindex"); 24 | using (var iwriter = new IndexWriter(directory, analyzer, true, new IndexWriter.MaxFieldLength(25000))) 25 | { 26 | var doc = new Document(); 27 | var text = "This is the text to be indexed."; 28 | doc.Add(new Field("fieldname", text, Field.Store.YES, Field.Index.ANALYZED)); 29 | iwriter.AddDocument(doc); 30 | } 31 | 32 | // Now search the index: 33 | using (IndexSearcher isearcher = new IndexSearcher(directory, true)) 34 | { 35 | // read-only=true 36 | // Parse a simple query that searches for "text": 37 | QueryParser parser = new QueryParser(Version.LUCENE_30, "fieldname", analyzer); 38 | Query query = parser.Parse("text"); 39 | ScoreDoc[] hits = isearcher.Search(query, null, 1000).ScoreDocs; 40 | Assert.AreEqual(1, hits.Length); 41 | // Iterate through the results: 42 | for (int i = 0; i < hits.Length; i++) 43 | { 44 | Document hitDoc = isearcher.Doc(hits[i].Doc); 45 | Assert.AreEqual(hitDoc.Get("fieldname"), "This is the text to be indexed."); 46 | } 47 | } 48 | } 49 | } 50 | } 51 | } -------------------------------------------------------------------------------- /LuceneNetSqlDirectory.Tests/Tests_from_Lucene/TestDirectoryReader.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | 19 | using System; 20 | using Lucene.Net.Index; 21 | using Microsoft.VisualStudio.TestTools.UnitTesting; 22 | using StandardAnalyzer = Lucene.Net.Analysis.Standard.StandardAnalyzer; 23 | using Document = Lucene.Net.Documents.Document; 24 | using Field = Lucene.Net.Documents.Field; 25 | using Directory = Lucene.Net.Store.Directory; 26 | 27 | namespace LuceneNetSqlDirectory.Tests 28 | { 29 | 30 | [TestClass] 31 | public class TestDirectoryReader : LuceneTestCase 32 | { 33 | protected internal SqlServerDirectory Dir; 34 | private Document _doc1; 35 | private Document _doc2; 36 | protected internal SegmentReader[] Readers = new SegmentReader[2]; 37 | protected internal SegmentInfos Sis; 38 | 39 | 40 | [TestInitialize] 41 | public override void TestInitialize() 42 | { 43 | base.TestInitialize(); 44 | Dir = new SqlServerDirectory(Connection, new Options()); 45 | _doc1 = new Document(); 46 | _doc2 = new Document(); 47 | DocHelper.SetupDoc(_doc1); 48 | DocHelper.SetupDoc(_doc2); 49 | DocHelper.WriteDoc(Dir, _doc1); 50 | DocHelper.WriteDoc(Dir, _doc2); 51 | Sis = new SegmentInfos(); 52 | Sis.Read(Dir); 53 | } 54 | 55 | protected internal virtual IndexReader OpenReader() 56 | { 57 | IndexReader reader; 58 | reader = IndexReader.Open(Dir, false); 59 | Assert.IsTrue(reader is DirectoryReader); 60 | 61 | Assert.IsTrue(Dir != null); 62 | Assert.IsTrue(Sis != null); 63 | Assert.IsTrue(reader != null); 64 | 65 | return reader; 66 | } 67 | 68 | [TestMethod] 69 | public virtual void Test() 70 | { 71 | DoTestDocument(); 72 | DoTestUndeleteAll(); 73 | } 74 | 75 | public virtual void DoTestDocument() 76 | { 77 | Sis.Read(Dir); 78 | IndexReader reader = OpenReader(); 79 | Assert.IsTrue(reader != null); 80 | Document newDoc1 = reader.Document(0); 81 | Assert.IsTrue(newDoc1 != null); 82 | Assert.IsTrue(DocHelper.NumFields(newDoc1) == DocHelper.NumFields(_doc1) - DocHelper.Unstored.Count); 83 | Document newDoc2 = reader.Document(1); 84 | Assert.IsTrue(newDoc2 != null); 85 | Assert.IsTrue(DocHelper.NumFields(newDoc2) == DocHelper.NumFields(_doc2) - DocHelper.Unstored.Count); 86 | ITermFreqVector vector = reader.GetTermFreqVector(0, DocHelper.TextField2Key); 87 | Assert.IsTrue(vector != null); 88 | TestSegmentReader.CheckNorms(reader); 89 | } 90 | 91 | public virtual void DoTestUndeleteAll() 92 | { 93 | Sis.Read(Dir); 94 | IndexReader reader = OpenReader(); 95 | Assert.IsTrue(reader != null); 96 | Assert.AreEqual(2, reader.NumDocs()); 97 | reader.DeleteDocument(0); 98 | Assert.AreEqual(1, reader.NumDocs()); 99 | reader.UndeleteAll(); 100 | Assert.AreEqual(2, reader.NumDocs()); 101 | 102 | // Ensure undeleteAll survives commit/close/reopen: 103 | reader.Commit(); 104 | reader.Close(); 105 | 106 | if (reader is MultiReader) 107 | // MultiReader does not "own" the directory so it does 108 | // not write the changes to sis on commit: 109 | Sis.Commit(Dir); 110 | 111 | Sis.Read(Dir); 112 | reader = OpenReader(); 113 | Assert.AreEqual(2, reader.NumDocs()); 114 | 115 | reader.DeleteDocument(0); 116 | Assert.AreEqual(1, reader.NumDocs()); 117 | reader.Commit(); 118 | reader.Close(); 119 | if (reader is MultiReader) 120 | // MultiReader does not "own" the directory so it does 121 | // not write the changes to sis on commit: 122 | Sis.Commit(Dir); 123 | Sis.Read(Dir); 124 | reader = OpenReader(); 125 | Assert.AreEqual(1, reader.NumDocs()); 126 | } 127 | 128 | 129 | public virtual void _testTermVectors() 130 | { 131 | MultiReader reader = new MultiReader(Readers); 132 | Assert.IsTrue(reader != null); 133 | } 134 | 135 | 136 | 137 | [TestMethod] 138 | public virtual void TestMultiTermDocs() 139 | { 140 | SqlServerDirectory.ProvisionDatabase(Connection, "test1", true); 141 | SqlServerDirectory.ProvisionDatabase(Connection, "test2", true); 142 | SqlServerDirectory.ProvisionDatabase(Connection, "test3", true); 143 | 144 | var ramDir1 = new SqlServerDirectory(Connection, new Options() { SchemaName = "test1" }); 145 | AddDoc(ramDir1, "test foo", true); 146 | var ramDir2 = new SqlServerDirectory(Connection, new Options() { SchemaName = "test2" }); 147 | AddDoc(ramDir2, "test blah", true); 148 | var ramDir3 = new SqlServerDirectory(Connection, new Options() { SchemaName = "test3" }); 149 | AddDoc(ramDir3, "test wow", true); 150 | 151 | IndexReader[] readers1 = new[] { IndexReader.Open(ramDir1, false), IndexReader.Open(ramDir3, false) }; 152 | IndexReader[] readers2 = new[] { IndexReader.Open(ramDir1, false), IndexReader.Open(ramDir2, false), IndexReader.Open(ramDir3, false) }; 153 | MultiReader mr2 = new MultiReader(readers1); 154 | MultiReader mr3 = new MultiReader(readers2); 155 | 156 | // test mixing up TermDocs and TermEnums from different readers. 157 | TermDocs td2 = mr2.TermDocs(); 158 | TermEnum te3 = mr3.Terms(new Term("body", "wow")); 159 | td2.Seek(te3); 160 | int ret = 0; 161 | 162 | // This should blow up if we forget to check that the TermEnum is from the same 163 | // reader as the TermDocs. 164 | while (td2.Next()) 165 | ret += td2.Doc; 166 | td2.Close(); 167 | te3.Close(); 168 | 169 | // really a dummy assert to ensure that we got some docs and to ensure that 170 | // nothing is optimized out. 171 | Assert.IsTrue(ret > 0); 172 | } 173 | 174 | [TestMethod] 175 | public virtual void TestAllTermDocs() 176 | { 177 | IndexReader reader = OpenReader(); 178 | int numDocs = 2; 179 | TermDocs td = reader.TermDocs(null); 180 | for (int i = 0; i < numDocs; i++) 181 | { 182 | Assert.IsTrue(td.Next()); 183 | Assert.AreEqual(i, td.Doc); 184 | Assert.AreEqual(1, td.Freq); 185 | } 186 | td.Close(); 187 | reader.Close(); 188 | } 189 | 190 | private void AddDoc(SqlServerDirectory ramDir1, System.String s, bool create) 191 | { 192 | IndexWriter iw = new IndexWriter(ramDir1, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_CURRENT), create, IndexWriter.MaxFieldLength.LIMITED); 193 | Document doc = new Document(); 194 | doc.Add(new Field("body", s, Field.Store.YES, Field.Index.ANALYZED)); 195 | iw.AddDocument(doc); 196 | iw.Close(); 197 | } 198 | } 199 | } -------------------------------------------------------------------------------- /LuceneNetSqlDirectory.Tests/Tests_from_Lucene/TestSegmentReader.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | using Lucene.Net.Documents; 19 | using Lucene.Net.Index; 20 | using Microsoft.VisualStudio.TestTools.UnitTesting; 21 | using Document = Lucene.Net.Documents.Document; 22 | using DefaultSimilarity = Lucene.Net.Search.DefaultSimilarity; 23 | 24 | namespace LuceneNetSqlDirectory.Tests 25 | { 26 | 27 | [TestClass] 28 | public class TestSegmentReader:LuceneTestCase 29 | { 30 | private SqlServerDirectory _dir; 31 | private Document _testDoc = new Document(); 32 | private SegmentReader _reader = null; 33 | 34 | 35 | //TODO: Setup the reader w/ multiple documents 36 | [TestInitialize] 37 | public override void TestInitialize() 38 | { 39 | base.TestInitialize(); 40 | _testDoc = new Document(); 41 | 42 | _dir = new SqlServerDirectory(Connection,new Options()); 43 | DocHelper.SetupDoc(_testDoc); 44 | SegmentInfo info = DocHelper.WriteDoc(_dir, _testDoc); 45 | _reader = SegmentReader.Get(true, info, 1); 46 | } 47 | 48 | [TestMethod] 49 | public virtual void Test() 50 | { 51 | Assert.IsTrue(_dir != null); 52 | Assert.IsTrue(_reader != null); 53 | Assert.IsTrue(DocHelper.NameValues.Count > 0); 54 | Assert.IsTrue(DocHelper.NumFields(_testDoc) == DocHelper.All.Count); 55 | } 56 | 57 | [TestMethod] 58 | public virtual void TestDocument() 59 | { 60 | Assert.IsTrue(_reader.NumDocs() == 1); 61 | Assert.IsTrue(_reader.MaxDoc >= 1); 62 | Document result = _reader.Document(0); 63 | Assert.IsTrue(result != null); 64 | //There are 2 unstored fields on the document that are not preserved across writing 65 | Assert.IsTrue(DocHelper.NumFields(result) == DocHelper.NumFields(_testDoc) - DocHelper.Unstored.Count); 66 | 67 | var fields = result.GetFields(); 68 | foreach (var field in fields) 69 | { 70 | Assert.IsTrue(field != null); 71 | Assert.IsTrue(DocHelper.NameValues.Contains(field.Name)); 72 | } 73 | } 74 | 75 | [TestMethod] 76 | public virtual void TestDelete() 77 | { 78 | Document docToDelete = new Document(); 79 | DocHelper.SetupDoc(docToDelete); 80 | SegmentInfo info = DocHelper.WriteDoc(_dir, docToDelete); 81 | SegmentReader deleteReader = SegmentReader.Get(false, info, 1); 82 | Assert.IsTrue(deleteReader != null); 83 | Assert.IsTrue(deleteReader.NumDocs() == 1); 84 | deleteReader.DeleteDocument(0); 85 | Assert.IsTrue(deleteReader.IsDeleted(0) == true); 86 | Assert.IsTrue(deleteReader.HasDeletions == true); 87 | Assert.IsTrue(deleteReader.NumDocs() == 0); 88 | } 89 | 90 | [TestMethod] 91 | public virtual void TestGetFieldNameVariations() 92 | { 93 | System.Collections.Generic.ICollection result = _reader.GetFieldNames(IndexReader.FieldOption.ALL); 94 | Assert.IsTrue(result != null); 95 | Assert.IsTrue(result.Count == DocHelper.All.Count); 96 | for (System.Collections.IEnumerator iter = result.GetEnumerator(); iter.MoveNext(); ) 97 | { 98 | System.String s = (System.String) iter.Current; 99 | //System.out.println("Name: " + s); 100 | Assert.IsTrue(DocHelper.NameValues.Contains(s) == true || s.Equals("")); 101 | } 102 | result = _reader.GetFieldNames(IndexReader.FieldOption.INDEXED); 103 | Assert.IsTrue(result != null); 104 | Assert.IsTrue(result.Count == DocHelper.Indexed.Count); 105 | for (System.Collections.IEnumerator iter = result.GetEnumerator(); iter.MoveNext(); ) 106 | { 107 | System.String s = (System.String) iter.Current; 108 | Assert.IsTrue(DocHelper.Indexed.Contains(s) == true || s.Equals("")); 109 | } 110 | 111 | result = _reader.GetFieldNames(IndexReader.FieldOption.UNINDEXED); 112 | Assert.IsTrue(result != null); 113 | Assert.IsTrue(result.Count == DocHelper.Unindexed.Count); 114 | //Get all indexed fields that are storing term vectors 115 | result = _reader.GetFieldNames(IndexReader.FieldOption.INDEXED_WITH_TERMVECTOR); 116 | Assert.IsTrue(result != null); 117 | Assert.IsTrue(result.Count == DocHelper.Termvector.Count); 118 | 119 | result = _reader.GetFieldNames(IndexReader.FieldOption.INDEXED_NO_TERMVECTOR); 120 | Assert.IsTrue(result != null); 121 | Assert.IsTrue(result.Count == DocHelper.Notermvector.Count); 122 | } 123 | 124 | [TestMethod] 125 | public virtual void TestTerms() 126 | { 127 | TermEnum terms = _reader.Terms(); 128 | Assert.IsTrue(terms != null); 129 | while (terms.Next() == true) 130 | { 131 | Term term = terms.Term; 132 | Assert.IsTrue(term != null); 133 | //System.out.println("Term: " + term); 134 | System.String fieldValue = (System.String) DocHelper.NameValues[term.Field]; 135 | Assert.IsTrue(fieldValue.IndexOf(term.Text) != - 1); 136 | } 137 | 138 | TermDocs termDocs = _reader.TermDocs(); 139 | Assert.IsTrue(termDocs != null); 140 | termDocs.Seek(new Term(DocHelper.TextField1Key, "field")); 141 | Assert.IsTrue(termDocs.Next() == true); 142 | 143 | termDocs.Seek(new Term(DocHelper.NoNormsKey, DocHelper.NoNormsText)); 144 | Assert.IsTrue(termDocs.Next() == true); 145 | 146 | 147 | TermPositions positions = _reader.TermPositions(); 148 | positions.Seek(new Term(DocHelper.TextField1Key, "field")); 149 | Assert.IsTrue(positions != null); 150 | Assert.IsTrue(positions.Doc == 0); 151 | Assert.IsTrue(positions.NextPosition() >= 0); 152 | } 153 | 154 | public static void CheckNorms(IndexReader reader) 155 | { 156 | // test omit norms 157 | for (int i = 0; i < DocHelper.Fields.Length; i++) 158 | { 159 | IFieldable f = DocHelper.Fields[i]; 160 | if (f.IsIndexed) 161 | { 162 | Assert.AreEqual(reader.HasNorms(f.Name), !f.OmitNorms); 163 | Assert.AreEqual(reader.HasNorms(f.Name), !DocHelper.NoNorms.Contains(f.Name)); 164 | if (!reader.HasNorms(f.Name)) 165 | { 166 | // test for fake norms of 1.0 or null depending on the flag 167 | byte[] norms = reader.Norms(f.Name); 168 | byte norm1 = DefaultSimilarity.EncodeNorm(1.0f); 169 | Assert.IsNull(norms); 170 | norms = new byte[reader.MaxDoc]; 171 | reader.Norms(f.Name, norms, 0); 172 | for (int j = 0; j < reader.MaxDoc; j++) 173 | { 174 | Assert.AreEqual(norms[j], norm1); 175 | } 176 | } 177 | } 178 | } 179 | } 180 | 181 | [TestMethod] 182 | public virtual void TestTermVectors() 183 | { 184 | ITermFreqVector result = _reader.GetTermFreqVector(0, DocHelper.TextField2Key); 185 | Assert.IsTrue(result != null); 186 | System.String[] terms = result.GetTerms(); 187 | int[] freqs = result.GetTermFrequencies(); 188 | Assert.IsTrue(terms != null && terms.Length == 3 && freqs != null && freqs.Length == 3); 189 | for (int i = 0; i < terms.Length; i++) 190 | { 191 | System.String term = terms[i]; 192 | int freq = freqs[i]; 193 | Assert.IsTrue(DocHelper.Field2Text.IndexOf(term) != - 1); 194 | Assert.IsTrue(freq > 0); 195 | } 196 | 197 | ITermFreqVector[] results = _reader.GetTermFreqVectors(0); 198 | Assert.IsTrue(results != null); 199 | Assert.IsTrue(results.Length == 3, "We do not have 3 term freq vectors, we have: " + results.Length); 200 | } 201 | } 202 | } -------------------------------------------------------------------------------- /LuceneNetSqlDirectory.Tests/Tests_from_Lucene/TestTermScorer.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | 19 | using Lucene.Net.Search; 20 | using Microsoft.VisualStudio.TestTools.UnitTesting; 21 | using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer; 22 | using Document = Lucene.Net.Documents.Document; 23 | using Field = Lucene.Net.Documents.Field; 24 | using IndexReader = Lucene.Net.Index.IndexReader; 25 | using IndexWriter = Lucene.Net.Index.IndexWriter; 26 | using Term = Lucene.Net.Index.Term; 27 | 28 | namespace LuceneNetSqlDirectory.Tests 29 | { 30 | 31 | [TestClass] 32 | public class TestTermScorer : LuceneTestCase 33 | { 34 | private class AnonymousClassCollector : Collector 35 | { 36 | public AnonymousClassCollector(System.Collections.IList docs, TestTermScorer enclosingInstance) 37 | { 38 | InitBlock(docs, enclosingInstance); 39 | } 40 | private void InitBlock(System.Collections.IList docs, TestTermScorer enclosingInstance) 41 | { 42 | this._docs = docs; 43 | this._enclosingInstance = enclosingInstance; 44 | } 45 | private System.Collections.IList _docs; 46 | private TestTermScorer _enclosingInstance; 47 | public TestTermScorer EnclosingInstance 48 | { 49 | get 50 | { 51 | return _enclosingInstance; 52 | } 53 | 54 | } 55 | private int _baseRenamed = 0; 56 | private Scorer _scorer; 57 | public override void SetScorer(Scorer scorer) 58 | { 59 | this._scorer = scorer; 60 | } 61 | 62 | public override void Collect(int doc) 63 | { 64 | float score = _scorer.Score(); 65 | doc = doc + _baseRenamed; 66 | _docs.Add(new TestHit(_enclosingInstance, doc, score)); 67 | Assert.IsTrue(score > 0, "score " + score + " is not greater than 0"); 68 | Assert.IsTrue(doc == 0 || doc == 5, "Doc: " + doc + " does not equal 0 or doc does not equal 5"); 69 | } 70 | public override void SetNextReader(IndexReader reader, int docBase) 71 | { 72 | _baseRenamed = docBase; 73 | } 74 | 75 | public override bool AcceptsDocsOutOfOrder 76 | { 77 | get { return true; } 78 | } 79 | } 80 | protected internal SqlServerDirectory Directory; 81 | private const System.String FIELD = "field"; 82 | 83 | protected internal System.String[] Values = new System.String[] { "all", "dogs dogs", "like", "playing", "fetch", "all" }; 84 | protected internal IndexSearcher IndexSearcher; 85 | protected internal IndexReader indexReader; 86 | 87 | 88 | [TestInitialize] 89 | public override void TestInitialize() 90 | { 91 | base.TestInitialize(); 92 | Directory = new SqlServerDirectory(Connection, new Options()); 93 | 94 | 95 | IndexWriter writer = new IndexWriter(Directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); 96 | for (int i = 0; i < Values.Length; i++) 97 | { 98 | Document doc = new Document(); 99 | doc.Add(new Field(FIELD, Values[i], Field.Store.YES, Field.Index.ANALYZED)); 100 | writer.AddDocument(doc); 101 | } 102 | writer.Close(); 103 | IndexSearcher = new IndexSearcher(Directory, false); 104 | indexReader = IndexSearcher.IndexReader; 105 | } 106 | 107 | [TestMethod] 108 | public virtual void Test() 109 | { 110 | 111 | Term allTerm = new Term(FIELD, "all"); 112 | TermQuery termQuery = new TermQuery(allTerm); 113 | 114 | Weight weight = termQuery.Weight(IndexSearcher); 115 | 116 | TermScorer ts = new TermScorer(weight, indexReader.TermDocs(allTerm), IndexSearcher.Similarity, indexReader.Norms(FIELD)); 117 | //we have 2 documents with the term all in them, one document for all the other values 118 | System.Collections.IList docs = new System.Collections.ArrayList(); 119 | //must call next first 120 | 121 | 122 | ts.Score(new AnonymousClassCollector(docs, this)); 123 | Assert.IsTrue(docs.Count == 2, "docs Size: " + docs.Count + " is not: " + 2); 124 | TestHit doc0 = (TestHit)docs[0]; 125 | TestHit doc5 = (TestHit)docs[1]; 126 | //The scores should be the same 127 | Assert.IsTrue(doc0.Score == doc5.Score, doc0.Score + " does not equal: " + doc5.Score); 128 | /* 129 | Score should be (based on Default Sim.: 130 | All floats are approximate 131 | tf = 1 132 | numDocs = 6 133 | docFreq(all) = 2 134 | idf = ln(6/3) + 1 = 1.693147 135 | idf ^ 2 = 2.8667 136 | boost = 1 137 | lengthNorm = 1 //there is 1 term in every document 138 | coord = 1 139 | sumOfSquaredWeights = (idf * boost) ^ 2 = 1.693147 ^ 2 = 2.8667 140 | queryNorm = 1 / (sumOfSquaredWeights)^0.5 = 1 /(1.693147) = 0.590 141 | 142 | score = 1 * 2.8667 * 1 * 1 * 0.590 = 1.69 143 | 144 | */ 145 | Assert.IsTrue(doc0.Score == 1.6931472f, doc0.Score + " does not equal: " + 1.6931472f); 146 | } 147 | 148 | [TestMethod] 149 | public virtual void TestNext() 150 | { 151 | 152 | Term allTerm = new Term(FIELD, "all"); 153 | TermQuery termQuery = new TermQuery(allTerm); 154 | 155 | Weight weight = termQuery.Weight(IndexSearcher); 156 | 157 | TermScorer ts = new TermScorer(weight, indexReader.TermDocs(allTerm), IndexSearcher.Similarity, indexReader.Norms(FIELD)); 158 | Assert.IsTrue(ts.NextDoc() != DocIdSetIterator.NO_MORE_DOCS, "next did not return a doc"); 159 | Assert.IsTrue(ts.Score() == 1.6931472f, "score is not correct"); 160 | Assert.IsTrue(ts.NextDoc() != DocIdSetIterator.NO_MORE_DOCS, "next did not return a doc"); 161 | Assert.IsTrue(ts.Score() == 1.6931472f, "score is not correct"); 162 | Assert.IsTrue(ts.NextDoc() == DocIdSetIterator.NO_MORE_DOCS, "next returned a doc and it should not have"); 163 | } 164 | 165 | [TestMethod] 166 | public virtual void TestSkipTo() 167 | { 168 | 169 | Term allTerm = new Term(FIELD, "all"); 170 | TermQuery termQuery = new TermQuery(allTerm); 171 | 172 | Weight weight = termQuery.Weight(IndexSearcher); 173 | 174 | TermScorer ts = new TermScorer(weight, indexReader.TermDocs(allTerm), IndexSearcher.Similarity, indexReader.Norms(FIELD)); 175 | Assert.IsTrue(ts.Advance(3) != DocIdSetIterator.NO_MORE_DOCS, "Didn't skip"); 176 | //The next doc should be doc 5 177 | Assert.IsTrue(ts.DocID() == 5, "doc should be number 5"); 178 | } 179 | 180 | private class TestHit 181 | { 182 | private void InitBlock(TestTermScorer enclosingInstance) 183 | { 184 | this._enclosingInstance = enclosingInstance; 185 | } 186 | private TestTermScorer _enclosingInstance; 187 | public TestTermScorer EnclosingInstance 188 | { 189 | get 190 | { 191 | return _enclosingInstance; 192 | } 193 | 194 | } 195 | public int Doc; 196 | public float Score; 197 | 198 | public TestHit(TestTermScorer enclosingInstance, int doc, float score) 199 | { 200 | InitBlock(enclosingInstance); 201 | this.Doc = doc; 202 | this.Score = score; 203 | } 204 | 205 | public override System.String ToString() 206 | { 207 | return "TestHit{" + "doc=" + Doc + ", score=" + Score + "}"; 208 | } 209 | } 210 | } 211 | } -------------------------------------------------------------------------------- /LuceneNetSqlDirectory.Tests/Tests_from_Lucene/TestWildcard.cs: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | 19 | using System; 20 | using Lucene.Net.Search; 21 | using Microsoft.VisualStudio.TestTools.UnitTesting; 22 | using SimpleAnalyzer = Lucene.Net.Analysis.SimpleAnalyzer; 23 | using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer; 24 | using Document = Lucene.Net.Documents.Document; 25 | using Field = Lucene.Net.Documents.Field; 26 | using IndexWriter = Lucene.Net.Index.IndexWriter; 27 | using Term = Lucene.Net.Index.Term; 28 | using QueryParser = Lucene.Net.QueryParsers.QueryParser; 29 | 30 | namespace LuceneNetSqlDirectory.Tests 31 | { 32 | /// TestWildcard tests the '*' and '?' wildcard characters. 33 | [TestClass] 34 | public class TestWildcard : LuceneTestCase 35 | { 36 | [TestMethod] 37 | public virtual void TestEquals() 38 | { 39 | WildcardQuery wq1 = new WildcardQuery(new Term("field", "b*a")); 40 | WildcardQuery wq2 = new WildcardQuery(new Term("field", "b*a")); 41 | WildcardQuery wq3 = new WildcardQuery(new Term("field", "b*a")); 42 | 43 | // reflexive? 44 | Assert.AreEqual(wq1, wq2); 45 | Assert.AreEqual(wq2, wq1); 46 | 47 | // transitive? 48 | Assert.AreEqual(wq2, wq3); 49 | Assert.AreEqual(wq1, wq3); 50 | 51 | Assert.IsFalse(wq1.Equals(null)); 52 | 53 | FuzzyQuery fq = new FuzzyQuery(new Term("field", "b*a")); 54 | Assert.IsFalse(wq1.Equals(fq)); 55 | Assert.IsFalse(fq.Equals(wq1)); 56 | } 57 | 58 | /// Tests if a WildcardQuery that has no wildcard in the term is rewritten to a single 59 | /// TermQuery. The boost should be prserved, and the rewrite should return 60 | /// a ConstantScoreQuery if the WildcardQuery had a ConstantScore rewriteMethod. 61 | /// 62 | [TestMethod] 63 | public virtual void TestTermWithoutWildcard() 64 | { 65 | var indexStore = GetIndexStore("field", new System.String[] { "nowildcard", "nowildcardx" }); 66 | IndexSearcher searcher = new IndexSearcher(indexStore, true); 67 | 68 | MultiTermQuery wq = new WildcardQuery(new Term("field", "nowildcard")); 69 | AssertMatches(searcher, wq, 1); 70 | 71 | wq.RewriteMethod = MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE; 72 | wq.Boost = 0.1f; 73 | Query q = searcher.Rewrite(wq); 74 | Assert.IsTrue(q is TermQuery); 75 | Assert.AreEqual(q.Boost, wq.Boost); 76 | 77 | wq.RewriteMethod = MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE; 78 | wq.Boost = 0.2f; 79 | q = searcher.Rewrite(wq); 80 | Assert.IsTrue(q is ConstantScoreQuery); 81 | Assert.AreEqual(q.Boost, wq.Boost); 82 | 83 | wq.RewriteMethod = MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT; 84 | wq.Boost = 0.3F; 85 | q = searcher.Rewrite(wq); 86 | Assert.IsTrue(q is ConstantScoreQuery); 87 | Assert.AreEqual(q.Boost, wq.Boost); 88 | 89 | wq.RewriteMethod = MultiTermQuery.CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE; 90 | wq.Boost = 0.4F; 91 | q = searcher.Rewrite(wq); 92 | Assert.IsTrue(q is ConstantScoreQuery); 93 | Assert.AreEqual(q.Boost, wq.Boost); 94 | } 95 | 96 | /// 97 | /// Tests if a WildcardQuery with an empty term is rewritten to an empty BooleanQuery 98 | /// 99 | [TestMethod] 100 | public void TestEmptyTerm() 101 | { 102 | var indexStore = GetIndexStore("field", new String[] { "nowildcard", "nowildcardx" }); 103 | IndexSearcher searcher = new IndexSearcher(indexStore, true); 104 | 105 | MultiTermQuery wq = new WildcardQuery(new Term("field", "")); 106 | wq.RewriteMethod = MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE; 107 | AssertMatches(searcher, wq, 0); 108 | BooleanQuery expected = new BooleanQuery(true); 109 | Assert.AreEqual(searcher.Rewrite(expected), searcher.Rewrite(wq)); 110 | } 111 | 112 | /// 113 | /// Tests if a WildcardQuery that has only a trailing * in the term is 114 | /// rewritten to a single PrefixQuery. The boost and rewriteMethod should be 115 | /// preserved. 116 | /// 117 | [TestMethod] 118 | public void TestPrefixTerm() 119 | { 120 | var indexStore = GetIndexStore("field", new String[] { "prefix", "prefixx" }); 121 | IndexSearcher searcher = new IndexSearcher(indexStore, true); 122 | 123 | MultiTermQuery wq = new WildcardQuery(new Term("field", "prefix*")); 124 | AssertMatches(searcher, wq, 2); 125 | 126 | MultiTermQuery expected = new PrefixQuery(new Term("field", "prefix")); 127 | wq.RewriteMethod = MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE; 128 | wq.Boost = 0.1F; 129 | expected.RewriteMethod = wq.RewriteMethod; 130 | expected.Boost = wq.Boost; 131 | Assert.AreEqual(searcher.Rewrite(expected), searcher.Rewrite(wq)); 132 | 133 | wq.RewriteMethod = MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE; 134 | wq.Boost = 0.2F; 135 | expected.RewriteMethod = wq.RewriteMethod; 136 | expected.Boost = wq.Boost; 137 | Assert.AreEqual(searcher.Rewrite(expected), searcher.Rewrite(wq)); 138 | 139 | wq.RewriteMethod = MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT; 140 | wq.Boost = 0.3F; 141 | expected.RewriteMethod = wq.RewriteMethod; 142 | expected.Boost = wq.Boost; 143 | Assert.AreEqual(searcher.Rewrite(expected), searcher.Rewrite(wq)); 144 | 145 | wq.RewriteMethod = MultiTermQuery.CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE; 146 | wq.Boost = 0.4F; 147 | expected.RewriteMethod = wq.RewriteMethod; 148 | expected.Boost = wq.Boost; 149 | Assert.AreEqual(searcher.Rewrite(expected), searcher.Rewrite(wq)); 150 | } 151 | 152 | /// Tests Wildcard queries with an asterisk. 153 | [TestMethod] 154 | public virtual void TestAsterisk() 155 | { 156 | var indexStore = GetIndexStore("body", new System.String[] { "metal", "metals" }); 157 | IndexSearcher searcher = new IndexSearcher(indexStore, true); 158 | Query query1 = new TermQuery(new Term("body", "metal")); 159 | Query query2 = new WildcardQuery(new Term("body", "metal*")); 160 | Query query3 = new WildcardQuery(new Term("body", "m*tal")); 161 | Query query4 = new WildcardQuery(new Term("body", "m*tal*")); 162 | Query query5 = new WildcardQuery(new Term("body", "m*tals")); 163 | 164 | BooleanQuery query6 = new BooleanQuery(); 165 | query6.Add(query5, Occur.SHOULD); 166 | 167 | BooleanQuery query7 = new BooleanQuery(); 168 | query7.Add(query3, Occur.SHOULD); 169 | query7.Add(query5, Occur.SHOULD); 170 | 171 | // Queries do not automatically lower-case search terms: 172 | Query query8 = new WildcardQuery(new Term("body", "M*tal*")); 173 | 174 | AssertMatches(searcher, query1, 1); 175 | AssertMatches(searcher, query2, 2); 176 | AssertMatches(searcher, query3, 1); 177 | AssertMatches(searcher, query4, 2); 178 | AssertMatches(searcher, query5, 1); 179 | AssertMatches(searcher, query6, 1); 180 | AssertMatches(searcher, query7, 2); 181 | AssertMatches(searcher, query8, 0); 182 | AssertMatches(searcher, new WildcardQuery(new Term("body", "*tall")), 0); 183 | AssertMatches(searcher, new WildcardQuery(new Term("body", "*tal")), 1); 184 | AssertMatches(searcher, new WildcardQuery(new Term("body", "*tal*")), 2); 185 | } 186 | 187 | 188 | /* 189 | * LUCENE-2620 190 | */ 191 | [TestMethod] 192 | public void TestLotsOfAsterisks() 193 | { 194 | var indexStore = GetIndexStore("body", new String[] { "metal", "metals" }); 195 | IndexSearcher searcher = new IndexSearcher(indexStore, true); 196 | System.Text.StringBuilder term = new System.Text.StringBuilder(); 197 | term.Append("m"); 198 | for (int i = 0; i < 512; i++) 199 | term.Append("*"); 200 | term.Append("tal"); 201 | Query query3 = new WildcardQuery(new Term("body", term.ToString())); 202 | 203 | AssertMatches(searcher, query3, 1); 204 | searcher.Close(); 205 | indexStore.Close(); 206 | } 207 | 208 | /// Tests Wildcard queries with a question mark. 209 | /// 210 | /// 211 | /// IOException if an error occurs 212 | [TestMethod] 213 | public virtual void TestQuestionmark() 214 | { 215 | var indexStore = GetIndexStore("body", new System.String[] { "metal", "metals", "mXtals", "mXtXls" }); 216 | IndexSearcher searcher = new IndexSearcher(indexStore, true); 217 | Query query1 = new WildcardQuery(new Term("body", "m?tal")); 218 | Query query2 = new WildcardQuery(new Term("body", "metal?")); 219 | Query query3 = new WildcardQuery(new Term("body", "metals?")); 220 | Query query4 = new WildcardQuery(new Term("body", "m?t?ls")); 221 | Query query5 = new WildcardQuery(new Term("body", "M?t?ls")); 222 | Query query6 = new WildcardQuery(new Term("body", "meta??")); 223 | 224 | AssertMatches(searcher, query1, 1); 225 | AssertMatches(searcher, query2, 1); 226 | AssertMatches(searcher, query3, 0); 227 | AssertMatches(searcher, query4, 3); 228 | AssertMatches(searcher, query5, 0); 229 | AssertMatches(searcher, query6, 1); // Query: 'meta??' matches 'metals' not 'metal' 230 | } 231 | 232 | private SqlServerDirectory GetIndexStore(System.String field, System.String[] contents) 233 | { 234 | var indexStore = new SqlServerDirectory(Connection, new Options()); 235 | IndexWriter writer = new IndexWriter(indexStore, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); 236 | for (int i = 0; i < contents.Length; ++i) 237 | { 238 | Document doc = new Document(); 239 | doc.Add(new Field(field, contents[i], Field.Store.YES, Field.Index.ANALYZED)); 240 | writer.AddDocument(doc); 241 | } 242 | writer.Optimize(); 243 | writer.Close(); 244 | 245 | return indexStore; 246 | } 247 | 248 | private void AssertMatches(IndexSearcher searcher, Query q, int expectedMatches) 249 | { 250 | ScoreDoc[] result = searcher.Search(q, null, 1000).ScoreDocs; 251 | Assert.AreEqual(expectedMatches, result.Length); 252 | } 253 | 254 | /// Test that wild card queries are parsed to the correct type and are searched correctly. 255 | /// This test looks at both parsing and execution of wildcard queries. 256 | /// Although placed here, it also tests prefix queries, verifying that 257 | /// prefix queries are not parsed into wild card queries, and viceversa. 258 | /// 259 | /// Exception 260 | [TestMethod] 261 | public virtual void TestParsingAndSearching() 262 | { 263 | System.String field = "content"; 264 | bool dbg = false; 265 | QueryParser qp = new QueryParser(Lucene.Net.Util.Version.LUCENE_CURRENT, field, new WhitespaceAnalyzer()); 266 | qp.AllowLeadingWildcard = true; 267 | System.String[] docs = new System.String[] { "\\ abcdefg1", "\\79 hijklmn1", "\\\\ opqrstu1" }; 268 | // queries that should find all docs 269 | System.String[] matchAll = new System.String[] { "*", "*1", "**1", "*?", "*?1", "?*1", "**", "***", "\\\\*" }; 270 | // queries that should find no docs 271 | System.String[] matchNone = new System.String[] { "a*h", "a?h", "*a*h", "?a", "a?" }; 272 | // queries that should be parsed to prefix queries 273 | System.String[][] matchOneDocPrefix = new System.String[][] { new System.String[] { "a*", "ab*", "abc*" }, new System.String[] { "h*", "hi*", "hij*", "\\\\7*" }, new System.String[] { "o*", "op*", "opq*", "\\\\\\\\*" } }; 274 | // queries that should be parsed to wildcard queries 275 | System.String[][] matchOneDocWild = new System.String[][] { new System.String[] { "*a*", "*ab*", "*abc**", "ab*e*", "*g?", "*f?1", "abc**" }, new System.String[] { "*h*", "*hi*", "*hij**", "hi*k*", "*n?", "*m?1", "hij**" }, new System.String[] { "*o*", "*op*", "*opq**", "op*q*", "*u?", "*t?1", "opq**" } }; 276 | 277 | // prepare the index 278 | var dir = new SqlServerDirectory(Connection,new Options()); 279 | IndexWriter iw = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); 280 | for (int i = 0; i < docs.Length; i++) 281 | { 282 | Document doc = new Document(); 283 | doc.Add(new Field(field, docs[i], Field.Store.NO, Field.Index.ANALYZED)); 284 | iw.AddDocument(doc); 285 | } 286 | iw.Close(); 287 | 288 | IndexSearcher searcher = new IndexSearcher(dir, true); 289 | 290 | // test queries that must find all 291 | for (int i = 0; i < matchAll.Length; i++) 292 | { 293 | System.String qtxt = matchAll[i]; 294 | Query q = qp.Parse(qtxt); 295 | if (dbg) 296 | { 297 | System.Console.Out.WriteLine("matchAll: qtxt=" + qtxt + " q=" + q + " " + q.GetType().FullName); 298 | } 299 | ScoreDoc[] hits = searcher.Search(q, null, 1000).ScoreDocs; 300 | Assert.AreEqual(docs.Length, hits.Length); 301 | } 302 | 303 | // test queries that must find none 304 | for (int i = 0; i < matchNone.Length; i++) 305 | { 306 | System.String qtxt = matchNone[i]; 307 | Query q = qp.Parse(qtxt); 308 | if (dbg) 309 | { 310 | System.Console.Out.WriteLine("matchNone: qtxt=" + qtxt + " q=" + q + " " + q.GetType().FullName); 311 | } 312 | ScoreDoc[] hits = searcher.Search(q, null, 1000).ScoreDocs; 313 | Assert.AreEqual(0, hits.Length); 314 | } 315 | 316 | // test queries that must be prefix queries and must find only one doc 317 | for (int i = 0; i < matchOneDocPrefix.Length; i++) 318 | { 319 | for (int j = 0; j < matchOneDocPrefix[i].Length; j++) 320 | { 321 | System.String qtxt = matchOneDocPrefix[i][j]; 322 | Query q = qp.Parse(qtxt); 323 | if (dbg) 324 | { 325 | System.Console.Out.WriteLine("match 1 prefix: doc=" + docs[i] + " qtxt=" + qtxt + " q=" + q + " " + q.GetType().FullName); 326 | } 327 | Assert.AreEqual(typeof(PrefixQuery), q.GetType()); 328 | ScoreDoc[] hits = searcher.Search(q, null, 1000).ScoreDocs; 329 | Assert.AreEqual(1, hits.Length); 330 | Assert.AreEqual(i, hits[0].Doc); 331 | } 332 | } 333 | 334 | // test queries that must be wildcard queries and must find only one doc 335 | for (int i = 0; i < matchOneDocPrefix.Length; i++) 336 | { 337 | for (int j = 0; j < matchOneDocWild[i].Length; j++) 338 | { 339 | System.String qtxt = matchOneDocWild[i][j]; 340 | Query q = qp.Parse(qtxt); 341 | if (dbg) 342 | { 343 | System.Console.Out.WriteLine("match 1 wild: doc=" + docs[i] + " qtxt=" + qtxt + " q=" + q + " " + q.GetType().FullName); 344 | } 345 | Assert.AreEqual(typeof(WildcardQuery), q.GetType()); 346 | ScoreDoc[] hits = searcher.Search(q, null, 1000).ScoreDocs; 347 | Assert.AreEqual(1, hits.Length); 348 | Assert.AreEqual(i, hits[0].Doc); 349 | } 350 | } 351 | 352 | searcher.Close(); 353 | } 354 | } 355 | } -------------------------------------------------------------------------------- /LuceneNetSqlDirectory.Tests/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /LuceneNetSqlDirectory/App.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /LuceneNetSqlDirectory/AutoStopWatch.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | using Lucene.Net.Analysis; 4 | using Lucene.Net.Documents; 5 | using Lucene.Net.Index; 6 | using Lucene.Net.Store; 7 | 8 | namespace LuceneNetSqlDirectory 9 | { 10 | public class AutoStopWatch : IDisposable 11 | { 12 | private readonly Stopwatch _stopwatch; 13 | private readonly string _message; 14 | public AutoStopWatch(string message) 15 | { 16 | _message = message; 17 | Console.WriteLine("{0} starting ", message); 18 | _stopwatch = Stopwatch.StartNew(); 19 | } 20 | 21 | 22 | #region IDisposable Members 23 | public void Dispose() 24 | { 25 | 26 | _stopwatch.Stop(); 27 | long ms = _stopwatch.ElapsedMilliseconds; 28 | 29 | Console.WriteLine("{0} Finished {1} ms", _message, ms); 30 | } 31 | #endregion 32 | } 33 | } -------------------------------------------------------------------------------- /LuceneNetSqlDirectory/Database.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace LuceneNetSqlDirectory 4 | { 5 | public class Database 6 | { 7 | public static IEnumerable Tables 8 | { 9 | get 10 | { 11 | yield return "FileMetaData"; 12 | yield return "Locks"; 13 | yield return "FileContents"; 14 | } 15 | } 16 | 17 | public static IEnumerable Structure(string schemaName) 18 | { 19 | yield return $"CREATE TABLE {schemaName}.[FileMetaData] ( SequenceId BIGINT IDENTITY(1,1), IsDeleted BIT DEFAULT 0, [Name] NVARCHAR(400) NOT NULL,LastTouchedTimestamp DATETIME2 NOT NULL)"; 20 | yield return $"CREATE CLUSTERED INDEX IC_FileMetaData ON {schemaName}.[FileMetaData] (SequenceId)"; 21 | yield return $"CREATE NONCLUSTERED INDEX NIC_FileMetaData ON {schemaName}.[FileMetaData] ([Name],[IsDeleted])"; 22 | yield return $"CREATE TABLE {schemaName}.[Locks] ( Name NVARCHAR(400) NOT NULL, LockReleaseTimestamp DATETIME2 NOT NULL)"; 23 | yield return $"ALTER TABLE {schemaName}.[Locks] ADD CONSTRAINT PK_Locks PRIMARY KEY NONCLUSTERED ([Name] ASC)"; 24 | yield return $"CREATE TABLE {schemaName}.[FileContents] (SequenceId BIGINT IDENTITY(1,1), IsDeleted BIT DEFAULT 0, [Name] NVARCHAR(400) NOT NULL,[Content] varbinary(max) NULL)"; 25 | yield return $"CREATE CLUSTERED INDEX IC_FileContents ON {schemaName}.[FileContents] (SequenceId)"; 26 | yield return $"CREATE NONCLUSTERED INDEX NIC_Contents ON {schemaName}.[FileContents] ([Name],[IsDeleted])"; 27 | } 28 | } 29 | } -------------------------------------------------------------------------------- /LuceneNetSqlDirectory/Helpers/AutoStopwatch.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | 8 | namespace LuceneNetSqlDirectory.Helpers 9 | { 10 | public class AutoStopWatch : IDisposable 11 | { 12 | private readonly Stopwatch _stopwatch; 13 | private readonly string _message; 14 | public AutoStopWatch(string message) 15 | { 16 | _message = message; 17 | Console.WriteLine("{0} starting ", message); 18 | _stopwatch = Stopwatch.StartNew(); 19 | } 20 | 21 | 22 | #region IDisposable Members 23 | public void Dispose() 24 | { 25 | 26 | _stopwatch.Stop(); 27 | long ms = _stopwatch.ElapsedMilliseconds; 28 | 29 | Console.WriteLine("{0} Finished {1} ms", _message, ms); 30 | } 31 | #endregion 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /LuceneNetSqlDirectory/Helpers/HigherOrderFunctions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace LuceneNetSqlDirectory.Helpers 5 | { 6 | static class HigherOrderFunctions 7 | { 8 | public static void ForEach(this IEnumerable items, Action action) 9 | { 10 | foreach (var item in items) 11 | { 12 | action(item); 13 | } 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /LuceneNetSqlDirectory/Helpers/SqlConnectionHelper.cs: -------------------------------------------------------------------------------- 1 | using System.Data.SqlClient; 2 | using Dapper; 3 | 4 | namespace LuceneNetSqlDirectory.Helpers 5 | { 6 | static class SqlConnectionHelper 7 | { 8 | public static void DropTableIfExists(this SqlConnection connection, string schemaName, string tableName) 9 | { 10 | var exists = 1 == connection.ExecuteScalar(@"SELECT count(0) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='BASE TABLE' AND TABLE_NAME=@tableName AND TABLE_Schema = @schemaName", new { schemaName = SqlHelper.RemoveBrackets(schemaName), tableName = SqlHelper.RemoveBrackets(tableName) }); 11 | if (exists) 12 | { 13 | connection.Execute($"DROP TABLE {schemaName}.{tableName}"); 14 | } 15 | } 16 | 17 | public static bool SchemaExists(this SqlConnection connection, string schemaName) 18 | { 19 | return 1 == connection.ExecuteScalar("SELECT count(0) FROM information_schema.schemata WHERE schema_name = @schemaName", new { schemaName = SqlHelper.RemoveBrackets(schemaName) }); 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /LuceneNetSqlDirectory/Helpers/SqlHelper.cs: -------------------------------------------------------------------------------- 1 | namespace LuceneNetSqlDirectory.Helpers 2 | { 3 | static class SqlHelper 4 | { 5 | public static string RemoveBrackets(string s) 6 | { 7 | return s.Replace("[", "").Replace("]", ""); 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /LuceneNetSqlDirectory/LuceneNetSqlDirectory.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {931814DD-8065-41E8-8472-8641FD1CF110} 8 | Library 9 | Properties 10 | LuceneNetSqlDirectory 11 | LuceneNetSqlDirectory 12 | v4.5 13 | 512 14 | true 15 | SAK 16 | SAK 17 | SAK 18 | SAK 19 | 20 | 21 | 22 | AnyCPU 23 | true 24 | full 25 | false 26 | bin\Debug\ 27 | DEBUG;TRACE 28 | prompt 29 | 4 30 | false 31 | 32 | 33 | AnyCPU 34 | pdbonly 35 | true 36 | bin\Release\ 37 | TRACE 38 | prompt 39 | 4 40 | false 41 | 42 | 43 | 44 | 45 | 46 | 47 | ..\packages\Dapper.1.42\lib\net45\Dapper.dll 48 | 49 | 50 | ..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll 51 | 52 | 53 | ..\packages\Lucene.Net.3.0.3\lib\NET40\Lucene.Net.dll 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 90 | -------------------------------------------------------------------------------- /LuceneNetSqlDirectory/LuceneNetSqlDirectory.nuspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | LuceneNetSqlDirectory 5 | 1.2.0.0 6 | LuceneNetSqlDirectory 7 | Tim Mahy 8 | Tim Mahy 9 | false 10 | https://github.com/MahyTim/LuceneNetSqlDirectory/blob/master/LICENSE 11 | https://github.com/MahyTim/LuceneNetSqlDirectory 12 | Store your Lucene.NET files in a SQLServer database. No more need for a shared file system when operatring in a webfarm. 13 | Lucene LuceneNET Lucene.NET SQLServer SQL Directory LuceneNetSqlDirectory 14 | Tim Mahy 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /LuceneNetSqlDirectory/NuGet.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahyTim/LuceneNetSqlDirectory/a2cf88a554426237f3dddfd0095a30b801217713/LuceneNetSqlDirectory/NuGet.exe -------------------------------------------------------------------------------- /LuceneNetSqlDirectory/Options.cs: -------------------------------------------------------------------------------- 1 | namespace LuceneNetSqlDirectory 2 | { 3 | public class Options 4 | { 5 | /// 6 | /// The database schema name in which the structure is provisioned. By default this is 'dbo'. 7 | /// 8 | public string SchemaName { get; set; } 9 | /// 10 | /// Locks are automatically released after a certain time window, by default this is 10 minutes. 11 | /// If you need to do batch style jobs for adding/deleting/updating documents, increase this setting! 12 | /// 13 | public int LockTimeoutInSeconds { get; set; } 14 | 15 | public Options() 16 | { 17 | SchemaName = "[search]"; 18 | LockTimeoutInSeconds = 10 * 60;//10 minutes 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /LuceneNetSqlDirectory/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.InteropServices; 3 | 4 | [assembly: AssemblyTitle("LuceneNetSqlDirectory")] 5 | [assembly: AssemblyDescription("Lucene.NET Directory implementation that stores all the files in SQLServer")] 6 | [assembly: AssemblyConfiguration("")] 7 | [assembly: AssemblyCompany("Tim Mahy")] 8 | [assembly: AssemblyProduct("LuceneNetSqlDirectory")] 9 | [assembly: AssemblyCopyright("Tim Mahy")] 10 | [assembly: AssemblyTrademark("")] 11 | [assembly: AssemblyCulture("")] 12 | [assembly: ComVisible(false)] 13 | [assembly: Guid("2e17939d-8402-4cf8-a954-eaf9beae344d")] 14 | [assembly: AssemblyVersion("1.2.0.0")] 15 | [assembly: AssemblyFileVersion("1.2.0.0")] 16 | 17 | -------------------------------------------------------------------------------- /LuceneNetSqlDirectory/SqlServerDirectory.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Configuration; 5 | using System.Data; 6 | using System.Data.SqlClient; 7 | using System.Linq; 8 | using Dapper; 9 | using Lucene.Net.Store; 10 | using Lucene.Net.Support; 11 | using LuceneNetSqlDirectory.Helpers; 12 | 13 | namespace LuceneNetSqlDirectory 14 | { 15 | public sealed class SqlServerDirectory : Directory 16 | { 17 | private readonly SqlConnection _connection; 18 | private readonly Options _options; 19 | 20 | public SqlServerDirectory(SqlConnection connection, Options options) 21 | { 22 | if (connection == null) 23 | throw new ArgumentNullException(nameof(connection)); 24 | if (options == null) 25 | throw new ArgumentNullException(nameof(options)); 26 | 27 | _connection = connection; 28 | _options = options; 29 | ValidateConfiguration(); 30 | SetLockFactory(new SqlServerLockFactory(connection, options)); 31 | } 32 | 33 | private void ValidateConfiguration() 34 | { 35 | { // Connection management should be done outside this library 36 | if (_connection.State != ConnectionState.Open) 37 | { 38 | throw new ConfigurationErrorsException($"The connection is not open. SQLServerDirectory does not perform any connection management (opening, disposing or closing), this should be handled by the calling application."); 39 | } 40 | } 41 | { // Validate if the required database structure is available 42 | var tables = _connection.GetSchema("Tables"); 43 | var alltablesAreAvailable = tables.Select($"(TABLE_NAME = 'Locks' OR TABLE_NAME = 'FileMetaData' OR TABLE_NAME = 'FileContents' ) AND ( TABLE_SCHEMA = '{ SqlHelper.RemoveBrackets(_options.SchemaName)}')").Count() == 3; 44 | if (false == alltablesAreAvailable) 45 | { 46 | throw new ConfigurationErrorsException($"The database structure required for the SQLServerDirectory are not available in database : '{_connection.Database}'"); 47 | } 48 | } 49 | { // Validate if MARS is enabled because we need this to read efficiently! 50 | var connectionString = new SqlConnectionStringBuilder(_connection.ConnectionString); 51 | if (connectionString.MultipleActiveResultSets == false) 52 | { 53 | throw new ConfigurationErrorsException($"The given connection does not have 'MultipleActiveResultSets' enabled. SQLServerDirectory requires this feature in order to read efficiently from the database using multiple readers. Please add the following in the connectionstring that is used for the given connection : 'MultipleActiveResultSets=True;'"); 54 | 55 | } 56 | } // We require MS SQLServer 2008 or higher (the lowest dependency today is DateTime2 type) 57 | { 58 | if (Convert.ToInt16(_connection.ServerVersion.Split('.')[0]) < 10) 59 | { 60 | throw new ConfigurationErrorsException($"The database server used for the SQLServerDirectory should be at least a MSSQLServer 2008. Required version: 10, current version: {_connection.ServerVersion}"); 61 | } 62 | } 63 | } 64 | 65 | public static void ProvisionDatabase(SqlConnection connection, string schemaName = "[dbo]", bool dropExisting = false) 66 | { 67 | if (dropExisting) 68 | { 69 | Database.Tables.ForEach(z => connection.DropTableIfExists(schemaName, z)); 70 | } 71 | if (false == connection.SchemaExists(schemaName)) 72 | { 73 | connection.Execute($"CREATE SCHEMA {schemaName}"); 74 | } 75 | Database.Structure(schemaName).ForEach(z => connection.Execute(z)); 76 | } 77 | 78 | public override string[] ListAll() 79 | { 80 | return _connection.Query($"SELECT Name FROM {_options.SchemaName}.FileMetaData").ToArray(); 81 | } 82 | 83 | public override bool FileExists(string name) 84 | { 85 | return _connection.ExecuteScalar($"SELECT COUNT(0) FROM {_options.SchemaName}.FileMetaData WHERE Name = @name", new { name }) != 0; 86 | } 87 | 88 | public override long FileModified(string name) 89 | { 90 | var lastTouched = _connection.ExecuteScalar($"SElECT TOP 1 LastTouchedTimestamp FROM {_options.SchemaName}.FileMetaData WHERE name = @name", new { name }); 91 | return lastTouched.UtcTicks; 92 | } 93 | 94 | public override void TouchFile(string name) 95 | { 96 | _connection.Execute($"UPDATE {_options.SchemaName}.FileMetaData SET LastTouchedTimestamp = SYSUTCDATETIME() WHERE name = @name ", new { name }); 97 | } 98 | 99 | public override void DeleteFile(string name) 100 | { 101 | SqlServerIndexOutput runningOutput; 102 | if (_runningOutputs.TryRemove(name, out runningOutput)) 103 | { 104 | runningOutput.Dispose(); 105 | } 106 | SqlServerIndexInput runningInput; 107 | if (_runningInputs.TryRemove(name, out runningInput)) 108 | { 109 | runningInput.Dispose(); 110 | } 111 | 112 | _connection.Execute($"UPDATE {_options.SchemaName}.FileMetaData SET name = NEWID(), IsDeleted = 1 WHERE name = @name", new { name }); 113 | _connection.Execute($"UPDATE {_options.SchemaName}.FileContents SET name = NEWID(), IsDeleted = 1 WHERE name = @name", new { name }); 114 | } 115 | 116 | private readonly ConcurrentDictionary _runningInputs = new ConcurrentDictionary(); 117 | private readonly ConcurrentDictionary _runningOutputs = new ConcurrentDictionary(); 118 | 119 | public override long FileLength(string name) 120 | { 121 | return _connection.ExecuteScalar($"SELECT DATALENGTH(Content) FROM {_options.SchemaName}.FileContents WHERE name = @name", new { name }); 122 | } 123 | 124 | public override IndexOutput CreateOutput(string name) 125 | { 126 | SqlServerIndexOutput runningOutput; 127 | if (_runningOutputs.TryRemove(name, out runningOutput)) 128 | { 129 | runningOutput.Dispose(); 130 | } 131 | 132 | if (0 == _connection.ExecuteScalar($"SELECT COUNT(0) FROM {_options.SchemaName}.FileContents WHERE Name = @name", new { name })) 133 | { 134 | _connection.Execute($"INSERT INTO {_options.SchemaName}.FileContents (Name,Content) VALUES (@name,@content)", new { name, content = new byte[0] }); 135 | } 136 | if (0 == _connection.ExecuteScalar($"SELECT COUNT(0) FROM {_options.SchemaName}.FileMetaData WHERE Name = @name", new { name })) 137 | { 138 | _connection.Execute($"INSERT INTO {_options.SchemaName}.FileMetaData (Name,LastTouchedTimestamp) VALUES (@name,SYSUTCDATETIME())", new { name }); 139 | } 140 | 141 | var result = new SqlServerIndexOutput(_connection, name, _options); 142 | _runningOutputs.TryAdd(name, result); 143 | 144 | return result; 145 | } 146 | 147 | public override IndexInput OpenInput(string name) 148 | { 149 | SqlServerIndexInput runningInput; 150 | if (_runningInputs.TryRemove(name, out runningInput)) 151 | { 152 | runningInput.Dispose(); 153 | } 154 | 155 | var result = new SqlServerIndexInput(_connection, name, _options); 156 | _runningInputs.TryAdd(name, result); 157 | return result; 158 | } 159 | 160 | protected override void Dispose(bool disposing) 161 | { 162 | _runningInputs.Values.ForEach(z => z.Dispose()); 163 | _runningOutputs.Values.ForEach(z => z.Dispose()); 164 | _connection.Dispose(); 165 | } 166 | } 167 | } -------------------------------------------------------------------------------- /LuceneNetSqlDirectory/SqlServerIndexInput.cs: -------------------------------------------------------------------------------- 1 | using System.Data; 2 | using System.Data.SqlClient; 3 | using Dapper; 4 | using Lucene.Net.Store; 5 | using LuceneNetSqlDirectory.Helpers; 6 | 7 | namespace LuceneNetSqlDirectory 8 | { 9 | internal class SqlServerIndexInput : BufferedIndexInput 10 | { 11 | private readonly SqlConnection _connection; 12 | private readonly string _name; 13 | private readonly Options _options; 14 | private long _position; 15 | 16 | private SqlCommand _command; 17 | private SqlDataReader _reader; 18 | 19 | internal SqlServerIndexInput(SqlConnection connection, string name, Options options) 20 | { 21 | _connection = connection; 22 | _name = name; 23 | _options = options; 24 | } 25 | 26 | protected override void Dispose(bool disposing) 27 | { 28 | _reader?.Dispose(); 29 | _command?.Dispose(); 30 | _reader = null; 31 | _command = null; 32 | } 33 | 34 | public override void ReadInternal(byte[] b, int offset, int length) 35 | { 36 | if (b.Length == 0) 37 | return; 38 | 39 | if (_command == null || _reader == null || _reader.IsClosed || offset < _position) 40 | { 41 | _reader?.Dispose(); 42 | _command?.Dispose(); 43 | _command = new SqlCommand($"SELECT Content FROM {_options.SchemaName}.[FileContents] WHERE Name = @name", _connection); 44 | _command.Parameters.AddWithValue("name", _name); 45 | _reader = _command.ExecuteReader(CommandBehavior.SequentialAccess); 46 | _reader.Read(); 47 | } 48 | if (false == _reader.HasRows) 49 | { 50 | return; 51 | } 52 | if (false == _reader.IsDBNull(0)) 53 | { 54 | _reader.GetBytes(0, _position, b, offset, length); 55 | } 56 | _position += length; 57 | } 58 | 59 | public override void SeekInternal(long pos) 60 | { 61 | _position = pos; 62 | } 63 | 64 | public override long Length() 65 | { 66 | return _connection.ExecuteScalar($"SELECT DATALENGTH(Content) FROM {_options.SchemaName}.FileContents WHERE name = @name", new { name = _name }); 67 | } 68 | 69 | public override long FilePointer => _position; 70 | } 71 | } -------------------------------------------------------------------------------- /LuceneNetSqlDirectory/SqlServerIndexOutput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Data; 3 | using System.Data.SqlClient; 4 | using Dapper; 5 | using Lucene.Net.Store; 6 | using LuceneNetSqlDirectory.Helpers; 7 | 8 | namespace LuceneNetSqlDirectory 9 | { 10 | class SqlServerIndexOutput : BufferedIndexOutput 11 | { 12 | private readonly SqlConnection _connection; 13 | private readonly string _name; 14 | private readonly Options _options; 15 | private SqlCommand _updateCommand; 16 | private SqlParameter _parameterName; 17 | private SqlParameter _parameterData; 18 | private SqlParameter _parameterIndex; 19 | private SqlParameter _parameterLen; 20 | public SqlServerIndexOutput(SqlConnection connection, string name, Options options) 21 | { 22 | _connection = connection; 23 | _name = name; 24 | _options = options; 25 | } 26 | 27 | public override void FlushBuffer(byte[] b, int offset, int len) 28 | { 29 | var segment = new byte[len]; 30 | Buffer.BlockCopy(b, offset, segment, 0, len); 31 | 32 | _updateCommand = _updateCommand ?? new SqlCommand($"UPDATE {_options.SchemaName}.[FileContents] SET [Content].WRITE(@chunk, @index, @len) WHERE [Name] = @name", _connection); 33 | _parameterName = _parameterName ?? _updateCommand.Parameters.Add("@name", SqlDbType.NVarChar); 34 | _parameterData = _parameterData ?? _updateCommand.Parameters.Add("@chunk", SqlDbType.VarBinary, -1); 35 | _parameterIndex = _parameterIndex ?? _updateCommand.Parameters.Add("index", SqlDbType.BigInt); 36 | _parameterLen = _parameterLen ?? _updateCommand.Parameters.Add("len", SqlDbType.BigInt); 37 | 38 | _parameterName.Value = _name; 39 | _parameterData.Value = segment; 40 | _parameterIndex.Value = FilePointer - len; 41 | _parameterLen.Value = len; 42 | 43 | _updateCommand.ExecuteNonQuery(); 44 | } 45 | 46 | protected override void Dispose(bool disposing) 47 | { 48 | base.Dispose(disposing); 49 | _updateCommand?.Dispose(); 50 | _updateCommand = null; 51 | } 52 | 53 | public override long Length => _connection.ExecuteScalar($"SELECT DATALENGTH(Content) FROM {_options.SchemaName}.FileContents WHERE name = @name", new { name = _name }); 54 | } 55 | } -------------------------------------------------------------------------------- /LuceneNetSqlDirectory/SqlServerLock.cs: -------------------------------------------------------------------------------- 1 | using System.Data.SqlClient; 2 | using Dapper; 3 | using Lucene.Net.Store; 4 | 5 | namespace LuceneNetSqlDirectory 6 | { 7 | internal class SqlServerLock : Lock 8 | { 9 | private readonly SqlConnection _connection; 10 | private readonly string _lockName; 11 | private readonly Options _options; 12 | 13 | public SqlServerLock(SqlConnection connection, string lockName, Options options) 14 | { 15 | _connection = connection; 16 | _lockName = lockName; 17 | _options = options; 18 | } 19 | 20 | public override bool Obtain() 21 | { 22 | ReleaseLocksByReleaseTimestamp(); 23 | if (IsLocked()) 24 | return false; 25 | try 26 | { 27 | _connection.Execute($"INSERT INTO {_options.SchemaName}.Locks (Name,LockReleaseTimestamp) VALUES (@name,DATEADD(SECOND, @minutesToAdd, SYSUTCDATETIME()))", new { name = _lockName, minutesToAdd = _options.LockTimeoutInSeconds }); 28 | } 29 | catch (SqlException ex) when (ex.Number == 2627) // Duplicate key --> duplicate lock 30 | { 31 | return false; 32 | } 33 | return true; 34 | } 35 | 36 | private void ReleaseLocksByReleaseTimestamp() 37 | { 38 | _connection.Execute($"DELETE FROM {_options.SchemaName}.[Locks] WHERE LockReleaseTimestamp < SYSUTCDATETIME()"); 39 | } 40 | 41 | public override void Release() 42 | { 43 | _connection.Execute($"DELETE FROM {_options.SchemaName}.[Locks] WHERE Name = @name", new { name = _lockName }); 44 | ReleaseLocksByReleaseTimestamp(); 45 | } 46 | 47 | public override bool IsLocked() 48 | { 49 | return _connection.ExecuteScalar($"SELECT COUNT(1) FROM {_options.SchemaName}.[Locks] WHERE Name = @name AND LockReleaseTimestamp > SYSUTCDATETIME()", new { name = _lockName }) != 0; 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /LuceneNetSqlDirectory/SqlServerLockFactory.cs: -------------------------------------------------------------------------------- 1 | using System.Data.SqlClient; 2 | using Dapper; 3 | using Lucene.Net.Store; 4 | 5 | namespace LuceneNetSqlDirectory 6 | { 7 | internal class SqlServerLockFactory : LockFactory 8 | { 9 | private readonly SqlConnection _connection; 10 | private readonly Options _options; 11 | 12 | internal SqlServerLockFactory(SqlConnection connection, Options options) 13 | { 14 | _connection = connection; 15 | _options = options; 16 | } 17 | 18 | public override Lock MakeLock(string lockName) 19 | { 20 | return new SqlServerLock(_connection, lockName, _options); 21 | } 22 | 23 | public override void ClearLock(string lockName) 24 | { 25 | _connection.Execute($"DELETE FROM {_options.SchemaName}.Locks Where Name = @name", new { name = lockName }); 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /LuceneNetSqlDirectory/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LuceneNetSqlDirectory 2 | Store your Lucene.NET files in a SQLServer using this Directory implementation. 3 | 4 | ## Why ## 5 | [Lucene.NET](https://lucenenet.apache.org/) is a very powerful [full text search](https://en.wikipedia.org/wiki/Full_text_search) solution with advanced '[faceted search](https://en.wikipedia.org/wiki/Faceted_search)' options. Lucene.NET stores it's metadata and inverted index using 'files', and the default implementations shipped with the framework are all file system based. However this has some major drawbacks if you want to use the solution in a clustered webfarm environment and encounter a more traditional operations division. From our experience most large scale non-devops oranizations using the MS stack tend to have an operations division that does not like shared file systems and cannot come up with decent backup strategies that are aligned with database backups. 6 | 7 | Being fed up of having to go into discussions on shared filesystems we decided to store the index in a Microsoft SQLServer database. In contrast to the Java JDBCDirectory this implementation is only aimed at storing it in MS SQLServer 2008 or higher, using it's advanced random read/write capabilities. 8 | 9 | Please notice that this implementation only intends to give a solution to indexes for less than 2 GB and that for high & concurrent write and near realtime search we suggest to look at [ElasticSearch](https://www.elastic.co/) or [Solr](http://lucene.apache.org/solr/) or managed services such as [Azure Search](http://azure.microsoft.com/en-us/services/search/ "Azure Search"). 10 | 11 | ## How to use ## 12 | using (var connection = new SqlConnection(@"......")) { 13 | connection.Open(); 14 | 15 | var directory = new SqlServerDirectory(connection, new Options() { SchemaName = "[search]" }); 16 | var exists = !IndexReader.IndexExists(directory); 17 | var indexWriter = new IndexWriter(directory, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30),exists,new Lucene.Net.Index.IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH)); 18 | 19 | var doc = new Document(); 20 | doc.Add(new Field("id", "1", Field.Store.YES,Field.Index.ANALYZED, Field.TermVector.NO)); 21 | doc.Add(new Field("Title", "this is my title", Field.Store.NO, Field.Index.ANALYZED,Field.TermVector.NO)); 22 | indexWriter.AddDocument(doc); 23 | 24 | searcher = new IndexSearcher(directory); 25 | var hits = searcher.Search(new TermQuery(new Term("Title", "find me")), 100); 26 | 27 | 28 | The SqlServerDirectory class takes in: 29 | 30 | 1. a connection (should be opened before because the implementation does not do any connection management such as disposing, opening or closing the given connection) 31 | 2. an Options instance in which the user can define in which database schema the required structure can be found (default = 'dbo' and a writer lock timeout (default 10 minutes) 32 | 33 | The required database structure can be provisioned using the following codesnippet: 34 | 35 | using (var connection = new SqlConnection("....")){ 36 | connection.Open(); 37 | SqlServerDirectory.ProvisionDatabase(connection, schemaName: "[search]", dropExisting: true); 38 | 39 | 40 | ## Limits ## 41 | 1. An individual index file cannot grow beyond 2GB. 42 | 1. There is a single writer locking factory included that is default used. 43 | 2. MultipleActiveResultSets should be enabled on the connection given to the SQLServerDirectory 44 | 45 | 46 | ## How to compile/contribute ## 47 | The codebase is currently writen in C# 6.0 and has a solution file for Microsoft Visual Studio 2015. Any contribution is welcome. 48 | -------------------------------------------------------------------------------- /SqlDirectory.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 14 4 | VisualStudioVersion = 14.0.25420.1 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LuceneNetSqlDirectory", "LuceneNetSqlDirectory\LuceneNetSqlDirectory.csproj", "{931814DD-8065-41E8-8472-8641FD1CF110}" 7 | EndProject 8 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LuceneNetSqlDirectory.Tests", "LuceneNetSqlDirectory.Tests\LuceneNetSqlDirectory.Tests.csproj", "{0BA12033-05D5-4DA7-8E43-A89596CB5A4C}" 9 | EndProject 10 | Global 11 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 12 | Debug|Any CPU = Debug|Any CPU 13 | Debug35|Any CPU = Debug35|Any CPU 14 | Release|Any CPU = Release|Any CPU 15 | Release35|Any CPU = Release35|Any CPU 16 | EndGlobalSection 17 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 18 | {931814DD-8065-41E8-8472-8641FD1CF110}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 19 | {931814DD-8065-41E8-8472-8641FD1CF110}.Debug|Any CPU.Build.0 = Debug|Any CPU 20 | {931814DD-8065-41E8-8472-8641FD1CF110}.Debug35|Any CPU.ActiveCfg = Debug|Any CPU 21 | {931814DD-8065-41E8-8472-8641FD1CF110}.Debug35|Any CPU.Build.0 = Debug|Any CPU 22 | {931814DD-8065-41E8-8472-8641FD1CF110}.Release|Any CPU.ActiveCfg = Release|Any CPU 23 | {931814DD-8065-41E8-8472-8641FD1CF110}.Release|Any CPU.Build.0 = Release|Any CPU 24 | {931814DD-8065-41E8-8472-8641FD1CF110}.Release35|Any CPU.ActiveCfg = Release|Any CPU 25 | {931814DD-8065-41E8-8472-8641FD1CF110}.Release35|Any CPU.Build.0 = Release|Any CPU 26 | {0BA12033-05D5-4DA7-8E43-A89596CB5A4C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 27 | {0BA12033-05D5-4DA7-8E43-A89596CB5A4C}.Debug|Any CPU.Build.0 = Debug|Any CPU 28 | {0BA12033-05D5-4DA7-8E43-A89596CB5A4C}.Debug35|Any CPU.ActiveCfg = Debug|Any CPU 29 | {0BA12033-05D5-4DA7-8E43-A89596CB5A4C}.Debug35|Any CPU.Build.0 = Debug|Any CPU 30 | {0BA12033-05D5-4DA7-8E43-A89596CB5A4C}.Release|Any CPU.ActiveCfg = Release|Any CPU 31 | {0BA12033-05D5-4DA7-8E43-A89596CB5A4C}.Release|Any CPU.Build.0 = Release|Any CPU 32 | {0BA12033-05D5-4DA7-8E43-A89596CB5A4C}.Release35|Any CPU.ActiveCfg = Release|Any CPU 33 | {0BA12033-05D5-4DA7-8E43-A89596CB5A4C}.Release35|Any CPU.Build.0 = Release|Any CPU 34 | EndGlobalSection 35 | GlobalSection(SolutionProperties) = preSolution 36 | HideSolutionNode = FALSE 37 | EndGlobalSection 38 | EndGlobal 39 | --------------------------------------------------------------------------------