├── .gitignore ├── EngineConsole ├── .vscode │ ├── launch.json │ └── tasks.json ├── EngineConsole.csproj ├── EngineTest.cs ├── ObjectSearch.cs └── Program.cs ├── FTServer ├── .vscode │ ├── launch.json │ └── tasks.json ├── Code │ ├── App.cs │ ├── ConcurrentLinkedDeque.cs │ ├── Config.cs │ ├── EasyOR.cs │ ├── FCSharpBridge.cs │ ├── FKeyWord.cs │ ├── FLang.cs │ ├── FStringUtil.cs │ ├── FTSEngine.cs │ ├── Html.cs │ ├── IndexAPI.cs │ ├── IndexFields.cs │ ├── IndexPage.cs │ ├── IndexServer.cs │ └── ReadonlyList.cs ├── Controllers │ ├── BookController.cs │ └── HomeController.cs ├── FTServer.csproj ├── Models │ ├── About.cshtml.cs │ ├── AdminModel.cs │ ├── Book.cshtml.cs │ ├── Error.cshtml.cs │ └── ResultPartial.cshtml.cs ├── Program.cs ├── Properties │ └── launchSettings.json ├── Startup.cs ├── Views │ ├── Book │ │ └── Index.cshtml │ ├── Home │ │ ├── About.cshtml │ │ ├── Admin.cshtml │ │ ├── Index.cshtml │ │ └── ResultPartial.cshtml │ ├── Shared │ │ ├── Error.cshtml │ │ └── _Layout.cshtml │ ├── _ViewImports.cshtml │ └── _ViewStart.cshtml ├── appsettings.Development.json ├── appsettings.json ├── fts.ico └── wwwroot │ ├── css │ ├── semantic.min.css │ └── themes │ │ └── default │ │ └── assets │ │ └── fonts │ │ └── icons.woff2 │ └── favicon.ico └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | Data/* 2 | Obj/* 3 | Bin/* 4 | data/* 5 | obj/* 6 | bin/* 7 | *_Bak 8 | */obj/* 9 | */bin/* 10 | */Data/* 11 | */data/* 12 | FTServer/Code/ExampleCode.txt 13 | *.box 14 | *.box.swp 15 | DATA_FTS_CS_140/* 16 | -------------------------------------------------------------------------------- /EngineConsole/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": ".NET Core Launch (console)", 9 | "type": "coreclr", 10 | "request": "launch", 11 | "preLaunchTask": "build", 12 | "program": "${workspaceFolder}/bin/Debug/net6.0/EngineConsole.dll", 13 | "args": [], 14 | "cwd": "${workspaceFolder}", 15 | "console": "internalConsole", 16 | "stopAtEntry": false 17 | }, 18 | { 19 | "name": ".NET Core Attach", 20 | "type": "coreclr", 21 | "request": "attach" 22 | } 23 | ] 24 | } -------------------------------------------------------------------------------- /EngineConsole/.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "label": "build", 6 | "command": "dotnet", 7 | "type": "process", 8 | "args": [ 9 | "build", 10 | "${workspaceFolder}/EngineConsole.csproj", 11 | "/property:GenerateFullPaths=true", 12 | "/consoleloggerparameters:NoSummary" 13 | ], 14 | "problemMatcher": "$msCompile" 15 | }, 16 | { 17 | "label": "publish", 18 | "command": "dotnet", 19 | "type": "process", 20 | "args": [ 21 | "publish", 22 | "${workspaceFolder}/EngineConsole.csproj", 23 | "/property:GenerateFullPaths=true", 24 | "/consoleloggerparameters:NoSummary" 25 | ], 26 | "problemMatcher": "$msCompile" 27 | }, 28 | { 29 | "label": "watch", 30 | "command": "dotnet", 31 | "type": "process", 32 | "args": [ 33 | "watch", 34 | "run", 35 | "${workspaceFolder}/EngineConsole.csproj", 36 | "/property:GenerateFullPaths=true", 37 | "/consoleloggerparameters:NoSummary" 38 | ], 39 | "problemMatcher": "$msCompile" 40 | } 41 | ] 42 | } -------------------------------------------------------------------------------- /EngineConsole/EngineConsole.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | net8.0 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /EngineConsole/EngineTest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using System.IO; 5 | using System.Threading; 6 | using System.Threading.Tasks; 7 | 8 | using IBoxDB.LocalServer; 9 | 10 | namespace FTServer 11 | { 12 | public class EngineTest 13 | { 14 | 15 | public static void test_order() 16 | { 17 | DB.Root("/tmp/"); 18 | 19 | 20 | BoxSystem.DBDebug.DeleteDBFiles(3); 21 | DB db = new DB(3); 22 | Engine engine = Engine.Instance; 23 | engine.Config(db.GetConfig()); 24 | 25 | AutoBox auto = db.Open(); 26 | 27 | int count = 100; 28 | String[] ts = new String[count]; 29 | for (int i = 0; i < count; i++) 30 | { 31 | ts[i] = "test " + i; 32 | } 33 | for (int i = 0; i < ts.Length; i++) 34 | { 35 | using (IBox box = auto.Cube()) 36 | { 37 | engine.indexText(box, i, ts[i], false); 38 | box.Commit().Assert(); 39 | } 40 | } 41 | 42 | bool doagain = true; 43 | long startId = long.MaxValue; 44 | long tcount = 0; 45 | while (doagain && (startId >= 0)) 46 | { 47 | doagain = false; 48 | using (IBox box = auto.Cube()) 49 | { 50 | foreach (KeyWord kw in engine.searchDistinct(box, "test", startId, 9)) 51 | { 52 | Console.WriteLine(engine.getDesc(ts[(int)kw.I], kw, 20)); 53 | tcount++; 54 | doagain = true; 55 | startId = kw.I - 1; 56 | } 57 | } 58 | Console.WriteLine(); 59 | Console.WriteLine(startId); 60 | } 61 | Console.WriteLine(count + " == " + tcount); 62 | auto.GetDatabase().Dispose(); 63 | 64 | } 65 | 66 | public static void test_main() 67 | { 68 | DB.Root("/tmp/"); 69 | 70 | 71 | String[] ts = new String[] { 72 | //ID=0 73 | "Setting up Git\n" 74 | + "\n" 75 | + "Download and install the latest version of GitHub Desktop. " 76 | + "This will automatically install Git and keep it up-to-date for you.\n" 77 | + "On your computer, open the Git Shell application.\n" 78 | + "Tell Git your name so your commits will be properly labeled. Type everything after the $ here:\n" 79 | + "\n babies " 80 | + "git config --global user.name \"YOUR NAME\"\n" 81 | + "Tell Git the email address that will be associated with your Git commits. " 82 | + "The email you specify should be the same one found in your email settings. " 83 | + "To keep your email address hidden," 84 | + " 关于 see \"Keeping your C# Java NoSQL email address abc@global.com private\".", 85 | //ID=1 86 | "关于版本控制\n" 87 | + "什么是“版本控制”？我为什么要关心它呢？ 版本控制是一种记录一个或若干文件内容变化，1234567890ABCDEFGH " 88 | + "以便将来查阅特定版本修订情况的系统。 在本书所展示的例子中，我们对保存着软件源代码的文件作版本控制，" 89 | + "但实际上，C lang IT 你可以对任何类型的文件进行版本控制。", 90 | //ID=2 91 | "バージョン管理に関して\n" 92 | + "\n" 93 | + "「バージョン管理」とは何でしょうか。また、なぜそれを気にする必要があるのでしょうか。 " 94 | + "バージョン管理とは、一つのファイルやファイルの集合に対して時間とともに加えられていく変更を記録するシステムで、" 95 | + "後で特定バージョンを呼び出すことができるようにするためのものです。" 96 | + " 本書の例では、バージョン管理されるファイルとしてソフトウェアのソースコードを用いていますが、" 97 | + "実際にはコンピューター上のあらゆる種類のファイルをバージョン管理のもとに置くことができます。", 98 | //ID=3 99 | "關於版本控制\n" 100 | + "什麼是版本控制？ 以及為什麼讀者會在意它？ 美食" 101 | + "版本控制是一個能夠記錄一個或一組檔案在某一段時間的變更，" 102 | + "使得讀者以後能取回特定版本的系統。has NoSQL" 103 | + "在本書的範例中，android 讀者會學到如何對軟體的原始碼做版本控制。" 104 | + " 即使實際上讀者幾乎可以針對電腦上任意型態的檔案做版本控制。", 105 | //ID=4 106 | "Git 简史\n" 107 | + "同生活中的许多伟大事物一样，Git 诞生于一个极富纷争大举创新的年代。nosql \n" 108 | + "\n" 109 | + "Linux 内核开源项目有着为数众广的参与者。 绝大多数的 Linux 内核维护工作都花在了提交补丁和保存归档的" 110 | + "繁琐事务上（1991－2002年间）。 到 2002 年，" 111 | + "整个项目组开始启用一个专有的分布式版本控制系统 BitKeeper 来管理和维护代码。\n" 112 | + "\n" 113 | + "到了 2005 年，开发 BitKeeper 的商业公司同 Linux 内核开源社区的合作关系结束，" 114 | + "他们收回了 Linux 内核社区免费使用 BitKeeper 的权力。" 115 | + " 这就迫使 Linux 开源社区（特别是 Linux 的缔造者 Linux Torvalds）基于使用 BitKcheper 时的" 116 | + "经验教训，开发出自己的版本系统。 他们对新的系统制订了若干目标：", 117 | //ID=5 118 | "버전 관리란?\n\n버전 관리는 무엇이고 우리는 왜 이것을 알아야 할까? 버전 관리 시스템은 파일 변화를 시간에 따라 " + 119 | "기록했다가 나중에 특정 시점의 버전을 다시 꺼내올 수 있는 시스템이다. 이 책에서는 버전 관리하는 예제로 소프트웨어 " + 120 | "소스 코드만 보여주지만, 실제로 거의 모든 컴퓨터 파일의 버전을 관리할 수 있다.\n\n그래픽 디자이너나" + 121 | "웹 디자이너도 버전 관리 시스템(VCS - Version Control System)을 사용할 수 있다. VCS로 이미지나 레이아웃의" + 122 | "버전(변경 이력 혹은 수정 내용)을 관리하는 것은 매우 현명하다. VCS를 사용하면 각 파일을 이전 상태로 되돌릴 수 있고," + 123 | "프로젝트를 통째로 이전 is 상태로 되돌릴 수 있고, 시간에 따라 수정 내용을 비교해 볼 수 있고," + 124 | "누가 문제를 일으켰는지도 추적할 수 있고, 누가 언제 만들어낸 이슈인지도 알 수 있다. VCS를 사용하면 파일을 잃어버리거나" + 125 | "잘못 고쳤을 때도 쉽게 복구할 수 있다. HAS GIT 이런 모든 장점을 큰 노력 없이 이용할 수 있다.", 126 | //ID=6 127 | @" 128 | 1.1 شروع به کار - دربارهٔ کنترل نسخه 129 | 130 | این فصل راجع به آغاز به کار با گیت خواهد بود. در آغاز پیرامون تاریخچهٔ ابزارهای کنترل نسخه توضیحاتی خواهیم داد، سپس به چگونگی راه‌اندازی گیت بر روی سیستم‌تان خواهیم پرداخت و در پایان به تنظیم گیت و کار با آن. در پایان این فصل خواننده علت وجود و استفاده از گیت را خواهد دانست و خواهد توانست محیط کار با گیت را فراهم کند. 131 | دربارهٔ کنترل نسخه 132 | 133 | «کنترل نسخه» چیست و چرا باید بدان پرداخت؟ کنترل نسخه سیستمی است که تغییرات را در فایل یا دسته‌ای از فایل‌ها ذخیره می‌کند و به شما این امکان را می‌دهد که در آینده به نسخه و نگارش خاصی برگردید. برای مثال‌های این کتاب، شما از سورس کد نرم‌افزار به عنوان فایل‌هایی که نسخه آنها کنترل می‌شود استفاده می‌کنید. اگرچه در واقع می‌توانید تقریباً از هر فایلی استفاده کنید. 134 | 135 | اگر شما یک گرافیست یا طراح وب هستید و می‌خواهید نسخه‌های متفاوت از عکس‌ها و قالب‌های خود داشته باشید (که احتمالاً می‌خواهید)، یک سیستم کنترل نسخه (Version Control System (VCS)) انتخاب خردمندانه‌ای است. یک VCS به شما این امکان را می‌دهد که فایل‌های انتخابی یا کل پروژه را به یک حالت قبلی خاص برگردانید، روند تغییرات را بررسی کنید، ببینید چه کسی آخرین بار تغییری ایجاد کرده که احتمالاً مشکل آفرین شده، چه کسی، چه وقت مشکلی را اعلام کرده و…​ استفاده از یک VCS همچنین به این معناست که اگر شما در حین کار چیزی را خراب کردید و یا فایل‌هایی از دست رفت، به سادگی می‌توانید کارهای انجام شده را بازیابی نمایید. همچنین مقداری سربار به فایل‌های پروژه‌تان افزوده می‌شود. 136 | سیستم‌های کنترل نسخهٔ محلی 137 | 138 | روش اصلی کنترل نسخهٔ کثیری از افراد کپی کردن فایل‌ها به پوشه‌ای دیگر است (احتمالاً با تاریخ‌گذاری، اگر خیلی باهوش باشند). این رویکرد به علت سادگی بسیار رایج است هرچند خطا آفرینی بالایی دارد. فراموش کردن اینکه در کدام پوشه بوده‌اید و نوشتن اشتباهی روی فایل یا فایل‌هایی که نمی‌خواستید روی آن بنویسید بسیار آسان است. 139 | 140 | برای حل این مشکل، سال‌ها قبل VCSهای محلی را توسعه دادند که پایگاه داده‌ای ساده داشت که تمام تغییرات فایل‌های تحت مراقبتش را نگهداری می‌کرد. 141 | Local version control diagram 142 | نمودار 1. کنترل نسخه محلی. 143 | 144 | یکی از شناخته‌شده‌ترین ابزاری‌های کنترل نسخه، سیستمی به نام RCS بود که حتی امروز، با بسیاری از کامپیوترها توزیع می‌شود. RCS با نگه داشتن مجموعه‌هایی از پچ‌ها (Patch/وصله) — همان تفاوت‌های بین نگارش‌های گوناگون فایل‌ها — در قالبی ویژه کار می‌کند؛ پس از این، با اعمال پچ‌ها می‌تواند هر نسخه‌ای از فایل که مربوط به هر زمان دلخواه است را بازسازی کند. 145 | سیستم‌های کنترل نسخهٔ متمرکز 146 | 147 | چالش بزرگ دیگری که مردم با آن روبرو می‌شوند نیاز به همکاری با توسعه‌دهندگانی است که با سیستم‌های دیگر کار می‌کنند. دربرخورد با این چالش سیستم‌های کنترل نسخه متمرکز (Centralized Version Control System (CVCS)) ایجاد شدند. این قبیل سیستم‌ها (مثل CVS، ساب‌ورژن و Preforce) یک سرور دارند که تمام فایل‌های نسخه‌بندی شده را در بر دارد و تعدادی کلاینت (Client/خدمت‌گیرنده) که فایل‌هایی را از آن سرور چک‌اوت (Checkout/وارسی) می‌کنند. سال‌های سال این روش استاندارد کنترل نسخه بوده است. 148 | Centralized version control diagram 149 | نمودار 2. کنترل نسخه متمرکز. 150 | 151 | این ساماندهی به ویژه برای VCSهای محلی منافع و مزایای بسیاری دارد. به طور مثال هر کسی به میزان مشخصی از فعالیت‌های دیگران روی پروژه آگاهی دارد. مدیران دسترسی و کنترل مناسبی بر این دارند که چه کسی چه کاری می‌تواند انجام دهد؛ همچنین مدیریت یک CVCS خیلی آسان‌تر از درگیری با پایگاه‌داده‌های محلی روی تک تک کلاینت‌هاست. 152 | 153 | هرچند که این گونه ساماندهی معایب جدی نیز دارد. واضح‌ترین آن رخدادن خطا در سروری که نسخه‌ها در آن متمرکز شده‌اند است. اگر که سرور برای یک ساعت غیرفعال باشد، در طول این یک ساعت هیچ‌کس نمی‌تواند همکاری یا تغییراتی که انجام داده است را ذخیره نماید. اگر هارددیسک سرور مرکزی دچار مشکلی شود و پشتیبان مناسبی هم تهیه نشده باشد همه چیز (تاریخچه کامل پروژه بجز اسنپ‌شات‌هایی که یک کلاینت ممکن است روی کامپیوتر خود ذخیره کرده باشد) از دست خواهد رفت. VCSهای محلی نیز همگی این مشکل را دارند — هرگاه کل تاریخچه پروژه را در یک مکان واحد ذخیره کنید، خطر از دست دادن همه چیز را به جان می‌خرید. 154 | سیستم‌های کنترل نسخه توزیع‌شده 155 | 156 | اینجا است که سیستم‌های کنترل نسخه توزیع‌شده (Distributed Version Control System (DVCS)) نمود پیدا می‌کنند. در یک DVCS (مانند گیت، Mercurial، Bazaar یا Darcs) کلاینت‌ها صرفاً به چک‌اوت کردن آخرین اسنپ‌شات فایل‌ها اکتفا نمی‌کنند؛ بلکه آن‌ها کل مخزن (Repository) را کپی عینی یا آینه (Mirror) می‌کنند که شامل تاریخچه کامل آن هم می‌شود. بنابراین اگر هر سروری که سیستم‌ها به واسطه آن در حال تعامل با یکدیگر هستند متوقف شده و از کار بیافتد، با کپی مخرن هر کدام از کاربران بر روی سرور، می‌توان آن را بازیابی کرد. در واقع هر کلون، پشتیبان کاملی از تمامی داده‌ها است. 157 | Distributed version control diagram 158 | نمودار 3. کنترل نسخه توزیع‌شده. 159 | 160 | علاوه بر آن اکثر این سیستم‌ها تعامل کاری خوبی با مخازن متعدد خارجی دارند و از آن استقبال می‌کنند، در نتیجه شما می‌توانید با گروه‌های مختلفی به روش‌های مختلفی در قالب پروژه‌ای یکسان به‌صورت همزمان همکاری کنید. این قابلیت این امکان را به کاربر می‌دهد که چندین جریان کاری متنوع، مانند مدل‌های سلسه مراتبی، را پیاده‌سازی کند که انجام آن در سیستم‌های متمرکز امکان‌پذیر نیست. 161 | prev | next 162 | ", 163 | //ID=7 164 | "حال باید درک پایه‌ای از اینکه گیت چیست و چه تفاوتی با سایر سیستم‌های کنترل نسخه متمرکز قدیمی دارد داشته باشید. همچنین حالا باید یک نسخه کاری از گیت که با هویت شخصی شما تنظیم شده را روی سیستم خود داشته باشید. اکنون وقت آن رسیده که کمی از مقدمات گیت را فرابگیرید" 165 | }; 166 | for (int tran = 0; tran < 2; tran++) 167 | { 168 | BoxSystem.DBDebug.DeleteDBFiles(3); 169 | DB db = new DB(3); 170 | Engine engine = Engine.Instance; 171 | engine.Config(db.GetConfig()); 172 | 173 | AutoBox auto = db.Open(); 174 | 175 | 176 | for (int i = 0; i < ts.Length; i++) 177 | { 178 | if (tran == 0) 179 | { 180 | using (var box = auto.Cube()) 181 | { 182 | engine.indexText(box, i, ts[i], false); 183 | box.Commit().Assert(); 184 | } 185 | } 186 | else 187 | { 188 | engine.indexTextNoTran(auto, 3, i, ts[i], false); 189 | } 190 | } 191 | 192 | using (var box = auto.Cube()) 193 | { 194 | 195 | //engine.indexText(box, 4, ts[4], true); 196 | box.Commit().Assert(); 197 | } 198 | 199 | String[] teststr = new String[] { 200 | "실제로 거의 모든", 201 | "هویت", 202 | "یکسان به‌صورت همزمان", 203 | "\"" + "متعدد خارجی دارند و از" + "\"" 204 | ,"nosql has 電 原始碼" }; 205 | using (var box = auto.Cube()) 206 | { 207 | foreach (var str in teststr) 208 | { 209 | // searchDistinct() search() 210 | Console.WriteLine("for " + str); 211 | foreach (KeyWord kw in engine.search(box, str)) 212 | { 213 | Console.WriteLine(kw.ToFullString()); 214 | Console.WriteLine(engine.getDesc(ts[(int)kw.I], kw, 20)); 215 | Console.WriteLine(); 216 | } 217 | } 218 | foreach (String skw in engine.discoverEN(box, 219 | 'n', 's', 2)) 220 | { 221 | Console.WriteLine(skw); 222 | } 223 | foreach (String skw in engine.discoverCN(box, 224 | '\u2E80', '\u9fa5', 2)) 225 | { 226 | Console.WriteLine(skw); 227 | } 228 | } 229 | auto.GetDatabase().Dispose(); 230 | Console.WriteLine("----------------------------------"); 231 | } 232 | } 233 | 234 | public static void test_big_n() 235 | { 236 | String book = "/hero.txt"; 237 | long dbid = 1; 238 | char split = '。'; 239 | 240 | //set this true 241 | bool rebuild = false; 242 | int notranCount = -1;//10; 243 | String strkw = "黄蓉 郭靖 洪七公"; 244 | strkw = "洪七公 黄蓉 郭靖"; 245 | strkw = "黄蓉 郭靖 公"; 246 | strkw = "郭靖 黄蓉"; 247 | strkw = "黄蓉"; 248 | 249 | strkw = "时察"; 250 | strkw = "的"; 251 | strkw = "七十二路"; 252 | strkw = "十八掌"; 253 | strkw = "日日夜夜无穷无尽的"; 254 | strkw = "牛家村边绕 日日夜夜无穷无尽的"; 255 | strkw = "这几天"; 256 | strkw = "有 这几天"; 257 | strkw = "这几天 有"; 258 | test_big(book, dbid, rebuild, split, strkw, notranCount); 259 | } 260 | 261 | public static void test_big_e() 262 | { 263 | String book = "/phoenix.txt"; 264 | long dbid = 2; 265 | char split = '.'; 266 | 267 | //set true 268 | bool rebuild = false; 269 | int notranCount = 10; //-1; 270 | 271 | String strkw = "Harry"; 272 | strkw = "Harry he"; 273 | strkw = "He Harry"; 274 | strkw = "Harry Philosopher"; 275 | strkw = "Philosopher"; 276 | strkw = "\"Harry Philosopher\""; 277 | strkw = "\"He looks\""; 278 | strkw = "He looks"; 279 | strkw = "\"he drove toward town he thought\""; 280 | strkw = "\"he drove toward\""; 281 | strkw = "\"he thought\""; 282 | strkw = "\"he thought\" toward"; 283 | strkw = "toward \"he thought\""; 284 | strkw = "he thought"; 285 | strkw = "he thought toward"; 286 | strkw = "He"; 287 | test_big(book, dbid, rebuild, split, strkw, notranCount); 288 | } 289 | 290 | private static void test_big(String book, long dbid, bool rebuild, 291 | char split, String strkw, int notranCount) 292 | { 293 | DB.Root("/tmp/"); 294 | 295 | if (rebuild) 296 | { 297 | BoxSystem.DBDebug.DeleteDBFiles(dbid); 298 | } 299 | DB db = new DB(dbid); 300 | 301 | String[] tstmp = File.OpenText(System.Environment.GetFolderPath(Environment.SpecialFolder.Personal) + 302 | "/github" + book).ReadToEnd().Split(split); 303 | 304 | //three times data 305 | List list = new List(); 306 | for (int i = 0; i < 3; i++) 307 | { 308 | foreach (String str in tstmp) 309 | { 310 | list.Add(str); 311 | } 312 | } 313 | String[] ts = list.ToArray(); 314 | 315 | 316 | Engine engine = Engine.Instance; 317 | engine.Config(db.GetConfig()); 318 | //engine.maxSearchTime = 1000; 319 | 320 | AutoBox auto = db.Open(); 321 | 322 | var b = DateTime.Now; 323 | if (rebuild) 324 | { 325 | long rbcount = 0; 326 | 327 | Parallel.For(0, ts.Length, (i) => 328 | { 329 | if (notranCount < 1) 330 | { 331 | using (var box = auto.Cube()) 332 | { 333 | Interlocked.Add(ref rbcount, engine.indexText(box, i, ts[i], false)); 334 | box.Commit().Assert(); 335 | } 336 | } 337 | else 338 | { 339 | Interlocked.Add(ref rbcount, engine.indexTextNoTran(auto, notranCount, i, ts[i], false)); 340 | } 341 | }); 342 | 343 | Console.WriteLine("Index " + (DateTime.Now - b).TotalSeconds + " -" + rbcount); 344 | } 345 | 346 | b = DateTime.Now; 347 | int c = 0; 348 | for (int i = 0; i < 20; i++) 349 | { 350 | b = DateTime.Now; 351 | c = 0; 352 | using (var box = auto.Cube()) 353 | { 354 | foreach (KeyWord kw in engine.searchDistinct(box, strkw)) 355 | { 356 | c++; 357 | } 358 | } 359 | Console.WriteLine("DB: " + c + " , " + (DateTime.Now - b).TotalSeconds + "s"); 360 | } 361 | 362 | 363 | StringUtil sutil = StringUtil.Instance; 364 | for (int i = 0; i < ts.Length; i++) 365 | { 366 | ts[i] = ts[i].ToLower() + " "; 367 | ts[i] = " " + new String(sutil.clear(ts[i])) + " "; 368 | } 369 | 370 | strkw = strkw.ToLower(); 371 | String[] kws = strkw.Split(new char[] { ' ' }); 372 | String tmp_kws = null; 373 | for (int i = 0; i < kws.Length; i++) 374 | { 375 | if (kws[i].length() < 1) 376 | { 377 | kws[i] = null; 378 | continue; 379 | } 380 | if (tmp_kws == null) 381 | { 382 | if (kws[i].StartsWith("\"")) 383 | { 384 | tmp_kws = kws[i]; 385 | kws[i] = null; 386 | } 387 | } 388 | else if (tmp_kws != null) 389 | { 390 | tmp_kws += (" " + kws[i]); 391 | 392 | if (kws[i].EndsWith("\"")) 393 | { 394 | kws[i] = tmp_kws.substring(1, tmp_kws.length() - 1); 395 | tmp_kws = null; 396 | } 397 | else 398 | { 399 | kws[i] = null; 400 | } 401 | } 402 | } 403 | 404 | 405 | b = DateTime.Now; 406 | c = 0; 407 | int starti = 0; 408 | Test: 409 | while (starti < ts.Length) 410 | { 411 | int i = starti++; 412 | for (int j = 0; j < kws.Length; j++) 413 | { 414 | if (kws[j] == null) 415 | { 416 | continue; 417 | } 418 | int p = 0; 419 | Test_P: 420 | while (p >= 0) 421 | { 422 | p = ts[i].IndexOf(kws[j], p + 1); 423 | if (p < 0) 424 | { 425 | goto Test; 426 | } 427 | if (onlyPart(ts[i], kws[j], p)) 428 | { 429 | goto Test_P; 430 | } 431 | break; 432 | } 433 | } 434 | c++; 435 | 436 | } 437 | Console.WriteLine(strkw); 438 | Console.WriteLine("MEM: " + c + " , " + (DateTime.Now - b).TotalSeconds + "s -" + ts.Length); 439 | 440 | auto.GetDatabase().Dispose(); 441 | } 442 | 443 | private static bool onlyPart(String str, String wd, int p) 444 | { 445 | char last = wd[wd.Length - 1]; 446 | if (last > 256) 447 | { 448 | return false; 449 | } 450 | 451 | char pc = str[p + wd.length()]; 452 | if (pc >= 'a' && pc <= 'z') 453 | { 454 | return true; 455 | } 456 | if (pc == '-') 457 | { 458 | return true; 459 | } 460 | 461 | int bef = p; 462 | Test: 463 | while (bef > 0) 464 | { 465 | pc = str[bef - 1]; 466 | if (pc >= 'a' && pc <= 'z') 467 | { 468 | return true; 469 | } 470 | if (pc == '-') 471 | { 472 | bef--; 473 | goto Test; 474 | } 475 | break; 476 | } 477 | 478 | return false; 479 | } 480 | 481 | public class TA 482 | { 483 | public int a; 484 | public int b; 485 | 486 | public override string ToString() 487 | { 488 | return a + "-" + b; 489 | } 490 | } 491 | 492 | public static void test_db() 493 | { 494 | DB db = new DB(new byte[0]); 495 | db.GetConfig().EnsureTable("TA", "a", "b"); 496 | var auto = db.Open(); 497 | using (var box = auto.Cube()) 498 | { 499 | for (int i = 0; i < 2; i++) 500 | { 501 | for (int j = 0; j < 9; j++) 502 | { 503 | box["TA"].Insert(new TA { a = i, b = j }); 504 | } 505 | } 506 | box.Commit().Assert(); 507 | } 508 | 509 | foreach (var ta in auto.Select("from TA where a>= ? & a< ? & b >= ? & b("TextObject", "ID"); 17 | 18 | Engine engine = Engine.Instance; 19 | engine.Config(db.GetConfig()); 20 | 21 | AutoBox auto = db.Open(); 22 | 23 | { 24 | TextObject to = new TextObject(); 25 | to.ID = 1; 26 | to.Season = 2018; 27 | to.Group = 3; 28 | to.Time = new DateTime(2018, 3, 5); 29 | to.Keys = new string[] { "COOL", "FAST" }; 30 | to.KVDesc = new Dictionary { { "Name", "X-MAN" }, { "Speed", 3000 } }; 31 | auto.Insert("TextObject", to); 32 | engine.indexTextNoTran(auto, int.MaxValue, to.ID, to.ToString(), false); 33 | 34 | TextObject to2 = new TextObject(); 35 | to2.ID = 2; 36 | to2.Season = 2018; 37 | to2.Group = 4; 38 | to2.Time = new DateTime(2018, 3, 6); 39 | to2.Keys = new string[] { "Sharp" }; 40 | to2.KVDesc = new Dictionary { { "Speed", "gt4000" } }; 41 | auto.Insert("TextObject", to2); 42 | engine.indexTextNoTran(auto, int.MaxValue, to2.ID, to2.ToString(), false); 43 | } 44 | //SQL and 45 | //auto.Select ("from TextObject where Season==?"); 46 | //auto.Select ("from TextObject where Group==?"); 47 | //auto.Select ("from TextObject where Season==? & Group==?"); 48 | //auto.Select ("from TextObject where Time==?"); 49 | //auto.Select ("from TextObject where Season==? & Time==?"); 50 | 51 | //Full text search -and 52 | using (var box = auto.Cube()) 53 | { 54 | Console.WriteLine("Search: Season=2018"); 55 | String searchText = "SE-" + 2018; 56 | foreach (var kw in engine.searchDistinct(box, searchText, long.MaxValue, 200)) 57 | { 58 | Console.WriteLine(box["TextObject", kw.I].Select()); 59 | } 60 | 61 | Console.WriteLine("\r\nSearch: KVDesc={ \"Name\", \"X-MAN\" } , Keys={COOL}"); 62 | searchText = "KV-" + "Name" + "-" + "X-MAN" + " KE-" + "COOL"; 63 | foreach (var kw in engine.searchDistinct(box, searchText, long.MaxValue, 200)) 64 | { 65 | Console.WriteLine(box["TextObject", kw.I].Select()); 66 | } 67 | } 68 | 69 | Console.WriteLine(""); 70 | //Full text search -or 71 | using (var box = auto.Cube()) 72 | { 73 | HashSet ids = new HashSet(); 74 | Console.WriteLine("Search: Time=2018-3-6 OR KVDesc={ \"Name\", \"X-MAN\" } , Keys={COOL}"); 75 | 76 | String searchText = "TI-" + TextObject.DateTimeToStringShort(new DateTime(2018, 3, 6)); 77 | foreach (var kw in engine.searchDistinct(box, searchText, long.MaxValue, 200)) 78 | { 79 | ids.add(kw.I); 80 | } 81 | 82 | searchText = "KV-" + "Name" + "-" + "X-MAN" + " KE-" + "COOL"; 83 | foreach (var kw in engine.searchDistinct(box, searchText, long.MaxValue, 200)) 84 | { 85 | ids.add(kw.I); 86 | } 87 | 88 | long[] idslong = new long[ids.Count]; 89 | ids.CopyTo(idslong); 90 | Array.Sort(idslong); 91 | 92 | for (var i = idslong.Length - 1; i >= 0; i--) 93 | { 94 | Console.WriteLine(box["TextObject", idslong[i]].Select()); 95 | } 96 | } 97 | 98 | auto.GetDatabase().Dispose(); 99 | } 100 | } 101 | 102 | public class TextObject 103 | { 104 | public string Content 105 | { 106 | get; 107 | set; 108 | } 109 | 110 | public int Group 111 | { 112 | get; 113 | set; 114 | } 115 | 116 | public Dictionary KVDesc 117 | { 118 | get; 119 | set; 120 | } 121 | 122 | public string[] Keys 123 | { 124 | get; 125 | set; 126 | } 127 | 128 | public DateTime Time 129 | { 130 | get; 131 | set; 132 | } 133 | 134 | public int Season 135 | { 136 | get; 137 | set; 138 | } 139 | 140 | public long ID 141 | { 142 | get; 143 | set; 144 | } 145 | 146 | public override string ToString() 147 | { 148 | StringBuilder sb = new StringBuilder(); 149 | sb.append("ID-" + ID + " "); 150 | sb.append("SE-" + Season + " "); 151 | sb.append("GR-" + Group + " "); 152 | sb.append("TI-" + DateTimeToStringShort(Time) + " "); 153 | if (Keys != null) 154 | { 155 | foreach (String str in Keys) 156 | { 157 | sb.append("KE-" + str + " "); 158 | } 159 | } 160 | if (KVDesc != null) 161 | { 162 | foreach (var kv in KVDesc) 163 | { 164 | sb.append("KV-" + kv.Key + "-" + kv.Value + " "); 165 | } 166 | } 167 | sb.append(Content); 168 | return sb.ToString(); 169 | } 170 | 171 | public static String DateTimeToStringShort(DateTime dt) 172 | { 173 | return dt.Year + "-" + dt.Month + "-" + dt.Day; 174 | } 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /EngineConsole/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using FTServer; 3 | 4 | namespace EngineConsole 5 | { 6 | class Program 7 | { 8 | static void Main(string[] args) 9 | { 10 | //FrenchTest(); 11 | 12 | FTServer.EngineTest.test_main(); 13 | //FTServer.EngineTest.test_order(); 14 | //FTServer.ObjectSearch.test_main(); 15 | 16 | //FTServer.EngineTest.test_big_n(); 17 | //FTServer.EngineTest.test_big_e(); 18 | 19 | } 20 | 21 | static void FrenchTest() 22 | { 23 | String str = "l’étranger ls’étranger S’inscrire S'Étatà d'étranger wouldn't I'm l'Europe l’Europe"; 24 | Console.WriteLine(StringUtil.Instance.fromatFrenchInput(str)); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /FTServer/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": ".NET Core Launch (web)", 9 | "type": "coreclr", 10 | "request": "launch", 11 | "preLaunchTask": "build", 12 | "program": "${workspaceFolder}/bin/Debug/net8.0/FTServer.dll", 13 | "args": [], 14 | "cwd": "${workspaceFolder}", 15 | "stopAtEntry": false, 16 | "serverReadyAction": { 17 | //"action": "openExternally", 18 | //"pattern": "\\bNow listening on:\\s+(https?://\\S+)" 19 | }, 20 | "env": { 21 | "ASPNETCORE_ENVIRONMENT": "Development" 22 | }, 23 | "sourceFileMap": { 24 | "/Views": "${workspaceFolder}/Views" 25 | } 26 | } 27 | 28 | ] 29 | } -------------------------------------------------------------------------------- /FTServer/.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=733558 3 | // for the documentation about the tasks.json format 4 | "version": "2.0.0", 5 | "tasks": [ 6 | { 7 | "label": "build", 8 | "command": "dotnet", 9 | "type": "shell", 10 | "args": [ 11 | "build", 12 | // Ask dotnet build to generate full paths for file names. 13 | "/property:GenerateFullPaths=true", 14 | // Do not generate summary otherwise it leads to duplicate errors in Problems panel 15 | "/consoleloggerparameters:NoSummary" 16 | ], 17 | "group": "build", 18 | "presentation": { 19 | "reveal": "silent" 20 | }, 21 | "problemMatcher": "$msCompile" 22 | } 23 | ] 24 | } -------------------------------------------------------------------------------- /FTServer/Code/App.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using IBoxDB.LocalServer; 3 | 4 | namespace FTServer 5 | { 6 | 7 | public class App 8 | { 9 | internal readonly static bool IsAndroid = false; 10 | public static int HttpPort = 5066; 11 | 12 | //for Application 13 | public static AutoBox Item; 14 | 15 | //for New Index 16 | public static AutoBox Index; 17 | 18 | //for Readonly PageIndex 19 | public static readonly ReadonlyList Indices = new ReadonlyList(); 20 | 21 | public static void Log(String msg) 22 | { 23 | Console.WriteLine(msg); 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /FTServer/Code/ConcurrentLinkedDeque.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using IBoxDB.LocalServer; 5 | 6 | namespace FTServer 7 | { 8 | public class ConcurrentLinkedDeque where T : class 9 | { 10 | 11 | LinkedList list = new LinkedList(); 12 | 13 | public int size() 14 | { 15 | lock (this) 16 | { 17 | return list.Count; 18 | } 19 | } 20 | public T pollFirst() 21 | { 22 | lock (this) 23 | { 24 | if (list.Count > 0) 25 | { 26 | T f = list.First.Value; 27 | list.RemoveFirst(); 28 | return f; 29 | } 30 | return default(T); 31 | } 32 | } 33 | 34 | public void addFirst(T t) 35 | { 36 | lock (this) 37 | { 38 | list.AddFirst(t); 39 | } 40 | } 41 | 42 | public void addLast(T t) 43 | { 44 | lock (this) 45 | { 46 | list.AddLast(t); 47 | } 48 | } 49 | } 50 | 51 | } -------------------------------------------------------------------------------- /FTServer/Code/Config.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace FTServer 4 | { 5 | 6 | //Default for 4GB Setting 7 | public class Config 8 | { 9 | 10 | public static long mb(long len) 11 | { 12 | return 1024L * 1024L * len; 13 | } 14 | public static readonly long lowReadonlyCache = Config.mb(8); 15 | 16 | public static readonly long DSize = 1L; 17 | 18 | //only index description, not full text, faster indexing 19 | //if wanting more Pages, dotn't care the content, set it to True 20 | public static bool DescriptionOnly = false; 21 | 22 | public static long Index_CacheLength = mb(800L) / DSize; 23 | 24 | //this should set bigger than 500MB. 25 | //for DescriptionOnly, it can set bigger, because it might load Only 10% to Memory 26 | public static long SwitchToReadonlyIndexLength = mb(750L * (DescriptionOnly ? 3 : 1)) / DSize; 27 | 28 | //Readonly Cache after Switch One Database To Readonly 29 | public static long Readonly_CacheLength = SwitchToReadonlyIndexLength / 23L; 30 | 31 | //How Many Readonly Databases Having long Cache 32 | //Set 1000 MB Readonly Index Cache 33 | public static long Readonly_MaxDBCount = mb(1000) / Readonly_CacheLength / DSize; 34 | 35 | public static long ShortCacheLength = mb(32L * (DescriptionOnly ? 2 : 1)); 36 | 37 | //HTML Page Cache, this should set bigger, if having more memory. 38 | public static long ItemConfig_CacheLength = mb(256); 39 | public static int ItemConfig_SwapFileBuffer = (int)mb(20); 40 | 41 | //this should less than 1/2 MaxMemory 42 | public static long minCache() 43 | { 44 | return Index_CacheLength + Readonly_CacheLength * Readonly_MaxDBCount + ItemConfig_CacheLength 45 | + ItemConfig_SwapFileBuffer * 2; 46 | } 47 | 48 | } 49 | 50 | 51 | } 52 | -------------------------------------------------------------------------------- /FTServer/Code/EasyOR.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Text; 3 | 4 | 5 | namespace FTServer 6 | { 7 | // the easy way convert to OR search,by removing one word 8 | public class EasyOR 9 | { 10 | 11 | static String[] removedWords; 12 | 13 | static EasyOR() 14 | { 15 | removedWords = new String[] { "\"", " of " }; // , "的" 16 | } 17 | 18 | internal static ArrayList toOrCondition(String str) 19 | { 20 | if (str == null || str.length() == 0) 21 | { 22 | return new ArrayList(); 23 | } 24 | foreach (String s in removedWords) 25 | { 26 | str = str.Replace(s, " "); 27 | } 28 | char[] encs = StringUtil.Instance.clear(str); 29 | char[] cncs = Arrays.copyOf(encs, encs.Length); 30 | 31 | for (int i = 0; i < encs.Length; i++) 32 | { 33 | if (StringUtil.Instance.isWord(encs[i])) 34 | { 35 | 36 | } 37 | else 38 | { 39 | encs[i] = ' '; 40 | } 41 | } 42 | 43 | for (int i = 0; i < cncs.Length; i++) 44 | { 45 | if (StringUtil.Instance.isWord(cncs[i])) 46 | { 47 | cncs[i] = ' '; 48 | } 49 | } 50 | 51 | String en = compress(encs); 52 | String cn = compress(cncs); 53 | 54 | ArrayList result = new ArrayList(); 55 | if (en.length() > 0 && cn.length() > 0) 56 | { 57 | result.add(en); 58 | result.add(cn); 59 | } 60 | else if (en.length() > 0) 61 | { 62 | result.addAll(removeOneEN(en)); 63 | } 64 | else if (cn.contains(" ")) 65 | { 66 | result.addAll(removeOneEN(cn)); 67 | } 68 | else 69 | { 70 | result.addAll(removeOneCN(cn)); 71 | } 72 | 73 | return filter(result); 74 | } 75 | 76 | private static String compress(char[] cs) 77 | { 78 | StringBuilder r = new StringBuilder(); 79 | foreach (char c in cs) 80 | { 81 | if (r.length() > 0 && r.charAt(r.length() - 1) == ' ' && c == ' ') 82 | { 83 | continue; 84 | } 85 | r.append(c); 86 | } 87 | return r.toString().trim(); 88 | } 89 | 90 | private static ArrayList removeOneCN(String str) 91 | { 92 | ArrayList r = new ArrayList(); 93 | 94 | StringBuilder sb = new StringBuilder(); 95 | for (int i = 0; i < str.length() - 2; i += 2) 96 | { 97 | sb.append(str.substring(i, i + 2) + " "); 98 | } 99 | r.add(sb.toString().trim()); 100 | 101 | sb = new StringBuilder(); 102 | for (int i = str.length(); i > 2; i -= 2) 103 | { 104 | sb.append(str.substring(i - 2, i) + " "); 105 | } 106 | r.add(sb.toString().trim()); 107 | 108 | return r; 109 | } 110 | 111 | private static ArrayList removeOneEN(String str) 112 | { 113 | ArrayList r = new ArrayList(); 114 | String[] sps = str.split(" "); 115 | if (sps.Length <= 1) 116 | { 117 | return r; 118 | } 119 | else if (sps.Length == 2) 120 | { 121 | r.add(sps[0]); 122 | r.add(sps[1]); 123 | } 124 | else 125 | { 126 | for (int i = 0; i < sps.Length; i++) 127 | { 128 | StringBuilder sb = new StringBuilder(); 129 | for (int j = 0; j < sps.Length; j++) 130 | { 131 | if (i == j) 132 | { 133 | continue; 134 | } 135 | if (sps[j].length() < 2) 136 | { 137 | continue; 138 | } 139 | sb.append(" " + sps[j]); 140 | } 141 | r.add(sb.toString().trim()); 142 | } 143 | } 144 | return r; 145 | } 146 | 147 | 148 | 149 | private static ArrayList filter(ArrayList src) 150 | { 151 | ArrayList r = new ArrayList(); 152 | foreach (String s in src) 153 | { 154 | if (s != null) 155 | { 156 | String s2 = s.trim(); 157 | if (s2.length() > 1 && (!r.contains(s2))) 158 | { 159 | r.add(s2); 160 | } 161 | } 162 | } 163 | return r; 164 | } 165 | private static String link(ArrayList aas) 166 | { 167 | StringBuilder sb = new StringBuilder(); 168 | foreach (String s in aas) 169 | { 170 | sb.append(s + " "); 171 | } 172 | return sb.toString().trim(); 173 | } 174 | 175 | } 176 | 177 | } 178 | 179 | -------------------------------------------------------------------------------- /FTServer/Code/FCSharpBridge.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using System.Collections.Concurrent; 5 | using System.Threading; 6 | 7 | using IBoxDB.LocalServer; 8 | 9 | 10 | namespace FTServer 11 | { 12 | 13 | //Ported from 14 | internal static class CSharpBridge 15 | { 16 | public static String toString(this T o) 17 | { 18 | return o.ToString(); 19 | } 20 | public static bool equals(this T o, T o2) 21 | { 22 | return o.Equals(o2); 23 | } 24 | public static StringBuilder append(this StringBuilder self, char str) 25 | { 26 | return self.Append(str); 27 | } 28 | public static StringBuilder append(this StringBuilder self, string str) 29 | { 30 | return self.Append(str); 31 | } 32 | 33 | public static int length(this StringBuilder self) 34 | { 35 | return self.Length; 36 | } 37 | 38 | public static char charAt(this StringBuilder self, int pos) 39 | { 40 | return self[pos]; 41 | } 42 | 43 | public static void insert(this StringBuilder self, int pos, char c) 44 | { 45 | self.Insert(pos, c); 46 | } 47 | public static bool isEmpty(this String self) 48 | { 49 | return self.Length == 0; 50 | } 51 | public static bool contains(this String self, string s) 52 | { 53 | return self.Contains(s); 54 | } 55 | public static int length(this String self) 56 | { 57 | return self.Length; 58 | } 59 | 60 | public static string[] split(this String self, String s) 61 | { 62 | return self.Split(s); 63 | } 64 | 65 | public static char[] toCharArray(this String self) 66 | { 67 | return self.ToCharArray(); 68 | } 69 | 70 | public static string toLowerCase(this String self) 71 | { 72 | return self.ToLower(); 73 | } 74 | 75 | public static string substring(this String self, int start, int end) 76 | { 77 | return self.Substring(start, end - start); 78 | } 79 | public static string substring(this String self, int start) 80 | { 81 | return self.Substring(start); 82 | } 83 | public static string trim(this string self) 84 | { 85 | return self.Trim(); 86 | } 87 | public static char charAt(this string self, int index) 88 | { 89 | return self[index]; 90 | } 91 | public static int lastIndexOf(this string self, char c, int index) 92 | { 93 | return self.LastIndexOf(c, index); 94 | } 95 | public static int nextInt(this Random self, int value) 96 | { 97 | return self.Next(value); 98 | } 99 | 100 | public static bool add(this HashSet self, T v) 101 | { 102 | return self.Add(v); 103 | } 104 | public static bool remove(this HashSet self, T v) 105 | { 106 | return self.Remove(v); 107 | } 108 | public static bool contains(this HashSet self, T v) 109 | { 110 | return self.Contains(v); 111 | } 112 | public static int size(this ICollection self) 113 | { 114 | return self.Count; 115 | } 116 | public static void remove(this ArrayList self, int pos) 117 | { 118 | self.RemoveAt(pos); 119 | } 120 | /* 121 | public static int size(this ArrayList self) 122 | { 123 | return self.Count; 124 | } 125 | */ 126 | public static T[] toArray(this ArrayList self) 127 | { 128 | return self.ToArray(); 129 | } 130 | 131 | public static T get(this ArrayList self, int pos) 132 | { 133 | return self[pos]; 134 | } 135 | 136 | public static void add(this ConcurrentQueue self, T obj) 137 | { 138 | self.Enqueue(obj); 139 | } 140 | public static void remove(this ConcurrentQueue self) 141 | { 142 | T o; 143 | self.TryDequeue(out o); 144 | } 145 | public static int size(this ConcurrentQueue self) 146 | { 147 | return self.Count; 148 | } 149 | 150 | } 151 | 152 | internal class ArrayList : List 153 | { 154 | public bool isEmpty() 155 | { 156 | return this.Count == 0; 157 | } 158 | 159 | public void add(T t) 160 | { 161 | this.Add(t); 162 | } 163 | public void addAll(IEnumerable t) 164 | { 165 | base.AddRange(t); 166 | } 167 | public void add(int index, T p) 168 | { 169 | this.Insert(index, p); 170 | } 171 | public bool contains(T t) 172 | { 173 | return this.Contains(t); 174 | } 175 | } 176 | 177 | public class LinkedHashSet : SortedSet 178 | { 179 | public int size() 180 | { 181 | return base.Count; 182 | } 183 | 184 | public void add(T t) 185 | { 186 | base.Add(t); 187 | } 188 | } 189 | 190 | internal class EngineIterator : Iterator 191 | { 192 | } 193 | 194 | internal class Iterator : IEnumerator 195 | { 196 | public delegate bool MoveNextDelegate(); 197 | 198 | public delegate T CurrentDelegate(); 199 | 200 | public MoveNextDelegate hasNext; 201 | public CurrentDelegate next; 202 | 203 | public bool MoveNext() 204 | { 205 | return hasNext(); 206 | } 207 | 208 | public T Current 209 | { 210 | get 211 | { 212 | return next(); 213 | } 214 | } 215 | 216 | void System.Collections.IEnumerator.Reset() 217 | { 218 | 219 | } 220 | 221 | object System.Collections.IEnumerator.Current 222 | { 223 | get 224 | { 225 | return this.Current; 226 | } 227 | } 228 | 229 | void IDisposable.Dispose() 230 | { 231 | 232 | } 233 | } 234 | 235 | internal class Iterable : IEnumerable 236 | { 237 | 238 | 239 | public IEnumerator GetEnumerator() 240 | { 241 | return iterator; 242 | } 243 | 244 | System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() 245 | { 246 | return iterator; 247 | } 248 | 249 | public EngineIterator iterator; 250 | } 251 | 252 | internal class Arrays 253 | { 254 | public static T[] copyOf(T[] kws, int len) 255 | { 256 | T[] condition = new T[len]; 257 | Array.Copy(kws, 0, condition, 0, Math.Min(kws.Length, condition.Length)); 258 | return condition; 259 | } 260 | } 261 | 262 | } -------------------------------------------------------------------------------- /FTServer/Code/FKeyWord.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using IBoxDB.LocalServer; 3 | 4 | namespace FTServer 5 | { 6 | 7 | public abstract class KeyWord 8 | { 9 | 10 | public readonly static int MAX_WORD_LENGTH = 16; 11 | 12 | public static void config(DatabaseConfig c) 13 | { 14 | // English Language or Word (max=16) 15 | c.EnsureTable("/E", "K(" + MAX_WORD_LENGTH + ")", "I", "P"); 16 | 17 | // Non-English Language or Character 18 | c.EnsureTable("/N", "K", "I", "P"); 19 | 20 | } 21 | public abstract int size(); 22 | //Position 23 | public int P; 24 | 25 | //Document ID 26 | public long I; 27 | 28 | [NotColumn] 29 | public KeyWord previous; 30 | [NotColumn] 31 | public bool isLinked; 32 | [NotColumn] 33 | public bool isLinkedEnd; 34 | 35 | public String ToFullString() 36 | { 37 | return (previous != null ? previous.ToFullString() + " -> " : "") + ToString(); 38 | } 39 | 40 | 41 | } 42 | 43 | public sealed class KeyWordE : KeyWord 44 | { 45 | //Key Word 46 | public String K; 47 | 48 | 49 | public void keyWord(String k) 50 | { 51 | if (k.length() > KeyWord.MAX_WORD_LENGTH) 52 | { 53 | return; 54 | } 55 | K = k; 56 | } 57 | 58 | public override int size() 59 | { 60 | return K.length(); 61 | } 62 | 63 | public override String ToString() 64 | { 65 | return K + " Pos=" + P + ", ID=" + I + " E"; 66 | } 67 | } 68 | 69 | public sealed class KeyWordN : KeyWord 70 | { 71 | //Key Word 72 | public long K; 73 | 74 | 75 | public override int size() 76 | { 77 | if ((K & CMASK) != 0L) 78 | { 79 | return 3; 80 | } 81 | if ((K & (CMASK << 16)) != 0L) 82 | { 83 | return 2; 84 | } 85 | return 1; 86 | } 87 | 88 | const long CMASK = 0xFFFF; 89 | 90 | private static String KtoString(long k) 91 | { 92 | char c0 = (char)((k & (CMASK << 32)) >> 32); 93 | char c1 = (char)((k & (CMASK << 16)) >> 16); 94 | char c2 = (char)(k & CMASK); 95 | 96 | if (c2 != 0) 97 | { 98 | return new String(new char[] { c0, c1, c2 }); 99 | } 100 | if (c1 != 0) 101 | { 102 | return new String(new char[] { c0, c1 }); 103 | } 104 | return c0.ToString(); 105 | } 106 | 107 | public void longKeyWord(char c0, char c1, char c2) 108 | { 109 | long k = (0L | c0) << 32; 110 | if (c1 != 0) 111 | { 112 | k |= ((0L | c1) << 16); 113 | if (c2 != 0) 114 | { 115 | k |= (0L | c2); 116 | } 117 | } 118 | K = k; 119 | } 120 | 121 | public String toKString() 122 | { 123 | return KtoString(K); 124 | } 125 | 126 | public override String ToString() 127 | { 128 | return toKString() + " Pos=" + P + ", ID=" + I + " N"; 129 | } 130 | } 131 | } -------------------------------------------------------------------------------- /FTServer/Code/FLang.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace FTServer 5 | { 6 | //Language 7 | public class Lang 8 | { 9 | public static Lang Instance = new Lang(); 10 | 11 | HashSet set; 12 | 13 | private Lang() 14 | { 15 | 16 | String s = "!\"@$%&'()*+,./:;<=>?[\\]^_`{|}~\r\n"; //@- 17 | s += "， 　，《。》、？；：‘’“”【｛】｝——=+、｜·～！￥%……&*（）"; //@-# 18 | s += "｀～！＠￥％……—×（）——＋－＝【】｛｝：；’＇”＂，．／＜＞？’‘”“";//＃ 19 | s += "� ★☆,。？,　！"; 20 | s += "©»¥「」"; 21 | s += "[¡, !, \", ', (, ), -, °, :, ;, ?]-\"#"; 22 | 23 | set = new HashSet(); 24 | foreach (char c in s.toCharArray()) 25 | { 26 | if (isWord(c)) 27 | { 28 | continue; 29 | } 30 | set.add(c); 31 | } 32 | 33 | set.add(' '); 34 | set.add('　'); 35 | 36 | set.add((char)0); 37 | set.add((char)0x09); 38 | set.add((char)8203); 39 | // http://www.unicode-symbol.com/block/Punctuation.html 40 | for (int i = 0x2000; i <= 0x206F; i++) 41 | { 42 | set.add((char)i); 43 | } 44 | set.add((char)0x0E00);//Thai 45 | 46 | //https://unicode-table.com/en/blocks/arabic/ 47 | //Punctuation Arabic 48 | set.add((char)0x0609); 49 | set.add((char)0x060A); 50 | set.add((char)0x060B); 51 | set.add((char)0x060C); 52 | set.add((char)0x060D); 53 | 54 | set.add((char)0x061B); 55 | set.add((char)0x061E); 56 | set.add((char)0x061F); 57 | 58 | set.add((char)0x066A); 59 | set.add((char)0x066B); 60 | set.add((char)0x066C); 61 | set.add((char)0x066D); 62 | 63 | set.add((char)0x06D4); 64 | 65 | //https://unicode-table.com/en/blocks/hebrew/ 66 | set.add((char)0x05BE); 67 | set.add((char)0x05C0); 68 | set.add((char)0x05C3); 69 | 70 | set.add((char)0x05C6); 71 | 72 | set.add((char)0x05F3); 73 | set.add((char)0x05F4); 74 | 75 | //Devanagari 76 | set.add((char)0x0964); 77 | set.add((char)0x0965); 78 | 79 | //Katakana 80 | set.add((char)0x30A0); 81 | set.add((char)0x30FB); 82 | set.add((char)0x30FC); 83 | } 84 | 85 | public bool isPunctuation(char c) 86 | { 87 | return set.contains(c); 88 | } 89 | public bool isWord(char c) 90 | { 91 | // https://unicode-table.com/en/blocks/basic-latin/ 92 | // 0-9 93 | if (c >= 0x30 && c <= 0x39) 94 | { 95 | return true; 96 | } 97 | // A - Z 98 | if (c >= 0x41 && c <= 0x5A) 99 | { 100 | return true; 101 | } 102 | // a - z 103 | if (c >= 0x61 && c <= 0x7A) 104 | { 105 | return true; 106 | } 107 | 108 | // https://unicode-table.com/en/blocks/latin-1-supplement/ 109 | if (c >= 0xC0 && c <= 0xFF) 110 | { 111 | return true; 112 | } 113 | 114 | // https://unicode-table.com/en/blocks/latin-extended-a/ 115 | if (c >= 0x0100 && c <= 0x017F) 116 | { 117 | return true; 118 | } 119 | 120 | // https://unicode-table.com/en/blocks/latin-extended-b/ 121 | if (c >= 0x0180 && c <= 0x024F) 122 | { 123 | return true; 124 | } 125 | 126 | // https://unicode-table.com/en/blocks/ipa-extensions/ 127 | if (c >= 0x0250 && c <= 0x02AF) 128 | { 129 | return true; 130 | } 131 | 132 | // https://unicode-table.com/en/blocks/combining-diacritical-marks/ 133 | if (c >= 0x0300 && c <= 0x036F) 134 | { 135 | return true; 136 | } 137 | 138 | // https://unicode-table.com/en/blocks/greek-coptic/ 139 | if (c >= 0x0370 && c <= 0x03FF) 140 | { 141 | return true; 142 | } 143 | 144 | //Russian 145 | // https://unicode-table.com/en/blocks/cyrillic/ 146 | // https://unicode-table.com/en/blocks/cyrillic-supplement/ 147 | if (c >= 0x0400 && c <= 0x052F) 148 | { 149 | return true; 150 | } 151 | 152 | // https://unicode-table.com/en/blocks/armenian/ 153 | if (c >= 0x0530 && c <= 0x058F) 154 | { 155 | return true; 156 | } 157 | 158 | if (isWordRight2Left(c)) 159 | { 160 | return true; 161 | } 162 | 163 | // https://unicode-table.com/en/blocks/devanagari/ 164 | // India 165 | if (c >= 0x0900 && c <= 0x097F) 166 | { 167 | return true; 168 | } 169 | 170 | //Korean 171 | // https://unicode-table.com/en/blocks/hangul-jamo/ 172 | if (c >= 0x1100 && c <= 0x11FF) 173 | { 174 | return true; 175 | } 176 | //https://unicode-table.com/en/blocks/hangul-jamo-extended-b/ 177 | if (c >= 0xD7B0 && c <= 0xD7FF) 178 | { 179 | return true; 180 | } 181 | //https://unicode-table.com/en/blocks/hangul-syllables/ 182 | if (c >= 0xAC00 && c <= 0xD7AF) 183 | { 184 | return true; 185 | } 186 | 187 | //Japanese 188 | /* 189 | if (c >= 0x3040 && c <= 0x312F) 190 | { 191 | return true; 192 | } 193 | */ 194 | 195 | 196 | // https://unicode-table.com/en/blocks/latin-extended-additional/ 197 | if (c >= 0x1E00 && c <= 0x1EFF) 198 | { 199 | return true; 200 | } 201 | // https://unicode-table.com/en/blocks/greek-extended/ 202 | if (c >= 0x1F00 && c <= 0x1FFF) 203 | { 204 | return true; 205 | } 206 | 207 | //special 208 | return c == '-' || c == '#'; 209 | } 210 | 211 | private bool isWordRight2Left(char c) 212 | { 213 | // https://unicode-table.com/en/blocks/hebrew/ 214 | // https://www.compart.com/en/unicode/block/U+0590 215 | if (c >= 0x0590 && c <= 0x05FF) 216 | { 217 | return true; 218 | } 219 | // https://unicode-table.com/en/blocks/arabic/ 220 | // https://www.compart.com/en/unicode/bidiclass/AL 221 | if (c >= 0x0600 && c <= 0x06FF) 222 | { 223 | return true; 224 | } 225 | 226 | // https://unicode-table.com/en/blocks/arabic-supplement/ 227 | if (c >= 0x0750 && c <= 0x077F) 228 | { 229 | return true; 230 | } 231 | // https://unicode-table.com/en/blocks/arabic-extended-a/ 232 | if (c >= 0x08A0 && c <= 0x08FF) 233 | { 234 | return true; 235 | } 236 | 237 | return false; 238 | } 239 | 240 | 241 | } 242 | } -------------------------------------------------------------------------------- /FTServer/Code/FStringUtil.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Text; 3 | using System.Text.RegularExpressions; 4 | 5 | namespace FTServer 6 | { 7 | class StringUtil 8 | { 9 | public static StringUtil Instance = new StringUtil(); 10 | private StringUtil() 11 | { 12 | 13 | } 14 | public bool isWord(char c) 15 | { 16 | return Lang.Instance.isWord(c); 17 | } 18 | public char[] clear(String str) 19 | { 20 | char[] cs = (str + " ").toLowerCase().toCharArray(); 21 | for (int i = 0; i < cs.Length; i++) 22 | { 23 | if (cs[i] == '"') 24 | { 25 | continue; 26 | } 27 | if (Lang.Instance.isPunctuation(cs[i])) 28 | { 29 | cs[i] = ' '; 30 | } 31 | } 32 | return cs; 33 | } 34 | 35 | public ArrayList fromString(long id, char[] str, bool forIndex) 36 | { 37 | 38 | ArrayList kws = new ArrayList(); 39 | 40 | KeyWordE k = null; 41 | int linkedCount = 0; 42 | int lastNPos = -2; 43 | for (int i = 0; i < str.Length; i++) 44 | { 45 | char c = str[i]; 46 | if (c == ' ') 47 | { 48 | if (k != null) 49 | { 50 | kws.add(k); 51 | } 52 | k = null; 53 | 54 | } 55 | else if (c == '"') 56 | { 57 | if (k != null) 58 | { 59 | kws.add(k); 60 | } 61 | k = null; 62 | 63 | if (linkedCount > 0) 64 | { 65 | linkedCount = 0; 66 | setLinkEnd(kws); 67 | } 68 | else 69 | { 70 | linkedCount = 1; 71 | } 72 | } 73 | else if (isWord(c)) 74 | { 75 | if (k == null && c != '-' && c != '#') 76 | { 77 | k = new KeyWordE(); 78 | k.I = id; 79 | k.keyWord(""); 80 | k.P = i; 81 | if (linkedCount > 0) 82 | { 83 | linkedCount++; 84 | } 85 | if (linkedCount > 2) 86 | { 87 | k.isLinked = true; 88 | } 89 | } 90 | if (k != null) 91 | { 92 | k.keyWord(k.K + c.ToString()); 93 | } 94 | } 95 | else 96 | { 97 | if (k != null) 98 | { 99 | kws.add(k); 100 | } 101 | k = null; 102 | 103 | KeyWordN n = new KeyWordN(); 104 | n.I = id; 105 | n.P = i; 106 | n.longKeyWord(c, (char)0, (char)0); 107 | n.isLinked = i == (lastNPos + 1); 108 | kws.add(n); 109 | 110 | char c1 = str[i + 1]; 111 | if ((c1 != ' ' && c1 != '"') && (!isWord(c1))) 112 | { 113 | n = new KeyWordN(); 114 | n.I = id; 115 | n.P = i; 116 | n.longKeyWord(c, c1, (char)0); 117 | n.isLinked = i == (lastNPos + 1); 118 | kws.add(n); 119 | if (!forIndex) 120 | { 121 | kws.remove(kws.size() - 2); 122 | i++; 123 | } 124 | } 125 | 126 | if (c1 == ' ' || c1 == '"') 127 | { 128 | setLinkEnd(kws); 129 | } 130 | 131 | lastNPos = i; 132 | 133 | } 134 | } 135 | setLinkEnd(kws); 136 | return kws; 137 | } 138 | 139 | private void setLinkEnd(ArrayList kws) 140 | { 141 | if (kws.size() > 1) 142 | { 143 | KeyWord last = kws.get(kws.size() - 1); 144 | if (last.isLinked) 145 | { 146 | last.isLinkedEnd = true; 147 | } 148 | } 149 | } 150 | 151 | public String getDesc(String str, KeyWord kw, int length) 152 | { 153 | ArrayList list = new ArrayList(); 154 | while (kw != null) 155 | { 156 | list.add(kw); 157 | kw = kw.previous; 158 | } 159 | 160 | KeyWord[] ps = list.toArray(); 161 | Array.Sort(ps, (KeyWord o1, KeyWord o2) => 162 | { 163 | return o1.P - o2.P; 164 | }); 165 | 166 | 167 | int start = -1; 168 | int end = -1; 169 | StringBuilder sb = new StringBuilder(); 170 | for (int i = 0; i < ps.Length; i++) 171 | { 172 | int len = ps[i].size(); 173 | 174 | start = ps[i].P; 175 | if ((start + len) <= end) 176 | { 177 | continue; 178 | } 179 | if (start >= str.length()) 180 | { 181 | continue; 182 | } 183 | 184 | end = start + length; 185 | if (end > str.length()) 186 | { 187 | end = str.length(); 188 | } 189 | sb.append(str.substring(start, end)) 190 | .append("... "); 191 | } 192 | return sb.ToString(); 193 | 194 | } 195 | 196 | public String fromatFrenchInput(String str) 197 | { 198 | if (str == null) 199 | { 200 | return ""; 201 | } 202 | if (str.contains("\"")) 203 | { 204 | return str; 205 | } 206 | 207 | Regex p = new Regex("\\s(\\w+)([’'])(\\w+)"); 208 | 209 | return p.Replace(" " + str.trim() + " ", " \"$1$2$3\"").Trim(); 210 | 211 | } 212 | } 213 | } -------------------------------------------------------------------------------- /FTServer/Code/FTSEngine.cs: -------------------------------------------------------------------------------- 1 | /* iBoxDB FTServer Bruce Yang CL-N */ 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Threading; 5 | 6 | using IBoxDB.LocalServer; 7 | 8 | namespace FTServer 9 | { 10 | public class Engine 11 | { 12 | public readonly static Engine Instance = new Engine(); 13 | public static long KeyWordMaxScan = long.MaxValue; 14 | 15 | private Engine() 16 | { 17 | 18 | } 19 | public void Config(DatabaseConfig config) 20 | { 21 | KeyWord.config(config); 22 | } 23 | 24 | 25 | public long indexText(IBox box, long id, String str, bool isRemove, ThreadStart delay = null) 26 | { 27 | if (id == -1) 28 | { 29 | return -1; 30 | } 31 | long itCount = 0; 32 | char[] cs = StringUtil.Instance.clear(str); 33 | List map = StringUtil.Instance.fromString(id, cs, true); 34 | 35 | 36 | foreach (KeyWord kw in map) 37 | { 38 | delay?.Invoke(); 39 | insertToBox(box, kw, isRemove); 40 | itCount++; 41 | } 42 | return itCount; 43 | } 44 | 45 | public long indexTextNoTran(AutoBox auto, int commitCount, long id, String str, bool isRemove) 46 | { 47 | if (id == -1) 48 | { 49 | return -1; 50 | } 51 | long itCount = 0; 52 | char[] cs = StringUtil.Instance.clear(str); 53 | List map = StringUtil.Instance.fromString(id, cs, true); 54 | 55 | 56 | IBox box = null; 57 | int ccount = 0; 58 | foreach (KeyWord kw in map) 59 | { 60 | if (box == null) 61 | { 62 | box = auto.Cube(); 63 | ccount = commitCount; 64 | } 65 | insertToBox(box, kw, isRemove); 66 | itCount++; 67 | if (--ccount < 1) 68 | { 69 | box.Commit().Assert(); 70 | box = null; 71 | } 72 | } 73 | if (box != null) 74 | { 75 | box.Commit().Assert(); 76 | } 77 | return itCount; 78 | } 79 | 80 | static void insertToBox(IBox box, KeyWord kw, bool isRemove) 81 | { 82 | Binder binder; 83 | if (kw is KeyWordE) 84 | { 85 | binder = box["/E", ((KeyWordE)kw).K, kw.I, kw.P]; 86 | } 87 | else 88 | { 89 | binder = box["/N", ((KeyWordN)kw).K, kw.I, kw.P]; 90 | } 91 | if (isRemove) 92 | { 93 | binder.Delete(); 94 | } 95 | else 96 | { 97 | if (binder.TableName == "/E") 98 | { 99 | binder.Insert((KeyWordE)kw); 100 | } 101 | else 102 | { 103 | binder.Insert((KeyWordN)kw); 104 | } 105 | } 106 | } 107 | 108 | private Random ran = new Random(); 109 | 110 | public LinkedHashSet discoverEN(IBox box, 111 | char efrom, char eto, int elength) 112 | { 113 | LinkedHashSet list = new LinkedHashSet(); 114 | if (elength > 0) 115 | { 116 | int len = ran.nextInt(KeyWord.MAX_WORD_LENGTH) + 1; 117 | char[] cs = new char[len]; 118 | for (int i = 0; i < cs.Length; i++) 119 | { 120 | cs[i] = (char)(ran.nextInt(eto - efrom) + efrom); 121 | } 122 | KeyWordE kw = new KeyWordE(); 123 | kw.keyWord(new String(cs)); 124 | foreach (KeyWordE tkw in lessMatch(box, kw)) 125 | { 126 | String str = tkw.K; 127 | if (str.charAt(0) < efrom) 128 | { 129 | break; 130 | } 131 | int c = list.size(); 132 | list.add(str); 133 | if (list.size() > c) 134 | { 135 | elength--; 136 | if (elength <= 0) 137 | { 138 | break; 139 | } 140 | } 141 | } 142 | } 143 | return list; 144 | } 145 | 146 | public LinkedHashSet discoverCN(IBox box, 147 | char nfrom, char nto, int nlength) 148 | { 149 | 150 | LinkedHashSet list = new LinkedHashSet(); 151 | if (nlength > 0) 152 | { 153 | char[] cs = new char[2]; 154 | for (int i = 0; i < cs.Length; i++) 155 | { 156 | cs[i] = (char)(ran.nextInt(nto - nfrom) + nfrom); 157 | } 158 | KeyWordN kw = new KeyWordN(); 159 | kw.longKeyWord(cs[0], cs[1], (char)0); 160 | foreach (KeyWord tkw in lessMatch(box, kw)) 161 | { 162 | String str = ((KeyWordN)tkw).toKString(); 163 | if (str.charAt(0) < nfrom) 164 | { 165 | break; 166 | } 167 | int c = list.size(); 168 | list.add(str); 169 | if (list.size() > c) 170 | { 171 | nlength--; 172 | if (nlength <= 0) 173 | { 174 | break; 175 | } 176 | } 177 | } 178 | } 179 | return list; 180 | } 181 | 182 | 183 | public IEnumerable searchDistinct(IBox box, String str) 184 | { 185 | return searchDistinct(box, str, long.MaxValue, long.MaxValue); 186 | } 187 | // startId -> descending order 188 | public IEnumerable searchDistinct(IBox box, String str, long startId, long len) 189 | { 190 | long c_id = -1; 191 | foreach (KeyWord kw in search(box, str, startId)) 192 | { 193 | if (len < 1) 194 | { 195 | break; 196 | } 197 | if (kw.I == c_id) 198 | { 199 | continue; 200 | } 201 | c_id = kw.I; 202 | len--; 203 | yield return kw; 204 | } 205 | } 206 | 207 | public String getDesc(String str, KeyWord kw, int length) 208 | { 209 | return StringUtil.Instance.getDesc(str, kw, length); 210 | } 211 | 212 | public IEnumerable search(IBox box, String str) 213 | { 214 | return search(box, str, long.MaxValue); 215 | } 216 | 217 | public IEnumerable search(IBox box, String str, long startId) 218 | { 219 | if (startId < 0) 220 | { 221 | return new ArrayList(); 222 | } 223 | char[] cs = StringUtil.Instance.clear(str); 224 | ArrayList map = StringUtil.Instance.fromString(-1, cs, false); 225 | 226 | if (map.size() > KeyWord.MAX_WORD_LENGTH || map.isEmpty()) 227 | { 228 | return new ArrayList(); 229 | } 230 | 231 | MaxID maxId = new MaxID(); 232 | maxId.id = startId; 233 | maxId.jumpTime = 0; 234 | 235 | IEnumerator cd = search(box, map.ToArray(), maxId).GetEnumerator(); 236 | return new Iterable() 237 | { 238 | 239 | iterator = new EngineIterator() 240 | { 241 | hasNext = () => 242 | { 243 | if (cd.MoveNext()) 244 | { 245 | maxId.jumpTime = 0; 246 | return true; 247 | } 248 | return false; 249 | }, 250 | next = () => 251 | { 252 | return cd.Current; 253 | } 254 | } 255 | }; 256 | } 257 | 258 | private IEnumerable search(IBox box, KeyWord[] kws, MaxID maxId) 259 | { 260 | 261 | if (kws.Length == 1) 262 | { 263 | return search(box, kws[0], (KeyWord)null, maxId); 264 | } 265 | 266 | return search(box, kws[kws.Length - 1], 267 | search(box, Arrays.copyOf(kws, kws.Length - 1), maxId), 268 | maxId); 269 | } 270 | 271 | private IEnumerable search(IBox box, KeyWord nw, 272 | IEnumerable condition, MaxID maxId) 273 | { 274 | IEnumerator cd = condition.GetEnumerator(); 275 | 276 | IEnumerator r1 = null; 277 | 278 | KeyWord r1_con = null; 279 | long r1_id = -1; 280 | 281 | 282 | return new Iterable() 283 | { 284 | 285 | iterator = new EngineIterator() 286 | { 287 | 288 | 289 | hasNext = () => 290 | { 291 | if (r1 != null && r1.MoveNext()) 292 | { 293 | return true; 294 | } 295 | while (cd.MoveNext()) 296 | { 297 | r1_con = cd.Current; 298 | 299 | if (r1_id == r1_con.I) 300 | { 301 | continue; 302 | } 303 | if (!nw.isLinked) 304 | { 305 | r1_id = r1_con.I; 306 | } 307 | 308 | r1 = search(box, nw, r1_con, maxId).GetEnumerator(); 309 | if (r1.MoveNext()) 310 | { 311 | return true; 312 | } 313 | 314 | } 315 | return false; 316 | }, 317 | 318 | next = () => 319 | { 320 | KeyWord k = r1.Current; 321 | k.previous = r1_con; 322 | return k; 323 | } 324 | } 325 | 326 | }; 327 | 328 | } 329 | 330 | private static IEnumerable search(IBox box, 331 | KeyWord kw, KeyWord con, MaxID maxId) 332 | { 333 | 334 | 335 | if (kw is KeyWordE && con is KeyWordE) 336 | { 337 | if (((KeyWordE)kw).K.equals(((KeyWordE)con).K)) 338 | { 339 | maxId.id = -1; 340 | return new List(); 341 | } 342 | } 343 | if (kw is KeyWordN && con is KeyWordN) 344 | { 345 | if (((KeyWordN)kw).K == ((KeyWordN)con).K) 346 | { 347 | maxId.id = -1; 348 | return new List(); 349 | } 350 | } 351 | 352 | String ql = kw is KeyWordE 353 | ? "from /E where K==? & I<=?" 354 | : "from /N where K==? & I<=?"; 355 | 356 | 357 | int linkPos = kw.isLinked ? (con.P + con.size() 358 | + (kw is KeyWordE ? 1 : 0)) : -1; 359 | 360 | long currentMaxId = long.MaxValue; 361 | KeyWord cache = null; 362 | IEnumerator iter = null; 363 | bool isLinkEndMet = false; 364 | 365 | return new Iterable() 366 | { 367 | iterator = new EngineIterator() 368 | { 369 | 370 | hasNext = () => 371 | { 372 | if (maxId.id == -1) 373 | { 374 | return false; 375 | } 376 | 377 | if (iter == null || currentMaxId > (maxId.id)) 378 | { 379 | currentMaxId = maxId.id; 380 | iter = kw is KeyWordE ? 381 | (IEnumerator)box.Scale(ql, ((KeyWordE)kw).K, maxId.id).GetEnumerator() : 382 | box.Scale(ql, ((KeyWordN)kw).K, maxId.id).GetEnumerator(); 383 | } 384 | 385 | while (iter.MoveNext()) 386 | { 387 | 388 | cache = iter.Current; 389 | 390 | maxId.id = cache.I; 391 | maxId.jumpTime++; 392 | if (maxId.jumpTime > Engine.KeyWordMaxScan) 393 | { 394 | break; 395 | } 396 | 397 | currentMaxId = maxId.id; 398 | if (con != null && con.I != maxId.id) 399 | { 400 | return false; 401 | } 402 | 403 | if (isLinkEndMet) 404 | { 405 | continue; 406 | } 407 | 408 | if (linkPos == -1) 409 | { 410 | return true; 411 | } 412 | 413 | int cpos = cache.P; 414 | if (cpos > linkPos) 415 | { 416 | continue; 417 | } 418 | if (cpos == linkPos) 419 | { 420 | if (kw.isLinkedEnd) 421 | { 422 | isLinkEndMet = true; 423 | } 424 | return true; 425 | } 426 | return false; 427 | } 428 | 429 | maxId.id = -1; 430 | return false; 431 | 432 | }, 433 | 434 | next = () => 435 | { 436 | return cache; 437 | } 438 | 439 | } 440 | }; 441 | 442 | 443 | } 444 | 445 | private static IEnumerable lessMatch(IBox box, KeyWord kw) 446 | { 447 | if (kw is KeyWordE) 448 | { 449 | return box.Scale("from /E where K<=? limit 0, 50", ((KeyWordE)kw).K); 450 | 451 | } 452 | else 453 | { 454 | return box.Scale("from /N where K<=? limit 0, 50", ((KeyWordN)kw).K); 455 | } 456 | } 457 | 458 | private sealed class MaxID 459 | { 460 | public long id = long.MaxValue; 461 | public long jumpTime = 0; 462 | 463 | } 464 | } 465 | 466 | } 467 | 468 | -------------------------------------------------------------------------------- /FTServer/Code/Html.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text.RegularExpressions; 4 | using AngleSharp; 5 | using AngleSharp.Dom; 6 | using AngleSharp.Html.Dom; 7 | using AngleSharp.Dom.Events; 8 | 9 | using static FTServer.App; 10 | 11 | namespace FTServer 12 | { 13 | public class Html 14 | { 15 | public static String getUrl(String name) 16 | { 17 | int p = name.IndexOf("http://"); 18 | if (p < 0) 19 | { 20 | p = name.IndexOf("https://"); 21 | } 22 | if (p >= 0) 23 | { 24 | name = name.substring(p).Trim(); 25 | var t = name.IndexOf("#"); 26 | if (t > 0) 27 | { 28 | name = name.substring(0, t); 29 | } 30 | return name; 31 | } 32 | return ""; 33 | } 34 | 35 | private static String getMetaContentByName(IDocument doc, String name) 36 | { 37 | String description = null; 38 | try 39 | { 40 | description = doc.QuerySelector("meta[name='" + name + "']").Attributes["content"].Value; 41 | } 42 | catch 43 | { 44 | 45 | } 46 | 47 | try 48 | { 49 | if (description == null) 50 | { 51 | description = doc.QuerySelector("meta[property='og:" + name + "']").Attributes["content"].Value; 52 | } 53 | } 54 | catch 55 | { 56 | 57 | } 58 | 59 | name = name.substring(0, 1).ToUpper() + name.substring(1); 60 | try 61 | { 62 | if (description == null) 63 | { 64 | description = doc.QuerySelector("meta[name='" + name + "']").Attributes["content"].Value; 65 | } 66 | } 67 | catch 68 | { 69 | 70 | } 71 | 72 | if (description == null) 73 | { 74 | description = ""; 75 | } 76 | 77 | return replace(description); 78 | } 79 | //static String splitWords = " ,.　，。"; 80 | public static Page Get(String url, HashSet subUrls) 81 | { 82 | try 83 | { 84 | if (url == null || url.Length > Page.MAX_URL_LENGTH || url.Length < 8) 85 | { 86 | Log("URL Length: " + url + " :" + (url != null ? url.length() : "")); 87 | return null; 88 | } 89 | 90 | var config = Configuration.Default.WithDefaultLoader(); 91 | var context = BrowsingContext.New(config); 92 | 93 | context.AddEventListener(AngleSharp.Dom.EventNames.Requested, (s, e) => 94 | { 95 | var r = ((RequestEvent)e).Response; 96 | //Content-Type = text/html; charset=utf-8 97 | bool isHTML = false; 98 | foreach (var h in r.Headers) 99 | { 100 | if (h.Value == null) { continue; } 101 | isHTML |= h.Value.Contains("text/html", StringComparison.InvariantCultureIgnoreCase); 102 | } 103 | if (!isHTML) 104 | { 105 | String Location = "Location"; 106 | if (r.Headers.ContainsKey(Location)) 107 | { 108 | // https://blog.nuget.org/ 109 | Log(r.StatusCode + " Location: " + r.Headers[Location] + " from " + r.Address.Href); 110 | } 111 | else 112 | { 113 | // https://www.nuget.org/api/v2/package/iBoxDB/3.5.0 114 | Log(r.StatusCode + " ContextType Not HTML, " + r.Address.Href); 115 | r.Dispose(); 116 | } 117 | } 118 | }); 119 | var doc = context.OpenAsync(url).GetAwaiter().GetResult(); 120 | if (doc == null) 121 | { 122 | return null; 123 | } 124 | if (doc.StatusCode >= System.Net.HttpStatusCode.BadRequest) 125 | { 126 | Log("Not OK " + url); 127 | return null; 128 | } 129 | if (doc.ContentType != null && doc.ContentType.toLowerCase().equals("text/xml")) 130 | { 131 | Log("XML " + url); 132 | return null; 133 | } 134 | if (doc.ContentType != null && (doc.ContentType.toLowerCase().IndexOf("text/html") < 0)) 135 | { 136 | Log("Not HTML " + doc.ContentType); 137 | return null; 138 | } 139 | if (subUrls != null) 140 | { 141 | var host = doc.BaseUrl.Host; 142 | var links = doc.QuerySelectorAll("a[href]"); 143 | foreach (var link in links) 144 | { 145 | //if (link.Host.Equals(host)) 146 | { 147 | String ss = link.Href; 148 | if (ss != null && ss.length() > 8) 149 | { 150 | ss = getUrl(ss); 151 | subUrls.add(ss); 152 | } 153 | } 154 | } 155 | } 156 | //Log(doc.ToHtml()); 157 | fixSpan(doc); 158 | 159 | Page page = new Page(); 160 | page.url = url; 161 | String text = replace(doc.Body.Text()); 162 | if (text.length() < 10) 163 | { 164 | //some website can't get html 165 | Log("No HTML " + url); 166 | return null; 167 | } 168 | if (text.length() > 100_000) 169 | { 170 | Log("BIG HTML " + url); 171 | return null; 172 | } 173 | if (text.length() > 50_000) 174 | { 175 | Log("[BigURL] " + url); 176 | } 177 | page.text = text; 178 | 179 | 180 | 181 | String title = null; 182 | String keywords = null; 183 | String description = null; 184 | 185 | try 186 | { 187 | title = doc.Title; 188 | } 189 | catch 190 | { 191 | 192 | } 193 | if (title == null) 194 | { 195 | title = ""; 196 | } 197 | if (title.length() < 1) 198 | { 199 | //title = url; 200 | //ignore no title 201 | Log("No Title " + url); 202 | return null; 203 | } 204 | title = replace(title); 205 | if (title.length() > 200) 206 | { 207 | title = title.substring(0, 200); 208 | } 209 | 210 | keywords = getMetaContentByName(doc, "keywords"); 211 | keywords = keywords.Replace("，", ","); 212 | 213 | if (keywords.length() > 200) 214 | { 215 | keywords = keywords.substring(0, 200); 216 | } 217 | 218 | description = getMetaContentByName(doc, "description"); 219 | if (description.length() == 0) 220 | { 221 | Log("Can't find description " + url); 222 | page.text += " " + title; 223 | } 224 | if (description.length() > 500) 225 | { 226 | //En 227 | description = description.substring(0, 500); 228 | } 229 | if (description.length() > 300) 230 | { 231 | if (!StringUtil.Instance.isWord(description.charAt(0))) 232 | { 233 | //CN 234 | description = description.substring(0, 300); 235 | } 236 | } 237 | 238 | page.title = title; 239 | page.keywords = keywords; 240 | page.description = description; 241 | 242 | if (Config.DescriptionOnly) 243 | { 244 | page.text = ""; 245 | } 246 | return page; 247 | } 248 | catch (Exception ex) 249 | { 250 | Log(ex.Message); 251 | return null; 252 | } 253 | } 254 | 255 | public static PageText getDefaultText(Page page, long id) 256 | { 257 | PageText pt = PageText.fromId(id); 258 | pt.url = page.url; 259 | pt.title = page.title; 260 | pt.createTime = page.createTime; 261 | if (pt.priority >= PageText.descriptionPriority) 262 | { 263 | pt.keywords = page.keywords; 264 | } 265 | if (pt.priority == PageText.userPriority) 266 | { 267 | pt.text = page.userDescription; 268 | } 269 | if (pt.priority == PageText.descriptionPriority || pt.priority == PageText.descriptionKeyPriority) 270 | { 271 | pt.text = page.description; 272 | } 273 | 274 | if (pt.priority == PageText.contextPriority) 275 | { 276 | pt.text = page.text; 277 | } 278 | return pt; 279 | } 280 | public static List getDefaultTexts(Page page) 281 | { 282 | if (page.textOrder < 1) 283 | { 284 | //no id; 285 | return null; 286 | } 287 | 288 | ArrayList result = new ArrayList(); 289 | 290 | if (page.userDescription != null && page.userDescription.Length > 0) 291 | { 292 | result.add(getDefaultText(page, PageText.toId(page.textOrder, PageText.userPriority))); 293 | } 294 | if (page.description != null && page.description.Length > 0) 295 | { 296 | long p = page.isKeyPage ? PageText.descriptionKeyPriority : PageText.descriptionPriority; 297 | result.add(getDefaultText(page, PageText.toId(page.textOrder, p))); 298 | } 299 | if (page.text != null && page.text.Length > 0) 300 | { 301 | result.add(getDefaultText(page, PageText.toId(page.textOrder, PageText.contextPriority))); 302 | } 303 | 304 | return result; 305 | } 306 | 307 | 308 | public static string replace(String content) 309 | { 310 | content = content.Replace("　", " ").Replace(((char)8203).ToString(), " "); 311 | content = Regex.Replace(content, "\t|\r|\n|<|>", " "); 312 | content = Regex.Replace(content, "\\$", " "); 313 | content = Regex.Replace(content, "\\s+", " "); 314 | content = content.Trim(); 315 | return content; 316 | } 317 | private static void fixSpan(IDocument doc) 318 | { 319 | foreach (var s in new string[] { "script", "style", "textarea", "noscript", "code" }) 320 | { 321 | foreach (var c in new List(doc.GetElementsByTagName(s))) 322 | { 323 | c.Parent.RemoveElement(c); 324 | } 325 | } 326 | foreach (var s in new string[] { 327 | "span", "td", "th", "li", "a", "option", "p", 328 | "div", "h1","h2","h3","h4","h5", "pre" }) 329 | { 330 | foreach (var c in doc.GetElementsByTagName(s)) 331 | { 332 | if (c.ChildNodes.Length == 1 && c.ChildNodes[0].NodeType == NodeType.Text) 333 | { 334 | try 335 | { 336 | c.TextContent = " " + c.TextContent + " "; 337 | } 338 | catch (Exception e) 339 | { 340 | Log(e.ToString()); 341 | } 342 | } 343 | } 344 | } 345 | } 346 | 347 | } 348 | } -------------------------------------------------------------------------------- /FTServer/Code/IndexAPI.cs: -------------------------------------------------------------------------------- 1 | /* iBoxDB FTServer Bruce Yang CL-N */ 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Text; 5 | using IBoxDB.LocalServer; 6 | using static FTServer.App; 7 | 8 | namespace FTServer 9 | { 10 | 11 | public class IndexAPI 12 | { 13 | private class StartIdParam 14 | { 15 | //andBox,orBox, ids... 16 | public long[] startId; 17 | 18 | public StartIdParam(long[] id) 19 | { 20 | if (id.Length == 1) 21 | { 22 | startId = new long[] { App.Indices.length() - 1, -1, id[0] }; 23 | } 24 | else 25 | { 26 | startId = id; 27 | } 28 | } 29 | public bool isAnd() 30 | { 31 | if (startId[0] >= 0) 32 | { 33 | if (startId[2] >= 0) { return true; } 34 | startId[0]--; 35 | startId[2] = long.MaxValue; 36 | return isAnd(); 37 | } 38 | return false; 39 | } 40 | 41 | internal virtual ArrayList ToOrCondition(String name) 42 | { 43 | 44 | ArrayList ors = EasyOR.toOrCondition(name); 45 | 46 | //----SET----// 47 | ors.add(0, null); //and box 48 | ors.add(1, null); //or box 49 | ors.add(2, null); //and startId 50 | //full search 51 | ors.add(3, name); 52 | 53 | if (startId.Length != ors.size()) 54 | { 55 | startId = new long[ors.size()]; 56 | startId[0] = -1; 57 | startId[1] = App.Indices.length() - 1;//or box 58 | startId[2] = -1; 59 | for (int i = 3; i < startId.Length; i++) 60 | { 61 | startId[i] = long.MaxValue; 62 | } 63 | } 64 | 65 | if (ors.Count > 16 || ors.size() < 5 || stringEqual(ors[3].ToString(), ors[4].ToString())) 66 | { 67 | for (int i = 0; i < startId.Length; i++) 68 | { 69 | startId[i] = -1; 70 | } 71 | } 72 | 73 | return ors; 74 | } 75 | 76 | public bool isOr() 77 | { 78 | if (startId[1] >= 0) 79 | { 80 | for (int i = 3; i < startId.Length; i++) 81 | { 82 | if (startId[i] >= 0) { return true; } 83 | } 84 | startId[1]--; 85 | for (int i = 3; i < startId.Length; i++) 86 | { 87 | startId[i] = long.MaxValue; 88 | } 89 | return isOr(); 90 | } 91 | return false; 92 | } 93 | } 94 | 95 | public static readonly string IndexingMessage = "BackgroundIndexRunning-SeeConsoleOutput"; 96 | public static long[] Search(List outputPages, 97 | String name, long[] t_startId, long pageCount) 98 | { 99 | name = StringUtil.Instance.fromatFrenchInput(name); 100 | 101 | if (name.length() == 0 || name.Length > 150 102 | || name == IndexingMessage 103 | || name == IndexPage.SystemShutdown 104 | ) { return new long[] { -1, -1, -1 }; } 105 | 106 | 107 | long maxTime = 1000 * 2; 108 | if (pageCount == 1) 109 | { 110 | maxTime = 500; 111 | } 112 | StartIdParam startId = new StartIdParam(t_startId); 113 | long beginTime = Environment.TickCount64; 114 | 115 | //And 116 | while (startId.isAnd()) 117 | { 118 | DelayService.delayIndex(); 119 | AutoBox auto = App.Indices.get((int)startId.startId[0]); 120 | startId.startId[2] = SearchAnd(auto, outputPages, name, startId.startId[2], pageCount - outputPages.Count); 121 | App.Indices.tryCloseOutOfCache(auto); 122 | foreach (var pt in outputPages) 123 | { 124 | if (pt.dbOrder < 0) 125 | { 126 | pt.dbOrder = startId.startId[0] + IndexServer.IndexDBStart; 127 | } 128 | } 129 | if (outputPages.Count >= pageCount) 130 | { 131 | return startId.startId; 132 | } 133 | if ((Environment.TickCount64 - beginTime) > maxTime) 134 | { 135 | return startId.startId; 136 | } 137 | } 138 | 139 | 140 | //OR 141 | ArrayList ors = startId.ToOrCondition(name); 142 | while (startId.isOr()) 143 | { 144 | DelayService.delayIndex(); 145 | AutoBox auto = App.Indices.get((int)startId.startId[1]); 146 | SearchOr(auto, outputPages, ors, startId.startId, pageCount); 147 | App.Indices.tryCloseOutOfCache(auto); 148 | foreach (var pt in outputPages) 149 | { 150 | if (pt.dbOrder < 0) 151 | { 152 | pt.dbOrder = startId.startId[1] + IndexServer.IndexDBStart; 153 | } 154 | } 155 | if (outputPages.Count >= pageCount) 156 | { 157 | break; 158 | } 159 | if ((Environment.TickCount64 - beginTime) > maxTime) 160 | { 161 | break; 162 | } 163 | } 164 | return startId.startId; 165 | } 166 | 167 | private static long SearchAnd(AutoBox auto, List pages, 168 | String name, long startId, long pageCount) 169 | { 170 | name = name.Trim(); 171 | 172 | using (var box = auto.Cube()) 173 | { 174 | foreach (KeyWord kw in Engine.Instance.searchDistinct(box, name, startId, pageCount)) 175 | { 176 | pageCount--; 177 | startId = kw.I - 1; 178 | 179 | long id = kw.I; 180 | PageText pt = PageText.fromId(id); 181 | Page p = getPage(pt.textOrder); 182 | if (p.show) 183 | { 184 | pt = Html.getDefaultText(p, id); 185 | pt.keyWord = kw; 186 | pt.page = p; 187 | pt.isAndSearch = true; 188 | pages.Add(pt); 189 | } 190 | } 191 | 192 | return pageCount == 0 ? startId : -1; 193 | } 194 | } 195 | private static void SearchOr(AutoBox auto, List outputPages, 196 | ArrayList ors, long[] startId, long pageCount) 197 | { 198 | 199 | using (IBox box = auto.Cube()) 200 | { 201 | 202 | IEnumerator[] iters = new IEnumerator[ors.size()]; 203 | 204 | for (int i = 0; i < ors.size(); i++) 205 | { 206 | String sbkw = ors.get(i); 207 | if (sbkw == null || sbkw.Length < 1) 208 | { 209 | iters[i] = null; 210 | continue; 211 | } 212 | //never set Long.MAX 213 | long subCount = pageCount * 10; 214 | iters[i] = Engine.Instance.searchDistinct(box, sbkw, startId[i], subCount).GetEnumerator(); 215 | } 216 | 217 | int orStartPos = 3; 218 | KeyWord[] kws = new KeyWord[iters.Length]; 219 | 220 | int mPos = maxPos(startId); 221 | while (mPos >= orStartPos) 222 | { 223 | 224 | for (int i = orStartPos; i < iters.Length; i++) 225 | { 226 | if (kws[i] == null) 227 | { 228 | if (iters[i] != null && iters[i].MoveNext()) 229 | { 230 | kws[i] = iters[i].Current; 231 | startId[i] = kws[i].I; 232 | } 233 | else 234 | { 235 | iters[i] = null; 236 | startId[i] = -1; 237 | } 238 | } 239 | } 240 | 241 | if (outputPages.Count >= pageCount) 242 | { 243 | break; 244 | } 245 | 246 | mPos = maxPos(startId); 247 | 248 | if (mPos > orStartPos) 249 | { 250 | KeyWord kw = kws[mPos]; 251 | 252 | long id = kw.I; 253 | 254 | PageText pt = PageText.fromId(id); 255 | Page p = getPage(pt.textOrder); 256 | if (p.show) 257 | { 258 | pt = Html.getDefaultText(p, id); 259 | pt.keyWord = kw; 260 | pt.page = p; 261 | pt.isAndSearch = false; 262 | outputPages.Add(pt); 263 | } 264 | } 265 | 266 | long maxId = startId[mPos]; 267 | for (int i = orStartPos; i < startId.Length; i++) 268 | { 269 | if (startId[i] == maxId) 270 | { 271 | kws[i] = null; 272 | } 273 | } 274 | 275 | } 276 | 277 | } 278 | 279 | } 280 | 281 | private static int maxPos(long[] ids) 282 | { 283 | int orStartPos = 3; 284 | orStartPos--; 285 | for (int i = orStartPos; i < ids.Length; i++) 286 | { 287 | if (ids[i] > ids[orStartPos]) 288 | { 289 | orStartPos = i; 290 | } 291 | } 292 | return orStartPos; 293 | } 294 | 295 | private static bool stringEqual(String a, String b) 296 | { 297 | if (a.Equals(b)) { return true; } 298 | if (a.Equals("\"" + b + "\"")) { return true; } 299 | if (b.Equals("\"" + a + "\"")) { return true; } 300 | return false; 301 | } 302 | public static Page getPage(long textOrder) 303 | { 304 | return App.Item.Get("Page", textOrder); 305 | } 306 | 307 | public static long addPage(Page page) 308 | { 309 | /* 310 | Page oldPage = GetOldPage(page.url); 311 | if (oldPage != null && oldPage.show) 312 | { 313 | if (oldPage.text.equals(page.text)) 314 | { 315 | Log("Page is not changed. " + page.url); 316 | return -1L; 317 | } 318 | else 319 | { 320 | Log("Page is changed. " + page.url); 321 | } 322 | } 323 | */ 324 | using (var box = App.Item.Cube()) 325 | { 326 | page.createTime = DateTime.Now; 327 | page.textOrder = box.NewId(); 328 | box["Page"].Insert(page, 1); 329 | if (box.Commit() == CommitResult.OK) 330 | { 331 | return page.textOrder; 332 | } 333 | return -1L; 334 | } 335 | } 336 | 337 | 338 | public static bool addPageIndex(long textOrder) 339 | { 340 | Page page = getPage(textOrder); 341 | if (page == null) { return false; } 342 | long HuggersMemory = int.MaxValue / 2; 343 | List ptlist = Html.getDefaultTexts(page); 344 | 345 | int count = 0; 346 | 347 | if (ptlist.size() == 0) 348 | { 349 | Log("[No Description]"); 350 | } 351 | foreach (PageText pt in ptlist) 352 | { 353 | count++; 354 | addPageTextIndex(pt, count == ptlist.Count ? 0 : HuggersMemory); 355 | } 356 | 357 | return true; 358 | } 359 | 360 | private static void addPageTextIndex(PageText pt, long huggers = 0) 361 | { 362 | using (IBox box = App.Index.Cube()) 363 | { 364 | Engine.Instance.indexText(box, pt.id, pt.indexedText(), false, DelayService.delay); 365 | CommitResult cr = box.Commit(huggers); 366 | Log("MEM: " + cr.GetMemoryLength(box).ToString("#,#")); 367 | } 368 | } 369 | 370 | public static void DisableOldPage(String url) 371 | { 372 | using (var box = App.Item.Cube()) 373 | { 374 | List page = new List(); 375 | 376 | foreach (var p in box.Scale("from Page where url==? limit 1,10", url)) 377 | { 378 | if (!p.show) { break; } 379 | page.Add(p); 380 | } 381 | foreach (var p in page) 382 | { 383 | p.show = false; 384 | box["Page"].Update(p); 385 | } 386 | box.Commit().Assert(); 387 | } 388 | } 389 | 390 | } 391 | 392 | } -------------------------------------------------------------------------------- /FTServer/Code/IndexFields.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using IBoxDB.LocalServer; 3 | 4 | //the db classes 5 | namespace FTServer 6 | { 7 | public class PageSearchTerm 8 | { 9 | 10 | public readonly static int MAX_TERM_LENGTH = 64; 11 | 12 | public DateTime time; 13 | public String keywords; 14 | public Guid uid; 15 | } 16 | public class PageText 17 | { 18 | 19 | public static readonly long userPriority = 12; 20 | 21 | public static readonly long descriptionKeyPriority = 11; 22 | 23 | //this is the center of Priorities, under is Body.Text, upper is user's input 24 | public static readonly long descriptionPriority = 10; 25 | 26 | public static readonly long contextPriority = 9; 27 | 28 | 29 | private static readonly int priorityOffset = 50; 30 | 31 | public static PageText fromId(long id) 32 | { 33 | PageText pt = new PageText(); 34 | pt.priority = id >> priorityOffset; 35 | pt.textOrder = id - (pt.priority << priorityOffset); 36 | 37 | pt.text = String.Empty; 38 | pt.keywords = String.Empty; 39 | pt.url = String.Empty; 40 | pt.title = String.Empty; 41 | return pt; 42 | } 43 | public static long toId(long textOrder, long priority) 44 | { 45 | return textOrder | (priority << priorityOffset); 46 | } 47 | 48 | public long id 49 | { 50 | get 51 | { 52 | return toId(textOrder, priority); 53 | } 54 | set 55 | { 56 | //ignore set 57 | } 58 | } 59 | 60 | 61 | public long textOrder; 62 | public long priority; 63 | 64 | public String url; 65 | 66 | public String title; 67 | 68 | public String text; 69 | 70 | //keywords 71 | public String keywords; 72 | 73 | public DateTime createTime; 74 | 75 | [NotColumn] 76 | public String indexedText() 77 | { 78 | if (priority >= descriptionPriority) 79 | { 80 | return text + " " + title; 81 | } 82 | 83 | if (priority == contextPriority) 84 | { 85 | return text + " " + decodeTry(url).Replace("-", " "); 86 | } 87 | 88 | return text; 89 | } 90 | public static String decodeTry(String str) 91 | { 92 | try 93 | { 94 | return System.Net.WebUtility.UrlDecode(str); 95 | } 96 | catch 97 | { 98 | return str; 99 | } 100 | } 101 | 102 | [NotColumn] 103 | public bool isAndSearch = true; 104 | 105 | [NotColumn] 106 | public KeyWord keyWord; 107 | 108 | 109 | [NotColumn] 110 | public Page page; 111 | 112 | [NotColumn] 113 | public long dbOrder = -1; 114 | } 115 | 116 | public partial class Page 117 | { 118 | public const int MAX_URL_LENGTH = 512; 119 | 120 | public String url; 121 | 122 | public long textOrder; 123 | 124 | // too too big this html 125 | //public String html; 126 | public String text; 127 | 128 | public DateTime createTime; 129 | public bool isKeyPage = false; 130 | 131 | public String title; 132 | public String keywords; 133 | public String description; 134 | 135 | public string userDescription; 136 | 137 | public bool show = true; 138 | 139 | } 140 | public partial class Page 141 | { 142 | 143 | private static Random RAN = new Random(); 144 | 145 | 146 | public String getRandomContent(int length) 147 | { 148 | int len = text.length() - length; 149 | if (len <= 0) 150 | { 151 | return text; 152 | } 153 | 154 | int s = RAN.nextInt(len); 155 | 156 | int end = s + length; 157 | if (end > text.length()) 158 | { 159 | end = text.length(); 160 | } 161 | 162 | return text.substring(s, end); 163 | } 164 | } 165 | 166 | 167 | } 168 | 169 | 170 | -------------------------------------------------------------------------------- /FTServer/Code/IndexPage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Threading; 4 | using System.Runtime.CompilerServices; 5 | 6 | using static FTServer.App; 7 | 8 | namespace FTServer 9 | { 10 | 11 | public class IndexPage 12 | { 13 | public static readonly String SystemShutdown = "SystemShutdown"; 14 | 15 | public static void addSearchTerm(String keywords) 16 | { 17 | if (keywords == null) 18 | { 19 | return; 20 | } 21 | if (keywords.length() > PageSearchTerm.MAX_TERM_LENGTH) 22 | { 23 | keywords = keywords.substring(0, PageSearchTerm.MAX_TERM_LENGTH - 1); 24 | } 25 | 26 | PageSearchTerm pst = new PageSearchTerm(); 27 | pst.time = DateTime.Now; 28 | pst.keywords = keywords; 29 | pst.uid = Guid.NewGuid(); 30 | 31 | long huggersMem = 1024L * 1024L * 3L; 32 | 33 | bool isShutdown = SystemShutdown.Equals(keywords); 34 | if (isShutdown) { huggersMem = 0; } 35 | using (var box = App.Item.Cube()) 36 | { 37 | box["/PageSearchTerm"].Insert(pst); 38 | box.Commit(huggersMem); 39 | } 40 | 41 | } 42 | 43 | 44 | public static List getSearchTerm(int len) 45 | { 46 | return App.Item.Select("from /PageSearchTerm limit 0 , ?", len); 47 | } 48 | 49 | public static String getDesc(String str, KeyWord kw, int length) 50 | { 51 | if (kw.I == -1) 52 | { 53 | return str; 54 | } 55 | return Engine.Instance.getDesc(str, kw, length); 56 | } 57 | public static List discover() 58 | { 59 | using (var box = App.Index.Cube()) 60 | { 61 | ArrayList result = new ArrayList(); 62 | 63 | //English 64 | result.addAll(Engine.Instance.discoverEN(box, (char)0x0061, (char)0x007A, 2)); 65 | 66 | //Russian 67 | result.addAll(Engine.Instance.discoverEN(box, (char)0x0410, (char)0x044F, 2)); 68 | 69 | //arabic 70 | result.addAll(Engine.Instance.discoverEN(box, (char)0x0621, (char)0x064A, 2)); 71 | 72 | //India 73 | result.addAll(Engine.Instance.discoverEN(box, (char)0x0900, (char)0x097F, 2)); 74 | 75 | //Japanese Hiragana(0x3040-309F), Katakana(0x30A0-30FF) 76 | result.addAll(Engine.Instance.discoverCN(box, (char)0x3040, (char)0x30FF, 2)); 77 | 78 | //Chinese 79 | result.addAll(Engine.Instance.discoverCN(box, (char)0x4E00, (char)0x9FFF, 2)); 80 | 81 | //Korean 82 | result.addAll(Engine.Instance.discoverEN(box, (char)0xAC00, (char)0xD7AF, 2)); 83 | 84 | 85 | return result; 86 | } 87 | } 88 | 89 | 90 | public static String addPage(String url, string userDescription, bool isKeyPage) 91 | { 92 | if (!isKeyPage) 93 | { 94 | if (App.Item.Count("from Page where url==? limit 0,1", url) > 0) 95 | { 96 | return null; 97 | } 98 | } 99 | 100 | HashSet subUrls = new HashSet(); 101 | 102 | DateTime begin = DateTime.Now; 103 | Page p = Html.Get(url, subUrls); 104 | DateTime ioend = DateTime.Now; 105 | if (subUrls.size() > 0) 106 | { 107 | subUrls.remove(url); 108 | subUrls.remove(url + "/"); 109 | subUrls.remove(url.substring(0, url.length() - 1)); 110 | runBGTask(subUrls, isKeyPage); 111 | } 112 | if (p == null) 113 | { 114 | return "Temporarily Unreachable"; 115 | } 116 | else 117 | { 118 | if (userDescription != null) 119 | { 120 | userDescription = Html.replace(userDescription); 121 | } 122 | p.userDescription = userDescription; 123 | p.show = true; 124 | p.isKeyPage = isKeyPage; 125 | long textOrder = IndexAPI.addPage(p); 126 | if (textOrder >= 0 && IndexAPI.addPageIndex(textOrder)) 127 | { 128 | IndexAPI.DisableOldPage(url); 129 | } 130 | long dbaddr = App.Indices.length() + IndexServer.IndexDBStart - 1; 131 | DateTime indexend = DateTime.Now; 132 | Log("TIME IO:" + (ioend - begin).TotalSeconds 133 | + " INDEX:" + (indexend - ioend).TotalSeconds + " TEXTORDER:" + textOrder + " (" + dbaddr + ") "); 134 | 135 | return url; 136 | } 137 | } 138 | 139 | 140 | [MethodImpl(MethodImplOptions.Synchronized)] 141 | public static void runBGTask(String url, String customContent = null) 142 | { 143 | backgroundThreadQueue.addFirst(() => 144 | { 145 | var bc = Console.BackgroundColor; 146 | Console.BackgroundColor = ConsoleColor.DarkRed; 147 | Log("(KeyPage) For:" + url + " ," + backgroundThreadQueue.size()); 148 | Console.BackgroundColor = bc; 149 | String r = addPage(url, customContent, true); 150 | backgroundLog(url, r); 151 | }); 152 | } 153 | 154 | [MethodImpl(MethodImplOptions.Synchronized)] 155 | private static void runBGTask(HashSet subUrls, bool isKeyPage) 156 | { 157 | if (subUrls == null || isshutdown) 158 | { 159 | return; 160 | } 161 | bool atNight = true; 162 | 163 | int max_background = atNight ? 500 : 0; 164 | if (App.IsAndroid && max_background > 50) 165 | { 166 | max_background = 50; 167 | } 168 | 169 | if (isKeyPage) 170 | { 171 | max_background *= 2; 172 | } 173 | 174 | 175 | foreach (String vurl in subUrls) 176 | { 177 | if (backgroundThreadQueue.size() > max_background) 178 | { 179 | break; 180 | } 181 | var url = Html.getUrl(vurl); 182 | backgroundThreadQueue.addLast(() => 183 | { 184 | Log("For:" + url + " ," + backgroundThreadQueue.size()); 185 | String r = addPage(url, null, false); 186 | backgroundLog(url, r); 187 | }); 188 | } 189 | 190 | } 191 | 192 | public static void backgroundLog(String url, String output) 193 | { 194 | if (output == null) 195 | { 196 | Log("Has indexed:" + url); 197 | } 198 | else if (url.equals(output)) 199 | { 200 | Log("Indexed:" + url); 201 | } 202 | else 203 | { 204 | Log("Retry:" + url); 205 | } 206 | Log(""); 207 | } 208 | 209 | 210 | private static ConcurrentLinkedDeque backgroundThreadQueue = new ConcurrentLinkedDeque(); 211 | private static bool isshutdown = false; 212 | 213 | public static int HttpGet_SleepTime = 1000; 214 | private static Thread backgroundTasks = new Func(() => 215 | { 216 | var bt = new Thread(() => 217 | { 218 | while (!isshutdown) 219 | { 220 | ThreadStart act = backgroundThreadQueue.pollFirst(); 221 | if (act != null) 222 | { 223 | try 224 | { 225 | act(); 226 | } 227 | catch (Exception e) 228 | { 229 | Console.WriteLine(e.ToString()); 230 | } 231 | } 232 | else 233 | { 234 | Thread.Sleep(2000); 235 | } 236 | 237 | if (!isshutdown) 238 | { 239 | Thread.Sleep(HttpGet_SleepTime); 240 | } 241 | } 242 | }); 243 | bt.Priority = ThreadPriority.Lowest; 244 | bt.IsBackground = true; 245 | bt.Start(); 246 | return bt; 247 | })(); 248 | 249 | public static void Shutdown() 250 | { 251 | if (backgroundTasks != null) 252 | { 253 | isshutdown = true; 254 | backgroundTasks.Priority = ThreadPriority.Highest; 255 | backgroundTasks.Join(); 256 | backgroundTasks = null; 257 | } 258 | Log("Background Task Ended"); 259 | } 260 | } 261 | } -------------------------------------------------------------------------------- /FTServer/Code/IndexServer.cs: -------------------------------------------------------------------------------- 1 | 2 | using System; 3 | using System.Collections.Generic; 4 | using System.IO; 5 | using System.Threading; 6 | using IBoxDB.LocalServer; 7 | using IBoxDB.LocalServer.IO; 8 | 9 | using static FTServer.App; 10 | 11 | namespace FTServer 12 | { 13 | public class DelayService 14 | { 15 | 16 | private static DateTime pageIndexDelay = DateTime.MinValue; 17 | 18 | public static void delayIndex(int seconds = 5) 19 | { 20 | pageIndexDelay = DateTime.Now.AddSeconds(seconds); 21 | } 22 | 23 | public static void delay() 24 | { 25 | if (pageIndexDelay == DateTime.MinValue) { return; } 26 | 27 | while (DateTime.Now < pageIndexDelay) 28 | { 29 | var d = (pageIndexDelay - DateTime.Now).TotalSeconds; 30 | if (d < 0) { d = 0; } 31 | if (d > 120) { d = 120; } 32 | 33 | Thread.Sleep((int)(d * 1000)); 34 | } 35 | } 36 | } 37 | 38 | public class ReadonlyIndexServer : LocalDatabaseServer 39 | { 40 | 41 | public bool OutOfCache = false; 42 | protected override DatabaseConfig BuildDatabaseConfig(long address) 43 | { 44 | return new ReadonlyConfig(address, OutOfCache); 45 | } 46 | public class ReadonlyConfig : ReadonlyStreamConfig 47 | { 48 | private long address; 49 | public bool OutOfCache; 50 | public ReadonlyConfig(long address, bool outOfCache) : base(GetStreamsImpl(address, outOfCache)) 51 | { 52 | this.address = address; 53 | this.OutOfCache = outOfCache; 54 | this.CacheLength = Config.Readonly_CacheLength; 55 | if (outOfCache) 56 | { 57 | this.CacheLength = Config.ShortCacheLength; 58 | } 59 | if (this.CacheLength < Config.lowReadonlyCache) 60 | { 61 | this.CacheLength = Config.lowReadonlyCache; 62 | } 63 | } 64 | 65 | private static Stream[] GetStreamsImpl(long address, bool outOfCache) 66 | { 67 | string pa = DatabaseConfig.GetFileName(address); 68 | Stream[] os = new Stream[outOfCache ? 1 : 2]; 69 | for (int i = 0; i < os.Length; i++) 70 | { 71 | os[i] = new FileStream(pa, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); 72 | } 73 | return os; 74 | } 75 | 76 | } 77 | } 78 | public class IndexServer : LocalDatabaseServer 79 | { 80 | public static long ItemDB = 2L; 81 | 82 | public static long IndexDBStart = 10L; 83 | public IndexServer() 84 | { 85 | } 86 | 87 | protected override DatabaseConfig BuildDatabaseConfig(long address) 88 | { 89 | if (address >= IndexDBStart) 90 | { 91 | return new IndexConfig(); 92 | } 93 | if (address == ItemDB) 94 | { 95 | return new ItemConfig(); 96 | } 97 | return null; 98 | } 99 | 100 | private class ItemConfig : BoxFileStreamConfig 101 | { 102 | public ItemConfig() 103 | { 104 | CacheLength = Config.ItemConfig_CacheLength; 105 | SwapFileBuffer = Config.ItemConfig_SwapFileBuffer; 106 | FileIncSize = Config.ItemConfig_SwapFileBuffer; 107 | EnsureTable("/PageSearchTerm", "time", "keywords(" + PageSearchTerm.MAX_TERM_LENGTH + ")", "uid"); 108 | 109 | EnsureTable("Page", "textOrder"); 110 | //the 'textOrder' is used to control url's order 111 | EnsureIndex("Page", "url(" + Page.MAX_URL_LENGTH + ")", "textOrder"); 112 | 113 | 114 | Log("ItemConfig CacheLength = " + (CacheLength / 1024L / 1024L) + " MB"); 115 | Log("ItemConfig SwapFileBuffer = " + (SwapFileBuffer / 1024L / 1024L) + " MB"); 116 | Log("ItemConfig ReadStreamCount = " + ReadStreamCount); 117 | } 118 | } 119 | 120 | 121 | //this IndexConfig will use IndexStream() to delay index-write, 122 | //other application Tables, place into ItemConfig 123 | private class IndexConfig : BoxFileStreamConfig 124 | { 125 | public IndexConfig() 126 | { 127 | CacheLength = Config.Index_CacheLength; 128 | SwapFileBuffer = Config.ItemConfig_SwapFileBuffer; 129 | //this size trigger "SWITCH" in Flush() 130 | FileIncSize = Config.ItemConfig_SwapFileBuffer; 131 | 132 | Engine.Instance.Config(this); 133 | 134 | Log("IndexConfig CacheLength = " + (CacheLength / 1024L / 1024L) + " MB"); 135 | Log("IndexConfig FileSwitchLength = " + (Config.SwitchToReadonlyIndexLength / 1024L / 1024L) + " MB"); 136 | 137 | } 138 | 139 | public override IBStream CreateStream(string path, StreamAccess access) 140 | { 141 | IBStream s = base.CreateStream(path, access); 142 | if (access == StreamAccess.ReadWrite) 143 | { 144 | return new IndexStream(s); 145 | } 146 | return s; 147 | } 148 | 149 | } 150 | private class IndexStream : IBStreamWrapper 151 | { 152 | 153 | public IndexStream(IBStream iBStream) : base(iBStream) 154 | { 155 | 156 | } 157 | public override void BeginWrite(long appID, int maxLen) 158 | { 159 | DelayService.delay(); 160 | base.BeginWrite(appID, maxLen); 161 | } 162 | 163 | long length = 0; 164 | public override void SetLength(long value) 165 | { 166 | base.SetLength(value); 167 | length = value; 168 | } 169 | 170 | public override void Flush() 171 | { 172 | base.Flush(); 173 | if (length > Config.SwitchToReadonlyIndexLength) 174 | { 175 | 176 | App.Indices.switchIndexToReadonly(); 177 | long addr = App.Index.GetDatabase().LocalAddress; 178 | addr++; 179 | Log("\r\nSwitch To DB (" + addr + ")"); 180 | App.Indices.add(addr, false); 181 | 182 | App.Index = App.Indices.get(App.Indices.length() - 1); 183 | 184 | System.GC.Collect(); 185 | } 186 | } 187 | 188 | } 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /FTServer/Code/ReadonlyList.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using IBoxDB.LocalServer; 5 | 6 | namespace FTServer 7 | { 8 | 9 | public class ReadonlyList : IDisposable 10 | { 11 | 12 | public class AutoBoxHolder 13 | { 14 | 15 | public readonly long address; 16 | public readonly AutoBox Auto; 17 | 18 | public AutoBoxHolder(AutoBox auto, long a) 19 | { 20 | Auto = auto; 21 | address = a; 22 | } 23 | } 24 | 25 | AutoBoxHolder[] list = new AutoBoxHolder[0]; 26 | 27 | public int length() 28 | { 29 | return list.Length; 30 | } 31 | 32 | public void switchIndexToReadonly() 33 | { 34 | App.Log("Switch Readonly DB " + list.Length); 35 | long addr = list[list.Length - 1].address; 36 | AutoBox auto = CreateAutoBox(addr, true); 37 | list[list.Length - 1] = new AutoBoxHolder(auto, addr); 38 | } 39 | 40 | public AutoBox get(int pos) 41 | { 42 | if (Config.Readonly_MaxDBCount < 2) 43 | { 44 | Config.Readonly_MaxDBCount = 2; 45 | } 46 | if (pos < (list.Length - Config.Readonly_MaxDBCount)) 47 | { 48 | AutoBoxHolder o = list[pos]; 49 | if (o.Auto != null) 50 | { 51 | App.Log("Out of Cache " + (pos) + " / " + list.Length + " "); 52 | list[pos] = new AutoBoxHolder(null, o.address); 53 | } 54 | } 55 | AutoBoxHolder a = list[pos]; 56 | if (a.Auto != null) 57 | { 58 | return a.Auto; 59 | } 60 | ReadonlyIndexServer server = new ReadonlyIndexServer(); 61 | server.OutOfCache = true; 62 | if (Config.DSize > 1) 63 | { 64 | App.Log("Use No Cache DB " + a.address); 65 | } 66 | return server.GetInstance(a.address).Get(); 67 | } 68 | 69 | public void tryCloseOutOfCache(AutoBox auto) 70 | { 71 | DatabaseConfig cfg = auto.GetDatabase().GetConfig(); 72 | if (cfg is ReadonlyIndexServer.ReadonlyConfig) 73 | { 74 | if (((ReadonlyIndexServer.ReadonlyConfig)cfg).OutOfCache) 75 | { 76 | if (Config.DSize > 1) 77 | { 78 | App.Log("close No Cache DB " + auto.GetDatabase().LocalAddress); 79 | } 80 | auto.GetDatabase().Dispose(); 81 | } 82 | } 83 | } 84 | 85 | public void add(long addr, bool isReadonly) 86 | { 87 | AutoBox auto = CreateAutoBox(addr, isReadonly); 88 | AutoBoxHolder[] t = Arrays.copyOf(list, list.Length + 1); 89 | t[t.Length - 1] = new AutoBoxHolder(auto, addr); 90 | list = t; 91 | } 92 | 93 | private AutoBox CreateAutoBox(long addr, bool isReadonly) 94 | { 95 | AutoBox auto; 96 | if (isReadonly) 97 | { 98 | if (Config.Readonly_CacheLength < Config.lowReadonlyCache) 99 | { 100 | auto = null; 101 | } 102 | else 103 | { 104 | ReadonlyIndexServer server = new ReadonlyIndexServer(); 105 | auto = server.GetInstance(addr).Get(); 106 | 107 | } 108 | } 109 | else 110 | { 111 | auto = new IndexServer().GetInstance(addr).Get(); 112 | } 113 | return auto; 114 | } 115 | 116 | public void Dispose() 117 | { 118 | if (list != null) 119 | { 120 | foreach (AutoBoxHolder a in list) 121 | { 122 | if (a.Auto != null) 123 | { 124 | a.Auto.GetDatabase().Dispose(); 125 | } 126 | } 127 | } 128 | list = null; 129 | } 130 | } 131 | 132 | } -------------------------------------------------------------------------------- /FTServer/Controllers/BookController.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using Microsoft.AspNetCore.Mvc; 4 | using FTServer.Models; 5 | using System.Text.RegularExpressions; 6 | 7 | namespace FTServer.Controllers 8 | { 9 | public class BookController : Controller 10 | { 11 | public static String[] Books = null; 12 | public static int Base = 100; 13 | 14 | //UTF-8 Text 15 | private static String book1_path = "/home/user/github/hero.txt"; 16 | private static String book2_path = "/home/user/github/phoenix.txt"; 17 | 18 | [ResponseCache(Duration = 0, Location = ResponseCacheLocation.None, NoStore = true)] 19 | public IActionResult Index(int book = 0, float start = 0, int length = 2000, string ex = "") 20 | { 21 | if (Books == null) 22 | { 23 | String[] tmp = new String[2]; 24 | if (System.IO.File.Exists(book1_path) && System.IO.File.Exists(book2_path)) 25 | { 26 | StreamReader sr = new StreamReader(book1_path); 27 | tmp[0] = sr.ReadToEnd(); 28 | sr.Close(); 29 | 30 | sr = new StreamReader(book2_path); 31 | tmp[1] = sr.ReadToEnd(); 32 | sr.Close(); 33 | } 34 | else 35 | { 36 | tmp[0] = RamdomBook(0x4E00, 0x9FFF, (int)'　', 150, 600000); 37 | tmp[1] = RamdomBook(0x0061, 0x007A, (int)' ', 16, 1000000); 38 | } 39 | Books = tmp; 40 | } 41 | 42 | start = start / (float)Base; 43 | 44 | int startIndex = (int)(start * Books[book].Length); 45 | int endIndex = startIndex + length; 46 | if (endIndex > Books[book].Length) 47 | { 48 | endIndex = Books[book].Length; 49 | } 50 | 51 | String content = Books[book].Substring(startIndex, endIndex - startIndex); 52 | 53 | String title = content.Length > 200 ? content.Substring(0, 200) : content; 54 | String description = content.length() > 1000 ? content.substring(200, 1000) : content; 55 | String text = content.Length > 500 ? content.Substring(300) : content; 56 | 57 | 58 | String keywords = "keyword1 keywords2,keyword3 hello"; 59 | 60 | title = Regex.Replace(title, "\t|\r|\n|�|<|>|\\s+", " "); 61 | description = Regex.Replace(description, "\t|\r|\n|�|<|>|\\s+", " "); 62 | 63 | 64 | var m = new BookModel(); 65 | m.Title = title; 66 | m.Description = description; 67 | m.Keywords = keywords; 68 | if (book == 0) 69 | { 70 | text += " " + DateTime.Now; 71 | } 72 | m.Text = text; 73 | m.Ex = ex; 74 | return View(m); 75 | } 76 | 77 | private static String RamdomBook(int startChar, int endChar, int emptyChar, int emptylen, int maxlen) 78 | { 79 | Random ran = new Random(); 80 | char[] cs = new char[maxlen]; 81 | for (int i = 0; i < cs.Length; i++) 82 | { 83 | if (ran.nextInt(emptylen) == 0) 84 | { 85 | cs[i] = (char)emptyChar; 86 | } 87 | else 88 | { 89 | char c = (char)(ran.nextInt(endChar - startChar) + startChar); 90 | cs[i] = c; 91 | } 92 | } 93 | return new String(cs); 94 | } 95 | } 96 | 97 | 98 | } -------------------------------------------------------------------------------- /FTServer/Controllers/HomeController.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | using Microsoft.AspNetCore.Mvc; 4 | using FTServer.Models; 5 | 6 | namespace FTServer.Controllers 7 | { 8 | public class HomeController : Controller 9 | { 10 | 11 | public HomeController() 12 | { 13 | 14 | } 15 | 16 | [ResponseCache(Duration = 0, Location = ResponseCacheLocation.None, NoStore = true)] 17 | public IActionResult Index() 18 | { 19 | return View(); 20 | } 21 | 22 | [ResponseCache(Duration = 0, Location = ResponseCacheLocation.None, NoStore = true)] 23 | public IActionResult About(string q) 24 | { 25 | if (q == null || q.Length == 0) { return NotFound(); } 26 | //App.Log("Search " + q); 27 | var m = new AboutModel(); 28 | q = q.Replace("<", "").Replace(">", "").Trim(); 29 | 30 | 31 | IndexPage.addSearchTerm(q); 32 | 33 | m.Result = new ResultPartialModel 34 | { 35 | Query = q, 36 | StartId = null 37 | }; 38 | 39 | 40 | return View(m); 41 | } 42 | 43 | [ResponseCache(Duration = 0, Location = ResponseCacheLocation.None, NoStore = true)] 44 | public IActionResult Result(String q, String s) 45 | { 46 | q = q.Replace("<", "").Replace(">", "").Trim(); 47 | 48 | long[] ids = new long[] { long.MaxValue }; 49 | if (s != null) 50 | { 51 | String[] ss = s.Trim().Split("_"); 52 | ids = new long[ss.Length]; 53 | for (int i = 0; i < ss.Length; i++) 54 | { 55 | ids[i] = long.Parse(ss[i]); 56 | } 57 | } 58 | 59 | var Result = new ResultPartialModel 60 | { 61 | Query = q, 62 | StartId = ids 63 | }; 64 | return View("ResultPartial", Result); 65 | } 66 | 67 | [ResponseCache(Duration = 0, Location = ResponseCacheLocation.None, NoStore = true)] 68 | public IActionResult Admin(String url = null, String msg = null) 69 | { 70 | if (url != null) 71 | { 72 | url = url.trim(); 73 | bool ishttp = false; 74 | 75 | if (url.StartsWith("http://") || url.StartsWith("https://")) 76 | { 77 | ishttp = true; 78 | } 79 | 80 | if (ishttp) 81 | { 82 | String furl = Html.getUrl(url); 83 | IndexPage.runBGTask(furl, msg); 84 | url = IndexAPI.IndexingMessage; 85 | } 86 | } 87 | var m = new AdminModel(); 88 | m.url = url; 89 | m.msg = msg; 90 | 91 | return View(m); 92 | } 93 | 94 | [ResponseCache(Duration = 0, Location = ResponseCacheLocation.None, NoStore = true)] 95 | public IActionResult Error() 96 | { 97 | return View(new ErrorViewModel { RequestId = Activity.Current?.Id ?? HttpContext.TraceIdentifier }); 98 | } 99 | } 100 | } 101 | 102 | -------------------------------------------------------------------------------- /FTServer/FTServer.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net8.0 5 | fts.ico 6 | 2.0 7 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /FTServer/Models/About.cshtml.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Threading.Tasks; 6 | using Microsoft.AspNetCore.Mvc.RazorPages; 7 | 8 | namespace FTServer.Models 9 | { 10 | public class AboutModel 11 | { 12 | 13 | public ResultPartialModel Result { get; set; } 14 | 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /FTServer/Models/AdminModel.cs: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | namespace FTServer.Models 5 | { 6 | public class AdminModel 7 | { 8 | public string url; 9 | public string msg; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /FTServer/Models/Book.cshtml.cs: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | using System; 5 | 6 | namespace FTServer.Models 7 | { 8 | public class BookModel 9 | { 10 | public static readonly Random ran = new Random(); 11 | public string Title { get; set; } 12 | public string Description { get; set; } 13 | 14 | public string Keywords { get; set; } 15 | 16 | public string Text { get; set; } 17 | 18 | public string Ex { get; set; } 19 | } 20 | } -------------------------------------------------------------------------------- /FTServer/Models/Error.cshtml.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace FTServer.Models 4 | { 5 | public class ErrorViewModel 6 | { 7 | public string RequestId { get; set; } 8 | 9 | public bool ShowRequestId => !string.IsNullOrEmpty(RequestId); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /FTServer/Models/ResultPartial.cshtml.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using System.Linq; 5 | 6 | namespace FTServer.Models 7 | { 8 | public class ResultPartialModel 9 | { 10 | //?q= 11 | public string Query { get; set; } 12 | 13 | //&s= 14 | public long[] StartId { get; set; } 15 | public List pages; 16 | public DateTime begin; 17 | public bool isFirstLoad; 18 | 19 | // reset in ResultPartial.cshtml 20 | public long pageCount = 8; 21 | public String IdToString() 22 | { 23 | long[] ids = StartId; 24 | char p = '_'; 25 | StringBuilder sb = new StringBuilder(); 26 | for (int i = 0; i < ids.Length; i++) 27 | { 28 | if (i > 0) 29 | { 30 | sb.Append(p); 31 | } 32 | sb.Append(ids[i]); 33 | } 34 | return sb.ToString(); 35 | } 36 | public String ToKeyWordString() 37 | { 38 | HashSet hc = new HashSet(); 39 | 40 | ArrayList kws = new ArrayList(); 41 | foreach (var pg in pages) 42 | { 43 | KeyWord kw = pg.keyWord; 44 | while (kw is KeyWordN) 45 | { 46 | kws.add(kw); 47 | kw = kw.previous; 48 | } 49 | } 50 | foreach (KeyWord kw in kws) 51 | { 52 | if (kw is KeyWordE e) 53 | { 54 | hc.Add(e.K); 55 | } 56 | if (kw is KeyWordN n) 57 | { 58 | String ks = n.toKString(); 59 | if (ks.length() > 1) 60 | hc.Add(ks); 61 | } 62 | } 63 | 64 | 65 | var ids = hc.ToArray(); 66 | char p = ' '; 67 | StringBuilder sb = new StringBuilder(); 68 | for (int i = 0; i < ids.Length; i++) 69 | { 70 | if (i > 0) 71 | { 72 | sb.Append(p); 73 | } 74 | sb.Append(ids[i]); 75 | } 76 | String r = sb.ToString(); 77 | //App.Log(r); 78 | return r; 79 | } 80 | 81 | public void Init() 82 | { 83 | if (StartId == null) 84 | { 85 | StartId = new long[] { long.MaxValue }; 86 | } 87 | 88 | isFirstLoad = StartId[0] == long.MaxValue; 89 | if (isFirstLoad) { pageCount = 1; } 90 | 91 | pages = new List(); 92 | 93 | begin = DateTime.Now; 94 | 95 | StartId = IndexAPI.Search(pages, Query, StartId, pageCount); 96 | 97 | } 98 | public bool IsEnd() 99 | { 100 | return StartId[0] < 0 && StartId[1] < 0; 101 | } 102 | } 103 | } -------------------------------------------------------------------------------- /FTServer/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Threading.Tasks; 6 | using System.Net; 7 | using System.Threading; 8 | using Microsoft.AspNetCore; 9 | using Microsoft.AspNetCore.Hosting; 10 | using Microsoft.Extensions.Configuration; 11 | using Microsoft.Extensions.Logging; 12 | using Microsoft.Extensions.Hosting; 13 | 14 | using IBoxDB.LocalServer; 15 | using static FTServer.App; 16 | using System.Collections.Concurrent; 17 | using System.Diagnostics; 18 | using System.Net.NetworkInformation; 19 | using System.Net.Sockets; 20 | 21 | namespace FTServer 22 | { 23 | 24 | public class Program 25 | { 26 | 27 | public static void Main(string[] args) 28 | { 29 | ServicePointManager.ServerCertificateValidationCallback = (sender, certificate, chain, sslPolicyErrors) => 30 | { 31 | return true; 32 | }; 33 | var task = Task.Run(() => 34 | { 35 | #region Path 36 | String dir = "DATA_FTS_CS_161"; 37 | 38 | //String path = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Personal), dir); 39 | 40 | //path = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), dir); 41 | //path = Path.Combine("/mnt/hgfs/DB", dir); 42 | String path = Path.Combine("../", dir); 43 | 44 | Directory.CreateDirectory(path); 45 | Log("DBPath=" + Path.GetFullPath(path)); 46 | DB.Root(path); 47 | 48 | #endregion 49 | 50 | Log("ReadOnly CacheLength = " + (Config.Readonly_CacheLength / 1024L / 1024L) + " MB (" + Config.Readonly_CacheLength + ")"); 51 | Log("ReadOnly Max DB Count = " + Config.Readonly_MaxDBCount); 52 | Log("ReadOnly ShortCacheLength = " + (Config.ShortCacheLength / 1024L / 1024L) + " MB "); 53 | 54 | App.Item = new IndexServer().GetInstance(IndexServer.ItemDB).Get(); 55 | 56 | long start = IndexServer.IndexDBStart; 57 | foreach (var f in Directory.GetFiles(path)) 58 | { 59 | var fn = Path.GetFileNameWithoutExtension(f).Replace("db", ""); 60 | long.TryParse(fn, out long r); 61 | if (r > start) { start = r; } 62 | } 63 | 64 | for (long l = IndexServer.IndexDBStart; l < start; l++) 65 | { 66 | App.Indices.add(l, true); 67 | } 68 | App.Indices.add(start, false); 69 | 70 | App.Index = App.Indices.get(App.Indices.length() - 1); 71 | 72 | Log("Index Description Only = " + Config.DescriptionOnly); 73 | Log("ReadOnly Index DB (" + start + "), start from " + IndexServer.IndexDBStart); 74 | Log("MinCache = " + (Config.minCache() / 1024L / 1024L) + " MB"); 75 | //bigger will more accurate, smaller faster will jump some pages 76 | Engine.KeyWordMaxScan = 10_000; 77 | Log("KeyWordMaxScan = " + Engine.KeyWordMaxScan); 78 | IndexPage.HttpGet_SleepTime = 0; 79 | Log("HttpGet_SleepTime = " + IndexPage.HttpGet_SleepTime + "ms"); 80 | return Task.FromResult(null); 81 | }); 82 | 83 | var host = CreateHostBuilder(args).Build(); 84 | 85 | task.GetAwaiter().GetResult(); 86 | 87 | Task.Delay(2000).ContinueWith((t) => 88 | { 89 | try 90 | { 91 | foreach (var nw in Dns.GetHostEntry(Dns.GetHostName()).AddressList) 92 | { 93 | if (nw.AddressFamily == AddressFamily.InterNetwork) 94 | { 95 | Console.WriteLine("http://" + nw.ToString() + ":" + App.HttpPort); 96 | } 97 | } 98 | } 99 | catch 100 | { 101 | 102 | } 103 | try 104 | { 105 | 106 | var resultsPath = "http://127.0.0.1:" + App.HttpPort; 107 | Console.WriteLine("use Browser to Open " + resultsPath); 108 | var psi = new ProcessStartInfo(resultsPath); 109 | psi.UseShellExecute = true; 110 | Process.Start(psi); 111 | } 112 | catch 113 | { 114 | 115 | } 116 | return Task.FromResult(null); 117 | }); 118 | 119 | host.Run(); 120 | 121 | IndexPage.Shutdown(); 122 | IndexPage.addSearchTerm(IndexPage.SystemShutdown); 123 | App.Item.GetDatabase().Dispose(); 124 | App.Indices.Dispose(); 125 | Log("DB Closed"); 126 | } 127 | 128 | public static IHostBuilder CreateHostBuilder(string[] args) => 129 | Host.CreateDefaultBuilder(args) 130 | .ConfigureWebHostDefaults(webBuilder => 131 | { 132 | webBuilder.UseSockets((soc) => 133 | { 134 | //Server doesn't wait clients. 135 | soc.Backlog = 2; 136 | Log("Backlog: " + soc.Backlog + ", IOQueueCount: " + soc.IOQueueCount + ", NoDelay: " + soc.NoDelay); 137 | }); 138 | webBuilder.ConfigureKestrel((cfg) => 139 | { 140 | ThreadPool.SetMinThreads(2, 2); 141 | ThreadPool.SetMaxThreads(25, 25); 142 | cfg.Limits.MaxConcurrentConnections = 15; 143 | cfg.Limits.MaxConcurrentUpgradedConnections = 15; 144 | }); 145 | webBuilder.ConfigureLogging(logging => 146 | { 147 | logging.AddFilter((name, lev) => 148 | { 149 | if (lev == LogLevel.Information) 150 | { 151 | switch (name) 152 | { 153 | case "Microsoft.AspNetCore.Routing.EndpointMiddleware": 154 | case "Microsoft.AspNetCore.Mvc.ViewFeatures.ViewResultExecutor": 155 | case "Microsoft.AspNetCore.Mvc.Infrastructure.ControllerActionInvoker": 156 | case "Microsoft.AspNetCore.StaticFiles.StaticFileMiddleware": 157 | case "Microsoft.AspNetCore.Hosting.Diagnostics": 158 | return false; 159 | default: 160 | return true; 161 | } 162 | } 163 | return false; 164 | }); 165 | }); 166 | webBuilder.UseStartup() 167 | .UseUrls("http://*:" + App.HttpPort); 168 | }); 169 | } 170 | 171 | 172 | 173 | } 174 | -------------------------------------------------------------------------------- /FTServer/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "iisSettings": { 3 | "windowsAuthentication": false, 4 | "anonymousAuthentication": true, 5 | "iisExpress": { 6 | "applicationUrl": "http://localhost:3636", 7 | "sslPort": 44359 8 | } 9 | }, 10 | "profiles": { 11 | "IIS Express": { 12 | "commandName": "IISExpress", 13 | "launchBrowser": true, 14 | "environmentVariables": { 15 | "ASPNETCORE_ENVIRONMENT": "Development" 16 | } 17 | }, 18 | "FTServer": { 19 | "commandName": "Project", 20 | "launchBrowser": true, 21 | "applicationUrl": "http://localhost:5000;https://localhost:5001", 22 | "environmentVariables": { 23 | "ASPNETCORE_ENVIRONMENT": "Development" 24 | } 25 | } 26 | } 27 | } -------------------------------------------------------------------------------- /FTServer/Startup.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using Microsoft.AspNetCore.Builder; 6 | using Microsoft.AspNetCore.Hosting; 7 | using Microsoft.AspNetCore.HttpsPolicy; 8 | using Microsoft.Extensions.Configuration; 9 | using Microsoft.Extensions.DependencyInjection; 10 | using Microsoft.Extensions.Hosting; 11 | 12 | 13 | namespace FTServer 14 | { 15 | 16 | 17 | public class Startup 18 | { 19 | public Startup(IConfiguration configuration) 20 | { 21 | Configuration = configuration; 22 | } 23 | 24 | public IConfiguration Configuration { get; } 25 | 26 | // This method gets called by the runtime. Use this method to add services to the container. 27 | public void ConfigureServices(IServiceCollection services) 28 | { 29 | services.AddControllersWithViews(); 30 | } 31 | 32 | // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. 33 | public void Configure(IApplicationBuilder app, IWebHostEnvironment env) 34 | { 35 | /* 36 | app.Use(async (context, next) => 37 | { 38 | context.Response.Headers.Add("Cache-Control", "non-cache, no-store, must-revalidate"); 39 | await next(); 40 | }); 41 | */ 42 | Console.WriteLine(env.ApplicationName); 43 | Console.WriteLine(env.EnvironmentName); 44 | if (env.IsDevelopment()) 45 | { 46 | app.UseDeveloperExceptionPage(); 47 | } 48 | else 49 | { 50 | app.UseExceptionHandler("/Home/Error"); 51 | // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts. 52 | //app.UseHsts(); 53 | } 54 | //app.UseHttpsRedirection(); 55 | app.UseStaticFiles(); 56 | 57 | app.UseRouting(); 58 | 59 | app.UseAuthorization(); 60 | 61 | app.UseEndpoints(endpoints => 62 | { 63 | endpoints.MapControllerRoute( 64 | name: "default", 65 | pattern: "{controller=Home}/{action=Index}/{id?}"); 66 | 67 | 68 | endpoints.MapControllerRoute( 69 | name: "JustActions", 70 | pattern: "{action}", 71 | defaults: new { controller = "Home" }); 72 | }); 73 | 74 | 75 | } 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /FTServer/Views/Book/Index.cshtml: -------------------------------------------------------------------------------- 1 | @model BookModel 2 | 3 | @{ 4 | Layout = ""; 5 | 6 | Random ran = BookModel.ran; 7 | bool nodesc = ran.Next(5) == 0; 8 | if (nodesc) { 9 | Model.Description = ""; 10 | } 11 | } 12 | 13 | 14 | 15 | 16 | 17 | @Model.Title 18 | @if (Model.Description.length() >0){ 19 | 20 | 21 | } 22 | 23 | 24 |

@Model.Ex

25 |
26 |

Hello World!

27 |

@Model.Text

28 |
29 |
30 | 
CodeStart  WinWinWin.GetDatabase().CopyTo(new ShowMirror(bakAddr, bakRoot), buffer)
31 |
32 |
33 |
34 | Front Page 35 | 36 | 37 | PngPng 38 | 39 |
40 | @{ 41 | 42 | String[] books = BookController.Books; 43 | int bbase = BookController.Base; 44 | for (int i = 0; i < 10; i++) { 45 | int book = ran.Next(books.Length); 46 | int start = ran.Next(bbase); 47 | int length = ran.Next(600) * 100 + 100; 48 | if(book==0){ 49 | length = ran.Next(26000); 50 | } 51 | String url = "?book=" + book + "&start=" + start + "&length=" + length; 52 |  
53 | 54 | } 55 | } 56 | 57 | XEnd 58 | 59 | 60 | -------------------------------------------------------------------------------- /FTServer/Views/Home/About.cshtml: -------------------------------------------------------------------------------- 1 | @model AboutModel 2 | 3 | @{ 4 | ViewData["Title"] = Model.Result.Query.Replace("\"", " ").Replace(",", " ") + ","; 5 | } 6 | 7 | @section Styles{ 8 | 60 | } 61 | 62 | 63 | @section Scripts{ 64 | 80 | 99 | 147 | 201 | } 202 | 203 | 204 |
205 |
206 |
207 |
208 | 209 |
210 | 211 | 212 |
213 |
214 |
215 |
216 | 224 |
225 |
226 | 227 |
228 |
229 |
230 | @{ 231 | //Html.RenderPartial( "ResultPartial", @Model.Result ); 232 | } 233 | 234 |
235 |
236 | 237 |
238 | 239 |
-------------------------------------------------------------------------------- /FTServer/Views/Home/Admin.cshtml: -------------------------------------------------------------------------------- 1 | @model AdminModel 2 | 3 | @{ 4 | ViewData["Title"] = "Add Page Index"; 5 | @if (Model.url == null) 6 | { 7 | Model.url = "https://www.iboxdb.com/"; 8 | } 9 | } 10 | 11 | @section Styles{ 12 | 26 | } 27 | 28 | 29 | @section Scripts{ 30 | 33 | } 34 | 35 | 36 |
37 |
38 |
39 |
40 | 41 |
42 | 43 | 44 |
45 | 46 |
Description:
47 |
48 | 49 | 50 |
51 | 52 |
53 |
54 | 55 |

Input HTTP or HTTPS ://URL

56 | 57 | The indexing process will continue until all be done, check the Log for details, 58 | it needs bandwidth and time. 59 | Close the App can stop it. 60 |
61 | Some pages may not finish the Indexing process because the App has stopped, try re-add again. 62 |
63 | Some pages were designed to not be indexed, it would never show up. 64 |
65 | Search Format:
66 | [Word1 Word2 Word3] = text has Word1 and Word2 and Word3
67 | ["Word1 Word2 Word3"] = text has "Word1 Word2 Word3" as a whole
68 |
69 | 70 | 80 |
81 |
-------------------------------------------------------------------------------- /FTServer/Views/Home/Index.cshtml: -------------------------------------------------------------------------------- 1 | @{ 2 | ViewData["Title"] = "iBoxDB"; 3 | 4 | DateTime begin = DateTime.Now; 5 | } 6 | 7 | @section Styles{ 8 | 28 | } 29 |
30 |
31 | 32 |

33 | Full Text Search Server 34 |

35 |
36 |
37 |
38 | 39 | 40 |
41 |
42 |
43 | 51 | 52 |
53 | Add Page Index
54 |
55 | @foreach (var t in IndexPage.getSearchTerm(10)) 56 | { 57 | var str = t.keywords; 58 | if (str.Equals(IndexPage.SystemShutdown)) 59 | { 60 | continue; 61 | } 62 | @:[@str]   63 | } 64 | 65 | 66 | 67 |

Refresh Discoveries: 
68 | @foreach (String str in IndexPage.discover()) 69 | { 70 | @:[@str]   71 | } 72 |
73 |
74 |
75 | Load Time: @((DateTime.Now - begin).TotalSeconds) s 76 |
77 |
78 | -------------------------------------------------------------------------------- /FTServer/Views/Home/ResultPartial.cshtml: -------------------------------------------------------------------------------- 1 | @model ResultPartialModel 2 | 3 | @{ 4 | Layout = ""; 5 | } 6 | 7 | @{ 8 | //Model.pageCount = 2; 9 | Model.Init(); 10 | } 11 | 12 | @functions { 13 | String decodeTry(String str) 14 | { 15 | return PageText.decodeTry(str); 16 | } 17 | } 18 | 19 |
20 | @{ 21 | var pages = Model.pages; 22 | //var isFirstLoad = Model.isFirstLoad; 23 | //var pageCount = Model.pageCount; 24 | 25 | foreach (var p in pages) 26 | { 27 | if (p.priority == 0) 28 | { 29 | continue; 30 | } 31 | int minSize = 200; 32 | if (p.keyWord is KeyWordE) 33 | { 34 | minSize = 300; 35 | } 36 | bool isdesc = p.priority >= PageText.descriptionPriority; 37 | String content = isdesc ? p.text 38 | : IndexPage.getDesc(p.text, p.keyWord, minSize - 20); 39 | 40 | if (content.Length < minSize && p.page != null) 41 | { 42 | content += "... " + p.page.getRandomContent(minSize); 43 | } 44 | String sp = p.keywords.IndexOf(',') > 0 ? "," : " "; 45 | String[] keywords = p.keywords.Split(sp); 46 | 47 | 48 |
49 |
@p.title
50 | 51 |
@content
52 | @{ 53 | var css = isdesc ? "gt" : "gtt"; 54 | } 55 |
56 | [@p.dbOrder-@p.textOrder] 57 | @if (p.isAndSearch) 58 | { 59 | } 60 | else 61 | { 62 | * 63 | } 64 | 65 | @decodeTry(p.url) 66 | 67 | @p.createTime 68 | 69 | @if (keywords.Length > 0) 70 | { 71 | @:
72 | } 73 | @foreach (var tkw in keywords) 74 | { 75 | var kw = tkw.Trim(); 76 | if (kw == null || kw.Length < 1) { continue; } 77 | if (kw.IndexOf(" ") > 0) 78 | { 79 | //kw = "\"" + kw + "\""; 80 | } 81 |   @kw   82 |   83 | } 84 |
85 |
86 |
87 | } 88 | } 89 |
90 | 91 |
92 | @{ 93 | String tcontent = (DateTime.Now - Model.begin).TotalSeconds + "s, " 94 | + "MEM:" + (System.GC.GetTotalMemory(false) / 1024 / 1024) + "MB "; 95 | 96 | @: @Model.Query, TIME: @tcontent, 97 | 98 | @if (!@Model.IsEnd()) 99 | { 100 | @:"Loading" 101 | 102 | } 103 | else 104 | { 105 | @:"END" 106 | 107 | } 108 | 109 | } 110 |
111 |
112 | -------------------------------------------------------------------------------- /FTServer/Views/Shared/Error.cshtml: -------------------------------------------------------------------------------- 1 | @model ErrorViewModel 2 | @{ 3 | ViewData["Title"] = "Error"; 4 | } 5 | 6 |

Error.

7 |

An error occurred while processing your request.

8 | 9 | @if (Model.ShowRequestId) 10 | { 11 |

12 | Request ID: @Model.RequestId 13 |

14 | } 15 | 16 |

Development Mode

17 |

18 | Swapping to Development environment will display more detailed information about the error that occurred. 19 |

20 |

21 | The Development environment shouldn't be enabled for deployed applications. 22 | It can result in displaying sensitive information from exceptions to end users. 23 | For local debugging, enable the Development environment by setting the ASPNETCORE_ENVIRONMENT environment variable to Development 24 | and restarting the app. 25 |

26 | -------------------------------------------------------------------------------- /FTServer/Views/Shared/_Layout.cshtml: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | @ViewData["Title"] - iBoxDB NoSQL Database Full Text Search Engine Server 8 | 9 | 10 | 11 | 12 | 13 | @RenderSection("Styles", required: false) 14 | @RenderSection("Scripts", required: false) 15 | 16 | 17 | 18 | @RenderBody() 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /FTServer/Views/_ViewImports.cshtml: -------------------------------------------------------------------------------- 1 | @using FTServer 2 | @using FTServer.Models 3 | @using FTServer.Controllers; 4 | @using System.Text.Encodings.Web 5 | 6 | 7 | @addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers 8 | -------------------------------------------------------------------------------- /FTServer/Views/_ViewStart.cshtml: -------------------------------------------------------------------------------- 1 | @{ 2 | Layout = "_Layout"; 3 | } 4 | -------------------------------------------------------------------------------- /FTServer/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Warning", 5 | "Microsoft.Hosting.Lifetime": "Information", 6 | "Microsoft.AspNetCore.Hosting.Diagnostics": "Warning" 7 | } 8 | }, 9 | "AllowedHosts": "*" 10 | } -------------------------------------------------------------------------------- /FTServer/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Warning", 5 | "Microsoft.Hosting.Lifetime": "Information", 6 | "Microsoft.AspNetCore.Hosting.Diagnostics": "Warning" 7 | } 8 | }, 9 | "AllowedHosts": "*" 10 | } -------------------------------------------------------------------------------- /FTServer/fts.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iboxdb/ftserver-cs/ff795ed0d6ddf7d6d312245a61b43567617e52b7/FTServer/fts.ico -------------------------------------------------------------------------------- /FTServer/wwwroot/css/themes/default/assets/fonts/icons.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iboxdb/ftserver-cs/ff795ed0d6ddf7d6d312245a61b43567617e52b7/FTServer/wwwroot/css/themes/default/assets/fonts/icons.woff2 -------------------------------------------------------------------------------- /FTServer/wwwroot/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/iboxdb/ftserver-cs/ff795ed0d6ddf7d6d312245a61b43567617e52b7/FTServer/wwwroot/favicon.ico -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Full Text Search Engine Server for CSharp 2 | 3 | 4 | ### User Guide 5 | 6 | #### Setup 7 | 8 | 1. [Install .NET](https://dotnet.microsoft.com/download) 9 | 10 | 2. Download this Project. 11 | 12 | 3. Run 13 | 14 | ```sh 15 | cd FTServer 16 | dotnet run -c Release 17 | ``` 18 | 19 | 4. Open [http://127.0.0.1:5066/](http://127.0.0.1:5066/) 20 | 21 | 5. Press [Ctrl+C] to shut down. 22 | 23 | 24 | ![](../../../../iboxdb/ftserver/raw/master/FTServer/src/main/webapp/css/fts.png) 25 | 26 | 27 | Input a Full URL to index the Page, then search. 28 | 29 | Move page forward by re-indexing the page. 30 | 31 | 32 | #### Search Format 33 | 34 | [Word1 Word2 Word3] => text has **Word1** and **Word2** and **Word3** 35 | 36 | ["Word1 Word2 Word3"] => text has **"Word1 Word2 Word3"** as a whole 37 | 38 | Search [https] or [http] => get almost all pages 39 | 40 | 41 | 42 | ### Developer Guide 43 | 44 | [Download Visual Studio Code](https://code.visualstudio.com/) 45 | 46 | #### Dependencies 47 | 48 | [iBoxDB](http://www.iboxdb.com/) 49 | 50 | [AngleSharp](https://github.com/AngleSharp/AngleSharp) 51 | 52 | [Semantic-UI](http://semantic-ui.com/) 53 | 54 | 55 | 56 | #### The Results Order 57 | The results order based on the **id()** number in **class PageText**, descending order. 58 | 59 | A Page has many PageTexts. if don't need multiple Texts, modify **Html.getDefaultTexts(Page)**, returns only one PageText (the page description text only, **Config.DescriptionOnly=true** ). 60 | 61 | the Page.GetRandomContent() method is used to keep the Search-Page-Content always changing, doesn't affect the real PageText order. 62 | 63 | Use the ID number to control the order instead of loading all pages to memory. 64 | 65 | 66 | #### Search Method 67 | search (... String keywords, long **startId**, long **count**) 68 | 69 | **startId** => which ID(the id when you created PageText) to start, 70 | use (startId=Long.MaxValue) to read from the top, descending order 71 | 72 | **count** => records to read, **important parameter**, the search speed depends on this parameter, not how big the data is. 73 | 74 | 75 | ##### Next Page 76 | set the startId as the last id from the results of search minus one 77 | 78 | ```cs 79 | startId = search( "keywords", startId, count); 80 | nextpage_startId = startId - 1 // this 'minus one' has done inside search() 81 | ... 82 | //read next page 83 | search("keywords", nextpage_startId, count) 84 | ``` 85 | 86 | mostly, the nextpage_startId is posted from client browser when user reached the end of webpage, 87 | and set the default nextpage_startId=Long.MaxValue, 88 | in javascript the big number have to write as String ("'" + nextpage_startId + "'") 89 | 90 | 91 | 92 | #### Private Server 93 | Open 94 | ```cs 95 | public Page Html.Get(String url); 96 | ``` 97 | Set your private WebSite text 98 | ```cs 99 | Page page = new Page(); 100 | page.url = url; 101 | page.title = title; 102 | page.text = bodyText 103 | page... = ... 104 | return page; 105 | ``` 106 | 107 | #### Configure Cache 108 | 109 | Setting Index Readonly Cache (Readonly_MaxDBCount) from [FTServer/Code/Config.cs](FTServer/Code/Config.cs) . 110 | 111 | 112 | 113 | #### Set Maximum Opened Files 114 | 115 | 116 | ```sh 117 | [user@localhost ~]$ cat /proc/sys/fs/file-max 118 | 803882 119 | [user@localhost ~]$ ulimit -a | grep files 120 | open files (-n) 500000 121 | [user@localhost ~]$ ulimit -Hn 122 | 500000 123 | [user@localhost ~]$ ulimit -Sn 124 | 500000 125 | [user@localhost ~]$ 126 | 127 | 128 | $ vi /etc/security/limits.conf 129 | * hard nofile 500000 130 | * soft nofile 500000 131 | root hard nofile 500000 132 | root soft nofile 500000 133 | 134 | 135 | [user@localhost ~]$ firewall-cmd --add-port=5066/tcp --permanent 136 | 137 | ``` 138 | 139 | 140 | #### Stop Tracker daemon 141 | 142 | [Why does Tracker consume resources on my PC?](https://gnome.pages.gitlab.gnome.org/tracker/faq/#why-does-tracker-consume-resources-on-my-pc) 143 | 144 | ```sh 145 | [user@localhost ~]$ tracker daemon -k 146 | 147 | [user@localhost project]$ tracker reset --hard 148 | ``` 149 | 150 | 151 | 152 | #### More 153 | [Transplant from Full Text Search Java JSP Version](https://github.com/iboxdb/ftserver) 154 | 155 | 156 | 157 |
158 | 159 | ![Flag](https://s05.flagcounter.com/count2/Ep/bg_373737/txt_F2F2F2/border_373737/columns_3/maxflags_12/viewers_0/labels_0/pageviews_1/flags_0/percent_0/) 160 | 161 | 162 | --------------------------------------------------------------------------------