- √异步、多线程地实时爬取多个网站。
- √可以非明文地缓存历史搜索结果，在本地历史纪录中更快速地搜索关键字。
- √绿色天然无公害，远离弹窗，远离广告。
9 |
10 | # Customize
11 | MagnetX提供了一个不错的框架，如果你有兴趣也可以实现自己的数据源，自给自足，丰衣足食。
12 | 只要在程序集下的类同时具有以下3个特点，你定义的源的实现即可通过C#的Reflection机制直接被识别：
13 | 1.继承自`MagnetX.Searcher.Searcher`类
14 | 2.类具有`MagnetX.Searcher.SearcherEnabledAttribute`属性
15 | 3.具有无参数构造函数
16 | ## 继承MagnetX.Searcher.SimpleWebSearcher类
17 | 对于网页爬虫源，最快捷的实现方式为继承`MagnetX.Searcher.SimpleWebSearcher`类，并给出解析源的关键实现。
18 | ```csharp
19 | [SearcherEnabled] //标记源为启用的属性
20 | class FooWebSearcher : SimpleWebSearcher //通过SimpleWebSearcher间接继承Searcher类
21 | {
22 | //必须覆盖的方法
23 | //定义word关键词第i页结果所在的URL
24 | protected override async Task", //每一个搜索结果的前导特征字符串。基类会对结果用这个字符串进行切割，并处理从下标1开始的字串。
35 | new Regex("target=\"_blank\">(.+?)", RegexOptions.Compiled), //资源名称的正则表达式。
36 | new Regex("(.+?)<", RegexOptions.Compiled)) //资源文件大小的正则表达式。
38 | {
39 |
40 | }
41 |
42 | //可选的覆盖方法
43 | //在这里可以对基类得到的结果进行进一步处理，例如去除一些标签。
44 | //有时正则表达式无法直接得到最终结果（例如，需要参考资源详情页面才能获得资源哈希值），
45 | //也可以在这个方法里做进一步处理。
46 | protected override async Task GetResultAsync(string part)
47 | {
48 | Result result = await base.GetResultAsync(part).ConfigureAwait(false); //获得基类的得到的中间结果
49 | if (result == null) return null; //注意基类结果发生错误时结果为null
50 |
51 | result.Name = result.Name.Replace("", "").Replace("", "");
52 | return result;
53 | }
54 | }
55 | ```
56 |
--------------------------------------------------------------------------------
/Searcher/HistorySearcher/HistoryRecorder.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Data.OleDb;
4 | using System.IO;
5 | using System.IO.Compression;
6 | using System.Linq;
7 | using System.Text;
8 | using System.Threading;
9 | using System.Threading.Tasks;
10 |
11 | namespace MagnetX.Searcher.HistorySearcher
12 | {
13 | ///
14 | /// 历史记录的记录类。
15 | ///
16 | class HistoryRecorder
17 | {
18 | ///
19 | /// 记录的文件流。
20 | ///
21 | protected BufferedStream Stream { get; } = new BufferedStream(Utils.HistoryStreamForWrite);
22 |
23 | ///
24 | /// 判断重复记录的哈希表。
25 | ///
26 | protected HashSet Set { get; } = new HashSet();
27 |
28 | ///
29 | /// 单线程插入的信号量。
30 | ///
31 | protected SemaphoreSlim Lock { get; } = new SemaphoreSlim(0);
32 |
33 | protected async void Prepare()
34 | {
35 | using (Stream stream = new BufferedStream(Utils.HistoryStreamForRead))
36 | {
37 | Result result = await Utils.ReadResult(stream).ConfigureAwait(false);
38 | while (result != null)
39 | {
40 | Set.Add(result.Magnet);
41 |
42 | result = await Utils.ReadResult(stream).ConfigureAwait(false);
43 | }
44 | }
45 |
46 | Lock.Release();
47 | }
48 |
49 | public HistoryRecorder()
50 | {
51 | Prepare();
52 | }
53 |
54 | ///
55 | /// 插入一组历史纪录。
56 | ///
57 | public async Task Insert(IEnumerable results)
58 | {
59 | await Lock.WaitAsync().ConfigureAwait(false);
60 | {
61 | int count = 0;
62 | Stream.Seek(0, SeekOrigin.End);
63 | foreach (Result result in results)
64 | {
65 | if (!Set.Contains(result.Magnet))
66 | {
67 | await Utils.WriteResult(Stream, result).ConfigureAwait(false);
68 | count++;
69 | }
70 | }
71 | if (count != 0) await Stream.FlushAsync().ConfigureAwait(false);
72 | }
73 | Lock.Release();
74 | }
75 | }
76 | }
77 |
--------------------------------------------------------------------------------
/Searcher/HistorySearcher/HistorySearcher.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Linq;
4 | using System.Text;
5 | using System.Threading.Tasks;
6 | using System.Data.OleDb;
7 | using System.Threading;
8 | using System.Windows.Forms;
9 | using System.IO;
10 | using System.IO.Compression;
11 |
12 | namespace MagnetX.Searcher.HistorySearcher
13 | {
14 | ///
15 | /// 历史记录的搜索类。
16 | ///
17 | [SearcherEnabled]
18 | class HistorySearcher : Searcher
19 | {
20 | protected bool Matches(string[] words, string name)
21 | {
22 | foreach(string word in words)
23 | {
24 | if (name.IndexOf(word) == -1) return false;
25 | }
26 | return true;
27 | }
28 |
29 | public override string Name
30 | {
31 | get
32 | {
33 | return "本地记录";
34 | }
35 | }
36 |
37 | public override async void SearchAsync(string word)
38 | {
39 | string[] words = word.Split(' ', '\t');
40 | List list = new List();
41 | try
42 | {
43 | using (BufferedStream fs = new BufferedStream(Utils.HistoryStreamForRead))
44 | {
45 | while (true)
46 | {
47 | Result result = await Utils.ReadResult(fs).ConfigureAwait(false);
48 | if (result == null) break;
49 |
50 | result.From = Name;
51 | if (Matches(words,result.Name))
52 | {
53 | list.Add(result);
54 | }
55 | if (list.Count == 100)
56 | {
57 | if (OnResults == null) break;
58 | if (!OnResults.Invoke(this, list)) break;
59 | list.Clear();
60 | }
61 | }
62 | }
63 | }
64 | catch { }
65 |
66 | try
67 | {
68 | if (list.Count != 0)
69 | {
70 | if (OnResults != null) OnResults.Invoke(this, list);
71 | list.Clear();
72 | }
73 | }
74 | catch { }
75 | }
76 |
77 | public override async Task TestAsync()
78 | {
79 | return TestResults.OK;
80 | }
81 | }
82 | }
83 |
--------------------------------------------------------------------------------
/Searcher/HistorySearcher/Utils.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Data.OleDb;
4 | using System.Linq;
5 | using System.Text;
6 | using System.Threading.Tasks;
7 | using System.IO;
8 | using System.IO.Compression;
9 |
10 | namespace MagnetX.Searcher.HistorySearcher
11 | {
12 | ///
13 | /// 历史纪录的工具类。
14 | /// 提供了对历史文件流的操作。
15 | ///
16 | static class Utils
17 | {
18 | private static string History { get; } = "History.bin";
19 | private static Encoding Encoding { get; } = Encoding.UTF8;
20 | public static FileStream HistoryStreamForRead { get { return new FileStream(History, FileMode.OpenOrCreate, FileAccess.Read, FileShare.ReadWrite); } }
21 | public static FileStream HistoryStreamForWrite { get { return new FileStream(History, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read); } }
22 |
23 | public static async Task ReadResult(Stream stream)
24 | {
25 | try
26 | {
27 | int size = (stream.ReadByte() << 24)
28 | + (stream.ReadByte() << 16)
29 | + (stream.ReadByte() << 8)
30 | + stream.ReadByte();
31 | byte[] buffer = new byte[size];
32 |
33 | await stream.ReadAsync(buffer, 0, size).ConfigureAwait(false);
34 | using (BinaryReader br = new BinaryReader(new DeflateStream(new MemoryStream(buffer), CompressionMode.Decompress), Encoding))
35 | {
36 | Result result = new Result();
37 | result.Name = br.ReadString();
38 | result.Magnet = br.ReadString();
39 | result.Size = br.ReadString();
40 |
41 | return result;
42 | }
43 | }
44 | catch (Exception ex)
45 | {
46 | return null;
47 | }
48 | }
49 |
50 | public static async Task WriteResult(Stream stream, Result result)
51 | {
52 | try
53 | {
54 | using (MemoryStream ms = new MemoryStream())
55 | {
56 | using (BinaryWriter bw = new BinaryWriter(new DeflateStream(ms, CompressionMode.Compress, true), Encoding))
57 | {
58 | bw.Write(result.Name);
59 | bw.Write(result.Magnet);
60 | bw.Write(result.Size);
61 | }
62 |
63 | ms.Seek(0, SeekOrigin.Begin);
64 | int size = (int)ms.Length;
65 | await stream.WriteAsync(new byte[] {
66 | (byte)(size >> 24),
67 | (byte)(size >> 16),
68 | (byte)(size >> 8),
69 | (byte)(size),
70 | }, 0, 4).ConfigureAwait(false);
71 | await ms.CopyToAsync(stream).ConfigureAwait(false);
72 |
73 | return true;
74 | }
75 | }
76 | catch (Exception ex)
77 | {
78 | return false;
79 | }
80 | }
81 | }
82 | }
83 |
--------------------------------------------------------------------------------
/Searcher/Searcher.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Linq;
4 | using System.Text;
5 | using System.Threading.Tasks;
6 |
7 | namespace MagnetX.Searcher
8 | {
9 | ///
10 | /// 搜索结果。
11 | ///
12 | public class Result
13 | {
14 | public string Name { get; set; }
15 | public string Magnet { get; set; }
16 | public string Size { get; set; }
17 | public string From { get; set; }
18 | }
19 |
20 | ///
21 | ///
23 | public enum TestResults
24 | {
25 | OK,
26 | Timeout,
27 | ServerError,
28 | FormatError,
29 | UnknownError,
30 | }
31 |
32 | ///
33 | /// 标记
35 | [AttributeUsage(AttributeTargets.Class, AllowMultiple = false, Inherited = false)]
36 | public class SearcherEnabledAttribute : System.Attribute
37 | {
38 |
39 | }
40 |
41 | ///
42 | ///
44 | /// 返回搜索结果的指示是否继续搜索。
47 | delegate bool SearcherResultsDelegate(Searcher sender, List results);
48 |
49 | ///
50 | /// 数据源的抽象类。
51 | ///
52 | abstract class Searcher
53 | {
54 | public SearcherResultsDelegate OnResults = null;
55 |
56 | public object Tag { get; set; } = null;
57 | public abstract string Name { get; }
58 | public abstract void SearchAsync(string word);
59 | public abstract Task TestAsync();
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/Searcher/WebSearcher/Bt177WebSearcher.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Linq;
4 | using System.Text;
5 | using System.Text.RegularExpressions;
6 | using System.Threading.Tasks;
7 |
8 | namespace MagnetX.Searcher.WebSearcher
9 | {
10 | [SearcherEnabled]
11 | class Bt177WebSearcher : SimpleWebSearcher
12 | {
13 | protected override async Task GetURLAsync(string word, int page)
14 | {
15 | string text = Uri.EscapeUriString(word);
16 | return "http://www.bt177.biz/word/" + text + "_" + page + ".html";
17 | }
18 |
19 | public Bt177WebSearcher()
20 | : base("bt177.biz", "
",
21 | new Regex("title=\"(.+?)\"", RegexOptions.Compiled),
22 | new Regex("href=\"/read/(.+?)\\.html", RegexOptions.Compiled),
23 | new Regex("大小.+?>(.+?)", RegexOptions.Compiled))
24 | {
25 |
26 | }
27 | }
28 | }
29 |
--------------------------------------------------------------------------------
/Searcher/WebSearcher/BtcatWebSearcher.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Text.RegularExpressions;
4 | using System.Threading.Tasks;
5 |
6 | namespace MagnetX.Searcher.WebSearcher
7 | {
8 | [SearcherEnabled]
9 | class BtcatWebSearcher : SimpleWebSearcher
10 | {
11 | protected override async Task GetURLAsync(string word, int page)
12 | {
13 | string text = Uri.EscapeUriString(word);
14 | return "https://btcat.org/search/" + text + "/rela-" + page + ".html";
15 | }
16 |
17 | public BtcatWebSearcher()
18 | : base("btcat.org", "
",
19 | new Regex("(.+?)<", RegexOptions.Compiled))
22 | {
23 |
24 | }
25 | }
26 | }
27 |
--------------------------------------------------------------------------------
/Searcher/WebSearcher/BtsoWebSearcher.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Net.Http;
4 | using System.Text.RegularExpressions;
5 | using System.Threading.Tasks;
6 |
7 | namespace MagnetX.Searcher.WebSearcher
8 | {
9 | [SearcherEnabled]
10 | class BtsoWebSearcher : SimpleWebSearcher
11 | {
12 | protected override async Task GetURLAsync(string word, int page)
13 | {
14 | string text = Uri.EscapeUriString(word);
15 | return "https://btso.pw/search/" + text + "/page/" + page;
16 | }
17 |
18 | public BtsoWebSearcher()
19 | : base("btso.pw", "
",
20 | new Regex("title=\"(.+?)\"", RegexOptions.Compiled),
21 | new Regex("/hash/(.+?)\"", RegexOptions.Compiled),
22 | new Regex("size\">(.+?)<", RegexOptions.Compiled))
23 | {
24 |
25 | }
26 |
27 | protected override HttpClient CreateHttpClient()
28 | {
29 | HttpClient httpClient = base.CreateHttpClient();
30 | httpClient.DefaultRequestHeaders.Add("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
31 | return httpClient;
32 | }
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/Searcher/WebSearcher/BturlWebSearcher.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Linq;
4 | using System.Text;
5 | using System.Text.RegularExpressions;
6 | using System.Threading.Tasks;
7 |
8 | namespace MagnetX.Searcher.WebSearcher
9 | {
10 | [SearcherEnabled]
11 | class BturlWebSearcher : SimpleWebSearcher
12 | {
13 | protected override async Task GetURLAsync(string word, int page)
14 | {
15 | string name = Uri.EscapeUriString(word);
16 | return "https://www.bturl.cc/search/" + name + "_click_" + page + ".html";
17 | }
18 |
19 | public BturlWebSearcher()
20 | : base("bturl.cc", "",
21 | new Regex("
",
21 | new Regex("(.+)(|\n)", RegexOptions.Compiled),
22 | new Regex("href=\"/(.+?)\\.html", RegexOptions.Compiled),
23 | new Regex("文件大小.+?>(.+?)", RegexOptions.Compiled))
24 | {
25 |
26 | }
27 | }
28 | }
29 |
--------------------------------------------------------------------------------
/Searcher/WebSearcher/CilibaWebSearcher.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Linq;
4 | using System.Text;
5 | using System.Text.RegularExpressions;
6 | using System.Threading.Tasks;
7 |
8 | namespace MagnetX.Searcher.WebSearcher
9 | {
10 | [SearcherEnabled]
11 | class CilibaWebSearcher : SimpleWebSearcher
12 | {
13 | protected override async Task GetURLAsync(string word, int page)
14 | {
15 | string name = Uri.EscapeUriString(word);
16 | return "https://www.ciliba.biz/s/" + name + "_rel_" + page + ".html";
17 | }
18 |
19 | public CilibaWebSearcher()
20 | : base("ciliba.biz", "",
21 | new Regex("target=\"_blank\">(.+?)", RegexOptions.Compiled),
22 | new Regex("(.+?)<", RegexOptions.Compiled))
24 | {
25 |
26 | }
27 |
28 | protected override async Task GetResultAsync(string part)
29 | {
30 | Result result = await base.GetResultAsync(part).ConfigureAwait(false);
31 | if (result == null) return null;
32 |
33 | result.Name = result.Name.Replace("", "").Replace("", "");
34 | return result;
35 | }
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/Searcher/WebSearcher/CililianxWebSearcher.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Linq;
4 | using System.Net.Http;
5 | using System.Text;
6 | using System.Text.RegularExpressions;
7 | using System.Threading.Tasks;
8 |
9 | namespace MagnetX.Searcher.WebSearcher
10 | {
11 | [SearcherEnabled]
12 | class CililianxWebSearcher : SimpleWebSearcher
13 | {
14 | protected override async Task GetURLAsync(string word, int page)
15 | {
16 | string name = Uri.EscapeUriString(word);
17 | return "http://cililianx.com/list/" + name + "/" + page + ".html";
18 | }
19 |
20 | public CililianxWebSearcher()
21 | : base("cililianx.com", "",
22 | new Regex("(.+?)", RegexOptions.Compiled),
23 | new Regex("/xiangxi/(.+?)\"", RegexOptions.Compiled),
24 | new Regex("种子大小[\\D]+\\>\\s*(\\d[^\\<]+)", RegexOptions.Compiled))
25 | {
26 |
27 | }
28 |
29 | protected override async Task GetResultAsync(string part)
30 | {
31 | Result result = await base.GetResultAsync(part).ConfigureAwait(false);
32 | if (result == null) return null;
33 |
34 | result.Name = result.Name.Replace("", "").Replace("", "");
35 | return result;
36 | }
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/Searcher/WebSearcher/CilimaoWebSearcher.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Linq;
4 | using System.Text;
5 | using System.Text.RegularExpressions;
6 | using System.Threading.Tasks;
7 |
8 | namespace MagnetX.Searcher.WebSearcher
9 | {
10 | [SearcherEnabled]
11 | class CilimaoWebSearcher : SimpleWebSearcher
12 | {
13 | protected override async Task GetURLAsync(string word, int page)
14 | {
15 | string name = Uri.EscapeUriString(word);
16 | return "https://www.cilimao.me/search?word=" + name + "&sortProperties=download_count&page=" + page;
17 | }
18 |
19 | public CilimaoWebSearcher()
20 | : base("cilimao.me", "(.+?)", RegexOptions.Compiled),
22 | new Regex("href=\"/information/(.+?)\\?", RegexOptions.Compiled),
23 | new Regex("文件大小.+?-->(.+?)
",
21 | new Regex("target=\"_blank\">(.+?)", RegexOptions.Compiled),
22 | new Regex("(.+?)<", RegexOptions.Compiled))
24 | {
25 |
26 | }
27 |
28 | protected override async Task GetResultAsync(string part)
29 | {
30 | Result result = await base.GetResultAsync(part).ConfigureAwait(false);
31 | if (result == null) return null;
32 |
33 | result.Name = result.Name.Replace("", "").Replace("", "");
34 | return result;
35 | }
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/Searcher/WebSearcher/CililianxWebSearcher.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Linq;
4 | using System.Net.Http;
5 | using System.Text;
6 | using System.Text.RegularExpressions;
7 | using System.Threading.Tasks;
8 |
9 | namespace MagnetX.Searcher.WebSearcher
10 | {
11 | [SearcherEnabled]
12 | class CililianxWebSearcher : SimpleWebSearcher
13 | {
14 | protected override async Task GetURLAsync(string word, int page)
15 | {
16 | string name = Uri.EscapeUriString(word);
17 | return "http://cililianx.com/list/" + name + "/" + page + ".html";
18 | }
19 |
20 | public CililianxWebSearcher()
21 | : base("cililianx.com", "
",
22 | new Regex("(.+?)", RegexOptions.Compiled),
23 | new Regex("/xiangxi/(.+?)\"", RegexOptions.Compiled),
24 | new Regex("种子大小[\\D]+\\>\\s*(\\d[^\\<]+)", RegexOptions.Compiled))
25 | {
26 |
27 | }
28 |
29 | protected override async Task GetResultAsync(string part)
30 | {
31 | Result result = await base.GetResultAsync(part).ConfigureAwait(false);
32 | if (result == null) return null;
33 |
34 | result.Name = result.Name.Replace("", "").Replace("", "");
35 | return result;
36 | }
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/Searcher/WebSearcher/CilimaoWebSearcher.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Linq;
4 | using System.Text;
5 | using System.Text.RegularExpressions;
6 | using System.Threading.Tasks;
7 |
8 | namespace MagnetX.Searcher.WebSearcher
9 | {
10 | [SearcherEnabled]
11 | class CilimaoWebSearcher : SimpleWebSearcher
12 | {
13 | protected override async Task GetURLAsync(string word, int page)
14 | {
15 | string name = Uri.EscapeUriString(word);
16 | return "https://www.cilimao.me/search?word=" + name + "&sortProperties=download_count&page=" + page;
17 | }
18 |
19 | public CilimaoWebSearcher()
20 | : base("cilimao.me", "
(.+?)", RegexOptions.Compiled),
22 | new Regex("href=\"/information/(.+?)\\?", RegexOptions.Compiled),
23 | new Regex("文件大小.+?-->(.+?)