├── .gitattributes ├── .gitignore ├── Bencode ├── BDict.cs ├── BInt.cs ├── BList.cs ├── BString.cs ├── BencodingUtils.cs └── IBencodingType.cs ├── DHTBucketManager.cs ├── DHTCrawler.csproj ├── DHTCrawler.sln ├── DHTNode.cs ├── MySQL.cs ├── Program.cs ├── Properties └── AssemblyInfo.cs └── app.config /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | 7 | # Standard to msysgit 8 | *.doc diff=astextplain 9 | *.DOC diff=astextplain 10 | *.docx diff=astextplain 11 | *.DOCX diff=astextplain 12 | *.dot diff=astextplain 13 | *.DOT diff=astextplain 14 | *.pdf diff=astextplain 15 | *.PDF diff=astextplain 16 | *.rtf diff=astextplain 17 | *.RTF diff=astextplain 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.userosscache 8 | *.sln.docstates 9 | 10 | # Build results 11 | [Dd]ebug/ 12 | [Dd]ebugPublic/ 13 | [Rr]elease/ 14 | [Rr]eleases/ 15 | x64/ 16 | x86/ 17 | build/ 18 | bld/ 19 | [Bb]in/ 20 | [Oo]bj/ 21 | 22 | # Roslyn cache directories 23 | *.ide/ 24 | 25 | # MSTest test Results 26 | [Tt]est[Rr]esult*/ 27 | [Bb]uild[Ll]og.* 28 | 29 | #NUNIT 30 | *.VisualState.xml 31 | TestResult.xml 32 | 33 | # Build Results of an ATL Project 34 | [Dd]ebugPS/ 35 | [Rr]eleasePS/ 36 | dlldata.c 37 | 38 | *_i.c 39 | *_p.c 40 | *_i.h 41 | *.ilk 42 | *.meta 43 | *.obj 44 | *.pch 45 | *.pdb 46 | *.pgc 47 | *.pgd 48 | *.rsp 49 | *.sbr 50 | *.tlb 51 | *.tli 52 | *.tlh 53 | *.tmp 54 | *.tmp_proj 55 | *.log 56 | *.vspscc 57 | *.vssscc 58 | .builds 59 | *.pidb 60 | *.svclog 61 | *.scc 62 | 63 | # Chutzpah Test files 64 | _Chutzpah* 65 | 66 | # Visual C++ cache files 67 | ipch/ 68 | *.aps 69 | *.ncb 70 | *.opensdf 71 | *.sdf 72 | *.cachefile 73 | 74 | # Visual Studio profiler 75 | *.psess 76 | *.vsp 77 | *.vspx 78 | 79 | # TFS 2012 Local Workspace 80 | $tf/ 81 | 82 | # Guidance Automation Toolkit 83 | *.gpState 84 | 85 | # ReSharper is a .NET coding add-in 86 | _ReSharper*/ 87 | *.[Rr]e[Ss]harper 88 | *.DotSettings.user 89 | 90 | # JustCode is a .NET coding addin-in 91 | .JustCode 92 | 93 | # TeamCity is a build add-in 94 | _TeamCity* 95 | 96 | # DotCover is a Code Coverage Tool 97 | *.dotCover 98 | 99 | # NCrunch 100 | _NCrunch_* 101 | .*crunch*.local.xml 102 | 103 | # MightyMoose 104 | *.mm.* 105 | AutoTest.Net/ 106 | 107 | # Web workbench (sass) 108 | .sass-cache/ 109 | 110 | # Installshield output folder 111 | [Ee]xpress/ 112 | 113 | # DocProject is a documentation generator add-in 114 | DocProject/buildhelp/ 115 | DocProject/Help/*.HxT 116 | DocProject/Help/*.HxC 117 | DocProject/Help/*.hhc 118 | DocProject/Help/*.hhk 119 | DocProject/Help/*.hhp 120 | DocProject/Help/Html2 121 | DocProject/Help/html 122 | 123 | # Click-Once directory 124 | publish/ 125 | 126 | # Publish Web Output 127 | *.[Pp]ublish.xml 128 | *.azurePubxml 129 | # TODO: Comment the next line if you want to checkin your web deploy settings 130 | # but database connection strings (with potential passwords) will be unencrypted 131 | *.pubxml 132 | *.publishproj 133 | 134 | # NuGet Packages 135 | *.nupkg 136 | # The packages folder can be ignored because of Package Restore 137 | **/packages/* 138 | # except build/, which is used as an MSBuild target. 139 | !**/packages/build/ 140 | # If using the old MSBuild-Integrated Package Restore, uncomment this: 141 | #!**/packages/repositories.config 142 | 143 | # Windows Azure Build Output 144 | csx/ 145 | *.build.csdef 146 | 147 | # Windows Store app package directory 148 | AppPackages/ 149 | 150 | # Others 151 | sql/ 152 | *.Cache 153 | ClientBin/ 154 | [Ss]tyle[Cc]op.* 155 | ~$* 156 | *~ 157 | *.dbmdl 158 | *.dbproj.schemaview 159 | *.pfx 160 | *.publishsettings 161 | node_modules/ 162 | 163 | # RIA/Silverlight projects 164 | Generated_Code/ 165 | 166 | # Backup & report files from converting an old project file 167 | # to a newer Visual Studio version. Backup files are not needed, 168 | # because we have git ;-) 169 | _UpgradeReport_Files/ 170 | Backup*/ 171 | UpgradeLog*.XML 172 | UpgradeLog*.htm 173 | 174 | # SQL Server files 175 | *.mdf 176 | *.ldf 177 | 178 | # Business Intelligence projects 179 | *.rdl.data 180 | *.bim.layout 181 | *.bim_*.settings 182 | 183 | # Microsoft Fakes 184 | FakesAssemblies/ 185 | 186 | # ========================= 187 | # Operating System Files 188 | # ========================= 189 | 190 | # OSX 191 | # ========================= 192 | 193 | .DS_Store 194 | .AppleDouble 195 | .LSOverride 196 | 197 | # Thumbnails 198 | ._* 199 | 200 | # Files that might appear on external disk 201 | .Spotlight-V100 202 | .Trashes 203 | 204 | # Directories potentially created on remote AFP share 205 | .AppleDB 206 | .AppleDesktop 207 | Network Trash Folder 208 | Temporary Items 209 | .apdisk 210 | 211 | # Windows 212 | # ========================= 213 | 214 | # Windows image file caches 215 | Thumbs.db 216 | ehthumbs.db 217 | 218 | # Folder config file 219 | Desktop.ini 220 | 221 | # Recycle Bin used on file shares 222 | $RECYCLE.BIN/ 223 | 224 | # Windows Installer files 225 | *.cab 226 | *.msi 227 | *.msm 228 | *.msp 229 | 230 | # Windows shortcuts 231 | *.lnk 232 | -------------------------------------------------------------------------------- /Bencode/BDict.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | 5 | namespace BencodeLibrary 6 | { 7 | public class BDict : Dictionary, IEquatable, IEquatable>, IBencodingType 8 | { 9 | /// 10 | /// Decode the next token as a dictionary. 11 | /// Assumes the next token is a dictionary. 12 | /// 13 | /// 14 | /// Decoded dictionary 15 | public static BDict Decode(BinaryReader inputStream) 16 | { 17 | // Get past 'd' 18 | inputStream.Read(); 19 | 20 | BDict res = new BDict(); 21 | 22 | // Read elements till an 'e' 23 | while (inputStream.PeekChar() != 'e') 24 | { 25 | // Key 26 | BString key = BString.Decode(inputStream); 27 | 28 | // Value 29 | IBencodingType value = BencodingUtils.Decode(inputStream); 30 | 31 | res[key.Value] = value; 32 | } 33 | 34 | // Get past 'e' 35 | inputStream.Read(); 36 | 37 | return res; 38 | } 39 | 40 | public void Encode(BinaryWriter writer) 41 | { 42 | // Write header 43 | writer.Write('d'); 44 | 45 | // Write elements 46 | foreach (KeyValuePair item in this) 47 | { 48 | // Write key 49 | BString key = new BString(); 50 | key.Value = item.Key; 51 | 52 | key.Encode(writer); 53 | 54 | // Write value 55 | item.Value.Encode(writer); 56 | } 57 | 58 | // Write footer 59 | writer.Write('e'); 60 | } 61 | 62 | public bool Equals(BDict obj) 63 | { 64 | Dictionary other = obj as Dictionary; 65 | 66 | return Equals(other); 67 | } 68 | public bool Equals(Dictionary other) 69 | { 70 | if (other == null) 71 | return false; 72 | 73 | if (other.Count != Count) 74 | return false; 75 | 76 | foreach (string key in Keys) 77 | { 78 | if (!other.ContainsKey(key)) 79 | return false; 80 | 81 | // Dictionaries cannot have nulls 82 | if (!other[key].Equals(this[key])) 83 | { 84 | // Not ok 85 | return false; 86 | } 87 | } 88 | 89 | return true; 90 | } 91 | public override bool Equals(object obj) 92 | { 93 | BDict other = obj as BDict; 94 | 95 | return Equals(other); 96 | } 97 | 98 | /// 99 | /// Adds a specified value. 100 | /// Must not be null. 101 | /// 102 | /// 103 | /// 104 | /// If the value is null 105 | public new void Add(string key, IBencodingType value) 106 | { 107 | if (value == null) 108 | throw new ArgumentNullException("value"); 109 | 110 | base.Add(key, value); 111 | } 112 | 113 | /// 114 | /// Gets or sets a value. 115 | /// Values must not be null. 116 | /// 117 | /// 118 | /// 119 | /// If the value is null 120 | public IBencodingType this[string index] 121 | { 122 | get { return base[index]; } 123 | set 124 | { 125 | if (value == null) 126 | throw new ArgumentNullException("value"); 127 | 128 | base[index] = value; 129 | } 130 | } 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /Bencode/BInt.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | 4 | namespace BencodeLibrary 5 | { 6 | public class BInt : IBencodingType, IComparable, IEquatable, IEquatable, IComparable 7 | { 8 | private BInt() 9 | { 10 | Value = 0; 11 | } 12 | public BInt(long value) 13 | { 14 | Value = value; 15 | } 16 | 17 | public long Value { get; set; } 18 | 19 | /// 20 | /// Decode the next token as a int. 21 | /// Assumes the next token is a int. 22 | /// 23 | /// 24 | /// Decoded int 25 | public static BInt Decode(BinaryReader inputStream) 26 | { 27 | // Get past 'i' 28 | inputStream.Read(); 29 | 30 | // Read numbers till an 'e' 31 | string number = ""; 32 | char ch; 33 | 34 | while ((ch = inputStream.ReadChar()) != 'e') 35 | { 36 | number += ch; 37 | } 38 | 39 | BInt res = new BInt { Value = long.Parse(number) }; 40 | 41 | return res; 42 | } 43 | 44 | public void Encode(BinaryWriter writer) 45 | { 46 | // Write header 47 | writer.Write('i'); 48 | 49 | // Write value 50 | writer.Write(Value.ToString().ToCharArray()); 51 | 52 | // Write footer 53 | writer.Write('e'); 54 | } 55 | 56 | public int CompareTo(long other) 57 | { 58 | if (Value < other) 59 | return -1; 60 | 61 | if (Value > other) 62 | return 1; 63 | 64 | return 0; 65 | } 66 | public int CompareTo(BInt other) 67 | { 68 | if (other == null) 69 | throw new ArgumentNullException("other"); 70 | 71 | if (Value < other.Value) 72 | return -1; 73 | 74 | if (Value > other.Value) 75 | return 1; 76 | 77 | return 0; 78 | } 79 | 80 | public override bool Equals(object obj) 81 | { 82 | BInt other = obj as BInt; 83 | 84 | return Equals(other); 85 | } 86 | public bool Equals(BInt other) 87 | { 88 | if (other == null) 89 | return false; 90 | 91 | return Equals(other.Value); 92 | } 93 | public bool Equals(long other) 94 | { 95 | return Value == other; 96 | } 97 | public override int GetHashCode() 98 | { 99 | return Value.GetHashCode(); 100 | } 101 | } 102 | } -------------------------------------------------------------------------------- /Bencode/BList.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | 5 | namespace BencodeLibrary 6 | { 7 | public class BList : List, IEquatable, IEquatable>, IBencodingType 8 | { 9 | /// 10 | /// Decode the next token as a list. 11 | /// Assumes the next token is a list. 12 | /// 13 | /// 14 | /// Decoded list 15 | public static BList Decode(BinaryReader inputStream) 16 | { 17 | // Get past 'l' 18 | inputStream.Read(); 19 | 20 | BList res = new BList(); 21 | 22 | // Read elements till an 'e' 23 | while (inputStream.PeekChar() != 'e') 24 | { 25 | res.Add(BencodingUtils.Decode(inputStream)); 26 | } 27 | 28 | // Get past 'e' 29 | inputStream.Read(); 30 | 31 | return res; 32 | } 33 | 34 | public void Encode(BinaryWriter writer) 35 | { 36 | // Write header 37 | writer.Write('l'); 38 | 39 | // Write elements 40 | foreach (IBencodingType item in this) 41 | { 42 | item.Encode(writer); 43 | } 44 | 45 | // Write footer 46 | writer.Write('e'); 47 | } 48 | 49 | public bool Equals(BList obj) 50 | { 51 | IList other = obj as IList; 52 | 53 | return Equals(other); 54 | } 55 | public bool Equals(IList other) 56 | { 57 | if (other == null) 58 | return false; 59 | 60 | if (other.Count != Count) 61 | return false; 62 | 63 | for (int i = 0; i < Count; i++) 64 | { 65 | // Lists cannot have nulls 66 | if (!other[i].Equals(this[i])) 67 | { 68 | // Not ok 69 | return false; 70 | } 71 | } 72 | 73 | return true; 74 | } 75 | public override bool Equals(object obj) 76 | { 77 | BList other = obj as BList; 78 | 79 | return Equals(other); 80 | } 81 | 82 | /// 83 | /// Adds a specified value. 84 | /// Must not be null. 85 | /// 86 | /// 87 | /// If the value is null 88 | public new void Add(IBencodingType value) 89 | { 90 | if (value == null) 91 | throw new ArgumentNullException("value"); 92 | 93 | base.Add(value); 94 | } 95 | 96 | /// 97 | /// Adds a range of specified values. 98 | /// None of them must be null. 99 | /// 100 | /// 101 | /// If any of the values is null 102 | public new void AddRange(IEnumerable values) 103 | { 104 | if (values == null) 105 | throw new ArgumentNullException("values"); 106 | 107 | foreach (IBencodingType value in values) 108 | { 109 | if (value == null) 110 | throw new ArgumentNullException("values"); 111 | 112 | base.Add(value); 113 | } 114 | } 115 | 116 | /// 117 | /// Gets or sets a specified value. 118 | /// The value must not be null. 119 | /// 120 | /// 121 | /// 122 | /// If the value is null 123 | public IBencodingType this[int index] 124 | { 125 | get { return base[index]; } 126 | set 127 | { 128 | if (value == null) 129 | throw new ArgumentNullException("value"); 130 | 131 | base[index] = value; 132 | } 133 | } 134 | } 135 | } -------------------------------------------------------------------------------- /Bencode/BString.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | 4 | namespace BencodeLibrary 5 | { 6 | /// 7 | /// Represents a String object. It cannot contain a null value. 8 | /// 9 | public class BString : IEquatable, IEquatable, IComparable, IComparable, IBencodingType 10 | { 11 | internal BString() 12 | { 13 | } 14 | public BString(string value) 15 | { 16 | Value = value; 17 | } 18 | 19 | private string _value = string.Empty; 20 | public string Value 21 | { 22 | get { return _value; } 23 | set 24 | { 25 | if (value != null) 26 | _value = value; 27 | } 28 | } 29 | 30 | /// 31 | /// Decode the next token as a string. 32 | /// Assumes the next token is a string. 33 | /// 34 | /// 35 | /// Decoded string 36 | public static BString Decode(BinaryReader inputStream) 37 | { 38 | // Read up to ':' 39 | string numberLength = ""; 40 | char ch; 41 | 42 | while ((ch = inputStream.ReadChar()) != ':') 43 | { 44 | numberLength += ch; 45 | } 46 | 47 | // Read chars out 48 | //char[] stringData = new char[int.Parse(numberLength)]; 49 | //inputStream.Read(stringData, 0, stringData.Length); 50 | 51 | byte[] stringData = inputStream.ReadBytes(int.Parse(numberLength)); 52 | 53 | // Return 54 | return new BString { Value = BencodingUtils.ExtendedASCIIEncoding.GetString(stringData) }; 55 | } 56 | 57 | public void Encode(BinaryWriter writer) 58 | { 59 | byte[] ascii = BencodingUtils.ExtendedASCIIEncoding.GetBytes(Value); 60 | 61 | // Write length 62 | writer.Write(BencodingUtils.ExtendedASCIIEncoding.GetBytes(ascii.Length.ToString())); 63 | 64 | // Write seperator 65 | writer.Write(':'); 66 | 67 | // Write ASCII representation 68 | writer.Write(ascii); 69 | } 70 | 71 | public int CompareTo(string other) 72 | { 73 | return StringComparer.InvariantCulture.Compare(Value, other); 74 | } 75 | public int CompareTo(BString other) 76 | { 77 | if (other == null) 78 | throw new ArgumentNullException("other"); 79 | 80 | return CompareTo(other.Value); 81 | } 82 | 83 | public override bool Equals(object obj) 84 | { 85 | BString other = obj as BString; 86 | 87 | if (other == null) 88 | return false; 89 | 90 | return Equals(other); 91 | } 92 | public bool Equals(BString other) 93 | { 94 | if (other == null) 95 | return false; 96 | 97 | if (other == this) 98 | return true; 99 | 100 | return Equals(other.Value, Value); 101 | } 102 | public bool Equals(string other) 103 | { 104 | if (other == null) 105 | return false; 106 | 107 | return Equals(Value, other); 108 | } 109 | public override int GetHashCode() 110 | { 111 | // Value should never be null 112 | return Value.GetHashCode(); 113 | } 114 | } 115 | } -------------------------------------------------------------------------------- /Bencode/BencodingUtils.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | using System.Security.Cryptography; 3 | using System.Text; 4 | 5 | namespace BencodeLibrary 6 | { 7 | public static class BencodingUtils 8 | { 9 | /// 10 | /// The encoding used by Bencoding files. As Strings are allowed to have binary 11 | /// data, we need to have an encoding that will allow this. UTF encodings won't 12 | /// work, as it may interpret several bytes as being a single character. Bencoding 13 | /// itself is ASCII based, and uses the full 256 character set (called "Extended ASCII"). 14 | /// This Encoding is also named "IBM 437", and is referred here for further use. 15 | /// If you need to read strings and decode them, you should load them with this encoding. 16 | /// 17 | /// 18 | /// To load a string from somewhere else, we need to specify that the source must be 19 | /// interpreted as Extended ASCII. Else the string will get messed up, and 20 | /// the parsing won't work. 21 | /// 22 | /// 23 | /// // Read a file using the extended ASCII encoding. 24 | /// string origTorrentString = File.ReadAllText(@"./torrentFile.torrent", BencodingUtils.ExtendedASCIIEncoding); 25 | /// 26 | /// // Parse the string as a bencoded torrent 27 | /// IBencodingType torrent = BencodingUtils.Decode(origTorrentString); 28 | /// 29 | /// 30 | public static Encoding ExtendedASCIIEncoding { get; private set; } 31 | 32 | static BencodingUtils() 33 | { 34 | // Extended ASCII encoding - http://stackoverflow.com/questions/4623650/encode-to-single-byte-extended-ascii-values 35 | ExtendedASCIIEncoding = Encoding.GetEncoding(437); 36 | } 37 | 38 | /// 39 | /// Read a file, and parse it a bencoded object. 40 | /// 41 | /// The path to the file. 42 | /// A bencoded object. 43 | public static IBencodingType DecodeFile(string fileName) 44 | { 45 | using (FileStream fileStream = File.OpenRead(fileName)) 46 | { 47 | return Decode(fileStream); 48 | } 49 | } 50 | 51 | /// 52 | /// Parse a bencoded object from a string. 53 | /// Warning: Beware of encodings. 54 | /// 55 | /// 56 | /// The bencoded string to parse. 57 | /// A bencoded object. 58 | public static IBencodingType Decode(string inputString) 59 | { 60 | byte[] byteArray = ExtendedASCIIEncoding.GetBytes(inputString); 61 | 62 | return Decode(new MemoryStream(byteArray)); 63 | } 64 | /// 65 | /// Parse a bencoded stream (for example a file). 66 | /// 67 | /// The bencoded stream to parse. 68 | /// A bencoded object. 69 | public static IBencodingType Decode(Stream inputStream) 70 | { 71 | using (BinaryReader sr = new BinaryReader(inputStream, ExtendedASCIIEncoding)) 72 | { 73 | return Decode(sr); 74 | } 75 | } 76 | internal static IBencodingType Decode(BinaryReader inputStream) 77 | { 78 | char next = (char)inputStream.PeekChar(); 79 | 80 | switch (next) 81 | { 82 | case 'i': 83 | // Integer 84 | return BInt.Decode(inputStream); 85 | 86 | case 'l': 87 | // List 88 | return BList.Decode(inputStream); 89 | 90 | case 'd': 91 | // List 92 | return BDict.Decode(inputStream); 93 | 94 | case '0': 95 | case '1': 96 | case '2': 97 | case '3': 98 | case '4': 99 | case '5': 100 | case '6': 101 | case '7': 102 | case '8': 103 | case '9': 104 | // String 105 | return BString.Decode(inputStream); 106 | } 107 | 108 | return null; 109 | } 110 | 111 | /// 112 | /// Encode the given object to a stream. 113 | /// 114 | /// The object to encode. 115 | /// The stream to write to. 116 | public static void Encode(IBencodingType bencode, Stream output) 117 | { 118 | BinaryWriter writer = new BinaryWriter(output, ExtendedASCIIEncoding); 119 | 120 | bencode.Encode(writer); 121 | 122 | writer.Flush(); 123 | } 124 | 125 | /// 126 | /// Encode the given object to a string. 127 | /// Warning: Beware of encodings, take special care when using it further. 128 | /// 129 | /// 130 | /// The bencode object to encode. 131 | /// A bencoded string with the object. 132 | public static string EncodeString(IBencodingType bencode) 133 | { 134 | MemoryStream ms = new MemoryStream(); 135 | Encode(bencode, ms); 136 | ms.Position = 0; 137 | 138 | return new StreamReader(ms, ExtendedASCIIEncoding).ReadToEnd(); 139 | } 140 | 141 | /// 142 | /// Encode the given object to a series of bytes. 143 | /// 144 | /// The bencode object to encode. 145 | /// A bencoded string of the object in Extended ASCII Encoding. 146 | public static byte[] EncodeBytes(IBencodingType bencode) 147 | { 148 | MemoryStream ms = new MemoryStream(); 149 | Encode(bencode, ms); 150 | ms.Position = 0; 151 | 152 | return new BinaryReader(ms, ExtendedASCIIEncoding).ReadBytes((int)ms.Length); 153 | } 154 | 155 | /// 156 | /// Calculates the InfoHash from a torrent. You must supply the "info" dictionary from the torrent. 157 | /// 158 | /// The "info" dictionary. 159 | /// 160 | /// This example, will load a torrent, take the "info" dictionary out of the root dictionary and hash this. 161 | /// 162 | /// BDict torrent = BencodingUtils.DecodeFile(@"torrentFile.torrent") as BDict; 163 | /// byte[] infoHash = BencodingUtils.CalculateTorrentInfoHash(torrent["info"] as BDict); 164 | /// 165 | /// 166 | /// The "infoHash" byte array now contains 20 bytes with the SHA-1 infoHash. 167 | /// 168 | /// 169 | public static byte[] CalculateTorrentInfoHash(BDict torrentInfoDict) 170 | { 171 | // Take the "info" dictionary provided, and encode it 172 | byte[] infoBytes = EncodeBytes(torrentInfoDict); 173 | 174 | // Hash the encoded dictionary 175 | return new SHA1CryptoServiceProvider().ComputeHash(infoBytes); 176 | } 177 | } 178 | } -------------------------------------------------------------------------------- /Bencode/IBencodingType.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | 3 | namespace BencodeLibrary 4 | { 5 | public interface IBencodingType 6 | { 7 | /// 8 | /// Encodes the current object onto the specified binary writer. 9 | /// 10 | /// The writer to write to - must not be null 11 | void Encode(BinaryWriter writer); 12 | } 13 | } -------------------------------------------------------------------------------- /DHTBucketManager.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Net; 6 | using System.Net.Sockets; 7 | using BencodeLibrary; 8 | using System.IO; 9 | using System.Runtime.Serialization; 10 | using System.Runtime.Serialization.Formatters.Binary; 11 | using System.Threading; 12 | using System.Collections; 13 | using System.Diagnostics; 14 | 15 | namespace DHTCrawler 16 | { 17 | /// 18 | /// Holds IP/Port/NodeID of all/most nodes in the DHT network 19 | /// Needs to be refreshed occasionally, could also be refreshed using get peer request answers. 20 | /// 21 | class DHTBucketManager 22 | { 23 | // List of all DHT nodes, could be extended with a timestamp 24 | public List>> lDHTNodes; 25 | public List locks = Enumerable.Repeat("x", Program.DHT_BUCKETS).ToList(); 26 | public int nofNodes = 0; 27 | private int countRecvPeerPackets; 28 | private int countRecvNodePackets; 29 | private int countRecvQuery; 30 | private int getPeerReqSent; 31 | private int localPort; 32 | private string infoHHex; 33 | private string nodeIDHex; 34 | private string conversationID; 35 | private static readonly Random random = new Random(); 36 | private Thread receiveThread; 37 | private IPEndPoint receiveFrom; 38 | private UdpClient udpClientReceive; 39 | private UdpClient udpClientSend; 40 | private Socket s; 41 | private const int SIO_UDP_CONNRESET = -1744830452; 42 | 43 | /// 44 | /// Initializes lDHTNodes with starting nodes, the list is loaded from disk if available 45 | /// 46 | /// 47 | /// 48 | public DHTBucketManager(IPEndPoint bootstrapAdr, int localport) 49 | { 50 | localPort = localport; 51 | 52 | // Read starting nodes from disk - slower than getting them from network! 53 | //LoadFromDiskFormatter(); 54 | //LoadFromDiskCustom(); 55 | if (lDHTNodes == null) 56 | { 57 | lDHTNodes = new List>>(); 58 | for (int i = 0; i < Program.DHT_BUCKETS; i++) 59 | { 60 | HashSet> hs = new HashSet>(); 61 | lDHTNodes.Add(hs); 62 | } 63 | } 64 | Console.WriteLine("Thread " + Thread.CurrentThread.ManagedThreadId+": Getting random starting nodes from the network!"); 65 | for (int c1 = 0; c1 < lDHTNodes.Count; c1++) 66 | { 67 | nofNodes += lDHTNodes[c1].Count; 68 | } 69 | udpClientReceive = new UdpClient(); 70 | udpClientReceive.Client.ReceiveBufferSize = 10000000; // in bytes 71 | udpClientReceive.Client.IOControl((IOControlCode)SIO_UDP_CONNRESET, new byte[] { 0, 0, 0, 0 }, null); 72 | udpClientReceive.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); 73 | udpClientReceive.Client.Ttl = 255; 74 | udpClientReceive.Client.Bind(new IPEndPoint(IPAddress.Any, localPort)); 75 | 76 | s = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); 77 | s.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); 78 | s.Bind(new IPEndPoint(IPAddress.Any, localPort)); 79 | 80 | udpClientSend = new UdpClient(); 81 | udpClientSend.Client.SendBufferSize = 10000000; 82 | udpClientSend.Client.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); 83 | udpClientSend.Client.Ttl = 255; 84 | udpClientSend.Client.Bind(new IPEndPoint(IPAddress.Any, localPort)); 85 | 86 | receiveThread = new Thread(new ThreadStart(ReceivePackets)); 87 | receiveThread.IsBackground = true; 88 | receiveThread.Start(); 89 | 90 | nodeIDHex = getRandomID(); 91 | conversationID = getRandomConvID(); 92 | 93 | // Get some random nodes from the bootstrap nodes 94 | while (nofNodes < Program.MAX_CONTACT_NODES) 95 | { 96 | infoHHex = getRandomID(); 97 | GetPeers(bootstrapAdr, infoHHex); 98 | getPeerReqSent++; 99 | Wait(); 100 | } 101 | DateTime startTime = DateTime.Now; 102 | DateTime endTime; 103 | 104 | // Continually query all nodes for random infohashes 105 | int cycleCount = 1; 106 | while (nofNodes < Program.MAX_STARTING_NODES) 107 | { 108 | for (int i = 0; i < Program.DHT_BUCKETS && nofNodes < Program.MAX_STARTING_NODES; i++) 109 | { 110 | endTime = DateTime.Now; 111 | TimeSpan span = endTime.Subtract(startTime); 112 | // Perdiodically display node count 113 | if (span.Seconds >= 10) 114 | { 115 | Console.WriteLine("Thread " + Thread.CurrentThread.ManagedThreadId + ": {0}/{1}", nofNodes, Program.MAX_STARTING_NODES); 116 | startTime = DateTime.Now; 117 | } 118 | Stopwatch stopwatch = new Stopwatch(); 119 | double getPeerReqSentCurrentTimespan = 0; 120 | lock (locks[i]) 121 | { 122 | stopwatch.Start(); 123 | 124 | //Stopwatch sw = new Stopwatch(); 125 | //sw.Start(); 126 | foreach (Tuple t in lDHTNodes[i]) 127 | { 128 | GetPeersBurst(t.Item2, infoHHex); 129 | getPeerReqSent += Program.MAX_GET_PEERS_BURST; 130 | getPeerReqSentCurrentTimespan += Program.MAX_GET_PEERS_BURST; 131 | } 132 | //Console.writeline(sw.elapsedmilliseconds * 8); 133 | //sw.stop(); 134 | } 135 | 136 | while ((1000*getPeerReqSentCurrentTimespan)/ (1+(double)stopwatch.ElapsedMilliseconds) > Program.MAX_PACKETS_PER_SECOND_BUCKET_MANAGER) { } 137 | stopwatch.Stop(); 138 | 139 | // Don't kill gateways with too many UDP connections 140 | if(getPeerReqSent > cycleCount*5000*Program.MAX_GET_PEERS_BURST) 141 | { 142 | Wait(Program.WAIT_BETWEEN_NODE_COLLECTION_CYCLES); 143 | cycleCount++; 144 | } 145 | } 146 | } 147 | Console.WriteLine("Thread " + Thread.CurrentThread.ManagedThreadId + ": {0}/{1}", nofNodes, Program.MAX_STARTING_NODES); 148 | Wait(3000); 149 | receiveThread.Abort(); 150 | 151 | //Log("Saving to " + Program.DHT_FILE_NAME); 152 | //stream = new FileStream(@Program.DHT_FILE_NAME, FileMode.Open); 153 | //formatter.Serialize(stream, lDHTNodes); 154 | //stream.Close(); 155 | } 156 | private void LoadFromDiskCustom() 157 | { 158 | FileStream stream; 159 | 160 | if (!File.Exists(Program.DHT_FILE_NAME)) 161 | { 162 | lDHTNodes = new List>>(); 163 | for (int i = 0; i < Program.DHT_BUCKETS; i++) 164 | { 165 | HashSet> hs = new HashSet>(); 166 | lDHTNodes.Add(hs); 167 | } 168 | stream = new FileStream(@Program.DHT_FILE_NAME, FileMode.Create); 169 | stream.Close(); 170 | Console.WriteLine("Created " + Program.DHT_FILE_NAME + "!"); 171 | } 172 | else 173 | { 174 | Console.WriteLine("Reading from " + Program.DHT_FILE_NAME + "!"); 175 | } 176 | string[] readText = File.ReadAllLines(@Program.DHT_FILE_NAME); 177 | } 178 | private void LoadFromDiskFormatter() 179 | { 180 | FileStream stream; 181 | BinaryFormatter formatter = new BinaryFormatter(); 182 | 183 | if (!File.Exists(Program.DHT_FILE_NAME)) 184 | { 185 | lDHTNodes = new List>>(); 186 | for (int i = 0; i < Program.DHT_BUCKETS; i++) 187 | { 188 | HashSet> hs = new HashSet>(); 189 | lDHTNodes.Add(hs); 190 | } 191 | stream = new FileStream(@Program.DHT_FILE_NAME, FileMode.Create); 192 | formatter.Serialize(stream, lDHTNodes); 193 | stream.Close(); 194 | Console.WriteLine("Created " + Program.DHT_FILE_NAME + "!"); 195 | } 196 | else 197 | { 198 | Console.WriteLine("Reading from " + Program.DHT_FILE_NAME + "!"); 199 | } 200 | stream = new FileStream(@Program.DHT_FILE_NAME, FileMode.Open); 201 | formatter = new BinaryFormatter(); 202 | lDHTNodes = formatter.Deserialize(stream) as List>>; 203 | stream.Close(); 204 | } 205 | private void GetPeers(IPEndPoint toAdr, string infohash) 206 | { 207 | infoHHex = getRandomID(); 208 | string getPeersMsg = "d1:ad2:id20:" + HexSToString(nodeIDHex) + "9:info_hash20:" + HexSToString(infohash) + "e1:q9:get_peers1:t2:" + HexSToString(conversationID) + "1:v4:abcd1:y1:qe"; 209 | //Log("Sent GetPeers request to " + toAdr); 210 | SendMessage(toAdr, getPeersMsg); 211 | } 212 | private void GetPeersBurst(IPEndPoint toAdr, string infohash) 213 | { 214 | for (int i = 0; i < Program.MAX_GET_PEERS_BURST; i++) 215 | { 216 | infoHHex = getRandomID(); 217 | GetPeers(toAdr, infoHHex); 218 | } 219 | } 220 | private void SendMessage(IPEndPoint destination, string message) 221 | { 222 | //Stopwatch sw = new Stopwatch(); 223 | //sw.Start(); 224 | byte[] send_buffer = BencodingUtils.ExtendedASCIIEncoding.GetBytes(message); 225 | try 226 | { 227 | // Poor performance when remote host address changes between individual packets! OS limitation? 228 | //udpClientSend.SendAsync(send_buffer, send_buffer.Length,new IPEndPoint(IPAddress.Parse("123.123.123.123"),2323));// destination); 229 | // udpClientSend.SendAsync(send_buffer, send_buffer.Length, destination);// destination); 230 | udpClientSend.Send(send_buffer,send_buffer.Length, destination); 231 | //Log(message); 232 | } 233 | catch (Exception ex) 234 | { 235 | } 236 | //Console.WriteLine(sw.ElapsedMilliseconds); 237 | //sw.Stop(); 238 | } 239 | /// 240 | /// Convert string in hex to string - assumes length of string is 2n 241 | /// 242 | /// 243 | /// 244 | private string HexSToString(string hexString) 245 | { 246 | int length = hexString.Length / 2; 247 | byte[] arr = new byte[length]; 248 | for (int i = 0; i < length; i++) 249 | { 250 | arr[i] = Convert.ToByte(hexString.Substring(2 * i, 2), 16); 251 | } 252 | return BencodingUtils.ExtendedASCIIEncoding.GetString(arr); 253 | } 254 | /// 255 | /// Updates the list of contact nodes with closer nodes 256 | /// 257 | /// 258 | private void UpdateContactList(BString nodeIpPortString) 259 | { 260 | byte[] arr = BencodingUtils.ExtendedASCIIEncoding.GetBytes(nodeIpPortString.Value); 261 | byte[] ip = new byte[4]; 262 | byte[] port = new byte[2]; 263 | byte[] nodeID = new byte[20]; 264 | byte[] currLock; 265 | int bucketNr; 266 | for (int i = 0; i < arr.Length / 26; i++) 267 | { 268 | currLock = new byte[4] { 0, 0, 0, 0 }; 269 | Array.Copy(arr, i * 26, nodeID, 0, 20); 270 | Array.Copy(arr, i * 26, currLock, 2, 2); 271 | Array.Copy(arr, i * 26 + 20, ip, 0, 4); 272 | Array.Copy(arr, i * 26 + 24, port, 0, 2); 273 | Array.Reverse(currLock); 274 | bucketNr = BitConverter.ToInt32(currLock, 0); 275 | Array.Reverse(port); 276 | IPEndPoint ipEndP = new IPEndPoint((Int64)BitConverter.ToUInt32(ip, 0), (Int32)BitConverter.ToUInt16(port, 0)); 277 | string sNodeID = ByteArrayToHexString(nodeID); 278 | 279 | if ((Int64)BitConverter.ToUInt32(ip, 0) != 0 && (Int32)BitConverter.ToUInt16(port, 0) != 0) 280 | { 281 | lock (locks[bucketNr]) 282 | { 283 | if (!lDHTNodes[bucketNr].Contains(Tuple.Create(sNodeID, ipEndP))) 284 | { 285 | lDHTNodes[bucketNr].Add(Tuple.Create(sNodeID, ipEndP)); 286 | nofNodes++; 287 | } 288 | } 289 | } 290 | } 291 | } 292 | /// 293 | /// Background thread handling all incoming traffic 294 | /// 295 | private void ReceivePackets() 296 | { 297 | while (true) 298 | { 299 | try 300 | { 301 | // Get a datagram 302 | receiveFrom = new IPEndPoint(IPAddress.Any, localPort); 303 | // Stopwatch sw = new Stopwatch(); 304 | // sw.Start(); 305 | byte[] data = udpClientReceive.Receive(ref receiveFrom); 306 | //if (sw.ElapsedMilliseconds > 10) { Console.WriteLine(sw.ElapsedMilliseconds); } 307 | //sw.Stop(); 308 | 309 | // Decode the message 310 | Stream stream = new MemoryStream(data); 311 | IBencodingType receivedMsg = BencodingUtils.Decode(stream); 312 | string decoded = BencodingUtils.ExtendedASCIIEncoding.GetString(data.ToArray()); 313 | //Log("Received message!"); 314 | 315 | // t is transaction id 316 | // y is e for error, r for reply, q for query 317 | if (receivedMsg is BDict) // throws error.. todo: fix 318 | { 319 | BDict dictMsg = (BDict)receivedMsg; 320 | if (dictMsg.ContainsKey("y")) 321 | { 322 | if (dictMsg["y"].Equals(new BString("e"))) 323 | { 324 | //Log("Received error! (ignored)"); 325 | } 326 | else if (dictMsg["y"].Equals(new BString("r"))) 327 | { 328 | // received reply 329 | if (dictMsg.ContainsKey("r")) 330 | { 331 | if (dictMsg["r"] is BDict) 332 | { 333 | BDict dictMsg2 = (BDict)dictMsg["r"]; 334 | if (dictMsg2.ContainsKey("values")) 335 | { 336 | //Log("Received list of peers for torrent!"); 337 | countRecvPeerPackets++; 338 | } 339 | else if (dictMsg2.ContainsKey("nodes")) 340 | { 341 | // could be an answer to find node or get peers 342 | //Log("Received list of nodeID & IP & port!"); 343 | countRecvNodePackets++; 344 | BString nodeIDString = (BString)dictMsg2["nodes"]; 345 | UpdateContactList(nodeIDString); 346 | } 347 | else 348 | { 349 | // no values and no nodes, assuming its a ping, 350 | // at least some form of response 351 | 352 | } 353 | } 354 | else 355 | { 356 | } 357 | } 358 | } 359 | else if (dictMsg["y"].Equals(new BString("q"))) 360 | { 361 | // received query 362 | countRecvQuery++; 363 | //Log("Received query! (ignored)"); 364 | } 365 | } 366 | } 367 | } 368 | catch (Exception ex) 369 | { 370 | //Log("Error receiving data: " + ex.ToString()); 371 | } 372 | } 373 | } 374 | /// 375 | /// Checks whether A is closer to C than B using the XOR metric 376 | /// 377 | /// 378 | /// 379 | /// 380 | /// 381 | private bool isAcloserToCThanB(BString nodeIDA, BString nodeIDB, BString nodeIDC) 382 | { 383 | byte[] arrA = BencodingUtils.ExtendedASCIIEncoding.GetBytes(nodeIDA.Value); 384 | byte[] arrB = BencodingUtils.ExtendedASCIIEncoding.GetBytes(nodeIDB.Value); 385 | byte[] arrC = BencodingUtils.ExtendedASCIIEncoding.GetBytes(nodeIDC.Value); 386 | bool[] bitsA = arrA.SelectMany(GetBits).ToArray(); 387 | bool[] bitsB = arrB.SelectMany(GetBits).ToArray(); 388 | bool[] bitsC = arrC.SelectMany(GetBits).ToArray(); 389 | for (int i = bitsA.Length - 1; i >= 0; i--) 390 | { 391 | if (bitsA[i] ^ bitsC[i] == true && bitsB[i] ^ bitsC[i] == false) { return false; } 392 | else if (bitsA[i] ^ bitsC[i] == false && bitsB[i] ^ bitsC[i] == true) { return true; } 393 | } 394 | return false; 395 | } 396 | /// 397 | /// Checks whether A is closer to C than B using the XOR metric 398 | /// 399 | /// 400 | /// 401 | /// 402 | /// 403 | private bool isAcloserToCThanB(string nodeIDA, string nodeIDB, string nodeIDC) 404 | { 405 | byte[] arrA = Encoding.ASCII.GetBytes(nodeIDA); 406 | byte[] arrB = Encoding.ASCII.GetBytes(nodeIDB); 407 | byte[] arrC = Encoding.ASCII.GetBytes(nodeIDC); 408 | bool[] bitsA = arrA.SelectMany(GetBits).ToArray(); 409 | bool[] bitsB = arrB.SelectMany(GetBits).ToArray(); 410 | bool[] bitsC = arrC.SelectMany(GetBits).ToArray(); 411 | for (int i = bitsA.Length - 1; i >= 0; i--) 412 | { 413 | if (bitsA[i] ^ bitsC[i] == true && bitsB[i] ^ bitsC[i] == false) { return false; } 414 | else if (bitsA[i] ^ bitsC[i] == false && bitsB[i] ^ bitsC[i] == true) { return true; } 415 | } 416 | return false; 417 | } 418 | private string getRandomID() 419 | { 420 | byte[] b = new byte[20]; 421 | random.NextBytes(b); 422 | return ByteArrayToHexString(b); 423 | } 424 | private string getRandomConvID() 425 | { 426 | byte[] b = new byte[2]; 427 | random.NextBytes(b); 428 | return ByteArrayToHexString(b); 429 | } 430 | IEnumerable GetBits(byte b) 431 | { 432 | for (int i = 0; i < 8; i++) 433 | { 434 | yield return (b & 0x80) != 0; 435 | b *= 2; 436 | } 437 | } 438 | /// 439 | /// Converts nodeID from byte array to displayable hex string 440 | /// 441 | /// 442 | /// 443 | private string ByteArrayToHexString(byte[] arr) 444 | { 445 | string hex = BitConverter.ToString(arr); 446 | return hex.Replace("-", ""); 447 | } 448 | private void Log(string message) 449 | { 450 | if (Program.debug) 451 | Console.WriteLine(message); 452 | } 453 | /// 454 | /// Wait for specified ms 455 | /// 456 | /// 457 | private void Wait(int ms) 458 | { 459 | DateTime called = DateTime.Now; 460 | while (DateTime.Now < called.Add(new TimeSpan(ms * 10000))) { } 461 | } 462 | /// 463 | /// Wait for Program.MAX_SYNC_WAIT ms 464 | /// 465 | private void Wait() 466 | { 467 | DateTime called = DateTime.Now; 468 | while (DateTime.Now < called.Add(Program.MAX_SYNC_WAIT)) { } 469 | } 470 | } 471 | } 472 | -------------------------------------------------------------------------------- /DHTCrawler.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Debug 5 | x86 6 | 8.0.30703 7 | 2.0 8 | {DD198B53-FA7B-4C7D-B14C-73BA20DA3416} 9 | Exe 10 | Properties 11 | DHTCrawler 12 | DHTCrawler 13 | v4.0 14 | 15 | 16 | 512 17 | publish\ 18 | true 19 | Disk 20 | false 21 | Foreground 22 | 7 23 | Days 24 | false 25 | false 26 | true 27 | 0 28 | 1.0.0.%2a 29 | false 30 | false 31 | true 32 | 33 | 34 | x86 35 | true 36 | full 37 | false 38 | bin\Debug\ 39 | DEBUG;TRACE 40 | prompt 41 | 4 42 | false 43 | 44 | 45 | x86 46 | pdbonly 47 | true 48 | bin\Release\ 49 | TRACE 50 | prompt 51 | 4 52 | false 53 | 54 | 55 | x64 56 | bin\x64\Debug\ 57 | 58 | 59 | x64 60 | bin\x64\Release\ 61 | 62 | 63 | 64 | bin\x64\Debug\MySql.Data.dll 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | False 93 | Microsoft .NET Framework 4.5 %28x86 and x64%29 94 | true 95 | 96 | 97 | False 98 | .NET Framework 3.5 SP1 Client Profile 99 | false 100 | 101 | 102 | False 103 | .NET Framework 3.5 SP1 104 | false 105 | 106 | 107 | 108 | 115 | -------------------------------------------------------------------------------- /DHTCrawler.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 2013 4 | VisualStudioVersion = 12.0.31101.0 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DHTCrawler", "DHTCrawler.csproj", "{DD198B53-FA7B-4C7D-B14C-73BA20DA3416}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|x64 = Debug|x64 11 | Debug|x86 = Debug|x86 12 | Release|x64 = Release|x64 13 | Release|x86 = Release|x86 14 | EndGlobalSection 15 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 16 | {DD198B53-FA7B-4C7D-B14C-73BA20DA3416}.Debug|x64.ActiveCfg = Debug|x64 17 | {DD198B53-FA7B-4C7D-B14C-73BA20DA3416}.Debug|x64.Build.0 = Debug|x64 18 | {DD198B53-FA7B-4C7D-B14C-73BA20DA3416}.Debug|x86.ActiveCfg = Debug|x86 19 | {DD198B53-FA7B-4C7D-B14C-73BA20DA3416}.Debug|x86.Build.0 = Debug|x86 20 | {DD198B53-FA7B-4C7D-B14C-73BA20DA3416}.Release|x64.ActiveCfg = Release|x64 21 | {DD198B53-FA7B-4C7D-B14C-73BA20DA3416}.Release|x64.Build.0 = Release|x64 22 | {DD198B53-FA7B-4C7D-B14C-73BA20DA3416}.Release|x86.ActiveCfg = Release|x86 23 | {DD198B53-FA7B-4C7D-B14C-73BA20DA3416}.Release|x86.Build.0 = Release|x86 24 | EndGlobalSection 25 | GlobalSection(SolutionProperties) = preSolution 26 | HideSolutionNode = FALSE 27 | EndGlobalSection 28 | EndGlobal 29 | -------------------------------------------------------------------------------- /DHTNode.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Net; 6 | using System.Net.Sockets; 7 | using BencodeLibrary; 8 | using System.IO; 9 | using System.Runtime.Serialization; 10 | using System.Runtime.Serialization.Formatters.Binary; 11 | using System.Threading; 12 | using System.Collections; 13 | using System.Diagnostics; 14 | using System.Runtime.InteropServices; 15 | 16 | namespace DHTCrawler 17 | { 18 | /// 19 | /// A node in the DHT network 20 | /// 21 | class DHTNode 22 | { 23 | private DHTBucketManager dhtBucketM; 24 | private string nodeIDHex; 25 | private int infohashCount = 0; 26 | private int localPort; 27 | private int nofPeersFound = 0; 28 | private Thread receiveThread; 29 | private UdpClient udpClient; 30 | private List linfoHHex; 31 | private List linfoHHex2; 32 | private List linfoHHexForMySQLInsert; 33 | private List lockContactNodes = Enumerable.Repeat("c", Program.MAX_INFOHASHES_PER_NODE).ToList(); 34 | private List>> lContactNodesA; 35 | private List>> lContactNodesB; 36 | private bool useA = true; 37 | private List> lTorrentPeers; 38 | private List> lTorrentPeersForQueries; 39 | private List bucketIndices; 40 | 41 | private IPEndPoint receiveFrom; 42 | private const int SIO_UDP_CONNRESET = -1744830452; 43 | 44 | private int countRecvPeerPackets = 0; 45 | private int countRecvNodePackets = 0; 46 | private int countGetPeerReqSent = 0; 47 | 48 | private static readonly Random random = new Random(); 49 | 50 | public DHTNode(int localPort) 51 | { 52 | this.localPort = localPort; 53 | useA = true; 54 | lContactNodesA = new List>>(); 55 | lContactNodesB = new List>>(); 56 | lTorrentPeers = new List>(); 57 | lTorrentPeersForQueries = new List>(); 58 | linfoHHex = new List(); 59 | linfoHHex2 = new List(); 60 | bucketIndices = new List(); 61 | 62 | udpClient = new UdpClient(localPort); 63 | udpClient.Client.ReceiveBufferSize = 10000; // in bytes 64 | udpClient.Client.IOControl((IOControlCode)SIO_UDP_CONNRESET, new byte[] { 0, 0, 0, 0 }, null); 65 | 66 | receiveThread = new Thread(new ThreadStart(ReceivePackets)); 67 | receiveThread.IsBackground = true; 68 | } 69 | public void Run() 70 | { 71 | // Get starting nodes from network 72 | dhtBucketM = new DHTBucketManager(Program.bootstrapAdr, localPort+1); 73 | 74 | // 1. Get oldest infohashes from DB 75 | /* 76 | Console.WriteLine("Thread " + Thread.CurrentThread.ManagedThreadId + ": Fetching " + Program.MAX_INFOHASHES_PER_NODE.ToString() + " new infohashes from DB!"); 77 | InitInfoHashes(false);*/ 78 | 79 | while(true) 80 | { 81 | // 1. Get oldest infohashes from DB 82 | Console.WriteLine("Thread " + Thread.CurrentThread.ManagedThreadId + ": Fetching " + Program.MAX_INFOHASHES_PER_NODE.ToString() + " new infohashes from DB!"); 83 | linfoHHex.Clear(); 84 | InitInfoHashes(false); 85 | 86 | infohashCount = linfoHHex.Count; 87 | //Console.WriteLine("Thread " + Thread.CurrentThread.ManagedThreadId + ": Getting " + Program.MAX_INFOHASHES_PER_NODE.ToString() + 88 | // " new infohashes from DB (background thread)!"); 89 | //Thread bg2 = new Thread(o => { InitInfoHashes(true); }); 90 | //bg2.Start(); 91 | 92 | // Clear old data 93 | nodeIDHex = GetRandomID(); 94 | nofPeersFound = 0; 95 | bucketIndices.Clear(); 96 | lTorrentPeers.Clear(); 97 | lContactNodesA.Clear(); 98 | lContactNodesB.Clear(); 99 | 100 | for (int k = 0; k < Program.MAX_INFOHASHES_PER_NODE; k++) 101 | { 102 | List lt1 = new List(); 103 | lTorrentPeers.Add(lt1); 104 | List lt0 = new List(); 105 | lTorrentPeersForQueries.Add(lt0); 106 | List> lt2 = new List>(); 107 | lContactNodesA.Add(lt2); 108 | List> lt3 = new List>(); 109 | lContactNodesB.Add(lt3); 110 | } 111 | 112 | receiveThread = new Thread(new ThreadStart(ReceivePackets)); 113 | receiveThread.IsBackground = true; 114 | receiveThread.Start(); 115 | 116 | // 2. Send get_peers request to all nodes in corresponding buckets for each infohash 117 | Stopwatch sw = new Stopwatch(); 118 | 119 | for (int curr = -Program.MAX_BUCKET_RADIUS; curr < Program.MAX_BUCKET_RADIUS+1; curr++) 120 | { 121 | for (int i = 0; i < infohashCount; i++) 122 | { 123 | string conversationID = GetConversatioID(i); 124 | bucketIndices.Add(GetBucketIndexFromInfohash(linfoHHex[i])); 125 | sw.Start(); 126 | int getPeerReqSentCurrentTimespan = 0; 127 | 128 | if (bucketIndices[i] + curr < 0) break; 129 | if (bucketIndices[i] + curr >= 65536) break; 130 | lock (dhtBucketM.locks[bucketIndices[i] + curr]) 131 | { 132 | foreach (Tuple t in dhtBucketM.lDHTNodes[bucketIndices[i] + curr]) 133 | { 134 | GetPeers(t.Item2, linfoHHex[i], conversationID); 135 | countGetPeerReqSent++; getPeerReqSentCurrentTimespan++; 136 | } 137 | } 138 | 139 | while ((1000 * getPeerReqSentCurrentTimespan) / (1 + (double)sw.ElapsedMilliseconds) > Program.MAX_PACKETS_PER_SECOND_DHT_NODE) { } 140 | sw.Stop(); 141 | 142 | dhtBucketM.nofNodes = 0; 143 | for (int u = 0; u < dhtBucketM.lDHTNodes.Count; u++) 144 | { 145 | dhtBucketM.nofNodes += dhtBucketM.lDHTNodes[u].Count; 146 | } 147 | } 148 | } 149 | 150 | useA = !useA; 151 | Wait(300); 152 | receiveThread.Abort(); 153 | Merge(); 154 | 155 | //3. Send get peer requests to all new contact nodes 156 | for (int passes = 0; passes < 4; passes++) 157 | { 158 | receiveThread = new Thread(new ThreadStart(ReceivePackets)); 159 | receiveThread.IsBackground = true; 160 | receiveThread.Start(); 161 | 162 | // Wait for responses to be processed 163 | for (int i = 0; i < infohashCount; i++) 164 | { 165 | string conversationID = GetConversatioID(i); 166 | sw.Start(); 167 | int getPeerReqSentCurrentTimespan = 0; 168 | 169 | if (!useA) 170 | { 171 | foreach (Tuple t in lContactNodesA[i]) 172 | { 173 | GetPeers(t.Item3, linfoHHex[i], conversationID); 174 | countGetPeerReqSent++; getPeerReqSentCurrentTimespan++; 175 | } 176 | lContactNodesA[i].Clear(); 177 | } 178 | else 179 | { 180 | foreach (Tuple t in lContactNodesB[i]) 181 | { 182 | GetPeers(t.Item3, linfoHHex[i], conversationID); 183 | countGetPeerReqSent++; getPeerReqSentCurrentTimespan++; 184 | } 185 | lContactNodesB[i].Clear(); 186 | } 187 | 188 | while ((1000 * getPeerReqSentCurrentTimespan) / (1 + (double)sw.ElapsedMilliseconds) > Program.MAX_PACKETS_PER_SECOND_DHT_NODE) { } 189 | sw.Stop(); 190 | 191 | dhtBucketM.nofNodes = 0; 192 | for (int u = 0; u < dhtBucketM.lDHTNodes.Count; u++) 193 | { 194 | dhtBucketM.nofNodes += dhtBucketM.lDHTNodes[u].Count; 195 | } 196 | } 197 | useA = !useA; 198 | Wait(300); 199 | receiveThread.Abort(); 200 | Merge(); 201 | } 202 | Console.WriteLine("Thread " + Thread.CurrentThread.ManagedThreadId + ": Nodes/peers found: {0}/{1} for {2} infohashes", 203 | dhtBucketM.nofNodes, nofPeersFound, Program.MAX_INFOHASHES_PER_NODE); 204 | 205 | Wait(5000); 206 | receiveThread.Abort(); 207 | // Wait for get infohash thread to finish 208 | //while (bg2.IsAlive) { } 209 | /*linfoHHexForMySQLInsert = new List(linfoHHex); 210 | lTorrentPeersForQueries = new List>(lTorrentPeers); 211 | linfoHHex.Clear(); 212 | linfoHHex = new List(linfoHHex2); 213 | linfoHHex2.Clear();*/ 214 | Console.WriteLine("Thread " + Thread.CurrentThread.ManagedThreadId + ": Inserting new peers into DB !"); 215 | Program.mySQL.InsertPeers(linfoHHex, lTorrentPeers); 216 | /*Thread bg = new Thread(o => { Program.mySQL.InsertPeers(linfoHHexForMySQLInsert, lTorrentPeersForQueries); }); 217 | bg.Start();*/ 218 | } 219 | 220 | //Log("Contact nodes: " + lContactNodes.Count); 221 | //Log("Peers found: " + lTorrentPeers.Count); 222 | //Log("Peer requests sent: " + countGetPeerReqSent); 223 | //Log("Received peer packets: " + countRecvPeerPackets); 224 | //Log("Received node packets: " + countRecvNodePackets); 225 | } 226 | private void Merge() 227 | { 228 | if (!useA) 229 | { 230 | for (int i = 0; i < lContactNodesA.Count; i++) 231 | { 232 | for (int j = 0; j < lContactNodesA[i].Count; j++) 233 | { 234 | if (!dhtBucketM.lDHTNodes[lContactNodesA[i][j].Item1].Contains(Tuple.Create( 235 | lContactNodesA[i][j].Item2, lContactNodesA[i][j].Item3))) 236 | { 237 | dhtBucketM.lDHTNodes[lContactNodesA[i][j].Item1].Add(Tuple.Create( 238 | lContactNodesA[i][j].Item2, lContactNodesA[i][j].Item3)); 239 | dhtBucketM.nofNodes++; 240 | } 241 | else 242 | { 243 | lContactNodesA[i].RemoveAt(j); 244 | } 245 | } 246 | } 247 | } 248 | else 249 | { 250 | for (int i = 0; i < lContactNodesB.Count; i++) 251 | { 252 | for (int j = 0; j < lContactNodesB[i].Count; j++) 253 | { 254 | if (!dhtBucketM.lDHTNodes[lContactNodesB[i][j].Item1].Contains(Tuple.Create( 255 | lContactNodesB[i][j].Item2, lContactNodesB[i][j].Item3))) 256 | { 257 | dhtBucketM.lDHTNodes[lContactNodesB[i][j].Item1].Add(Tuple.Create( 258 | lContactNodesB[i][j].Item2, lContactNodesB[i][j].Item3)); 259 | dhtBucketM.nofNodes++; 260 | } 261 | else 262 | { 263 | lContactNodesB[i].RemoveAt(j); 264 | } 265 | } 266 | } 267 | } 268 | } 269 | private int GetIndexFromConvID(string conversationID) 270 | { 271 | byte[] b = System.Text.Encoding.ASCII.GetBytes(conversationID); 272 | return Int32.Parse(conversationID, System.Globalization.NumberStyles.HexNumber); 273 | } 274 | private int GetBucketIndexFromInfohash(string infohash) 275 | { 276 | string infohash2 = HexSToString(infohash); 277 | byte[] b = BencodingUtils.ExtendedASCIIEncoding.GetBytes(infohash2); 278 | byte[] b2 = new byte[4]{0,0,0,0}; 279 | Array.Copy(b, 0, b2, 2, 2); 280 | Array.Reverse(b2); 281 | return BitConverter.ToInt32(b2, 0); 282 | } 283 | private void GetPeers(IPEndPoint toAdr, string infohash, string conversationID) 284 | { 285 | string getPeersMsg = "d1:ad2:id20:" + HexSToString(nodeIDHex) + "9:info_hash20:" + HexSToString(infohash) + "e1:q9:get_peers1:t2:" + HexSToString(conversationID) + "1:v4:abcd1:y1:qe"; 286 | Log("Sent GetPeers request to " + toAdr); 287 | SendMessage(toAdr, getPeersMsg); 288 | } 289 | private void FindNode(IPEndPoint toAdr, string findNodeID, string conversationID) 290 | { 291 | string findNodeMsg = "d1:ad2:id20:" + HexSToString(nodeIDHex) + "6:target20:" + HexSToString(findNodeID) + "e1:q9:find_node1:t2:" + HexSToString(conversationID) + "1:v4:abcd1:y1:qe"; 292 | Log("Sent FindNode request to " + toAdr); 293 | SendMessage(toAdr, findNodeMsg); 294 | } 295 | private void SendMessage(IPEndPoint destination, string message) 296 | { 297 | byte[] send_buffer = BencodingUtils.ExtendedASCIIEncoding.GetBytes(message); 298 | 299 | udpClient.Send(send_buffer, send_buffer.Length, destination); 300 | 301 | } 302 | /// 303 | /// Not thread safe, call when all threads have finished 304 | /// 305 | private void OutputContactList() 306 | { 307 | Log("Contact list:"); 308 | for (int index = 0; index < lContactNodesA.Count; index++) 309 | { 310 | //Log(index.ToString() + ": " + lContactNodes.ElementAt(index).Item1 + " " + lContactNodes.ElementAt(index).Item2.ToString()); 311 | } 312 | } 313 | private void OutputTorrentPeerList() 314 | { 315 | Log("Torrent peer list:"); 316 | for (int i = 0; i < lTorrentPeers.Count; i++) 317 | { 318 | Log(i.ToString() + ": " + lTorrentPeers[i].ToString()); 319 | } 320 | } 321 | private void Log(string message) 322 | { 323 | //if (Program.debug) 324 | // Console.WriteLine(message); 325 | } 326 | /// 327 | /// Updates the buckets with closer nodes 328 | /// 329 | /// 330 | private void UpdateContactList(BString nodeIpPortString, BString transactionID) 331 | { 332 | byte[] arr = BencodingUtils.ExtendedASCIIEncoding.GetBytes(nodeIpPortString.Value); 333 | byte[] arr2 = BencodingUtils.ExtendedASCIIEncoding.GetBytes(transactionID.Value); 334 | byte[] transID = new byte[2]; 335 | Array.Copy(arr2, 0, transID, 0, 2); 336 | string stransID = ByteArrayToHexString(transID); 337 | int index = GetIndexFromConvID(stransID); 338 | byte[] ip = new byte[4]; 339 | byte[] port = new byte[2]; 340 | byte[] nodeID = new byte[20]; 341 | byte[] currLock; 342 | int bucketNr; 343 | 344 | for (int i = 0; i < arr.Length / 26; i++) 345 | { 346 | currLock = new byte[4] { 0, 0, 0, 0 }; 347 | Array.Copy(arr, i * 26, currLock, 2, 2); 348 | Array.Copy(arr, i * 26, nodeID, 0, 20); 349 | Array.Copy(arr, i * 26 + 20, ip, 0, 4); 350 | Array.Copy(arr, i * 26 + 24, port, 0, 2); 351 | Array.Reverse(port); 352 | Array.Reverse(currLock); 353 | bucketNr = BitConverter.ToInt32(currLock, 0); 354 | IPEndPoint ipEndP = new IPEndPoint((Int64)BitConverter.ToUInt32(ip, 0), (Int32)BitConverter.ToUInt16(port, 0)); 355 | string sNodeID = ByteArrayToHexString(nodeID); 356 | 357 | if ((Int64)BitConverter.ToUInt32(ip, 0) != 0 && (Int32)BitConverter.ToUInt16(port, 0) != 0) 358 | { 359 | Tuple toInsert = Tuple.Create(bucketNr, sNodeID, ipEndP); 360 | if (useA && CheckIfBucketCloseEnough(toInsert, index)) 361 | lContactNodesA[index].Add(toInsert); 362 | else if (!useA && CheckIfBucketCloseEnough(toInsert, index)) 363 | lContactNodesB[index].Add(toInsert); 364 | } 365 | } 366 | } 367 | private bool CheckIfBucketCloseEnough(Tuple toInsert,int index) 368 | { 369 | int bucketIndex = GetBucketIndexFromInfohash(linfoHHex[index]); 370 | if (toInsert.Item1 > bucketIndex - 1 && toInsert.Item1 < bucketIndex + 1) 371 | { 372 | return true; 373 | } 374 | else return false; 375 | } 376 | /// 377 | /// Updates Peer list for given infohash 378 | /// 379 | /// 380 | /// 381 | private void UpdateTorrentPeerList(BList lIPPortString, BString transactionID) 382 | { 383 | byte[] ip = new byte[4]; 384 | byte[] port = new byte[2]; 385 | byte[] arr2 = BencodingUtils.ExtendedASCIIEncoding.GetBytes(transactionID.Value); 386 | byte[] transID = new byte[2]; 387 | Array.Copy(arr2, 0, transID, 0, 2); 388 | string stransID = ByteArrayToHexString(transID); 389 | int index = GetIndexFromConvID(stransID); 390 | 391 | for (int k = 0; k < lIPPortString.Count; k++) 392 | { 393 | BString tempBS = (BString)lIPPortString[k]; 394 | byte[] arr = BencodingUtils.ExtendedASCIIEncoding.GetBytes(tempBS.Value); 395 | Array.Copy(arr, 0, ip, 0, 4); 396 | Array.Copy(arr, 4, port, 0, 2); 397 | Array.Reverse(port); 398 | IPEndPoint ipEndP = new IPEndPoint((Int64)BitConverter.ToUInt32(ip, 0), (Int32)BitConverter.ToUInt16(port, 0)); 399 | 400 | if ((Int64)BitConverter.ToUInt32(ip, 0) != 0 && (Int32)BitConverter.ToUInt16(port, 0) != 0) 401 | { 402 | if (!lTorrentPeers[index].Contains(ipEndP)) 403 | { 404 | lTorrentPeers[index].Add(ipEndP); 405 | nofPeersFound++; 406 | } 407 | } 408 | } 409 | } 410 | /// 411 | /// Converts nodeID from BString to displayable hex string 412 | /// 413 | /// 414 | /// 415 | private string BStringToHexNodeID(BString nodeID) 416 | { 417 | string hex = BitConverter.ToString(BencodingUtils.ExtendedASCIIEncoding.GetBytes(nodeID.Value)); 418 | return hex.Replace("-", ""); 419 | } 420 | /// 421 | /// Converts nodeID from byte array to displayable hex string 422 | /// 423 | /// 424 | /// 425 | private string ByteArrayToHexString(byte[] arr) 426 | { 427 | string hex = BitConverter.ToString(arr); 428 | return hex.Replace("-", ""); 429 | } 430 | /// 431 | /// Convert string in hex to string - assumes length of string is 2n 432 | /// 433 | /// 434 | /// 435 | private string HexSToString(string hexString) 436 | { 437 | int length = hexString.Length / 2; 438 | byte[] arr = new byte[length]; 439 | for (int i = 0; i < length; i++) 440 | { 441 | arr[i] = Convert.ToByte(hexString.Substring(2 * i, 2), 16); 442 | } 443 | return BencodingUtils.ExtendedASCIIEncoding.GetString(arr); 444 | } 445 | /// 446 | /// Wait for specified ms 447 | /// 448 | /// 449 | private void Wait(int ms) 450 | { 451 | DateTime called = DateTime.Now; 452 | while (DateTime.Now < called.Add(new TimeSpan(ms * 10000))) { } 453 | } 454 | /// 455 | /// Wait for Program.MAX_SYNC_WAIT 456 | /// 457 | private void Wait() 458 | { 459 | DateTime called = DateTime.Now; 460 | while (DateTime.Now < called.Add(Program.MAX_SYNC_WAIT)) { } 461 | } 462 | private string GetRandomID() 463 | { 464 | byte[] b = new byte[20]; 465 | random.NextBytes(b); 466 | return ByteArrayToHexString(b); 467 | } 468 | private string GetRandomConvID() 469 | { 470 | byte[] b = new byte[2]; 471 | random.NextBytes(b); 472 | return ByteArrayToHexString(b); 473 | } 474 | IEnumerable GetBits(byte b) 475 | { 476 | for (int i = 0; i < 8; i++) 477 | { 478 | yield return (b & 0x80) != 0; 479 | b *= 2; 480 | } 481 | } 482 | private string GetConversatioID(int index) 483 | { 484 | //byte[] b = new byte[2]; 485 | //b[1] = (byte)(index & 0xFF); 486 | //b[0] = (byte)((index >> 8) & 0xFF); 487 | //return ByteArrayToHexString(b); 488 | // return index.ToString(); 489 | return String.Format("{0:X4}", index); 490 | } 491 | private void InitInfoHashes(bool useSecondary) 492 | { 493 | List infohashes = new List(); 494 | infohashes = Program.mySQL.getNextInfoHashes(Program.MAX_INFOHASHES_PER_NODE); 495 | 496 | for (int i = 0; i < infohashes.Count; i++) 497 | { 498 | if (infohashes[i] != null && !linfoHHex.Contains(infohashes[i])) // todo 499 | { 500 | if (useSecondary) 501 | { 502 | linfoHHex2.Add(infohashes[i]); 503 | } 504 | else 505 | { 506 | linfoHHex.Add(infohashes[i]); 507 | } 508 | } 509 | else 510 | { break; } 511 | } 512 | } 513 | /// 514 | /// Background thread handling all incoming traffic 515 | /// 516 | private void ReceivePackets() 517 | { 518 | while (true) 519 | { 520 | try 521 | { 522 | // Get a datagram 523 | receiveFrom = new IPEndPoint(IPAddress.Any, localPort); 524 | byte[] data = udpClient.Receive(ref receiveFrom); 525 | 526 | // Decode the message 527 | Stream stream = new MemoryStream(data); 528 | IBencodingType receivedMsg = BencodingUtils.Decode(stream); 529 | string decoded = BencodingUtils.ExtendedASCIIEncoding.GetString(data.ToArray()); 530 | Log("Received message!"); 531 | //Log(decoded); 532 | 533 | // t is transaction id : todo: check, since ports are reused 534 | // y is e for error, r for reply, q for query 535 | if (receivedMsg is BDict) // throws error.. todo: fix 536 | { 537 | BDict dictMsg = (BDict)receivedMsg; 538 | if (dictMsg.ContainsKey("y")) 539 | { 540 | if (dictMsg["y"].Equals(new BString("e"))) 541 | { 542 | // received error 543 | Log("Received error! (ignored)"); 544 | } 545 | else if (dictMsg["y"].Equals(new BString("r"))) 546 | { 547 | // received reply 548 | if (dictMsg.ContainsKey("r")) 549 | { 550 | if (dictMsg["r"] is BDict) 551 | { 552 | BDict dictMsg2 = (BDict)dictMsg["r"]; 553 | if (dictMsg2.ContainsKey("values") && dictMsg.ContainsKey("t")) 554 | { 555 | Log("Received list of peers for torrent!"); 556 | countRecvPeerPackets++; 557 | BList peerAdrs = (BList)dictMsg2["values"]; 558 | UpdateTorrentPeerList(peerAdrs, (BString)dictMsg["t"]); 559 | } 560 | else if (dictMsg2.ContainsKey("nodes") && dictMsg.ContainsKey("t")) 561 | { 562 | // could be an answer to find node or get peers 563 | Log("Received list of nodeID & IP & port!"); 564 | countRecvNodePackets++; 565 | BString nodeIDString = (BString)dictMsg2["nodes"]; 566 | UpdateContactList(nodeIDString, (BString)dictMsg["t"]); 567 | } 568 | else 569 | { 570 | } 571 | } 572 | else 573 | { 574 | } 575 | } 576 | } 577 | else if (dictMsg["y"].Equals(new BString("q"))) 578 | { 579 | // received query 580 | Log("Received query! (ignored)"); 581 | } 582 | } 583 | } 584 | } 585 | catch (Exception ex) 586 | { 587 | Log("Error receiving data: " + ex.ToString()); 588 | } 589 | } 590 | } 591 | } 592 | } -------------------------------------------------------------------------------- /MySQL.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using MySql.Data.MySqlClient; 6 | using System.Net; 7 | 8 | namespace DHTCrawler 9 | { 10 | class MySQL 11 | { 12 | private MySqlConnection connection; 13 | private MySqlCommand command; 14 | private MySqlDataReader reader; 15 | 16 | public MySQL() 17 | { 18 | //string sConnection = "SERVER=localhost;" + "DATABASE=bachelor;" + "UID=root;" + "PASSWORD=salami;" +"Connection Timeout=10000"; 19 | //string sConnection = "SERVER=pc-10129.ethz.ch;" + "DATABASE=bt_dht_recommender;" + "UID=bt_dht_recommend;" + "PASSWORD=xSXLQHfSj9hUsmj4;" + "Connection Timeout=10000"; 20 | string sConnection = "SERVER=localhost;" + "DATABASE=bachelor;" + "UID=root;" + "PASSWORD=xSXLQHfSj9hUsmj4;" + "Connection Timeout=10000"; 21 | 22 | connection = new MySqlConnection(sConnection); 23 | command = connection.CreateCommand(); 24 | } 25 | public List getNextInfoHashes(int nofInfohashes) 26 | { 27 | List lResult = new List(); 28 | lock (Program.dbLock) 29 | { 30 | // Get oldest torrent 31 | command.CommandText = "SELECT `infohash`,`id` FROM `torrents45` ORDER BY updated ASC LIMIT " 32 | + nofInfohashes.ToString() + ";"; 33 | command.CommandTimeout = 60000; 34 | connection.Open(); 35 | 36 | reader = command.ExecuteReader(); 37 | if (!reader.HasRows) return null; 38 | while (reader.Read()) 39 | { 40 | lResult.Add(reader.GetString("infohash")); 41 | } 42 | 43 | reader.Close(); 44 | connection.Close(); 45 | connection.Open(); 46 | 47 | command.CommandText = "UPDATE `torrents45` SET updated = CURRENT_TIMESTAMP WHERE infohash = '" + lResult[0] + "' OR "; 48 | for (int i = 1; i < lResult.Count-1; i++) 49 | { 50 | command.CommandText += "infohash = '" + lResult[i] + "' OR "; 51 | } 52 | command.CommandText += "infohash = '" + lResult[lResult.Count-1] + "';"; 53 | command.CommandTimeout = 60000; 54 | command.ExecuteNonQuery(); 55 | connection.Close(); 56 | } 57 | return lResult; 58 | } 59 | public void InsertPeers(List linfoHHex, List> lTorrentPeers) 60 | { 61 | lock (Program.dbLock) 62 | { 63 | connection.Open(); 64 | // Insert all peers 65 | for (int c = 0; c < lTorrentPeers.Count; c++) 66 | { 67 | if (lTorrentPeers[c].Count != 0) 68 | { 69 | command.CommandText = "INSERT IGNORE INTO `peers` (ipAddress) VALUES ('" + lTorrentPeers[c][0].ToString() + "')"; 70 | command.CommandTimeout = 60000; 71 | for (int peers = 1; peers < lTorrentPeers[c].Count; peers++) 72 | { 73 | command.CommandText += ", ('" + lTorrentPeers[c][peers].ToString() + "')"; 74 | } 75 | command.ExecuteNonQuery(); 76 | } 77 | } 78 | // Insert torrent-peer associations 79 | command.CommandTimeout = 6000; 80 | command.CommandText = "INSERT INTO associated (torrentID, peerID) (SELECT torrents.id,peers.id " + 81 | "FROM torrents, peers WHERE "; 82 | for (int c = 0; c < lTorrentPeers.Count; c++) 83 | { 84 | if (lTorrentPeers[c].Count != 0) 85 | { 86 | for (int peers = 0; peers < lTorrentPeers[c].Count; peers++) 87 | { 88 | command.CommandText += "torrents.infohash = '" + linfoHHex[c].ToString() + "' " + 89 | "AND peers.ipAddress = '" + lTorrentPeers[c][peers].ToString() + "' OR "; 90 | } 91 | } 92 | } 93 | if (command.CommandText.Length > 200) 94 | { 95 | // Dont execute invalid SQL commands 96 | command.CommandText += "torrents.infohash = 'abc' AND peers.ipAddress = 'abc') " + 97 | "ON DUPLICATE KEY UPDATE `updated` = CURRENT_TIMESTAMP;"; 98 | command.ExecuteNonQuery(); 99 | } 100 | 101 | /* 102 | if (lTorrentPeers[c].Count != 0) 103 | { 104 | // Get torrentID 105 | command.CommandText = "SELECT `id` FROM `torrents45` WHERE `infohash` = '" + linfoHHex[c].ToString() + "'"; 106 | command.CommandTimeout = 6000; 107 | reader = command.ExecuteReader(); 108 | reader.Read(); 109 | torrentID = (int)reader.GetValue(0); 110 | reader.Close(); 111 | 112 | for (int peers = 0; peers < lTorrentPeers[c].Count; peers++) 113 | { 114 | // Get peerID 115 | command.CommandText = "SELECT `id` FROM `peers` WHERE `ipAddress` = '" + lTorrentPeers[c][peers].ToString() + "'"; 116 | command.CommandTimeout = 6000; 117 | reader = command.ExecuteReader(); 118 | reader.Read(); 119 | peerID = (int)reader.GetValue(0); 120 | reader.Close(); 121 | 122 | // Insert association 123 | command.CommandText = "INSERT INTO `associated` (torrentID,peerID) VALUES (" + torrentID.ToString() + "," + 124 | peerID.ToString() + ") ON DUPLICATE KEY UPDATE `updated` = CURRENT_TIMESTAMP"; 125 | command.CommandTimeout = 6000; 126 | command.ExecuteNonQuery(); 127 | } 128 | }*/ 129 | 130 | connection.Close(); 131 | } 132 | } 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Net; 6 | using System.Net.Sockets; 7 | using BencodeLibrary; 8 | using System.IO; 9 | using System.Runtime.Serialization; 10 | using System.Runtime.Serialization.Formatters.Binary; 11 | using System.Threading; 12 | 13 | namespace DHTCrawler 14 | { 15 | class Program 16 | { 17 | static public TimeSpan MAX_SYNC_WAIT = new TimeSpan(20000000); // 10000 = 1ms 18 | static public TimeSpan MAX_WAIT_AFTER_IDLE = new TimeSpan(60000000);//new TimeSpan(60000000); 19 | static public int MAX_STARTING_NODES; // 1500000 20 | static public int MAX_PACKETS_PER_SECOND_BUCKET_MANAGER = 10000; // Only sending // 4000 21 | static public int MAX_PACKETS_PER_SECOND_DHT_NODE = 10000; //2500 5000 22 | static public int MAX_INFOHASHES_PER_NODE = 100; // Must be below 65536 for 2 byte conversation IDs! 23 | static public int DHT_BUCKETS = 65536; 24 | static public int MAX_DHT_NODES; 25 | static public int MAX_BUCKET_RADIUS = 1; // default 1 for 1.5m 26 | static public int MAX_PASSES = 3; 27 | static public string DHT_FILE_NAME = "DHT_NODES.dat"; 28 | static public int MAX_CONTACT_NODES = 100; 29 | static public int MAX_GET_PEERS_BURST = 100; 30 | static public int WAIT_BETWEEN_NODE_COLLECTION_CYCLES = 0; // 1200 31 | static public MySQL mySQL = new MySQL(); 32 | static public bool debug = true; 33 | static public IPAddress bootstrapIP = IPAddress.Parse("67.215.246.10"); // router.bittorrent.com:6881 34 | static public IPEndPoint bootstrapAdr = new IPEndPoint(bootstrapIP, 6881); 35 | static private List threads; 36 | public static string dbLock = "a"; 37 | 38 | static void Main(string[] args) 39 | { 40 | threads = new List(); 41 | 42 | // Get number of threads 43 | Console.Write("Enter number of sending threads (default 4): "); 44 | string input = Console.ReadLine(); 45 | bool result = Int32.TryParse(input, out MAX_DHT_NODES); 46 | if (!result) 47 | MAX_DHT_NODES = 4; 48 | 49 | // Get number of packets per second sending 50 | Console.Write("Enter number of packets send per second for each thread (default 10000): "); 51 | input = Console.ReadLine(); 52 | result = Int32.TryParse(input, out MAX_PACKETS_PER_SECOND_DHT_NODE); 53 | if (!result) 54 | MAX_PACKETS_PER_SECOND_DHT_NODE = 10000; 55 | 56 | // Get number of starting nodes per thread 57 | Console.Write("Enter number of starting nodes per thread (default 1.5 million): "); 58 | input = Console.ReadLine(); 59 | result = Int32.TryParse(input, out MAX_STARTING_NODES); 60 | if (!result) 61 | MAX_STARTING_NODES = 1500000; /* 1500000*/ 62 | 63 | // Get number of starting nodes per thread 64 | Console.Write("Enter number of infohashes queried per loop (default 20 - determines size of SQL queries): "); 65 | input = Console.ReadLine(); 66 | result = Int32.TryParse(input, out MAX_INFOHASHES_PER_NODE); 67 | if (!result) 68 | MAX_INFOHASHES_PER_NODE = 20; /* 1500000*/ 69 | 70 | // Get bucket radius 71 | Console.Write("Enter number of neighbor buckets to check in each direction (default 1): "); 72 | input = Console.ReadLine(); 73 | result = Int32.TryParse(input, out MAX_BUCKET_RADIUS); 74 | if (!result) 75 | MAX_BUCKET_RADIUS = 1; 76 | 77 | // Get number of passes 78 | Console.Write("Enter number of times to ask all nodes (default 3): "); 79 | input = Console.ReadLine(); 80 | result = Int32.TryParse(input, out MAX_PASSES); 81 | if (!result) 82 | MAX_PASSES = 3; 83 | 84 | 85 | for (int i = 0; i < MAX_DHT_NODES; ++i) 86 | { 87 | DHTNode dhtNode = new DHTNode(i * 2 + 3244); 88 | Thread child = new Thread(t => { dhtNode.Run(); }); 89 | threads.Add(child); 90 | child.Start(); 91 | } 92 | 93 | // None of the childs will return 94 | foreach(Thread t in threads) 95 | t.Join(); 96 | } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("DHTCrawler")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("DHTCrawler")] 13 | [assembly: AssemblyCopyright("Copyright © 2014")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("d76c1c7a-cb59-4f33-a01e-1db1bd17f94f")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /app.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | --------------------------------------------------------------------------------