├── .gitattributes ├── .gitignore ├── BetterHttpClient.sln ├── BetterHttpClient ├── BetterHttpClient.csproj ├── CheckService │ ├── ProxyCheckService.cs │ └── ProxyJudgeProxyCheckService.cs ├── HttpClient.cs ├── Properties │ └── AssemblyInfo.cs ├── Proxy.cs ├── ProxyManager.cs └── Socks │ ├── NeverEndingStream.cs │ ├── SocksHttpWebRequest.cs │ ├── SocksHttpWebResponse.cs │ └── TaskExtensions.cs ├── LICENSE ├── README.md └── UnitTestBetterHttpClient ├── Properties └── AssemblyInfo.cs ├── UnitTestBetterHttpClient.csproj ├── UnitTestHttpClient.cs └── packages.config /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.userosscache 8 | *.sln.docstates 9 | 10 | # User-specific files (MonoDevelop/Xamarin Studio) 11 | *.userprefs 12 | 13 | # Build results 14 | [Dd]ebug/ 15 | [Dd]ebugPublic/ 16 | [Rr]elease/ 17 | [Rr]eleases/ 18 | x64/ 19 | x86/ 20 | build/ 21 | bld/ 22 | [Bb]in/ 23 | [Oo]bj/ 24 | 25 | # Visual Studio 2015 cache/options directory 26 | .vs/ 27 | 28 | # MSTest test Results 29 | [Tt]est[Rr]esult*/ 30 | [Bb]uild[Ll]og.* 31 | 32 | # NUNIT 33 | *.VisualState.xml 34 | TestResult.xml 35 | 36 | # Build Results of an ATL Project 37 | [Dd]ebugPS/ 38 | [Rr]eleasePS/ 39 | dlldata.c 40 | 41 | # DNX 42 | project.lock.json 43 | artifacts/ 44 | 45 | *_i.c 46 | *_p.c 47 | *_i.h 48 | *.ilk 49 | *.meta 50 | *.obj 51 | *.pch 52 | *.pdb 53 | *.pgc 54 | *.pgd 55 | *.rsp 56 | *.sbr 57 | *.tlb 58 | *.tli 59 | *.tlh 60 | *.tmp 61 | *.tmp_proj 62 | *.log 63 | *.vspscc 64 | *.vssscc 65 | .builds 66 | *.pidb 67 | *.svclog 68 | *.scc 69 | 70 | # Chutzpah Test files 71 | _Chutzpah* 72 | 73 | # Visual C++ cache files 74 | ipch/ 75 | *.aps 76 | *.ncb 77 | *.opensdf 78 | *.sdf 79 | *.cachefile 80 | 81 | # Visual Studio profiler 82 | *.psess 83 | *.vsp 84 | *.vspx 85 | 86 | # TFS 2012 Local Workspace 87 | $tf/ 88 | 89 | # Guidance Automation Toolkit 90 | *.gpState 91 | 92 | # ReSharper is a .NET coding add-in 93 | _ReSharper*/ 94 | *.[Rr]e[Ss]harper 95 | *.DotSettings.user 96 | 97 | # JustCode is a .NET coding add-in 98 | .JustCode 99 | 100 | # TeamCity is a build add-in 101 | _TeamCity* 102 | 103 | # DotCover is a Code Coverage Tool 104 | *.dotCover 105 | 106 | # NCrunch 107 | _NCrunch_* 108 | .*crunch*.local.xml 109 | 110 | # MightyMoose 111 | *.mm.* 112 | AutoTest.Net/ 113 | 114 | # Web workbench (sass) 115 | .sass-cache/ 116 | 117 | # Installshield output folder 118 | [Ee]xpress/ 119 | 120 | # DocProject is a documentation generator add-in 121 | DocProject/buildhelp/ 122 | DocProject/Help/*.HxT 123 | DocProject/Help/*.HxC 124 | DocProject/Help/*.hhc 125 | DocProject/Help/*.hhk 126 | DocProject/Help/*.hhp 127 | DocProject/Help/Html2 128 | DocProject/Help/html 129 | 130 | # Click-Once directory 131 | publish/ 132 | 133 | # Publish Web Output 134 | *.[Pp]ublish.xml 135 | *.azurePubxml 136 | ## TODO: Comment the next line if you want to checkin your 137 | ## web deploy settings but do note that will include unencrypted 138 | ## passwords 139 | #*.pubxml 140 | 141 | *.publishproj 142 | 143 | # NuGet Packages 144 | *.nupkg 145 | # The packages folder can be ignored because of Package Restore 146 | **/packages/* 147 | # except build/, which is used as an MSBuild target. 148 | !**/packages/build/ 149 | # Uncomment if necessary however generally it will be regenerated when needed 150 | #!**/packages/repositories.config 151 | 152 | # Windows Azure Build Output 153 | csx/ 154 | *.build.csdef 155 | 156 | # Windows Store app package directory 157 | AppPackages/ 158 | 159 | # Visual Studio cache files 160 | # files ending in .cache can be ignored 161 | *.[Cc]ache 162 | # but keep track of directories ending in .cache 163 | !*.[Cc]ache/ 164 | 165 | # Others 166 | ClientBin/ 167 | [Ss]tyle[Cc]op.* 168 | ~$* 169 | *~ 170 | *.dbmdl 171 | *.dbproj.schemaview 172 | *.pfx 173 | *.publishsettings 174 | node_modules/ 175 | orleans.codegen.cs 176 | 177 | # RIA/Silverlight projects 178 | Generated_Code/ 179 | 180 | # Backup & report files from converting an old project file 181 | # to a newer Visual Studio version. Backup files are not needed, 182 | # because we have git ;-) 183 | _UpgradeReport_Files/ 184 | Backup*/ 185 | UpgradeLog*.XML 186 | UpgradeLog*.htm 187 | 188 | # SQL Server files 189 | *.mdf 190 | *.ldf 191 | 192 | # Business Intelligence projects 193 | *.rdl.data 194 | *.bim.layout 195 | *.bim_*.settings 196 | 197 | # Microsoft Fakes 198 | FakesAssemblies/ 199 | 200 | # Node.js Tools for Visual Studio 201 | .ntvs_analysis.dat 202 | 203 | # Visual Studio 6 build log 204 | *.plg 205 | 206 | # Visual Studio 6 workspace options file 207 | *.opt 208 | 209 | # LightSwitch generated files 210 | GeneratedArtifacts/ 211 | _Pvt_Extensions/ 212 | ModelManifest.xml 213 | -------------------------------------------------------------------------------- /BetterHttpClient.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 14 4 | VisualStudioVersion = 14.0.23107.0 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BetterHttpClient", "BetterHttpClient\BetterHttpClient.csproj", "{FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}" 7 | EndProject 8 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UnitTestBetterHttpClient", "UnitTestBetterHttpClient\UnitTestBetterHttpClient.csproj", "{59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}" 9 | EndProject 10 | Global 11 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 12 | Debug|Any CPU = Debug|Any CPU 13 | Debug|ARM = Debug|ARM 14 | Debug|x64 = Debug|x64 15 | Debug|x86 = Debug|x86 16 | Release|Any CPU = Release|Any CPU 17 | Release|ARM = Release|ARM 18 | Release|x64 = Release|x64 19 | Release|x86 = Release|x86 20 | EndGlobalSection 21 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 22 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 23 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|Any CPU.Build.0 = Debug|Any CPU 24 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|ARM.ActiveCfg = Debug|Any CPU 25 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|ARM.Build.0 = Debug|Any CPU 26 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|x64.ActiveCfg = Debug|Any CPU 27 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|x64.Build.0 = Debug|Any CPU 28 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|x86.ActiveCfg = Debug|Any CPU 29 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|x86.Build.0 = Debug|Any CPU 30 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|Any CPU.ActiveCfg = Release|Any CPU 31 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|Any CPU.Build.0 = Release|Any CPU 32 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|ARM.ActiveCfg = Release|Any CPU 33 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|ARM.Build.0 = Release|Any CPU 34 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|x64.ActiveCfg = Release|Any CPU 35 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|x64.Build.0 = Release|Any CPU 36 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|x86.ActiveCfg = Release|Any CPU 37 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|x86.Build.0 = Release|Any CPU 38 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 39 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|Any CPU.Build.0 = Debug|Any CPU 40 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|ARM.ActiveCfg = Debug|Any CPU 41 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|ARM.Build.0 = Debug|Any CPU 42 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|x64.ActiveCfg = Debug|Any CPU 43 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|x64.Build.0 = Debug|Any CPU 44 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|x86.ActiveCfg = Debug|Any CPU 45 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|x86.Build.0 = Debug|Any CPU 46 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|Any CPU.ActiveCfg = Release|Any CPU 47 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|Any CPU.Build.0 = Release|Any CPU 48 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|ARM.ActiveCfg = Release|Any CPU 49 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|ARM.Build.0 = Release|Any CPU 50 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|x64.ActiveCfg = Release|Any CPU 51 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|x64.Build.0 = Release|Any CPU 52 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|x86.ActiveCfg = Release|Any CPU 53 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|x86.Build.0 = Release|Any CPU 54 | EndGlobalSection 55 | GlobalSection(SolutionProperties) = preSolution 56 | HideSolutionNode = FALSE 57 | EndGlobalSection 58 | EndGlobal 59 | -------------------------------------------------------------------------------- /BetterHttpClient/BetterHttpClient.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3} 8 | Library 9 | Properties 10 | BetterHttpClient 11 | BetterHttpClient 12 | v4.5 13 | 512 14 | 15 | 16 | 17 | true 18 | full 19 | false 20 | bin\Debug\ 21 | DEBUG;TRACE 22 | prompt 23 | 4 24 | 25 | 26 | pdbonly 27 | true 28 | bin\Release\ 29 | TRACE 30 | prompt 31 | 4 32 | bin\Release\BetterHttpClient.XML 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | Component 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 65 | -------------------------------------------------------------------------------- /BetterHttpClient/CheckService/ProxyCheckService.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace BetterHttpClient.CheckService 4 | { 5 | public abstract class ProxyJudgeServiceAbstract 6 | { 7 | private string _myIp = null; 8 | private readonly object _syncLock = new object(); 9 | 10 | public string MyIp 11 | { 12 | get 13 | { 14 | lock (_syncLock) 15 | { 16 | if(_myIp == null) 17 | _myIp = GetMyIp(); 18 | } 19 | 20 | return _myIp; 21 | } 22 | } 23 | /// 24 | /// Set number of attempts which can be made to verify proxy. 25 | /// 26 | public int NumberOfAttempts { get; set; } 27 | /// 28 | /// Abstract method. Should return true if proxy is hiding your real ip address. 29 | /// 30 | /// 31 | /// 32 | public abstract bool IsProxyAnonymous(Proxy proxy); 33 | /// 34 | /// Abstract method. Should return real ip addres of user (so without any proxy). 35 | /// 36 | /// 37 | protected abstract string GetMyIp(); 38 | } 39 | public class GetMyIpException : Exception 40 | { 41 | } 42 | } -------------------------------------------------------------------------------- /BetterHttpClient/CheckService/ProxyJudgeProxyCheckService.cs: -------------------------------------------------------------------------------- 1 | using System.Net; 2 | using System.Text; 3 | using System.Text.RegularExpressions; 4 | 5 | namespace BetterHttpClient.CheckService 6 | { 7 | public class ProxyJudgeService : ProxyJudgeServiceAbstract 8 | { 9 | public override bool IsProxyAnonymous(Proxy proxy) 10 | { 11 | HttpClient client = new HttpClient(proxy, Encoding.UTF8) 12 | { 13 | NumberOfAttempts = NumberOfAttempts 14 | }; 15 | string page = null; 16 | 17 | try 18 | { 19 | page = client.Get("http://proxyjudge.info/"); 20 | } 21 | catch (WebException) 22 | { 23 | proxy.IsOnline = false; 24 | } 25 | 26 | if (page == null) 27 | return false; 28 | if (page.Contains("Proxyjudge.info") && !page.Contains(MyIp)) 29 | return true; 30 | return false; 31 | } 32 | 33 | protected override string GetMyIp() 34 | { 35 | HttpClient client = new HttpClient(Encoding.UTF8); 36 | string page = client.Get("http://proxyjudge.info/"); 37 | 38 | Match match = Regex.Match(page, "REMOTE_ADDR = (.*?)\\n"); 39 | if (!match.Success) 40 | throw new GetMyIpException(); 41 | else 42 | return match.Groups[1].Value; 43 | } 44 | } 45 | } -------------------------------------------------------------------------------- /BetterHttpClient/HttpClient.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Specialized; 3 | using System.Net; 4 | using System.Text; 5 | using BetterHttpClient.Socks; 6 | 7 | namespace BetterHttpClient 8 | { 9 | public class HttpClient : WebClient 10 | { 11 | private int _numberOfAttempts = 4; 12 | private TimeSpan _timeout = TimeSpan.FromSeconds(60); 13 | private Proxy _proxy; 14 | 15 | /// 16 | /// Cookie container. 17 | /// 18 | public CookieContainer Cookies { get; set; } = new CookieContainer(); 19 | 20 | /// 21 | /// Proxy which should be used for request. 22 | /// Set null or proxy with type ProxyTypeEnum.None if you want to perform request without proxy. 23 | /// 24 | public new Proxy Proxy 25 | { 26 | get { return _proxy; } 27 | set { _proxy = value ?? new Proxy(); } 28 | } 29 | /// 30 | /// Timeout for request. 31 | /// 32 | /// Has to be greater than 5 milliseconds. 33 | /// Default value: 60 seconds. 34 | public TimeSpan Timeout 35 | { 36 | get 37 | { 38 | return _timeout; 39 | } 40 | set 41 | { 42 | if (value.TotalMilliseconds < 5) 43 | throw new ArgumentOutOfRangeException("Timeout has to be greater or equal than 5 milliseconds."); 44 | _timeout = value; 45 | } 46 | } 47 | /// 48 | /// Set number of attempts that can be made to execute request. 49 | /// 50 | /// Default value: 4 attempts. 51 | /// Should be greater than 1 52 | public int NumberOfAttempts 53 | { 54 | get { return _numberOfAttempts; } 55 | set 56 | { 57 | if (value < 1) 58 | throw new ArgumentOutOfRangeException("Value has to be greater than one."); 59 | 60 | _numberOfAttempts = value; 61 | } 62 | } 63 | 64 | /// 65 | /// Set User-Agent header. 66 | /// 67 | /// Default value: "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0" 68 | public string UserAgent { get; set; } = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0"; 69 | /// 70 | /// Set Accept header. 71 | /// 72 | /// Default value: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" 73 | public string Accept { get; set; } = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; 74 | /// 75 | /// Set Referer header. 76 | /// 77 | /// Default value: null 78 | public string Referer { get; set; } 79 | /// 80 | /// Set Accept-Language header. 81 | /// 82 | /// Default value: "en-US;q=0.7,en;q=0.3" 83 | public string AcceptLanguage { get; set; } = "en-US;q=0.7,en;q=0.3"; 84 | /// 85 | /// Set default Accept-Encoding header. 86 | /// 87 | /// Default value: "gzip, deflate" 88 | public string AcceptEncoding { get; set; } = "gzip, deflate"; 89 | /// 90 | /// Set encoding for request. 91 | /// 92 | /// Default value: UTF-8 93 | public new Encoding Encoding 94 | { 95 | get { return base.Encoding; } 96 | set { base.Encoding = value; } 97 | } 98 | /// 99 | /// Enabled automatic redirect. Default: true 100 | /// 101 | public bool AllowAutoRedirect { get; set; } = true; 102 | 103 | /// 104 | /// Headers collection that will be added to each request 105 | /// 106 | public NameValueCollection CustomHeaders { get; set; } 107 | public HttpClient(Proxy proxy) : this(proxy, Encoding.UTF8) { } 108 | 109 | public HttpClient() : this(new Proxy(), Encoding.UTF8) { } 110 | public HttpClient(Encoding encoding) : this(new Proxy(), encoding) { } 111 | public HttpClient(Proxy proxy, Encoding encoding) 112 | { 113 | Encoding = encoding; 114 | Proxy = proxy; 115 | } 116 | 117 | protected override WebRequest GetWebRequest(Uri address) 118 | { 119 | WebRequest request = null; 120 | 121 | if (Proxy.ProxyType != ProxyTypeEnum.Socks) 122 | { 123 | request = base.GetWebRequest(address); 124 | } 125 | else 126 | { 127 | request = SocksHttpWebRequest.Create(address); 128 | request.Method = base.GetWebRequest(address).Method; 129 | request.ContentLength = base.GetWebRequest(address).ContentLength; 130 | request.ContentType = base.GetWebRequest(address).ContentType; 131 | } 132 | 133 | 134 | request.Headers.Add("Cookie", Cookies.GetCookieHeader(address)); 135 | request.Headers.Add("Accept-Language", AcceptLanguage); 136 | request.Headers.Add("Accept-Encoding", AcceptEncoding); 137 | if (CustomHeaders != null) 138 | { 139 | foreach (string key in CustomHeaders.AllKeys) 140 | { 141 | request.Headers.Add(key, CustomHeaders[key]); 142 | } 143 | } 144 | 145 | 146 | if (Proxy.ProxyType != ProxyTypeEnum.Socks) 147 | { 148 | var httpRequest = (request as HttpWebRequest); 149 | httpRequest.UserAgent = UserAgent; 150 | httpRequest.Accept = Accept; 151 | httpRequest.Referer = Referer; 152 | httpRequest.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip; 153 | httpRequest.AllowAutoRedirect = AllowAutoRedirect; 154 | } 155 | else if (Proxy.ProxyType == ProxyTypeEnum.Socks) 156 | { 157 | var socksRequest = (request as SocksHttpWebRequest); 158 | socksRequest.UserAgent = UserAgent; 159 | socksRequest.Accept = Accept; 160 | socksRequest.Referer = Referer; 161 | socksRequest.AllowAutoRedirect = AllowAutoRedirect; 162 | } 163 | 164 | request.Timeout = (int) Timeout.TotalMilliseconds; 165 | request.Proxy = Proxy.ProxyItem; 166 | 167 | return request; 168 | } 169 | 170 | protected override WebResponse GetWebResponse(WebRequest request) 171 | { 172 | var response = base.GetWebResponse(request); 173 | try 174 | { 175 | string setCookies = response.Headers["Set-Cookie"]; 176 | Cookies.SetCookies(request.RequestUri, setCookies); 177 | } 178 | catch (Exception) 179 | { 180 | 181 | } 182 | 183 | return response; 184 | } 185 | /// 186 | /// Execute GET request. 187 | /// 188 | /// 189 | /// 190 | public string Get(string url) 191 | { 192 | return Encoding.GetString(DownloadBytes(url, null)); 193 | } 194 | /// 195 | /// Execute POST request. 196 | /// 197 | /// 198 | /// 199 | /// 200 | public string Post(string url, NameValueCollection data) 201 | { 202 | return Encoding.GetString(DownloadBytes(url, data)); 203 | } 204 | 205 | /// 206 | /// Execute GET request. 207 | /// 208 | /// 209 | /// 210 | public byte[] DownloadBytes(string url) 211 | { 212 | return DownloadBytes(url, null); 213 | } 214 | /// 215 | /// Excecute POST request. 216 | /// 217 | /// 218 | /// 219 | /// 220 | public byte[] DownloadBytes(string url, NameValueCollection data) 221 | { 222 | int counter = 0; 223 | WebException lastWebException = null; 224 | bool unkownProxy = Proxy.ProxyType == ProxyTypeEnum.Unknown; 225 | 226 | while (counter < NumberOfAttempts + (NumberOfAttempts < 2 && unkownProxy ? 1 : 0)) // min two try for unkonwn proxy type 227 | { 228 | try 229 | { 230 | if (unkownProxy && counter % 2 == 0) // every odd try is as http proxy 231 | Proxy.ProxyType = ProxyTypeEnum.Http; 232 | else if (unkownProxy) 233 | Proxy.ProxyType = ProxyTypeEnum.Socks; 234 | 235 | byte[] result = data == null ? Encoding.GetBytes(DownloadString(url)) : UploadValues(url, data); 236 | return result; 237 | } 238 | catch (WebException e) 239 | { 240 | lastWebException = e; 241 | counter++; 242 | } 243 | } 244 | 245 | if (unkownProxy) 246 | Proxy.ProxyType = ProxyTypeEnum.Unknown; 247 | // ReSharper disable once PossibleNullReferenceException 248 | throw lastWebException; 249 | } 250 | } 251 | } 252 | -------------------------------------------------------------------------------- /BetterHttpClient/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.InteropServices; 3 | 4 | // General Information about an assembly is controlled through the following 5 | // set of attributes. Change these attribute values to modify the information 6 | // associated with an assembly. 7 | 8 | [assembly: AssemblyTitle("HttpClient")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("HttpClient")] 13 | [assembly: AssemblyCopyright("Copyright © 2015")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | 21 | [assembly: ComVisible(false)] 22 | 23 | // The following GUID is for the ID of the typelib if this project is exposed to COM 24 | 25 | [assembly: Guid("fbc5f420-3f80-4a20-a9ec-b6574e18a1d3")] 26 | 27 | // Version information for an assembly consists of the following four values: 28 | // 29 | // Major Version 30 | // Minor Version 31 | // Build Number 32 | // Revision 33 | // 34 | // You can specify all the values or you can default the Build and Revision Numbers 35 | // by using the '*' as shown below: 36 | // [assembly: AssemblyVersion("1.0.*")] 37 | 38 | [assembly: AssemblyVersion("1.0.7")] 39 | [assembly: AssemblyFileVersion("1.0.0")] -------------------------------------------------------------------------------- /BetterHttpClient/Proxy.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Net; 3 | using BetterHttpClient.CheckService; 4 | 5 | namespace BetterHttpClient 6 | { 7 | public class Proxy : ICloneable 8 | { 9 | private volatile bool _isBusy = false; 10 | private bool _isAnonymous; 11 | private bool _isChecked = false; 12 | 13 | /// 14 | /// Check if proxy is busy. 15 | /// 16 | public bool IsBusy 17 | { 18 | get { return _isBusy; } 19 | internal set { _isBusy = value; } 20 | } 21 | 22 | 23 | public bool IsOnline { get; set; } = true; 24 | public ProxyTypeEnum ProxyType { get; internal set; } 25 | internal WebProxy ProxyItem { get; set; } 26 | 27 | public Proxy() 28 | { 29 | ProxyItem = new WebProxy(); 30 | ProxyType = ProxyTypeEnum.None; 31 | } 32 | 33 | public Proxy(string address) 34 | { 35 | ProxyItem = new WebProxy(address); 36 | } 37 | 38 | public Proxy(string ip, int port) 39 | { 40 | ProxyItem = new WebProxy(ip, port); 41 | } 42 | 43 | public Proxy(string ip, int port, ProxyTypeEnum proxyType) 44 | { 45 | ProxyItem = new WebProxy(ip, port); 46 | ProxyType = proxyType; 47 | } 48 | 49 | /// 50 | /// Returns true if proxy can hide your ip address 51 | /// 52 | public bool IsAnonymous(ProxyJudgeService service) 53 | { 54 | if (_isChecked) 55 | return _isAnonymous; 56 | 57 | _isAnonymous = service.IsProxyAnonymous(this); 58 | _isChecked = true; 59 | IsOnline = _isAnonymous; 60 | return _isAnonymous; 61 | } 62 | 63 | public object Clone() 64 | { 65 | var proxy = new Proxy 66 | { 67 | _isAnonymous = _isAnonymous, 68 | _isBusy = _isBusy, 69 | _isChecked = _isChecked, 70 | IsOnline = IsOnline, 71 | ProxyItem = ProxyItem, 72 | ProxyType = ProxyType 73 | }; 74 | return proxy; 75 | } 76 | } 77 | 78 | public enum ProxyTypeEnum 79 | { 80 | Unknown, 81 | None, 82 | Http, 83 | Socks 84 | } 85 | } -------------------------------------------------------------------------------- /BetterHttpClient/ProxyManager.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Collections.Specialized; 4 | using System.IO; 5 | using System.Linq; 6 | using System.Net; 7 | using System.Text; 8 | using System.Threading; 9 | using BetterHttpClient.CheckService; 10 | 11 | namespace BetterHttpClient 12 | { 13 | public class ProxyManager 14 | { 15 | private List _proxies = new List(); 16 | private string _requiredString = string.Empty; 17 | private int _numberOfAttemptsPerRequest; 18 | private TimeSpan _timeout = TimeSpan.FromSeconds(10); 19 | private int _numberOfAttempts = 4; 20 | private ProxyJudgeService _proxyJudgeService; 21 | 22 | /// 23 | /// Set User-Agent header. 24 | /// 25 | /// Default value: "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0" 26 | public string UserAgent { get; set; } = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0"; 27 | /// 28 | /// Set Accept header. 29 | /// 30 | /// Default value: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" 31 | public string Accept { get; set; } = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; 32 | /// 33 | /// Set Referer header. 34 | /// 35 | /// Default value: null 36 | public string Referer { get; set; } 37 | /// 38 | /// Set Accept-Language header. 39 | /// 40 | /// Default value: "en-US;q=0.7,en;q=0.3" 41 | public string AcceptLanguage { get; set; } = "en-US;q=0.7,en;q=0.3"; 42 | /// 43 | /// Set default Accept-Encoding header. 44 | /// 45 | /// Default value: "gzip, deflate" 46 | public string AcceptEncoding { get; set; } = "gzip, deflate"; 47 | 48 | /// 49 | /// Set proxy check service. 50 | /// Must derive from ProxyCheckService class. 51 | /// 52 | public ProxyJudgeService ProxyJudgeService 53 | { 54 | get { return _proxyJudgeService; } 55 | set 56 | { 57 | if(value == null) 58 | throw new ArgumentNullException(); 59 | _proxyJudgeService = value; 60 | } 61 | } 62 | /// 63 | /// Downloaded page has to contain this tring. 64 | /// It helps to check if returned page is the page which we watned to receive. 65 | /// Proxy sometimes are returing some other pages. 66 | /// Default value string.Empty 67 | /// 68 | public string RequiredString 69 | { 70 | get { return _requiredString; } 71 | set { _requiredString = value ?? string.Empty; } 72 | } 73 | /// 74 | /// Set encoding for request. 75 | /// 76 | /// Default value: UTF-8 77 | public Encoding Encoding { get; set; } = Encoding.UTF8; 78 | /// 79 | /// Set this to true if you want to use only anonymous proxies. 80 | /// 81 | /// Default value: false 82 | public bool AnonymousProxyOnly { get; } = false; 83 | /// 84 | /// Timeout for request. 85 | /// 86 | /// Has to be greater than 5 milliseconds. 87 | /// Default value: 10 seconds. 88 | public TimeSpan Timeout 89 | { 90 | get 91 | { 92 | return _timeout; 93 | } 94 | set 95 | { 96 | if (value.TotalMilliseconds < 5) 97 | throw new ArgumentOutOfRangeException("Timeout has to be greater or equal than 5 milliseconds."); 98 | _timeout = value; 99 | } 100 | } 101 | /// 102 | /// Set how many attempts can be made to execute request on one proxy. 103 | /// Default value is default value is equal 4 104 | /// 105 | public int NumberOfAttempts 106 | { 107 | get { return _numberOfAttempts; } 108 | set 109 | { 110 | if (value < 1) 111 | throw new ArgumentOutOfRangeException("Value has to be greater than one."); 112 | 113 | _numberOfAttempts = value; 114 | _proxyJudgeService.NumberOfAttempts = _numberOfAttempts; 115 | } 116 | } 117 | 118 | /// Proxy list 119 | /// Set true if you want to filter proxy list and use only anonymous only 120 | /// Proxy judge service is used to determine proxy anonymity level 121 | /// 122 | public ProxyManager(IEnumerable proxies, bool anonymousOnly, ProxyJudgeService proxyJudgeService) : 123 | this(ParseProxies(proxies), anonymousOnly, proxyJudgeService) 124 | { 125 | } 126 | 127 | /// Proxy list 128 | /// Set true if you want to filter proxy list and use only anonymous only 129 | /// Proxy judge service is used to determine proxy anonymity level 130 | /// 131 | public ProxyManager(IEnumerable proxies, bool anonymousOnly, ProxyJudgeService proxyJudgeService) 132 | { 133 | if (proxies == null || proxyJudgeService == null) 134 | throw new ArgumentNullException(); 135 | foreach (Proxy proxy in proxies) 136 | { 137 | try 138 | { 139 | _proxies.Add(proxy); 140 | } 141 | catch (UriFormatException) 142 | { 143 | // parsing exception 144 | } 145 | } 146 | AnonymousProxyOnly = anonymousOnly; 147 | _proxyJudgeService = proxyJudgeService; 148 | _numberOfAttemptsPerRequest = _proxies.Count + 1; 149 | _proxyJudgeService.NumberOfAttempts = _numberOfAttempts; 150 | } 151 | public ProxyManager(string file) : this(File.ReadLines(file), false, new ProxyJudgeService()) { } 152 | public ProxyManager(string file, bool anonymousOnly) : this(File.ReadLines(file), anonymousOnly, new ProxyJudgeService()) { } 153 | public ProxyManager(string file, bool anonymousOnly, ProxyJudgeService service) : this(File.ReadLines(file), anonymousOnly, service) { } 154 | 155 | /// 156 | /// Downloads url using GET. 157 | /// 158 | /// Url of webpage 159 | /// Cookies for request. Left null if you don't want to use cookies 160 | /// Specify custom headers for this request 161 | /// 162 | /// Page has returned 404 not found 163 | public string GetPage(string url, string requiredString = null, CookieContainer cookies = null, NameValueCollection customHeaders = null) 164 | { 165 | return PostPage(url, null, requiredString, cookies, customHeaders); 166 | } 167 | 168 | /// 169 | /// Downloads url using POST. 170 | /// 171 | /// Url of webpage 172 | /// Post values 173 | /// Cookies for request. Left null if you don't want to use cookies 174 | /// Specify custom headers for this request 175 | /// Page has returned 404 not found 176 | /// 177 | public string PostPage(string url, NameValueCollection data, string requiredString = null, CookieContainer cookies = null, NameValueCollection customHeaders = null) 178 | { 179 | if (requiredString == null) 180 | requiredString = RequiredString; 181 | 182 | string page = null; 183 | int limit = 0; 184 | 185 | do 186 | { 187 | Proxy proxy = GetAvalibleProxy(); 188 | 189 | try 190 | { 191 | if (AnonymousProxyOnly) 192 | { 193 | if (AnonymousProxyOnly && !proxy.IsAnonymous(ProxyJudgeService)) 194 | { 195 | continue; 196 | } 197 | } 198 | 199 | var bytes = DownloadBytes(url, data, proxy, cookies, customHeaders); 200 | 201 | if (bytes != null) 202 | { 203 | page = Encoding.GetString(bytes); 204 | if (!page.Contains(requiredString)) 205 | { 206 | proxy.IsOnline = false; 207 | } 208 | else 209 | { 210 | return page; 211 | } 212 | } 213 | } 214 | finally 215 | { 216 | limit++; 217 | proxy.IsBusy = false; 218 | } 219 | 220 | limit++; 221 | } while (limit < _numberOfAttemptsPerRequest); 222 | 223 | throw new AllProxiesBannedException(); 224 | } 225 | 226 | /// 227 | /// Downloads url using POST. 228 | /// 229 | /// Url of webpage 230 | /// Post values 231 | /// Cookies for request. Left null if you don't want to use cookies 232 | /// Specify custom headers for this request 233 | /// Page has returned 404 not found 234 | /// 235 | public byte[] DownloadBytes(string url, NameValueCollection data, CookieContainer cookies = null, NameValueCollection customHeaders = null) 236 | { 237 | int limit = 0; 238 | 239 | do 240 | { 241 | Proxy proxy = GetAvalibleProxy(); 242 | 243 | try 244 | { 245 | if (AnonymousProxyOnly && !proxy.IsAnonymous(ProxyJudgeService)) 246 | { 247 | continue; 248 | } 249 | 250 | byte[] result = DownloadBytes(url, data, proxy, cookies, customHeaders); 251 | 252 | if (result != null) 253 | { 254 | return result; 255 | } 256 | } 257 | finally 258 | { 259 | limit++; 260 | proxy.IsBusy = false; 261 | } 262 | 263 | limit++; 264 | } while (limit < _numberOfAttemptsPerRequest); 265 | 266 | throw new AllProxiesBannedException(); 267 | } 268 | 269 | /// 270 | /// Downloads url using GET. 271 | /// 272 | /// 273 | /// 274 | /// Specify custom headers for this request 275 | /// 276 | /// Page has returned 404 not found 277 | public byte[] DownloadBytes(string url, CookieContainer cookies = null, NameValueCollection customHeaders = null) 278 | { 279 | return DownloadBytes(url, null, cookies, customHeaders); 280 | } 281 | /// 282 | /// Returns first free (but busy) and working proxy. 283 | /// 284 | /// 285 | /// All proxies are banned. You can't make request. 286 | public Proxy GetAvalibleProxy() 287 | { 288 | lock (_proxies) 289 | { 290 | Proxy selectedProxy = null; 291 | 292 | do 293 | { 294 | if (_proxies.Count(t => t.IsOnline) == 0) 295 | throw new AllProxiesBannedException(); 296 | 297 | selectedProxy = _proxies.FirstOrDefault(t => t.IsOnline && !t.IsBusy); 298 | if (selectedProxy == null) 299 | { 300 | Thread.Sleep(35); 301 | } 302 | 303 | } while (selectedProxy == null); 304 | 305 | selectedProxy.IsBusy = true; 306 | return selectedProxy; 307 | } 308 | } 309 | /// 310 | /// Sets all proxies IsOnline property to true. 311 | /// 312 | public void SetAllProxyAsOnline() 313 | { 314 | lock (_proxies) 315 | { 316 | _proxies.ForEach(t => t.IsOnline = true); 317 | } 318 | } 319 | 320 | /// 321 | /// Return all proxies 322 | /// 323 | /// 324 | public List GetAllProxies() 325 | { 326 | lock(_proxies) 327 | return CloneProxyList(_proxies); 328 | } 329 | 330 | 331 | private List CloneProxyList(List proxyInput) 332 | { 333 | List proxies = new List(proxyInput.Count); 334 | proxies.AddRange(proxyInput.Select(proxy => (Proxy) proxy.Clone())); 335 | return proxies; 336 | } 337 | 338 | private byte[] DownloadBytes(string url, NameValueCollection data, Proxy proxy, CookieContainer cookies = null, NameValueCollection customHeaders = null) 339 | { 340 | HttpClient client = CreateHttpClient(customHeaders); 341 | client.Proxy = proxy; 342 | if (cookies != null) client.Cookies = cookies; 343 | 344 | try 345 | { 346 | return client.DownloadBytes(url, data); 347 | } 348 | catch (WebException e) 349 | { 350 | if (e.Response != null && (e.Response as HttpWebResponse).StatusCode == HttpStatusCode.NotFound) 351 | throw new WebPageNotFoundException(); 352 | proxy.IsOnline = false; 353 | } 354 | 355 | return null; 356 | } 357 | private HttpClient CreateHttpClient(NameValueCollection customHeaders = null) 358 | { 359 | HttpClient client = new HttpClient(Encoding) 360 | { 361 | Accept = Accept, 362 | AcceptEncoding = AcceptEncoding, 363 | AcceptLanguage = AcceptLanguage, 364 | NumberOfAttempts = NumberOfAttempts, 365 | UserAgent = UserAgent, 366 | Referer = Referer, 367 | Timeout = Timeout 368 | }; 369 | 370 | if (customHeaders != null) 371 | { 372 | if (customHeaders.AllKeys.Contains("Accept")) 373 | { 374 | client.Accept = customHeaders["Accept"]; 375 | customHeaders.Remove("Accept"); 376 | } 377 | if (customHeaders.AllKeys.Contains("Accept-Encoding")) 378 | { 379 | client.AcceptEncoding = customHeaders["Accept-Encoding"]; 380 | customHeaders.Remove("Accept-Encoding"); 381 | } 382 | if (customHeaders.AllKeys.Contains("Accept-Language")) 383 | { 384 | client.AcceptLanguage = customHeaders["Accept-Language"]; 385 | customHeaders.Remove("Accept-Language"); 386 | } 387 | if (customHeaders.AllKeys.Contains("User-Agent")) 388 | { 389 | client.UserAgent = customHeaders["User-Agente"]; 390 | customHeaders.Remove("User-Agent"); 391 | } 392 | if (customHeaders.AllKeys.Contains("Referer")) 393 | { 394 | client.Referer = customHeaders["Referer"]; 395 | customHeaders.Remove("Referer"); 396 | } 397 | 398 | client.CustomHeaders = customHeaders; 399 | } 400 | 401 | return client; 402 | } 403 | private static IEnumerable ParseProxies(IEnumerable proxies) 404 | { 405 | IEnumerable proxyParsed = proxies.Select(t => 406 | { 407 | try 408 | { 409 | Proxy proxy = new Proxy(t); 410 | return proxy; 411 | } 412 | catch (UriFormatException) 413 | { 414 | // parsing exception 415 | return null; 416 | } 417 | }).Where(t => t != null); 418 | return proxyParsed; 419 | } 420 | } 421 | 422 | public class AllProxiesBannedException : Exception 423 | { 424 | } 425 | public class WebPageNotFoundException : Exception 426 | { 427 | } 428 | } 429 | -------------------------------------------------------------------------------- /BetterHttpClient/Socks/NeverEndingStream.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | 8 | namespace BetterHttpClient.Socks 9 | { 10 | public class NeverEndingStream : MemoryStream 11 | { 12 | public override void Close() 13 | { 14 | // Ignore 15 | } 16 | 17 | public void ForceClose() 18 | { 19 | base.Close(); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /BetterHttpClient/Socks/SocksHttpWebRequest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Collections.Specialized; 4 | using System.IO; 5 | using System.Linq; 6 | using System.Net; 7 | using System.Net.Security; 8 | using System.Net.Sockets; 9 | using System.Text; 10 | using System.Threading.Tasks; 11 | using BetterHttpClient.Socks.Extensions; 12 | 13 | namespace BetterHttpClient.Socks 14 | { 15 | internal class SocksHttpWebRequest : WebRequest 16 | { 17 | private static readonly StringCollection validHttpVerbs = new StringCollection { "GET", "HEAD", "POST", "PUT", "DELETE", "TRACE", "OPTIONS" }; 18 | private WebHeaderCollection _HttpRequestHeaders; 19 | private string _method; 20 | private byte[] _requestContentBuffer; 21 | private NeverEndingStream _requestContentStream; 22 | private SocksHttpWebResponse _response; 23 | 24 | private SocksHttpWebRequest(Uri requestUri) 25 | { 26 | RequestUri = requestUri; 27 | } 28 | 29 | public override int Timeout { get; set; } 30 | 31 | public string UserAgent 32 | { 33 | get { return _HttpRequestHeaders["User-Agent"]; } 34 | set { SetSpecialHeaders("User-Agent", value ?? string.Empty); } 35 | } 36 | 37 | public string Referer 38 | { 39 | get { return _HttpRequestHeaders["Referer"]; } 40 | set { SetSpecialHeaders("Referer", value ?? string.Empty); } 41 | } 42 | 43 | public string Accept 44 | { 45 | get { return _HttpRequestHeaders["Accept"]; } 46 | set { SetSpecialHeaders("Accept", value ?? string.Empty); } 47 | } 48 | 49 | public DecompressionMethods AutomaticDecompression 50 | { 51 | get 52 | { 53 | var result = DecompressionMethods.None; 54 | string encoding = _HttpRequestHeaders["Accept-Encoding"] ?? string.Empty; 55 | foreach (string value in encoding.Split(',')) 56 | { 57 | switch (value.Trim()) 58 | { 59 | case "gzip": 60 | result |= DecompressionMethods.GZip; 61 | break; 62 | 63 | case "deflate": 64 | result |= DecompressionMethods.Deflate; 65 | break; 66 | } 67 | } 68 | 69 | return result; 70 | } 71 | set 72 | { 73 | string result = string.Empty; 74 | if ((value & DecompressionMethods.GZip) != 0) 75 | result = "gzip"; 76 | if ((value & DecompressionMethods.Deflate) != 0) 77 | { 78 | if (!string.IsNullOrEmpty(result)) 79 | result += ", "; 80 | result += "deflate"; 81 | } 82 | 83 | SetSpecialHeaders("Accept-Encoding", result); 84 | } 85 | } 86 | 87 | public override Uri RequestUri { get; } 88 | 89 | public override IWebProxy Proxy { get; set; } 90 | 91 | public override WebHeaderCollection Headers 92 | { 93 | get { return _HttpRequestHeaders ?? (_HttpRequestHeaders = new WebHeaderCollection()); } 94 | set 95 | { 96 | if (RequestSubmitted) 97 | { 98 | throw new InvalidOperationException("This operation cannot be performed after the request has been submitted."); 99 | } 100 | _HttpRequestHeaders = value; 101 | } 102 | } 103 | 104 | public bool RequestSubmitted { get; private set; } 105 | 106 | public override string Method 107 | { 108 | get { return _method ?? "GET"; } 109 | set 110 | { 111 | if (validHttpVerbs.Contains(value)) 112 | { 113 | _method = value; 114 | } 115 | else 116 | { 117 | throw new ArgumentOutOfRangeException(nameof(value), $"'{value}' is not a known HTTP verb."); 118 | } 119 | } 120 | } 121 | 122 | public override long ContentLength { get; set; } 123 | 124 | public override string ContentType { get; set; } 125 | public bool AllowAutoRedirect { get; set; } = true; 126 | 127 | public override WebResponse GetResponse() 128 | { 129 | if (Proxy == null) 130 | { 131 | throw new InvalidOperationException("Proxy property cannot be null."); 132 | } 133 | if (string.IsNullOrEmpty(Method)) 134 | { 135 | throw new InvalidOperationException("Method has not been set."); 136 | } 137 | 138 | if (RequestSubmitted) 139 | { 140 | return _response; 141 | } 142 | _response = InternalGetResponse(); 143 | RequestSubmitted = true; 144 | return _response; 145 | } 146 | 147 | public override Stream GetRequestStream() 148 | { 149 | if (RequestSubmitted) 150 | { 151 | throw new InvalidOperationException("This operation cannot be performed after the request has been submitted."); 152 | } 153 | 154 | if (_requestContentBuffer == null) 155 | { 156 | if (ContentLength == 0) 157 | { 158 | _requestContentStream = new NeverEndingStream(); 159 | return _requestContentStream; 160 | } 161 | 162 | _requestContentBuffer = new byte[ContentLength]; 163 | } 164 | else if (ContentLength == default(long)) 165 | { 166 | _requestContentBuffer = new byte[int.MaxValue]; 167 | } 168 | else if (_requestContentBuffer.Length != ContentLength) 169 | { 170 | Array.Resize(ref _requestContentBuffer, (int)ContentLength); 171 | } 172 | return new MemoryStream(_requestContentBuffer); 173 | } 174 | 175 | public override IAsyncResult BeginGetResponse(AsyncCallback callback, object state) 176 | { 177 | if (Proxy == null) 178 | { 179 | throw new InvalidOperationException("Proxy property cannot be null."); 180 | } 181 | if (string.IsNullOrEmpty(Method)) 182 | { 183 | throw new InvalidOperationException("Method has not been set."); 184 | } 185 | 186 | var task = Task.Run(() => 187 | { 188 | if (RequestSubmitted) 189 | { 190 | return _response; 191 | } 192 | _response = InternalGetResponse(); 193 | RequestSubmitted = true; 194 | return _response; 195 | }); 196 | 197 | return task.AsApm(callback, state); 198 | } 199 | 200 | public override WebResponse EndGetResponse(IAsyncResult asyncResult) 201 | { 202 | var task = asyncResult as Task; 203 | 204 | try 205 | { 206 | return task.Result; 207 | } 208 | catch (AggregateException ex) 209 | { 210 | if (ex.InnerException is IOException || ex.InnerException is System.Net.Sockets.SocketException) 211 | throw new WebException("Proxy error " + ex.InnerException.Message, ex.InnerException, WebExceptionStatus.ConnectFailure, 212 | SocksHttpWebResponse.CreateErrorResponse(HttpStatusCode.GatewayTimeout)); 213 | throw ex.InnerException; 214 | } 215 | } 216 | 217 | public new static WebRequest Create(string requestUri) 218 | { 219 | return new SocksHttpWebRequest(new Uri(requestUri)); 220 | } 221 | 222 | public new static WebRequest Create(Uri requestUri) 223 | { 224 | return new SocksHttpWebRequest(requestUri); 225 | } 226 | 227 | private void SetSpecialHeaders(string headerName, string value) 228 | { 229 | _HttpRequestHeaders.Remove(headerName); 230 | if (value.Length != 0) 231 | { 232 | _HttpRequestHeaders.Add(headerName, value); 233 | } 234 | } 235 | 236 | private string BuildHttpRequestMessage(Uri requestUri) 237 | { 238 | if (RequestSubmitted) 239 | { 240 | throw new InvalidOperationException("This operation cannot be performed after the request has been submitted."); 241 | } 242 | 243 | // See if we have a stream instead of byte array 244 | if (_requestContentBuffer == null && _requestContentStream != null) 245 | { 246 | _requestContentBuffer = _requestContentStream.ToArray(); 247 | _requestContentStream.ForceClose(); 248 | _requestContentStream.Dispose(); 249 | _requestContentStream = null; 250 | ContentLength = _requestContentBuffer.Length; 251 | } 252 | 253 | var message = new StringBuilder(); 254 | message.AppendFormat("{0} {1} HTTP/1.1\r\nHost: {2}\r\n", Method, requestUri, requestUri.Host); 255 | 256 | Headers.Set(HttpRequestHeader.Connection, "close"); 257 | 258 | // add the headers 259 | foreach (var key in Headers.Keys) 260 | { 261 | string value = Headers[key.ToString()]; 262 | if (!string.IsNullOrEmpty(value)) 263 | message.AppendFormat("{0}: {1}\r\n", key, value); 264 | } 265 | 266 | if (!string.IsNullOrEmpty(ContentType)) 267 | { 268 | message.AppendFormat("Content-Type: {0}\r\n", ContentType); 269 | } 270 | if (ContentLength > 0) 271 | { 272 | message.AppendFormat("Content-Length: {0}\r\n", ContentLength); 273 | } 274 | 275 | // add a blank line to indicate the end of the headers 276 | message.Append("\r\n"); 277 | 278 | // add content 279 | if (_requestContentBuffer != null && _requestContentBuffer.Length > 0) 280 | { 281 | using (var stream = new MemoryStream(_requestContentBuffer, false)) 282 | { 283 | using (var reader = new StreamReader(stream)) 284 | { 285 | message.Append(reader.ReadToEnd()); 286 | } 287 | } 288 | } 289 | else if (_requestContentStream != null) 290 | { 291 | _requestContentStream.Position = 0; 292 | 293 | using (var reader = new StreamReader(_requestContentStream)) 294 | { 295 | message.Append(reader.ReadToEnd()); 296 | } 297 | 298 | 299 | _requestContentStream.ForceClose(); 300 | _requestContentStream.Dispose(); 301 | } 302 | 303 | return message.ToString(); 304 | } 305 | 306 | private SocksHttpWebResponse InternalGetResponse() 307 | { 308 | Uri requestUri = RequestUri; 309 | 310 | int redirects = 0; 311 | const int maxAutoredirectCount = 10; 312 | while (redirects++ < maxAutoredirectCount) 313 | { 314 | // Loop while redirecting 315 | 316 | var proxyUri = Proxy.GetProxy(requestUri); 317 | var ipAddress = GetProxyIpAddress(proxyUri); 318 | var response = new List(); 319 | 320 | using (var client = new TcpClient(ipAddress.ToString(), proxyUri.Port)) 321 | { 322 | int timeout = Timeout; 323 | if (timeout == 0) 324 | timeout = 30 * 1000; 325 | client.ReceiveTimeout = timeout; 326 | client.SendTimeout = timeout; 327 | var networkStream = client.GetStream(); 328 | // auth 329 | var buf = new byte[300]; 330 | buf[0] = 0x05; // Version 331 | buf[1] = 0x01; // NMETHODS 332 | buf[2] = 0x00; // No auth-method 333 | networkStream.Write(buf, 0, 3); 334 | 335 | networkStream.Read(buf, 0, 2); 336 | if (buf[0] != 0x05) 337 | { 338 | throw new IOException("Invalid Socks Version"); 339 | } 340 | if (buf[1] == 0xff) 341 | { 342 | throw new IOException("Socks Server does not support no-auth"); 343 | } 344 | if (buf[1] != 0x00) 345 | { 346 | throw new Exception("Socks Server did choose bogus auth"); 347 | } 348 | 349 | // connect 350 | var destIP = Dns.GetHostEntry(requestUri.DnsSafeHost).AddressList[0]; 351 | var index = 0; 352 | buf[index++] = 0x05; // version 5 . 353 | buf[index++] = 0x01; // command = connect. 354 | buf[index++] = 0x00; // Reserve = must be 0x00 355 | 356 | buf[index++] = 0x01; // Address is full-qualified domain name. 357 | var rawBytes = destIP.GetAddressBytes(); 358 | rawBytes.CopyTo(buf, index); 359 | index += (ushort)rawBytes.Length; 360 | 361 | var portBytes = BitConverter.GetBytes(Uri.UriSchemeHttps == requestUri.Scheme ? 443 : 80); 362 | for (var i = portBytes.Length - 3; i >= 0; i--) 363 | buf[index++] = portBytes[i]; 364 | 365 | 366 | networkStream.Write(buf, 0, index); 367 | 368 | networkStream.Read(buf, 0, 4); 369 | if (buf[0] != 0x05) 370 | { 371 | throw new IOException("Invalid Socks Version"); 372 | } 373 | if (buf[1] != 0x00) 374 | { 375 | throw new IOException($"Socks Error {buf[1]:X}"); 376 | } 377 | 378 | var rdest = string.Empty; 379 | switch (buf[3]) 380 | { 381 | case 0x01: // IPv4 382 | networkStream.Read(buf, 0, 4); 383 | var v4 = BitConverter.ToUInt32(buf, 0); 384 | rdest = new IPAddress(v4).ToString(); 385 | break; 386 | case 0x03: // Domain name 387 | networkStream.Read(buf, 0, 1); 388 | if (buf[0] == 0xff) 389 | { 390 | throw new IOException("Invalid Domain Name"); 391 | } 392 | networkStream.Read(buf, 1, buf[0]); 393 | rdest = Encoding.ASCII.GetString(buf, 1, buf[0]); 394 | break; 395 | case 0x04: // IPv6 396 | var octets = new byte[16]; 397 | networkStream.Read(octets, 0, 16); 398 | rdest = new IPAddress(octets).ToString(); 399 | break; 400 | default: 401 | throw new IOException("Invalid Address type"); 402 | } 403 | networkStream.Read(buf, 0, 2); 404 | var rport = (ushort)IPAddress.NetworkToHostOrder((short)BitConverter.ToUInt16(buf, 0)); 405 | 406 | Stream readStream = null; 407 | if (Uri.UriSchemeHttps == requestUri.Scheme) 408 | { 409 | var ssl = new SslStream(networkStream); 410 | ssl.AuthenticateAsClient(requestUri.DnsSafeHost); 411 | readStream = ssl; 412 | } 413 | else 414 | { 415 | readStream = networkStream; 416 | } 417 | 418 | string requestString = BuildHttpRequestMessage(requestUri); 419 | 420 | var request = Encoding.ASCII.GetBytes(requestString); 421 | readStream.Write(request, 0, request.Length); 422 | readStream.Flush(); 423 | 424 | var buffer = new byte[client.ReceiveBufferSize]; 425 | 426 | var readlen = 0; 427 | do 428 | { 429 | readlen = readStream.Read(buffer, 0, buffer.Length); 430 | response.AddRange(buffer.Take(readlen)); 431 | } while (readlen != 0); 432 | 433 | readStream.Close(); 434 | } 435 | 436 | var webResponse = new SocksHttpWebResponse(requestUri, response.ToArray()); 437 | 438 | if (webResponse.StatusCode == HttpStatusCode.Moved || webResponse.StatusCode == HttpStatusCode.MovedPermanently) 439 | { 440 | string redirectUrl = webResponse.Headers["Location"]; 441 | if (string.IsNullOrEmpty(redirectUrl)) 442 | throw new WebException("Missing location for redirect"); 443 | 444 | requestUri = new Uri(requestUri, redirectUrl); 445 | if (AllowAutoRedirect) 446 | { 447 | continue; 448 | } 449 | return webResponse; 450 | } 451 | 452 | if ((int)webResponse.StatusCode < 200 || (int)webResponse.StatusCode > 299) 453 | throw new WebException(webResponse.StatusDescription, null, WebExceptionStatus.UnknownError, webResponse); 454 | 455 | return webResponse; 456 | } 457 | 458 | throw new WebException("Too many redirects", null, WebExceptionStatus.ProtocolError, SocksHttpWebResponse.CreateErrorResponse(HttpStatusCode.BadRequest)); 459 | } 460 | 461 | private static IPAddress GetProxyIpAddress(Uri proxyUri) 462 | { 463 | IPAddress ipAddress; 464 | if (!IPAddress.TryParse(proxyUri.Host, out ipAddress)) 465 | { 466 | try 467 | { 468 | return Dns.GetHostEntry(proxyUri.Host).AddressList[0]; 469 | } 470 | catch (Exception e) 471 | { 472 | throw new InvalidOperationException($"Unable to resolve proxy hostname '{proxyUri.Host}' to a valid IP address.", e); 473 | } 474 | } 475 | return ipAddress; 476 | } 477 | } 478 | } -------------------------------------------------------------------------------- /BetterHttpClient/Socks/SocksHttpWebResponse.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.IO.Compression; 4 | using System.Net; 5 | using System.Text; 6 | 7 | // Credit: https://github.com/Yozer/BetterHttpClient 8 | 9 | namespace BetterHttpClient.Socks 10 | { 11 | public class SocksHttpWebResponse : WebResponse 12 | { 13 | #region Member Variables 14 | 15 | private WebHeaderCollection _httpResponseHeaders; 16 | private Uri responseUri; 17 | 18 | #endregion 19 | 20 | #region Constructors 21 | 22 | public SocksHttpWebResponse(Uri responseUri, byte[] httpResponseMessage) 23 | { 24 | this.responseUri = responseUri; 25 | SetHeadersAndResponseContent(httpResponseMessage); 26 | } 27 | 28 | private SocksHttpWebResponse() 29 | { 30 | } 31 | 32 | public static SocksHttpWebResponse CreateErrorResponse(HttpStatusCode statusCode) 33 | { 34 | var response = new SocksHttpWebResponse 35 | { 36 | StatusCode = statusCode 37 | }; 38 | 39 | return response; 40 | } 41 | 42 | #endregion 43 | 44 | #region Methods 45 | 46 | private void SetHeadersAndResponseContent(byte[] responseMessage) 47 | { 48 | if (responseMessage == null || responseMessage.Length < 4) 49 | return; 50 | 51 | var headers = string.Empty; 52 | // get headers part; 53 | var headerEnd = 0; 54 | for (; headerEnd < responseMessage.Length - 3; headerEnd++) 55 | { 56 | if (responseMessage[headerEnd] == 13 && responseMessage[headerEnd + 1] == 10 && responseMessage[headerEnd + 2] == 13 && responseMessage[headerEnd + 3] == 10) 57 | { 58 | headers = Encoding.ASCII.GetString(responseMessage, 0, headerEnd); 59 | break; 60 | } 61 | } 62 | if (string.IsNullOrEmpty(headers)) 63 | return; 64 | 65 | var headerValues = headers.Split(new[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries); 66 | // ignore the first line in the header since it is the HTTP response code 67 | for (var i = 1; i < headerValues.Length; i++) 68 | { 69 | var headerEntry = headerValues[i].Split(':'); 70 | Headers.Add(headerEntry[0], headerEntry[1]); 71 | } 72 | 73 | var statusRegex = new System.Text.RegularExpressions.Regex(@"^HTTP/\d+\.\d+ (\d+)(.*)$"); 74 | var matches = statusRegex.Match(headerValues[0]); 75 | if (matches.Groups.Count < 3) 76 | throw new InvalidDataException(); 77 | StatusCode = (HttpStatusCode)int.Parse(matches.Groups[1].Value); 78 | StatusDescription = matches.Groups[2].Value; 79 | 80 | ResponseContent = new byte[responseMessage.Length - headerEnd - 4]; 81 | Array.Copy(responseMessage, headerEnd + 4, ResponseContent, 0, ResponseContent.Length); 82 | 83 | string transferEncoding = Headers["Transfer-Encoding"]; 84 | 85 | switch (transferEncoding) 86 | { 87 | case "chunked": 88 | DechunkContent(); 89 | break; 90 | } 91 | 92 | string contentEncoding = Headers["Content-Encoding"]; 93 | 94 | switch (contentEncoding) 95 | { 96 | case "gzip": 97 | ResponseContent = Decompress(ResponseContent) ?? ResponseContent; 98 | break; 99 | } 100 | } 101 | 102 | private void DechunkContent() 103 | { 104 | using (var output = new MemoryStream()) 105 | using (var ms = new MemoryStream(ResponseContent)) 106 | { 107 | while (true) 108 | { 109 | var infoLine = new StringBuilder(); 110 | while (true) 111 | { 112 | int b = ms.ReadByte(); 113 | 114 | if (b == -1) 115 | // End of stream 116 | break; 117 | 118 | if (b == 13) 119 | // Ignore 120 | continue; 121 | 122 | if (b == 10) 123 | { 124 | if (infoLine.Length == 0) 125 | // Trim 126 | continue; 127 | else 128 | break; 129 | } 130 | 131 | infoLine.Append((char)b); 132 | } 133 | if (infoLine.Length == 0) 134 | break; 135 | 136 | int chunkLength = int.Parse(infoLine.ToString(), System.Globalization.NumberStyles.HexNumber); 137 | if (chunkLength == 0) 138 | break; 139 | 140 | var buffer = new byte[chunkLength]; 141 | ms.Read(buffer, 0, chunkLength); 142 | output.Write(buffer, 0, buffer.Length); 143 | } 144 | 145 | ResponseContent = output.ToArray(); 146 | } 147 | } 148 | 149 | #endregion 150 | 151 | #region WebResponse Members 152 | 153 | public HttpStatusCode StatusCode { get; private set; } 154 | 155 | public string StatusDescription { get; private set; } 156 | 157 | public override Uri ResponseUri 158 | { 159 | get 160 | { 161 | return this.responseUri; 162 | } 163 | } 164 | 165 | private static readonly byte[] GZipHeaderBytes = { 0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 4, 0 }; 166 | private static readonly byte[] GZipLevel10HeaderBytes = { 0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 2, 0 }; 167 | 168 | public static bool IsPossiblyGZippedBytes(byte[] a) 169 | { 170 | var yes = a.Length > 10; 171 | 172 | if (!yes) 173 | { 174 | return false; 175 | } 176 | 177 | var fail = false; 178 | for (var i = 0; i < 10; i++) 179 | { 180 | if (a[i] != GZipHeaderBytes[i]) 181 | { 182 | fail = true; 183 | break; 184 | } 185 | } 186 | 187 | if (!fail) 188 | return true; 189 | 190 | for (var i = 0; i < 10; i++) 191 | { 192 | if (a[i] != GZipLevel10HeaderBytes[i]) 193 | { 194 | return false; 195 | } 196 | } 197 | 198 | return true; 199 | } 200 | 201 | private byte[] Decompress(byte[] gzip) 202 | { 203 | try 204 | { 205 | using (var stream = new GZipStream(new MemoryStream(gzip), CompressionMode.Decompress)) 206 | { 207 | const int size = 4096; 208 | var buffer = new byte[size]; 209 | using (var memory = new MemoryStream()) 210 | { 211 | var count = 0; 212 | do 213 | { 214 | count = stream.Read(buffer, 0, size); 215 | if (count > 0) 216 | { 217 | memory.Write(buffer, 0, count); 218 | } 219 | } while (count > 0); 220 | return memory.ToArray(); 221 | } 222 | } 223 | } 224 | catch (Exception) 225 | { 226 | return null; 227 | } 228 | } 229 | 230 | public override Stream GetResponseStream() 231 | { 232 | return ResponseContent.Length == 0 ? Stream.Null : new MemoryStream(ResponseContent); 233 | } 234 | 235 | public override void Close() 236 | { 237 | /* the base implementation throws an exception */ 238 | } 239 | 240 | public override WebHeaderCollection Headers 241 | { 242 | get 243 | { 244 | if (_httpResponseHeaders == null) 245 | { 246 | _httpResponseHeaders = new WebHeaderCollection(); 247 | } 248 | return _httpResponseHeaders; 249 | } 250 | } 251 | 252 | public override long ContentLength 253 | { 254 | get { return ResponseContent.Length; } 255 | set { throw new NotSupportedException(); } 256 | } 257 | 258 | #endregion 259 | 260 | #region Properties 261 | 262 | private byte[] ResponseContent { get; set; } 263 | 264 | public Encoding CorrectEncoding { get; set; } 265 | 266 | #endregion 267 | } 268 | } -------------------------------------------------------------------------------- /BetterHttpClient/Socks/TaskExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | 7 | namespace BetterHttpClient.Socks.Extensions 8 | { 9 | public static class TaskExtensions 10 | { 11 | public static IAsyncResult AsApm(this Task task, 12 | AsyncCallback callback, 13 | object state) 14 | { 15 | if (task == null) 16 | throw new ArgumentNullException("task"); 17 | 18 | var tcs = new TaskCompletionSource(state); 19 | task.ContinueWith(t => 20 | { 21 | if (t.IsFaulted) 22 | tcs.TrySetException(t.Exception.InnerExceptions); 23 | else if (t.IsCanceled) 24 | tcs.TrySetCanceled(); 25 | else 26 | tcs.TrySetResult(t.Result); 27 | 28 | if (callback != null) 29 | callback(tcs.Task); 30 | }, TaskScheduler.Default); 31 | return tcs.Task; 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Dominik Baran 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BetterHttpClient 2 | 3 | BetterHttpClient will help you making http and https requests with full cookie support. 4 | 5 | It supports three types of proxy servers: 6 | - HTTP 7 | - HTTPS 8 | - Socks5 9 | 10 | # Http Client 11 | - Http GET request 12 | ```cs 13 | HttpClient client = new HttpClient 14 | { 15 | UserAgent = "My custom user-agent", 16 | Referer = "My referer", 17 | Encoding = Encoding.UTF8, 18 | AcceptEncoding = "gzip, deflate" // setup transfer compression algorithm 19 | }; 20 | string page = client.Get("http://www.google.com"); 21 | ``` 22 | 23 | - Http POST request 24 | ```cs 25 | HttpClient client = new HttpClient(); // if no headers are specified defaults will be used 26 | string page = client.Post("https://httpbin.org/post", new NameValueCollection 27 | { 28 | {"custname", "Some post data"}, 29 | }); 30 | ``` 31 | 32 | - Https GET request using proxy 33 | 34 | HttpClient will detect proxy type (http/s, socks5) if you don't provide such info. 35 | ```cs 36 | string proxyAddress = "218.200.66.196:8080"; 37 | HttpClient client = new HttpClient(new Proxy(proxyAddress)) 38 | { 39 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0" 40 | }; 41 | 42 | string page = client.Get("https://httpbin.org/get"); 43 | ``` 44 | 45 | # Proxy Manager 46 | This class will help you managing large amount of proxy servers. 47 | 48 | You can load proxy list from file and then execute GET or POST requests. 49 | 50 | If some proxy will be dead ProxyManager will mark them as offline and won't use in future requests. 51 | 52 | To perform http/s request it will use first Not Busy and Online proxy (and check for an anonymity level if anonymousOnly is true). 53 | 54 | Set anonymousOnly to true if you want to force ProxyManager to use only proxies that really hide your ip. For more info look at ProxyJudgeService class. 55 | Methods 56 | - GetPage 57 | - PostPage 58 | - DownloadBytes 59 | 60 | are thread safe. 61 | 62 | Example usage: 63 | ```cs 64 | ProxyManager proxyManager = new ProxyManager("proxy_list.txt", anonymousOnly: true) 65 | { 66 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0" 67 | PreserveCookies = false, // if true it will use the same cookies between distinct requests 68 | Timeout = TimeSpan.FromSeconds(10), 69 | NumberOfAttempts = 2 // max number of attempts that can be made per one request 70 | }; 71 | 72 | // ProxyManager will choose first working proxy and use it to perform GET request 73 | string page = proxyManager.GetPage("http://example.com"); 74 | ``` 75 | -------------------------------------------------------------------------------- /UnitTestBetterHttpClient/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("UnitTestBetterHttpClient")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("UnitTestBetterHttpClient")] 13 | [assembly: AssemblyCopyright("Copyright © 2015")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("59acc7d1-547c-4e1c-8de3-ba5351c1a00b")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /UnitTestBetterHttpClient/UnitTestBetterHttpClient.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Debug 5 | AnyCPU 6 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B} 7 | Library 8 | Properties 9 | UnitTestBetterHttpClient 10 | UnitTestBetterHttpClient 11 | v4.5.2 12 | 512 13 | {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} 14 | 10.0 15 | $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) 16 | $(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages 17 | False 18 | UnitTest 19 | 20 | 21 | true 22 | full 23 | false 24 | bin\Debug\ 25 | DEBUG;TRACE 26 | prompt 27 | 4 28 | 29 | 30 | pdbonly 31 | true 32 | bin\Release\ 33 | TRACE 34 | prompt 35 | 4 36 | 37 | 38 | 39 | ..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll 40 | True 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | {fbc5f420-3f80-4a20-a9ec-b6574e18a1d3} 64 | BetterHttpClient 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | False 75 | 76 | 77 | False 78 | 79 | 80 | False 81 | 82 | 83 | False 84 | 85 | 86 | 87 | 88 | 89 | 90 | 97 | -------------------------------------------------------------------------------- /UnitTestBetterHttpClient/UnitTestHttpClient.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Specialized; 3 | using System.Text; 4 | using BetterHttpClient; 5 | using Microsoft.VisualStudio.TestTools.UnitTesting; 6 | using Newtonsoft.Json; 7 | 8 | namespace UnitTestBetterHttpClient 9 | { 10 | [TestClass] 11 | public class UnitTestHttpClient 12 | { 13 | private const string HttpsProxy = "210.245.25.229:3128"; 14 | private const string Socksproxy = "200.239.9.161:10000"; 15 | [TestMethod] 16 | public void TestGet() 17 | { 18 | HttpClient client = new HttpClient 19 | { 20 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0" 21 | }; 22 | 23 | string page = client.Get("http://www.google.com"); 24 | Assert.IsTrue(page.Contains("Google")); 25 | } 26 | 27 | [TestMethod] 28 | public void TestPost() 29 | { 30 | HttpClient client = new HttpClient 31 | { 32 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0" 33 | }; 34 | var size = "medium"; 35 | var topping = "cheese"; 36 | var customerName = "TestName"; 37 | var phone = "TestPhone"; 38 | var email = "testmail@example.com"; 39 | var delivery = "now"; 40 | var comments = "fast"; 41 | 42 | string page = client.Post("https://httpbin.org/post", new NameValueCollection 43 | { 44 | {"custname", customerName}, 45 | {"custtel", phone}, 46 | {"custemail", email}, 47 | {"size", size}, 48 | {"topping", topping}, 49 | {"delivery", delivery}, 50 | {"comments", comments} 51 | }); 52 | 53 | Form root = JsonConvert.DeserializeObject(page).form; 54 | 55 | Assert.AreEqual(root.custname, customerName); 56 | Assert.AreEqual(root.custtel, phone); 57 | Assert.AreEqual(root.custemail, email); 58 | Assert.AreEqual(root.size, size); 59 | Assert.AreEqual(root.topping, topping); 60 | Assert.AreEqual(root.delivery, delivery); 61 | Assert.AreEqual(root.comments, comments); 62 | 63 | } 64 | 65 | [TestMethod] 66 | public void TestUserAgent() 67 | { 68 | HttpClient client = new HttpClient 69 | { 70 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0" 71 | }; 72 | 73 | string page = client.Get("https://httpbin.org/user-agent"); 74 | Assert.IsTrue(page.Contains(client.UserAgent)); 75 | } 76 | 77 | [TestMethod] 78 | public void TestGzipDecodingAndReferer() 79 | { 80 | HttpClient client = new HttpClient 81 | { 82 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0", 83 | Referer = "https://httpbin.org/" 84 | }; 85 | 86 | string page = client.Get("https://httpbin.org/gzip"); 87 | Assert.IsTrue(page.Contains(client.UserAgent)); 88 | // check for referer 89 | Assert.IsTrue(page.Contains("https://httpbin.org/")); 90 | } 91 | 92 | [TestMethod] 93 | public void TestHttpProxy() 94 | { 95 | HttpClient client = new HttpClient(new Proxy(HttpsProxy)) 96 | { 97 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0", 98 | Encoding = Encoding.GetEncoding("iso-8859-2"), 99 | AcceptEncoding = "gzip" 100 | }; 101 | 102 | string page = client.Get("http://darkwarez.pl"); 103 | Assert.IsTrue(page.Contains("Polskie Forum Warez! Najnowsze linki")); 104 | } 105 | [TestMethod] 106 | public void TestHttpsProxy() 107 | { 108 | HttpClient client = new HttpClient(new Proxy(HttpsProxy)) 109 | { 110 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0" 111 | }; 112 | 113 | string page = client.Get("https://httpbin.org/get"); 114 | Assert.IsTrue(page.Contains(client.UserAgent)); 115 | } 116 | [TestMethod] 117 | public void TestSocksHttpProxy() 118 | { 119 | HttpClient client = new HttpClient(new Proxy(Socksproxy)) 120 | { 121 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0", 122 | Encoding = Encoding.GetEncoding("iso-8859-2"), 123 | AcceptEncoding = "deflate" 124 | }; 125 | 126 | string page = client.Get("http://darkwarez.pl/forum/"); 127 | Assert.IsTrue(page.Contains("darkwarez.pl - Gry, Muzyka, Filmy, Download")); 128 | } 129 | [TestMethod] 130 | public void TestSocksHttpsProxyDeflateEncoding() 131 | { 132 | HttpClient client = new HttpClient(new Proxy(Socksproxy)) 133 | { 134 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0", 135 | AcceptEncoding = "deflate" 136 | }; 137 | 138 | string page = client.Get("https://httpbin.org/get"); 139 | Assert.IsTrue(page.Contains(client.UserAgent)); 140 | } 141 | [TestMethod] 142 | public void TestSocksHttpsProxyGzipEndcoding() 143 | { 144 | HttpClient client = new HttpClient(new Proxy(Socksproxy)) 145 | { 146 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0", 147 | AcceptEncoding = "gzip" 148 | }; 149 | 150 | string page = client.Get("https://httpbin.org/get"); 151 | Assert.IsTrue(page.Contains(client.UserAgent)); 152 | } 153 | public class Form 154 | { 155 | public string comments { get; set; } 156 | public string custemail { get; set; } 157 | public string custname { get; set; } 158 | public string custtel { get; set; } 159 | public string delivery { get; set; } 160 | public string size { get; set; } 161 | public string topping { get; set; } 162 | } 163 | 164 | public class RootObject 165 | { 166 | public object args { get; set; } 167 | public string data { get; set; } 168 | public object files { get; set; } 169 | public Form form { get; set; } 170 | public object headers { get; set; } 171 | public object json { get; set; } 172 | public string origin { get; set; } 173 | public string url { get; set; } 174 | } 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /UnitTestBetterHttpClient/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | --------------------------------------------------------------------------------