├── .gitattributes
├── .gitignore
├── BetterHttpClient.sln
├── BetterHttpClient
├── BetterHttpClient.csproj
├── CheckService
│ ├── ProxyCheckService.cs
│ └── ProxyJudgeProxyCheckService.cs
├── HttpClient.cs
├── Properties
│ └── AssemblyInfo.cs
├── Proxy.cs
├── ProxyManager.cs
└── Socks
│ ├── NeverEndingStream.cs
│ ├── SocksHttpWebRequest.cs
│ ├── SocksHttpWebResponse.cs
│ └── TaskExtensions.cs
├── LICENSE
├── README.md
└── UnitTestBetterHttpClient
├── Properties
└── AssemblyInfo.cs
├── UnitTestBetterHttpClient.csproj
├── UnitTestHttpClient.cs
└── packages.config
/.gitattributes:
--------------------------------------------------------------------------------
1 | ###############################################################################
2 | # Set default behavior to automatically normalize line endings.
3 | ###############################################################################
4 | * text=auto
5 |
6 | ###############################################################################
7 | # Set default behavior for command prompt diff.
8 | #
9 | # This is need for earlier builds of msysgit that does not have it on by
10 | # default for csharp files.
11 | # Note: This is only used by command line
12 | ###############################################################################
13 | #*.cs diff=csharp
14 |
15 | ###############################################################################
16 | # Set the merge driver for project and solution files
17 | #
18 | # Merging from the command prompt will add diff markers to the files if there
19 | # are conflicts (Merging from VS is not affected by the settings below, in VS
20 | # the diff markers are never inserted). Diff markers may cause the following
21 | # file extensions to fail to load in VS. An alternative would be to treat
22 | # these files as binary and thus will always conflict and require user
23 | # intervention with every merge. To do so, just uncomment the entries below
24 | ###############################################################################
25 | #*.sln merge=binary
26 | #*.csproj merge=binary
27 | #*.vbproj merge=binary
28 | #*.vcxproj merge=binary
29 | #*.vcproj merge=binary
30 | #*.dbproj merge=binary
31 | #*.fsproj merge=binary
32 | #*.lsproj merge=binary
33 | #*.wixproj merge=binary
34 | #*.modelproj merge=binary
35 | #*.sqlproj merge=binary
36 | #*.wwaproj merge=binary
37 |
38 | ###############################################################################
39 | # behavior for image files
40 | #
41 | # image files are treated as binary by default.
42 | ###############################################################################
43 | #*.jpg binary
44 | #*.png binary
45 | #*.gif binary
46 |
47 | ###############################################################################
48 | # diff behavior for common document formats
49 | #
50 | # Convert binary document formats to text before diffing them. This feature
51 | # is only available from the command line. Turn it on by uncommenting the
52 | # entries below.
53 | ###############################################################################
54 | #*.doc diff=astextplain
55 | #*.DOC diff=astextplain
56 | #*.docx diff=astextplain
57 | #*.DOCX diff=astextplain
58 | #*.dot diff=astextplain
59 | #*.DOT diff=astextplain
60 | #*.pdf diff=astextplain
61 | #*.PDF diff=astextplain
62 | #*.rtf diff=astextplain
63 | #*.RTF diff=astextplain
64 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | ## Ignore Visual Studio temporary files, build results, and
2 | ## files generated by popular Visual Studio add-ons.
3 |
4 | # User-specific files
5 | *.suo
6 | *.user
7 | *.userosscache
8 | *.sln.docstates
9 |
10 | # User-specific files (MonoDevelop/Xamarin Studio)
11 | *.userprefs
12 |
13 | # Build results
14 | [Dd]ebug/
15 | [Dd]ebugPublic/
16 | [Rr]elease/
17 | [Rr]eleases/
18 | x64/
19 | x86/
20 | build/
21 | bld/
22 | [Bb]in/
23 | [Oo]bj/
24 |
25 | # Visual Studio 2015 cache/options directory
26 | .vs/
27 |
28 | # MSTest test Results
29 | [Tt]est[Rr]esult*/
30 | [Bb]uild[Ll]og.*
31 |
32 | # NUNIT
33 | *.VisualState.xml
34 | TestResult.xml
35 |
36 | # Build Results of an ATL Project
37 | [Dd]ebugPS/
38 | [Rr]eleasePS/
39 | dlldata.c
40 |
41 | # DNX
42 | project.lock.json
43 | artifacts/
44 |
45 | *_i.c
46 | *_p.c
47 | *_i.h
48 | *.ilk
49 | *.meta
50 | *.obj
51 | *.pch
52 | *.pdb
53 | *.pgc
54 | *.pgd
55 | *.rsp
56 | *.sbr
57 | *.tlb
58 | *.tli
59 | *.tlh
60 | *.tmp
61 | *.tmp_proj
62 | *.log
63 | *.vspscc
64 | *.vssscc
65 | .builds
66 | *.pidb
67 | *.svclog
68 | *.scc
69 |
70 | # Chutzpah Test files
71 | _Chutzpah*
72 |
73 | # Visual C++ cache files
74 | ipch/
75 | *.aps
76 | *.ncb
77 | *.opensdf
78 | *.sdf
79 | *.cachefile
80 |
81 | # Visual Studio profiler
82 | *.psess
83 | *.vsp
84 | *.vspx
85 |
86 | # TFS 2012 Local Workspace
87 | $tf/
88 |
89 | # Guidance Automation Toolkit
90 | *.gpState
91 |
92 | # ReSharper is a .NET coding add-in
93 | _ReSharper*/
94 | *.[Rr]e[Ss]harper
95 | *.DotSettings.user
96 |
97 | # JustCode is a .NET coding add-in
98 | .JustCode
99 |
100 | # TeamCity is a build add-in
101 | _TeamCity*
102 |
103 | # DotCover is a Code Coverage Tool
104 | *.dotCover
105 |
106 | # NCrunch
107 | _NCrunch_*
108 | .*crunch*.local.xml
109 |
110 | # MightyMoose
111 | *.mm.*
112 | AutoTest.Net/
113 |
114 | # Web workbench (sass)
115 | .sass-cache/
116 |
117 | # Installshield output folder
118 | [Ee]xpress/
119 |
120 | # DocProject is a documentation generator add-in
121 | DocProject/buildhelp/
122 | DocProject/Help/*.HxT
123 | DocProject/Help/*.HxC
124 | DocProject/Help/*.hhc
125 | DocProject/Help/*.hhk
126 | DocProject/Help/*.hhp
127 | DocProject/Help/Html2
128 | DocProject/Help/html
129 |
130 | # Click-Once directory
131 | publish/
132 |
133 | # Publish Web Output
134 | *.[Pp]ublish.xml
135 | *.azurePubxml
136 | ## TODO: Comment the next line if you want to checkin your
137 | ## web deploy settings but do note that will include unencrypted
138 | ## passwords
139 | #*.pubxml
140 |
141 | *.publishproj
142 |
143 | # NuGet Packages
144 | *.nupkg
145 | # The packages folder can be ignored because of Package Restore
146 | **/packages/*
147 | # except build/, which is used as an MSBuild target.
148 | !**/packages/build/
149 | # Uncomment if necessary however generally it will be regenerated when needed
150 | #!**/packages/repositories.config
151 |
152 | # Windows Azure Build Output
153 | csx/
154 | *.build.csdef
155 |
156 | # Windows Store app package directory
157 | AppPackages/
158 |
159 | # Visual Studio cache files
160 | # files ending in .cache can be ignored
161 | *.[Cc]ache
162 | # but keep track of directories ending in .cache
163 | !*.[Cc]ache/
164 |
165 | # Others
166 | ClientBin/
167 | [Ss]tyle[Cc]op.*
168 | ~$*
169 | *~
170 | *.dbmdl
171 | *.dbproj.schemaview
172 | *.pfx
173 | *.publishsettings
174 | node_modules/
175 | orleans.codegen.cs
176 |
177 | # RIA/Silverlight projects
178 | Generated_Code/
179 |
180 | # Backup & report files from converting an old project file
181 | # to a newer Visual Studio version. Backup files are not needed,
182 | # because we have git ;-)
183 | _UpgradeReport_Files/
184 | Backup*/
185 | UpgradeLog*.XML
186 | UpgradeLog*.htm
187 |
188 | # SQL Server files
189 | *.mdf
190 | *.ldf
191 |
192 | # Business Intelligence projects
193 | *.rdl.data
194 | *.bim.layout
195 | *.bim_*.settings
196 |
197 | # Microsoft Fakes
198 | FakesAssemblies/
199 |
200 | # Node.js Tools for Visual Studio
201 | .ntvs_analysis.dat
202 |
203 | # Visual Studio 6 build log
204 | *.plg
205 |
206 | # Visual Studio 6 workspace options file
207 | *.opt
208 |
209 | # LightSwitch generated files
210 | GeneratedArtifacts/
211 | _Pvt_Extensions/
212 | ModelManifest.xml
213 |
--------------------------------------------------------------------------------
/BetterHttpClient.sln:
--------------------------------------------------------------------------------
1 |
2 | Microsoft Visual Studio Solution File, Format Version 12.00
3 | # Visual Studio 14
4 | VisualStudioVersion = 14.0.23107.0
5 | MinimumVisualStudioVersion = 10.0.40219.1
6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BetterHttpClient", "BetterHttpClient\BetterHttpClient.csproj", "{FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}"
7 | EndProject
8 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UnitTestBetterHttpClient", "UnitTestBetterHttpClient\UnitTestBetterHttpClient.csproj", "{59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}"
9 | EndProject
10 | Global
11 | GlobalSection(SolutionConfigurationPlatforms) = preSolution
12 | Debug|Any CPU = Debug|Any CPU
13 | Debug|ARM = Debug|ARM
14 | Debug|x64 = Debug|x64
15 | Debug|x86 = Debug|x86
16 | Release|Any CPU = Release|Any CPU
17 | Release|ARM = Release|ARM
18 | Release|x64 = Release|x64
19 | Release|x86 = Release|x86
20 | EndGlobalSection
21 | GlobalSection(ProjectConfigurationPlatforms) = postSolution
22 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
23 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|Any CPU.Build.0 = Debug|Any CPU
24 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|ARM.ActiveCfg = Debug|Any CPU
25 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|ARM.Build.0 = Debug|Any CPU
26 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|x64.ActiveCfg = Debug|Any CPU
27 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|x64.Build.0 = Debug|Any CPU
28 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|x86.ActiveCfg = Debug|Any CPU
29 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|x86.Build.0 = Debug|Any CPU
30 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|Any CPU.ActiveCfg = Release|Any CPU
31 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|Any CPU.Build.0 = Release|Any CPU
32 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|ARM.ActiveCfg = Release|Any CPU
33 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|ARM.Build.0 = Release|Any CPU
34 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|x64.ActiveCfg = Release|Any CPU
35 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|x64.Build.0 = Release|Any CPU
36 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|x86.ActiveCfg = Release|Any CPU
37 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|x86.Build.0 = Release|Any CPU
38 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
39 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|Any CPU.Build.0 = Debug|Any CPU
40 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|ARM.ActiveCfg = Debug|Any CPU
41 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|ARM.Build.0 = Debug|Any CPU
42 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|x64.ActiveCfg = Debug|Any CPU
43 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|x64.Build.0 = Debug|Any CPU
44 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|x86.ActiveCfg = Debug|Any CPU
45 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|x86.Build.0 = Debug|Any CPU
46 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|Any CPU.ActiveCfg = Release|Any CPU
47 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|Any CPU.Build.0 = Release|Any CPU
48 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|ARM.ActiveCfg = Release|Any CPU
49 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|ARM.Build.0 = Release|Any CPU
50 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|x64.ActiveCfg = Release|Any CPU
51 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|x64.Build.0 = Release|Any CPU
52 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|x86.ActiveCfg = Release|Any CPU
53 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|x86.Build.0 = Release|Any CPU
54 | EndGlobalSection
55 | GlobalSection(SolutionProperties) = preSolution
56 | HideSolutionNode = FALSE
57 | EndGlobalSection
58 | EndGlobal
59 |
--------------------------------------------------------------------------------
/BetterHttpClient/BetterHttpClient.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Debug
6 | AnyCPU
7 | {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}
8 | Library
9 | Properties
10 | BetterHttpClient
11 | BetterHttpClient
12 | v4.5
13 | 512
14 |
15 |
16 |
17 | true
18 | full
19 | false
20 | bin\Debug\
21 | DEBUG;TRACE
22 | prompt
23 | 4
24 |
25 |
26 | pdbonly
27 | true
28 | bin\Release\
29 | TRACE
30 | prompt
31 | 4
32 | bin\Release\BetterHttpClient.XML
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 | Component
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
65 |
--------------------------------------------------------------------------------
/BetterHttpClient/CheckService/ProxyCheckService.cs:
--------------------------------------------------------------------------------
1 | using System;
2 |
3 | namespace BetterHttpClient.CheckService
4 | {
5 | public abstract class ProxyJudgeServiceAbstract
6 | {
7 | private string _myIp = null;
8 | private readonly object _syncLock = new object();
9 |
10 | public string MyIp
11 | {
12 | get
13 | {
14 | lock (_syncLock)
15 | {
16 | if(_myIp == null)
17 | _myIp = GetMyIp();
18 | }
19 |
20 | return _myIp;
21 | }
22 | }
23 | ///
24 | /// Set number of attempts which can be made to verify proxy.
25 | ///
26 | public int NumberOfAttempts { get; set; }
27 | ///
28 | /// Abstract method. Should return true if proxy is hiding your real ip address.
29 | ///
30 | ///
31 | ///
32 | public abstract bool IsProxyAnonymous(Proxy proxy);
33 | ///
34 | /// Abstract method. Should return real ip addres of user (so without any proxy).
35 | ///
36 | ///
37 | protected abstract string GetMyIp();
38 | }
39 | public class GetMyIpException : Exception
40 | {
41 | }
42 | }
--------------------------------------------------------------------------------
/BetterHttpClient/CheckService/ProxyJudgeProxyCheckService.cs:
--------------------------------------------------------------------------------
1 | using System.Net;
2 | using System.Text;
3 | using System.Text.RegularExpressions;
4 |
5 | namespace BetterHttpClient.CheckService
6 | {
7 | public class ProxyJudgeService : ProxyJudgeServiceAbstract
8 | {
9 | public override bool IsProxyAnonymous(Proxy proxy)
10 | {
11 | HttpClient client = new HttpClient(proxy, Encoding.UTF8)
12 | {
13 | NumberOfAttempts = NumberOfAttempts
14 | };
15 | string page = null;
16 |
17 | try
18 | {
19 | page = client.Get("http://proxyjudge.info/");
20 | }
21 | catch (WebException)
22 | {
23 | proxy.IsOnline = false;
24 | }
25 |
26 | if (page == null)
27 | return false;
28 | if (page.Contains("
Proxyjudge.info") && !page.Contains(MyIp))
29 | return true;
30 | return false;
31 | }
32 |
33 | protected override string GetMyIp()
34 | {
35 | HttpClient client = new HttpClient(Encoding.UTF8);
36 | string page = client.Get("http://proxyjudge.info/");
37 |
38 | Match match = Regex.Match(page, "REMOTE_ADDR = (.*?)\\n");
39 | if (!match.Success)
40 | throw new GetMyIpException();
41 | else
42 | return match.Groups[1].Value;
43 | }
44 | }
45 | }
--------------------------------------------------------------------------------
/BetterHttpClient/HttpClient.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Specialized;
3 | using System.Net;
4 | using System.Text;
5 | using BetterHttpClient.Socks;
6 |
7 | namespace BetterHttpClient
8 | {
9 | public class HttpClient : WebClient
10 | {
11 | private int _numberOfAttempts = 4;
12 | private TimeSpan _timeout = TimeSpan.FromSeconds(60);
13 | private Proxy _proxy;
14 |
15 | ///
16 | /// Cookie container.
17 | ///
18 | public CookieContainer Cookies { get; set; } = new CookieContainer();
19 |
20 | ///
21 | /// Proxy which should be used for request.
22 | /// Set null or proxy with type ProxyTypeEnum.None if you want to perform request without proxy.
23 | ///
24 | public new Proxy Proxy
25 | {
26 | get { return _proxy; }
27 | set { _proxy = value ?? new Proxy(); }
28 | }
29 | ///
30 | /// Timeout for request.
31 | ///
32 | /// Has to be greater than 5 milliseconds.
33 | /// Default value: 60 seconds.
34 | public TimeSpan Timeout
35 | {
36 | get
37 | {
38 | return _timeout;
39 | }
40 | set
41 | {
42 | if (value.TotalMilliseconds < 5)
43 | throw new ArgumentOutOfRangeException("Timeout has to be greater or equal than 5 milliseconds.");
44 | _timeout = value;
45 | }
46 | }
47 | ///
48 | /// Set number of attempts that can be made to execute request.
49 | ///
50 | /// Default value: 4 attempts.
51 | /// Should be greater than 1
52 | public int NumberOfAttempts
53 | {
54 | get { return _numberOfAttempts; }
55 | set
56 | {
57 | if (value < 1)
58 | throw new ArgumentOutOfRangeException("Value has to be greater than one.");
59 |
60 | _numberOfAttempts = value;
61 | }
62 | }
63 |
64 | ///
65 | /// Set User-Agent header.
66 | ///
67 | /// Default value: "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0"
68 | public string UserAgent { get; set; } = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0";
69 | ///
70 | /// Set Accept header.
71 | ///
72 | /// Default value: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
73 | public string Accept { get; set; } = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
74 | ///
75 | /// Set Referer header.
76 | ///
77 | /// Default value: null
78 | public string Referer { get; set; }
79 | ///
80 | /// Set Accept-Language header.
81 | ///
82 | /// Default value: "en-US;q=0.7,en;q=0.3"
83 | public string AcceptLanguage { get; set; } = "en-US;q=0.7,en;q=0.3";
84 | ///
85 | /// Set default Accept-Encoding header.
86 | ///
87 | /// Default value: "gzip, deflate"
88 | public string AcceptEncoding { get; set; } = "gzip, deflate";
89 | ///
90 | /// Set encoding for request.
91 | ///
92 | /// Default value: UTF-8
93 | public new Encoding Encoding
94 | {
95 | get { return base.Encoding; }
96 | set { base.Encoding = value; }
97 | }
98 | ///
99 | /// Enabled automatic redirect. Default: true
100 | ///
101 | public bool AllowAutoRedirect { get; set; } = true;
102 |
103 | ///
104 | /// Headers collection that will be added to each request
105 | ///
106 | public NameValueCollection CustomHeaders { get; set; }
107 | public HttpClient(Proxy proxy) : this(proxy, Encoding.UTF8) { }
108 |
109 | public HttpClient() : this(new Proxy(), Encoding.UTF8) { }
110 | public HttpClient(Encoding encoding) : this(new Proxy(), encoding) { }
111 | public HttpClient(Proxy proxy, Encoding encoding)
112 | {
113 | Encoding = encoding;
114 | Proxy = proxy;
115 | }
116 |
117 | protected override WebRequest GetWebRequest(Uri address)
118 | {
119 | WebRequest request = null;
120 |
121 | if (Proxy.ProxyType != ProxyTypeEnum.Socks)
122 | {
123 | request = base.GetWebRequest(address);
124 | }
125 | else
126 | {
127 | request = SocksHttpWebRequest.Create(address);
128 | request.Method = base.GetWebRequest(address).Method;
129 | request.ContentLength = base.GetWebRequest(address).ContentLength;
130 | request.ContentType = base.GetWebRequest(address).ContentType;
131 | }
132 |
133 |
134 | request.Headers.Add("Cookie", Cookies.GetCookieHeader(address));
135 | request.Headers.Add("Accept-Language", AcceptLanguage);
136 | request.Headers.Add("Accept-Encoding", AcceptEncoding);
137 | if (CustomHeaders != null)
138 | {
139 | foreach (string key in CustomHeaders.AllKeys)
140 | {
141 | request.Headers.Add(key, CustomHeaders[key]);
142 | }
143 | }
144 |
145 |
146 | if (Proxy.ProxyType != ProxyTypeEnum.Socks)
147 | {
148 | var httpRequest = (request as HttpWebRequest);
149 | httpRequest.UserAgent = UserAgent;
150 | httpRequest.Accept = Accept;
151 | httpRequest.Referer = Referer;
152 | httpRequest.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
153 | httpRequest.AllowAutoRedirect = AllowAutoRedirect;
154 | }
155 | else if (Proxy.ProxyType == ProxyTypeEnum.Socks)
156 | {
157 | var socksRequest = (request as SocksHttpWebRequest);
158 | socksRequest.UserAgent = UserAgent;
159 | socksRequest.Accept = Accept;
160 | socksRequest.Referer = Referer;
161 | socksRequest.AllowAutoRedirect = AllowAutoRedirect;
162 | }
163 |
164 | request.Timeout = (int) Timeout.TotalMilliseconds;
165 | request.Proxy = Proxy.ProxyItem;
166 |
167 | return request;
168 | }
169 |
170 | protected override WebResponse GetWebResponse(WebRequest request)
171 | {
172 | var response = base.GetWebResponse(request);
173 | try
174 | {
175 | string setCookies = response.Headers["Set-Cookie"];
176 | Cookies.SetCookies(request.RequestUri, setCookies);
177 | }
178 | catch (Exception)
179 | {
180 |
181 | }
182 |
183 | return response;
184 | }
185 | ///
186 | /// Execute GET request.
187 | ///
188 | ///
189 | ///
190 | public string Get(string url)
191 | {
192 | return Encoding.GetString(DownloadBytes(url, null));
193 | }
194 | ///
195 | /// Execute POST request.
196 | ///
197 | ///
198 | ///
199 | ///
200 | public string Post(string url, NameValueCollection data)
201 | {
202 | return Encoding.GetString(DownloadBytes(url, data));
203 | }
204 |
205 | ///
206 | /// Execute GET request.
207 | ///
208 | ///
209 | ///
210 | public byte[] DownloadBytes(string url)
211 | {
212 | return DownloadBytes(url, null);
213 | }
214 | ///
215 | /// Excecute POST request.
216 | ///
217 | ///
218 | ///
219 | ///
220 | public byte[] DownloadBytes(string url, NameValueCollection data)
221 | {
222 | int counter = 0;
223 | WebException lastWebException = null;
224 | bool unkownProxy = Proxy.ProxyType == ProxyTypeEnum.Unknown;
225 |
226 | while (counter < NumberOfAttempts + (NumberOfAttempts < 2 && unkownProxy ? 1 : 0)) // min two try for unkonwn proxy type
227 | {
228 | try
229 | {
230 | if (unkownProxy && counter % 2 == 0) // every odd try is as http proxy
231 | Proxy.ProxyType = ProxyTypeEnum.Http;
232 | else if (unkownProxy)
233 | Proxy.ProxyType = ProxyTypeEnum.Socks;
234 |
235 | byte[] result = data == null ? Encoding.GetBytes(DownloadString(url)) : UploadValues(url, data);
236 | return result;
237 | }
238 | catch (WebException e)
239 | {
240 | lastWebException = e;
241 | counter++;
242 | }
243 | }
244 |
245 | if (unkownProxy)
246 | Proxy.ProxyType = ProxyTypeEnum.Unknown;
247 | // ReSharper disable once PossibleNullReferenceException
248 | throw lastWebException;
249 | }
250 | }
251 | }
252 |
--------------------------------------------------------------------------------
/BetterHttpClient/Properties/AssemblyInfo.cs:
--------------------------------------------------------------------------------
1 | using System.Reflection;
2 | using System.Runtime.InteropServices;
3 |
4 | // General Information about an assembly is controlled through the following
5 | // set of attributes. Change these attribute values to modify the information
6 | // associated with an assembly.
7 |
8 | [assembly: AssemblyTitle("HttpClient")]
9 | [assembly: AssemblyDescription("")]
10 | [assembly: AssemblyConfiguration("")]
11 | [assembly: AssemblyCompany("")]
12 | [assembly: AssemblyProduct("HttpClient")]
13 | [assembly: AssemblyCopyright("Copyright © 2015")]
14 | [assembly: AssemblyTrademark("")]
15 | [assembly: AssemblyCulture("")]
16 |
17 | // Setting ComVisible to false makes the types in this assembly not visible
18 | // to COM components. If you need to access a type in this assembly from
19 | // COM, set the ComVisible attribute to true on that type.
20 |
21 | [assembly: ComVisible(false)]
22 |
23 | // The following GUID is for the ID of the typelib if this project is exposed to COM
24 |
25 | [assembly: Guid("fbc5f420-3f80-4a20-a9ec-b6574e18a1d3")]
26 |
27 | // Version information for an assembly consists of the following four values:
28 | //
29 | // Major Version
30 | // Minor Version
31 | // Build Number
32 | // Revision
33 | //
34 | // You can specify all the values or you can default the Build and Revision Numbers
35 | // by using the '*' as shown below:
36 | // [assembly: AssemblyVersion("1.0.*")]
37 |
38 | [assembly: AssemblyVersion("1.0.7")]
39 | [assembly: AssemblyFileVersion("1.0.0")]
--------------------------------------------------------------------------------
/BetterHttpClient/Proxy.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Net;
3 | using BetterHttpClient.CheckService;
4 |
5 | namespace BetterHttpClient
6 | {
7 | public class Proxy : ICloneable
8 | {
9 | private volatile bool _isBusy = false;
10 | private bool _isAnonymous;
11 | private bool _isChecked = false;
12 |
13 | ///
14 | /// Check if proxy is busy.
15 | ///
16 | public bool IsBusy
17 | {
18 | get { return _isBusy; }
19 | internal set { _isBusy = value; }
20 | }
21 |
22 |
23 | public bool IsOnline { get; set; } = true;
24 | public ProxyTypeEnum ProxyType { get; internal set; }
25 | internal WebProxy ProxyItem { get; set; }
26 |
27 | public Proxy()
28 | {
29 | ProxyItem = new WebProxy();
30 | ProxyType = ProxyTypeEnum.None;
31 | }
32 |
33 | public Proxy(string address)
34 | {
35 | ProxyItem = new WebProxy(address);
36 | }
37 |
38 | public Proxy(string ip, int port)
39 | {
40 | ProxyItem = new WebProxy(ip, port);
41 | }
42 |
43 | public Proxy(string ip, int port, ProxyTypeEnum proxyType)
44 | {
45 | ProxyItem = new WebProxy(ip, port);
46 | ProxyType = proxyType;
47 | }
48 |
49 | ///
50 | /// Returns true if proxy can hide your ip address
51 | ///
52 | public bool IsAnonymous(ProxyJudgeService service)
53 | {
54 | if (_isChecked)
55 | return _isAnonymous;
56 |
57 | _isAnonymous = service.IsProxyAnonymous(this);
58 | _isChecked = true;
59 | IsOnline = _isAnonymous;
60 | return _isAnonymous;
61 | }
62 |
63 | public object Clone()
64 | {
65 | var proxy = new Proxy
66 | {
67 | _isAnonymous = _isAnonymous,
68 | _isBusy = _isBusy,
69 | _isChecked = _isChecked,
70 | IsOnline = IsOnline,
71 | ProxyItem = ProxyItem,
72 | ProxyType = ProxyType
73 | };
74 | return proxy;
75 | }
76 | }
77 |
78 | public enum ProxyTypeEnum
79 | {
80 | Unknown,
81 | None,
82 | Http,
83 | Socks
84 | }
85 | }
--------------------------------------------------------------------------------
/BetterHttpClient/ProxyManager.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Collections.Specialized;
4 | using System.IO;
5 | using System.Linq;
6 | using System.Net;
7 | using System.Text;
8 | using System.Threading;
9 | using BetterHttpClient.CheckService;
10 |
11 | namespace BetterHttpClient
12 | {
13 | public class ProxyManager
14 | {
15 | private List _proxies = new List();
16 | private string _requiredString = string.Empty;
17 | private int _numberOfAttemptsPerRequest;
18 | private TimeSpan _timeout = TimeSpan.FromSeconds(10);
19 | private int _numberOfAttempts = 4;
20 | private ProxyJudgeService _proxyJudgeService;
21 |
22 | ///
23 | /// Set User-Agent header.
24 | ///
25 | /// Default value: "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0"
26 | public string UserAgent { get; set; } = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0";
27 | ///
28 | /// Set Accept header.
29 | ///
30 | /// Default value: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
31 | public string Accept { get; set; } = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
32 | ///
33 | /// Set Referer header.
34 | ///
35 | /// Default value: null
36 | public string Referer { get; set; }
37 | ///
38 | /// Set Accept-Language header.
39 | ///
40 | /// Default value: "en-US;q=0.7,en;q=0.3"
41 | public string AcceptLanguage { get; set; } = "en-US;q=0.7,en;q=0.3";
42 | ///
43 | /// Set default Accept-Encoding header.
44 | ///
45 | /// Default value: "gzip, deflate"
46 | public string AcceptEncoding { get; set; } = "gzip, deflate";
47 |
48 | ///
49 | /// Set proxy check service.
50 | /// Must derive from ProxyCheckService class.
51 | ///
52 | public ProxyJudgeService ProxyJudgeService
53 | {
54 | get { return _proxyJudgeService; }
55 | set
56 | {
57 | if(value == null)
58 | throw new ArgumentNullException();
59 | _proxyJudgeService = value;
60 | }
61 | }
62 | ///
63 | /// Downloaded page has to contain this tring.
64 | /// It helps to check if returned page is the page which we watned to receive.
65 | /// Proxy sometimes are returing some other pages.
66 | /// Default value string.Empty
67 | ///
68 | public string RequiredString
69 | {
70 | get { return _requiredString; }
71 | set { _requiredString = value ?? string.Empty; }
72 | }
73 | ///
74 | /// Set encoding for request.
75 | ///
76 | /// Default value: UTF-8
77 | public Encoding Encoding { get; set; } = Encoding.UTF8;
78 | ///
79 | /// Set this to true if you want to use only anonymous proxies.
80 | ///
81 | /// Default value: false
82 | public bool AnonymousProxyOnly { get; } = false;
83 | ///
84 | /// Timeout for request.
85 | ///
86 | /// Has to be greater than 5 milliseconds.
87 | /// Default value: 10 seconds.
88 | public TimeSpan Timeout
89 | {
90 | get
91 | {
92 | return _timeout;
93 | }
94 | set
95 | {
96 | if (value.TotalMilliseconds < 5)
97 | throw new ArgumentOutOfRangeException("Timeout has to be greater or equal than 5 milliseconds.");
98 | _timeout = value;
99 | }
100 | }
101 | ///
102 | /// Set how many attempts can be made to execute request on one proxy.
103 | /// Default value is default value is equal 4
104 | ///
105 | public int NumberOfAttempts
106 | {
107 | get { return _numberOfAttempts; }
108 | set
109 | {
110 | if (value < 1)
111 | throw new ArgumentOutOfRangeException("Value has to be greater than one.");
112 |
113 | _numberOfAttempts = value;
114 | _proxyJudgeService.NumberOfAttempts = _numberOfAttempts;
115 | }
116 | }
117 |
118 | /// Proxy list
119 | /// Set true if you want to filter proxy list and use only anonymous only
120 | /// Proxy judge service is used to determine proxy anonymity level
121 | ///
122 | public ProxyManager(IEnumerable proxies, bool anonymousOnly, ProxyJudgeService proxyJudgeService) :
123 | this(ParseProxies(proxies), anonymousOnly, proxyJudgeService)
124 | {
125 | }
126 |
127 | /// Proxy list
128 | /// Set true if you want to filter proxy list and use only anonymous only
129 | /// Proxy judge service is used to determine proxy anonymity level
130 | ///
131 | public ProxyManager(IEnumerable proxies, bool anonymousOnly, ProxyJudgeService proxyJudgeService)
132 | {
133 | if (proxies == null || proxyJudgeService == null)
134 | throw new ArgumentNullException();
135 | foreach (Proxy proxy in proxies)
136 | {
137 | try
138 | {
139 | _proxies.Add(proxy);
140 | }
141 | catch (UriFormatException)
142 | {
143 | // parsing exception
144 | }
145 | }
146 | AnonymousProxyOnly = anonymousOnly;
147 | _proxyJudgeService = proxyJudgeService;
148 | _numberOfAttemptsPerRequest = _proxies.Count + 1;
149 | _proxyJudgeService.NumberOfAttempts = _numberOfAttempts;
150 | }
151 | public ProxyManager(string file) : this(File.ReadLines(file), false, new ProxyJudgeService()) { }
152 | public ProxyManager(string file, bool anonymousOnly) : this(File.ReadLines(file), anonymousOnly, new ProxyJudgeService()) { }
153 | public ProxyManager(string file, bool anonymousOnly, ProxyJudgeService service) : this(File.ReadLines(file), anonymousOnly, service) { }
154 |
155 | ///
156 | /// Downloads url using GET.
157 | ///
158 | /// Url of webpage
159 | /// Cookies for request. Left null if you don't want to use cookies
160 | /// Specify custom headers for this request
161 | ///
162 | /// Page has returned 404 not found
163 | public string GetPage(string url, string requiredString = null, CookieContainer cookies = null, NameValueCollection customHeaders = null)
164 | {
165 | return PostPage(url, null, requiredString, cookies, customHeaders);
166 | }
167 |
168 | ///
169 | /// Downloads url using POST.
170 | ///
171 | /// Url of webpage
172 | /// Post values
173 | /// Cookies for request. Left null if you don't want to use cookies
174 | /// Specify custom headers for this request
175 | /// Page has returned 404 not found
176 | ///
177 | public string PostPage(string url, NameValueCollection data, string requiredString = null, CookieContainer cookies = null, NameValueCollection customHeaders = null)
178 | {
179 | if (requiredString == null)
180 | requiredString = RequiredString;
181 |
182 | string page = null;
183 | int limit = 0;
184 |
185 | do
186 | {
187 | Proxy proxy = GetAvalibleProxy();
188 |
189 | try
190 | {
191 | if (AnonymousProxyOnly)
192 | {
193 | if (AnonymousProxyOnly && !proxy.IsAnonymous(ProxyJudgeService))
194 | {
195 | continue;
196 | }
197 | }
198 |
199 | var bytes = DownloadBytes(url, data, proxy, cookies, customHeaders);
200 |
201 | if (bytes != null)
202 | {
203 | page = Encoding.GetString(bytes);
204 | if (!page.Contains(requiredString))
205 | {
206 | proxy.IsOnline = false;
207 | }
208 | else
209 | {
210 | return page;
211 | }
212 | }
213 | }
214 | finally
215 | {
216 | limit++;
217 | proxy.IsBusy = false;
218 | }
219 |
220 | limit++;
221 | } while (limit < _numberOfAttemptsPerRequest);
222 |
223 | throw new AllProxiesBannedException();
224 | }
225 |
226 | ///
227 | /// Downloads url using POST.
228 | ///
229 | /// Url of webpage
230 | /// Post values
231 | /// Cookies for request. Left null if you don't want to use cookies
232 | /// Specify custom headers for this request
233 | /// Page has returned 404 not found
234 | ///
235 | public byte[] DownloadBytes(string url, NameValueCollection data, CookieContainer cookies = null, NameValueCollection customHeaders = null)
236 | {
237 | int limit = 0;
238 |
239 | do
240 | {
241 | Proxy proxy = GetAvalibleProxy();
242 |
243 | try
244 | {
245 | if (AnonymousProxyOnly && !proxy.IsAnonymous(ProxyJudgeService))
246 | {
247 | continue;
248 | }
249 |
250 | byte[] result = DownloadBytes(url, data, proxy, cookies, customHeaders);
251 |
252 | if (result != null)
253 | {
254 | return result;
255 | }
256 | }
257 | finally
258 | {
259 | limit++;
260 | proxy.IsBusy = false;
261 | }
262 |
263 | limit++;
264 | } while (limit < _numberOfAttemptsPerRequest);
265 |
266 | throw new AllProxiesBannedException();
267 | }
268 |
269 | ///
270 | /// Downloads url using GET.
271 | ///
272 | ///
273 | ///
274 | /// Specify custom headers for this request
275 | ///
276 | /// Page has returned 404 not found
277 | public byte[] DownloadBytes(string url, CookieContainer cookies = null, NameValueCollection customHeaders = null)
278 | {
279 | return DownloadBytes(url, null, cookies, customHeaders);
280 | }
281 | ///
282 | /// Returns first free (but busy) and working proxy.
283 | ///
284 | ///
285 | /// All proxies are banned. You can't make request.
286 | public Proxy GetAvalibleProxy()
287 | {
288 | lock (_proxies)
289 | {
290 | Proxy selectedProxy = null;
291 |
292 | do
293 | {
294 | if (_proxies.Count(t => t.IsOnline) == 0)
295 | throw new AllProxiesBannedException();
296 |
297 | selectedProxy = _proxies.FirstOrDefault(t => t.IsOnline && !t.IsBusy);
298 | if (selectedProxy == null)
299 | {
300 | Thread.Sleep(35);
301 | }
302 |
303 | } while (selectedProxy == null);
304 |
305 | selectedProxy.IsBusy = true;
306 | return selectedProxy;
307 | }
308 | }
309 | ///
310 | /// Sets all proxies IsOnline property to true.
311 | ///
312 | public void SetAllProxyAsOnline()
313 | {
314 | lock (_proxies)
315 | {
316 | _proxies.ForEach(t => t.IsOnline = true);
317 | }
318 | }
319 |
320 | ///
321 | /// Return all proxies
322 | ///
323 | ///
324 | public List GetAllProxies()
325 | {
326 | lock(_proxies)
327 | return CloneProxyList(_proxies);
328 | }
329 |
330 |
331 | private List CloneProxyList(List proxyInput)
332 | {
333 | List proxies = new List(proxyInput.Count);
334 | proxies.AddRange(proxyInput.Select(proxy => (Proxy) proxy.Clone()));
335 | return proxies;
336 | }
337 |
338 | private byte[] DownloadBytes(string url, NameValueCollection data, Proxy proxy, CookieContainer cookies = null, NameValueCollection customHeaders = null)
339 | {
340 | HttpClient client = CreateHttpClient(customHeaders);
341 | client.Proxy = proxy;
342 | if (cookies != null) client.Cookies = cookies;
343 |
344 | try
345 | {
346 | return client.DownloadBytes(url, data);
347 | }
348 | catch (WebException e)
349 | {
350 | if (e.Response != null && (e.Response as HttpWebResponse).StatusCode == HttpStatusCode.NotFound)
351 | throw new WebPageNotFoundException();
352 | proxy.IsOnline = false;
353 | }
354 |
355 | return null;
356 | }
357 | private HttpClient CreateHttpClient(NameValueCollection customHeaders = null)
358 | {
359 | HttpClient client = new HttpClient(Encoding)
360 | {
361 | Accept = Accept,
362 | AcceptEncoding = AcceptEncoding,
363 | AcceptLanguage = AcceptLanguage,
364 | NumberOfAttempts = NumberOfAttempts,
365 | UserAgent = UserAgent,
366 | Referer = Referer,
367 | Timeout = Timeout
368 | };
369 |
370 | if (customHeaders != null)
371 | {
372 | if (customHeaders.AllKeys.Contains("Accept"))
373 | {
374 | client.Accept = customHeaders["Accept"];
375 | customHeaders.Remove("Accept");
376 | }
377 | if (customHeaders.AllKeys.Contains("Accept-Encoding"))
378 | {
379 | client.AcceptEncoding = customHeaders["Accept-Encoding"];
380 | customHeaders.Remove("Accept-Encoding");
381 | }
382 | if (customHeaders.AllKeys.Contains("Accept-Language"))
383 | {
384 | client.AcceptLanguage = customHeaders["Accept-Language"];
385 | customHeaders.Remove("Accept-Language");
386 | }
387 | if (customHeaders.AllKeys.Contains("User-Agent"))
388 | {
389 | client.UserAgent = customHeaders["User-Agente"];
390 | customHeaders.Remove("User-Agent");
391 | }
392 | if (customHeaders.AllKeys.Contains("Referer"))
393 | {
394 | client.Referer = customHeaders["Referer"];
395 | customHeaders.Remove("Referer");
396 | }
397 |
398 | client.CustomHeaders = customHeaders;
399 | }
400 |
401 | return client;
402 | }
403 | private static IEnumerable ParseProxies(IEnumerable proxies)
404 | {
405 | IEnumerable proxyParsed = proxies.Select(t =>
406 | {
407 | try
408 | {
409 | Proxy proxy = new Proxy(t);
410 | return proxy;
411 | }
412 | catch (UriFormatException)
413 | {
414 | // parsing exception
415 | return null;
416 | }
417 | }).Where(t => t != null);
418 | return proxyParsed;
419 | }
420 | }
421 |
422 | public class AllProxiesBannedException : Exception
423 | {
424 | }
425 | public class WebPageNotFoundException : Exception
426 | {
427 | }
428 | }
429 |
--------------------------------------------------------------------------------
/BetterHttpClient/Socks/NeverEndingStream.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.IO;
4 | using System.Linq;
5 | using System.Text;
6 | using System.Threading.Tasks;
7 |
8 | namespace BetterHttpClient.Socks
9 | {
10 | public class NeverEndingStream : MemoryStream
11 | {
12 | public override void Close()
13 | {
14 | // Ignore
15 | }
16 |
17 | public void ForceClose()
18 | {
19 | base.Close();
20 | }
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/BetterHttpClient/Socks/SocksHttpWebRequest.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Collections.Specialized;
4 | using System.IO;
5 | using System.Linq;
6 | using System.Net;
7 | using System.Net.Security;
8 | using System.Net.Sockets;
9 | using System.Text;
10 | using System.Threading.Tasks;
11 | using BetterHttpClient.Socks.Extensions;
12 |
13 | namespace BetterHttpClient.Socks
14 | {
15 | internal class SocksHttpWebRequest : WebRequest
16 | {
17 | private static readonly StringCollection validHttpVerbs = new StringCollection { "GET", "HEAD", "POST", "PUT", "DELETE", "TRACE", "OPTIONS" };
18 | private WebHeaderCollection _HttpRequestHeaders;
19 | private string _method;
20 | private byte[] _requestContentBuffer;
21 | private NeverEndingStream _requestContentStream;
22 | private SocksHttpWebResponse _response;
23 |
24 | private SocksHttpWebRequest(Uri requestUri)
25 | {
26 | RequestUri = requestUri;
27 | }
28 |
29 | public override int Timeout { get; set; }
30 |
31 | public string UserAgent
32 | {
33 | get { return _HttpRequestHeaders["User-Agent"]; }
34 | set { SetSpecialHeaders("User-Agent", value ?? string.Empty); }
35 | }
36 |
37 | public string Referer
38 | {
39 | get { return _HttpRequestHeaders["Referer"]; }
40 | set { SetSpecialHeaders("Referer", value ?? string.Empty); }
41 | }
42 |
43 | public string Accept
44 | {
45 | get { return _HttpRequestHeaders["Accept"]; }
46 | set { SetSpecialHeaders("Accept", value ?? string.Empty); }
47 | }
48 |
49 | public DecompressionMethods AutomaticDecompression
50 | {
51 | get
52 | {
53 | var result = DecompressionMethods.None;
54 | string encoding = _HttpRequestHeaders["Accept-Encoding"] ?? string.Empty;
55 | foreach (string value in encoding.Split(','))
56 | {
57 | switch (value.Trim())
58 | {
59 | case "gzip":
60 | result |= DecompressionMethods.GZip;
61 | break;
62 |
63 | case "deflate":
64 | result |= DecompressionMethods.Deflate;
65 | break;
66 | }
67 | }
68 |
69 | return result;
70 | }
71 | set
72 | {
73 | string result = string.Empty;
74 | if ((value & DecompressionMethods.GZip) != 0)
75 | result = "gzip";
76 | if ((value & DecompressionMethods.Deflate) != 0)
77 | {
78 | if (!string.IsNullOrEmpty(result))
79 | result += ", ";
80 | result += "deflate";
81 | }
82 |
83 | SetSpecialHeaders("Accept-Encoding", result);
84 | }
85 | }
86 |
87 | public override Uri RequestUri { get; }
88 |
89 | public override IWebProxy Proxy { get; set; }
90 |
91 | public override WebHeaderCollection Headers
92 | {
93 | get { return _HttpRequestHeaders ?? (_HttpRequestHeaders = new WebHeaderCollection()); }
94 | set
95 | {
96 | if (RequestSubmitted)
97 | {
98 | throw new InvalidOperationException("This operation cannot be performed after the request has been submitted.");
99 | }
100 | _HttpRequestHeaders = value;
101 | }
102 | }
103 |
104 | public bool RequestSubmitted { get; private set; }
105 |
106 | public override string Method
107 | {
108 | get { return _method ?? "GET"; }
109 | set
110 | {
111 | if (validHttpVerbs.Contains(value))
112 | {
113 | _method = value;
114 | }
115 | else
116 | {
117 | throw new ArgumentOutOfRangeException(nameof(value), $"'{value}' is not a known HTTP verb.");
118 | }
119 | }
120 | }
121 |
122 | public override long ContentLength { get; set; }
123 |
124 | public override string ContentType { get; set; }
125 | public bool AllowAutoRedirect { get; set; } = true;
126 |
127 | public override WebResponse GetResponse()
128 | {
129 | if (Proxy == null)
130 | {
131 | throw new InvalidOperationException("Proxy property cannot be null.");
132 | }
133 | if (string.IsNullOrEmpty(Method))
134 | {
135 | throw new InvalidOperationException("Method has not been set.");
136 | }
137 |
138 | if (RequestSubmitted)
139 | {
140 | return _response;
141 | }
142 | _response = InternalGetResponse();
143 | RequestSubmitted = true;
144 | return _response;
145 | }
146 |
147 | public override Stream GetRequestStream()
148 | {
149 | if (RequestSubmitted)
150 | {
151 | throw new InvalidOperationException("This operation cannot be performed after the request has been submitted.");
152 | }
153 |
154 | if (_requestContentBuffer == null)
155 | {
156 | if (ContentLength == 0)
157 | {
158 | _requestContentStream = new NeverEndingStream();
159 | return _requestContentStream;
160 | }
161 |
162 | _requestContentBuffer = new byte[ContentLength];
163 | }
164 | else if (ContentLength == default(long))
165 | {
166 | _requestContentBuffer = new byte[int.MaxValue];
167 | }
168 | else if (_requestContentBuffer.Length != ContentLength)
169 | {
170 | Array.Resize(ref _requestContentBuffer, (int)ContentLength);
171 | }
172 | return new MemoryStream(_requestContentBuffer);
173 | }
174 |
175 | public override IAsyncResult BeginGetResponse(AsyncCallback callback, object state)
176 | {
177 | if (Proxy == null)
178 | {
179 | throw new InvalidOperationException("Proxy property cannot be null.");
180 | }
181 | if (string.IsNullOrEmpty(Method))
182 | {
183 | throw new InvalidOperationException("Method has not been set.");
184 | }
185 |
186 | var task = Task.Run(() =>
187 | {
188 | if (RequestSubmitted)
189 | {
190 | return _response;
191 | }
192 | _response = InternalGetResponse();
193 | RequestSubmitted = true;
194 | return _response;
195 | });
196 |
197 | return task.AsApm(callback, state);
198 | }
199 |
200 | public override WebResponse EndGetResponse(IAsyncResult asyncResult)
201 | {
202 | var task = asyncResult as Task;
203 |
204 | try
205 | {
206 | return task.Result;
207 | }
208 | catch (AggregateException ex)
209 | {
210 | if (ex.InnerException is IOException || ex.InnerException is System.Net.Sockets.SocketException)
211 | throw new WebException("Proxy error " + ex.InnerException.Message, ex.InnerException, WebExceptionStatus.ConnectFailure,
212 | SocksHttpWebResponse.CreateErrorResponse(HttpStatusCode.GatewayTimeout));
213 | throw ex.InnerException;
214 | }
215 | }
216 |
217 | public new static WebRequest Create(string requestUri)
218 | {
219 | return new SocksHttpWebRequest(new Uri(requestUri));
220 | }
221 |
222 | public new static WebRequest Create(Uri requestUri)
223 | {
224 | return new SocksHttpWebRequest(requestUri);
225 | }
226 |
227 | private void SetSpecialHeaders(string headerName, string value)
228 | {
229 | _HttpRequestHeaders.Remove(headerName);
230 | if (value.Length != 0)
231 | {
232 | _HttpRequestHeaders.Add(headerName, value);
233 | }
234 | }
235 |
236 | private string BuildHttpRequestMessage(Uri requestUri)
237 | {
238 | if (RequestSubmitted)
239 | {
240 | throw new InvalidOperationException("This operation cannot be performed after the request has been submitted.");
241 | }
242 |
243 | // See if we have a stream instead of byte array
244 | if (_requestContentBuffer == null && _requestContentStream != null)
245 | {
246 | _requestContentBuffer = _requestContentStream.ToArray();
247 | _requestContentStream.ForceClose();
248 | _requestContentStream.Dispose();
249 | _requestContentStream = null;
250 | ContentLength = _requestContentBuffer.Length;
251 | }
252 |
253 | var message = new StringBuilder();
254 | message.AppendFormat("{0} {1} HTTP/1.1\r\nHost: {2}\r\n", Method, requestUri, requestUri.Host);
255 |
256 | Headers.Set(HttpRequestHeader.Connection, "close");
257 |
258 | // add the headers
259 | foreach (var key in Headers.Keys)
260 | {
261 | string value = Headers[key.ToString()];
262 | if (!string.IsNullOrEmpty(value))
263 | message.AppendFormat("{0}: {1}\r\n", key, value);
264 | }
265 |
266 | if (!string.IsNullOrEmpty(ContentType))
267 | {
268 | message.AppendFormat("Content-Type: {0}\r\n", ContentType);
269 | }
270 | if (ContentLength > 0)
271 | {
272 | message.AppendFormat("Content-Length: {0}\r\n", ContentLength);
273 | }
274 |
275 | // add a blank line to indicate the end of the headers
276 | message.Append("\r\n");
277 |
278 | // add content
279 | if (_requestContentBuffer != null && _requestContentBuffer.Length > 0)
280 | {
281 | using (var stream = new MemoryStream(_requestContentBuffer, false))
282 | {
283 | using (var reader = new StreamReader(stream))
284 | {
285 | message.Append(reader.ReadToEnd());
286 | }
287 | }
288 | }
289 | else if (_requestContentStream != null)
290 | {
291 | _requestContentStream.Position = 0;
292 |
293 | using (var reader = new StreamReader(_requestContentStream))
294 | {
295 | message.Append(reader.ReadToEnd());
296 | }
297 |
298 |
299 | _requestContentStream.ForceClose();
300 | _requestContentStream.Dispose();
301 | }
302 |
303 | return message.ToString();
304 | }
305 |
306 | private SocksHttpWebResponse InternalGetResponse()
307 | {
308 | Uri requestUri = RequestUri;
309 |
310 | int redirects = 0;
311 | const int maxAutoredirectCount = 10;
312 | while (redirects++ < maxAutoredirectCount)
313 | {
314 | // Loop while redirecting
315 |
316 | var proxyUri = Proxy.GetProxy(requestUri);
317 | var ipAddress = GetProxyIpAddress(proxyUri);
318 | var response = new List();
319 |
320 | using (var client = new TcpClient(ipAddress.ToString(), proxyUri.Port))
321 | {
322 | int timeout = Timeout;
323 | if (timeout == 0)
324 | timeout = 30 * 1000;
325 | client.ReceiveTimeout = timeout;
326 | client.SendTimeout = timeout;
327 | var networkStream = client.GetStream();
328 | // auth
329 | var buf = new byte[300];
330 | buf[0] = 0x05; // Version
331 | buf[1] = 0x01; // NMETHODS
332 | buf[2] = 0x00; // No auth-method
333 | networkStream.Write(buf, 0, 3);
334 |
335 | networkStream.Read(buf, 0, 2);
336 | if (buf[0] != 0x05)
337 | {
338 | throw new IOException("Invalid Socks Version");
339 | }
340 | if (buf[1] == 0xff)
341 | {
342 | throw new IOException("Socks Server does not support no-auth");
343 | }
344 | if (buf[1] != 0x00)
345 | {
346 | throw new Exception("Socks Server did choose bogus auth");
347 | }
348 |
349 | // connect
350 | var destIP = Dns.GetHostEntry(requestUri.DnsSafeHost).AddressList[0];
351 | var index = 0;
352 | buf[index++] = 0x05; // version 5 .
353 | buf[index++] = 0x01; // command = connect.
354 | buf[index++] = 0x00; // Reserve = must be 0x00
355 |
356 | buf[index++] = 0x01; // Address is full-qualified domain name.
357 | var rawBytes = destIP.GetAddressBytes();
358 | rawBytes.CopyTo(buf, index);
359 | index += (ushort)rawBytes.Length;
360 |
361 | var portBytes = BitConverter.GetBytes(Uri.UriSchemeHttps == requestUri.Scheme ? 443 : 80);
362 | for (var i = portBytes.Length - 3; i >= 0; i--)
363 | buf[index++] = portBytes[i];
364 |
365 |
366 | networkStream.Write(buf, 0, index);
367 |
368 | networkStream.Read(buf, 0, 4);
369 | if (buf[0] != 0x05)
370 | {
371 | throw new IOException("Invalid Socks Version");
372 | }
373 | if (buf[1] != 0x00)
374 | {
375 | throw new IOException($"Socks Error {buf[1]:X}");
376 | }
377 |
378 | var rdest = string.Empty;
379 | switch (buf[3])
380 | {
381 | case 0x01: // IPv4
382 | networkStream.Read(buf, 0, 4);
383 | var v4 = BitConverter.ToUInt32(buf, 0);
384 | rdest = new IPAddress(v4).ToString();
385 | break;
386 | case 0x03: // Domain name
387 | networkStream.Read(buf, 0, 1);
388 | if (buf[0] == 0xff)
389 | {
390 | throw new IOException("Invalid Domain Name");
391 | }
392 | networkStream.Read(buf, 1, buf[0]);
393 | rdest = Encoding.ASCII.GetString(buf, 1, buf[0]);
394 | break;
395 | case 0x04: // IPv6
396 | var octets = new byte[16];
397 | networkStream.Read(octets, 0, 16);
398 | rdest = new IPAddress(octets).ToString();
399 | break;
400 | default:
401 | throw new IOException("Invalid Address type");
402 | }
403 | networkStream.Read(buf, 0, 2);
404 | var rport = (ushort)IPAddress.NetworkToHostOrder((short)BitConverter.ToUInt16(buf, 0));
405 |
406 | Stream readStream = null;
407 | if (Uri.UriSchemeHttps == requestUri.Scheme)
408 | {
409 | var ssl = new SslStream(networkStream);
410 | ssl.AuthenticateAsClient(requestUri.DnsSafeHost);
411 | readStream = ssl;
412 | }
413 | else
414 | {
415 | readStream = networkStream;
416 | }
417 |
418 | string requestString = BuildHttpRequestMessage(requestUri);
419 |
420 | var request = Encoding.ASCII.GetBytes(requestString);
421 | readStream.Write(request, 0, request.Length);
422 | readStream.Flush();
423 |
424 | var buffer = new byte[client.ReceiveBufferSize];
425 |
426 | var readlen = 0;
427 | do
428 | {
429 | readlen = readStream.Read(buffer, 0, buffer.Length);
430 | response.AddRange(buffer.Take(readlen));
431 | } while (readlen != 0);
432 |
433 | readStream.Close();
434 | }
435 |
436 | var webResponse = new SocksHttpWebResponse(requestUri, response.ToArray());
437 |
438 | if (webResponse.StatusCode == HttpStatusCode.Moved || webResponse.StatusCode == HttpStatusCode.MovedPermanently)
439 | {
440 | string redirectUrl = webResponse.Headers["Location"];
441 | if (string.IsNullOrEmpty(redirectUrl))
442 | throw new WebException("Missing location for redirect");
443 |
444 | requestUri = new Uri(requestUri, redirectUrl);
445 | if (AllowAutoRedirect)
446 | {
447 | continue;
448 | }
449 | return webResponse;
450 | }
451 |
452 | if ((int)webResponse.StatusCode < 200 || (int)webResponse.StatusCode > 299)
453 | throw new WebException(webResponse.StatusDescription, null, WebExceptionStatus.UnknownError, webResponse);
454 |
455 | return webResponse;
456 | }
457 |
458 | throw new WebException("Too many redirects", null, WebExceptionStatus.ProtocolError, SocksHttpWebResponse.CreateErrorResponse(HttpStatusCode.BadRequest));
459 | }
460 |
461 | private static IPAddress GetProxyIpAddress(Uri proxyUri)
462 | {
463 | IPAddress ipAddress;
464 | if (!IPAddress.TryParse(proxyUri.Host, out ipAddress))
465 | {
466 | try
467 | {
468 | return Dns.GetHostEntry(proxyUri.Host).AddressList[0];
469 | }
470 | catch (Exception e)
471 | {
472 | throw new InvalidOperationException($"Unable to resolve proxy hostname '{proxyUri.Host}' to a valid IP address.", e);
473 | }
474 | }
475 | return ipAddress;
476 | }
477 | }
478 | }
--------------------------------------------------------------------------------
/BetterHttpClient/Socks/SocksHttpWebResponse.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.IO;
3 | using System.IO.Compression;
4 | using System.Net;
5 | using System.Text;
6 |
7 | // Credit: https://github.com/Yozer/BetterHttpClient
8 |
9 | namespace BetterHttpClient.Socks
10 | {
11 | public class SocksHttpWebResponse : WebResponse
12 | {
13 | #region Member Variables
14 |
15 | private WebHeaderCollection _httpResponseHeaders;
16 | private Uri responseUri;
17 |
18 | #endregion
19 |
20 | #region Constructors
21 |
22 | public SocksHttpWebResponse(Uri responseUri, byte[] httpResponseMessage)
23 | {
24 | this.responseUri = responseUri;
25 | SetHeadersAndResponseContent(httpResponseMessage);
26 | }
27 |
28 | private SocksHttpWebResponse()
29 | {
30 | }
31 |
32 | public static SocksHttpWebResponse CreateErrorResponse(HttpStatusCode statusCode)
33 | {
34 | var response = new SocksHttpWebResponse
35 | {
36 | StatusCode = statusCode
37 | };
38 |
39 | return response;
40 | }
41 |
42 | #endregion
43 |
44 | #region Methods
45 |
46 | private void SetHeadersAndResponseContent(byte[] responseMessage)
47 | {
48 | if (responseMessage == null || responseMessage.Length < 4)
49 | return;
50 |
51 | var headers = string.Empty;
52 | // get headers part;
53 | var headerEnd = 0;
54 | for (; headerEnd < responseMessage.Length - 3; headerEnd++)
55 | {
56 | if (responseMessage[headerEnd] == 13 && responseMessage[headerEnd + 1] == 10 && responseMessage[headerEnd + 2] == 13 && responseMessage[headerEnd + 3] == 10)
57 | {
58 | headers = Encoding.ASCII.GetString(responseMessage, 0, headerEnd);
59 | break;
60 | }
61 | }
62 | if (string.IsNullOrEmpty(headers))
63 | return;
64 |
65 | var headerValues = headers.Split(new[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries);
66 | // ignore the first line in the header since it is the HTTP response code
67 | for (var i = 1; i < headerValues.Length; i++)
68 | {
69 | var headerEntry = headerValues[i].Split(':');
70 | Headers.Add(headerEntry[0], headerEntry[1]);
71 | }
72 |
73 | var statusRegex = new System.Text.RegularExpressions.Regex(@"^HTTP/\d+\.\d+ (\d+)(.*)$");
74 | var matches = statusRegex.Match(headerValues[0]);
75 | if (matches.Groups.Count < 3)
76 | throw new InvalidDataException();
77 | StatusCode = (HttpStatusCode)int.Parse(matches.Groups[1].Value);
78 | StatusDescription = matches.Groups[2].Value;
79 |
80 | ResponseContent = new byte[responseMessage.Length - headerEnd - 4];
81 | Array.Copy(responseMessage, headerEnd + 4, ResponseContent, 0, ResponseContent.Length);
82 |
83 | string transferEncoding = Headers["Transfer-Encoding"];
84 |
85 | switch (transferEncoding)
86 | {
87 | case "chunked":
88 | DechunkContent();
89 | break;
90 | }
91 |
92 | string contentEncoding = Headers["Content-Encoding"];
93 |
94 | switch (contentEncoding)
95 | {
96 | case "gzip":
97 | ResponseContent = Decompress(ResponseContent) ?? ResponseContent;
98 | break;
99 | }
100 | }
101 |
102 | private void DechunkContent()
103 | {
104 | using (var output = new MemoryStream())
105 | using (var ms = new MemoryStream(ResponseContent))
106 | {
107 | while (true)
108 | {
109 | var infoLine = new StringBuilder();
110 | while (true)
111 | {
112 | int b = ms.ReadByte();
113 |
114 | if (b == -1)
115 | // End of stream
116 | break;
117 |
118 | if (b == 13)
119 | // Ignore
120 | continue;
121 |
122 | if (b == 10)
123 | {
124 | if (infoLine.Length == 0)
125 | // Trim
126 | continue;
127 | else
128 | break;
129 | }
130 |
131 | infoLine.Append((char)b);
132 | }
133 | if (infoLine.Length == 0)
134 | break;
135 |
136 | int chunkLength = int.Parse(infoLine.ToString(), System.Globalization.NumberStyles.HexNumber);
137 | if (chunkLength == 0)
138 | break;
139 |
140 | var buffer = new byte[chunkLength];
141 | ms.Read(buffer, 0, chunkLength);
142 | output.Write(buffer, 0, buffer.Length);
143 | }
144 |
145 | ResponseContent = output.ToArray();
146 | }
147 | }
148 |
149 | #endregion
150 |
151 | #region WebResponse Members
152 |
153 | public HttpStatusCode StatusCode { get; private set; }
154 |
155 | public string StatusDescription { get; private set; }
156 |
157 | public override Uri ResponseUri
158 | {
159 | get
160 | {
161 | return this.responseUri;
162 | }
163 | }
164 |
165 | private static readonly byte[] GZipHeaderBytes = { 0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 4, 0 };
166 | private static readonly byte[] GZipLevel10HeaderBytes = { 0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 2, 0 };
167 |
168 | public static bool IsPossiblyGZippedBytes(byte[] a)
169 | {
170 | var yes = a.Length > 10;
171 |
172 | if (!yes)
173 | {
174 | return false;
175 | }
176 |
177 | var fail = false;
178 | for (var i = 0; i < 10; i++)
179 | {
180 | if (a[i] != GZipHeaderBytes[i])
181 | {
182 | fail = true;
183 | break;
184 | }
185 | }
186 |
187 | if (!fail)
188 | return true;
189 |
190 | for (var i = 0; i < 10; i++)
191 | {
192 | if (a[i] != GZipLevel10HeaderBytes[i])
193 | {
194 | return false;
195 | }
196 | }
197 |
198 | return true;
199 | }
200 |
201 | private byte[] Decompress(byte[] gzip)
202 | {
203 | try
204 | {
205 | using (var stream = new GZipStream(new MemoryStream(gzip), CompressionMode.Decompress))
206 | {
207 | const int size = 4096;
208 | var buffer = new byte[size];
209 | using (var memory = new MemoryStream())
210 | {
211 | var count = 0;
212 | do
213 | {
214 | count = stream.Read(buffer, 0, size);
215 | if (count > 0)
216 | {
217 | memory.Write(buffer, 0, count);
218 | }
219 | } while (count > 0);
220 | return memory.ToArray();
221 | }
222 | }
223 | }
224 | catch (Exception)
225 | {
226 | return null;
227 | }
228 | }
229 |
230 | public override Stream GetResponseStream()
231 | {
232 | return ResponseContent.Length == 0 ? Stream.Null : new MemoryStream(ResponseContent);
233 | }
234 |
235 | public override void Close()
236 | {
237 | /* the base implementation throws an exception */
238 | }
239 |
240 | public override WebHeaderCollection Headers
241 | {
242 | get
243 | {
244 | if (_httpResponseHeaders == null)
245 | {
246 | _httpResponseHeaders = new WebHeaderCollection();
247 | }
248 | return _httpResponseHeaders;
249 | }
250 | }
251 |
252 | public override long ContentLength
253 | {
254 | get { return ResponseContent.Length; }
255 | set { throw new NotSupportedException(); }
256 | }
257 |
258 | #endregion
259 |
260 | #region Properties
261 |
262 | private byte[] ResponseContent { get; set; }
263 |
264 | public Encoding CorrectEncoding { get; set; }
265 |
266 | #endregion
267 | }
268 | }
--------------------------------------------------------------------------------
/BetterHttpClient/Socks/TaskExtensions.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Generic;
3 | using System.Linq;
4 | using System.Text;
5 | using System.Threading.Tasks;
6 |
7 | namespace BetterHttpClient.Socks.Extensions
8 | {
9 | public static class TaskExtensions
10 | {
11 | public static IAsyncResult AsApm(this Task task,
12 | AsyncCallback callback,
13 | object state)
14 | {
15 | if (task == null)
16 | throw new ArgumentNullException("task");
17 |
18 | var tcs = new TaskCompletionSource(state);
19 | task.ContinueWith(t =>
20 | {
21 | if (t.IsFaulted)
22 | tcs.TrySetException(t.Exception.InnerExceptions);
23 | else if (t.IsCanceled)
24 | tcs.TrySetCanceled();
25 | else
26 | tcs.TrySetResult(t.Result);
27 |
28 | if (callback != null)
29 | callback(tcs.Task);
30 | }, TaskScheduler.Default);
31 | return tcs.Task;
32 | }
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2018 Dominik Baran
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # BetterHttpClient
2 |
3 | BetterHttpClient will help you making http and https requests with full cookie support.
4 |
5 | It supports three types of proxy servers:
6 | - HTTP
7 | - HTTPS
8 | - Socks5
9 |
10 | # Http Client
11 | - Http GET request
12 | ```cs
13 | HttpClient client = new HttpClient
14 | {
15 | UserAgent = "My custom user-agent",
16 | Referer = "My referer",
17 | Encoding = Encoding.UTF8,
18 | AcceptEncoding = "gzip, deflate" // setup transfer compression algorithm
19 | };
20 | string page = client.Get("http://www.google.com");
21 | ```
22 |
23 | - Http POST request
24 | ```cs
25 | HttpClient client = new HttpClient(); // if no headers are specified defaults will be used
26 | string page = client.Post("https://httpbin.org/post", new NameValueCollection
27 | {
28 | {"custname", "Some post data"},
29 | });
30 | ```
31 |
32 | - Https GET request using proxy
33 |
34 | HttpClient will detect proxy type (http/s, socks5) if you don't provide such info.
35 | ```cs
36 | string proxyAddress = "218.200.66.196:8080";
37 | HttpClient client = new HttpClient(new Proxy(proxyAddress))
38 | {
39 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0"
40 | };
41 |
42 | string page = client.Get("https://httpbin.org/get");
43 | ```
44 |
45 | # Proxy Manager
46 | This class will help you managing large amount of proxy servers.
47 |
48 | You can load proxy list from file and then execute GET or POST requests.
49 |
50 | If some proxy will be dead ProxyManager will mark them as offline and won't use in future requests.
51 |
52 | To perform http/s request it will use first Not Busy and Online proxy (and check for an anonymity level if anonymousOnly is true).
53 |
54 | Set anonymousOnly to true if you want to force ProxyManager to use only proxies that really hide your ip. For more info look at ProxyJudgeService class.
55 | Methods
56 | - GetPage
57 | - PostPage
58 | - DownloadBytes
59 |
60 | are thread safe.
61 |
62 | Example usage:
63 | ```cs
64 | ProxyManager proxyManager = new ProxyManager("proxy_list.txt", anonymousOnly: true)
65 | {
66 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0"
67 | PreserveCookies = false, // if true it will use the same cookies between distinct requests
68 | Timeout = TimeSpan.FromSeconds(10),
69 | NumberOfAttempts = 2 // max number of attempts that can be made per one request
70 | };
71 |
72 | // ProxyManager will choose first working proxy and use it to perform GET request
73 | string page = proxyManager.GetPage("http://example.com");
74 | ```
75 |
--------------------------------------------------------------------------------
/UnitTestBetterHttpClient/Properties/AssemblyInfo.cs:
--------------------------------------------------------------------------------
1 | using System.Reflection;
2 | using System.Runtime.CompilerServices;
3 | using System.Runtime.InteropServices;
4 |
5 | // General Information about an assembly is controlled through the following
6 | // set of attributes. Change these attribute values to modify the information
7 | // associated with an assembly.
8 | [assembly: AssemblyTitle("UnitTestBetterHttpClient")]
9 | [assembly: AssemblyDescription("")]
10 | [assembly: AssemblyConfiguration("")]
11 | [assembly: AssemblyCompany("")]
12 | [assembly: AssemblyProduct("UnitTestBetterHttpClient")]
13 | [assembly: AssemblyCopyright("Copyright © 2015")]
14 | [assembly: AssemblyTrademark("")]
15 | [assembly: AssemblyCulture("")]
16 |
17 | // Setting ComVisible to false makes the types in this assembly not visible
18 | // to COM components. If you need to access a type in this assembly from
19 | // COM, set the ComVisible attribute to true on that type.
20 | [assembly: ComVisible(false)]
21 |
22 | // The following GUID is for the ID of the typelib if this project is exposed to COM
23 | [assembly: Guid("59acc7d1-547c-4e1c-8de3-ba5351c1a00b")]
24 |
25 | // Version information for an assembly consists of the following four values:
26 | //
27 | // Major Version
28 | // Minor Version
29 | // Build Number
30 | // Revision
31 | //
32 | // You can specify all the values or you can default the Build and Revision Numbers
33 | // by using the '*' as shown below:
34 | // [assembly: AssemblyVersion("1.0.*")]
35 | [assembly: AssemblyVersion("1.0.0.0")]
36 | [assembly: AssemblyFileVersion("1.0.0.0")]
37 |
--------------------------------------------------------------------------------
/UnitTestBetterHttpClient/UnitTestBetterHttpClient.csproj:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Debug
5 | AnyCPU
6 | {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}
7 | Library
8 | Properties
9 | UnitTestBetterHttpClient
10 | UnitTestBetterHttpClient
11 | v4.5.2
12 | 512
13 | {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}
14 | 10.0
15 | $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)
16 | $(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages
17 | False
18 | UnitTest
19 |
20 |
21 | true
22 | full
23 | false
24 | bin\Debug\
25 | DEBUG;TRACE
26 | prompt
27 | 4
28 |
29 |
30 | pdbonly
31 | true
32 | bin\Release\
33 | TRACE
34 | prompt
35 | 4
36 |
37 |
38 |
39 | ..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll
40 | True
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 | {fbc5f420-3f80-4a20-a9ec-b6574e18a1d3}
64 | BetterHttpClient
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 | False
75 |
76 |
77 | False
78 |
79 |
80 | False
81 |
82 |
83 | False
84 |
85 |
86 |
87 |
88 |
89 |
90 |
97 |
--------------------------------------------------------------------------------
/UnitTestBetterHttpClient/UnitTestHttpClient.cs:
--------------------------------------------------------------------------------
1 | using System;
2 | using System.Collections.Specialized;
3 | using System.Text;
4 | using BetterHttpClient;
5 | using Microsoft.VisualStudio.TestTools.UnitTesting;
6 | using Newtonsoft.Json;
7 |
8 | namespace UnitTestBetterHttpClient
9 | {
10 | [TestClass]
11 | public class UnitTestHttpClient
12 | {
13 | private const string HttpsProxy = "210.245.25.229:3128";
14 | private const string Socksproxy = "200.239.9.161:10000";
15 | [TestMethod]
16 | public void TestGet()
17 | {
18 | HttpClient client = new HttpClient
19 | {
20 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0"
21 | };
22 |
23 | string page = client.Get("http://www.google.com");
24 | Assert.IsTrue(page.Contains("Google"));
25 | }
26 |
27 | [TestMethod]
28 | public void TestPost()
29 | {
30 | HttpClient client = new HttpClient
31 | {
32 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0"
33 | };
34 | var size = "medium";
35 | var topping = "cheese";
36 | var customerName = "TestName";
37 | var phone = "TestPhone";
38 | var email = "testmail@example.com";
39 | var delivery = "now";
40 | var comments = "fast";
41 |
42 | string page = client.Post("https://httpbin.org/post", new NameValueCollection
43 | {
44 | {"custname", customerName},
45 | {"custtel", phone},
46 | {"custemail", email},
47 | {"size", size},
48 | {"topping", topping},
49 | {"delivery", delivery},
50 | {"comments", comments}
51 | });
52 |
53 | Form root = JsonConvert.DeserializeObject(page).form;
54 |
55 | Assert.AreEqual(root.custname, customerName);
56 | Assert.AreEqual(root.custtel, phone);
57 | Assert.AreEqual(root.custemail, email);
58 | Assert.AreEqual(root.size, size);
59 | Assert.AreEqual(root.topping, topping);
60 | Assert.AreEqual(root.delivery, delivery);
61 | Assert.AreEqual(root.comments, comments);
62 |
63 | }
64 |
65 | [TestMethod]
66 | public void TestUserAgent()
67 | {
68 | HttpClient client = new HttpClient
69 | {
70 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0"
71 | };
72 |
73 | string page = client.Get("https://httpbin.org/user-agent");
74 | Assert.IsTrue(page.Contains(client.UserAgent));
75 | }
76 |
77 | [TestMethod]
78 | public void TestGzipDecodingAndReferer()
79 | {
80 | HttpClient client = new HttpClient
81 | {
82 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0",
83 | Referer = "https://httpbin.org/"
84 | };
85 |
86 | string page = client.Get("https://httpbin.org/gzip");
87 | Assert.IsTrue(page.Contains(client.UserAgent));
88 | // check for referer
89 | Assert.IsTrue(page.Contains("https://httpbin.org/"));
90 | }
91 |
92 | [TestMethod]
93 | public void TestHttpProxy()
94 | {
95 | HttpClient client = new HttpClient(new Proxy(HttpsProxy))
96 | {
97 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0",
98 | Encoding = Encoding.GetEncoding("iso-8859-2"),
99 | AcceptEncoding = "gzip"
100 | };
101 |
102 | string page = client.Get("http://darkwarez.pl");
103 | Assert.IsTrue(page.Contains("Polskie Forum Warez! Najnowsze linki"));
104 | }
105 | [TestMethod]
106 | public void TestHttpsProxy()
107 | {
108 | HttpClient client = new HttpClient(new Proxy(HttpsProxy))
109 | {
110 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0"
111 | };
112 |
113 | string page = client.Get("https://httpbin.org/get");
114 | Assert.IsTrue(page.Contains(client.UserAgent));
115 | }
116 | [TestMethod]
117 | public void TestSocksHttpProxy()
118 | {
119 | HttpClient client = new HttpClient(new Proxy(Socksproxy))
120 | {
121 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0",
122 | Encoding = Encoding.GetEncoding("iso-8859-2"),
123 | AcceptEncoding = "deflate"
124 | };
125 |
126 | string page = client.Get("http://darkwarez.pl/forum/");
127 | Assert.IsTrue(page.Contains("darkwarez.pl - Gry, Muzyka, Filmy, Download"));
128 | }
129 | [TestMethod]
130 | public void TestSocksHttpsProxyDeflateEncoding()
131 | {
132 | HttpClient client = new HttpClient(new Proxy(Socksproxy))
133 | {
134 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0",
135 | AcceptEncoding = "deflate"
136 | };
137 |
138 | string page = client.Get("https://httpbin.org/get");
139 | Assert.IsTrue(page.Contains(client.UserAgent));
140 | }
141 | [TestMethod]
142 | public void TestSocksHttpsProxyGzipEndcoding()
143 | {
144 | HttpClient client = new HttpClient(new Proxy(Socksproxy))
145 | {
146 | UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0",
147 | AcceptEncoding = "gzip"
148 | };
149 |
150 | string page = client.Get("https://httpbin.org/get");
151 | Assert.IsTrue(page.Contains(client.UserAgent));
152 | }
153 | public class Form
154 | {
155 | public string comments { get; set; }
156 | public string custemail { get; set; }
157 | public string custname { get; set; }
158 | public string custtel { get; set; }
159 | public string delivery { get; set; }
160 | public string size { get; set; }
161 | public string topping { get; set; }
162 | }
163 |
164 | public class RootObject
165 | {
166 | public object args { get; set; }
167 | public string data { get; set; }
168 | public object files { get; set; }
169 | public Form form { get; set; }
170 | public object headers { get; set; }
171 | public object json { get; set; }
172 | public string origin { get; set; }
173 | public string url { get; set; }
174 | }
175 | }
176 | }
177 |
--------------------------------------------------------------------------------
/UnitTestBetterHttpClient/packages.config:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
--------------------------------------------------------------------------------