├── .github └── workflows │ └── dotnetcore.yml ├── .gitignore ├── HttpHandler ├── HttpHandler.csproj ├── Program.cs └── RequestRunner.cs ├── HttpParser.sln ├── HttpWebRequestExecutor ├── Extensions │ ├── HttpWebResponseExtensions.cs │ └── StreamExtensions.cs ├── Factories │ └── HttpWebRequestFactory.cs ├── HttpParser │ ├── Models │ │ ├── CouldNotParseHttpRequestException.cs │ │ ├── IgnoreHttpParserOptions.cs │ │ ├── ParsedHttpRequest.cs │ │ ├── RequestBody.cs │ │ ├── RequestCookies.cs │ │ ├── RequestHeaders.cs │ │ └── RequestLine.cs │ └── Parser.cs ├── HttpRequestBuilder │ ├── Extensions │ │ └── HttpWebRequestExtensions.cs │ └── HttpWebRequestBuilder.cs ├── HttpWebRequestExecutor.csproj ├── Interfaces │ ├── IHttpWebRequest.cs │ ├── IHttpWebRequestFactory.cs │ └── IHttpWebResponse.cs ├── Lib │ ├── HttpWebRequestWrapper.cs │ └── HttpWebResponseWrapper.cs ├── Models │ └── ParsedWebResponse.cs └── mitlicense.txt ├── NuGetPackages └── HttpWebRequestExecutor.1.1.5.nupkg ├── Tests ├── FakeData │ └── FakeRawRequests.cs ├── HttpWebRequestBuilderTests.cs ├── ParseRawHttpTests.cs ├── ParsedObjectConvertTests.cs ├── RequestLineTests.cs └── Tests.csproj └── readme.md /.github/workflows/dotnetcore.yml: -------------------------------------------------------------------------------- 1 | name: dotnet core - build 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | 9 | steps: 10 | - name: Get current date without leading zeros 11 | id: date 12 | run: echo "::set-output name=date::$(date +'%Y.%-m.%-d').$GITHUB_RUN_NUMBER" 13 | 14 | - name: Test with environment variables 15 | run: echo $TAG_NAME 16 | env: 17 | TAG_NAME: ${{ steps.date.outputs.date }} 18 | 19 | - uses: actions/checkout@v3 20 | 21 | - name: Setup .NET 22 | uses: actions/setup-dotnet@v3 23 | with: 24 | dotnet-version: '8.0.x' 25 | 26 | - name: Build 27 | run: dotnet build --configuration Release 28 | 29 | - name: Unit Tests 30 | run: dotnet test 31 | 32 | - name: Build NuGet Package 33 | run: dotnet pack ./HttpWebRequestExecutor/HttpWebRequestExecutor.csproj --configuration Release -o ./NuGetPackages -p:PackageVersion=${{ steps.date.outputs.date }} 34 | 35 | - name: List generated files (for debugging) 36 | run: ls -la ./NuGetPackages 37 | 38 | - name: Deploy NuGet Package 39 | run: dotnet nuget push ./NuGetPackages/HttpWebRequestExecutor.${{ steps.date.outputs.date }}.nupkg -k ${{ secrets.NUGET_API_KEY }} -s https://api.nuget.org/v3/index.json 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.suo 2 | *.csproj.user 3 | obj/ 4 | bin/ 5 | *.ncb 6 | *.log 7 | *.cache 8 | *.user 9 | */_* 10 | *Reports/_* 11 | doc/ 12 | *.InstallLog 13 | TestResult.xml 14 | *SSIS/_* 15 | *Resharper.* 16 | CompressedExe/ 17 | CompressedMSM/ 18 | CompressedMsi/ 19 | *.orig 20 | *.[mM][dD][fF] 21 | *.[lL][dD][fF] 22 | *.rdl.data 23 | Compressed/ 24 | _ReSharper* 25 | Artifacts/** 26 | Installer/Debug/** 27 | UpgradeLog.XML 28 | .vs 29 | 30 | # utilities 31 | !nunit*.exe 32 | !octo.exe 33 | *.nuspec 34 | -------------------------------------------------------------------------------- /HttpHandler/HttpHandler.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Exe 5 | net8.0 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /HttpHandler/Program.cs: -------------------------------------------------------------------------------- 1 | using HttpWebRequestExecutor.Factories; 2 | using HttpWebRequestExecutor.Interfaces; 3 | using System; 4 | using System.Net; 5 | 6 | namespace HttpHandler 7 | { 8 | class Program 9 | { 10 | static void Main(string[] args) 11 | { 12 | try 13 | { 14 | IHttpWebRequestFactory factory = new HttpWebRequestFactory(); 15 | var rr = new RequestRunner(factory); 16 | rr.Run(); 17 | } 18 | catch(WebException wex) 19 | { 20 | Console.WriteLine($"Web exception caught. {wex.Message}"); 21 | } 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /HttpHandler/RequestRunner.cs: -------------------------------------------------------------------------------- 1 | using HttpWebRequestExecutor.Interfaces; 2 | using System; 3 | 4 | namespace HttpHandler 5 | { 6 | public class RequestRunner 7 | { 8 | public const string GetWithoutQueryString = @"GET https://httpbin.org/get HTTP/1.1 9 | Host: httpbin.org 10 | Connection: keep-alive 11 | User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36 12 | Upgrade-Insecure-Requests: 1 13 | Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 14 | Accept-Encoding: gzip, deflate, br 15 | Accept-Language: en-US,en;q=0.9"; 16 | 17 | private readonly IHttpWebRequestFactory factory; 18 | 19 | public RequestRunner(IHttpWebRequestFactory factory) 20 | { 21 | this.factory = factory; 22 | } 23 | 24 | public void Run() 25 | { 26 | var parsed = HttpParser.Parser.ParseRawRequest(GetWithoutQueryString); 27 | var req = factory.BuildRequest(parsed); 28 | 29 | var resp = req.GetResponse(); 30 | Console.WriteLine(resp.GetParsedWebResponse().ResponseText); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /HttpParser.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.29102.190 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests", "Tests\Tests.csproj", "{D8B84F71-C268-4CFC-B732-099CF45AB3CB}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HttpWebRequestExecutor", "HttpWebRequestExecutor\HttpWebRequestExecutor.csproj", "{EB4B727A-1944-4CC1-B271-7057963DF385}" 9 | EndProject 10 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HttpHandler", "HttpHandler\HttpHandler.csproj", "{269714CF-0AC9-482C-A6CC-35D32B999896}" 11 | EndProject 12 | Global 13 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 14 | Debug|Any CPU = Debug|Any CPU 15 | Debug|x64 = Debug|x64 16 | Release|Any CPU = Release|Any CPU 17 | Release|x64 = Release|x64 18 | EndGlobalSection 19 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 20 | {D8B84F71-C268-4CFC-B732-099CF45AB3CB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 21 | {D8B84F71-C268-4CFC-B732-099CF45AB3CB}.Debug|Any CPU.Build.0 = Debug|Any CPU 22 | {D8B84F71-C268-4CFC-B732-099CF45AB3CB}.Debug|x64.ActiveCfg = Debug|Any CPU 23 | {D8B84F71-C268-4CFC-B732-099CF45AB3CB}.Debug|x64.Build.0 = Debug|Any CPU 24 | {D8B84F71-C268-4CFC-B732-099CF45AB3CB}.Release|Any CPU.ActiveCfg = Release|Any CPU 25 | {D8B84F71-C268-4CFC-B732-099CF45AB3CB}.Release|Any CPU.Build.0 = Release|Any CPU 26 | {D8B84F71-C268-4CFC-B732-099CF45AB3CB}.Release|x64.ActiveCfg = Release|Any CPU 27 | {D8B84F71-C268-4CFC-B732-099CF45AB3CB}.Release|x64.Build.0 = Release|Any CPU 28 | {EB4B727A-1944-4CC1-B271-7057963DF385}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 29 | {EB4B727A-1944-4CC1-B271-7057963DF385}.Debug|Any CPU.Build.0 = Debug|Any CPU 30 | {EB4B727A-1944-4CC1-B271-7057963DF385}.Debug|x64.ActiveCfg = Debug|Any CPU 31 | {EB4B727A-1944-4CC1-B271-7057963DF385}.Debug|x64.Build.0 = Debug|Any CPU 32 | {EB4B727A-1944-4CC1-B271-7057963DF385}.Release|Any CPU.ActiveCfg = Release|Any CPU 33 | {EB4B727A-1944-4CC1-B271-7057963DF385}.Release|Any CPU.Build.0 = Release|Any CPU 34 | {EB4B727A-1944-4CC1-B271-7057963DF385}.Release|x64.ActiveCfg = Release|Any CPU 35 | {EB4B727A-1944-4CC1-B271-7057963DF385}.Release|x64.Build.0 = Release|Any CPU 36 | {269714CF-0AC9-482C-A6CC-35D32B999896}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 37 | {269714CF-0AC9-482C-A6CC-35D32B999896}.Debug|Any CPU.Build.0 = Debug|Any CPU 38 | {269714CF-0AC9-482C-A6CC-35D32B999896}.Debug|x64.ActiveCfg = Debug|Any CPU 39 | {269714CF-0AC9-482C-A6CC-35D32B999896}.Debug|x64.Build.0 = Debug|Any CPU 40 | {269714CF-0AC9-482C-A6CC-35D32B999896}.Release|Any CPU.ActiveCfg = Release|Any CPU 41 | {269714CF-0AC9-482C-A6CC-35D32B999896}.Release|Any CPU.Build.0 = Release|Any CPU 42 | {269714CF-0AC9-482C-A6CC-35D32B999896}.Release|x64.ActiveCfg = Release|Any CPU 43 | {269714CF-0AC9-482C-A6CC-35D32B999896}.Release|x64.Build.0 = Release|Any CPU 44 | EndGlobalSection 45 | GlobalSection(SolutionProperties) = preSolution 46 | HideSolutionNode = FALSE 47 | EndGlobalSection 48 | GlobalSection(ExtensibilityGlobals) = postSolution 49 | SolutionGuid = {2906F202-1DE7-48B9-AD89-3FADD25D19F4} 50 | EndGlobalSection 51 | EndGlobal 52 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/Extensions/HttpWebResponseExtensions.cs: -------------------------------------------------------------------------------- 1 | using System.Net; 2 | 3 | namespace HttpWebRequestExecutor.Extensions 4 | { 5 | public static class HttpWebResponseExtensions 6 | { 7 | public static string ResponseString(this HttpWebResponse resp) 8 | { 9 | using(var stream = resp.GetResponseStream()) 10 | { 11 | if (resp.Headers["Content-Encoding"] == "gzip") 12 | { 13 | return stream.DecompressGzipStream().GetStringFromStream(); 14 | } 15 | 16 | return stream.GetStringFromStream(); 17 | } 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/Extensions/StreamExtensions.cs: -------------------------------------------------------------------------------- 1 | using ICSharpCode.SharpZipLib.GZip; 2 | using System.IO; 3 | 4 | namespace HttpWebRequestExecutor.Extensions 5 | { 6 | internal static class StreamExtensions 7 | { 8 | public static string GetStringFromStream(this Stream stream) 9 | { 10 | if (stream == null) return null; 11 | 12 | using (var streamReader = new StreamReader(stream)) 13 | { 14 | return streamReader.ReadToEnd(); 15 | } 16 | } 17 | 18 | public static Stream DecompressGzipStream(this Stream stream) 19 | { 20 | if (stream == null) return stream; 21 | 22 | Stream compressedStream = new GZipInputStream(stream); 23 | 24 | if (compressedStream == null) 25 | { 26 | return stream; 27 | } 28 | 29 | var decompressedStream = new MemoryStream(); 30 | var size = 2048; 31 | var writeData = new byte[size]; 32 | 33 | while (true) 34 | { 35 | size = compressedStream.Read(writeData, 0, size); 36 | if (size > 0) 37 | { 38 | decompressedStream.Write(writeData, 0, size); 39 | } 40 | else 41 | { 42 | break; 43 | } 44 | } 45 | 46 | decompressedStream.Seek(0, SeekOrigin.Begin); 47 | 48 | return decompressedStream; 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/Factories/HttpWebRequestFactory.cs: -------------------------------------------------------------------------------- 1 | using HttpBuilder; 2 | using HttpParser.Models; 3 | using HttpWebRequestExecutor.Interfaces; 4 | using HttpWebRequestExecutor.Lib; 5 | 6 | namespace HttpWebRequestExecutor.Factories 7 | { 8 | public class HttpWebRequestFactory : IHttpWebRequestFactory 9 | { 10 | public IHttpWebRequest BuildRequest(ParsedHttpRequest parsed) 11 | { 12 | var request = HttpWebRequestBuilder.InitializeWebRequest(parsed); 13 | return new HttpWebRequestWrapper(request); 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/HttpParser/Models/CouldNotParseHttpRequestException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace HttpParser.Models 4 | { 5 | public class CouldNotParseHttpRequestException : Exception 6 | { 7 | public CouldNotParseHttpRequestException(string message, string step, string component) 8 | : base($"{message} Method: {step}() Data: {component}") { } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/HttpParser/Models/IgnoreHttpParserOptions.cs: -------------------------------------------------------------------------------- 1 | namespace HttpParser.Models 2 | { 3 | public class IgnoreHttpParserOptions 4 | { 5 | public bool IgnoreUrl { get; set; } 6 | public bool IgnoreHeaders { get; set; } 7 | public bool IgnoreCookies { get; set; } 8 | public bool IgnoreRequestBody { get; set; } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/HttpParser/Models/ParsedHttpRequest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Net; 5 | using System.Text; 6 | 7 | namespace HttpParser.Models 8 | { 9 | public class ParsedHttpRequest 10 | { 11 | public string Url { get; set; } 12 | public Dictionary Headers { get; set; } 13 | public Dictionary Cookies { get; set; } 14 | public string RequestBody { get; set; } 15 | public Uri Uri { get; set; } 16 | public CookieContainer CookieContainer { get; set; } 17 | 18 | private IgnoreHttpParserOptions ignoreHttpParserOptions; 19 | public ParsedHttpRequest(IgnoreHttpParserOptions options = null) 20 | { 21 | ignoreHttpParserOptions = options; 22 | } 23 | 24 | public void ApplyIgnoreOptions() 25 | { 26 | if (ignoreHttpParserOptions == null) return; 27 | 28 | if (ignoreHttpParserOptions.IgnoreUrl) 29 | { 30 | Url = null; 31 | } 32 | if (ignoreHttpParserOptions.IgnoreHeaders) 33 | { 34 | Headers = null; 35 | } 36 | if (ignoreHttpParserOptions.IgnoreCookies) 37 | { 38 | Cookies = null; 39 | CookieContainer = null; 40 | } 41 | if (ignoreHttpParserOptions.IgnoreRequestBody) 42 | { 43 | RequestBody = null; 44 | } 45 | } 46 | 47 | public override string ToString() 48 | { 49 | var method = Headers["Method"]; 50 | var version = Headers["HttpVersion"]; 51 | 52 | var sb = new StringBuilder($"{method} {Url} {version}{Environment.NewLine}"); 53 | 54 | var headersToIgnore = new List { "Method", "HttpVersion" }; 55 | if (Cookies == null) headersToIgnore.Add("Cookie"); 56 | 57 | foreach(var header in Headers) 58 | { 59 | if (headersToIgnore.Contains(header.Key)) continue; 60 | sb.Append($"{header.Key}: {header.Value}{Environment.NewLine}"); 61 | } 62 | 63 | if (Cookies?.Count > 0) 64 | { 65 | var cookies = string.Join(";", Cookies 66 | .Select(cookie => $" {cookie.Key}={cookie.Value};")) 67 | .TrimEnd(';'); 68 | 69 | sb.Append($"Cookie:{cookies}{Environment.NewLine}"); 70 | } 71 | 72 | if (method == "POST") 73 | { 74 | sb.Append(Environment.NewLine); 75 | sb.Append(RequestBody); 76 | } 77 | 78 | return sb.ToString().Trim(); 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/HttpParser/Models/RequestBody.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | 3 | namespace HttpParser.Models 4 | { 5 | internal class RequestBody 6 | { 7 | public string Body { get; set; } 8 | 9 | public RequestBody(RequestLine requestLine, string[] lines) 10 | { 11 | if (requestLine.Method == "GET") 12 | { 13 | Body = SetBodyFromUrlGet(requestLine.Url); 14 | } 15 | if (requestLine.Method == "POST") 16 | { 17 | Body = SetBodyFromPost(lines); 18 | } 19 | } 20 | 21 | private static string SetBodyFromUrlGet(string url) 22 | { 23 | return url.Contains('?') 24 | ? url.Split('?')[1] 25 | : null; 26 | } 27 | 28 | private static string SetBodyFromPost(string[] lines) 29 | { 30 | var index = lines.Length; 31 | 32 | if (index == -1) 33 | { 34 | return null; 35 | } 36 | 37 | return lines[index - 1]; 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/HttpParser/Models/RequestCookies.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text.RegularExpressions; 5 | 6 | namespace HttpParser.Models 7 | { 8 | internal class RequestCookies 9 | { 10 | public Dictionary ParsedCookies = new Dictionary(); 11 | public RequestCookies(string[] lines) 12 | { 13 | var cookieLine = ExtractCookiesLine(lines); 14 | PopulateParsedCookies(cookieLine); 15 | } 16 | 17 | private string ExtractCookiesLine(string[] lines) 18 | { 19 | var cookieIndex = Array.FindLastIndex(lines, l => l.StartsWith("Cookie")); 20 | 21 | return cookieIndex > 0 ? lines[cookieIndex] : null; 22 | } 23 | 24 | private void PopulateParsedCookies(string cookiesLine) 25 | { 26 | if (string.IsNullOrEmpty(cookiesLine)) return; 27 | 28 | var matches = new Regex(@"Cookie:(?(.+))", RegexOptions.Singleline).Match(cookiesLine); 29 | var cookies = matches.Groups["Cookie"].ToString().Trim().Split(';'); 30 | 31 | if (cookies?.Length < 1 || cookies.Contains("")) 32 | { 33 | return; 34 | } 35 | 36 | foreach (var cookie in cookies) 37 | { 38 | var key = cookie.Split('=')[0].Trim(); 39 | var value = cookie.Split('=')[1].Trim(); 40 | ParsedCookies[key] = value; 41 | } 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/HttpParser/Models/RequestHeaders.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace HttpParser.Models 5 | { 6 | internal class RequestHeaders 7 | { 8 | public Dictionary Headers; 9 | 10 | public RequestHeaders(string[] lines) 11 | { 12 | InitializeHeaders(lines); 13 | } 14 | 15 | public void AddHeader(string key, string value) 16 | { 17 | Headers[key] = value; 18 | } 19 | 20 | public void RemoveHeader(string key) 21 | { 22 | if (Headers.ContainsKey(key)) 23 | Headers.Remove(key); 24 | } 25 | 26 | private void InitializeHeaders(string[] lines) 27 | { 28 | Headers = new Dictionary(); 29 | var lastIndex = DetectLastRowIndex(lines); 30 | for (int i = 1; i < lastIndex; i++) 31 | { 32 | var (key, value) = GetHeader(lines[i]); 33 | 34 | if (key == "Cookie") continue; 35 | 36 | Headers[key] = value; 37 | } 38 | } 39 | 40 | private static (string key, string value) GetHeader(string line) 41 | { 42 | var pieces = line.Split(new[] { ':' }, 2); 43 | 44 | return (pieces[0].Trim(), pieces[1].Trim()); 45 | } 46 | 47 | private static int DetectLastRowIndex(string[] lines) 48 | { 49 | var blankIndex = Array.IndexOf(lines, ""); 50 | return blankIndex == -1 ? lines.Length : blankIndex - 1; 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/HttpParser/Models/RequestLine.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Text.RegularExpressions; 4 | 5 | namespace HttpParser.Models 6 | { 7 | public class RequestLine 8 | { 9 | public string Method { get; set; } 10 | public string Url { get; set; } 11 | public string HttpVersion { get; set; } 12 | 13 | private readonly string[] validHttpVerbs = { "GET", "POST" }; 14 | 15 | public RequestLine(string[] lines) 16 | { 17 | var firstLine = lines[0].Split(' '); 18 | ValidateRequestLine(firstLine); 19 | 20 | SetHttpMethod(firstLine[0]); 21 | SetUrl(firstLine[1]); 22 | SetHttpVersion(firstLine[2]); 23 | } 24 | 25 | private void ValidateRequestLine(string[] firstLine) 26 | { 27 | if (firstLine.Length != 3) 28 | throw new CouldNotParseHttpRequestException("Request Line is not in a valid format", "ValidateRequestLine", string.Join(" ", firstLine)); 29 | } 30 | 31 | private void SetHttpMethod(string method) 32 | { 33 | method = method.Trim().ToUpper(); 34 | 35 | if (!validHttpVerbs.Contains(method)) 36 | throw new CouldNotParseHttpRequestException($"Not a valid HTTP Verb", "SetHttpMethod", method); 37 | 38 | Method = method; 39 | } 40 | 41 | private void SetUrl(string url) 42 | { 43 | if (!IsValidUri(url, out Uri _)) 44 | throw new CouldNotParseHttpRequestException($"URL is not in a valid format", "SetUrl", url); 45 | 46 | Url = url.Trim(); 47 | } 48 | 49 | private static bool IsValidUri(string url, out Uri uriResult) 50 | { 51 | return Uri.TryCreate(url, UriKind.Absolute, out uriResult) && (uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps); 52 | } 53 | 54 | private void SetHttpVersion(string version) 55 | { 56 | if (!Regex.IsMatch(version, @"HTTP/\d.\d")) 57 | { 58 | HttpVersion = "HTTP/1.1"; 59 | } 60 | 61 | HttpVersion = version.Trim(); 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/HttpParser/Parser.cs: -------------------------------------------------------------------------------- 1 | using HttpParser.Models; 2 | using System; 3 | 4 | namespace HttpParser 5 | { 6 | public static class Parser 7 | { 8 | public static ParsedHttpRequest ParseRawRequest(string raw, IgnoreHttpParserOptions options = null) 9 | { 10 | try 11 | { 12 | var lines = SplitLines(raw); 13 | 14 | var requestLine = new RequestLine(lines); 15 | var requestHeaders = new RequestHeaders(lines); 16 | requestHeaders.AddHeader("Method", requestLine.Method); 17 | requestHeaders.AddHeader("HttpVersion", requestLine.HttpVersion); 18 | 19 | var requestCookies = new RequestCookies(lines); 20 | var requestBody = new RequestBody(requestLine, lines); 21 | 22 | var parsed = new ParsedHttpRequest(options) 23 | { 24 | Url = requestLine.Url, 25 | Uri = new Uri(requestLine.Url), 26 | Headers = requestHeaders.Headers, 27 | Cookies = requestCookies.ParsedCookies, 28 | RequestBody = requestBody.Body 29 | }; 30 | 31 | parsed.ApplyIgnoreOptions(); 32 | 33 | return parsed; 34 | } 35 | catch (CouldNotParseHttpRequestException c) 36 | { 37 | Console.WriteLine($"Could not parse the raw request. {c.Message}"); 38 | throw; 39 | } 40 | catch (Exception e) 41 | { 42 | Console.WriteLine($"Unhandled error parsing the raw request: {raw}\r\nError {e.Message}"); 43 | throw; 44 | } 45 | } 46 | 47 | private static string[] SplitLines(string raw) 48 | { 49 | return raw 50 | .TrimEnd('\r', '\n') 51 | .Split(new[] { "\\n", "\n", "\r\n" }, StringSplitOptions.None); 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/HttpRequestBuilder/Extensions/HttpWebRequestExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Net; 4 | using System.Text; 5 | 6 | namespace HttpBuilder.Extensions 7 | { 8 | internal static class HttpWebRequestExtensions 9 | { 10 | public static void SetHttpHeaders(this HttpWebRequest request, Dictionary headers) 11 | { 12 | foreach (var header in headers) 13 | { 14 | try 15 | { 16 | request.SetHttpHeader(header.Key, header.Value); 17 | } 18 | catch(Exception) 19 | { 20 | Console.WriteLine($"Could not set HTTP header {header.Key}: {header.Value}"); 21 | } 22 | } 23 | } 24 | 25 | public static void SetHttpCookies(this HttpWebRequest request, Dictionary cookies, Uri uri) 26 | { 27 | if (cookies == null) return; 28 | 29 | request.CookieContainer = new CookieContainer(); 30 | 31 | foreach(var cookie in cookies) 32 | { 33 | request.CookieContainer.Add(uri, new Cookie(cookie.Key, cookie.Value)); 34 | } 35 | } 36 | 37 | public static void SetRequestData(this HttpWebRequest request, string requestData) 38 | { 39 | if (string.IsNullOrEmpty(requestData)) return; 40 | 41 | if (request.Method == "POST") 42 | { 43 | request.AddRequestData(requestData); 44 | } 45 | } 46 | 47 | private static HttpWebRequest SetHttpHeader(this HttpWebRequest request, string key, string value) 48 | { 49 | switch (key.ToLower()) 50 | { 51 | case "method": 52 | request.Method = value; 53 | break; 54 | case "accept": 55 | request.Accept = value; 56 | break; 57 | case "connection": 58 | request.KeepAlive = value.ToLower() == "keep-alive"; 59 | //req.Connection = value; 60 | // throws System.ArgumentException : "Keep-Alive and Close may not be set using this property." 61 | break; 62 | case "contenttype": 63 | case "content-type": 64 | request.ContentType = value; 65 | break; 66 | case "content-length": 67 | case "contentlength": 68 | request.ContentLength = Convert.ToInt64(value); 69 | break; 70 | case "date": 71 | request.Date = Convert.ToDateTime(value); 72 | break; 73 | case "expect": 74 | if (value == "100-continue") 75 | break; 76 | request.Expect = value; 77 | break; 78 | case "host": 79 | request.Host = value; 80 | break; 81 | case "httpversion": 82 | var version = Convert.ToString(value).Split('/')[1]; 83 | request.ProtocolVersion = Version.Parse(version); 84 | break; 85 | case "ifmodifiedsince": 86 | case "if-modified-since": 87 | request.IfModifiedSince = Convert.ToDateTime(value); 88 | break; 89 | case "keepalive": 90 | case "keep-alive": 91 | request.KeepAlive = Convert.ToBoolean(value); 92 | break; 93 | case "proxy-connection": 94 | break; 95 | case "referer": 96 | request.Referer = value; 97 | break; 98 | case "transferEncoding": 99 | request.TransferEncoding = value; 100 | break; 101 | case "useragent": 102 | case "user-agent": 103 | request.UserAgent = value; 104 | break; 105 | default: 106 | request.Headers[key] = value; 107 | break; 108 | } 109 | 110 | return request; 111 | } 112 | 113 | private static void AddRequestData(this WebRequest request, string value) 114 | { 115 | var data = Encoding.ASCII.GetBytes(value); 116 | request.ContentLength = data.Length; 117 | 118 | using (var streamWriter = request.GetRequestStream()) 119 | { 120 | streamWriter.Write(data, 0, data.Length); 121 | } 122 | } 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/HttpRequestBuilder/HttpWebRequestBuilder.cs: -------------------------------------------------------------------------------- 1 | using HttpBuilder.Extensions; 2 | using HttpParser.Models; 3 | using System; 4 | using System.Net; 5 | 6 | namespace HttpBuilder 7 | { 8 | public static class HttpWebRequestBuilder 9 | { 10 | public static HttpWebRequest InitializeWebRequest(ParsedHttpRequest parsed, Action callback = null) 11 | { 12 | var req = (HttpWebRequest)WebRequest.Create(parsed.Url); 13 | req.SetHttpHeaders(parsed.Headers); 14 | req.AddCookies(parsed); 15 | 16 | callback?.Invoke(req); 17 | 18 | req.SetRequestData(parsed.RequestBody); 19 | 20 | return req; 21 | } 22 | 23 | private static void AddCookies(this HttpWebRequest request, ParsedHttpRequest parsed) 24 | { 25 | if (parsed.CookieContainer != null) 26 | { 27 | request.CookieContainer = parsed.CookieContainer; 28 | return; 29 | } 30 | 31 | if (parsed.Cookies != null) 32 | { 33 | request.SetHttpCookies(parsed.Cookies, parsed.Uri); 34 | } 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/HttpWebRequestExecutor.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | net8.0 5 | Ryan Johnson 6 | Copyright 2019 7 | true 8 | false 9 | 10 | mitlicense.txt 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | True 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/Interfaces/IHttpWebRequest.cs: -------------------------------------------------------------------------------- 1 | namespace HttpWebRequestExecutor.Interfaces 2 | { 3 | public interface IHttpWebRequest 4 | { 5 | IHttpWebResponse GetResponse(); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/Interfaces/IHttpWebRequestFactory.cs: -------------------------------------------------------------------------------- 1 | using HttpParser.Models; 2 | 3 | namespace HttpWebRequestExecutor.Interfaces 4 | { 5 | public interface IHttpWebRequestFactory 6 | { 7 | IHttpWebRequest BuildRequest(ParsedHttpRequest parsed); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/Interfaces/IHttpWebResponse.cs: -------------------------------------------------------------------------------- 1 | using HttpWebRequestExecutor.Models; 2 | using System; 3 | using System.IO; 4 | 5 | namespace HttpWebRequestExecutor.Interfaces 6 | { 7 | public interface IHttpWebResponse : IDisposable 8 | { 9 | Stream GetResponseStream(); 10 | ParsedWebResponse GetParsedWebResponse(); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/Lib/HttpWebRequestWrapper.cs: -------------------------------------------------------------------------------- 1 | using HttpWebRequestExecutor.Interfaces; 2 | using System.Net; 3 | 4 | namespace HttpWebRequestExecutor.Lib 5 | { 6 | internal class HttpWebRequestWrapper : IHttpWebRequest 7 | { 8 | private readonly HttpWebRequest request; 9 | 10 | public HttpWebRequestWrapper(HttpWebRequest request) 11 | { 12 | this.request = request; 13 | } 14 | 15 | public IHttpWebResponse GetResponse() 16 | { 17 | return new HttpWebResponseWrapper((HttpWebResponse)request.GetResponse()); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/Lib/HttpWebResponseWrapper.cs: -------------------------------------------------------------------------------- 1 | using HttpWebRequestExecutor.Interfaces; 2 | using HttpWebRequestExecutor.Models; 3 | using System; 4 | using System.IO; 5 | using System.Net; 6 | 7 | namespace HttpWebRequestExecutor.Lib 8 | { 9 | internal class HttpWebResponseWrapper : IHttpWebResponse 10 | { 11 | private HttpWebResponse response; 12 | 13 | public HttpWebResponseWrapper(HttpWebResponse response) 14 | { 15 | this.response = response; 16 | } 17 | 18 | public Stream GetResponseStream() 19 | { 20 | return response.GetResponseStream(); 21 | } 22 | 23 | public ParsedWebResponse GetParsedWebResponse() 24 | { 25 | return new ParsedWebResponse(response); 26 | } 27 | 28 | public void Dispose() 29 | { 30 | Dispose(true); 31 | GC.SuppressFinalize(this); 32 | } 33 | 34 | private void Dispose(bool disposing) 35 | { 36 | if (!disposing || response == null) return; 37 | 38 | response.Dispose(); 39 | response = null; 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/Models/ParsedWebResponse.cs: -------------------------------------------------------------------------------- 1 | using HttpWebRequestExecutor.Extensions; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Net; 6 | 7 | namespace HttpWebRequestExecutor.Models 8 | { 9 | public class ParsedWebResponse 10 | { 11 | public string ResponseText { get; set; } 12 | public int StatusCode { get; set; } 13 | public string StatusDescription { get; set; } 14 | public string Cookies { get; set; } 15 | public Uri ResponseUri { get; set; } 16 | public Dictionary ResponseHeaders { get; set; } 17 | 18 | public ParsedWebResponse() { } 19 | public ParsedWebResponse(HttpWebResponse response) 20 | { 21 | ResponseText = response.ResponseString(); 22 | StatusCode = (int)response.StatusCode; 23 | StatusDescription = response.StatusDescription; 24 | ResponseHeaders = ConvertWebHeadersToDictionary(response.Headers); 25 | Cookies = response.Headers["Set-Cookie"]; 26 | ResponseUri = response.ResponseUri; 27 | } 28 | 29 | private Dictionary ConvertWebHeadersToDictionary(WebHeaderCollection headers) 30 | { 31 | return Enumerable.Range(0, headers.Count).ToDictionary(i => headers.Keys[i], headers.GetValues); 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /HttpWebRequestExecutor/mitlicense.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /NuGetPackages/HttpWebRequestExecutor.1.1.5.nupkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ryanxjohnson/http-parser/3371ec4e6391bfee05c2db825e595af13617fe91/NuGetPackages/HttpWebRequestExecutor.1.1.5.nupkg -------------------------------------------------------------------------------- /Tests/FakeData/FakeRawRequests.cs: -------------------------------------------------------------------------------- 1 | namespace Tests.FakeData 2 | { 3 | public class FakeRawRequests 4 | { 5 | public const string GetWithoutQueryString = @"GET https://httpbin.org/get HTTP/1.1 6 | Host: httpbin.org 7 | Connection: keep-alive 8 | User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36 9 | Upgrade-Insecure-Requests: 1 10 | Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 11 | Accept-Encoding: gzip, deflate, br 12 | Accept-Language: en-US,en;q=0.9"; 13 | 14 | 15 | public const string GetWithQueryString = @"GET https://httpbin.org/get?name=ryan HTTP/1.1 16 | Host: httpbin.org 17 | Connection: keep-alive 18 | User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36 19 | Upgrade-Insecure-Requests: 1 20 | Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 21 | Accept-Encoding: gzip, deflate, br 22 | Accept-Language: en-US,en;q=0.9"; 23 | 24 | public const string PostWithRequestBody = @"POST https://httpbin.org/post HTTP/1.1 25 | Host: httpbin.org 26 | User-Agent: curl/7.54.1 27 | Accept: */* 28 | Content-Type: application/x-www-form-urlencoded 29 | Cookie: ilikecookies=chocchip 30 | 31 | helloworld"; 32 | 33 | public const string BadlyFormattedRequest1 = @"GET www.httpbin.org/get HTTP/1.1 34 | Host: httpbin.org 35 | Connection: keep-alive 36 | User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36 37 | Upgrade-Insecure-Requests: 1 38 | Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 39 | Accept-Encoding: gzip, deflate, br 40 | Accept-Language: en-US,en;q=0.9"; 41 | 42 | public const string BadlyFormattedRequest2 = @"POST https://httpbin.org/post HTTP/1.1 43 | Host: httpbin.org 44 | User-Agent: curl/7.54.1 45 | Accept: */* 46 | Content-Type: application/x-www-form-urlencoded 47 | Cookie: ilikecookies=chocchip 48 | 49 | helloworld 50 | 51 | "; 52 | 53 | public const string RequestWithCookiesInTheWrongSpot = @"POST http://www.providerlookuponline.com/Coventry/po7/Client_FacetWebService.asmx/FillStateFacet HTTP/1.1 54 | Accept: application/json, text/javascript, */*; q=0.01 55 | Origin: http://www.providerlookuponline.com 56 | X-Requested-With: XMLHttpRequest 57 | User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36 58 | Content-Type: application/json; charset=UTF-8 59 | Referer: http://www.providerlookuponline.com/coventry/po7/Search.aspx 60 | Accept-Encoding: gzip, deflate 61 | Host: www.providerlookuponline.com 62 | Cookie: ASP.NET_SessionId=yayd4qyttoe1xgq0xgc24qmy; TS019e6c20=014b5a756ffb575a91f7fe2504052493bb4f1fba160f72df0ba31f8e13fd05c3d62743577fe69ccf443cff87a9d33c5a5f3cc5a96c9e6ba3933a2150b7925ed7a82f86fba4; TS019e6c20_28=014d91d154ab4251246f2b614bad8e3e3241d256a621c0c978e5b5b0c957f5a69eb8d190406800a83fe9c55d4a39c60544f14384e4; TS96fa379b_75=TS96fa379b_rc=0&TS96fa379b_id=2&TS96fa379b_cr=08826a2764ab2800a727f3fb31df5fa1f3d7126b7eb93d1972e2ff558b0f89972be6c6a81e4f0ecc37725679d56547d4:0801e2461a032000f07dc4289936b17d79f52446a01f35ddc8aedb7f8aec47131ce115d972fb6fb8&TS96fa379b_ef=&TS96fa379b_pg=0&TS96fa379b_ct=0&TS96fa379b_rf=0; TSPD_101=08826a2764ab2800a727f3fb31df5fa1f3d7126b7eb93d1972e2ff558b0f89972be6c6a81e4f0ecc37725679d56547d4:; GeoCookie=VisitorGUID=c121d0f4-b987-443d-905a-0d015b138eaa; InitialIntegrationComplete=True; __utma=1.1241581742.1539971847.1539971847.1539971847.1; __utmc=1; __utmz=1.1539971847.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __utmt=1; __utmb=1.1.10.1539971847; TS019e6c20_77=08826a2764ab2800c69276fe08e7da9f855eefc774f68568df90d2fb919995f7424c2808379f3342dcf98983f849918c08cbf3bf3d823800b90ffcee0cb1a32f5eb378991711baafb033fee7eedb1b0ceaec62f398876aef21bfefc413a3f8f81dc6e49b9a3ae3c2d6f47a28e1ba2d72; GeoCookie=VisitorGUID=c121d0f4-b987-443d-905a-0d015b138eaa; TS01b76a60=014b5a756f4c4111c831124ec77a44bbd45d6d53ed0f72df0ba31f8e13fd05c3d62743577f32f32b5e15dd86c365179a3c6bdbfcf5b69a009d8cebfcc98e7f845df02080ee; ASP.NET_SessionId=yayd4qyttoe1xgq0xgc24qmy; TS019e6c20=014b5a756ffb575a91f7fe2504052493bb4f1fba160f72df0ba31f8e13fd05c3d62743577fe69ccf443cff87a9d33c5a5f3cc5a96c9e6ba3933a2150b7925ed7a82f86fba4; TS01b76a60=014b5a756f4c4111c831124ec77a44bbd45d6d53ed0f72df0ba31f8e13fd05c3d62743577f32f32b5e15dd86c365179a3c6bdbfcf5b69a009d8cebfcc98e7f845df02080ee; TS019e6c20_28=014d91d154ab4251246f2b614bad8e3e3241d256a621c0c978e5b5b0c957f5a69eb8d190406800a83fe9c55d4a39c60544f14384e4 63 | Content-Length: 23 64 | 65 | #{{RequestBody}}"; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /Tests/HttpWebRequestBuilderTests.cs: -------------------------------------------------------------------------------- 1 | using HttpBuilder; 2 | using HttpParser; 3 | using NUnit.Framework; 4 | using System.IO; 5 | using System.Text; 6 | using Moq; 7 | using HttpParser.Models; 8 | using HttpWebRequestExecutor.Models; 9 | using Tests.FakeData; 10 | using HttpWebRequestExecutor.Interfaces; 11 | using FluentAssertions; 12 | 13 | namespace Tests 14 | { 15 | [TestFixture] 16 | public class HttpWebRequestBuilderTests 17 | { 18 | [Test] 19 | public void Should_Build_Get() 20 | { 21 | var parsed = Parser.ParseRawRequest(FakeRawRequests.GetWithoutQueryString); 22 | var req = HttpWebRequestBuilder.InitializeWebRequest(parsed); 23 | 24 | req.Should().BeOfType(); 25 | } 26 | 27 | [Test] 28 | public void Should_Run_Fake_Request() 29 | { 30 | // arrange 31 | var expected = "hello world"; 32 | 33 | var response = new Mock(); 34 | response.Setup(s => s.GetResponseStream()).Returns(FakeStream(expected)); 35 | 36 | var request = new Mock(); 37 | request.Setup(c => c.GetResponse()).Returns(response.Object); 38 | 39 | var factory = new Mock(); 40 | factory.Setup(c => c.BuildRequest(It.IsAny())).Returns(request.Object); 41 | 42 | var parsed = Parser.ParseRawRequest(FakeRawRequests.GetWithoutQueryString); 43 | 44 | // act 45 | var actualRequest = factory.Object.BuildRequest(parsed); 46 | 47 | string actual; 48 | 49 | using (var httpWebResponse = actualRequest.GetResponse()) 50 | { 51 | using (var streamReader = new StreamReader(httpWebResponse.GetResponseStream())) 52 | { 53 | actual = streamReader.ReadToEnd(); 54 | } 55 | } 56 | 57 | // assert 58 | actual.Should().Be(expected); 59 | } 60 | 61 | [Test] 62 | public void Should_Get_Fake_ParsedWebResponse() 63 | { 64 | // arrange 65 | var expected = "hello world"; 66 | 67 | var response = new Mock(); 68 | response.Setup(s => s.GetParsedWebResponse()).Returns(FakeParsedWebResponse(expected)); 69 | 70 | var request = new Mock(); 71 | request.Setup(c => c.GetResponse()).Returns(response.Object); 72 | 73 | var factory = new Mock(); 74 | factory.Setup(c => c.BuildRequest(It.IsAny())).Returns(request.Object); 75 | 76 | var parsed = Parser.ParseRawRequest(FakeRawRequests.GetWithoutQueryString); 77 | 78 | // act 79 | var actualRequest = factory.Object.BuildRequest(parsed); 80 | 81 | string actual; 82 | 83 | using (var httpWebResponse = actualRequest.GetResponse()) 84 | { 85 | actual = httpWebResponse.GetParsedWebResponse().ResponseText; 86 | } 87 | 88 | // assert 89 | actual.Should().BeEquivalentTo(expected); 90 | } 91 | 92 | private static MemoryStream FakeStream(string expected) 93 | { 94 | var expectedBytes = Encoding.UTF8.GetBytes(expected); 95 | var responseStream = new MemoryStream(); 96 | responseStream.Write(expectedBytes, 0, expectedBytes.Length); 97 | responseStream.Seek(0, SeekOrigin.Begin); 98 | 99 | return responseStream; 100 | } 101 | 102 | private static ParsedWebResponse FakeParsedWebResponse(string responseText) 103 | { 104 | return new ParsedWebResponse() 105 | { 106 | ResponseText = responseText 107 | }; 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /Tests/ParseRawHttpTests.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using HttpParser; 3 | using HttpParser.Models; 4 | using NUnit.Framework; 5 | using Tests.FakeData; 6 | 7 | namespace HttpParserTests 8 | { 9 | [TestFixture()] 10 | public class ParseRawHttpTests 11 | { 12 | [Test] 13 | public void Should_Parse_Get() 14 | { 15 | var parsed = Parser.ParseRawRequest(FakeRawRequests.GetWithoutQueryString); 16 | parsed.Url.Should().BeEquivalentTo("https://httpbin.org/get"); 17 | parsed.Headers["Method"].Should().BeEquivalentTo("GET"); 18 | parsed.RequestBody.Should().BeNull(); 19 | } 20 | 21 | [Test] 22 | public void Should_Parse_Get_With_QueryString() 23 | { 24 | var parsed = Parser.ParseRawRequest(FakeRawRequests.GetWithQueryString); 25 | 26 | parsed.Url.Should().BeEquivalentTo("https://httpbin.org/get?name=ryan"); 27 | parsed.Headers["Method"].Should().BeEquivalentTo("GET"); 28 | parsed.RequestBody.Should().BeEquivalentTo("name=ryan"); 29 | } 30 | 31 | [Test] 32 | public void Should_Parse_Get_With_QueryString_Ignored() 33 | { 34 | var options = new IgnoreHttpParserOptions 35 | { 36 | IgnoreRequestBody = true 37 | }; 38 | 39 | var parsed = Parser.ParseRawRequest(FakeRawRequests.GetWithQueryString, options); 40 | 41 | parsed.Url.Should().BeEquivalentTo("https://httpbin.org/get?name=ryan"); 42 | parsed.Headers["Method"].Should().BeEquivalentTo("GET"); 43 | parsed.RequestBody.Should().BeNull(); 44 | } 45 | 46 | [Test] 47 | public void Should_Parse_Post() 48 | { 49 | var parsed = Parser.ParseRawRequest(FakeRawRequests.PostWithRequestBody); 50 | 51 | parsed.Url.Should().BeEquivalentTo("https://httpbin.org/post"); 52 | parsed.Headers["Method"].Should().BeEquivalentTo("POST"); 53 | parsed.RequestBody.Should().BeEquivalentTo("helloworld"); 54 | } 55 | 56 | [Test] 57 | public void Should_Parse_Post_Ignore_RequestBody() 58 | { 59 | var options = new IgnoreHttpParserOptions 60 | { 61 | IgnoreRequestBody = true 62 | }; 63 | 64 | var parsed = Parser.ParseRawRequest(FakeRawRequests.PostWithRequestBody, options); 65 | 66 | parsed.Url.Should().BeEquivalentTo("https://httpbin.org/post"); 67 | parsed.Headers["Method"].Should().BeEquivalentTo("POST"); 68 | parsed.Cookies.Should().HaveCount(1); 69 | parsed.Cookies["ilikecookies"].Should().BeEquivalentTo("chocchip"); 70 | parsed.RequestBody.Should().BeNull(); 71 | } 72 | 73 | [TestCase(FakeRawRequests.BadlyFormattedRequest1, "URL is not in a valid format Method: SetUrl() Data: www.httpbin.org/get")] 74 | public void Should_Throw_For_Badly_Formatted_Request(string raw, string expectedMessage) 75 | { 76 | var ex = Assert.Throws(() => Parser.ParseRawRequest(raw)); 77 | 78 | ex.Message.Should().Be(expectedMessage); 79 | } 80 | 81 | [Test] 82 | public void Should_Not_Throw_For_Extra_Lines() 83 | { 84 | Assert.DoesNotThrow(() => Parser.ParseRawRequest(FakeRawRequests.BadlyFormattedRequest2)); 85 | } 86 | 87 | [Test] 88 | public void Should_Parse_Cookie_In_Wrong_Place() 89 | { 90 | var raw = FakeRawRequests.RequestWithCookiesInTheWrongSpot; 91 | var parsed = Parser.ParseRawRequest(raw); 92 | 93 | System.Console.WriteLine(parsed.Cookies.Count); 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /Tests/ParsedObjectConvertTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | 3 | namespace Tests 4 | { 5 | [TestFixture] 6 | public class ParsedObjectConvertTests 7 | { 8 | [TestCase(FakeData.FakeRawRequests.GetWithoutQueryString)] 9 | [TestCase(FakeData.FakeRawRequests.PostWithRequestBody)] 10 | public void Should_Convert_ParsedRequest_Back_To_String(string input) 11 | { 12 | var parsed = HttpParser.Parser.ParseRawRequest(input); 13 | 14 | Assert.That(input, Is.EqualTo(parsed.ToString())); 15 | } 16 | 17 | [TestCase(FakeData.FakeRawRequests.PostWithRequestBody)] 18 | public void Should_Strip_Cookies(string input) 19 | { 20 | var parsed = HttpParser.Parser.ParseRawRequest(input, new HttpParser.Models.IgnoreHttpParserOptions { IgnoreCookies = true }); ; 21 | 22 | Assert.That(requestCookiesStripped, Is.EqualTo(parsed.ToString())); 23 | } 24 | 25 | private readonly string requestCookiesStripped = @"POST https://httpbin.org/post HTTP/1.1 26 | Host: httpbin.org 27 | User-Agent: curl/7.54.1 28 | Accept: */* 29 | Content-Type: application/x-www-form-urlencoded 30 | 31 | helloworld"; 32 | } 33 | } -------------------------------------------------------------------------------- /Tests/RequestLineTests.cs: -------------------------------------------------------------------------------- 1 | using HttpParser.Models; 2 | using NUnit.Framework; 3 | using FluentAssertions; 4 | namespace HttpParserTests 5 | { 6 | [TestFixture] 7 | public class RequestLineTests 8 | { 9 | [Test] 10 | public void Should_Parse_Request_Line() 11 | { 12 | var line = new [] { "GET https://www.example.com HTTP/1.1" }; 13 | 14 | var requestLine = new RequestLine(line); 15 | 16 | requestLine.Method.Should().BeEquivalentTo("GET"); 17 | requestLine.Url.Should().BeEquivalentTo("https://www.example.com"); 18 | requestLine.HttpVersion.Should().BeEquivalentTo("HTTP/1.1"); 19 | } 20 | 21 | [Test] 22 | public void Should_Throw_For_Bad_Method() 23 | { 24 | var line = new[] { "PUT https://www.example.com HTTP/1.1" }; 25 | 26 | var ex = Assert.Throws(() => new RequestLine(line)); 27 | ex.Message.Should().BeEquivalentTo("Not a valid HTTP Verb Method: SetHttpMethod() Data: PUT"); 28 | } 29 | 30 | [Test] 31 | public void Should_Throw_For_Bad_Url() 32 | { 33 | var line = new[] { "GET www.example.com HTTP/1.1" }; 34 | 35 | var ex = Assert.Throws(() => new RequestLine(line)); 36 | ex.Message.Should().BeEquivalentTo("URL is not in a valid format Method: SetUrl() Data: www.example.com"); 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /Tests/Tests.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Exe 5 | net8.0 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Http Web Parser 2 | 3 | ## What this libary will accomplish: 4 | This library will parse raw HTTP request text to a C# object. 5 | This allows you save all the information about a particular web request in permanent storage, decoupling the request properties from any particular framework. 6 | 7 | This libary will also build a .NET `HttpWebRequest` object from the raw request text or the JSON object. 8 | 9 | IgnoreSerialization options allows the client to ignore certain headers like cookies, headers, etc. 10 | 11 | ## Parsing Usage: 12 | 13 | Sample Raw Web Request 14 | ``` 15 | var raw = " 16 | GET https://httpbin.org/get HTTP/1.1 17 | Host: httpbin.org 18 | Connection: keep-alive 19 | User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36 20 | Upgrade-Insecure-Requests: 1 21 | Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 22 | Accept-Encoding: gzip, deflate, br 23 | Accept-Language: en-US,en;q=0.9"; 24 | ``` 25 | 26 | #### IgnoreHttpParserOptions: 27 | You can specify what the parser should not serialize. Pass IgnoreHttpParserOptions as an optional parameter to the parser. For example, it might not make sense to parse the request body if you are replacing with it with another value. 28 | 29 | ``` 30 | IgnoreHttpParserOptions options = new IgnoreHttpParserOptions { IgnoreRequestBody = true }; 31 | ``` 32 | 33 | ### Parse to ParsedRequest object: 34 | ``` 35 | var parsed = Parser.ParseRawRequest(raw); 36 | ``` 37 | 38 | ## Build .NET HttpWebRequest Usage: 39 | 40 | ``` 41 | HttpWebRequest request = HttpWebRequestBuilder.InitializeWebRequest(parsed); 42 | ``` 43 | 44 | ## Invoking callback in InitializeWebRequest() 45 | Calling `request.GetRequestStream()` closes the request for adding headers, so unless you are positive you don't need to add any new headers after writing the request body, use the call back to defer this to your client 46 | 47 | ### What this looks like: 48 | ``` 49 | // build request 50 | HttpWebRequest request = ... build request 51 | 52 | // defer control back to the calling method 53 | callback?.Invoke(request); 54 | 55 | // add request body 56 | request.WritePostDataToRequestStream(requestBody); 57 | ``` 58 | 59 | ### Sample RequestBuilder invoking callback 60 | ``` 61 | void ClientBuildRequest() 62 | { 63 | // parse raw request... 64 | 65 | HttpWebRequest request = HttpWebRequestBuilder.InitializeWebRequest(parsed, AddMoreDynamicHeaders); 66 | 67 | // do stuff with the completed request 68 | } 69 | 70 | static void AddMoreDynamicHeaders(HttpWebRequest request) 71 | { 72 | // request.Headers.Add(...) 73 | } 74 | ``` 75 | 76 | ### Execute Web Request and capture response: 77 | ``` 78 | using (var httpWebResponse = request.GetResponse()) 79 | { 80 | httpWebResponse.GetParsedWebResponse(); 81 | } 82 | ``` 83 | 84 | ### ParsedResponse is a flattened version of HttpWebResponse 85 | ``` 86 | string ResponseText 87 | int StatusCode 88 | string StatusDescription 89 | string Cookies 90 | Uri ResponseUri 91 | Dictionary ResponseHeaders 92 | ``` 93 | 94 | 95 | ### Mocking Web Requests in unit tests with Moq 96 | 97 | ``` 98 | var response = new Mock(); 99 | response.Setup(s => s.GetParsedWebResponse()).Returns(new ParsedWebResponse { ResponseText = "Hello world" }); 100 | 101 | var request = new Mock(); 102 | request.Setup(c => c.GetResponse()).Returns(response.Object); 103 | 104 | var factory = new Mock(); 105 | factory.Setup(c => c.BuildRequest(It.IsAny())).Returns(request.Object); 106 | 107 | var parsed = Parser.ParseRawRequest("GET http://www.foo.com HTTP/1.1"); 108 | 109 | var result = factory.Object.BuildRequest(parsed).GetResponse(); 110 | 111 | Console.WriteLine(result); // "Hello world" 112 | ``` 113 | --------------------------------------------------------------------------------