├── global.json ├── pack.sh ├── pack.cmd ├── test.sh ├── tests ├── Extensions.cs ├── WebLinq.Tests.csproj ├── UriModuleTests.cs ├── TestTransport.cs ├── HttpConfigTests.cs └── SpawnTests.cs ├── test.cmd ├── .gitattributes ├── eg ├── Program.Main.cs └── WebLinq.Samples.csproj ├── README.md ├── src └── Core │ ├── AL.licenseheader │ ├── AssemblyInfo.cs │ ├── FileSystem.cs │ ├── Mapper.cs │ ├── HttpOptions.cs │ ├── Lazy.cs │ ├── TaskExtensions.cs │ ├── StringExtensions.cs │ ├── Xml │ └── XmlQuery.cs │ ├── Html │ ├── HtmlParser.cs │ ├── HtmlString.cs │ ├── HtmlFormControl.cs │ ├── HtmlObject.cs │ ├── HtmlQuery.cs │ ├── HapHtmlParser.cs │ ├── HtmlInputType.cs │ └── HtmlForm.cs │ ├── HttpRequestBuilder.cs │ ├── Zip │ ├── ZipQuery.cs │ └── Zip.cs │ ├── Collections │ ├── Map.cs │ └── MapBase.cs │ ├── Compatibility.cs │ ├── UriFormatProvider.cs │ ├── WebLinq.csproj │ ├── HttpHeaderCollection.cs │ ├── ArrayList.cs │ ├── Text │ └── TextQuery.cs │ ├── Xsv │ └── XsvQuery.cs │ ├── HashCodeCombiner.cs │ ├── FormatStringParser.cs │ ├── FormUrlEncodedContent.cs │ ├── LinqPadDumps.cs │ ├── StringBuilderCache.cs │ ├── HttpConfig.cs │ ├── ParsedValue.cs │ ├── Download.cs │ ├── HttpObservable.cs │ ├── HttpFetch.cs │ └── Sys │ ├── SpawnOptions.cs │ ├── Spawner.cs │ ├── PasteArguments.cs │ └── ProgramArguments.cs ├── lic └── corefx │ └── LICENSE.txt ├── .editorconfig ├── appveyor.yml ├── WebLinq.sln ├── .gitignore └── COPYING.txt /global.json: -------------------------------------------------------------------------------- 1 | { 2 | "sdk": { 3 | "version": "3.1.202" 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /pack.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | cd "$(dirname "$0")" 4 | if [ -n "$1" ]; then 5 | VERSION_SUFFIX="--version-suffix $1" 6 | else 7 | VERSION_SUFFIX= 8 | fi 9 | ./build.sh 10 | dotnet pack --no-restore --no-build -c Release $VERSION_SUFFIX 11 | -------------------------------------------------------------------------------- /pack.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | pushd "%~dp0" 3 | call :main %* 4 | popd 5 | goto :EOF 6 | 7 | :main 8 | setlocal 9 | set VERSION_SUFFIX= 10 | if not "%~1"=="" set VERSION_SUFFIX=--version-suffix %~1 11 | call build ^ 12 | && dotnet pack --no-build -c Release %VERSION_SUFFIX% 13 | goto :EOF 14 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | cd "$(dirname "$0")" 4 | ./build.sh 5 | dotnet test --no-build tests -c Debug -p:CollectCoverage=true \ 6 | -p:CoverletOutputFormat=opencover \ 7 | -p:Exclude=[NUnit*]* 8 | dotnet test --no-build tests -c Release 9 | 10 | -------------------------------------------------------------------------------- /tests/Extensions.cs: -------------------------------------------------------------------------------- 1 | namespace WebLinq.Tests 2 | { 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | 6 | static class Extensions 7 | { 8 | public static IEnumerable Except(this IEnumerable source, params T[] items) => 9 | source.Except(items.AsEnumerable()); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /test.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | pushd "%~dp0" 3 | call :main %* 4 | popd 5 | goto :EOF 6 | 7 | :main 8 | call build ^ 9 | && call :test Debug -p:CollectCoverage=true ^ 10 | -p:CoverletOutputFormat=opencover ^ 11 | -p:Exclude=[NUnit*]* ^ 12 | && call :test Release 13 | goto :EOF 14 | 15 | :test 16 | dotnet test --no-build tests -c %* 17 | goto :EOF 18 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | *.sh eol=lf 5 | 6 | # Custom for Visual Studio 7 | *.cs diff=csharp 8 | 9 | # Standard to msysgit 10 | *.doc diff=astextplain 11 | *.DOC diff=astextplain 12 | *.docx diff=astextplain 13 | *.DOCX diff=astextplain 14 | *.dot diff=astextplain 15 | *.DOT diff=astextplain 16 | *.pdf diff=astextplain 17 | *.PDF diff=astextplain 18 | *.rtf diff=astextplain 19 | *.RTF diff=astextplain 20 | -------------------------------------------------------------------------------- /eg/Program.Main.cs: -------------------------------------------------------------------------------- 1 | namespace WebLinq.Samples 2 | { 3 | using System; 4 | 5 | static partial class Program 6 | { 7 | static int Main(string[] args) 8 | { 9 | try 10 | { 11 | Wain(args); 12 | return 0; 13 | } 14 | catch (Exception e) 15 | { 16 | Console.Error.WriteLine(e); 17 | return 0xbad; 18 | } 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # WebLINQ 2 | 3 | [![Build Status][build-badge]][builds] 4 | [![NuGet][nuget-badge]][nuget-pkg] 5 | [![MyGet][myget-badge]][edge-pkgs] 6 | 7 | WebLINQ is a library for .NET to bring the web to LINQ. 8 | 9 | 10 | [build-badge]: https://img.shields.io/appveyor/ci/raboof/weblinq.svg 11 | [myget-badge]: https://img.shields.io/myget/raboof/v/WebLinq.svg?label=myget 12 | [edge-pkgs]: https://www.myget.org/feed/raboof/package/nuget/WebLinq 13 | [nuget-badge]: https://img.shields.io/nuget/v/WebLinq.svg 14 | [nuget-pkg]: https://www.nuget.org/packages/WebLinq 15 | [builds]: https://ci.appveyor.com/project/raboof/weblinq 16 | -------------------------------------------------------------------------------- /src/Core/AL.licenseheader: -------------------------------------------------------------------------------- 1 | extensions: designer.cs generated.cs g.cs .tt 2 | extensions: .cs 3 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | // 17 | #endregion 18 | -------------------------------------------------------------------------------- /src/Core/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | using System; 18 | using System.Runtime.InteropServices; 19 | 20 | [assembly: ComVisible(false)] 21 | [assembly: CLSCompliant(true)] 22 | [assembly: Guid("76fcf387-b33c-49b9-916f-f91862cb648c")] 23 | -------------------------------------------------------------------------------- /src/Core/FileSystem.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq 18 | { 19 | using System.IO; 20 | 21 | static class FileSystem 22 | { 23 | public static long GetFileSize(string path) => 24 | new FileInfo(path).Length; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /eg/WebLinq.Samples.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | netcoreapp3.1;netcoreapp2.1 6 | 8 7 | false 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /tests/WebLinq.Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp3.1;netcoreapp2.1 5 | 7.3 6 | false 7 | 8 | 9 | 10 | 11 | all 12 | runtime; build; native; contentfiles; analyzers 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /src/Core/Mapper.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq 18 | { 19 | using System; 20 | 21 | static class Mapper 22 | { 23 | public static TResult Map(this T input, Func mapper) 24 | { 25 | if (input == null) throw new ArgumentNullException(nameof(input)); 26 | if (mapper == null) throw new ArgumentNullException(nameof(mapper)); 27 | return mapper(input); 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /lic/corefx/LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) .NET Foundation and Contributors 4 | 5 | All rights reserved. 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy 8 | of this software and associated documentation files (the "Software"), to deal 9 | in the Software without restriction, including without limitation the rights 10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom the Software is 12 | furnished to do so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in all 15 | copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | SOFTWARE. 24 | -------------------------------------------------------------------------------- /tests/UriModuleTests.cs: -------------------------------------------------------------------------------- 1 | namespace WebLinq.Tests 2 | { 3 | using System; 4 | using NUnit.Framework; 5 | using Modules; 6 | 7 | [TestFixture] 8 | public class UriModuleTests 9 | { 10 | [Test] 11 | public void FormatUri() 12 | { 13 | var date = new DateTime(2007, 6, 29); 14 | var url = UriModule.FormatUri($@" 15 | http://www.example.com/ 16 | {date:yyyy}/ 17 | {date:MM}/ 18 | {date:dd}/ 19 | {{123_456_789}}/ 20 | {123_456_789}/ 21 | info.html 22 | ?h={"foo bar"} 23 | &date={date:MMM dd, yyyy}"); 24 | 25 | Assert.That(url.AbsoluteUri, 26 | Is.EqualTo("http://www.example.com/" 27 | + "2007/" 28 | + "06/" 29 | + "29/" 30 | + "%7B123_456_789%7D/" 31 | + "123456789/" 32 | + "info.html" 33 | + "?h=foo%20bar" 34 | + "&date=Jun%2029%2C%202007")); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/Core/HttpOptions.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq 18 | { 19 | public sealed class HttpOptions 20 | { 21 | public static readonly HttpOptions Default = new HttpOptions(returnErroneousFetch: false); 22 | 23 | public bool ReturnErroneousFetch { get; } 24 | 25 | HttpOptions(bool returnErroneousFetch) 26 | { 27 | ReturnErroneousFetch = returnErroneousFetch; 28 | } 29 | 30 | public HttpOptions WithReturnErroneousFetch(bool value) => 31 | ReturnErroneousFetch == value ? this : new HttpOptions(value); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/Core/Lazy.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq 18 | { 19 | using System; 20 | 21 | static class Lazy 22 | { 23 | public static TResult LazyGet(this T arg, 24 | ref (bool, TResult) storage, Func factory) 25 | { 26 | if (arg == null) throw new ArgumentNullException(nameof(arg)); 27 | if (factory == null) throw new ArgumentNullException(nameof(factory)); 28 | var (initialized, value) = storage; 29 | if (initialized) 30 | return value; 31 | (_, value) = storage = (true, factory(arg)); 32 | return value; 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/Core/TaskExtensions.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq 18 | { 19 | using System; 20 | using System.Runtime.CompilerServices; 21 | using System.Threading.Tasks; 22 | 23 | static class TaskExtensions 24 | { 25 | public static ConfiguredTaskAwaitable DontContinueOnCapturedContext(this Task task) 26 | { 27 | if (task == null) throw new ArgumentNullException(nameof(task)); 28 | return task.ConfigureAwait(continueOnCapturedContext: false); 29 | } 30 | 31 | public static ConfiguredTaskAwaitable DontContinueOnCapturedContext(this Task task) 32 | { 33 | if (task == null) throw new ArgumentNullException(nameof(task)); 34 | return task.ConfigureAwait(continueOnCapturedContext: false); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/Core/StringExtensions.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2019 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq 18 | { 19 | using System; 20 | 21 | static class StringExtensions 22 | { 23 | public static bool HasWhiteSpace(this string str) => 24 | HasWhiteSpace(str, 0, str.Length); 25 | 26 | public static bool HasWhiteSpace(this string str, int index, int length) 27 | { 28 | if (str == null) throw new ArgumentNullException(nameof(str)); 29 | if (index < 0 || index >= str.Length) throw new ArgumentOutOfRangeException(nameof(index), index, null); 30 | if (length < 0) throw new ArgumentOutOfRangeException(nameof(length), length, null); 31 | if (index + length > str.Length) throw new ArgumentOutOfRangeException(nameof(index), index, null); 32 | 33 | for (var i = index; i < length; i++) 34 | { 35 | if (char.IsWhiteSpace(str, i)) 36 | return true; 37 | } 38 | 39 | return false; 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/Core/Xml/XmlQuery.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq.Xml 18 | { 19 | using System; 20 | using System.Net.Http; 21 | using System.Reactive.Linq; 22 | using System.Xml.Linq; 23 | 24 | public static class XmlQuery 25 | { 26 | public static IObservable Xml(HttpContent content) => 27 | Xml(content, LoadOptions.None); 28 | 29 | public static IObservable Xml(HttpContent content, LoadOptions options) => 30 | from c in Observable.Return(content) 31 | from xml in c.ReadAsStringAsync() 32 | select XDocument.Parse(xml, options); 33 | 34 | public static IObservable> Xml(this IObservable> query) => 35 | query.Xml(LoadOptions.None); 36 | 37 | public static IObservable> Xml(this IObservable> query, LoadOptions options) => 38 | from e in query 39 | from xml in Xml(e.Content, options) 40 | select e.WithContent(xml); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/Core/Html/HtmlParser.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq.Html 18 | { 19 | using System; 20 | 21 | public interface IHtmlParser 22 | { 23 | ParsedHtml Parse(string html, Uri baseUrl); 24 | } 25 | 26 | public static class HtmlParser 27 | { 28 | public static IHtmlParser Default => new HapHtmlParser(); 29 | 30 | public static ParsedHtml Parse(this IHtmlParser parser, string html) => 31 | parser.Parse(html, null); 32 | 33 | public static IHtmlParser Wrap(this IHtmlParser parser, Func impl) => 34 | new DelegatingHtmlParser((html, baseUrl) => impl(parser, html, baseUrl)); 35 | 36 | sealed class DelegatingHtmlParser : IHtmlParser 37 | { 38 | readonly Func _parser; 39 | 40 | public DelegatingHtmlParser(Func parser) 41 | { 42 | _parser = parser; 43 | } 44 | 45 | public ParsedHtml Parse(string html, Uri baseUrl) => _parser(html, baseUrl); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/Core/HttpRequestBuilder.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq 18 | { 19 | using System; 20 | using System.Net.Http; 21 | 22 | static class SysNetHttpExtensions 23 | { 24 | public static HttpFetch ToHttpFetch(this HttpResponseMessage response, int id, IHttpClient http) 25 | { 26 | if (response == null) throw new ArgumentNullException(nameof(response)); 27 | var request = response.RequestMessage; 28 | return HttpFetch.Create(new HttpFetchInfo( 29 | id, http, 30 | response.Version, 31 | response.StatusCode, response.ReasonPhrase, 32 | HttpHeaderCollection.Empty.Set(response.Headers), 33 | HttpHeaderCollection.Empty.Set(response.Content.Headers), 34 | request.RequestUri, 35 | HttpHeaderCollection.Empty.Set(request.Headers)), 36 | response.Content); 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/Core/Html/HtmlString.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq.Html 18 | { 19 | using System; 20 | using System.Net; 21 | 22 | public struct HtmlString : IEquatable 23 | { 24 | string _encoded; 25 | string _decoded; 26 | 27 | public static HtmlString FromEncoded(string encoded) => new HtmlString(encoded: encoded, decoded: null); 28 | public static HtmlString FromDecoded(string decoded) => new HtmlString(decoded: decoded, encoded: null); 29 | 30 | HtmlString(string encoded, string decoded) 31 | { 32 | _encoded = encoded; 33 | _decoded = decoded; 34 | } 35 | 36 | public string Decoded => _decoded ?? (_encoded == null ? null : (_decoded = WebUtility.HtmlDecode(_encoded))); 37 | public string Encoded => _encoded ?? (_decoded == null ? null : (_encoded = WebUtility.HtmlEncode(_decoded))); 38 | 39 | public bool Equals(HtmlString other) => string.Equals(Decoded, other.Decoded); 40 | public override bool Equals(object obj) => obj is HtmlString s && Equals(s); 41 | public override int GetHashCode() => Decoded?.GetHashCode() ?? 0; 42 | public override string ToString() => Encoded; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org/ 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | insert_final_newline = true 8 | trim_trailing_whitespace = true 9 | 10 | [*.{csproj,vbproj,vcxproj,vcxproj.filters,proj,projitems,shproj}] 11 | indent_size = 2 12 | 13 | [*.{sln}] 14 | indent_style = tab 15 | 16 | [*.{json,yml}] 17 | indent_size = 2 18 | 19 | [*.{cs,tt}] 20 | charset = utf-8 21 | indent_style = space 22 | indent_size = 4 23 | max_line_length = 100 24 | 25 | [*.cs] 26 | # Prefer "var" everywhere 27 | csharp_style_var_for_built_in_types = true:suggestion 28 | csharp_style_var_when_type_is_apparent = true:suggestion 29 | csharp_style_var_elsewhere = true:suggestion 30 | 31 | # Prefer method-like constructs to have a block body 32 | csharp_style_expression_bodied_methods = false:none 33 | csharp_style_expression_bodied_constructors = false:none 34 | csharp_style_expression_bodied_operators = false:none 35 | 36 | # Prefer property-like constructs to have an expression-body 37 | csharp_style_expression_bodied_properties = true:none 38 | csharp_style_expression_bodied_indexers = true:none 39 | csharp_style_expression_bodied_accessors = true:none 40 | 41 | # Suggest more modern language features when available 42 | csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion 43 | csharp_style_pattern_matching_over_as_with_null_check = true:suggestion 44 | csharp_style_inlined_variable_declaration = true:suggestion 45 | csharp_style_throw_expression = true:suggestion 46 | csharp_style_conditional_delegate_call = true:suggestion 47 | csharp_prefer_simple_default_expression = true:suggestion 48 | 49 | # Spacing 50 | csharp_space_after_cast = true 51 | csharp_space_after_keywords_in_control_flow_statements = true 52 | csharp_space_between_method_declaration_parameter_list_parentheses = false 53 | 54 | # Wrapping 55 | csharp_preserve_single_line_statements = true 56 | csharp_preserve_single_line_blocks = true 57 | -------------------------------------------------------------------------------- /src/Core/Zip/ZipQuery.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq.Zip 18 | { 19 | using System; 20 | using System.Net.Http; 21 | using System.Reactive.Linq; 22 | 23 | public static class ZipQuery 24 | { 25 | [Obsolete("Use " + nameof(HttpObservable.Download) 26 | + " or " + nameof(HttpObservable.DownloadTemp) 27 | + " with " + nameof(AsZip) + " instead.")] 28 | public static IObservable> DownloadZip(this IHttpObservable query) => 29 | query.DownloadTemp("zip").AsZip(); 30 | 31 | [Obsolete("Use " + nameof(HttpObservable.Download) 32 | + " or " + nameof(HttpObservable.DownloadTemp) 33 | + " with " + nameof(AsZip) + " instead.")] 34 | public static IObservable> DownloadZip(this IObservable> query) => 35 | from fetch in query.DownloadTemp("zip") 36 | select fetch.WithContent(new Zip(fetch.Content.Path)); 37 | 38 | public static IObservable> AsZip(this IObservable> query) => 39 | from fetch in query 40 | select fetch.WithContent(new Zip(fetch.Content.Path)); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/Core/Collections/Map.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq.Collections 18 | { 19 | using System.Collections.Generic; 20 | 21 | public sealed class Map : MapBase 22 | { 23 | public static Map Empty = new Map(null); 24 | 25 | readonly Map _link; 26 | 27 | public Map(IEqualityComparer comparer) : 28 | base(comparer) {} 29 | 30 | public Map(Map link, TKey key, TValue value, IEqualityComparer comparer) : 31 | base(key, value, comparer ?? link?.Comparer) { _link = link; } 32 | 33 | public Map WithComparer(IEqualityComparer comparer) => 34 | new Map(_link, Key, Value, comparer); 35 | 36 | public Map Set(KeyValuePair pair) => 37 | Set(pair.Key, pair.Value); 38 | 39 | public Map Set(TKey key, TValue value) => 40 | new Map(this, key, value, null); 41 | 42 | public Map Remove(TKey key) => 43 | RemoveCore(Empty, key, (map, k, v) => map.Set(k, v)); 44 | 45 | public override bool IsEmpty => _link == null; 46 | 47 | protected override IEnumerable> Nodes => 48 | GetNodesCore(this, m => m._link); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/Core/Html/HtmlFormControl.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq.Html 18 | { 19 | public enum HtmlControlType { Input, Select, TextArea } 20 | 21 | public sealed partial class HtmlFormControl 22 | { 23 | bool? _isDisabled; 24 | bool? _isReadOnly; 25 | bool? _isChecked; 26 | bool? _isMultiple; 27 | 28 | public HtmlForm Form { get; } 29 | public HtmlObject Element { get; } 30 | public string Name { get; } 31 | public HtmlControlType ControlType { get; } 32 | public HtmlInputType InputType { get; } 33 | public bool IsDisabled => (_isDisabled ?? (_isDisabled = Element.IsAttributeFlagged("disabled"))) == true; 34 | public bool IsReadOnly => (_isReadOnly ?? (_isReadOnly = Element.IsAttributeFlagged("readonly"))) == true; 35 | public bool IsChecked => (_isChecked ?? (_isChecked = Element.IsAttributeFlagged("checked" ))) == true; 36 | public bool IsMultiple => (_isMultiple ?? (_isMultiple = Element.IsAttributeFlagged("multiple"))) == true; 37 | 38 | internal HtmlFormControl(HtmlForm form, HtmlObject element, string name, HtmlControlType controlType, HtmlInputType inputType) 39 | { 40 | Form = form; 41 | Element = element; 42 | Name = name; 43 | ControlType = controlType; 44 | InputType = inputType; 45 | } 46 | 47 | public override string ToString() => Element.ToString(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | version: '{build}' 2 | image: 3 | - Visual Studio 2019 4 | - Ubuntu 5 | stack: node 6 6 | skip_commits: 7 | files: 8 | - '*.md' 9 | - '*.txt' 10 | environment: 11 | DOTNET_CLI_TELEMETRY_OPTOUT: true 12 | DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true 13 | install: 14 | - ps: if ($isWindows) { Install-Product node 6 } 15 | - cmd: npm install -g eclint 16 | - cmd: git rm .editorconfig 17 | - cmd: eclint check -n "**/*.{cs,tt,cmd,sh,md,txt,yml}" 18 | - cmd: eclint check -w "**/*.{cs,tt,cmd,sh,md,txt,yml,json,sln,csproj,shfbproj}" 19 | - cmd: git reset --hard 20 | - cmd: curl -OsSL https://dot.net/v1/dotnet-install.ps1 21 | - ps: if ($isWindows) { ./dotnet-install.ps1 -JsonFile global.json } 22 | - ps: if ($isWindows) { ./dotnet-install.ps1 -Runtime dotnet -Version 2.1.30 -SkipNonVersionedFiles } 23 | - sh: curl -OsSL https://dot.net/v1/dotnet-install.sh 24 | - sh: chmod +x dotnet-install.sh 25 | - sh: ./dotnet-install.sh --jsonfile global.json 26 | - sh: ./dotnet-install.sh --runtime dotnet --version 2.1.30 --skip-non-versioned-files 27 | - sh: export PATH="$HOME/.dotnet:$PATH" 28 | skip_tags: true 29 | before_build: 30 | - dotnet --info 31 | build_script: 32 | - ps: | 33 | $id = ([datetimeoffset]$env:APPVEYOR_REPO_COMMIT_TIMESTAMP).ToUniversalTime().ToString('yyyyMMdd''t''HHmm') 34 | if ($isWindows) { 35 | cmd /c call .\pack.cmd ci-$id $env:APPVEYOR_REPO_COMMIT 36 | } else { 37 | ./pack.sh ci-$id $env:APPVEYOR_REPO_COMMIT 38 | } 39 | test_script: 40 | - cmd: test.cmd 41 | - ps: if ($isWindows) { Invoke-WebRequest -Uri https://uploader.codecov.io/latest/windows/codecov.exe -Outfile codecov.exe } 42 | - cmd: codecov.exe 43 | - sh: ./test.sh 44 | artifacts: 45 | - path: dist\*.nupkg 46 | deploy: 47 | - provider: NuGet 48 | server: https://www.myget.org/F/raboof/api/v2/package 49 | api_key: 50 | secure: fhGwXyO35FSshRzs5GWmF1LJTrd1sIqmS/jNCSfO2LfOciuYAKiXuFMYZFGiTAl+ 51 | symbol_server: https://www.myget.org/F/raboof/symbols/api/v2/package 52 | on: 53 | branch: master 54 | notifications: 55 | - provider: Email 56 | to: 57 | - raboof-ci@googlegroups.com 58 | on_build_success: true 59 | on_build_failure: true 60 | on_build_status_changed: false 61 | -------------------------------------------------------------------------------- /src/Core/Compatibility.cs: -------------------------------------------------------------------------------- 1 | namespace WebLinq 2 | { 3 | using System; 4 | using System.Data; 5 | using System.Linq; 6 | using Dsv; 7 | using Mannex; 8 | 9 | static class Compatibility 10 | { 11 | public static DataTable ParseXsvAsDataTable(this string xsv, string delimiter, bool quoted, 12 | params DataColumn[] columns) 13 | { 14 | var table = new DataTable(); 15 | var lines = xsv.SplitIntoLines(); 16 | var format = new Format(delimiter[0]).WithQuote(quoted ? '"' : (char?) null); 17 | 18 | if (columns.Length > 0) 19 | { 20 | table.Columns.AddRange(columns); 21 | 22 | var rows = 23 | from e in 24 | lines.ParseDsv(format, 25 | hr => columns.Select(c => hr.FindFirstIndex(c.ColumnName, StringComparison.OrdinalIgnoreCase)) 26 | .ToArray()) 27 | select 28 | from i in e.Header 29 | select i is int n && n < e.Row.Count ? (object) e.Row[n] : DBNull.Value; 30 | 31 | foreach (var row in rows) 32 | table.Rows.Add(row.ToArray()); 33 | } 34 | else 35 | { 36 | foreach (var row in lines.ParseDsv(format)) 37 | { 38 | if (row.LineNumber == 1) 39 | { 40 | foreach (var e in row) 41 | table.Columns.Add(new DataColumn(e)); 42 | } 43 | else 44 | { 45 | var newRow = table.NewRow(); 46 | var count = Math.Min(row.Count, table.Columns.Count); 47 | for (var i = 0; i < count; i++) 48 | newRow[i] = row[i]; 49 | table.Rows.Add(newRow); 50 | } 51 | } 52 | } 53 | 54 | table.AcceptChanges(); 55 | return table; 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/Core/UriFormatProvider.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2019 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq 18 | { 19 | using System; 20 | using System.Globalization; 21 | 22 | sealed class UriFormatProvider : IFormatProvider 23 | { 24 | public static readonly IFormatProvider InvariantCulture = 25 | new UriFormatProvider(CultureInfo.InvariantCulture); 26 | 27 | readonly IFormatProvider _baseProvider; 28 | 29 | public UriFormatProvider() : 30 | this(null) {} 31 | 32 | public UriFormatProvider(IFormatProvider baseProvider) => 33 | _baseProvider = baseProvider; 34 | 35 | public object GetFormat(Type formatType) 36 | => formatType == null ? throw new ArgumentNullException(nameof(formatType)) 37 | : formatType == typeof(ICustomFormatter) ? UriFormatter.Default 38 | : _baseProvider != null ? _baseProvider.GetFormat(formatType) 39 | : CultureInfo.CurrentCulture.GetFormat(formatType); 40 | 41 | sealed class UriFormatter : ICustomFormatter 42 | { 43 | public static readonly UriFormatter Default = new UriFormatter(); 44 | 45 | UriFormatter() {} 46 | 47 | public string Format(string format, object arg, IFormatProvider formatProvider) 48 | => arg == null ? string.Empty 49 | : Uri.EscapeDataString(arg is IFormattable formattable 50 | ? formattable.ToString(format, formatProvider) 51 | : arg.ToString()); 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/Core/WebLinq.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | LINQ to Web 5 | netstandard2.0 6 | 7.3 7 | 1.0.0 8 | Library enabling web scraping over LINQ 9 | Copyright (c) 2016 Atif Aziz. All rights reserved. Portions Copyright (c) .NET Foundation and Contributors. 10 | Atif Aziz 11 | Atif Aziz 12 | 13 | https://github.com/weblinq/WebLinq 14 | true 15 | linq;web;www;query;scraping;scrape;scraper 16 | COPYING.txt 17 | ..\..\dist 18 | true 19 | true 20 | $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb 21 | 22 | 23 | 24 | ..\..\bin\Debug\ 25 | 26 | 27 | ..\..\bin\Release\ 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /src/Core/HttpHeaderCollection.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq 18 | { 19 | #region Imports 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Diagnostics; 24 | using System.Linq; 25 | using System.Net.Http.Headers; 26 | using Collections; 27 | 28 | #endregion 29 | 30 | [DebuggerDisplay("Count = {Count}")] 31 | public sealed class HttpHeaderCollection : MapBase 32 | { 33 | public static readonly HttpHeaderCollection Empty = new HttpHeaderCollection(); 34 | 35 | readonly HttpHeaderCollection _link; 36 | 37 | public HttpHeaderCollection() : 38 | base(StringComparer.OrdinalIgnoreCase) {} 39 | 40 | HttpHeaderCollection(HttpHeaderCollection link, string key, Strings values) : 41 | base(key, values, link.Comparer) 42 | { 43 | if (key == null) throw new ArgumentNullException(nameof(key)); 44 | _link = link; 45 | } 46 | 47 | public HttpHeaderCollection Set(string key, Strings values) => 48 | new HttpHeaderCollection(this, key, values); 49 | 50 | internal HttpHeaderCollection Set(HttpHeaders headers) => 51 | headers.Aggregate(this, (h, e) => h.Set(e.Key, Strings.Sequence(e.Value))); 52 | 53 | public HttpHeaderCollection Remove(string key) => 54 | RemoveCore(Empty, key, (hs, k, vs) => hs.Set(k, vs)); 55 | 56 | public override bool IsEmpty => _link == null; 57 | 58 | protected override IEnumerable> Nodes => 59 | GetNodesCore(this, h => h._link); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/Core/ArrayList.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq 18 | { 19 | using System; 20 | using System.Linq; 21 | 22 | struct ArrayList 23 | { 24 | static readonly T[] Zero = new T[0]; 25 | 26 | T[] _items; 27 | 28 | public int Count { get; private set; } 29 | 30 | public void EnsureCapacity(int capacity) 31 | { 32 | if (capacity <= (Capacity ?? 0)) 33 | return; 34 | Array.Resize(ref _items, TwoPowers.RoundUpToClosest(capacity)); 35 | } 36 | 37 | public T this[int index] 38 | { 39 | get { return _items[index]; } 40 | set 41 | { 42 | EnsureCapacity(index + 1); 43 | _items[index] = value; 44 | Count = Math.Max(Count, index + 1); 45 | } 46 | } 47 | 48 | public T this[int index, T defaultValue] => 49 | index < Count ? this[index] : defaultValue; 50 | 51 | int? Capacity => _items?.Length; 52 | 53 | public T[] ToArray() 54 | { 55 | if (Count == 0) 56 | return Zero; 57 | Array.Resize(ref _items, Count); 58 | return _items; 59 | } 60 | } 61 | 62 | static class TwoPowers 63 | { 64 | static readonly int[] Cache = Enumerable.Range(0, 31).Select(p => 1 << p).ToArray(); 65 | 66 | public static int RoundUpToClosest(int x) 67 | { 68 | if (x < 0 || x > Cache[Cache.Length - 1]) 69 | throw new ArgumentOutOfRangeException(nameof(x), x, null); 70 | var i = Array.BinarySearch(Cache, x); 71 | return Cache[i >= 0 ? i : ~i]; 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/Core/Text/TextQuery.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq.Text 18 | { 19 | using System; 20 | using System.IO; 21 | using System.Net.Http; 22 | using System.Reactive.Linq; 23 | using System.Text; 24 | using Mannex.Collections.Generic; 25 | 26 | public static class TextQuery 27 | { 28 | public static IObservable Delimited(this IObservable query, string delimiter) => 29 | query.Select((e, i) => i.AsKeyTo(e)) 30 | .Aggregate( 31 | new StringBuilder(), 32 | (sb, e) => sb.Append(e.Key > 0 ? delimiter : null).Append(e.Value), 33 | sb => sb.ToString()); 34 | 35 | public static IObservable> Text(this IHttpObservable query) => 36 | query.ReadContent(f => f.Content.ReadAsStringAsync()); 37 | 38 | public static IObservable> Text(this IHttpObservable query, Encoding encoding) => 39 | query.ReadContent(async f => 40 | { 41 | using (var stream = await f.Content.ReadAsStreamAsync().DontContinueOnCapturedContext()) 42 | using (var reader = new StreamReader(stream, encoding)) 43 | return await reader.ReadToEndAsync().DontContinueOnCapturedContext(); 44 | }); 45 | 46 | public static IObservable> Text(this IObservable> query) => 47 | from fetch in query 48 | from text in fetch.Content.ReadAsStringAsync() 49 | select fetch.WithContent(text); 50 | 51 | public static IObservable> Text(this IObservable> query, Encoding encoding) => 52 | from fetch in query 53 | from bytes in fetch.Content.ReadAsByteArrayAsync() 54 | select fetch.WithContent(encoding.GetString(bytes)); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /WebLinq.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.27703.2035 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebLinq.Samples", "eg\WebLinq.Samples.csproj", "{B4C1F88A-ED2D-4CF1-807F-2B9C13D5C7FE}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebLinq", "src\Core\WebLinq.csproj", "{76FCF387-B33C-49B9-916F-F91862CB648C}" 9 | EndProject 10 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{F1FAE8C9-0AA0-471E-974F-A257ABB2F3E8}" 11 | ProjectSection(SolutionItems) = preProject 12 | .editorconfig = .editorconfig 13 | appveyor.yml = appveyor.yml 14 | build.cmd = build.cmd 15 | build.sh = build.sh 16 | COPYING.txt = COPYING.txt 17 | global.json = global.json 18 | pack.cmd = pack.cmd 19 | pack.sh = pack.sh 20 | README.md = README.md 21 | test.cmd = test.cmd 22 | test.sh = test.sh 23 | EndProjectSection 24 | EndProject 25 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebLinq.Tests", "tests\WebLinq.Tests.csproj", "{00CDA73A-CA20-4AEB-A29B-CEF37876941D}" 26 | EndProject 27 | Global 28 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 29 | Debug|Any CPU = Debug|Any CPU 30 | Release|Any CPU = Release|Any CPU 31 | EndGlobalSection 32 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 33 | {B4C1F88A-ED2D-4CF1-807F-2B9C13D5C7FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 34 | {B4C1F88A-ED2D-4CF1-807F-2B9C13D5C7FE}.Debug|Any CPU.Build.0 = Debug|Any CPU 35 | {B4C1F88A-ED2D-4CF1-807F-2B9C13D5C7FE}.Release|Any CPU.ActiveCfg = Release|Any CPU 36 | {B4C1F88A-ED2D-4CF1-807F-2B9C13D5C7FE}.Release|Any CPU.Build.0 = Release|Any CPU 37 | {76FCF387-B33C-49B9-916F-F91862CB648C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 38 | {76FCF387-B33C-49B9-916F-F91862CB648C}.Debug|Any CPU.Build.0 = Debug|Any CPU 39 | {76FCF387-B33C-49B9-916F-F91862CB648C}.Release|Any CPU.ActiveCfg = Release|Any CPU 40 | {76FCF387-B33C-49B9-916F-F91862CB648C}.Release|Any CPU.Build.0 = Release|Any CPU 41 | {00CDA73A-CA20-4AEB-A29B-CEF37876941D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 42 | {00CDA73A-CA20-4AEB-A29B-CEF37876941D}.Debug|Any CPU.Build.0 = Debug|Any CPU 43 | {00CDA73A-CA20-4AEB-A29B-CEF37876941D}.Release|Any CPU.ActiveCfg = Release|Any CPU 44 | {00CDA73A-CA20-4AEB-A29B-CEF37876941D}.Release|Any CPU.Build.0 = Release|Any CPU 45 | EndGlobalSection 46 | GlobalSection(SolutionProperties) = preSolution 47 | HideSolutionNode = FALSE 48 | EndGlobalSection 49 | GlobalSection(ExtensibilityGlobals) = postSolution 50 | SolutionGuid = {4C6B3CD7-B266-43BB-AD41-92C6B58842F2} 51 | EndGlobalSection 52 | EndGlobal 53 | -------------------------------------------------------------------------------- /src/Core/Xsv/XsvQuery.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq.Xsv 18 | { 19 | using System; 20 | using System.Data; 21 | using System.Net.Http; 22 | using System.Reactive.Linq; 23 | using Text; 24 | 25 | public static class XsvQuery 26 | { 27 | public static IObservable> XsvToDataTable(this IHttpObservable query, string delimiter, bool quoted, params DataColumn[] columns) => 28 | from f in query.Text() 29 | from csv in XsvToDataTable(f.Content, delimiter, quoted, columns) 30 | select f.WithContent(csv); 31 | 32 | public static IObservable> XsvToDataTable(this IObservable> query, string delimiter, bool quoted, params DataColumn[] columns) => 33 | from fetch in query.Text() 34 | select fetch.WithContent(fetch.Content.ParseXsvAsDataTable(delimiter, quoted, columns)); 35 | 36 | public static IObservable XsvToDataTable(this IObservable query, string delimiter, bool quoted, params DataColumn[] columns) => 37 | from xsv in query 38 | select xsv.ParseXsvAsDataTable(delimiter, quoted, columns); 39 | 40 | // TODO return plain DataTable 41 | public static IObservable XsvToDataTable(string text, string delimiter, bool quoted, params DataColumn[] columns) 42 | { 43 | DataTable dt = null; 44 | return Observable.Defer(() => Observable.Return(dt ?? (dt = text.ParseXsvAsDataTable(delimiter, quoted, columns)))); 45 | } 46 | 47 | public static IObservable> CsvToDataTable(this IHttpObservable query, params DataColumn[] columns) => 48 | query.XsvToDataTable(",", true, columns); 49 | 50 | public static IObservable> CsvToDataTable(this IObservable> query, params DataColumn[] columns) => 51 | query.XsvToDataTable(",", true, columns); 52 | 53 | public static IObservable CsvToDataTable(this IObservable query, params DataColumn[] columns) => 54 | query.XsvToDataTable(",", true, columns); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/Core/HashCodeCombiner.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using System.Collections; 5 | using System.Collections.Generic; 6 | using System.Runtime.CompilerServices; 7 | 8 | // ReSharper disable once CheckNamespace 9 | 10 | namespace Microsoft.Extensions.Internal 11 | { 12 | // Source: 13 | // https://github.com/aspnet/Common/blob/6ea261d07b013a93e8200d55668343dd7d6b305d/src/Microsoft.Extensions.HashCodeCombiner.Sources/HashCodeCombiner.cs 14 | 15 | internal struct HashCodeCombiner 16 | { 17 | private long _combinedHash64; 18 | 19 | public int CombinedHash 20 | { 21 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 22 | get { return _combinedHash64.GetHashCode(); } 23 | } 24 | 25 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 26 | HashCodeCombiner(long seed) 27 | { 28 | _combinedHash64 = seed; 29 | } 30 | 31 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 32 | public void Add(IEnumerable e) 33 | { 34 | if (e == null) 35 | { 36 | Add(0); 37 | } 38 | else 39 | { 40 | var count = 0; 41 | foreach (object o in e) 42 | { 43 | Add(o); 44 | count++; 45 | } 46 | Add(count); 47 | } 48 | } 49 | 50 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 51 | public static implicit operator int(HashCodeCombiner self) 52 | { 53 | return self.CombinedHash; 54 | } 55 | 56 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 57 | public void Add(int i) 58 | { 59 | _combinedHash64 = ((_combinedHash64 << 5) + _combinedHash64) ^ i; 60 | } 61 | 62 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 63 | public void Add(string s) 64 | { 65 | var hashCode = (s != null) ? s.GetHashCode() : 0; 66 | Add(hashCode); 67 | } 68 | 69 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 70 | public void Add(object o) 71 | { 72 | var hashCode = (o != null) ? o.GetHashCode() : 0; 73 | Add(hashCode); 74 | } 75 | 76 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 77 | public void Add(TValue value, IEqualityComparer comparer) 78 | { 79 | var hashCode = value != null ? comparer.GetHashCode(value) : 0; 80 | Add(hashCode); 81 | } 82 | 83 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 84 | public static HashCodeCombiner Start() 85 | { 86 | return new HashCodeCombiner(0x1505L); 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/Core/Html/HtmlObject.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | //[assembly: System.Runtime.CompilerServices.InternalsVisibleTo("LINQPadQeury")] 18 | 19 | namespace WebLinq.Html 20 | { 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | 25 | public abstract partial class HtmlObject 26 | { 27 | public abstract ParsedHtml Owner { get; } 28 | public abstract string Name { get; } 29 | public bool IsNamed(string name) => Name.Equals(name, StringComparison.OrdinalIgnoreCase); 30 | public virtual bool HasAttributes => AttributeNames.Any(); 31 | public abstract IEnumerable AttributeNames { get; } 32 | public abstract bool HasAttribute(string name); 33 | public virtual string GetAttributeValue(string name) => GetAttributeSourceValue(name).Decoded; 34 | public virtual bool AttributeValueEquals(string name, string value) => string.Equals(GetAttributeValue(name)?.Trim(), value, StringComparison.OrdinalIgnoreCase); 35 | public virtual bool IsAttributeFlagged(string name) => AttributeValueEquals(name, name); 36 | public abstract HtmlString GetAttributeSourceValue(string name); 37 | public abstract string OuterHtml { get; } 38 | public abstract string InnerHtml { get; } 39 | public virtual string InnerText => InnerTextSource.Decoded; 40 | public abstract HtmlString InnerTextSource { get; } 41 | public virtual bool HasChildElements => ChildElements.Any(); 42 | public abstract HtmlObject ParentElement { get; } 43 | public abstract IEnumerable ChildElements { get; } 44 | public override string ToString() => OuterHtml; 45 | 46 | public virtual IEnumerable QuerySelectorAll(string selector) => 47 | Owner.QuerySelectorAll(selector, this); 48 | 49 | public virtual HtmlObject QuerySelector(string selector) => 50 | Owner.QuerySelector(selector, this); 51 | 52 | string _class; 53 | HashSet _classes; 54 | HashSet Classes => _classes ?? (_classes = new HashSet(Class.Split(' '))); 55 | 56 | public string Class => _class ?? (_class = GetAttributeValue("class") ?? string.Empty); 57 | public bool HasClass(string className) => Classes.Contains(className); 58 | 59 | public virtual string NormalInnerText 60 | => string.IsNullOrEmpty(InnerText) 61 | ? InnerText 62 | : string.Join(" ", InnerText.Split((char[])null, StringSplitOptions.RemoveEmptyEntries)); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/Core/FormatStringParser.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2019 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq 18 | { 19 | using System; 20 | using System.Collections.Generic; 21 | 22 | static class FormatStringParser 23 | { 24 | public static IEnumerable Parse(string format, 25 | Func textSelector, 26 | Func formatItemSelector) 27 | { 28 | if (format == null) throw new ArgumentNullException(nameof(format)); 29 | if (textSelector == null) throw new ArgumentNullException(nameof(textSelector)); 30 | if (formatItemSelector == null) throw new ArgumentNullException(nameof(formatItemSelector)); 31 | 32 | var si = 0; 33 | var inFormatItem = false; 34 | for (var i = 0; i < format.Length; i++) 35 | { 36 | var ch = format[i]; 37 | if (inFormatItem) 38 | { 39 | if (ch == '}') 40 | { 41 | yield return formatItemSelector(format, si, i - si + 1); 42 | si = i + 1; 43 | inFormatItem = false; 44 | } 45 | } 46 | else if (ch == '{') 47 | { 48 | bool eoi; 49 | if ((eoi = i + 1 == format.Length) || format[i + 1] != '{') 50 | { 51 | var len = i - si; 52 | if (len > 0) 53 | yield return textSelector(format, si, len); 54 | if (eoi) 55 | throw new FormatException($"Missing close delimiter '}}' in format string for format item starting with '{{' at offset {i}."); 56 | si = i; 57 | inFormatItem = true; 58 | } 59 | else 60 | { 61 | i++; 62 | } 63 | } 64 | else if (ch == '}') 65 | { 66 | if (i + 1 == format.Length || format[i + 1] != '}') 67 | throw new FormatException($"Character '}}' at offset {i} must be escaped (by doubling) in a format string."); 68 | i++; 69 | } 70 | } 71 | 72 | if (inFormatItem) 73 | throw new FormatException($"Missing close delimiter '}}' in format string for format item starting with '{{' at offset {si}."); 74 | 75 | if (si < format.Length) 76 | yield return textSelector(format, si, format.Length - si); 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /tests/TestTransport.cs: -------------------------------------------------------------------------------- 1 | namespace WebLinq.Tests 2 | { 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Net; 6 | using System.Net.Http; 7 | using System.Text; 8 | using System.Threading.Tasks; 9 | using Modules; 10 | 11 | sealed class TestTransport 12 | { 13 | readonly Queue _responses; 14 | readonly Queue _requests; 15 | readonly Queue _requestConfigs; 16 | 17 | public TestTransport(params HttpResponseMessage[] responses) : 18 | this(HttpConfig.Default, responses) {} 19 | 20 | public TestTransport(HttpConfig config, params HttpResponseMessage[] responses) 21 | { 22 | _responses = new Queue(responses); 23 | _requests = new Queue(); 24 | _requestConfigs = new Queue(); 25 | 26 | Http = HttpModule.Http.WithConfig(config) 27 | .Wrap((_, req, cfg) => SendAsync(req, cfg)); 28 | } 29 | 30 | public TestTransport Enqueue(HttpResponseMessage response) 31 | { 32 | _responses.Enqueue(response); 33 | return this; 34 | } 35 | 36 | public TestTransport EnqueueHtml(string html, HttpStatusCode statusCode = HttpStatusCode.OK) => 37 | Enqueue(new HttpResponseMessage 38 | { 39 | StatusCode = statusCode, 40 | Content = new StringContent(html, Encoding.UTF8, "text/html") 41 | }); 42 | 43 | public TestTransport EnqueueText(string text, HttpStatusCode statusCode = HttpStatusCode.OK) => 44 | Enqueue(new HttpResponseMessage 45 | { 46 | StatusCode = statusCode, 47 | Content = new StringContent(text, Encoding.UTF8, "text/plain") 48 | }); 49 | 50 | public TestTransport EnqueueJson(string json, HttpStatusCode statusCode = HttpStatusCode.OK) => 51 | Enqueue(new HttpResponseMessage 52 | { 53 | StatusCode = statusCode, 54 | Content = new StringContent(json, Encoding.UTF8, "application/json") 55 | }); 56 | 57 | public TestTransport Enqueue(byte[] bytes, HttpStatusCode statusCode = HttpStatusCode.OK) => 58 | Enqueue(new HttpResponseMessage 59 | { 60 | StatusCode = statusCode, 61 | Content = new ByteArrayContent(bytes) 62 | }); 63 | 64 | public HttpRequestMessage DequeueRequestMessage() => 65 | DequeueRequest((rm, _) => rm); 66 | 67 | public T DequeueRequest(Func selector) 68 | { 69 | var config = _requestConfigs.Dequeue(); 70 | var request = _requests.Dequeue(); 71 | return selector(request, config); 72 | } 73 | 74 | public IHttpClient Http { get; } 75 | 76 | Task SendAsync(HttpRequestMessage request, HttpConfig config) 77 | { 78 | _requestConfigs.Enqueue(config); 79 | _requests.Enqueue(request); 80 | var response = _responses.Dequeue(); 81 | response.RequestMessage = request; 82 | return Task.FromResult(response); 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/Core/FormUrlEncodedContent.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) .NET Foundation and Contributors. All rights reserved. 2 | // 3 | // 4 | // The MIT License (MIT) 5 | // 6 | // Permission is hereby granted, free of charge, to any person obtaining a copy 7 | // of this software and associated documentation files (the "Software"), to deal 8 | // in the Software without restriction, including without limitation the rights 9 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | // copies of the Software, and to permit persons to whom the Software is 11 | // furnished to do so, subject to the following conditions: 12 | // 13 | // The above copyright notice and this permission notice shall be included in all 14 | // copies or substantial portions of the Software. 15 | // 16 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | // SOFTWARE. 23 | // 24 | // https://github.com/dotnet/corefx/blob/e0ba7aa8026280ee3571179cc06431baf1dfaaac/LICENSE.TXT 25 | // 26 | #endregion 27 | 28 | namespace WebLinq 29 | { 30 | using System; 31 | using System.Collections.Generic; 32 | using System.Text; 33 | using System.Net.Http.Headers; 34 | using System.Net; 35 | using System.Net.Http; 36 | 37 | // Inspiration & credit: 38 | // https://github.com/dotnet/corefx/blob/e0ba7aa8026280ee3571179cc06431baf1dfaaac/src/System.Net.Http/src/System/Net/Http/FormUrlEncodedContent.cs 39 | // 40 | // 41 | // - System.System.Net.Http.FormUrlEncodedContent(...) can't handle very long parameters/values 42 | // https://github.com/dotnet/corefx/issues/1936 43 | 44 | sealed class FormUrlEncodedContent : ByteArrayContent 45 | { 46 | public FormUrlEncodedContent(IEnumerable> nameValueCollection) : 47 | base(GetContentByteArray(nameValueCollection)) => 48 | Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded"); 49 | 50 | static readonly Encoding DefaultHttpEncoding = Encoding.GetEncoding("iso-8859-1"); 51 | 52 | static byte[] GetContentByteArray(IEnumerable> nameValueCollection) 53 | { 54 | if (nameValueCollection == null) 55 | throw new ArgumentNullException(nameof(nameValueCollection)); 56 | 57 | // Encode and concatenate data 58 | var builder = new StringBuilder(); 59 | foreach (var pair in nameValueCollection) 60 | { 61 | if (builder.Length > 0) 62 | builder.Append('&'); 63 | 64 | builder.Append(Encode(pair.Key)); 65 | builder.Append('='); 66 | builder.Append(Encode(pair.Value)); 67 | } 68 | 69 | return DefaultHttpEncoding.GetBytes(builder.ToString()); 70 | } 71 | 72 | static string Encode(string data) => 73 | string.IsNullOrEmpty(data) 74 | ? string.Empty 75 | : WebUtility.UrlEncode(data).Replace("%20", "+"); // Escape spaces as '+'. 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/Core/LinqPadDumps.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | // The methods in this file make WebLinq types friendlier to view in 18 | // LINQPad. See http://www.linqpad.net/CustomizingDump.aspx for more. 19 | 20 | namespace WebLinq 21 | { 22 | using System.Linq; 23 | 24 | partial class HttpFetch 25 | { 26 | internal object ToDump() => new 27 | { 28 | Id, 29 | HttpVersion = HttpVersion.ToString(), 30 | StatusCode, 31 | RequestUrl, 32 | 33 | Headers = 34 | from e in Headers 35 | select new 36 | { 37 | e.Key, 38 | Value = e.Value.Count == 1 39 | ? (object) e.Value.Single() 40 | : e.Value 41 | }, 42 | 43 | ContentHeaders = 44 | from e in ContentHeaders 45 | select new 46 | { 47 | e.Key, 48 | Value = e.Value.Count == 1 49 | ? (object)e.Value.Single() 50 | : e.Value 51 | }, 52 | 53 | Content, 54 | }; 55 | } 56 | } 57 | 58 | namespace WebLinq.Collections 59 | { 60 | partial struct Strings 61 | { 62 | internal object ToDump() 63 | => Count > 1 ? this : (object) ToString(); 64 | } 65 | } 66 | 67 | namespace WebLinq.Html 68 | { 69 | using System; 70 | using System.Linq; 71 | 72 | partial class HtmlObject 73 | { 74 | internal object ToDump() => new 75 | { 76 | Name, 77 | 78 | Attributes = 79 | from an in AttributeNames 80 | select new { Name = an, Value = GetAttributeValue(an) }, 81 | 82 | ChildElementCount = 83 | HasChildElements ? ChildElements.Count() : 0, 84 | 85 | OuterHtml = 86 | OuterHtml.Length <= 300 87 | ? (object)OuterHtml 88 | : new Lazy(() => OuterHtml) 89 | }; 90 | } 91 | 92 | partial class HtmlForm 93 | { 94 | internal object ToDump() => new 95 | { 96 | Name, 97 | Action, 98 | Method, 99 | EncType, 100 | Controls, 101 | Element = new Lazy(() => Element), 102 | }; 103 | } 104 | 105 | partial class HtmlFormControl 106 | { 107 | internal object ToDump() => new 108 | { 109 | Name, 110 | ControlType, 111 | InputType = InputType?.KnownType, 112 | IsDisabled, 113 | IsReadOnly, 114 | IsChecked, 115 | IsMultiple, 116 | Element = new Lazy(() => Element), 117 | Form = new Lazy(() => Form), 118 | }; 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /src/Core/Html/HtmlQuery.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq.Html 18 | { 19 | using System; 20 | using System.Data; 21 | using System.Net.Mime; 22 | using System.Reactive.Linq; 23 | 24 | public static class HtmlQuery 25 | { 26 | public static IObservable> Html(this IHttpObservable query, IHtmlParser parser) => 27 | query.Accept(MediaTypeNames.Text.Html) 28 | .ReadContent(async fetch => 29 | parser.Parse(await fetch.Content.ReadAsStringAsync() 30 | .DontContinueOnCapturedContext(), 31 | fetch.RequestUrl)); 32 | 33 | public static IObservable> Links(this IObservable> query) => 34 | query.Links(null); 35 | 36 | public static IObservable> Links(this IObservable> query, Func selector) => 37 | Links(query, null, selector); 38 | 39 | public static IObservable> Links(this IObservable> query, Uri baseUrl) => 40 | Links(query, baseUrl, (href, _) => href); 41 | 42 | public static IObservable> Links(this IObservable> query, Uri baseUrl, Func selector) => 43 | from html in query 44 | from link in html.Content.Links((href, ho) => html.WithContent(selector(href, ho))) 45 | select link; 46 | 47 | public static IObservable Tables(this IObservable query) => 48 | from html in query 49 | from t in html.Tables() 50 | select t; 51 | 52 | public static IObservable> Tables(this IObservable> query) => 53 | query.LiftTranslate(Tables); 54 | 55 | public static IObservable FormsAsDataTable(this IObservable query) => 56 | from html in query 57 | from forms in Observable.Return(html.FormsAsDataTable()) 58 | select forms; 59 | 60 | public static IObservable> FormsAsDataTable(this IObservable> query) => 61 | query.LiftTranslate(FormsAsDataTable); 62 | 63 | public static IObservable Forms(this IObservable query) => 64 | from html in query 65 | from forms in html.Forms.ToObservable() 66 | select forms; 67 | 68 | public static IObservable> Forms(this IObservable> query) => 69 | query.LiftTranslate(Forms); 70 | 71 | static IObservable> LiftTranslate(this IObservable> query, Func, IObservable> converter) => 72 | from input in query 73 | from output in converter(Observable.Return(input.Content)) 74 | select input.WithContent(output); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/Core/StringBuilderCache.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) .NET Foundation and Contributors 2 | // 3 | // The MIT License (MIT) 4 | // 5 | // All rights reserved. 6 | // 7 | // Permission is hereby granted, free of charge, to any person obtaining a copy 8 | // of this software and associated documentation files (the "Software"), to deal 9 | // in the Software without restriction, including without limitation the rights 10 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | // copies of the Software, and to permit persons to whom the Software is 12 | // furnished to do so, subject to the following conditions: 13 | // 14 | // The above copyright notice and this permission notice shall be included in all 15 | // copies or substantial portions of the Software. 16 | // 17 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | // SOFTWARE. 24 | // 25 | #endregion 26 | 27 | // https://github.com/dotnet/corefx/blob/3c127cf75ae1915b2b2cd7e898ac9f57524a9c79/src/Common/src/CoreLib/System/Text/StringBuilderCache.cs 28 | 29 | namespace System.Text 30 | { 31 | /// Provide a cached reusable instance of stringbuilder per thread. 32 | internal static class StringBuilderCache 33 | { 34 | // The value 360 was chosen in discussion with performance experts as a compromise between using 35 | // as litle memory per thread as possible and still covering a large part of short-lived 36 | // StringBuilder creations on the startup path of VS designers. 37 | private const int MaxBuilderSize = 360; 38 | private const int DefaultCapacity = 16; // == StringBuilder.DefaultCapacity 39 | 40 | // WARNING: We allow diagnostic tools to directly inspect this member (t_cachedInstance). 41 | // See https://github.com/dotnet/corert/blob/master/Documentation/design-docs/diagnostics/diagnostics-tools-contract.md for more details. 42 | // Please do not change the type, the name, or the semantic usage of this member without understanding the implication for tools. 43 | // Get in touch with the diagnostics team if you have questions. 44 | [ThreadStatic] 45 | private static StringBuilder t_cachedInstance; 46 | 47 | /// Get a StringBuilder for the specified capacity. 48 | /// If a StringBuilder of an appropriate size is cached, it will be returned and the cache emptied. 49 | public static StringBuilder Acquire(int capacity = DefaultCapacity) 50 | { 51 | if (capacity <= MaxBuilderSize) 52 | { 53 | StringBuilder sb = t_cachedInstance; 54 | if (sb != null) 55 | { 56 | // Avoid stringbuilder block fragmentation by getting a new StringBuilder 57 | // when the requested size is larger than the current capacity 58 | if (capacity <= sb.Capacity) 59 | { 60 | t_cachedInstance = null; 61 | sb.Clear(); 62 | return sb; 63 | } 64 | } 65 | } 66 | return new StringBuilder(capacity); 67 | } 68 | 69 | /// Place the specified builder in the cache if it is not too big. 70 | public static void Release(StringBuilder sb) 71 | { 72 | if (sb.Capacity <= MaxBuilderSize) 73 | { 74 | t_cachedInstance = sb; 75 | } 76 | } 77 | 78 | /// ToString() the stringbuilder, Release it to the cache, and return the resulting string. 79 | public static string GetStringAndRelease(StringBuilder sb) 80 | { 81 | string result = sb.ToString(); 82 | Release(sb); 83 | return result; 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | tests/coverage.* 2 | 3 | ### VisualStudio ### 4 | ## Ignore Visual Studio temporary files, build results, and 5 | ## files generated by popular Visual Studio add-ons. 6 | 7 | # User-specific files 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Build results 17 | [Dd]ebug/ 18 | [Dd]ebugPublic/ 19 | [Rr]elease/ 20 | [Rr]eleases/ 21 | x64/ 22 | x86/ 23 | bld/ 24 | [Bb]in/ 25 | [Oo]bj/ 26 | [Ll]og/ 27 | 28 | # Visual Studio 2015 cache/options directory 29 | .vs/ 30 | # Uncomment if you have tasks that create the project's static files in wwwroot 31 | #wwwroot/ 32 | 33 | # MSTest test Results 34 | [Tt]est[Rr]esult*/ 35 | [Bb]uild[Ll]og.* 36 | 37 | # NUNIT 38 | *.VisualState.xml 39 | TestResult.xml 40 | 41 | # DNX 42 | project.lock.json 43 | artifacts/ 44 | 45 | *_i.c 46 | *_p.c 47 | *_i.h 48 | *.ilk 49 | *.meta 50 | *.obj 51 | *.pch 52 | *.pdb 53 | *.pgc 54 | *.pgd 55 | *.rsp 56 | *.sbr 57 | *.tlb 58 | *.tli 59 | *.tlh 60 | *.tmp 61 | *.tmp_proj 62 | *.log 63 | *.vspscc 64 | *.vssscc 65 | .builds 66 | *.pidb 67 | *.svclog 68 | *.scc 69 | 70 | # Chutzpah Test files 71 | _Chutzpah* 72 | 73 | # Visual Studio profiler 74 | *.psess 75 | *.vsp 76 | *.vspx 77 | *.sap 78 | 79 | # TFS 2012 Local Workspace 80 | $tf/ 81 | 82 | # Guidance Automation Toolkit 83 | *.gpState 84 | 85 | # ReSharper is a .NET coding add-in 86 | _ReSharper*/ 87 | *.[Rr]e[Ss]harper 88 | *.DotSettings.user 89 | 90 | # JustCode is a .NET coding add-in 91 | .JustCode 92 | 93 | # TeamCity is a build add-in 94 | _TeamCity* 95 | 96 | # DotCover is a Code Coverage Tool 97 | *.dotCover 98 | 99 | # NCrunch 100 | _NCrunch_* 101 | .*crunch*.local.xml 102 | nCrunchTemp_* 103 | 104 | # MightyMoose 105 | *.mm.* 106 | AutoTest.Net/ 107 | 108 | # Web workbench (sass) 109 | .sass-cache/ 110 | 111 | # Installshield output folder 112 | [Ee]xpress/ 113 | 114 | # DocProject is a documentation generator add-in 115 | DocProject/buildhelp/ 116 | DocProject/Help/*.HxT 117 | DocProject/Help/*.HxC 118 | DocProject/Help/*.hhc 119 | DocProject/Help/*.hhk 120 | DocProject/Help/*.hhp 121 | DocProject/Help/Html2 122 | DocProject/Help/html 123 | 124 | # Click-Once directory 125 | publish/ 126 | 127 | # Publish Web Output 128 | *.[Pp]ublish.xml 129 | *.azurePubxml 130 | # TODO: Comment the next line if you want to checkin your web deploy settings 131 | # but database connection strings (with potential passwords) will be unencrypted 132 | *.pubxml 133 | *.publishproj 134 | 135 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 136 | # checkin your Azure Web App publish settings, but sensitive information contained 137 | # in these scripts will be unencrypted 138 | PublishScripts/ 139 | 140 | # NuGet Packages 141 | *.nupkg 142 | # The packages folder can be ignored because of Package Restore 143 | **/packages/* 144 | # except build/, which is used as an MSBuild target. 145 | !**/packages/build/ 146 | # Uncomment if necessary however generally it will be regenerated when needed 147 | #!**/packages/repositories.config 148 | # NuGet v3's project.json files produces more ignoreable files 149 | *.nuget.props 150 | *.nuget.targets 151 | 152 | # Microsoft Azure Build Output 153 | csx/ 154 | *.build.csdef 155 | 156 | # Microsoft Azure Emulator 157 | ecf/ 158 | rcf/ 159 | 160 | # Windows Store app package directories and files 161 | AppPackages/ 162 | BundleArtifacts/ 163 | Package.StoreAssociation.xml 164 | _pkginfo.txt 165 | 166 | # Visual Studio cache files 167 | # files ending in .cache can be ignored 168 | *.[Cc]ache 169 | # but keep track of directories ending in .cache 170 | !*.[Cc]ache/ 171 | 172 | # Others 173 | ClientBin/ 174 | ~$* 175 | *~ 176 | *.dbmdl 177 | *.dbproj.schemaview 178 | *.pfx 179 | *.publishsettings 180 | node_modules/ 181 | orleans.codegen.cs 182 | 183 | # Since there are multiple workflows, uncomment next line to ignore bower_components 184 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 185 | #bower_components/ 186 | 187 | # RIA/Silverlight projects 188 | Generated_Code/ 189 | 190 | # Backup & report files from converting an old project file 191 | # to a newer Visual Studio version. Backup files are not needed, 192 | # because we have git ;-) 193 | _UpgradeReport_Files/ 194 | Backup*/ 195 | UpgradeLog*.XML 196 | UpgradeLog*.htm 197 | 198 | # SQL Server files 199 | *.mdf 200 | *.ldf 201 | 202 | # Business Intelligence projects 203 | *.rdl.data 204 | *.bim.layout 205 | *.bim_*.settings 206 | 207 | # Microsoft Fakes 208 | FakesAssemblies/ 209 | 210 | # GhostDoc plugin setting file 211 | *.GhostDoc.xml 212 | 213 | # Node.js Tools for Visual Studio 214 | .ntvs_analysis.dat 215 | 216 | # Visual Studio 6 build log 217 | *.plg 218 | 219 | # Visual Studio 6 workspace options file 220 | *.opt 221 | 222 | # Visual Studio LightSwitch build output 223 | **/*.HTMLClient/GeneratedArtifacts 224 | **/*.DesktopClient/GeneratedArtifacts 225 | **/*.DesktopClient/ModelManifest.xml 226 | **/*.Server/GeneratedArtifacts 227 | **/*.Server/ModelManifest.xml 228 | _Pvt_Extensions 229 | 230 | # Paket dependency manager 231 | .paket/paket.exe 232 | paket-files/ 233 | 234 | # FAKE - F# Make 235 | .fake/ 236 | 237 | # JetBrains Rider 238 | .idea/ 239 | *.sln.iml 240 | -------------------------------------------------------------------------------- /src/Core/HttpConfig.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | #endregion 18 | 19 | namespace WebLinq 20 | { 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Net; 25 | 26 | partial class HttpConfig 27 | { 28 | public static readonly HttpConfig Default; 29 | 30 | static HttpConfig() 31 | { 32 | var req = WebRequest.CreateHttp("http://localhost/"); 33 | Default = new HttpConfig(HttpHeaderCollection.Empty, 34 | TimeSpan.FromMilliseconds(req.Timeout), 35 | req.UseDefaultCredentials, req.Credentials, 36 | req.UserAgent, 37 | req.AutomaticDecompression, 38 | cookies: null, 39 | ignoreInvalidServerCertificate: false); 40 | } 41 | } 42 | 43 | public sealed partial class HttpConfig 44 | { 45 | public static readonly IEnumerable ZeroCookies = new Cookie[0]; 46 | 47 | public HttpHeaderCollection Headers { get; private set; } 48 | public TimeSpan Timeout { get; private set; } 49 | public bool UseDefaultCredentials { get; private set; } 50 | public ICredentials Credentials { get; private set; } 51 | public string UserAgent { get; private set; } 52 | public DecompressionMethods AutomaticDecompression { get; private set; } 53 | public IReadOnlyCollection Cookies { get; private set; } 54 | public bool IgnoreInvalidServerCertificate { get; private set; } 55 | 56 | public HttpConfig(HttpHeaderCollection headers, 57 | TimeSpan timeout, 58 | bool useDefaultCredentials, ICredentials credentials, 59 | string userAgent, 60 | DecompressionMethods automaticDecompression, 61 | IReadOnlyCollection cookies, 62 | bool ignoreInvalidServerCertificate) 63 | { 64 | Headers = headers; 65 | Timeout = timeout; 66 | UserAgent = userAgent; 67 | AutomaticDecompression = automaticDecompression; 68 | Cookies = cookies; 69 | Credentials = credentials; 70 | UseDefaultCredentials = useDefaultCredentials; 71 | IgnoreInvalidServerCertificate = ignoreInvalidServerCertificate; 72 | } 73 | 74 | HttpConfig(HttpConfig other) : 75 | this(other.Headers, 76 | other.Timeout, 77 | other.UseDefaultCredentials, other.Credentials, 78 | other.UserAgent, 79 | other.AutomaticDecompression, 80 | other.Cookies, 81 | other.IgnoreInvalidServerCertificate) { } 82 | 83 | public HttpConfig WithHeader(string name, string value) => 84 | WithHeaders(Headers.Set(name, value)); 85 | 86 | public HttpConfig WithHeaders(HttpHeaderCollection value) => 87 | Headers == value ? this : new HttpConfig(this) { Headers = value }; 88 | 89 | public HttpConfig WithTimeout(TimeSpan value) => 90 | Timeout == value ? this : new HttpConfig(this) { Timeout = value }; 91 | 92 | public HttpConfig WithUseDefaultCredentials(bool value) => 93 | UseDefaultCredentials == value ? this : new HttpConfig(this) { UseDefaultCredentials = value }; 94 | 95 | public HttpConfig WithCredentials(ICredentials value) => 96 | Credentials == value ? this : new HttpConfig(this) { Credentials = value }; 97 | 98 | public HttpConfig WithUserAgent(string value) => 99 | WithUserAgentImpl(value ?? string.Empty); 100 | 101 | HttpConfig WithUserAgentImpl(string value) => 102 | UserAgent == value ? this : new HttpConfig(this) { UserAgent = value }; 103 | 104 | public HttpConfig WithAutomaticDecompression(DecompressionMethods value) => 105 | AutomaticDecompression == value ? this : new HttpConfig(this) { AutomaticDecompression = value }; 106 | 107 | public HttpConfig WithCookies(IReadOnlyCollection value) => 108 | ReferenceEquals(Cookies, value) || Cookies?.SequenceEqual(value ?? ZeroCookies) == true 109 | ? this 110 | : new HttpConfig(this) { Cookies = value }; 111 | 112 | public HttpConfig WithIgnoreInvalidServerCertificate(bool value) => 113 | IgnoreInvalidServerCertificate == value 114 | ? this 115 | : new HttpConfig(this) { IgnoreInvalidServerCertificate = value }; 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/Core/ParsedValue.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq 18 | { 19 | using System; 20 | using System.Collections.Generic; 21 | using System.Globalization; 22 | 23 | public static class ParsedValue 24 | { 25 | public static ParsedValue Create(TSource source, TValue value) => 26 | new ParsedValue(source, value); 27 | } 28 | 29 | public static class ValueParser 30 | { 31 | public static Func> Create(Func parser) => 32 | s => ParsedValue.Create(s, parser(s)); 33 | } 34 | 35 | public static class ValueTextParser 36 | { 37 | public static Func> Create(Func parser) => 38 | ValueParser.Create(parser); 39 | 40 | public static ParsedValue ParseInt32(this string source) => 41 | ParseInt32(source, null); 42 | 43 | public static ParsedValue ParseInt32(this string source, IFormatProvider provider) => 44 | ParseInt32(source, NumberStyles.Integer, provider); 45 | 46 | public static ParsedValue ParseInt32(this string source, NumberStyles styles) => 47 | ParseInt32(source, styles, null); 48 | 49 | public static ParsedValue ParseInt32(this string source, NumberStyles styles, IFormatProvider provider) => 50 | ParsedValue.Create(source, int.Parse(source, styles, CultureInfo.InvariantCulture)); 51 | 52 | public static ParsedValue ParseDouble(this string source) => 53 | ParseDouble(source, null); 54 | 55 | public static ParsedValue ParseDouble(this string source, IFormatProvider provider) => 56 | ParseDouble(source, NumberStyles.Float | NumberStyles.AllowThousands, provider); 57 | 58 | public static ParsedValue ParseDouble(this string source, NumberStyles styles) => 59 | ParseDouble(source, styles, null); 60 | 61 | public static ParsedValue ParseDouble(this string source, NumberStyles styles, IFormatProvider provider) => 62 | ParsedValue.Create(source, double.Parse(source, styles, provider)); 63 | 64 | public static ParsedValue ParseDecimal(this string source) => 65 | ParseDecimal(source, null); 66 | 67 | public static ParsedValue ParseDecimal(this string source, IFormatProvider provider) => 68 | ParseDecimal(source, NumberStyles.Number, provider); 69 | 70 | public static ParsedValue ParseDecimal(this string source, NumberStyles styles) => 71 | ParseDecimal(source, styles, null); 72 | 73 | public static ParsedValue ParseDecimal(this string source, NumberStyles styles, IFormatProvider provider) => 74 | ParsedValue.Create(source, decimal.Parse(source, styles, provider)); 75 | 76 | public static ParsedValue ParseDateTime(this string source) => 77 | ParseDateTime(source, (IFormatProvider) null); 78 | 79 | public static ParsedValue ParseDateTime(this string source, IFormatProvider provider) => 80 | ParsedValue.Create(source, DateTime.Parse(source, provider)); 81 | 82 | public static ParsedValue ParseDateTime(this string source, string format) => 83 | ParseDateTime(source, format, null); 84 | 85 | public static ParsedValue ParseDateTime(this string source, string format, IFormatProvider provider) => 86 | ParsedValue.Create(source, DateTime.ParseExact(source, format, provider)); 87 | } 88 | 89 | public struct ParsedValue : IEquatable> 90 | { 91 | public TSource Source { get; } 92 | public TValue Value { get; } 93 | 94 | public ParsedValue(TSource source, TValue value) 95 | { 96 | Source = source; 97 | Value = value; 98 | } 99 | 100 | public bool Equals(ParsedValue other) => 101 | EqualityComparer.Default.Equals(Source, other.Source) 102 | && EqualityComparer.Default.Equals(Value, other.Value); 103 | 104 | public override bool Equals(object obj) => 105 | obj is ParsedValue pv && Equals(pv); 106 | 107 | public override int GetHashCode() => 108 | unchecked((EqualityComparer.Default.GetHashCode(Source) * 397) 109 | ^ EqualityComparer.Default.GetHashCode(Value)); 110 | 111 | public override string ToString() => 112 | Value == null ? string.Empty : Value.ToString(); 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /src/Core/Collections/MapBase.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq.Collections 18 | { 19 | using System; 20 | using System.Collections; 21 | using System.Collections.Generic; 22 | using System.Linq; 23 | 24 | // An immutable dictionary implementation that maintains a linked list 25 | // of entries. Each node in the linked list contains a cached dictionary 26 | // that is only built on first use, e.g. when a look-up is made or the 27 | // entries are iterated. The cache is built by walking the list nodes 28 | // from tail to head, with each node only being added if not already in 29 | // the dictionary. This way, newer nodes with the same key shadow older 30 | // ones. 31 | 32 | public abstract class MapBase : IReadOnlyDictionary 33 | { 34 | protected TKey Key { get; } 35 | protected TValue Value { get; } 36 | public IEqualityComparer Comparer { get; } 37 | Dictionary _cache; 38 | 39 | protected MapBase(IEqualityComparer comparer) : 40 | this(default, default, comparer) {} 41 | 42 | protected MapBase(TKey key, TValue value) : 43 | this(key, value, null) {} 44 | 45 | protected MapBase(TKey key, TValue value, IEqualityComparer comparer) 46 | { 47 | Key = key; 48 | Value = value; 49 | Comparer = comparer ?? EqualityComparer.Default; 50 | } 51 | 52 | bool HasCache => _cache != null; 53 | 54 | public Dictionary Cache => 55 | _cache ?? (_cache = CreateCache()); 56 | 57 | protected virtual Dictionary CreateCache() 58 | { 59 | var map = new Dictionary(Comparer); 60 | foreach (var node in Nodes) 61 | { 62 | if (!map.ContainsKey(node.Key)) 63 | map.Add(node.Key, node.Value); 64 | } 65 | return map; 66 | } 67 | 68 | protected abstract IEnumerable> Nodes { get; } 69 | 70 | protected static IEnumerable> GetNodesCore(T initial, Func nextSelector) 71 | where T : MapBase 72 | { 73 | if (initial == null) throw new ArgumentNullException(nameof(initial)); 74 | if (nextSelector == null) throw new ArgumentNullException(nameof(nextSelector)); 75 | for (var node = initial; !node.IsEmpty; node = nextSelector(node)) 76 | yield return new KeyValuePair(node.Key, node.Value); 77 | } 78 | 79 | public IEnumerator> GetEnumerator() => 80 | (IsEmpty ? Enumerable.Empty>() : Cache).GetEnumerator(); 81 | 82 | IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); 83 | 84 | public abstract bool IsEmpty { get; } 85 | 86 | public int Count => IsEmpty ? 0 : Cache.Count; 87 | 88 | public bool ContainsKey(TKey key) => 89 | !IsEmpty && ((!HasCache && IsKey(key)) || Cache.ContainsKey(key)); 90 | 91 | bool IsKey(TKey key) => 92 | !IsEmpty && EqualityComparer.Default.Equals(Key, key); 93 | 94 | public bool TryGetValue(TKey key, out TValue value) 95 | { 96 | if (IsEmpty) 97 | { 98 | value = default; 99 | return false; 100 | } 101 | if (!HasCache && IsKey(key)) 102 | { 103 | value = Value; 104 | return true; 105 | } 106 | return Cache.TryGetValue(key, out value); 107 | } 108 | 109 | public TValue TryGetValue(TKey key) => 110 | TryGetValue(key, (found, value) => found ? value : default); 111 | 112 | public T TryGetValue(TKey key, Func selector) 113 | { 114 | if (selector == null) throw new ArgumentNullException(nameof(selector)); 115 | return TryGetValue(key, out var value) 116 | ? selector(true, value) 117 | : selector(false, default); 118 | } 119 | 120 | public TValue this[TKey key] 121 | { 122 | get 123 | { 124 | if (IsEmpty) throw new KeyNotFoundException(); 125 | return !HasCache && IsKey(key) ? Value : Cache[key]; 126 | } 127 | } 128 | 129 | public IEnumerable Keys => from e in this select e.Key; 130 | public IEnumerable Values => from e in this select e.Value; 131 | 132 | protected T RemoveCore(T initial, TKey key, Func setter) 133 | { 134 | // ReSharper disable once LoopCanBeConvertedToQuery 135 | foreach (var item in this) 136 | { 137 | if (!Comparer.Equals(key, item.Key)) 138 | initial = setter(initial, item.Key, item.Value); 139 | } 140 | return initial; 141 | } 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /src/Core/Download.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq 18 | { 19 | using System; 20 | using System.Collections.Generic; 21 | using System.Diagnostics; 22 | using System.IO; 23 | using System.Net; 24 | using System.Net.Http; 25 | using System.Reactive.Linq; 26 | using System.Threading.Tasks; 27 | using static FileSystem; 28 | 29 | public sealed class LocalFileContent : HttpContent 30 | { 31 | public string Path { get; } 32 | 33 | public LocalFileContent(string path) => 34 | Path = path ?? throw new ArgumentNullException(nameof(path)); 35 | 36 | protected override Task CreateContentReadStreamAsync() => 37 | Task.FromResult((Stream) File.OpenRead(Path)); 38 | 39 | protected override async Task SerializeToStreamAsync(Stream stream, TransportContext context) 40 | { 41 | using (var fs = File.OpenRead(Path)) 42 | await fs.CopyToAsync(stream).DontContinueOnCapturedContext(); 43 | } 44 | 45 | protected override bool TryComputeLength(out long length) 46 | { 47 | length = GetFileSize(Path); 48 | return true; 49 | } 50 | } 51 | 52 | partial class HttpObservable 53 | { 54 | public static IObservable> AsHttpContent(this IObservable> query) 55 | where T : HttpContent => 56 | from fetch in query 57 | select fetch.WithContent(fetch.Content); 58 | 59 | public static IObservable> Download(this IObservable> query, string path) => 60 | query.SelectMany(async f => f.WithContent(await DownloadAsync(f.Content, path))); 61 | 62 | public static IObservable> Download(this IHttpObservable query, string path) => 63 | query.ReadContent(f => DownloadAsync(f.Content, path)); 64 | 65 | static async Task DownloadAsync(HttpContent content, string path) 66 | { 67 | if (content == null) throw new ArgumentNullException(nameof(content)); 68 | if (path == null) throw new ArgumentNullException(nameof(path)); 69 | 70 | using (var output = File.OpenWrite(path)) 71 | { 72 | await content.CopyToAsync(output).DontContinueOnCapturedContext(); 73 | var localContent = new LocalFileContent(path); 74 | foreach (var e in content.Headers) 75 | localContent.Headers.TryAddWithoutValidation(e.Key, e.Value); 76 | return localContent; 77 | } 78 | } 79 | 80 | public static IObservable> DownloadTemp(this IObservable> query) => 81 | query.DownloadTemp(Path.GetTempPath()); 82 | 83 | public static IObservable> DownloadTemp(this IObservable> query, string path) => 84 | query.SelectMany(async f => f.WithContent(await DownloadTemp(f, path))); 85 | 86 | public static IObservable> DownloadTemp(this IHttpObservable query) => 87 | query.Download(Path.GetTempFileName()); 88 | 89 | public static IObservable> DownloadTemp(this IHttpObservable query, string path) => 90 | query.ReadContent(f => DownloadTemp(f, path)); 91 | 92 | static async Task DownloadTemp(HttpFetch fetch, string path) 93 | { 94 | string tempPath; 95 | 96 | var startTime = DateTime.Now; 97 | var exceptions = new List(); 98 | 99 | while (true) 100 | { 101 | // NOTE! Path.GetDirectoryName returns null for "/" or "\\" 102 | // but Path.IsPathRooted return true, so we use path verbatim 103 | // when it is just the root. 104 | 105 | var desiredDir = Path.GetDirectoryName(path); 106 | var dir = desiredDir is string dn && dn.Length > 0 ? dn 107 | : desiredDir == null && Path.IsPathRooted(path) ? path 108 | : Path.GetTempPath(); 109 | 110 | var fileName = Path.GetFileNameWithoutExtension(path); 111 | var discriminator = (Process.GetCurrentProcess().Id + Environment.TickCount).ToString("x16"); 112 | var tempName = (string.IsNullOrEmpty(fileName) ? "tmp" : fileName) 113 | + "-" + discriminator 114 | + Path.GetExtension(path); 115 | 116 | var tempTestPath = Path.Combine(dir, tempName); 117 | 118 | try 119 | { 120 | new FileStream(tempTestPath, FileMode.CreateNew).Close(); 121 | tempPath = tempTestPath; 122 | break; 123 | } 124 | catch (IOException e) 125 | { 126 | exceptions.Add(e); 127 | 128 | // There is no good cross-platform way to know if 129 | // IOException is due to file already existing or not 130 | // so we assume any IOException is due to file already 131 | // existing. However, if we've been looping for 5 132 | // seconds then something is seriously wrong and we 133 | // throw thereafter. 134 | 135 | if (DateTime.Now - startTime > TimeSpan.FromSeconds(5)) 136 | throw new IOException(e.Message, new AggregateException(exceptions)); 137 | } 138 | 139 | await Task.Delay(TimeSpan.FromSeconds(0.1)) 140 | .DontContinueOnCapturedContext(); 141 | } 142 | 143 | return await DownloadAsync(fetch.Content, tempPath).DontContinueOnCapturedContext(); 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /src/Core/HttpObservable.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq 18 | { 19 | using System; 20 | using System.Net; 21 | using System.Net.Http; 22 | using System.Reactive; 23 | using System.Reactive.Linq; 24 | using System.Threading.Tasks; 25 | using WebLinq; 26 | 27 | public interface IHttpObservable : IObservable> 28 | { 29 | HttpOptions Options { get; } 30 | IHttpObservable WithOptions(HttpOptions options); 31 | Func Configurer { get; } 32 | Func FilterPredicate { get; } 33 | IHttpObservable WithConfigurer(Func modifier); 34 | IHttpObservable WithFilterPredicate(Func predicate); 35 | IObservable> ReadContent(Func, Task> reader); 36 | } 37 | 38 | public static partial class HttpObservable 39 | { 40 | public static IHttpObservable ReturnErroneousFetch(this IHttpObservable query) => 41 | query.WithOptions(query.Options.WithReturnErroneousFetch(true)); 42 | 43 | [Obsolete("Use " + nameof(ReturnErroneousFetch) + " instead.")] 44 | public static IHttpObservable ReturnErrorneousFetch(this IHttpObservable query) => 45 | query.WithOptions(query.Options.WithReturnErroneousFetch(true)); 46 | 47 | public static IHttpObservable SkipErroneousFetch(this IHttpObservable query) => 48 | query.ReturnErroneousFetch() 49 | .Filter(f => f.IsSuccessStatusCode); 50 | 51 | public static IHttpObservable SetHeader(this IHttpObservable query, string name, string value) => 52 | query.WithConfigurer(c => query.Configurer(c).WithHeader(name, value)); 53 | 54 | public static IHttpObservable SetUserAgent(this IHttpObservable query, string value) => 55 | query.WithConfigurer(c => query.Configurer(c).WithUserAgent(value)); 56 | 57 | public static IHttpObservable AutomaticDecompression(this IHttpObservable query, DecompressionMethods value) => 58 | query.WithConfigurer(c => query.Configurer(c).WithAutomaticDecompression(value)); 59 | 60 | public static IObservable> Buffer(this IHttpObservable query) => 61 | query.ReadContent(async f => 62 | { 63 | await f.Content.LoadIntoBufferAsync().DontContinueOnCapturedContext(); 64 | return f.Content; 65 | }); 66 | 67 | public static IHttpObservable Do(this IHttpObservable query, Action action) => 68 | query.Filter(f => { action(f); return true; }); 69 | 70 | public static IHttpObservable Filter(this IHttpObservable query, Func predicate) => 71 | query.WithFilterPredicate(f => query.FilterPredicate(f) && predicate(f)); 72 | 73 | internal static IHttpObservable Return(Func>> query) => 74 | Return(HttpOptions.Default, _ => _, query, _ => true); 75 | 76 | static IHttpObservable 77 | Return(HttpOptions options, 78 | Func configurer, 79 | Func>> query, 80 | Func predicate) => 81 | new Impl(query, options, configurer, predicate); 82 | 83 | internal static IHttpObservable Return(IObservable query) => 84 | Return(ho => 85 | from q in query 86 | from e in q.WithConfigurer(ho.Configurer) 87 | .WithOptions(ho.Options) 88 | .WithFilterPredicate(ho.FilterPredicate) 89 | .ReadContent(f => Task.FromResult(f.Content)) 90 | select e); 91 | 92 | [Obsolete("Use " + nameof(IHttpObservable.ReadContent) + " instead.")] 93 | public static IObservable> WithReader(this IHttpObservable query, Func, Task> reader) => 94 | query.ReadContent(reader); 95 | 96 | sealed class Impl : IHttpObservable 97 | { 98 | readonly Func>> _query; 99 | 100 | public Impl(Func>> query, 101 | HttpOptions options, 102 | Func configurer, 103 | Func predicate) 104 | { 105 | _query = query; 106 | Options = options; 107 | Configurer = configurer; 108 | FilterPredicate = predicate; 109 | } 110 | 111 | public IDisposable Subscribe(IObserver> observer) => 112 | _query(this) 113 | .Do(f => f.Content.Dispose()) 114 | .Select(f => f.WithContent(new Unit())) 115 | .Where(f => FilterPredicate(f.Info)) 116 | .Subscribe(observer); 117 | 118 | public HttpOptions Options { get; } 119 | 120 | public IHttpObservable WithOptions(HttpOptions options) => 121 | new Impl(_query, options, Configurer, FilterPredicate); 122 | 123 | public Func Configurer { get; } 124 | 125 | public IHttpObservable WithConfigurer(Func configurer) => 126 | new Impl(_query, Options, configurer, FilterPredicate); 127 | 128 | public Func FilterPredicate { get; } 129 | 130 | public IHttpObservable WithFilterPredicate(Func predicate) => 131 | new Impl(_query, Options, Configurer, predicate); 132 | 133 | public IObservable> ReadContent(Func, Task> reader) => 134 | from f in _query(this) 135 | where FilterPredicate(f.Info) 136 | from c in reader(f) 137 | select f.WithContent(c); 138 | } 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /src/Core/HttpFetch.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq 18 | { 19 | using System; 20 | using System.Diagnostics; 21 | using System.Globalization; 22 | using System.Net; 23 | using System.Net.Http.Headers; 24 | using System.Text; 25 | 26 | public static class HttpFetch 27 | { 28 | public static HttpFetch Create(HttpFetchInfo info, T content) => 29 | new HttpFetch(info, content); 30 | } 31 | 32 | [DebuggerDisplay("Id = {Id}, StatusCode = {StatusCode} ({ReasonPhrase}), Content = {Content}")] 33 | public partial class HttpFetch 34 | { 35 | public HttpFetchInfo Info { get; } 36 | public T Content { get; } 37 | 38 | public int Id => Info.Id; 39 | public IHttpClient Client => Info.Client; 40 | public Version HttpVersion => Info.HttpVersion; 41 | public HttpStatusCode StatusCode => Info.StatusCode; 42 | public string ReasonPhrase => Info.ReasonPhrase; 43 | public HttpHeaderCollection Headers => Info.Headers; 44 | public HttpHeaderCollection ContentHeaders => Info.ContentHeaders; 45 | public Uri RequestUrl => Info.RequestUrl; 46 | public HttpHeaderCollection RequestHeaders => Info.RequestHeaders; 47 | 48 | public HttpFetch(HttpFetchInfo info, T content) 49 | { 50 | Info = info; 51 | Content = content; 52 | } 53 | 54 | public HttpFetch WithContent(TContent content) => 55 | new HttpFetch(Info, content); 56 | 57 | public bool IsSuccessStatusCode => Info.IsSuccessStatusCode; 58 | public bool IsSuccessStatusCodeInRange(int first, int last) => Info.IsSuccessStatusCodeInRange(first, last); 59 | 60 | public bool IsContentType(string type) => Info.IsContentType(type); 61 | public string ContentMediaType => Info.ContentMediaType; 62 | public string ContentCharSet => Info.ContentCharSet; 63 | public Encoding ContentCharSetEncoding => Info.ContentCharSetEncoding; 64 | 65 | public string ContentDispositionType => Info.ContentDispositionType; 66 | public string ContentDispositionFileName => Info.ContentDispositionFileName; 67 | 68 | public long? ContentLength => Info.ContentLength; 69 | } 70 | 71 | [DebuggerDisplay("Id = {Id}, StatusCode = {StatusCode} ({ReasonPhrase})")] 72 | public sealed class HttpFetchInfo 73 | { 74 | public int Id { get; } 75 | public IHttpClient Client { get; } 76 | public Version HttpVersion { get; } 77 | public HttpStatusCode StatusCode { get; } 78 | public string ReasonPhrase { get; } 79 | public HttpHeaderCollection Headers { get; } 80 | public HttpHeaderCollection ContentHeaders { get; } 81 | public Uri RequestUrl { get; } 82 | public HttpHeaderCollection RequestHeaders { get; } 83 | 84 | public HttpFetchInfo(int id, 85 | IHttpClient client, 86 | Version httpVersion, 87 | HttpStatusCode statusCode, 88 | string reasonPhrase, 89 | HttpHeaderCollection headers, 90 | HttpHeaderCollection contentHeaders, 91 | Uri requestUrl, 92 | HttpHeaderCollection requestHeaders) 93 | { 94 | Id = id; 95 | Client = client; 96 | HttpVersion = httpVersion; 97 | StatusCode = statusCode; 98 | ReasonPhrase = reasonPhrase; 99 | Headers = headers; 100 | ContentHeaders = contentHeaders; 101 | RequestUrl = requestUrl; 102 | RequestHeaders = requestHeaders; 103 | } 104 | 105 | public bool IsSuccessStatusCode => IsSuccessStatusCodeInRange(200, 299); 106 | public bool IsSuccessStatusCodeInRange(int first, int last) => (int)StatusCode >= first && (int)StatusCode <= last; 107 | 108 | (bool, MediaTypeHeaderValue Value) _contentType; 109 | 110 | MediaTypeHeaderValue ContentType => 111 | this.LazyGet(ref _contentType, 112 | it => it.ContentHeaders.TryGetValue("Content-Type", out var value) 113 | ? MediaTypeHeaderValue.Parse(value) 114 | : null); 115 | 116 | public bool IsContentType(string type) => 117 | string.Equals(ContentType?.MediaType, type, StringComparison.OrdinalIgnoreCase); 118 | 119 | public string ContentMediaType => ContentType?.MediaType; 120 | public string ContentCharSet => ContentType?.CharSet; 121 | 122 | public Encoding ContentCharSetEncoding 123 | => ContentType?.CharSet is string s 124 | ? Encoding.GetEncoding(s) 125 | : null; 126 | 127 | (bool, ContentDispositionHeaderValue Value) _contentDisposition; 128 | 129 | ContentDispositionHeaderValue ContentDisposition => 130 | this.LazyGet(ref _contentDisposition, 131 | it => it.ContentHeaders.TryGetValue("Content-Disposition", out var value) 132 | ? ContentDispositionHeaderValue.Parse(value) 133 | : null); 134 | 135 | public string ContentDispositionType => ContentDisposition?.DispositionType; 136 | 137 | static readonly char[] Quote = { '"' }; 138 | 139 | public string ContentDispositionFileName 140 | => ContentDisposition is ContentDispositionHeaderValue h 141 | ? (h.FileNameStar ?? h.FileName)?.Trim(Quote) 142 | : null; 143 | 144 | (bool, long? Value) _contentLength; 145 | 146 | public long? ContentLength => 147 | this.LazyGet(ref _contentLength, 148 | it => it.ContentHeaders.TryGetValue("Content-Length", out var str) 149 | ? long.Parse(str.Single(), NumberStyles.None, CultureInfo.InvariantCulture) 150 | : (long?) null); 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /src/Core/Html/HapHtmlParser.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq.Html 18 | { 19 | #region Imports 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Linq; 24 | using System.Runtime.CompilerServices; 25 | using Fizzler.Systems.HtmlAgilityPack; 26 | using HtmlAgilityPack; 27 | 28 | #endregion 29 | 30 | public sealed class HapHtmlParser : IHtmlParser 31 | { 32 | public void Register(Action registrationHandler) => 33 | registrationHandler(typeof(IHtmlParser), this); 34 | 35 | public ParsedHtml Parse(string html, Uri baseUrl) 36 | { 37 | var doc = new HtmlDocument(); 38 | doc.LoadHtml2(html); 39 | return new HapParsedHtml(doc, baseUrl); 40 | } 41 | 42 | sealed class HapParsedHtml : ParsedHtml 43 | { 44 | readonly HtmlDocument _document; 45 | readonly ConditionalWeakTable _map = new ConditionalWeakTable(); 46 | 47 | public HapParsedHtml(HtmlDocument document, Uri baseUrl) : 48 | base(baseUrl) 49 | { 50 | _document = document; 51 | } 52 | 53 | HtmlObject GetPublicObject(HtmlNode node) 54 | { 55 | if (!_map.TryGetValue(node, out var obj)) 56 | _map.Add(node, obj = "option".Equals(node.Name, StringComparison.Ordinal) 57 | ? new HapOptionElement(node, this) 58 | : new HapHtmlObject(node, this)); 59 | return obj; 60 | } 61 | 62 | public override IEnumerable QuerySelectorAll(string selector, HtmlObject context) => 63 | from node in (((HapHtmlObject)context)?.Node ?? _document.DocumentNode).QuerySelectorAll(selector) 64 | orderby node.StreamPosition 65 | select GetPublicObject(node); 66 | 67 | public override HtmlObject Root => GetPublicObject(_document.DocumentNode); 68 | 69 | class HapHtmlObject : HtmlObject 70 | { 71 | readonly HapParsedHtml _owner; 72 | 73 | public HapHtmlObject(HtmlNode node, HapParsedHtml owner) 74 | { 75 | Node = node; 76 | _owner = owner; 77 | } 78 | 79 | public override ParsedHtml Owner => _owner; 80 | public HtmlNode Node { get; } 81 | public override string Name => Node.Name; 82 | 83 | public override IEnumerable AttributeNames => 84 | from a in Node.Attributes select a.Name; 85 | 86 | public override bool HasAttribute(string name) => 87 | GetAttributeValue(name) != null; 88 | 89 | public override HtmlString GetAttributeSourceValue(string name) => 90 | HtmlString.FromEncoded(Node.GetAttributeValue(name, null)); 91 | 92 | public override string OuterHtml => Node.OuterHtml; 93 | public override string InnerHtml => Node.InnerHtml; 94 | public override HtmlString InnerTextSource => 95 | HtmlString.FromEncoded(Node.InnerText); 96 | 97 | public override HtmlObject ParentElement => 98 | Node.ParentNode.NodeType == HtmlNodeType.Element 99 | ? _owner.GetPublicObject(Node.ParentNode) 100 | : null; 101 | 102 | public override IEnumerable ChildElements => 103 | from e in Node.ChildNodes 104 | where e.NodeType == HtmlNodeType.Element 105 | select _owner.GetPublicObject(e); 106 | } 107 | 108 | sealed class HapOptionElement : HapHtmlObject 109 | { 110 | HtmlString? _innerTextSource; 111 | 112 | public HapOptionElement(HtmlNode node, HapParsedHtml owner) : 113 | base(node, owner) {} 114 | 115 | public override HtmlString InnerTextSource => 116 | (_innerTextSource ?? (_innerTextSource = GetInnerTextSourceCore())).Value; 117 | 118 | HtmlString GetInnerTextSourceCore() 119 | { 120 | // Workaround for HTML Agility Pack where OPTION elements 121 | // do not correctly return their inner text, especially 122 | // in the presence of a closing tag. 123 | // https://www.w3.org/TR/html-markup/option.html#option-tags 124 | // An option element's end tag may be omitted if the option 125 | // element is immediately followed by another OPTION 126 | // element, or if it is immediately followed by an 127 | // OPTGROUP element, or if there is no more content in the 128 | // parent element. 129 | 130 | var siblingElement = Node.Siblings().FirstOrDefault(s => s.NodeType == HtmlNodeType.Element); 131 | if (Node.InnerText.Length == 0 132 | && (siblingElement == null || "option".Equals(siblingElement.Name, StringComparison.OrdinalIgnoreCase) 133 | || "optgroup".Equals(siblingElement.Name, StringComparison.OrdinalIgnoreCase))) 134 | { 135 | var tns = 136 | from n in Node.Siblings().TakeWhile(n => n != siblingElement) 137 | where n.NodeType == HtmlNodeType.Text 138 | select n.InnerText; 139 | return HtmlString.FromEncoded(string.Join(null, tns)); 140 | } 141 | 142 | return base.InnerTextSource; 143 | } 144 | } 145 | } 146 | } 147 | 148 | static class HapExtensions 149 | { 150 | public static IEnumerable Siblings(this HtmlNode node) 151 | { 152 | if (node == null) throw new ArgumentNullException(nameof(node)); 153 | for (var sibling = node.NextSibling; sibling != null; sibling = sibling.NextSibling) 154 | yield return sibling; 155 | } 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /src/Core/Zip/Zip.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq.Zip 18 | { 19 | #region Imports 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.IO; 24 | using System.IO.Compression; 25 | using System.Linq; 26 | using System.Net.Http; 27 | using System.Net.Http.Headers; 28 | using System.Text; 29 | using Mannex.IO; 30 | using Mime; 31 | 32 | #endregion 33 | 34 | public sealed class ZipEntry 35 | { 36 | string _fileName; 37 | string _dirPath; 38 | 39 | public string Name { get; } 40 | public string DirPath => DirPathFileNameFault._dirPath; 41 | public string FileName => DirPathFileNameFault._fileName; 42 | public long Length { get; } 43 | public DateTimeOffset Time { get; } 44 | 45 | public ZipEntry(string name, long length, DateTimeOffset time) 46 | { 47 | Name = name; 48 | Length = length; 49 | Time = time; 50 | } 51 | 52 | ZipEntry DirPathFileNameFault 53 | { 54 | get 55 | { 56 | if (_fileName == null) 57 | SplitDirPathFileName(Name, out _dirPath, out _fileName); 58 | return this; 59 | } 60 | } 61 | 62 | static readonly char[] PathSeparators = { '/', '\\' }; 63 | 64 | static void SplitDirPathFileName(string name, out string dirPath, out string fileName) 65 | { 66 | var i = name.LastIndexOfAny(PathSeparators); 67 | if (i < 0) 68 | { 69 | dirPath = string.Empty; 70 | fileName = name; 71 | } 72 | else 73 | { 74 | dirPath = name.Substring(0, i + 1); 75 | fileName = name.Substring(i + 1); 76 | } 77 | } 78 | 79 | public override string ToString() => Name; 80 | } 81 | 82 | public sealed class Zip 83 | { 84 | readonly string _path; 85 | 86 | public Zip(string path) { _path = path; } 87 | 88 | public IEnumerable GetEntries() => 89 | GetEntries((name, length, _, time) => new ZipEntry(name, length, time)); 90 | 91 | public IEnumerable GetEntries(Func selector) 92 | { 93 | using (var zip = Open()) 94 | foreach (var e in zip.Entries) 95 | yield return selector(e.FullName, e.Length, e.CompressedLength, e.LastWriteTime); 96 | } 97 | 98 | public IEnumerable Extract(Func extractor) => 99 | Extract(null, extractor); 100 | 101 | public IEnumerable Extract(Func predicate) => 102 | Extract(predicate, ZipExtractors.Buffer); 103 | 104 | public IEnumerable Extract(Func predicate, 105 | Func extractor) 106 | { 107 | using (var zip = Open()) 108 | foreach (var e in from e in zip.Entries 109 | let ext = new ZipEntry(e.FullName, e.Length, e.LastWriteTime) 110 | where predicate?.Invoke(ext) ?? true 111 | select new { Int = e, Ext = ext }) 112 | { 113 | using (var s = e.Int.Open()) 114 | yield return extractor(e.Ext, s); 115 | } 116 | } 117 | 118 | ZipArchive Open() 119 | { 120 | var input = File.OpenRead(_path); 121 | try 122 | { 123 | var zip = new ZipArchive(input, ZipArchiveMode.Read, leaveOpen: false); 124 | input = null; // ownership transferred 125 | return zip; 126 | } 127 | finally 128 | { 129 | input?.Dispose(); 130 | } 131 | } 132 | 133 | public override string ToString() => _path; 134 | 135 | } 136 | 137 | public sealed class ZipExtractors 138 | { 139 | public static string Utf8Text(ZipEntry entry, Stream input) => 140 | Text(input, Encoding.UTF8); 141 | 142 | public static Func Text(Encoding encoding) => 143 | (_, input) => Text(input, encoding); 144 | 145 | static string Text(Stream input, Encoding encoding) 146 | { 147 | using (var reader = new StreamReader(input, encoding)) 148 | return reader.ReadToEnd(); 149 | } 150 | 151 | public static HttpContent HttpContent(ZipEntry entry, Stream input) => 152 | HttpContent(entry, input, null); 153 | 154 | public static Func HttpContent(Func mimeMapper) => 155 | (e, input) => HttpContent(e, input, mimeMapper); 156 | 157 | static HttpContent HttpContent(ZipEntry entry, Stream input, Func mimeMapper) 158 | { 159 | if (entry.FileName.Length == 0) throw new ArgumentException("ZIP entry must be a file.", nameof(entry)); 160 | 161 | var content = new StreamContent(input); 162 | var headers = content.Headers; 163 | headers.ContentLength = entry.Length; 164 | headers.ContentType = new MediaTypeHeaderValue((mimeMapper ?? MimeMapping.FindMimeTypeFromFileName)(entry.FileName)); 165 | headers.ContentDisposition = new ContentDispositionHeaderValue("attachment") 166 | { 167 | FileName = entry.FileName, 168 | ModificationDate = entry.Time 169 | }; 170 | return content; 171 | } 172 | 173 | public static Stream Memorize(ZipEntry entry, Stream input) => 174 | input.Memorize(); 175 | 176 | public static byte[] Buffer(ZipEntry entry, Stream input) 177 | { 178 | if (entry.Length > int.MaxValue) 179 | throw new Exception($"Uncompressed stream is too large ({entry.Length:N0} bytes) to buffer."); 180 | 181 | var length = (int) entry.Length; 182 | var buffer = new byte[length]; 183 | var offset = 0; 184 | int read; 185 | do 186 | { 187 | read = input.Read(buffer, offset, length); 188 | offset += read; 189 | length -= read; 190 | } 191 | while (read > 0 && length > 0); 192 | return buffer; 193 | } 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /src/Core/Sys/SpawnOptions.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq.Sys 18 | { 19 | #region Imports 20 | 21 | using System; 22 | using System.Collections; 23 | using System.Collections.Generic; 24 | using System.Collections.Immutable; 25 | using System.Diagnostics; 26 | using System.Linq; 27 | using System.Runtime.InteropServices; 28 | using Mannex.Collections.Generic; 29 | 30 | #endregion 31 | 32 | public sealed class SpawnOptions 33 | { 34 | public static SpawnOptions Create() => 35 | new SpawnOptions(ProgramArguments.Empty, 36 | System.Environment.CurrentDirectory, 37 | ImmutableArray.CreateRange(from DictionaryEntry e in System.Environment.GetEnvironmentVariables() 38 | select ((string)e.Key).AsKeyTo((string)e.Value))); 39 | 40 | SpawnOptions(ProgramArguments arguments, string workingDirectory, ImmutableArray> environment) 41 | { 42 | Arguments = arguments; 43 | WorkingDirectory = workingDirectory; 44 | Environment = environment; 45 | } 46 | 47 | public SpawnOptions(SpawnOptions other) : 48 | this(other.Arguments, other.WorkingDirectory, other.Environment) {} 49 | 50 | public ProgramArguments Arguments { get; private set; } 51 | public string WorkingDirectory { get; private set; } 52 | public ImmutableArray> Environment { get; private set; } 53 | 54 | public SpawnOptions WithEnvironment(ImmutableArray> value) => 55 | Environment.IsEmpty && value.IsEmpty ? this : new SpawnOptions(this) { Environment = value }; 56 | 57 | public SpawnOptions WithWorkingDirectory(string value) 58 | => value is null ? throw new ArgumentNullException(nameof(value)) 59 | : value == WorkingDirectory ? this 60 | : new SpawnOptions(this) { WorkingDirectory = value }; 61 | 62 | public SpawnOptions WithArguments(ProgramArguments value) 63 | => value is null ? throw new ArgumentNullException(nameof(value)) 64 | : value == Arguments || value.Count == 0 && Arguments.Count == 0 ? this 65 | : new SpawnOptions(this) { Arguments = value }; 66 | } 67 | 68 | public static class SpawnOptionsExtensions 69 | { 70 | public static SpawnOptions AddArgument(this SpawnOptions options, string value) 71 | => options is null 72 | ? throw new ArgumentNullException(nameof(options)) 73 | : options.WithArguments(ProgramArguments.From(options.Arguments.Append(value))); 74 | 75 | public static SpawnOptions AddArgument(this SpawnOptions options, params string[] values) => 76 | options.AddArguments(values); 77 | 78 | public static SpawnOptions AddArguments(this SpawnOptions options, IEnumerable values) 79 | => options is null 80 | ? throw new ArgumentNullException(nameof(options)) 81 | : options.WithArguments(ProgramArguments.From(options.Arguments.Concat(values))); 82 | 83 | public static SpawnOptions ClearArguments(this SpawnOptions options) 84 | => options is null 85 | ? throw new ArgumentNullException(nameof(options)) 86 | : options.WithArguments(ProgramArguments.Empty); 87 | 88 | public static SpawnOptions SetCommandLine(this SpawnOptions options, string value) 89 | => options is null 90 | ? throw new ArgumentNullException(nameof(options)) 91 | : options.WithArguments(ProgramArguments.Parse(value)); 92 | 93 | public static SpawnOptions AddEnvironment(this SpawnOptions options, string name, string value) 94 | { 95 | if (options is null) throw new ArgumentNullException(nameof(options)); 96 | if (name is null) throw new ArgumentNullException(nameof(name)); 97 | if (name.Length == 0) throw new ArgumentException(null, nameof(name)); 98 | if (value is null) throw new ArgumentNullException(nameof(value)); 99 | 100 | return options.WithEnvironment(options.Environment.Add(name.AsKeyTo(value))); 101 | } 102 | 103 | public static SpawnOptions ClearEnvironment(this SpawnOptions options) 104 | => options is null ? throw new ArgumentNullException(nameof(options)) 105 | : options.WithEnvironment(ImmutableArray>.Empty); 106 | 107 | public static SpawnOptions SetEnvironment(this SpawnOptions options, string name, string value) 108 | { 109 | if (options is null) throw new ArgumentNullException(nameof(options)); 110 | var updateOptions = options.UnsetEnvironment(name); 111 | return value is null ? updateOptions : updateOptions.AddEnvironment(name, value); 112 | } 113 | 114 | public static SpawnOptions UnsetEnvironment(this SpawnOptions options, string name) 115 | { 116 | if (options is null) throw new ArgumentNullException(nameof(options)); 117 | if (name is null) throw new ArgumentNullException(nameof(name)); 118 | if (name.Length == 0) throw new ArgumentException(null, nameof(name)); 119 | 120 | var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); 121 | var comparison = isWindows ? StringComparison.OrdinalIgnoreCase 122 | : StringComparison.Ordinal; 123 | 124 | var found = false; 125 | foreach (var e in options.Environment) 126 | { 127 | if (found = string.Equals(e.Key, name, comparison)) 128 | break; 129 | } 130 | 131 | return found 132 | ? options.WithEnvironment(ImmutableArray.CreateRange( 133 | from e in options.Environment 134 | where !string.Equals(e.Key, name, comparison) 135 | select e)) 136 | : options; 137 | } 138 | 139 | public static void Update(this SpawnOptions options, ProcessStartInfo startInfo) 140 | { 141 | if (options is null) throw new ArgumentNullException(nameof(options)); 142 | if (startInfo is null) throw new ArgumentNullException(nameof(startInfo)); 143 | 144 | startInfo.WorkingDirectory = options.WorkingDirectory; 145 | 146 | var environment = startInfo.Environment; 147 | environment.Clear(); 148 | foreach (var e in options.Environment) 149 | environment.Add(e.Key, e.Value); 150 | } 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /src/Core/Html/HtmlInputType.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq.Html 18 | { 19 | using System; 20 | using System.Collections.Generic; 21 | using System.Linq; 22 | using System.Reflection; 23 | using Mannex.Collections.Generic; 24 | 25 | /// 26 | /// Represents a control 27 | /// type that can be specified with the HTML INPUT tag. 28 | /// 29 | 30 | public enum KnownHtmlInputType 31 | { 32 | Other, 33 | // https://www.w3.org/TR/html5/forms.html#attr-input-type 34 | Hidden , // Hidden An arbitrary string 35 | Text , // Text Text with no line breaks 36 | // TODO Search , // Search Text with no line breaks 37 | Tel , // Telephone Text with no line breaks 38 | // TODO Url , // URL An absolute URL 39 | Email , // E-mail An e-mail address or list of e-mail addresses 40 | Password, // Password Text with no line breaks (sensitive information) 41 | Date , // Date A date (year, month, day) with no time zone 42 | Time , // Time A time (hour, minute, seconds, fractional seconds) with no time zone 43 | Number , // Number A numerical value 44 | // TODO Range , // Range A numerical value, with the extra semantic that the exact value is not important 45 | // TODO Color , // Color An sRGB color with 8-bit red, green, and blue components 46 | Checkbox, // Checkbox A set of zero or more values from a predefined list 47 | Radio , // Radio Button An enumerated value 48 | File , // File Upload Zero or more files each with a MIME type and optionally a file name 49 | Submit , // Submit Button An enumerated value, with the extra semantic that it must be the last value selected and initiates form submission 50 | Image , // Image Button A coordinate, relative to a particular image's size, with the extra semantic that it must be the last value selected and initiates form submission 51 | Reset , // Reset Button n/a 52 | Button , // Button n/a 53 | } 54 | 55 | public sealed class HtmlInputType : IEquatable 56 | { 57 | readonly string _type; 58 | 59 | public KnownHtmlInputType KnownType { get; } 60 | 61 | public static readonly HtmlInputType Text = new HtmlInputType(KnownHtmlInputType.Text , "text"); 62 | public static readonly HtmlInputType Email = new HtmlInputType(KnownHtmlInputType.Email , "email"); 63 | public static readonly HtmlInputType Tel = new HtmlInputType(KnownHtmlInputType.Tel , "tel"); 64 | public static readonly HtmlInputType Number = new HtmlInputType(KnownHtmlInputType.Number , "number"); 65 | public static readonly HtmlInputType Date = new HtmlInputType(KnownHtmlInputType.Date , "date"); 66 | public static readonly HtmlInputType Time = new HtmlInputType(KnownHtmlInputType.Time , "time"); 67 | public static readonly HtmlInputType Password = new HtmlInputType(KnownHtmlInputType.Password, "password"); 68 | public static readonly HtmlInputType Checkbox = new HtmlInputType(KnownHtmlInputType.Checkbox, "checkbox"); 69 | public static readonly HtmlInputType Radio = new HtmlInputType(KnownHtmlInputType.Radio , "radio"); 70 | public static readonly HtmlInputType Submit = new HtmlInputType(KnownHtmlInputType.Submit , "submit"); 71 | public static readonly HtmlInputType Reset = new HtmlInputType(KnownHtmlInputType.Reset , "reset"); 72 | public static readonly HtmlInputType File = new HtmlInputType(KnownHtmlInputType.File , "file"); 73 | public static readonly HtmlInputType Hidden = new HtmlInputType(KnownHtmlInputType.Hidden , "hidden"); 74 | public static readonly HtmlInputType Image = new HtmlInputType(KnownHtmlInputType.Image , "image"); 75 | public static readonly HtmlInputType Button = new HtmlInputType(KnownHtmlInputType.Button , "button"); 76 | 77 | public static HtmlInputType Default => Text; 78 | 79 | static Dictionary _lookup; 80 | static Dictionary Lookup => _lookup ?? (_lookup = CreateLookup()); 81 | 82 | static Dictionary CreateLookup() 83 | { 84 | var types = 85 | from f in typeof(HtmlInputType).GetFields(BindingFlags.Public | BindingFlags.Static) 86 | where f.FieldType == typeof(HtmlInputType) 87 | select (HtmlInputType) f.GetValue(null); 88 | 89 | return types.ToDictionary(e => e.ToString(), e => e, StringComparer.OrdinalIgnoreCase); 90 | } 91 | 92 | public static HtmlInputType Parse(string input) 93 | { 94 | if (input == null) throw new ArgumentNullException(nameof(input)); 95 | if (input.Length == 0) throw new ArgumentException(null, nameof(input)); 96 | 97 | var type = Lookup.Find(input); 98 | return type == null ? new HtmlInputType(KnownHtmlInputType.Other, input) 99 | : type.ToString() == input ? type 100 | : new HtmlInputType(type.KnownType, input); 101 | } 102 | 103 | public HtmlInputType(KnownHtmlInputType knowType, string type) 104 | { 105 | if (type == null) throw new ArgumentNullException(nameof(type)); 106 | if (type.Length == 0) throw new ArgumentException(null, nameof(type)); 107 | 108 | // ReSharper disable once ForCanBeConvertedToForeach 109 | // ReSharper disable once LoopCanBeConvertedToQuery 110 | for (var i = 0; i < type.Length; i++) 111 | { 112 | var ch = type[i] & ~0x20; 113 | if ((ch < '0' || ch > '9') && (ch < 'A' || ch > 'Z')) 114 | throw new ArgumentException(null, nameof(type)); 115 | } 116 | 117 | _type = type; 118 | KnownType = knowType; 119 | } 120 | 121 | public bool Equals(HtmlInputType other) => 122 | other != null 123 | && ((other.KnownType == KnownHtmlInputType.Other 124 | && string.Equals(_type, other._type, StringComparison.OrdinalIgnoreCase)) 125 | || KnownType == other.KnownType); 126 | 127 | public override bool Equals(object obj) => Equals(obj as HtmlInputType); 128 | 129 | public override int GetHashCode() => 130 | KnownType == KnownHtmlInputType.Other 131 | ? StringComparer.OrdinalIgnoreCase.GetHashCode(_type) * 397 132 | : (int) KnownType; 133 | 134 | public override string ToString() => _type; 135 | 136 | public static bool operator ==(HtmlInputType a, HtmlInputType b) => 137 | !ReferenceEquals(a, null) ? a.Equals(b) : ReferenceEquals(b, null); 138 | 139 | public static bool operator !=(HtmlInputType a, HtmlInputType b) => !(a == b); 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /tests/HttpConfigTests.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq.Tests 18 | { 19 | using NUnit.Framework; 20 | using System; 21 | using System.Collections.Generic; 22 | using System.Net; 23 | using MoreLinq; 24 | 25 | [TestFixture] 26 | public class HttpConfigTests 27 | { 28 | [Test] 29 | public void DefaultConfigHeadersIsEmpty() 30 | { 31 | Assert.That(HttpConfig.Default.Headers, Is.Empty); 32 | } 33 | 34 | [Test] 35 | public void DefaultConfigCookiesIsNull() 36 | { 37 | Assert.That(HttpConfig.Default.Cookies, Is.Null); 38 | } 39 | 40 | [Test] 41 | public void DefaultConfigIgnoreInvalidServerCertificateIsFalse() 42 | { 43 | Assert.That(HttpConfig.Default.IgnoreInvalidServerCertificate, Is.False); 44 | } 45 | 46 | [Test] 47 | public void DefaultConfigAutomaticCompressionIsNone() 48 | { 49 | Assert.That(HttpConfig.Default.AutomaticDecompression, Is.EqualTo(DecompressionMethods.None)); 50 | } 51 | 52 | [Test] 53 | public void WithHeader() 54 | { 55 | var config = HttpConfig.Default 56 | .WithHeader("name1", "value1") 57 | .WithHeader("name2", "value2") 58 | .WithHeader("name3", "value3"); 59 | 60 | var entry = config.Headers.Fold((e1, e2, e3) => new 61 | { 62 | First = e3, Second = e2, Third = e1, 63 | }); 64 | 65 | Assert.That(entry.First.Key, Is.EqualTo("name1")); 66 | Assert.That(entry.First.Value.Single(), Is.EqualTo("value1")); 67 | 68 | Assert.That(entry.Second.Key, Is.EqualTo("name2")); 69 | Assert.That(entry.Second.Value.Single(), Is.EqualTo("value2")); 70 | 71 | Assert.That(entry.Third.Key, Is.EqualTo("name3")); 72 | Assert.That(entry.Third.Value.Single(), Is.EqualTo("value3")); 73 | 74 | AssertDefaultConfigEqual(config, ConfigAssertion.All.Except(ConfigAssertion.Headers)); 75 | } 76 | 77 | [Test] 78 | public void WithHeaders() 79 | { 80 | var headers = HttpHeaderCollection.Empty 81 | .Set("name1", "value1") 82 | .Set("name2", "value2") 83 | .Set("name3", "value3"); 84 | 85 | var config = HttpConfig.Default.WithHeaders(headers); 86 | 87 | Assert.That(config.Headers, Is.SameAs(headers)); 88 | AssertDefaultConfigEqual(config, ConfigAssertion.All.Except(ConfigAssertion.Headers)); 89 | } 90 | 91 | [Test] 92 | public void WithTimeout() 93 | { 94 | var config = HttpConfig.Default.WithTimeout(new TimeSpan(0, 1, 0)); 95 | 96 | Assert.That(config.Timeout, Is.EqualTo(new TimeSpan(0, 1, 0))); 97 | AssertDefaultConfigEqual(config, ConfigAssertion.All.Except(ConfigAssertion.Timeout)); 98 | } 99 | 100 | [Test] 101 | public void WithUserAgent() 102 | { 103 | var config = HttpConfig.Default.WithUserAgent("Spider/1.0"); 104 | 105 | Assert.That(config.UserAgent, Is.EqualTo("Spider/1.0")); 106 | AssertDefaultConfigEqual(config, ConfigAssertion.All.Except(ConfigAssertion.UserAgent)); 107 | } 108 | 109 | [Test] 110 | public void WithAutomaticDecompression() 111 | { 112 | var deflate = DecompressionMethods.Deflate; 113 | var config = HttpConfig.Default.WithAutomaticDecompression(deflate); 114 | 115 | Assert.That(config.AutomaticDecompression, Is.EqualTo(deflate)); 116 | AssertDefaultConfigEqual(config, ConfigAssertion.All.Except(ConfigAssertion.AutomaticDecompression)); 117 | } 118 | 119 | [Test] 120 | public void WithCredentials() 121 | { 122 | var credentials = new NetworkCredential("admin", "admin"); 123 | var config = HttpConfig.Default.WithCredentials(credentials); 124 | 125 | Assert.That(config.Credentials, Is.SameAs(credentials)); 126 | AssertDefaultConfigEqual(config, ConfigAssertion.All.Except(ConfigAssertion.Credentials)); 127 | } 128 | 129 | [Test] 130 | public void WithUseDefaultCredentials() 131 | { 132 | var config = HttpConfig.Default.WithUseDefaultCredentials(true); 133 | 134 | Assert.That(config.UseDefaultCredentials, Is.True); 135 | Assert.That(config.Credentials, Is.EqualTo(HttpConfig.Default.Credentials)); 136 | AssertDefaultConfigEqual(config, ConfigAssertion.All.Except(ConfigAssertion.UseDefaultCredentials)); 137 | } 138 | 139 | [Test] 140 | public void WithCookies() 141 | { 142 | var cookies = new[] { new Cookie("name", "value") }; 143 | var config = HttpConfig.Default.WithCookies(cookies); 144 | 145 | Assert.That(config.Cookies, Is.SameAs(cookies)); 146 | AssertDefaultConfigEqual(config, ConfigAssertion.All.Except(ConfigAssertion.Cookies)); 147 | } 148 | 149 | [Test] 150 | public void WithIgnoreInvalidServerCertificate() 151 | { 152 | var config = HttpConfig.Default.WithIgnoreInvalidServerCertificate(true); 153 | 154 | Assert.That(config.IgnoreInvalidServerCertificate, Is.True); 155 | AssertDefaultConfigEqual(config, ConfigAssertion.All.Except(ConfigAssertion.IgnoreInvalidServerCertificate)); 156 | } 157 | 158 | static class ConfigAssertion 159 | { 160 | public static readonly Action Headers = (actual, expected) => Assert.That(actual.Headers, Is.EqualTo(expected.Headers)); 161 | public static readonly Action Timeout = (actual, expected) => Assert.That(actual.Timeout, Is.EqualTo(expected.Timeout)); 162 | public static readonly Action UseDefaultCredentials = (actual, expected) => Assert.That(actual.UseDefaultCredentials, Is.EqualTo(expected.UseDefaultCredentials)); 163 | public static readonly Action Credentials = (actual, expected) => Assert.That(actual.Credentials, Is.SameAs(expected.Credentials)); 164 | public static readonly Action UserAgent = (actual, expected) => Assert.That(actual.UserAgent, Is.EqualTo(expected.UserAgent)); 165 | public static readonly Action AutomaticDecompression = (actual, expected) => Assert.That(actual.AutomaticDecompression, Is.EqualTo(expected.AutomaticDecompression)); 166 | public static readonly Action Cookies = (actual, expected) => Assert.That(actual.Cookies, Is.SameAs(expected.Cookies)); 167 | public static readonly Action IgnoreInvalidServerCertificate = (actual, expected) => Assert.That(actual.IgnoreInvalidServerCertificate, Is.EqualTo(expected.IgnoreInvalidServerCertificate)); 168 | 169 | public static IEnumerable> All 170 | { 171 | get 172 | { 173 | yield return Headers; 174 | yield return Timeout; 175 | yield return UseDefaultCredentials; 176 | yield return Credentials; 177 | yield return UserAgent; 178 | yield return AutomaticDecompression; 179 | yield return Cookies; 180 | yield return IgnoreInvalidServerCertificate; 181 | } 182 | } 183 | } 184 | 185 | public static void AssertDefaultConfigEqual(HttpConfig config, IEnumerable> assertions) => 186 | assertions.ForEach(a => a(HttpConfig.Default, config)); 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /src/Core/Sys/Spawner.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq.Sys 18 | { 19 | #region Imports 20 | 21 | using System; 22 | using System.Collections.Concurrent; 23 | using System.Collections.Generic; 24 | using System.Diagnostics; 25 | using System.IO; 26 | using System.Linq; 27 | using System.Reactive.Linq; 28 | using System.Threading.Tasks; 29 | using Mannex; 30 | using Mannex.Collections.Generic; 31 | using Mannex.Diagnostics; 32 | 33 | #endregion 34 | 35 | public interface ISpawnObservable : IObservable 36 | { 37 | SpawnOptions Options { get; } 38 | ISpawnObservable WithOptions(SpawnOptions options); 39 | } 40 | 41 | static class SpawnObservable 42 | { 43 | public static ISpawnObservable Create(SpawnOptions options, Func, SpawnOptions, IDisposable> subscriber) => 44 | new Implementation(options, subscriber); 45 | 46 | sealed class Implementation : ISpawnObservable 47 | { 48 | readonly Func, SpawnOptions, IDisposable> _subscriber; 49 | 50 | public Implementation(SpawnOptions options, Func, SpawnOptions, IDisposable> subscriber) 51 | { 52 | Options = options ?? throw new ArgumentNullException(nameof(options)); 53 | _subscriber = subscriber ?? throw new ArgumentNullException(nameof(subscriber)); 54 | } 55 | 56 | public SpawnOptions Options { get; } 57 | 58 | public ISpawnObservable WithOptions(SpawnOptions value) => 59 | ReferenceEquals(Options, value) ? this : Create(value, _subscriber); 60 | 61 | public IDisposable Subscribe(IObserver observer) => 62 | _subscriber(observer, Options); 63 | } 64 | } 65 | 66 | public interface ISpawner 67 | { 68 | IObservable Spawn(string path, SpawnOptions options, Func stdoutSelector, Func stderrSelector); 69 | } 70 | 71 | public static class SpawnerExtensions 72 | { 73 | public static ISpawnObservable AddArgument(this ISpawnObservable source, string value) => 74 | source.WithOptions(source.Options.AddArgument(value)); 75 | 76 | public static ISpawnObservable AddArgument(this ISpawnObservable source, params string[] values) => 77 | source.WithOptions(source.Options.AddArgument(values)); 78 | 79 | public static ISpawnObservable AddArguments(this ISpawnObservable source, IEnumerable values) => 80 | source.WithOptions(source.Options.AddArguments(values)); 81 | 82 | public static ISpawnObservable ClearArguments(this ISpawnObservable source) => 83 | source.WithOptions(source.Options.ClearArguments()); 84 | 85 | public static ISpawnObservable SetCommandLine(this ISpawnObservable source, string value) => 86 | source.WithOptions(source.Options.SetCommandLine(value)); 87 | 88 | public static ISpawnObservable ClearEnvironment(this ISpawnObservable source) => 89 | source.WithOptions(source.Options.ClearEnvironment()); 90 | 91 | public static ISpawnObservable AddEnvironment(this ISpawnObservable source, string name, string value) => 92 | source.WithOptions(source.Options.AddEnvironment(name, value)); 93 | 94 | public static ISpawnObservable SetEnvironment(this ISpawnObservable source, string name, string value) => 95 | source.WithOptions(source.Options.SetEnvironment(name, value)); 96 | 97 | public static ISpawnObservable UnsetEnvironment(this ISpawnObservable source, string name) => 98 | source.WithOptions(source.Options.UnsetEnvironment(name)); 99 | 100 | public static ISpawnObservable WorkingDirectory(this ISpawnObservable source, string value) => 101 | source.WithOptions(source.Options.WithWorkingDirectory(value)); 102 | 103 | public static ISpawnObservable Spawn(this ISpawner spawner, string path, ProgramArguments args) => 104 | spawner.Spawn(path, args, output => output, null); 105 | 106 | public static ISpawnObservable> 107 | Spawn(this ISpawner spawner, 108 | string path, ProgramArguments args, 109 | T stdoutKey, T stderrKey) => 110 | spawner.Spawn(path, args, stdout => stdoutKey.AsKeyTo(stdout), 111 | stderr => stderrKey.AsKeyTo(stderr)); 112 | 113 | public static ISpawnObservable Spawn(this ISpawner spawner, string path, ProgramArguments args, Func stdoutSelector, Func stderrSelector) => 114 | SpawnObservable.Create(SpawnOptions.Create().WithArguments(args), 115 | (observer, options) => 116 | spawner.Spawn(path, options, stdoutSelector, stderrSelector).Subscribe(observer)); 117 | } 118 | 119 | public static class Spawner 120 | { 121 | public static ISpawner Default => new SysSpawner(); 122 | 123 | sealed class SysSpawner : ISpawner 124 | { 125 | public IObservable Spawn(string path, SpawnOptions options, Func stdoutSelector, Func stderrSelector) => 126 | SpawnCore(path, options, stdoutSelector, stderrSelector).ToObservable(); 127 | } 128 | 129 | // TODO Make true observable 130 | static IEnumerable SpawnCore(string path, SpawnOptions options, Func stdoutSelector, Func stderrSelector) 131 | { 132 | var psi = new ProcessStartInfo(path, options.Arguments.ToString()) 133 | { 134 | CreateNoWindow = true, 135 | UseShellExecute = false, 136 | RedirectStandardOutput = true, 137 | RedirectStandardError = true, 138 | }; 139 | 140 | options.Update(psi); 141 | 142 | using (var process = Process.Start(psi)) 143 | { 144 | Debug.Assert(process != null); 145 | 146 | var bc = new BlockingCollection>(); 147 | var drainer = process.BeginReadLineAsync(stdoutSelector != null ? (stdout => bc.Add(Tuple.Create(stdoutSelector(stdout), default(Exception)))) : (Action)null, 148 | stderrSelector != null ? (stderr => bc.Add(Tuple.Create(stderrSelector(stderr), default(Exception)))) : (Action)null); 149 | 150 | Task.Run(async () => // ReSharper disable AccessToDisposedClosure 151 | { 152 | try 153 | { 154 | var pid = process.Id; 155 | var error = await 156 | process.AsTask(dispose: false, 157 | p => p.ExitCode != 0 ? new Exception($"Process \"{Path.GetFileName(path)}\" (launched as the ID {pid}) ended with the non-zero exit code {p.ExitCode}.") 158 | : null, 159 | e => e, 160 | e => e) 161 | .DontContinueOnCapturedContext(); 162 | 163 | await drainer(null).DontContinueOnCapturedContext(); 164 | 165 | if (error != null) 166 | throw error; 167 | 168 | bc.CompleteAdding(); 169 | } 170 | catch (Exception e) 171 | { 172 | bc.Add(Tuple.Create(default(T), e)); 173 | } 174 | 175 | // ReSharper restore AccessToDisposedClosure 176 | }); 177 | 178 | foreach (var e in from e in bc.GetConsumingEnumerable() 179 | select e.Fold((res, err) => new { Result = res, Error = err })) 180 | { 181 | if (e.Error != null) 182 | throw e.Error; 183 | yield return e.Result; 184 | } 185 | } 186 | } 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /src/Core/Sys/PasteArguments.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) .NET Foundation and Contributors 2 | // 3 | // The MIT License (MIT) 4 | // 5 | // All rights reserved. 6 | // 7 | // Permission is hereby granted, free of charge, to any person obtaining a copy 8 | // of this software and associated documentation files (the "Software"), to deal 9 | // in the Software without restriction, including without limitation the rights 10 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | // copies of the Software, and to permit persons to whom the Software is 12 | // furnished to do so, subject to the following conditions: 13 | // 14 | // The above copyright notice and this permission notice shall be included in all 15 | // copies or substantial portions of the Software. 16 | // 17 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | // SOFTWARE. 24 | // 25 | #endregion 26 | 27 | // https://github.com/dotnet/corefx/blob/v2.1.4/src/Common/src/System/PasteArguments.cs 28 | 29 | using System.Text; 30 | using System.Collections.Generic; 31 | 32 | namespace System 33 | { 34 | internal static partial class PasteArguments 35 | { 36 | internal static void AppendArgument(StringBuilder stringBuilder, string argument) 37 | { 38 | if (stringBuilder.Length != 0) 39 | { 40 | stringBuilder.Append(' '); 41 | } 42 | 43 | // Parsing rules for non-argv[0] arguments: 44 | // - Backslash is a normal character except followed by a quote. 45 | // - 2N backslashes followed by a quote ==> N literal backslashes followed by unescaped quote 46 | // - 2N+1 backslashes followed by a quote ==> N literal backslashes followed by a literal quote 47 | // - Parsing stops at first whitespace outside of quoted region. 48 | // - (post 2008 rule): A closing quote followed by another quote ==> literal quote, and parsing remains in quoting mode. 49 | if (argument.Length != 0 && ContainsNoWhitespaceOrQuotes(argument)) 50 | { 51 | // Simple case - no quoting or changes needed. 52 | stringBuilder.Append(argument); 53 | } 54 | else 55 | { 56 | stringBuilder.Append(Quote); 57 | int idx = 0; 58 | while (idx < argument.Length) 59 | { 60 | char c = argument[idx++]; 61 | if (c == Backslash) 62 | { 63 | int numBackSlash = 1; 64 | while (idx < argument.Length && argument[idx] == Backslash) 65 | { 66 | idx++; 67 | numBackSlash++; 68 | } 69 | 70 | if (idx == argument.Length) 71 | { 72 | // We'll emit an end quote after this so must double the number of backslashes. 73 | stringBuilder.Append(Backslash, numBackSlash * 2); 74 | } 75 | else if (argument[idx] == Quote) 76 | { 77 | // Backslashes will be followed by a quote. Must double the number of backslashes. 78 | stringBuilder.Append(Backslash, numBackSlash * 2 + 1); 79 | stringBuilder.Append(Quote); 80 | idx++; 81 | } 82 | else 83 | { 84 | // Backslash will not be followed by a quote, so emit as normal characters. 85 | stringBuilder.Append(Backslash, numBackSlash); 86 | } 87 | 88 | continue; 89 | } 90 | 91 | if (c == Quote) 92 | { 93 | // Escape the quote so it appears as a literal. This also guarantees that we won't end up generating a closing quote followed 94 | // by another quote (which parses differently pre-2008 vs. post-2008.) 95 | stringBuilder.Append(Backslash); 96 | stringBuilder.Append(Quote); 97 | continue; 98 | } 99 | 100 | stringBuilder.Append(c); 101 | } 102 | 103 | stringBuilder.Append(Quote); 104 | } 105 | } 106 | 107 | private static bool ContainsNoWhitespaceOrQuotes(string s) 108 | { 109 | for (int i = 0; i < s.Length; i++) 110 | { 111 | char c = s[i]; 112 | if (char.IsWhiteSpace(c) || c == Quote) 113 | { 114 | return false; 115 | } 116 | } 117 | 118 | return true; 119 | } 120 | 121 | private const char Quote = '\"'; 122 | private const char Backslash = '\\'; 123 | } 124 | } 125 | 126 | // https://github.com/dotnet/corefx/blob/v2.1.4/src/Common/src/System/PasteArguments.Unix.cs 127 | 128 | namespace System 129 | { 130 | internal static partial class PasteArguments 131 | { 132 | /// 133 | /// Repastes a set of arguments into a linear string that parses back into the originals under pre- or post-2008 VC parsing rules. 134 | /// On Unix: the rules for parsing the executable name (argv[0]) are ignored. 135 | /// 136 | internal static string PasteForUnix(IEnumerable arguments, bool pasteFirstArgumentUsingArgV0Rules) 137 | { 138 | var stringBuilder = new StringBuilder(); 139 | foreach (string argument in arguments) 140 | { 141 | AppendArgument(stringBuilder, argument); 142 | } 143 | return stringBuilder.ToString(); 144 | } 145 | 146 | } 147 | } 148 | 149 | // https://github.com/dotnet/corefx/blob/v2.1.4/src/Common/src/System/PasteArguments.Windows.cs 150 | 151 | namespace System 152 | { 153 | internal static partial class PasteArguments 154 | { 155 | /// 156 | /// Repastes a set of arguments into a linear string that parses back into the originals under pre- or post-2008 VC parsing rules. 157 | /// The rules for parsing the executable name (argv[0]) are special, so you must indicate whether the first argument actually is argv[0]. 158 | /// 159 | internal static string PasteForWindows(IEnumerable arguments, bool pasteFirstArgumentUsingArgV0Rules) 160 | { 161 | var stringBuilder = new StringBuilder(); 162 | 163 | foreach (string argument in arguments) 164 | { 165 | if (pasteFirstArgumentUsingArgV0Rules) 166 | { 167 | pasteFirstArgumentUsingArgV0Rules = false; 168 | 169 | // Special rules for argv[0] 170 | // - Backslash is a normal character. 171 | // - Quotes used to include whitespace characters. 172 | // - Parsing ends at first whitespace outside quoted region. 173 | // - No way to get a literal quote past the parser. 174 | 175 | bool hasWhitespace = false; 176 | foreach (char c in argument) 177 | { 178 | if (c == Quote) 179 | { 180 | throw new ApplicationException("The argv[0] argument cannot include a double quote."); 181 | } 182 | if (char.IsWhiteSpace(c)) 183 | { 184 | hasWhitespace = true; 185 | } 186 | } 187 | if (argument.Length == 0 || hasWhitespace) 188 | { 189 | stringBuilder.Append(Quote); 190 | stringBuilder.Append(argument); 191 | stringBuilder.Append(Quote); 192 | } 193 | else 194 | { 195 | stringBuilder.Append(argument); 196 | } 197 | } 198 | else 199 | { 200 | AppendArgument(stringBuilder, argument); 201 | } 202 | } 203 | 204 | return stringBuilder.ToString(); 205 | } 206 | 207 | } 208 | } 209 | 210 | namespace System 211 | { 212 | using Runtime.InteropServices; 213 | 214 | internal static partial class PasteArguments 215 | { 216 | internal static string Paste(IEnumerable arguments, bool pasteFirstArgumentUsingArgV0Rules = false) 217 | => RuntimeInformation.IsOSPlatform(OSPlatform.Windows) 218 | ? PasteForWindows(arguments, pasteFirstArgumentUsingArgV0Rules) 219 | : PasteForUnix(arguments, pasteFirstArgumentUsingArgV0Rules); 220 | } 221 | } 222 | -------------------------------------------------------------------------------- /tests/SpawnTests.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq.Tests 18 | { 19 | using System; 20 | using NUnit.Framework; 21 | using Sys; 22 | using System.Reactive.Linq; 23 | using System.Collections.Generic; 24 | using System.Linq; 25 | using Mannex.Collections.Generic; 26 | 27 | [TestFixture] 28 | public class SpawnTests 29 | { 30 | enum LineKind { Output, Error } 31 | 32 | sealed class Spawner : ISpawner 33 | { 34 | public string Path; 35 | public SpawnOptions Options; 36 | public readonly List<(LineKind Kind, string Line)> Return = new List<(LineKind, string)>(); 37 | 38 | public IObservable Spawn(string path, SpawnOptions options, Func stdoutSelector, Func stderrSelector) 39 | { 40 | Path = path; 41 | Options = options; 42 | var output = 43 | from e in Return 44 | where e.Kind == LineKind.Output || stderrSelector != null 45 | select e.Kind == LineKind.Output ? stdoutSelector(e.Line) : stderrSelector(e.Line); 46 | return output.ToObservable(); 47 | } 48 | } 49 | 50 | [Test] 51 | public void Spawn() 52 | { 53 | var spawner = new Spawner 54 | { 55 | Return = 56 | { 57 | (LineKind.Output, "output"), 58 | (LineKind.Error, "error"), 59 | } 60 | }; 61 | 62 | var spawn = spawner.Spawn("program", ProgramArguments.Var("foo", "bar", "baz"), 63 | s => "STDOUT: " + s.ToUpperInvariant(), 64 | s => "STDERR: " + s.ToUpperInvariant()); 65 | var output = spawn.ToEnumerable().ToArray(); 66 | 67 | Assert.That(spawner.Path, Is.EqualTo("program")); 68 | Assert.That(spawner.Options.Arguments, Is.EqualTo(new[] { "foo", "bar", "baz" })); 69 | Assert.That(spawner.Options, Is.SameAs(spawn.Options)); 70 | Assert.That(output, Is.EqualTo(new[] { "STDOUT: OUTPUT", "STDERR: ERROR" })); 71 | } 72 | 73 | [Test] 74 | public void SpawnReturningOutputOnly() 75 | { 76 | var spawner = new Spawner 77 | { 78 | Return = 79 | { 80 | (LineKind.Output, "output"), 81 | (LineKind.Error, "error"), 82 | } 83 | }; 84 | 85 | var output = 86 | spawner.Spawn("program", ProgramArguments.Var("foo", "bar", "baz")) 87 | .ToEnumerable() 88 | .ToArray(); 89 | 90 | Assert.That(spawner.Path, Is.EqualTo("program")); 91 | Assert.That(spawner.Options.Arguments, Is.EqualTo(new[] { "foo", "bar", "baz" })); 92 | 93 | Assert.That(output, Is.EqualTo(new[] { "output" })); 94 | } 95 | 96 | [Test] 97 | public void SpawnReturningTagged() 98 | { 99 | var spawner = new Spawner 100 | { 101 | Return = 102 | { 103 | (LineKind.Output, "output"), 104 | (LineKind.Error, "error"), 105 | } 106 | }; 107 | 108 | var output = 109 | spawner.Spawn("program", ProgramArguments.Var("foo", "bar", "baz"), 1, 2) 110 | .ToEnumerable() 111 | .ToArray(); 112 | 113 | Assert.That(spawner.Path, Is.EqualTo("program")); 114 | Assert.That(spawner.Options.Arguments, Is.EqualTo(new[] { "foo", "bar", "baz" })); 115 | 116 | Assert.That(output, Is.EqualTo(new[] 117 | { 118 | 1.AsKeyTo("output"), 119 | 2.AsKeyTo("error") 120 | })); 121 | } 122 | 123 | static ISpawnObservable SpawnProgram() => 124 | new Spawner().Spawn("program", ProgramArguments.Empty); 125 | 126 | [Test] 127 | public void ClearEnvironment() 128 | { 129 | var spawn1 = SpawnProgram(); 130 | 131 | Assert.That(spawn1.Options.Environment, Is.Not.Empty); 132 | 133 | var spawn2 = spawn1.ClearEnvironment(); 134 | 135 | Assert.That(spawn2, Is.Not.SameAs(spawn1)); 136 | Assert.That(spawn2.Options, Is.Not.SameAs(spawn1.Options)); 137 | Assert.That(spawn2.Options.Environment, Is.Empty); 138 | } 139 | 140 | [Test] 141 | public void AddEnvironmentWithNullName() 142 | { 143 | var spawn = SpawnProgram(); 144 | 145 | var e = Assert.Throws(() => 146 | spawn.AddEnvironment(null, string.Empty)); 147 | Assert.That(e.ParamName, Is.EqualTo("name")); 148 | } 149 | 150 | [Test] 151 | public void AddEnvironmentWithEmptyName() 152 | { 153 | var spawn = SpawnProgram(); 154 | 155 | var e = Assert.Throws(() => 156 | spawn.AddEnvironment(string.Empty, string.Empty)); 157 | Assert.That(e.ParamName, Is.EqualTo("name")); 158 | } 159 | 160 | [Test] 161 | public void AddEnvironmentWithNullValue() 162 | { 163 | var spawn = SpawnProgram(); 164 | 165 | var e = Assert.Throws(() => 166 | spawn.AddEnvironment("FOO", null)); 167 | Assert.That(e.ParamName, Is.EqualTo("value")); 168 | } 169 | 170 | [Test] 171 | public void AddEnvironment() 172 | { 173 | var options = 174 | SpawnProgram() 175 | .ClearEnvironment() 176 | .AddEnvironment("FOO", "BAR") 177 | .Options; 178 | 179 | Assert.That(options.Environment, Is.EqualTo(new[] 180 | { 181 | KeyValuePair.Create("FOO", "BAR") 182 | })); 183 | 184 | options = options.AddEnvironment("FOO", "BAZ"); 185 | 186 | Assert.That(options.Environment, Is.EqualTo(new[] 187 | { 188 | KeyValuePair.Create("FOO", "BAR"), 189 | KeyValuePair.Create("FOO", "BAZ"), 190 | })); 191 | } 192 | 193 | [Test] 194 | public void SetEnvironmentWithNullName() 195 | { 196 | var spawn = SpawnProgram(); 197 | 198 | var e = Assert.Throws(() => 199 | spawn.SetEnvironment(null, string.Empty)); 200 | Assert.That(e.ParamName, Is.EqualTo("name")); 201 | } 202 | 203 | [Test] 204 | public void SetEnvironmentWithNullValueUnsets() 205 | { 206 | var options = 207 | SpawnProgram() 208 | .ClearEnvironment() 209 | .AddEnvironment("FOO", "BAR") 210 | .AddEnvironment("FOO", "BAZ") 211 | .Options; 212 | 213 | Assert.That(options.Environment, Is.EqualTo(new[] 214 | { 215 | KeyValuePair.Create("FOO", "BAR"), 216 | KeyValuePair.Create("FOO", "BAZ"), 217 | })); 218 | 219 | options = options.SetEnvironment("FOO", null); 220 | 221 | Assert.That(options.Environment, Is.Empty); 222 | } 223 | 224 | [Test] 225 | public void UnsetEnvironmentWithNullName() 226 | { 227 | var spawn = SpawnProgram(); 228 | 229 | var e = Assert.Throws(() => 230 | spawn.UnsetEnvironment(null)); 231 | 232 | Assert.That(e.ParamName, Is.EqualTo("name")); 233 | } 234 | 235 | [Test] 236 | public void UnsetEnvironmentWithEmptyName() 237 | { 238 | var spawn = SpawnProgram(); 239 | 240 | var e = Assert.Throws(() => 241 | spawn.UnsetEnvironment(string.Empty)); 242 | Assert.That(e.ParamName, Is.EqualTo("name")); 243 | } 244 | 245 | [Test] 246 | public void UnsetEnvironment() 247 | { 248 | var options = 249 | SpawnProgram() 250 | .ClearEnvironment() 251 | .AddEnvironment("FOO", "BAR") 252 | .AddEnvironment("FOO", "BAZ") 253 | .Options; 254 | 255 | Assert.That(options.Environment, Is.EqualTo(new[] 256 | { 257 | KeyValuePair.Create("FOO", "BAR"), 258 | KeyValuePair.Create("FOO", "BAZ"), 259 | })); 260 | 261 | options = options.UnsetEnvironment("FOO"); 262 | Assert.That(options.Environment, Is.Empty); 263 | } 264 | } 265 | } 266 | -------------------------------------------------------------------------------- /src/Core/Html/HtmlForm.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq.Html 18 | { 19 | #region Imports 20 | 21 | using System; 22 | using System.Collections.Generic; 23 | using System.Collections.ObjectModel; 24 | using System.Collections.Specialized; 25 | using System.Linq; 26 | using System.Net.Mime; 27 | using System.Text.RegularExpressions; 28 | 29 | #endregion 30 | 31 | public enum HtmlFormMethod { Get, Post } 32 | 33 | public sealed partial class HtmlForm 34 | { 35 | ReadOnlyCollection _controls; 36 | Dictionary _controlByElement; 37 | 38 | public HtmlObject Element { get; } 39 | public string Name { get; } 40 | public string Action { get; } 41 | public HtmlFormMethod Method { get; } 42 | public ContentType EncType { get; } 43 | 44 | public IReadOnlyList Controls => 45 | _controls ?? (_controls = Array.AsReadOnly(GetControlsCore().ToArray())); 46 | 47 | internal HtmlForm(HtmlObject element, string name, string action, HtmlFormMethod method, ContentType encType) 48 | { 49 | Element = element; 50 | Name = name; 51 | Action = action; 52 | Method = method; 53 | EncType = encType; 54 | } 55 | 56 | public override string ToString() => Element.ToString(); 57 | 58 | IEnumerable GetControlsCore() => 59 | // 60 | // Grab all INPUT and SELECT elements belonging to the form. 61 | // 62 | // TODO: BUTTON 63 | // TODO: formaction https://developer.mozilla.org/en-US/docs/Web/HTML/Element/button#attr-formaction 64 | // TODO: formenctype https://developer.mozilla.org/en-US/docs/Web/HTML/Element/button#attr-formenctype 65 | // TODO: formmethod https://developer.mozilla.org/en-US/docs/Web/HTML/Element/button#attr-formmethod 66 | // 67 | from e in Element.QuerySelectorAll("input, select, textarea") 68 | let name = e.GetAttributeValue("name")?.Trim() ?? string.Empty 69 | where name.Length > 0 70 | let controlType = "select".Equals(e.Name, StringComparison.OrdinalIgnoreCase) 71 | ? HtmlControlType.Select 72 | : "textarea".Equals(e.Name, StringComparison.OrdinalIgnoreCase) 73 | ? HtmlControlType.TextArea 74 | : HtmlControlType.Input 75 | let inputType = controlType == HtmlControlType.Input 76 | ? e.GetAttributeValue("type")?.Trim().Map(HtmlInputType.Parse) 77 | // Missing "type" attribute implies "text" since HTML 3.2 78 | ?? HtmlInputType.Default 79 | : null 80 | select new HtmlFormControl(this, e, name, controlType, inputType); 81 | 82 | Dictionary ControlByElement => 83 | _controlByElement ?? (_controlByElement = Controls.ToDictionary(c => c.Element, c => c)); 84 | 85 | public HtmlFormControl QuerySelector(string selector) => 86 | Element.QuerySelector(selector) is HtmlObject element 87 | ? ControlByElement.TryGetValue(element, out var control) ? control : null 88 | : null; 89 | 90 | public IEnumerable QuerySelectorAll(string selector) => 91 | from element in Element.QuerySelectorAll(selector) 92 | select ControlByElement.TryGetValue(element, out var control) ? control : null 93 | into control 94 | where control != null 95 | select control; 96 | 97 | public IEnumerable FindControls(string name) => 98 | FindControls(name, StringComparison.Ordinal); 99 | 100 | public IEnumerable FindControls(string name, StringComparison comparison) => 101 | from c in Controls 102 | where string.Equals(name, c.Name, comparison) 103 | select c; 104 | 105 | public IEnumerable MatchControls(string pattern) => 106 | MatchControls(pattern, RegexOptions.None); 107 | 108 | public IEnumerable MatchControls(string pattern, RegexOptions options) => 109 | from c in Controls 110 | where Regex.IsMatch(pattern, c.Name, options) 111 | select c; 112 | 113 | public NameValueCollection GetSubmissionData() => 114 | GetFormCore(data => data); 115 | 116 | public T GetForm(Func selector) => 117 | GetFormCore(selector2: selector); 118 | 119 | public T GetForm(Func selector) => 120 | GetFormCore(selector3: selector); 121 | 122 | T GetFormCore(Func selector1 = null, 123 | Func selector2 = null, 124 | Func selector3 = null) 125 | { 126 | // TODO Validate formElement is FORM 127 | // TODO formmethod, formaction, formenctype 128 | 129 | var all = selector3 != null ? new NameValueCollection() : null; 130 | var form = new NameValueCollection(); 131 | var submittables = selector1 == null ? new NameValueCollection() : null; 132 | 133 | // 134 | // Controls are collected into one or more of following buckets: 135 | // 136 | // - all (including disabled ones) 137 | // - form (enabled, non-submittables) 138 | // - submittables (just the enabled submittables) 139 | // 140 | // See section 4.10.19.6[1] (Form submission) in HTML5 141 | // specification as well as section 17.13[2] (Form submission) in 142 | // the older HTML 4.01 Specification for details. 143 | // 144 | // [1]: https://www.w3.org/TR/html5/forms.html#form-submission 145 | // [2]: http://www.w3.org/TR/html401/interact/forms.html#h-17.13 146 | 147 | foreach (var field in 148 | from c in Controls 149 | select new 150 | { 151 | c.Element, 152 | c.Name, 153 | c.ControlType, 154 | IsSelect = c.ControlType == HtmlControlType.Select, 155 | c.InputType, 156 | c.IsDisabled, 157 | c.IsReadOnly, 158 | }) 159 | { 160 | if (!field.IsSelect 161 | && field.ControlType != HtmlControlType.TextArea 162 | && field.InputType.KnownType == KnownHtmlInputType.Other) 163 | { 164 | throw new Exception($"Unexpected type of form field (\"{field.InputType}\")."); 165 | } 166 | 167 | // TODO select first of multiple checked in a radio button group 168 | // TODO multiple values handling in form data set 169 | // TODO multiple select with one or more selected options 170 | 171 | var value = field.IsSelect 172 | ? (field.Element.QuerySelector("option[selected]") ?? field.Element.QuerySelector("option"))?.GetAttributeValue("value") ?? string.Empty 173 | : field.InputType == HtmlInputType.Radio || field.InputType == HtmlInputType.Checkbox 174 | ? field.Element.HasAttribute("checked") ? field.Element.GetAttributeValue("value") ?? "on" : null 175 | : field.Element.GetAttributeValue("value") ?? string.Empty; 176 | 177 | all?.Add(field.Name, value); 178 | 179 | if (field.IsDisabled || value == null) 180 | continue; 181 | 182 | var bucket = field.InputType == HtmlInputType.Submit 183 | || field.InputType == HtmlInputType.Button 184 | || field.InputType == HtmlInputType.Image 185 | ? submittables 186 | : field.InputType != HtmlInputType.Reset 187 | && field.InputType != HtmlInputType.File 188 | ? form 189 | : null; 190 | 191 | bucket?.Add(field.Name, value); 192 | } 193 | 194 | return selector3 != null ? selector3(all, form, submittables) 195 | : selector2 != null ? selector2(form, submittables) 196 | : selector1 != null ? selector1(form) 197 | : default; 198 | } 199 | } 200 | } 201 | -------------------------------------------------------------------------------- /src/Core/Sys/ProgramArguments.cs: -------------------------------------------------------------------------------- 1 | #region Copyright (c) 2016 Atif Aziz. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | // 15 | #endregion 16 | 17 | namespace WebLinq.Sys 18 | { 19 | using System; 20 | using System.Collections; 21 | using System.Collections.Generic; 22 | using System.Collections.Immutable; 23 | using System.Diagnostics; 24 | using System.Text; 25 | 26 | // A union of process arguments stored as either a list or a single 27 | // string representing the arguments in a command-line. 28 | 29 | public sealed partial class ProgramArguments : ICollection 30 | { 31 | public static readonly ProgramArguments Empty = new ProgramArguments(null, ImmutableArray.Empty); 32 | 33 | readonly string _line; 34 | List _parsedLineArgs; 35 | readonly ImmutableArray _args; 36 | 37 | ProgramArguments(string line, ImmutableArray args) 38 | { 39 | Debug.Assert(args.IsDefault || line == null); 40 | _line = line; 41 | _args = args; 42 | } 43 | 44 | public static ProgramArguments Parse(string args) 45 | { 46 | if (args == null) throw new ArgumentNullException(nameof(args)); 47 | return args.Length == 0 ? Empty : new ProgramArguments(args, default); 48 | } 49 | 50 | public static ProgramArguments Var(params string[] args) => 51 | From(args); 52 | 53 | public static ProgramArguments From(IEnumerable args) 54 | { 55 | if (args == null) throw new ArgumentNullException(nameof(args)); 56 | return new ProgramArguments(null, ImmutableArray.CreateRange(args)); 57 | } 58 | 59 | ICollection Args 60 | => _line is string line 61 | ? line.Length == 0 ? (ICollection)Array.Empty() 62 | : _parsedLineArgs ?? (_parsedLineArgs = ParseArgumentsIntoList(line)) 63 | : _args; 64 | 65 | public int Count => _line is string s && s.Length == 0 ? 0 : Args.Count; 66 | 67 | IEnumerator IEnumerable.GetEnumerator() => 68 | GetEnumerator(); 69 | 70 | public IEnumerator GetEnumerator() => 71 | Args.GetEnumerator(); 72 | 73 | public override string ToString() 74 | => Count == 0 ? string.Empty : _line ?? PasteArguments.Paste(_args); 75 | 76 | bool ICollection.IsReadOnly => true; 77 | 78 | bool ICollection.Contains(string item) => Args.Contains(item); 79 | void ICollection.CopyTo(string[] array, int arrayIndex) => Args.CopyTo(array, arrayIndex); 80 | 81 | void ICollection.Add(string item) => throw new NotSupportedException(); 82 | void ICollection.Clear() => throw new NotSupportedException(); 83 | bool ICollection.Remove(string item) => throw new NotSupportedException(); 84 | } 85 | 86 | partial class ProgramArguments 87 | { 88 | static List ParseArgumentsIntoList(string arguments) 89 | { 90 | var list = new List(); 91 | ParseArgumentsIntoList(arguments, list); 92 | return list; 93 | } 94 | 95 | #region Copyright (c) .NET Foundation and Contributors 96 | // 97 | // The MIT License (MIT) 98 | // 99 | // All rights reserved. 100 | // 101 | // Permission is hereby granted, free of charge, to any person obtaining a copy 102 | // of this software and associated documentation files (the "Software"), to deal 103 | // in the Software without restriction, including without limitation the rights 104 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 105 | // copies of the Software, and to permit persons to whom the Software is 106 | // furnished to do so, subject to the following conditions: 107 | // 108 | // The above copyright notice and this permission notice shall be included in all 109 | // copies or substantial portions of the Software. 110 | // 111 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 112 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 113 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 114 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 115 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 116 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 117 | // SOFTWARE. 118 | // 119 | #endregion 120 | 121 | // https://github.com/dotnet/corefx/blob/96925bba1377b6042fc7322564b600a4e651f7fd/src/System.Diagnostics.Process/src/System/Diagnostics/Process.Unix.cs#L527-L626 122 | 123 | /// Parses a command-line argument string into a list of arguments. 124 | /// The argument string. 125 | /// The list into which the component arguments should be stored. 126 | /// 127 | /// This follows the rules outlined in "Parsing C++ Command-Line Arguments" at 128 | /// https://msdn.microsoft.com/en-us/library/17w5ykft.aspx. 129 | /// 130 | private static void ParseArgumentsIntoList(string arguments, List results) 131 | { 132 | // Iterate through all of the characters in the argument string. 133 | for (int i = 0; i < arguments.Length; i++) 134 | { 135 | while (i < arguments.Length && (arguments[i] == ' ' || arguments[i] == '\t')) 136 | i++; 137 | 138 | if (i == arguments.Length) 139 | break; 140 | 141 | results.Add(GetNextArgument(ref i)); 142 | } 143 | 144 | string GetNextArgument(ref int i) 145 | { 146 | var currentArgument = StringBuilderCache.Acquire(); 147 | bool inQuotes = false; 148 | 149 | while (i < arguments.Length) 150 | { 151 | // From the current position, iterate through contiguous backslashes. 152 | int backslashCount = 0; 153 | while (i < arguments.Length && arguments[i] == '\\') 154 | { 155 | i++; 156 | backslashCount++; 157 | } 158 | 159 | if (backslashCount > 0) 160 | { 161 | if (i >= arguments.Length || arguments[i] != '"') 162 | { 163 | // Backslashes not followed by a double quote: 164 | // they should all be treated as literal backslashes. 165 | currentArgument.Append('\\', backslashCount); 166 | } 167 | else 168 | { 169 | // Backslashes followed by a double quote: 170 | // - Output a literal slash for each complete pair of slashes 171 | // - If one remains, use it to make the subsequent quote a literal. 172 | currentArgument.Append('\\', backslashCount / 2); 173 | if (backslashCount % 2 != 0) 174 | { 175 | currentArgument.Append('"'); 176 | i++; 177 | } 178 | } 179 | 180 | continue; 181 | } 182 | 183 | char c = arguments[i]; 184 | 185 | // If this is a double quote, track whether we're inside of quotes or not. 186 | // Anything within quotes will be treated as a single argument, even if 187 | // it contains spaces. 188 | if (c == '"') 189 | { 190 | if (inQuotes && i < arguments.Length - 1 && arguments[i + 1] == '"') 191 | { 192 | // Two consecutive double quotes inside an inQuotes region should result in a literal double quote 193 | // (the parser is left in the inQuotes region). 194 | // This behavior is not part of the spec of code:ParseArgumentsIntoList, but is compatible with CRT 195 | // and .NET Framework. 196 | currentArgument.Append('"'); 197 | i++; 198 | } 199 | else 200 | { 201 | inQuotes = !inQuotes; 202 | } 203 | 204 | i++; 205 | continue; 206 | } 207 | 208 | // If this is a space/tab and we're not in quotes, we're done with the current 209 | // argument, it should be added to the results and then reset for the next one. 210 | if ((c == ' ' || c == '\t') && !inQuotes) 211 | { 212 | break; 213 | } 214 | 215 | // Nothing special; add the character to the current argument. 216 | currentArgument.Append(c); 217 | i++; 218 | } 219 | 220 | return StringBuilderCache.GetStringAndRelease(currentArgument); 221 | } 222 | } 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /COPYING.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | --------------------------------------------------------------------------------