├── .editorconfig ├── .gitignore ├── .paket └── Paket.Restore.targets ├── CHANGELOG.md ├── Directory.Build.props ├── LICENSE ├── PickAll.sln ├── README.md ├── appveyor.yml ├── assets └── icon.png ├── azure-pipelines.yml ├── paket.dependencies ├── paket.lock ├── samples └── PickAll.Sample │ ├── .editorconfig │ ├── Options.cs │ ├── OptionsExtensions.cs │ ├── PickAll.Sample.csproj │ ├── Program.cs │ └── paket.references ├── src └── PickAll │ ├── .editorconfig │ ├── Abstractions │ ├── IFetchedDocument.cs │ ├── IFetchingContext.cs │ ├── PostProcessor.cs │ ├── Searcher.cs │ └── Service.cs │ ├── AssemblyInfo.cs │ ├── ContextSettings.cs │ ├── Events.cs │ ├── FetchedDocument.cs │ ├── FetchedDocumentExtensions.cs │ ├── FetchingContext.cs │ ├── Internal │ ├── EventHelper.cs │ ├── Fuzzy.cs │ ├── Guard.cs │ ├── HtmlElementExtensions.cs │ ├── ObjectExtensions.cs │ └── TypeExtensions.cs │ ├── PickAll.csproj │ ├── PostProcessors │ ├── FuzzyMatch.cs │ ├── Improve.cs │ ├── Order.cs │ ├── Textify.cs │ └── Uniqueness.cs │ ├── ResultInfo.cs │ ├── ResultInfoExtensions.cs │ ├── RuntimeInfo.cs │ ├── SearchContext.cs │ ├── SearchContextExtensions.cs │ ├── Searchers │ ├── Bing.cs │ ├── BingNews.cs │ ├── DuckDuckGo.cs │ ├── Google.cs │ └── Yahoo.cs │ └── paket.references └── tests └── PickAll.Specs ├── .editorconfig ├── Fakes ├── Arbitrary.cs ├── ArbitrarySearcher.cs └── Marker.cs ├── Helpers ├── ResultInfoExtensions.cs └── ResultInfoHelper.cs ├── Outcomes ├── FetchedDocumentExtensionsSpecs.cs ├── FetchedDocumentSpecs.cs ├── FetchingContextSpecs.cs ├── FuzzyMatchSpecs.cs ├── ImproveSpecs.cs ├── OrderSpecs.cs ├── SearchContextExtensionsSpecs.cs ├── SearchContextSpecs.Events.cs ├── SearchContextSpecs.cs └── UniquenessSpecs.cs ├── PickAll.Specs.csproj └── paket.references /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [] 4 | end_of_line = crlf 5 | insert_final_newline = true 6 | 7 | [*.xml] 8 | indent_style = space 9 | indent_size = 4 10 | 11 | [*.{json,yml}] 12 | indent_style = space 13 | indent_size = 2 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | .DS_Store 3 | .vscode/ 4 | .vs 5 | tools/ 6 | paket-files/ 7 | [Oo]bj/ 8 | [Bb]in/ 9 | .nuget/ 10 | _ReSharper.* 11 | packages/ 12 | artifacts/ 13 | *.user 14 | *.suo 15 | *.userprefs 16 | *DS_Store 17 | *.sln.ide 18 | -------------------------------------------------------------------------------- /.paket/Paket.Restore.targets: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | $(MSBuildAllProjects);$(MSBuildThisFileFullPath) 8 | 9 | $(MSBuildVersion) 10 | 15.0.0 11 | false 12 | true 13 | 14 | true 15 | $(MSBuildThisFileDirectory) 16 | $(MSBuildThisFileDirectory)..\ 17 | $(PaketRootPath)paket-files\paket.restore.cached 18 | $(PaketRootPath)paket.lock 19 | classic 20 | proj 21 | assembly 22 | native 23 | /Library/Frameworks/Mono.framework/Commands/mono 24 | mono 25 | 26 | 27 | $(PaketRootPath)paket.bootstrapper.exe 28 | $(PaketToolsPath)paket.bootstrapper.exe 29 | $([System.IO.Path]::GetDirectoryName("$(PaketBootStrapperExePath)"))\ 30 | 31 | "$(PaketBootStrapperExePath)" 32 | $(MonoPath) --runtime=v4.0.30319 "$(PaketBootStrapperExePath)" 33 | 34 | 35 | 36 | 37 | true 38 | true 39 | 40 | 41 | True 42 | 43 | 44 | False 45 | 46 | $(BaseIntermediateOutputPath.TrimEnd('\').TrimEnd('\/')) 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | $(PaketRootPath)paket 56 | $(PaketToolsPath)paket 57 | 58 | 59 | 60 | 61 | 62 | $(PaketRootPath)paket.exe 63 | $(PaketToolsPath)paket.exe 64 | 65 | 66 | 67 | 68 | 69 | <_DotnetToolsJson Condition="Exists('$(PaketRootPath)/.config/dotnet-tools.json')">$([System.IO.File]::ReadAllText("$(PaketRootPath)/.config/dotnet-tools.json")) 70 | <_ConfigContainsPaket Condition=" '$(_DotnetToolsJson)' != ''">$(_DotnetToolsJson.Contains('"paket"')) 71 | <_ConfigContainsPaket Condition=" '$(_ConfigContainsPaket)' == ''">false 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | <_PaketCommand>dotnet paket 83 | 84 | 85 | 86 | 87 | 88 | $(PaketToolsPath)paket 89 | $(PaketBootStrapperExeDir)paket 90 | 91 | 92 | paket 93 | 94 | 95 | 96 | 97 | <_PaketExeExtension>$([System.IO.Path]::GetExtension("$(PaketExePath)")) 98 | <_PaketCommand Condition=" '$(_PaketCommand)' == '' AND '$(_PaketExeExtension)' == '.dll' ">dotnet "$(PaketExePath)" 99 | <_PaketCommand Condition=" '$(_PaketCommand)' == '' AND '$(OS)' != 'Windows_NT' AND '$(_PaketExeExtension)' == '.exe' ">$(MonoPath) --runtime=v4.0.30319 "$(PaketExePath)" 100 | <_PaketCommand Condition=" '$(_PaketCommand)' == '' ">"$(PaketExePath)" 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | true 122 | $(NoWarn);NU1603;NU1604;NU1605;NU1608 123 | false 124 | true 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | $([System.IO.File]::ReadAllText('$(PaketRestoreCacheFile)')) 134 | 135 | 136 | 137 | 138 | 139 | 141 | $([System.Text.RegularExpressions.Regex]::Split(`%(Identity)`, `": "`)[0].Replace(`"`, ``).Replace(` `, ``)) 142 | $([System.Text.RegularExpressions.Regex]::Split(`%(Identity)`, `": "`)[1].Replace(`"`, ``).Replace(` `, ``)) 143 | 144 | 145 | 146 | 147 | %(PaketRestoreCachedKeyValue.Value) 148 | %(PaketRestoreCachedKeyValue.Value) 149 | 150 | 151 | 152 | 153 | true 154 | false 155 | true 156 | 157 | 158 | 162 | 163 | true 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | $(PaketIntermediateOutputPath)\$(MSBuildProjectFile).paket.references.cached 183 | 184 | $(MSBuildProjectFullPath).paket.references 185 | 186 | $(MSBuildProjectDirectory)\$(MSBuildProjectName).paket.references 187 | 188 | $(MSBuildProjectDirectory)\paket.references 189 | 190 | false 191 | true 192 | true 193 | references-file-or-cache-not-found 194 | 195 | 196 | 197 | 198 | $([System.IO.File]::ReadAllText('$(PaketReferencesCachedFilePath)')) 199 | $([System.IO.File]::ReadAllText('$(PaketOriginalReferencesFilePath)')) 200 | references-file 201 | false 202 | 203 | 204 | 205 | 206 | false 207 | 208 | 209 | 210 | 211 | true 212 | target-framework '$(TargetFramework)' or '$(TargetFrameworks)' files @(PaketResolvedFilePaths) 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | false 224 | true 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',').Length) 236 | $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[0]) 237 | $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[1]) 238 | $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[4]) 239 | $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[5]) 240 | $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[6]) 241 | $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[7]) 242 | 243 | 244 | %(PaketReferencesFileLinesInfo.PackageVersion) 245 | All 246 | runtime 247 | $(ExcludeAssets);contentFiles 248 | $(ExcludeAssets);build;buildMultitargeting;buildTransitive 249 | true 250 | true 251 | 252 | 253 | 254 | 255 | $(PaketIntermediateOutputPath)/$(MSBuildProjectFile).paket.clitools 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | $([System.String]::Copy('%(PaketCliToolFileLines.Identity)').Split(',')[0]) 265 | $([System.String]::Copy('%(PaketCliToolFileLines.Identity)').Split(',')[1]) 266 | 267 | 268 | %(PaketCliToolFileLinesInfo.PackageVersion) 269 | 270 | 271 | 272 | 276 | 277 | 278 | 279 | 280 | 281 | false 282 | 283 | 284 | 285 | 286 | 287 | <_NuspecFilesNewLocation Include="$(PaketIntermediateOutputPath)\$(Configuration)\*.nuspec"/> 288 | 289 | 290 | 291 | 292 | 293 | $(MSBuildProjectDirectory)/$(MSBuildProjectFile) 294 | true 295 | false 296 | true 297 | false 298 | true 299 | false 300 | true 301 | false 302 | true 303 | false 304 | true 305 | $(PaketIntermediateOutputPath)\$(Configuration) 306 | $(PaketIntermediateOutputPath) 307 | 308 | 309 | 310 | <_NuspecFiles Include="$(AdjustedNuspecOutputPath)\*.$(PackageVersion.Split(`+`)[0]).nuspec"/> 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 | 322 | 370 | 371 | 420 | 421 | 466 | 467 | 511 | 512 | 555 | 556 | 557 | 558 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | ## [Unreleased] 9 | 10 | ## [1.3.0] - 2022-01-14 11 | 12 | - Added Bing news searcher as class `BingNews`. 13 | 14 | ## [1.2.2] - 2021-11-20 15 | 16 | - SharpX upgraded to version 1.0.3. 17 | -------------------------------------------------------------------------------- /Directory.Build.props: -------------------------------------------------------------------------------- 1 | 2 | 3 | $(MSBuildThisFileDirectory) 4 | false 5 | 6 | 7 | $(DefineConstants);NETFRAMEWORK 8 | 9 | 10 | 11 | 12 | runtime; build; native; contentfiles; analyzers 13 | all 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 - 2022 Giacomo Stelluti Scala 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /PickAll.sln: -------------------------------------------------------------------------------- 1 | Microsoft Visual Studio Solution File, Format Version 12.00 2 | # Visual Studio Version 16 3 | VisualStudioVersion = 16.0.31729.503 4 | MinimumVisualStudioVersion = 15.0.26124.0 5 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".paket", ".paket", "{F2416294-558A-41FA-A2CC-843435B27551}" 6 | ProjectSection(SolutionItems) = preProject 7 | paket.dependencies = paket.dependencies 8 | EndProjectSection 9 | EndProject 10 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PickAll", "src\PickAll\PickAll.csproj", "{B936C659-C656-40A8-A7D5-35EBCF27BA74}" 11 | EndProject 12 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PickAll.Specs", "tests\PickAll.Specs\PickAll.Specs.csproj", "{6CD5FED7-D697-413F-A763-EF95FEB77F8A}" 13 | EndProject 14 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PickAll.Sample", "samples\PickAll.Sample\PickAll.Sample.csproj", "{2DA9658E-F760-4122-8589-4B7CBD8AAD20}" 15 | EndProject 16 | Global 17 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 18 | Debug|Any CPU = Debug|Any CPU 19 | Release|Any CPU = Release|Any CPU 20 | EndGlobalSection 21 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 22 | {B936C659-C656-40A8-A7D5-35EBCF27BA74}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 23 | {B936C659-C656-40A8-A7D5-35EBCF27BA74}.Debug|Any CPU.Build.0 = Debug|Any CPU 24 | {B936C659-C656-40A8-A7D5-35EBCF27BA74}.Release|Any CPU.ActiveCfg = Release|Any CPU 25 | {B936C659-C656-40A8-A7D5-35EBCF27BA74}.Release|Any CPU.Build.0 = Release|Any CPU 26 | {6CD5FED7-D697-413F-A763-EF95FEB77F8A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 27 | {6CD5FED7-D697-413F-A763-EF95FEB77F8A}.Debug|Any CPU.Build.0 = Debug|Any CPU 28 | {6CD5FED7-D697-413F-A763-EF95FEB77F8A}.Release|Any CPU.ActiveCfg = Release|Any CPU 29 | {6CD5FED7-D697-413F-A763-EF95FEB77F8A}.Release|Any CPU.Build.0 = Release|Any CPU 30 | {2DA9658E-F760-4122-8589-4B7CBD8AAD20}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 31 | {2DA9658E-F760-4122-8589-4B7CBD8AAD20}.Debug|Any CPU.Build.0 = Debug|Any CPU 32 | {2DA9658E-F760-4122-8589-4B7CBD8AAD20}.Release|Any CPU.ActiveCfg = Release|Any CPU 33 | {2DA9658E-F760-4122-8589-4B7CBD8AAD20}.Release|Any CPU.Build.0 = Release|Any CPU 34 | EndGlobalSection 35 | GlobalSection(SolutionProperties) = preSolution 36 | HideSolutionNode = FALSE 37 | EndGlobalSection 38 | GlobalSection(ExtensibilityGlobals) = postSolution 39 | SolutionGuid = {FF6DB9EB-A535-4CCD-9E7A-1C9058EF5D99} 40 | EndGlobalSection 41 | EndGlobal 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://dev.azure.com/gsscoder/pickall/_apis/build/status/gsscoder.pickall?branchName=master)](https://dev.azure.com/gsscoder/pickall/_build/latest?definitionId=2&branchName=master) 2 | [![NuGet](https://img.shields.io/nuget/dt/pickall.svg)](https://nuget.org/packages/pickall) 3 | [![NuGet](https://img.shields.io/nuget/vpre/pickall.svg)](https://www.nuget.org/packages/pickall) 4 | [![Join the Gitter chat!](https://badges.gitter.im/gsscoder/pickall.svg)](https://gitter.im/pickallwebsearcher/community#) 5 | 6 | # PickAll 7 | 8 | ![alt text](/assets/icon.png "SharpX Logo") 9 | 10 | .NET agile and extensible web searching API. Built with [AngleSharp](https://anglesharp.github.io/). 11 | 12 | ## Philosophy 13 | 14 | PickAll is primarily designed to collect a limited amount of results (possibly the more relavant) from different sources and process these in a chain of steps. Results are essentially URLs and descriptions, but more data can be handled. 15 | 16 | ## Documentation 17 | 18 | Documentation is available in the project [wiki](https://github.com/gsscoder/pickall/wiki). 19 | 20 | ## Targets 21 | 22 | - .NET Standard 2.0 23 | - .NET Core 3.1 24 | - .NET 5.0 25 | 26 | ## Install via NuGet 27 | 28 | ```sh 29 | $ dotnet add package PickAll --version 1.3.1 30 | Determining projects to restore... 31 | ... 32 | ``` 33 | 34 | ## Build and sample 35 | 36 | ```sh 37 | # clone the repository 38 | $ git clone https://github.com/gsscoder/pickall.git 39 | 40 | # build the package 41 | $ cd pickall/src/PickAll 42 | $ dotnet build -c release 43 | 44 | # execute sample 45 | $ cd pickall/samples/PickAll.Sample 46 | $ dotnet build -c release 47 | $ cd ../../artifacts/PickAll.Sample/Release/netcoreapp3.0/PickAll.Sample 48 | ./PickAll.Sample "Steve Jobs" -e bing:duckduckgo 49 | Searching 'Steve Jobs' ... 50 | [0] Bing: "Steve Jobs - Wikipedia": "https://it.wikipedia.org/wiki/Steve_Jobs" 51 | [0] DuckDuckGo: "Steve Jobs - Wikipedia": "https://en.wikipedia.org/wiki/Steve_Jobs" 52 | [1] DuckDuckGo: "Steve Jobs - Apple, Family & Death - Biography": "https://www.biography.com/business-figure/steve-jobs" 53 | [2] Bing: "CC-BY-SA licenza": "http://creativecommons.org/licenses/by-sa/3.0/" 54 | [2] DuckDuckGo: "Steve Jobs - IMDb": "https://www.imdb.com/name/nm0423418/" 55 | [3] Bing: "Biografia di Steve Jobs - Biografieonline": "https://biografieonline.it/biografia.htm?BioID=1560&biografia=Steve+Jobs" 56 | ``` 57 | 58 | ## Test 59 | 60 | ```sh 61 | # change to tests directory 62 | $ cd pickall/tests/PickAll.Specs 63 | 64 | # build with debug configuration 65 | $ dotnet build -c debug 66 | ... 67 | 68 | # execute tests 69 | $ dotnet test 70 | ... 71 | ``` 72 | 73 | ## At a glance 74 | 75 | **CSharp:** 76 | ```csharp 77 | using PickAll; 78 | 79 | var context = new SearchContext() 80 | .WithEvents() 81 | .With() // search on google.com 82 | .With() // search on yahoo.com 83 | .With() // remove duplicates 84 | .With() // prioritize results 85 | // match Levenshtein distance with maximum of 15 86 | .With(new FuzzyMatchSettings { Text = "mechanics", MaximumDistance = 15 }); 87 | // repeat a search using more frequent words of previous results 88 | .With(new ImproveSettings { WordCount = 2, NoiseLength = 3 }) 89 | // scrape result pages and extract all text 90 | .With(new TextifySettings { IncludeTitle = true, NoiseLength = 3 }); 91 | // attach events 92 | context.ResultCreated += (sender, e) => Console.WriteLine($"Result created from {e.Result.Originator}"); 93 | // execute services (order of addition) 94 | var results = await context.SearchAsync("quantum physics"); 95 | // do anything you need with LINQ 96 | var scientific = results.Where(result => result.Url.Contains("wikipedia")); 97 | foreach (var result in scientific) { 98 | Console.WriteLine($"{result.Url} {result.Description}"); 99 | } 100 | ``` 101 | 102 | **FSharp:** 103 | ```fsharp 104 | let context = new SearchContext(typeof, 105 | typeof, 106 | typeof) 107 | let results = context.SearchAsync("quantum physics") 108 | |> Async.AwaitTask 109 | |> Async.RunSynchronously 110 | 111 | results |> Seq.iter (fun x -> printfn "%s %s" x.Url x.Description) 112 | ``` 113 | 114 | ## Libraries 115 | 116 | - [AngleSharp](https://github.com/AngleSharp/AngleSharp) 117 | - [AngleSharp.Io](https://github.com/AngleSharp/AngleSharp.Io) 118 | - [SharpX](https://github.com/gsscoder/sharpx) 119 | - [CommandLineParser](https://github.com/commandlineparser/commandline) 120 | - [xUnit.net](https://github.com/xunit/xunit) 121 | - [FluentAssertions](https://github.com/fluentassertions/fluentassertions) 122 | - [WaffleGenerator](https://github.com/SimonCropp/WaffleGenerator) 123 | - [Bogus](https://github.com/bchavez/Bogus) 124 | 125 | ## Tools 126 | 127 | - [Paket](https://github.com/fsprojects/Paket) 128 | 129 | ## Icon 130 | 131 | - [Search Engine](https://thenounproject.com/search/?q=search%20engine&i=2054907) icon designed by Vectors Market from [The Noun Project](https://thenounproject.com/). 132 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | version: 1.0.{build} 2 | build: 3 | verbosity: minimal 4 | -------------------------------------------------------------------------------- /assets/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gsscoder/pickall/070ac05614e2b538383b81a192b08b56281cc6fa/assets/icon.png -------------------------------------------------------------------------------- /azure-pipelines.yml: -------------------------------------------------------------------------------- 1 | trigger: 2 | - master 3 | 4 | pool: 5 | vmImage: 'windows-2019' 6 | 7 | steps: 8 | - task: UseDotNet@2 9 | displayName: '.NET Core: install 5.0.x' 10 | inputs: 11 | packageType: 'sdk' 12 | version: '5.0.x' 13 | 14 | - task: DotNetCoreCLI@2 15 | displayName: 'Paket: global install' 16 | inputs: 17 | command: 'custom' 18 | custom: 'tool' 19 | arguments: 'install -g paket' 20 | 21 | - task: PaketRestore@0 22 | displayName: 'Paket: restore' 23 | inputs: 24 | PaketPath: '.paket' 25 | 26 | - task: CmdLine@2 27 | displayName: '.NET Core CLI: restore' 28 | inputs: 29 | script: 'dotnet restore' 30 | 31 | - task: DotNetCoreCLI@2 32 | displayName: '.NET Core CLI: build' 33 | inputs: 34 | command: 'build' 35 | projects: 'src/**/*.csproj' 36 | arguments: '-c release' 37 | -------------------------------------------------------------------------------- /paket.dependencies: -------------------------------------------------------------------------------- 1 | group main 2 | source https://www.nuget.org/api/v2 3 | framework: netstandard20, netcoreapp31, net50 4 | nuget AngleSharp 0.14.0 5 | nuget AngleSharp.Io 0.14.0 6 | nuget SharpX 1.1.5 7 | 8 | group specs 9 | source https://www.nuget.org/api/v2 10 | framework: net50 11 | nuget Microsoft.NET.Test.Sdk 16.9.4 12 | nuget coverlet.collector 1.0.1 13 | nuget xunit 2.4.1 14 | nuget xunit.runner.visualstudio 2.4.3 15 | nuget FluentAssertions 6.2.0 16 | nuget SharpX 1.1.5 17 | nuget Bogus 33.1.1 18 | nuget WaffleGenerator 4.2.1 19 | nuget WaffleGenerator.Bogus 4.2.1 20 | 21 | group sample 22 | source https://www.nuget.org/api/v2 23 | framework: net50 24 | nuget CommandLineParser 2.7.82 25 | -------------------------------------------------------------------------------- /paket.lock: -------------------------------------------------------------------------------- 1 | RESTRICTION: || (== net5.0) (== netcoreapp3.1) (== netstandard2.0) 2 | NUGET 3 | remote: https://www.nuget.org/api/v2 4 | AngleSharp (0.14) 5 | System.Text.Encoding.CodePages (>= 4.5) 6 | AngleSharp.Io (0.14) 7 | AngleSharp (>= 0.14) 8 | FSharp.Core (6.0.1) 9 | Microsoft.NETCore.Platforms (3.1) - restriction: || (== net5.0) (== netcoreapp3.1) (&& (== netstandard2.0) (>= netcoreapp2.0)) (&& (== netstandard2.0) (>= netcoreapp3.1)) 10 | SharpX (1.1.5) 11 | FSharp.Core (>= 4.7) 12 | System.Runtime.CompilerServices.Unsafe (4.7) - restriction: || (&& (== net5.0) (>= net461)) (&& (== net5.0) (< netcoreapp2.0)) (&& (== net5.0) (< netcoreapp3.1)) (&& (== netcoreapp3.1) (>= net461)) (&& (== netcoreapp3.1) (< netcoreapp2.0)) (== netstandard2.0) 13 | System.Text.Encoding.CodePages (4.7) 14 | Microsoft.NETCore.Platforms (>= 3.1) - restriction: || (== net5.0) (== netcoreapp3.1) (&& (== netstandard2.0) (>= netcoreapp2.0)) (&& (== netstandard2.0) (>= netcoreapp3.1)) 15 | System.Runtime.CompilerServices.Unsafe (>= 4.7) - restriction: || (&& (== net5.0) (>= net461)) (&& (== net5.0) (< netcoreapp2.0)) (&& (== net5.0) (< netcoreapp3.1)) (&& (== netcoreapp3.1) (>= net461)) (&& (== netcoreapp3.1) (< netcoreapp2.0)) (== netstandard2.0) 16 | 17 | GROUP sample 18 | RESTRICTION: == net5.0 19 | NUGET 20 | remote: https://www.nuget.org/api/v2 21 | CommandLineParser (2.7.82) 22 | 23 | GROUP specs 24 | RESTRICTION: == net5.0 25 | NUGET 26 | remote: https://www.nuget.org/api/v2 27 | Bogus (33.1.1) 28 | coverlet.collector (1.0.1) 29 | FluentAssertions (6.2) 30 | System.Configuration.ConfigurationManager (>= 4.4) 31 | FSharp.Core (6.0.1) 32 | Microsoft.CodeCoverage (17.0) 33 | Microsoft.NET.Test.Sdk (16.9.4) 34 | Microsoft.CodeCoverage (>= 16.9.4) 35 | Microsoft.TestPlatform.TestHost (>= 16.9.4) 36 | Microsoft.NETCore.Platforms (5.0.4) 37 | Microsoft.TestPlatform.ObjectModel (17.0) 38 | NuGet.Frameworks (>= 5.0) 39 | System.Reflection.Metadata (>= 1.6) 40 | Microsoft.TestPlatform.TestHost (17.0) 41 | Microsoft.TestPlatform.ObjectModel (>= 17.0) 42 | Newtonsoft.Json (>= 9.0.1) 43 | Microsoft.Win32.SystemEvents (5.0) 44 | Microsoft.NETCore.Platforms (>= 5.0) 45 | NETStandard.Library (2.0.3) 46 | Microsoft.NETCore.Platforms (>= 1.1) 47 | Newtonsoft.Json (13.0.1) 48 | NuGet.Frameworks (5.11) 49 | SharpX (1.1.5) 50 | FSharp.Core (>= 4.7) 51 | System.Configuration.ConfigurationManager (5.0) 52 | System.Security.Cryptography.ProtectedData (>= 5.0) 53 | System.Security.Permissions (>= 5.0) 54 | System.Drawing.Common (5.0.2) 55 | Microsoft.Win32.SystemEvents (>= 5.0) 56 | System.Reflection.Metadata (5.0) 57 | System.Security.AccessControl (5.0) 58 | Microsoft.NETCore.Platforms (>= 5.0) 59 | System.Security.Principal.Windows (>= 5.0) 60 | System.Security.Cryptography.ProtectedData (5.0) 61 | System.Security.Permissions (5.0) 62 | System.Security.AccessControl (>= 5.0) 63 | System.Windows.Extensions (>= 5.0) 64 | System.Security.Principal.Windows (5.0) 65 | System.Windows.Extensions (5.0) 66 | System.Drawing.Common (>= 5.0) 67 | WaffleGenerator (4.2.1) 68 | WaffleGenerator.Bogus (4.2.1) 69 | Bogus (>= 33.0.2) 70 | WaffleGenerator (>= 4.2.1) 71 | xunit (2.4.1) 72 | xunit.analyzers (>= 0.10) 73 | xunit.assert (2.4.1) 74 | xunit.core (2.4.1) 75 | xunit.abstractions (2.0.3) 76 | xunit.analyzers (0.10) 77 | xunit.assert (2.4.1) 78 | NETStandard.Library (>= 1.6.1) 79 | xunit.core (2.4.1) 80 | xunit.extensibility.core (2.4.1) 81 | xunit.extensibility.execution (2.4.1) 82 | xunit.extensibility.core (2.4.1) 83 | NETStandard.Library (>= 1.6.1) 84 | xunit.abstractions (>= 2.0.3) 85 | xunit.extensibility.execution (2.4.1) 86 | NETStandard.Library (>= 1.6.1) 87 | xunit.extensibility.core (2.4.1) 88 | xunit.runner.visualstudio (2.4.3) 89 | -------------------------------------------------------------------------------- /samples/PickAll.Sample/.editorconfig: -------------------------------------------------------------------------------- 1 | [*.{csproj,config}] 2 | indent_style = space 3 | indent_size = 2 4 | 5 | # C# files 6 | [*.cs] 7 | 8 | #### Core EditorConfig Options #### 9 | 10 | # Indentation and spacing 11 | indent_size = 4 12 | indent_style = space 13 | tab_width = 4 14 | 15 | # New line preferences 16 | end_of_line = crlf 17 | insert_final_newline = false 18 | 19 | #### .NET Coding Conventions #### 20 | 21 | # Organize usings 22 | dotnet_separate_import_directive_groups = false 23 | dotnet_sort_system_directives_first = true 24 | file_header_template = unset 25 | 26 | # this. and Me. preferences 27 | dotnet_style_qualification_for_event = false 28 | dotnet_style_qualification_for_field = false 29 | dotnet_style_qualification_for_method = false 30 | dotnet_style_qualification_for_property = false 31 | 32 | # Language keywords vs BCL types preferences 33 | dotnet_style_predefined_type_for_locals_parameters_members = true 34 | dotnet_style_predefined_type_for_member_access = true 35 | 36 | # Parentheses preferences 37 | dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity 38 | dotnet_style_parentheses_in_other_binary_operators = always_for_clarity 39 | dotnet_style_parentheses_in_other_operators = never_if_unnecessary 40 | dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity 41 | 42 | # Modifier preferences 43 | dotnet_style_require_accessibility_modifiers = for_non_interface_members 44 | 45 | # Expression-level preferences 46 | dotnet_style_coalesce_expression = true 47 | dotnet_style_collection_initializer = true 48 | dotnet_style_explicit_tuple_names = true 49 | dotnet_style_null_propagation = true 50 | dotnet_style_object_initializer = true 51 | dotnet_style_operator_placement_when_wrapping = beginning_of_line 52 | dotnet_style_prefer_auto_properties = true 53 | dotnet_style_prefer_compound_assignment = true 54 | dotnet_style_prefer_conditional_expression_over_assignment = true 55 | dotnet_style_prefer_conditional_expression_over_return = true 56 | dotnet_style_prefer_inferred_anonymous_type_member_names = true 57 | dotnet_style_prefer_inferred_tuple_names = true 58 | dotnet_style_prefer_is_null_check_over_reference_equality_method = true 59 | dotnet_style_prefer_simplified_boolean_expressions = true 60 | dotnet_style_prefer_simplified_interpolation = true 61 | 62 | # Field preferences 63 | dotnet_style_readonly_field = true 64 | 65 | # Parameter preferences 66 | dotnet_code_quality_unused_parameters = all 67 | 68 | # Suppression preferences 69 | dotnet_remove_unnecessary_suppression_exclusions = none 70 | 71 | #### C# Coding Conventions #### 72 | 73 | # var preferences 74 | csharp_style_var_elsewhere = false 75 | csharp_style_var_for_built_in_types = false 76 | csharp_style_var_when_type_is_apparent = false 77 | 78 | # Expression-bodied members 79 | csharp_style_expression_bodied_accessors = true 80 | csharp_style_expression_bodied_constructors = false 81 | csharp_style_expression_bodied_indexers = true 82 | csharp_style_expression_bodied_lambdas = true 83 | csharp_style_expression_bodied_local_functions = false 84 | csharp_style_expression_bodied_methods = false 85 | csharp_style_expression_bodied_operators = false 86 | csharp_style_expression_bodied_properties = true 87 | 88 | # Pattern matching preferences 89 | csharp_style_pattern_matching_over_as_with_null_check = true 90 | csharp_style_pattern_matching_over_is_with_cast_check = true 91 | csharp_style_prefer_not_pattern = true 92 | csharp_style_prefer_pattern_matching = true 93 | csharp_style_prefer_switch_expression = true 94 | 95 | # Null-checking preferences 96 | csharp_style_conditional_delegate_call = true 97 | 98 | # Modifier preferences 99 | csharp_prefer_static_local_function = true 100 | csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async 101 | 102 | # Code-block preferences 103 | csharp_prefer_braces = true 104 | csharp_prefer_simple_using_statement = true 105 | 106 | # Expression-level preferences 107 | csharp_prefer_simple_default_expression = true 108 | csharp_style_deconstructed_variable_declaration = true 109 | csharp_style_implicit_object_creation_when_type_is_apparent = true 110 | csharp_style_inlined_variable_declaration = true 111 | csharp_style_pattern_local_over_anonymous_function = true 112 | csharp_style_prefer_index_operator = true 113 | csharp_style_prefer_range_operator = true 114 | csharp_style_throw_expression = true 115 | csharp_style_unused_value_assignment_preference = discard_variable 116 | csharp_style_unused_value_expression_statement_preference = discard_variable 117 | 118 | # 'using' directive preferences 119 | csharp_using_directive_placement = outside_namespace 120 | 121 | #### C# Formatting Rules #### 122 | 123 | # New line preferences 124 | csharp_new_line_before_catch = true 125 | csharp_new_line_before_else = true 126 | csharp_new_line_before_finally = true 127 | csharp_new_line_before_members_in_anonymous_types = true 128 | csharp_new_line_before_members_in_object_initializers = true 129 | csharp_new_line_before_open_brace = anonymous_methods,anonymous_types,lambdas,methods,object_collection_array_initializers,properties,types 130 | csharp_new_line_between_query_expression_clauses = true 131 | 132 | # Indentation preferences 133 | csharp_indent_block_contents = true 134 | csharp_indent_braces = false 135 | csharp_indent_case_contents = true 136 | csharp_indent_case_contents_when_block = true 137 | csharp_indent_labels = one_less_than_current 138 | csharp_indent_switch_labels = true 139 | 140 | # Space preferences 141 | csharp_space_after_cast = false 142 | csharp_space_after_colon_in_inheritance_clause = true 143 | csharp_space_after_comma = true 144 | csharp_space_after_dot = false 145 | csharp_space_after_keywords_in_control_flow_statements = true 146 | csharp_space_after_semicolon_in_for_statement = true 147 | csharp_space_around_binary_operators = before_and_after 148 | csharp_space_around_declaration_statements = false 149 | csharp_space_before_colon_in_inheritance_clause = true 150 | csharp_space_before_comma = false 151 | csharp_space_before_dot = false 152 | csharp_space_before_open_square_brackets = false 153 | csharp_space_before_semicolon_in_for_statement = false 154 | csharp_space_between_empty_square_brackets = false 155 | csharp_space_between_method_call_empty_parameter_list_parentheses = false 156 | csharp_space_between_method_call_name_and_opening_parenthesis = false 157 | csharp_space_between_method_call_parameter_list_parentheses = false 158 | csharp_space_between_method_declaration_empty_parameter_list_parentheses = false 159 | csharp_space_between_method_declaration_name_and_open_parenthesis = false 160 | csharp_space_between_method_declaration_parameter_list_parentheses = false 161 | csharp_space_between_parentheses = false 162 | csharp_space_between_square_brackets = false 163 | 164 | # Wrapping preferences 165 | csharp_preserve_single_line_blocks = true 166 | csharp_preserve_single_line_statements = true 167 | 168 | #### Naming styles #### 169 | 170 | # Naming rules 171 | 172 | dotnet_naming_rule.interface_should_be_begins_with_i.severity = suggestion 173 | dotnet_naming_rule.interface_should_be_begins_with_i.symbols = interface 174 | dotnet_naming_rule.interface_should_be_begins_with_i.style = begins_with_i 175 | 176 | dotnet_naming_rule.types_should_be_pascal_case.severity = suggestion 177 | dotnet_naming_rule.types_should_be_pascal_case.symbols = types 178 | dotnet_naming_rule.types_should_be_pascal_case.style = pascal_case 179 | 180 | dotnet_naming_rule.non_field_members_should_be_pascal_case.severity = suggestion 181 | dotnet_naming_rule.non_field_members_should_be_pascal_case.symbols = non_field_members 182 | dotnet_naming_rule.non_field_members_should_be_pascal_case.style = pascal_case 183 | 184 | # Symbol specifications 185 | 186 | dotnet_naming_symbols.interface.applicable_kinds = interface 187 | dotnet_naming_symbols.interface.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected 188 | dotnet_naming_symbols.interface.required_modifiers = 189 | 190 | dotnet_naming_symbols.types.applicable_kinds = class, struct, interface, enum 191 | dotnet_naming_symbols.types.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected 192 | dotnet_naming_symbols.types.required_modifiers = 193 | 194 | dotnet_naming_symbols.non_field_members.applicable_kinds = property, event, method 195 | dotnet_naming_symbols.non_field_members.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected 196 | dotnet_naming_symbols.non_field_members.required_modifiers = 197 | 198 | # Naming styles 199 | 200 | dotnet_naming_style.pascal_case.required_prefix = 201 | dotnet_naming_style.pascal_case.required_suffix = 202 | dotnet_naming_style.pascal_case.word_separator = 203 | dotnet_naming_style.pascal_case.capitalization = pascal_case 204 | 205 | dotnet_naming_style.begins_with_i.required_prefix = I 206 | dotnet_naming_style.begins_with_i.required_suffix = 207 | dotnet_naming_style.begins_with_i.word_separator = 208 | dotnet_naming_style.begins_with_i.capitalization = pascal_case 209 | -------------------------------------------------------------------------------- /samples/PickAll.Sample/Options.cs: -------------------------------------------------------------------------------- 1 | using CommandLine; 2 | using System.Collections.Generic; 3 | 4 | sealed class Options 5 | { 6 | [Value(0, MetaName = "search query", HelpText = "Query to submit to search engines", 7 | Required = true)] 8 | public string Query { get; set; } 9 | 10 | [Option("timeout", HelpText = "Maximum timeout for HTTP requests in seconds")] 11 | public uint? Timeout { get; set; } 12 | 13 | [Option('f', "fuzzy", HelpText = "Fuzzy matching of Levenshtein distance 0-10")] 14 | public string FuzzyMatch { get; set; } 15 | 16 | [Option('i', "improve", HelpText = "Enable improve search post processor")] 17 | public bool Improve { get; set; } 18 | 19 | [Option('t', "textify", HelpText = "Enable textify post processor")] 20 | public bool Wordify { get; set; } 21 | 22 | [Option('e', "engines", HelpText = "Search engines to use separated by ':'", 23 | Separator = ':')] 24 | public IEnumerable Engines { get; set; } 25 | } 26 | -------------------------------------------------------------------------------- /samples/PickAll.Sample/OptionsExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using PickAll; 4 | 5 | static class OptionsExtensions 6 | { 7 | public static SearchContext ToContext(this Options options) 8 | { 9 | SearchContext context; 10 | if (!options.Engines.Any()) { 11 | context = SearchContext.Default; 12 | } 13 | else { 14 | context = new SearchContext(); 15 | foreach (var engine in options.Engines) { 16 | context = context.With(engine); 17 | } 18 | context = context 19 | .With() 20 | .With(); 21 | } 22 | context = context.WithEvents(); 23 | if (options.Timeout.HasValue) { 24 | context = context.WithConfiguration( 25 | new ContextSettings { 26 | Timeout = TimeSpan.FromSeconds(options.Timeout.Value) }, 27 | merge: true); 28 | } 29 | if (!string.IsNullOrEmpty(options.FuzzyMatch)) { 30 | context = context.With( 31 | new FuzzyMatchSettings { 32 | Text = options.FuzzyMatch, 33 | MaximumDistance = 10 }); 34 | } 35 | if (options.Improve) { 36 | context = context.With( 37 | new ImproveSettings { 38 | WordCount = 2, 39 | NoiseLength = 3}); 40 | } 41 | if (options.Wordify) { 42 | context = context.With( 43 | new TextifySettings { 44 | SanitizeText = true, 45 | NoiseLength = 3}); 46 | } 47 | return context; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /samples/PickAll.Sample/PickAll.Sample.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | Exe 8 | net5.0 9 | 9.0 10 | gsscoder 11 | PickAll 12 | gsscoder 13 | false 14 | 15 | 16 | ../../artifacts/PickAll.Sample/Debug 17 | 18 | 19 | ../../artifacts/PickAll.Sample/Release 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /samples/PickAll.Sample/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using CommandLine; 4 | 5 | sealed class Program 6 | { 7 | const int exitOK = 0; 8 | const int exitFail = 1; 9 | 10 | static int Main(string[] args) 11 | { 12 | return Parser.Default.ParseArguments(args) 13 | .MapResult(options => ExecuteSearch(options).Result, 14 | _ => exitFail); 15 | } 16 | 17 | static async Task ExecuteSearch(Options options) 18 | { 19 | var context = options.ToContext(); 20 | context.SearchBegin += (sender, e) => Console.WriteLine($"Searching '{e.Query}' ..."); 21 | var results = await context.SearchAsync(options.Query); 22 | foreach (var result in results) { 23 | Console.WriteLine( 24 | $"[{result.Index}] {result.Originator}: \"{result.Description}\": \"{result.Url}\""); 25 | if (result.Data != null) { 26 | Console.WriteLine( 27 | $"\tData:\n\t\t{result.Data}"); 28 | } 29 | } 30 | return exitOK; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /samples/PickAll.Sample/paket.references: -------------------------------------------------------------------------------- 1 | group sample 2 | CommandLineParser 3 | -------------------------------------------------------------------------------- /src/PickAll/.editorconfig: -------------------------------------------------------------------------------- 1 | [*.{csproj,config}] 2 | indent_style = space 3 | indent_size = 2 4 | 5 | # C# files 6 | [*.cs] 7 | 8 | #### Core EditorConfig Options #### 9 | 10 | # Indentation and spacing 11 | indent_size = 4 12 | indent_style = space 13 | tab_width = 4 14 | 15 | # New line preferences 16 | end_of_line = crlf 17 | insert_final_newline = false 18 | 19 | #### .NET Coding Conventions #### 20 | 21 | # Organize usings 22 | dotnet_separate_import_directive_groups = false 23 | dotnet_sort_system_directives_first = true 24 | file_header_template = unset 25 | 26 | # this. and Me. preferences 27 | dotnet_style_qualification_for_event = false 28 | dotnet_style_qualification_for_field = false 29 | dotnet_style_qualification_for_method = false 30 | dotnet_style_qualification_for_property = false 31 | 32 | # Language keywords vs BCL types preferences 33 | dotnet_style_predefined_type_for_locals_parameters_members = true 34 | dotnet_style_predefined_type_for_member_access = true 35 | 36 | # Parentheses preferences 37 | dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity 38 | dotnet_style_parentheses_in_other_binary_operators = always_for_clarity 39 | dotnet_style_parentheses_in_other_operators = never_if_unnecessary 40 | dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity 41 | 42 | # Modifier preferences 43 | dotnet_style_require_accessibility_modifiers = for_non_interface_members 44 | 45 | # Expression-level preferences 46 | dotnet_style_coalesce_expression = true 47 | dotnet_style_collection_initializer = true 48 | dotnet_style_explicit_tuple_names = true 49 | dotnet_style_null_propagation = true 50 | dotnet_style_object_initializer = true 51 | dotnet_style_operator_placement_when_wrapping = beginning_of_line 52 | dotnet_style_prefer_auto_properties = true 53 | dotnet_style_prefer_compound_assignment = true 54 | dotnet_style_prefer_conditional_expression_over_assignment = true 55 | dotnet_style_prefer_conditional_expression_over_return = true 56 | dotnet_style_prefer_inferred_anonymous_type_member_names = true 57 | dotnet_style_prefer_inferred_tuple_names = true 58 | dotnet_style_prefer_is_null_check_over_reference_equality_method = true 59 | dotnet_style_prefer_simplified_boolean_expressions = true 60 | dotnet_style_prefer_simplified_interpolation = true 61 | 62 | # Field preferences 63 | dotnet_style_readonly_field = true 64 | 65 | # Parameter preferences 66 | dotnet_code_quality_unused_parameters = all 67 | 68 | # Suppression preferences 69 | dotnet_remove_unnecessary_suppression_exclusions = none 70 | 71 | #### C# Coding Conventions #### 72 | 73 | # var preferences 74 | csharp_style_var_elsewhere = false 75 | csharp_style_var_for_built_in_types = false 76 | csharp_style_var_when_type_is_apparent = false 77 | 78 | # Expression-bodied members 79 | csharp_style_expression_bodied_accessors = true 80 | csharp_style_expression_bodied_constructors = false 81 | csharp_style_expression_bodied_indexers = true 82 | csharp_style_expression_bodied_lambdas = true 83 | csharp_style_expression_bodied_local_functions = false 84 | csharp_style_expression_bodied_methods = false 85 | csharp_style_expression_bodied_operators = false 86 | csharp_style_expression_bodied_properties = true 87 | 88 | # Pattern matching preferences 89 | csharp_style_pattern_matching_over_as_with_null_check = true 90 | csharp_style_pattern_matching_over_is_with_cast_check = true 91 | csharp_style_prefer_not_pattern = true 92 | csharp_style_prefer_pattern_matching = true 93 | csharp_style_prefer_switch_expression = true 94 | 95 | # Null-checking preferences 96 | csharp_style_conditional_delegate_call = true 97 | 98 | # Modifier preferences 99 | csharp_prefer_static_local_function = true 100 | csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async 101 | 102 | # Code-block preferences 103 | csharp_prefer_braces = true 104 | csharp_prefer_simple_using_statement = true 105 | 106 | # Expression-level preferences 107 | csharp_prefer_simple_default_expression = true 108 | csharp_style_deconstructed_variable_declaration = true 109 | csharp_style_implicit_object_creation_when_type_is_apparent = true 110 | csharp_style_inlined_variable_declaration = true 111 | csharp_style_pattern_local_over_anonymous_function = true 112 | csharp_style_prefer_index_operator = true 113 | csharp_style_prefer_range_operator = true 114 | csharp_style_throw_expression = true 115 | csharp_style_unused_value_assignment_preference = discard_variable 116 | csharp_style_unused_value_expression_statement_preference = discard_variable 117 | 118 | # 'using' directive preferences 119 | csharp_using_directive_placement = outside_namespace 120 | 121 | #### C# Formatting Rules #### 122 | 123 | # New line preferences 124 | csharp_new_line_before_catch = true 125 | csharp_new_line_before_else = true 126 | csharp_new_line_before_finally = true 127 | csharp_new_line_before_members_in_anonymous_types = true 128 | csharp_new_line_before_members_in_object_initializers = true 129 | csharp_new_line_before_open_brace = anonymous_methods,anonymous_types,lambdas,methods,object_collection_array_initializers,properties,types 130 | csharp_new_line_between_query_expression_clauses = true 131 | 132 | # Indentation preferences 133 | csharp_indent_block_contents = true 134 | csharp_indent_braces = false 135 | csharp_indent_case_contents = true 136 | csharp_indent_case_contents_when_block = true 137 | csharp_indent_labels = one_less_than_current 138 | csharp_indent_switch_labels = true 139 | 140 | # Space preferences 141 | csharp_space_after_cast = false 142 | csharp_space_after_colon_in_inheritance_clause = true 143 | csharp_space_after_comma = true 144 | csharp_space_after_dot = false 145 | csharp_space_after_keywords_in_control_flow_statements = true 146 | csharp_space_after_semicolon_in_for_statement = true 147 | csharp_space_around_binary_operators = before_and_after 148 | csharp_space_around_declaration_statements = false 149 | csharp_space_before_colon_in_inheritance_clause = true 150 | csharp_space_before_comma = false 151 | csharp_space_before_dot = false 152 | csharp_space_before_open_square_brackets = false 153 | csharp_space_before_semicolon_in_for_statement = false 154 | csharp_space_between_empty_square_brackets = false 155 | csharp_space_between_method_call_empty_parameter_list_parentheses = false 156 | csharp_space_between_method_call_name_and_opening_parenthesis = false 157 | csharp_space_between_method_call_parameter_list_parentheses = false 158 | csharp_space_between_method_declaration_empty_parameter_list_parentheses = false 159 | csharp_space_between_method_declaration_name_and_open_parenthesis = false 160 | csharp_space_between_method_declaration_parameter_list_parentheses = false 161 | csharp_space_between_parentheses = false 162 | csharp_space_between_square_brackets = false 163 | 164 | # Wrapping preferences 165 | csharp_preserve_single_line_blocks = true 166 | csharp_preserve_single_line_statements = true 167 | 168 | #### Naming styles #### 169 | 170 | # Naming rules 171 | 172 | dotnet_naming_rule.interface_should_be_begins_with_i.severity = suggestion 173 | dotnet_naming_rule.interface_should_be_begins_with_i.symbols = interface 174 | dotnet_naming_rule.interface_should_be_begins_with_i.style = begins_with_i 175 | 176 | dotnet_naming_rule.types_should_be_pascal_case.severity = suggestion 177 | dotnet_naming_rule.types_should_be_pascal_case.symbols = types 178 | dotnet_naming_rule.types_should_be_pascal_case.style = pascal_case 179 | 180 | dotnet_naming_rule.non_field_members_should_be_pascal_case.severity = suggestion 181 | dotnet_naming_rule.non_field_members_should_be_pascal_case.symbols = non_field_members 182 | dotnet_naming_rule.non_field_members_should_be_pascal_case.style = pascal_case 183 | 184 | # Symbol specifications 185 | 186 | dotnet_naming_symbols.interface.applicable_kinds = interface 187 | dotnet_naming_symbols.interface.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected 188 | dotnet_naming_symbols.interface.required_modifiers = 189 | 190 | dotnet_naming_symbols.types.applicable_kinds = class, struct, interface, enum 191 | dotnet_naming_symbols.types.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected 192 | dotnet_naming_symbols.types.required_modifiers = 193 | 194 | dotnet_naming_symbols.non_field_members.applicable_kinds = property, event, method 195 | dotnet_naming_symbols.non_field_members.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected 196 | dotnet_naming_symbols.non_field_members.required_modifiers = 197 | 198 | # Naming styles 199 | 200 | dotnet_naming_style.pascal_case.required_prefix = 201 | dotnet_naming_style.pascal_case.required_suffix = 202 | dotnet_naming_style.pascal_case.word_separator = 203 | dotnet_naming_style.pascal_case.capitalization = pascal_case 204 | 205 | dotnet_naming_style.begins_with_i.required_prefix = I 206 | dotnet_naming_style.begins_with_i.required_suffix = 207 | dotnet_naming_style.begins_with_i.word_separator = 208 | dotnet_naming_style.begins_with_i.capitalization = pascal_case 209 | -------------------------------------------------------------------------------- /src/PickAll/Abstractions/IFetchedDocument.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Text; 3 | 4 | namespace PickAll 5 | { 6 | /// Represents a document fetched without HTML DOM. 7 | public interface IFetchedDocument : IEquatable 8 | { 9 | byte[] Content { get; } 10 | 11 | int Length { get; } 12 | 13 | long LongLength { get; } 14 | 15 | string Text(); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/PickAll/Abstractions/IFetchingContext.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | 3 | namespace PickAll 4 | { 5 | /// Represents a context in which the a document without HTML DOM is fetched. 6 | public interface IFetchingContext 7 | { 8 | Task FetchAsync(string address); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/PickAll/Abstractions/PostProcessor.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace PickAll 5 | { 6 | /// Represents a post processor service managed by SearchContext. 7 | public abstract class PostProcessor : Service 8 | { 9 | public PostProcessor(object settings) 10 | { 11 | Settings = settings; 12 | } 13 | 14 | public virtual bool PublishEvents => false; 15 | 16 | public abstract IEnumerable Process(IEnumerable results); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/PickAll/Abstractions/Searcher.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Threading.Tasks; 4 | 5 | namespace PickAll 6 | { 7 | /// Represents a searching service managed by SearchContext. 8 | public abstract class Searcher : Service 9 | { 10 | public Searcher(object settings) 11 | { 12 | Settings = settings; 13 | Name = GetType().Name; 14 | } 15 | 16 | internal event EventHandler ResultCreated; 17 | 18 | /// The searcher identifier set to class name. 19 | public string Name { get; private set; } 20 | 21 | /// Performs the actual search and returns a sequence of ResultInfo. 22 | public abstract Task> SearchAsync(string query); 23 | 24 | protected ResultInfo CreateResult( 25 | int index, string url, string description, object data = null) 26 | { 27 | var result = new ResultInfo(Name, index, url, description, data); 28 | EventHelper.RaiseEvent(this, ResultCreated, 29 | () => new ResultHandledEventArgs(result, ServiceType.Searcher), Context.Settings.EnableRaisingEvents); 30 | return result; 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/PickAll/Abstractions/Service.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace PickAll 4 | { 5 | /// Represents a service managed by SearchContext. 6 | public abstract class Service 7 | { 8 | SearchContext _context; 9 | internal event EventHandler Load; 10 | 11 | public SearchContext Context 12 | { 13 | get { return _context; } 14 | set 15 | { 16 | _context = value; 17 | // Guard against raising load event before configuration happens. 18 | // A service is loaded when is bound to a search context. 19 | if (_context == null) return; 20 | EventHelper.RaiseEvent(this, Load, EventArgs.Empty, _context.Settings.EnableRaisingEvents); 21 | } 22 | } 23 | 24 | public RuntimeInfo Runtime { get; internal set; } 25 | 26 | protected object Settings { get; set; } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/PickAll/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | #if DEBUG 2 | using System; 3 | [assembly: CLSCompliant(true)] 4 | #endif 5 | -------------------------------------------------------------------------------- /src/PickAll/ContextSettings.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace PickAll 4 | { 5 | /// Settings for a search context. 6 | public struct ContextSettings 7 | { 8 | int? _maximumResults; 9 | 10 | /// Maximum results a search is allowed to return. 11 | public int? MaximumResults 12 | { 13 | get { return _maximumResults; } 14 | set 15 | { 16 | if (value.HasValue) Guard.AgainstNegative("MaximumResults", value.Value); 17 | _maximumResults = value; 18 | } 19 | } 20 | 21 | /// Timeout for each HTTP request performed. 22 | public TimeSpan? Timeout { get; set; } 23 | 24 | /// Enables events in search context and services. 25 | public bool EnableRaisingEvents { get; set; } 26 | 27 | internal ContextSettings Clone() 28 | { 29 | return new ContextSettings 30 | { 31 | MaximumResults = MaximumResults, 32 | Timeout = Timeout, 33 | EnableRaisingEvents = EnableRaisingEvents 34 | }; 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/PickAll/Events.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace PickAll 4 | { 5 | /// Defines the service type. 6 | public enum ServiceType 7 | { 8 | /// Searcher service. 9 | Searcher, 10 | /// Post processor service. 11 | PostProcessor 12 | } 13 | 14 | /// Holds event data for the SearchBegin event. 15 | public sealed class SearchBeginEventArgs : EventArgs 16 | { 17 | public SearchBeginEventArgs(string query) => Query = query; 18 | 19 | public string Query { get; private set; } 20 | } 21 | 22 | /// Holds event data for the ResultProcessed event. 23 | public sealed class ResultHandledEventArgs : EventArgs 24 | { 25 | public ResultHandledEventArgs(ResultInfo result, ServiceType type) 26 | { 27 | Result = result; 28 | Type = type; 29 | } 30 | 31 | public ResultInfo Result { get; private set; } 32 | 33 | public ServiceType Type { get; private set; } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/PickAll/FetchedDocument.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using System.Text; 3 | 4 | namespace PickAll 5 | { 6 | /// Default implementation of a document fetched without HTML DOM. 7 | public sealed class FetchedDocument : IFetchedDocument 8 | { 9 | public static readonly IFetchedDocument Empty = new FetchedDocument(); 10 | 11 | FetchedDocument() => Content = new byte[] {}; 12 | 13 | #if DEBUG 14 | public FetchedDocument(byte[] content) => Content = content; 15 | #else 16 | internal FetchedDocument(byte[] content) => Content = content; 17 | #endif 18 | 19 | public override bool Equals(object value) => 20 | value is FetchedDocument f && 21 | Enumerable.SequenceEqual(f.Content, Content) && 22 | f.Length.Equals(Length) && 23 | f.LongLength.Equals(LongLength); 24 | 25 | public bool Equals(IFetchedDocument other) => 26 | Enumerable.SequenceEqual(other.Content, Content) && 27 | other.Length.Equals(Length) && 28 | other.LongLength.Equals(LongLength); 29 | 30 | public override int GetHashCode() 31 | { 32 | unchecked { 33 | var hash = 17; 34 | hash = hash * 31 + Content.GetHashCode(); 35 | hash = hash * 31 + Length.GetHashCode(); 36 | return hash; 37 | } 38 | } 39 | 40 | public byte[] Content { get; private set; } 41 | 42 | public string Text() => Encoding.UTF8.GetString(Content); 43 | 44 | public int Length => Content.Length; 45 | 46 | public long LongLength => Content.LongLength; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/PickAll/FetchedDocumentExtensions.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Text.RegularExpressions; 3 | using System.Linq; 4 | 5 | namespace PickAll 6 | { 7 | public static class FetchedDocumentExtensions 8 | { 9 | public static string ElementSelector(this IFetchedDocument document, string tag) 10 | { 11 | Guard.AgainstNull(nameof(document), document); 12 | 13 | return document.ElementSelectorAll(tag).SingleOrDefault() ?? string.Empty; 14 | } 15 | 16 | public static IEnumerable ElementSelectorAll(this IFetchedDocument document, string tag) 17 | { 18 | Guard.AgainstNull(nameof(document), document); 19 | Guard.AgainstNull(nameof(tag), tag); 20 | Guard.AgainstEmptyWhiteSpace(nameof(tag), tag); 21 | 22 | var getElement = new Regex($@"(?<=<{tag}>)(.|\n)*?(?=<\/{tag}>)", 23 | RegexOptions.Compiled | RegexOptions.Multiline); 24 | var matches = getElement.Matches(document.Text()); 25 | var contents = new List(); 26 | foreach (Match match in matches) { 27 | contents.Add(match.Value); 28 | } 29 | return contents; 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/PickAll/FetchingContext.cs: -------------------------------------------------------------------------------- 1 | using System.Net.Http; 2 | using System.Threading.Tasks; 3 | 4 | namespace PickAll 5 | { 6 | /// The context in which the a document without HTML DOM is fetched. 7 | public sealed class FetchingContext : IFetchingContext 8 | { 9 | private readonly HttpClient _client; 10 | 11 | #if DEBUG 12 | public FetchingContext(HttpClient httpClient) => _client = httpClient; 13 | #else 14 | internal FetchingContext(HttpClient httpClient) => _client = httpClient; 15 | #endif 16 | 17 | public async Task FetchAsync(string address) 18 | { 19 | Guard.AgainstNull(nameof(address), address); 20 | Guard.AgainstEmptyWhiteSpace(nameof(address), address); 21 | 22 | try { 23 | var response = await _client.GetAsync(address); 24 | if (!response.IsSuccessStatusCode) { 25 | return FetchedDocument.Empty; 26 | } 27 | return new FetchedDocument(await response.Content.ReadAsByteArrayAsync()); 28 | } 29 | catch (HttpRequestException) { 30 | return FetchedDocument.Empty; 31 | } 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/PickAll/Internal/EventHelper.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Runtime.CompilerServices; 3 | 4 | static class EventHelper 5 | { 6 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 7 | public static void RaiseEvent(object sender, EventHandler handler, EventArgs args, bool enabled) 8 | { 9 | if (enabled && handler != null) { 10 | handler(sender, args); 11 | } 12 | } 13 | 14 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 15 | public static void RaiseEvent(object sender, EventHandler handler, Func args, bool enabled) 16 | where T : EventArgs 17 | { 18 | if (enabled && handler != null) { 19 | handler(sender, args()); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/PickAll/Internal/Fuzzy.cs: -------------------------------------------------------------------------------- 1 | // From: https://github.com/gsscoder/sharprhythm/blob/master/src/SharpRhythm/Algorithms/LevenshteinFuzzyMatch.cs 2 | 3 | using System; 4 | 5 | interface IFuzzyMatch 6 | { 7 | uint Compare(string first, string second); 8 | } 9 | 10 | sealed class LevenshteinFuzzyMatch : IFuzzyMatch 11 | { 12 | public uint Compare(string first, string second) 13 | { 14 | if (first == null) throw new ArgumentNullException(nameof(first)); 15 | if (second == null) throw new ArgumentNullException(nameof(second)); 16 | 17 | uint n = (uint)first.Length; 18 | uint m = (uint)second.Length; 19 | uint[,] d = new uint[n + 1, m + 1]; 20 | 21 | // Step 1 22 | if (n == 0) { 23 | return m; 24 | } 25 | if (m == 0) { 26 | return n; 27 | } 28 | // Step 2 29 | for (uint i = 0; i <= n; d[i, 0] = i++) { 30 | } 31 | for (uint j = 0; j <= m; d[0, j] = j++) { 32 | } 33 | // Step 3 34 | for (uint i = 1; i <= n; i++) { 35 | //Step 4 36 | for (uint j = 1; j <= m; j++) { 37 | // Step 5 38 | uint cost = (second[(int)j - 1] == first[(int)i - 1]) ? 0 : 1u; 39 | // Step 6 40 | d[i, j] = Math.Min( 41 | Math.Min(d[i - 1, j] + 1, d[i, j - 1] + 1), 42 | d[i - 1, j - 1] + cost); 43 | } 44 | } 45 | // Step 7 46 | return d[n, m]; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/PickAll/Internal/Guard.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Runtime.CompilerServices; 4 | 5 | static class Guard 6 | { 7 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 8 | public static void AgainstNull(string argumentName, object value) 9 | { 10 | if (value == null) throw new ArgumentNullException(argumentName, 11 | $"{argumentName} cannot be null."); 12 | } 13 | 14 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 15 | public static void AgainstEmptyWhiteSpace(string argumentName, string value) 16 | { 17 | if (value.Trim() == string.Empty) throw new ArgumentException( 18 | $"{argumentName} cannot be empty or contains only white spaces.", argumentName); 19 | } 20 | 21 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 22 | public static void AgainstNegative(string argumentName, int value) 23 | { 24 | if (value < 0) throw new ArgumentException(argumentName, 25 | $"{argumentName} cannot be lesser than zero."); 26 | } 27 | 28 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 29 | public static void AgainstSubclassExcept(string argumentName, object value) 30 | { 31 | if (!value.GetType().IsSubclassOf(typeof(T))) throw new NotSupportedException( 32 | $"{argumentName} must inherit from {nameof(T)}."); 33 | } 34 | 35 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 36 | public static void AgainstSubclassExcept(string argumentName, params Type[] types) 37 | { 38 | if (types.Any(t => !t.IsSubclassOf(typeof(T)))) throw new NotSupportedException( 39 | $"All {argumentName} must inherit from {nameof(T)}."); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/PickAll/Internal/HtmlElementExtensions.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using AngleSharp.Dom; 3 | using AngleSharp.Html.Dom; 4 | 5 | static class HtmlElementExtensions 6 | { 7 | public static string FirstChildText(this IHtmlElement element, 8 | params string[] selectorsGroup) 9 | { 10 | foreach (var selectors in selectorsGroup) { 11 | var selected = element.QuerySelector(selectors); 12 | if (selected != null) { 13 | if (selected.ChildNodes.Count() > 0) { 14 | return selected.FirstChild.Text(); 15 | } 16 | } 17 | } 18 | return string.Empty; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/PickAll/Internal/ObjectExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | static class ObjectExtensions 5 | { 6 | public static IEnumerable Add(this IEnumerable collection, T newElement) 7 | { 8 | foreach (var element in collection) { 9 | yield return element; 10 | } 11 | yield return newElement; 12 | } 13 | 14 | public static IEnumerable Map(this IEnumerable collection, Func func, 15 | Func predicate = null) 16 | { 17 | foreach (var element in collection) { 18 | if (element.GetType().EqualsOrSubtype() && 19 | (predicate == null || 20 | (predicate != null && predicate((T)element)))) { 21 | yield return func((T)element); 22 | } 23 | else { 24 | yield return element; 25 | } 26 | } 27 | } 28 | 29 | public static IEnumerable Remove(this IEnumerable collection, Type type) 30 | { 31 | bool removed = false; 32 | foreach (var element in collection) { 33 | if (!element.GetType().Equals(type)) { 34 | yield return element; 35 | } 36 | else { 37 | if (!removed) { 38 | removed = true; 39 | } 40 | else { 41 | yield return element; 42 | } 43 | } 44 | } 45 | } 46 | 47 | public static IEnumerable Remove( 48 | this IEnumerable collection) => collection.Remove(typeof(T)); 49 | 50 | public static IEnumerable RemoveAll(this IEnumerable collection) 51 | { 52 | foreach (var element in collection) { 53 | if (!element.GetType().IsSubclassOf(typeof(T))) { 54 | yield return element; 55 | } 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/PickAll/Internal/TypeExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | static class TypeExtensions 4 | { 5 | public static bool EqualsOrSubtype(this Type type) => type.Equals(typeof(T)) || 6 | type.IsSubclassOf(typeof(T)); 7 | } 8 | -------------------------------------------------------------------------------- /src/PickAll/PickAll.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netstandard2.0;netcoreapp3.1;net5.0 5 | 9.0 6 | .NET agile and extensible web searching API 7 | .NET agile and extensible web searching API 8 | 1.3.1 9 | gsscoder 10 | Copyright © Giacomo Stelluti Scala, 2019-2021 11 | https://github.com/gsscoder/pickall 12 | https://github.com/gsscoder/pickall 13 | MIT 14 | web;scraping;api;library 15 | icon.png 16 | README.md 17 | 18 | 19 | ../../artifacts/PickAll/Debug 20 | 21 | 22 | ../../artifacts/PickAll/Release 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /src/PickAll/PostProcessors/FuzzyMatch.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | 5 | namespace PickAll 6 | { 7 | /// Settings for FuzzyMatch post processor. 8 | public struct FuzzyMatchSettings 9 | { 10 | int _minimumDistance; 11 | int _maximumDistance; 12 | 13 | /// String to compare against descriptions. 14 | public string Text { get; set; } 15 | 16 | /// Minimum distance of permutations. 17 | public int MinimumDistance 18 | { 19 | get { return _minimumDistance; } 20 | set 21 | { 22 | Guard.AgainstNegative("MinimumDistance", value); 23 | _minimumDistance = value; 24 | } 25 | } 26 | 27 | /// Maximum distance of permutations. 28 | public int MaximumDistance 29 | { 30 | get { return _maximumDistance; } 31 | set 32 | { 33 | Guard.AgainstNegative("MaximumDistance", value); 34 | _maximumDistance = value; 35 | } 36 | } 37 | } 38 | 39 | /// Compares a string against results descriptions. 40 | public class FuzzyMatch : PostProcessor 41 | { 42 | readonly FuzzyMatchSettings _settings; 43 | 44 | public FuzzyMatch(object settings) : base(settings) 45 | { 46 | if (!(Settings is FuzzyMatchSettings)) { 47 | throw new NotSupportedException($"{nameof(settings)} must be of FuzzyMatchSettings type"); 48 | } 49 | _settings = (FuzzyMatchSettings)Settings; 50 | } 51 | 52 | public override IEnumerable Process(IEnumerable results) 53 | { 54 | var fuzzyMatch = new LevenshteinFuzzyMatch(); 55 | return 56 | from computed in 57 | from result in results 58 | select new {result = result, 59 | distance = fuzzyMatch.Compare(_settings.Text, result.Description)} 60 | where computed.distance >= _settings.MinimumDistance && 61 | computed.distance <= _settings.MaximumDistance 62 | select computed.result; 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/PickAll/PostProcessors/Improve.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Text; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using SharpX; 6 | using SharpX.Extensions; 7 | 8 | namespace PickAll 9 | { 10 | /// Settings for Improve post processor. 11 | public struct ImproveSettings 12 | { 13 | int _wordCount; 14 | int _noiseLength; 15 | 16 | /// Number of word with highest frequency to use in subsequent search. 17 | public int WordCount 18 | { 19 | get { return _wordCount; } 20 | set 21 | { 22 | Guard.AgainstNegative("WordCount", value); 23 | _wordCount = value; 24 | } 25 | } 26 | 27 | /// Length of words to be considered noise. 28 | public int NoiseLength 29 | { 30 | get { return _noiseLength; } 31 | set 32 | { 33 | Guard.AgainstNegative("NoiseLength", value); 34 | _noiseLength = value; 35 | } 36 | } 37 | } 38 | 39 | /// Improves results computing word frequency to perform a subsequent search. 40 | public class Improve : PostProcessor 41 | { 42 | readonly ImproveSettings _settings; 43 | 44 | public Improve(object settings) : base(settings) 45 | { 46 | if (!(settings is ImproveSettings)) { 47 | throw new NotSupportedException( 48 | $"{nameof(settings)} must be of ImproveSettings type"); 49 | } 50 | _settings = (ImproveSettings)Settings; 51 | } 52 | 53 | #if DEBUG 54 | public 55 | #endif 56 | IEnumerable FoldDescriptions(IEnumerable results) 57 | { 58 | Func couldBeNoise = _settings.NoiseLength == 0 59 | ? couldBeNoise = _ => false 60 | : w => w.Length <= _settings.NoiseLength; 61 | var words = from result in results 62 | from word in result.Description.Split() 63 | where word.IsAlphanumeric() 64 | select word; 65 | var folded = from w in 66 | from word in words 67 | group word by word into g 68 | select new Tuple(g.Key, g.Count()) 69 | orderby w.Item2 descending 70 | select w; 71 | IEnumerable> refined; 72 | var query = Runtime.Query ?? string.Empty; 73 | var queryWords = query.ToLower().Split(); 74 | refined = from computed in folded 75 | where !queryWords.Contains(computed.Item1.ToLower()) 76 | && !couldBeNoise.Invoke(computed.Item1) 77 | select computed; 78 | 79 | return (from computed in refined 80 | select computed.Item1).Take(_settings.WordCount); 81 | } 82 | 83 | public override bool PublishEvents { get { return true; } } 84 | 85 | public override IEnumerable Process(IEnumerable results) 86 | { 87 | var builder = new StringBuilder(); 88 | builder.Append(string.Join(" ", FoldDescriptions(results).ToArray())); 89 | builder.Append(' '); 90 | builder.Append(Runtime.Query); 91 | 92 | return Context 93 | .WithoutAll() 94 | .With() 95 | .With() 96 | .SearchAsync(builder.ToString()) 97 | .Result; 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/PickAll/PostProcessors/Order.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | 4 | namespace PickAll 5 | { 6 | /// Orders results placing indexes of same number close by each other. 7 | public class Order : PostProcessor 8 | { 9 | public Order(object settings) : base(settings) 10 | { 11 | } 12 | 13 | public override IEnumerable Process(IEnumerable results) 14 | { 15 | return results.OrderBy(result => result.Index); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/PickAll/PostProcessors/Textify.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Text; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using SharpX; 6 | using SharpX.Extensions; 7 | 8 | namespace PickAll 9 | { 10 | /// Settings for Textify post processor. 11 | public struct TextifySettings 12 | { 13 | int _noiseLength; 14 | int? _maximumLength; 15 | 16 | /// If set to true, page title will be included in result. 17 | public bool IncludeTitle { get; set; } 18 | 19 | /// If set to true, extracted text is sanitized. 20 | public bool SanitizeText { get; set; } 21 | 22 | /// Maximum allowed length of the page to scrape. If null, will be to to a default 23 | /// of 100000. 24 | /// An high limit with numerous pages to scrape can be resource intensive. 25 | public int? MaximumLength 26 | { 27 | get { return _maximumLength; } 28 | set 29 | { 30 | if (value.HasValue) Guard.AgainstNegative("MaximumLength", value.Value); 31 | _maximumLength = value; 32 | } 33 | } 34 | 35 | /// Length of words to be considered noise. 36 | public int NoiseLength 37 | { 38 | get { return _noiseLength; } 39 | set 40 | { 41 | Guard.AgainstNegative("NoiseLength", value); 42 | _noiseLength = value; 43 | } 44 | } 45 | } 46 | 47 | /// Data produced by Textify post processor. 48 | public struct TextifyData 49 | { 50 | public TextifyData(string text) => Text = text; 51 | 52 | public string Text { get; private set; } 53 | 54 | public override string ToString() => Text; 55 | } 56 | 57 | /// Extracts all text from results URLs. 58 | public class Textify : PostProcessor 59 | { 60 | readonly TextifySettings _settings; 61 | 62 | 63 | public Textify(object settings) : base(settings) 64 | { 65 | if (!(settings is TextifySettings)) { 66 | throw new NotSupportedException( 67 | $"{nameof(settings)} must be of {nameof(TextifySettings)} type"); 68 | } 69 | _settings = (TextifySettings)Settings; 70 | } 71 | 72 | public override bool PublishEvents { get { return true; } } 73 | 74 | public override IEnumerable Process(IEnumerable results) 75 | { 76 | var builder = new StringBuilder(512); 77 | var limit = _settings.MaximumLength ?? 100000; 78 | foreach (var result in results) { 79 | var document = Context.Fetching.FetchAsync(result.Url).Result; 80 | if (document.Equals(FetchedDocument.Empty)) continue; 81 | if (document.Length > limit) continue; 82 | 83 | if (_settings.IncludeTitle) { 84 | builder.Append(document.ElementSelector("title")); 85 | builder.Append(' '); 86 | } 87 | builder.Append(AllTextContent(document)); 88 | yield return result.Clone(new TextifyData(builder.ToString().TrimEnd())); 89 | } 90 | 91 | string AllTextContent(IFetchedDocument document) 92 | { 93 | var content = new StringBuilder(512); 94 | content.Append(JoinAndRefine(document.ElementSelectorAll("div"))); 95 | content.Append(JoinAndRefine(document.ElementSelectorAll("p"))); 96 | content.Append(JoinAndRefine(document.ElementSelectorAll("h1"))); 97 | content.Append(JoinAndRefine(document.ElementSelectorAll("h2"))); 98 | content.Append(JoinAndRefine(document.ElementSelectorAll("h3"))); 99 | content.Append(JoinAndRefine(document.ElementSelectorAll("h4"))); 100 | content.Append(JoinAndRefine(document.ElementSelectorAll("h5"))); 101 | content.Append(JoinAndRefine(document.ElementSelectorAll("h6"))); 102 | return content.ToString(); 103 | string JoinAndRefine(IEnumerable texts) { 104 | return string.Concat( 105 | string.Join(" ", 106 | from text in texts 107 | select RemoveNoise(Sanitize(text.StripTag().NormalizeWhiteSpace()))), 108 | " "); 109 | string Sanitize(string text) { 110 | if (!_settings.SanitizeText) return text; 111 | return text.Sanitize(); 112 | } 113 | string RemoveNoise(string text) { 114 | if (_settings.NoiseLength == 0) return text; 115 | return text.StripByLength(_settings.NoiseLength); 116 | } 117 | } 118 | } 119 | } 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/PickAll/PostProcessors/Uniqueness.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using SharpX.Extensions; 3 | 4 | namespace PickAll 5 | { 6 | /// Removes duplicate results by URL. 7 | public class Uniqueness : PostProcessor 8 | { 9 | public Uniqueness(object settings) : base(settings) 10 | { 11 | } 12 | 13 | public override IEnumerable Process(IEnumerable results) 14 | { 15 | return results.DistinctBy(result => result.Url); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/PickAll/ResultInfo.cs: -------------------------------------------------------------------------------- 1 | namespace PickAll 2 | { 3 | /// Models a Searcher result record. 4 | public class ResultInfo 5 | { 6 | #if DEBUG 7 | public ResultInfo() { } 8 | #endif 9 | /// Initializes a new instance of ResultInfo. 10 | public ResultInfo(string originator, int index, string url, string description, object data) 11 | { 12 | Guard.AgainstNull(nameof(originator), originator); 13 | Guard.AgainstEmptyWhiteSpace(nameof(originator), originator); 14 | Guard.AgainstNegative(nameof(index), index); 15 | Guard.AgainstNull(nameof(url), url); 16 | Guard.AgainstEmptyWhiteSpace(nameof(url), url); 17 | Guard.AgainstNull(nameof(description), description); 18 | 19 | Originator = originator; 20 | Index = index; 21 | Url = url; 22 | Description = description; 23 | Data = data; 24 | } 25 | 26 | /// The Searcher which originated the result. 27 | public string Originator 28 | { 29 | get; 30 | #if DEBUG 31 | private set; 32 | #else 33 | internal set; 34 | #endif 35 | } 36 | 37 | /// The result index. 38 | public int Index 39 | { 40 | get; 41 | #if !DEBUG 42 | private set; 43 | #else 44 | internal set; 45 | #endif 46 | } 47 | 48 | /// The result URL. 49 | public string Url 50 | { 51 | get; 52 | #if !DEBUG 53 | private set; 54 | #else 55 | internal set; 56 | #endif 57 | } 58 | 59 | /// The result description. 60 | public string Description 61 | { 62 | get; 63 | #if !DEBUG 64 | private set; 65 | #else 66 | internal set; 67 | #endif 68 | } 69 | /// Additional data supplied by the service. 70 | public object Data 71 | { 72 | get; 73 | #if !DEBUG 74 | private set; 75 | #else 76 | internal set; 77 | #endif 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/PickAll/ResultInfoExtensions.cs: -------------------------------------------------------------------------------- 1 | namespace PickAll 2 | { 3 | public static class ResultInfoExtensions 4 | { 5 | public static ResultInfo Clone(this ResultInfo resultInfo, object data = null) 6 | { 7 | Guard.AgainstNull(nameof(resultInfo), resultInfo); 8 | 9 | var _data = data ?? resultInfo.Data; 10 | return new ResultInfo( 11 | resultInfo.Originator, 12 | resultInfo.Index, 13 | resultInfo.Url, 14 | resultInfo.Description, 15 | _data); 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/PickAll/RuntimeInfo.cs: -------------------------------------------------------------------------------- 1 | namespace PickAll 2 | { 3 | /// Models runtime informations managed by a Service. 4 | public struct RuntimeInfo 5 | { 6 | internal RuntimeInfo(string query, int? maximumResults) 7 | { 8 | Query = query; 9 | MaximumResults = maximumResults; 10 | } 11 | 12 | public string Query { get; set; } 13 | 14 | public int? MaximumResults { get; private set; } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/PickAll/SearchContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using System.Net.Http; 6 | using AngleSharp; 7 | using AngleSharp.Io.Network; 8 | 9 | namespace PickAll 10 | { 11 | /// Manages Searcher and PostProcessor instances to gather 12 | /// and elaborate results. 13 | public sealed class SearchContext 14 | { 15 | readonly Lazy _browsing; 16 | readonly Lazy _fetching; 17 | static readonly Lazy _default = new Lazy( 18 | () => new SearchContext( 19 | typeof(Google), 20 | typeof(DuckDuckGo), 21 | typeof(Uniqueness), 22 | typeof(Order))); 23 | 24 | internal SearchContext(IEnumerable services, ContextSettings settings) 25 | { 26 | Services = services; 27 | Settings = settings; 28 | _browsing = new Lazy( 29 | () => BuildBrowsingContext(settings.Timeout, () => BuildHttpClient(settings.Timeout))); 30 | _fetching = new Lazy( 31 | () => new FetchingContext(BuildHttpClient(settings.Timeout, new HttpClient()))); 32 | #if DEBUG 33 | EnforceMaximumResults = true; 34 | #endif 35 | } 36 | 37 | public SearchContext(ContextSettings settings) 38 | : this(Enumerable.Empty(), settings) { } 39 | 40 | public SearchContext() 41 | : this(Enumerable.Empty(), new ContextSettings()) { } 42 | 43 | public SearchContext(int maximumResults) 44 | : this(Enumerable.Empty(), new ContextSettings { MaximumResults = maximumResults }) { } 45 | 46 | public SearchContext(TimeSpan timeout) 47 | : this(Enumerable.Empty(), new ContextSettings { Timeout = timeout }) { } 48 | 49 | /// Builds a new search context with a given types. 50 | public SearchContext(params Type[] services) : this() 51 | { 52 | Guard.AgainstSubclassExcept(nameof(services), services); 53 | 54 | Services = Enumerable.Empty(); 55 | foreach (var type in services) { 56 | var instance = Activator.CreateInstance(type, new object[] { null }); 57 | Services = Services.Add(instance); 58 | } 59 | } 60 | 61 | /// Occurs when search begins. 62 | public event EventHandler SearchBegin; 63 | /// Occurs when search terminates. 64 | public event EventHandler SearchEnd; 65 | /// Occurs when a service is loaded. 66 | public event EventHandler ServiceLoad; 67 | /// Occurs a ResultInfo is created. 68 | public event EventHandler ResultCreated; 69 | /// Occurs a ResultInfo is processed. 70 | public event EventHandler ResultProcessed; 71 | #pragma warning disable CS3003 72 | /// Current IBrowsingContext instance. 73 | public IBrowsingContext Browsing => _browsing.Value; 74 | #pragma warning restore CS3003 75 | /// Current IFetchingContext instance. 76 | public IFetchingContext Fetching => _fetching.Value; 77 | #if !DEBUG 78 | internal IEnumerable Services { get; private set; } 79 | internal ContextSettings Settings { get; private set; } 80 | #else 81 | public IEnumerable Services { get; private set; } 82 | public ContextSettings Settings { get; private set; } 83 | public bool EnforceMaximumResults { get; set; } // Debug only 84 | #endif 85 | 86 | /// Executes a search using the given query, invoking all Searcher 87 | /// services asynchronously and then PostProcessor services in chain. Returns a 88 | /// sequence of ResultInfo. 89 | public async Task> SearchAsync(string query) 90 | { 91 | Guard.AgainstNull(nameof(query), query); 92 | Guard.AgainstEmptyWhiteSpace(nameof(query), query); 93 | 94 | EventHelper.RaiseEvent(this, SearchBegin, 95 | () => new SearchBeginEventArgs(query), Settings.EnableRaisingEvents); 96 | // Bind context and partition maximum results 97 | Services = Configure(query, this); 98 | // Invoke searchers in parallel 99 | var resultGroup = await Task.WhenAll( 100 | from searcher in Services.OfType() 101 | select searcher.SearchAsync(query)); 102 | var results = resultGroup.SelectMany(group => group).ToList(); 103 | if (Settings.MaximumResults != null) { 104 | #if !DEBUG 105 | // Default behaviour 106 | results = new List(results.Take((int)Settings.MaximumResults.Value)); 107 | #else 108 | // Useful for debugging 109 | if (EnforceMaximumResults) { 110 | results = new List(results.Take((int)Settings.MaximumResults.Value)); 111 | } 112 | #endif 113 | } 114 | // Invoke post processors in sync 115 | foreach (var processor in Services.OfType()) { 116 | var publish = Settings.EnableRaisingEvents && processor.PublishEvents; 117 | var current = processor.Process(results); 118 | results = new List(); 119 | foreach (var result in current) { 120 | EventHelper.RaiseEvent(processor, ResultProcessed, 121 | () => new ResultHandledEventArgs(result, ServiceType.PostProcessor), publish); 122 | results.Add(result); 123 | } 124 | } 125 | EventHelper.RaiseEvent(this, SearchEnd, EventArgs.Empty, Settings.EnableRaisingEvents); 126 | return results; 127 | } 128 | 129 | /// Builds a SearchContext instance with default services. 130 | public static SearchContext Default = _default.Value; 131 | 132 | static IEnumerable Configure(string query, SearchContext context) 133 | { 134 | var searchers = context.Services.OfType(); 135 | var first = searchers.FirstOrDefault(); 136 | var maximumResults = context.Settings.MaximumResults.HasValue 137 | ? context.Settings.MaximumResults / searchers.Count() 138 | : null; 139 | var services = context.Services 140 | .Map(service => 141 | { 142 | service.Load += context.ServiceLoad; 143 | service.Context = context; 144 | service.Runtime = new RuntimeInfo(query, maximumResults); 145 | return service; 146 | }) 147 | .Map(searcher => 148 | { 149 | searcher.ResultCreated += context.ResultCreated; 150 | return searcher; 151 | }); 152 | if (first != null) { 153 | // First service maybe burdened of handling extra results 154 | services = services.Map(searcher => 155 | { 156 | searcher.Runtime = new RuntimeInfo( 157 | query, 158 | searcher.Runtime.MaximumResults + 159 | context.Settings.MaximumResults % searchers.Count()); 160 | return searcher; 161 | }, 162 | searcher => searcher.GetHashCode().Equals(first.GetHashCode())); 163 | } 164 | return services; 165 | } 166 | 167 | static HttpClient BuildHttpClient(TimeSpan? timeout, HttpClient defaultClient = null) 168 | { 169 | if (timeout.HasValue) { 170 | var client = new HttpClient(); 171 | client.Timeout = timeout.Value; 172 | return client; 173 | } 174 | return defaultClient; 175 | } 176 | 177 | static IBrowsingContext BuildBrowsingContext( 178 | TimeSpan? timeout, Func client) 179 | { 180 | if (timeout.HasValue) { 181 | var requester = new HttpClientRequester(client()); 182 | return BrowsingContext.New( 183 | Configuration.Default 184 | .WithRequester(requester) 185 | .WithDefaultLoader()); 186 | } 187 | return BrowsingContext.New(Configuration.Default.WithDefaultLoader()); 188 | } 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /src/PickAll/SearchContextExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using System.Reflection; 4 | 5 | namespace PickAll 6 | { 7 | /// A set of useful extensions for SearchContext. 8 | public static class SearchContextExtensions 9 | { 10 | /// Registers an instance of Searcher or PostProcessor without settings, 11 | /// using its type. 12 | public static SearchContext With(this SearchContext context, object settings = null) 13 | where T : Service 14 | { 15 | Guard.AgainstNull(nameof(context), context); 16 | 17 | var service = (T)Activator.CreateInstance(typeof(T), settings); 18 | return new SearchContext( 19 | context.Services.Add(service), 20 | context.Settings.Clone()); 21 | } 22 | 23 | /// Registers an instance of Searcher or PostProcessor without settings, 24 | /// using its type name. 25 | public static SearchContext With(this SearchContext context, string serviceName, 26 | object settings = null) 27 | { 28 | Guard.AgainstNull(nameof(context), context); 29 | Guard.AgainstNull(nameof(serviceName), serviceName); 30 | Guard.AgainstEmptyWhiteSpace(nameof(serviceName), serviceName); 31 | 32 | var type = context.GetType().GetTypeInfo().Assembly.GetTypes().Where( 33 | @this => @this.Name.Equals(serviceName, StringComparison.OrdinalIgnoreCase)) 34 | .SingleOrDefault(); 35 | if (type == null) { 36 | throw new NotSupportedException($"{serviceName} service not found"); 37 | } 38 | 39 | var service = Activator.CreateInstance(type, settings); 40 | Guard.AgainstSubclassExcept(nameof(serviceName), service); 41 | return new SearchContext( 42 | context.Services.Add(service), 43 | context.Settings.Clone()); 44 | } 45 | 46 | /// Unregisters first instance of Searcher or PostProcessor, using its 47 | /// type. 48 | public static SearchContext Without(this SearchContext context) 49 | where T : Service 50 | { 51 | Guard.AgainstNull(nameof(context), context); 52 | 53 | return new SearchContext( 54 | context.Services.Remove(), 55 | context.Settings.Clone()); 56 | } 57 | 58 | /// Unregisters first instance of Searcher or PostProcessor, using its 59 | /// type name. 60 | public static SearchContext Without(this SearchContext context, string serviceName) 61 | { 62 | Guard.AgainstNull(nameof(context), context); 63 | Guard.AgainstNull(nameof(serviceName), serviceName); 64 | Guard.AgainstEmptyWhiteSpace(nameof(serviceName), serviceName); 65 | 66 | var service = (from @this in context.Services 67 | where @this.GetType().Name.Equals( 68 | serviceName, StringComparison.OrdinalIgnoreCase) 69 | select @this).FirstOrDefault(); 70 | if (service == null) { 71 | throw new InvalidOperationException($"{serviceName} not registred as service"); 72 | } 73 | return new SearchContext( 74 | context.Services.Remove(service.GetType()), 75 | context.Settings.Clone()); 76 | } 77 | 78 | /// Unregisters all instances of types that inherits from Searcher 79 | /// or PostProcessor. 80 | public static SearchContext WithoutAll(this SearchContext context) 81 | where T : Service 82 | { 83 | Guard.AgainstNull(nameof(context), context); 84 | 85 | return new SearchContext( 86 | context.Services.RemoveAll(), 87 | context.Settings.Clone()); 88 | } 89 | 90 | /// Configures a search context with a ContextSettings instance. If merge 91 | /// is true the settings instance is merged to the actul one. 92 | public static SearchContext WithConfiguration(this SearchContext context, 93 | ContextSettings settings, bool merge = false) 94 | { 95 | Guard.AgainstNull(nameof(context), context); 96 | 97 | if (merge) { 98 | var merged = context.Settings; 99 | if (settings.MaximumResults != default(int?)) merged.MaximumResults = settings.MaximumResults; 100 | if (settings.Timeout != default(TimeSpan)) merged.Timeout = settings.Timeout; 101 | if (settings.EnableRaisingEvents != default(bool)) merged.EnableRaisingEvents = settings.EnableRaisingEvents; 102 | return new SearchContext(context.Services, merged); 103 | } 104 | return new SearchContext(context.Services, settings); 105 | } 106 | 107 | /// Configures a search context able to raise events. 108 | public static SearchContext WithEvents(this SearchContext context) 109 | { 110 | Guard.AgainstNull(nameof(context), context); 111 | 112 | return context.WithConfiguration( 113 | new ContextSettings { EnableRaisingEvents = true }, merge: true); 114 | } 115 | 116 | /// Builds a new search context with same services of the current. 117 | public static SearchContext Clone(this SearchContext context) 118 | { 119 | Guard.AgainstNull(nameof(context), context); 120 | 121 | return new SearchContext( 122 | context.Services.Map(service => 123 | { 124 | service.Context = null; 125 | return service; 126 | }), 127 | new ContextSettings { MaximumResults = context.Settings.MaximumResults }); 128 | } 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /src/PickAll/Searchers/Bing.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using AngleSharp; 6 | using AngleSharp.Html.Dom; 7 | using AngleSharp.Dom; 8 | 9 | namespace PickAll 10 | { 11 | /// Searcher that searches on Bing search engine. 12 | public class Bing : Searcher 13 | { 14 | public Bing(object settings) : base(settings) 15 | { 16 | } 17 | 18 | public override async Task> SearchAsync(string query) 19 | { 20 | using var document = await Context.Browsing.OpenAsync("https://www.bing.com/"); 21 | var form = document.QuerySelector("#sb_form"); 22 | ((IHtmlInputElement)form["sb_form_q"]).Value = query; 23 | using var result = await form.SubmitAsync(form); 24 | // Select only actual results 25 | var links = from link in result.QuerySelectorAll("li.b_algo a") 26 | where link.Attributes["href"].Value.StartsWith( 27 | "http", 28 | StringComparison.OrdinalIgnoreCase) 29 | select link; 30 | 31 | return links.Select((link, index) => 32 | CreateResult((ushort)index, link.Attributes["href"].Value, link.Text)); 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/PickAll/Searchers/BingNews.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using AngleSharp; 6 | using AngleSharp.Html.Dom; 7 | using AngleSharp.Dom; 8 | 9 | namespace PickAll 10 | { 11 | /// Searcher that searches on Bing search engine for news. 12 | public class BingNews : Searcher 13 | { 14 | const string _searchUrl = "https://www.bing.com/news"; 15 | 16 | public BingNews(object settings) : base(settings) 17 | { 18 | } 19 | 20 | public override async Task> SearchAsync(string query) 21 | { 22 | using var document = await Context.Browsing.OpenAsync(_searchUrl); 23 | var form = document.QuerySelector("#sb_form"); 24 | ((IHtmlInputElement)form["sb_form_q"]).Value = query; 25 | using var result = await form.SubmitAsync(form); 26 | // Select only actual results 27 | var links = from l in 28 | (from link in result.QuerySelectorAll("a[h*='news']") 29 | where link.Href.StartsWith("http", StringComparison.OrdinalIgnoreCase) 30 | select link) 31 | where !l.Href.Contains("go.microsoft.com") && 32 | !l.Href.Equals($"{_searchUrl}/search#", StringComparison.OrdinalIgnoreCase) && 33 | !string.IsNullOrWhiteSpace(l.Text) 34 | select l; 35 | 36 | return links.Select((link, index) => 37 | CreateResult((ushort)index, link.Href, link.Text)); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/PickAll/Searchers/DuckDuckGo.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using AngleSharp; 5 | using AngleSharp.Html.Dom; 6 | using AngleSharp.Dom; 7 | 8 | namespace PickAll 9 | { 10 | /// Searcher that searches on DuckDuckGo search engine. 11 | public class DuckDuckGo : Searcher 12 | { 13 | public DuckDuckGo(object settings) : base(settings) 14 | { 15 | } 16 | 17 | public override async Task> SearchAsync(string query) 18 | { 19 | using var document = await Context.Browsing.OpenAsync("https://duckduckgo.com/"); 20 | var form = document.QuerySelector("#search_form_homepage"); 21 | using var result = await form.SubmitAsync( 22 | new { q = query }); var links = result.QuerySelectorAll("a.result__a"); 23 | 24 | return links.Select((link, index) => 25 | CreateResult((ushort)index, link.Attributes["href"].Value, link.Text)); 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/PickAll/Searchers/Google.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading.Tasks; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text.RegularExpressions; 6 | using AngleSharp; 7 | using AngleSharp.Html.Dom; 8 | using AngleSharp.Dom; 9 | 10 | namespace PickAll 11 | { 12 | /// Searcher that searches on Google search engine. 13 | public class Google : Searcher 14 | { 15 | static readonly Regex _normalize = new Regex(@"^/url\?q=([^&]*)&.*", RegexOptions.Compiled); 16 | 17 | public Google(object settings) : base(settings) 18 | { 19 | } 20 | 21 | public override async Task> SearchAsync(string query) 22 | { 23 | using var document = await Context.Browsing.OpenAsync("https://www.google.com/"); 24 | var form = document.QuerySelector("form[action='/search']"); 25 | using var result = await form.SubmitAsync(new { q = query }); 26 | // Take only valid URLs 27 | var links = from anchor in result.QuerySelectorAll("a") 28 | where Validate(anchor.Attributes["href"].Value) 29 | select anchor; 30 | // Create results normalizing URLs 31 | var results = links.Select((link, index) => 32 | CreateResult((ushort)index, Normalize(link.Attributes["href"].Value), 33 | link.FirstChildText("div", "span"))); 34 | 35 | // Discard ones without description (not actual results) 36 | return from @this in results 37 | where @this.Description.Trim() != string.Empty 38 | select @this; 39 | } 40 | 41 | static bool Validate(string url) => 42 | url.StartsWith( 43 | "/url?", StringComparison.OrdinalIgnoreCase) && 44 | !url.StartsWith( 45 | "/url?q=http://webcache.googleusercontent.com",StringComparison.OrdinalIgnoreCase); 46 | 47 | static string Normalize(string url) 48 | { 49 | var match = _normalize.Match(url); 50 | return match.Groups.Count == 2 ? match.Groups[1].Value : url; 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/PickAll/Searchers/Yahoo.cs: -------------------------------------------------------------------------------- 1 | using System.Threading.Tasks; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using AngleSharp; 5 | using AngleSharp.Html.Dom; 6 | using AngleSharp.Dom; 7 | 8 | namespace PickAll 9 | { 10 | /// Searcher that searches on Yahoo search engine. 11 | public class Yahoo : Searcher 12 | { 13 | public Yahoo(object settings) : base(settings) 14 | { 15 | } 16 | 17 | public override async Task> SearchAsync(string query) 18 | { 19 | using var document = await Context.Browsing.OpenAsync("https://yahoo.com/"); 20 | var form = document.QuerySelector("#uh-search-form"); 21 | ((IHtmlInputElement)form["uh-search-box"]).Value = query; 22 | using var result = await form.SubmitAsync(form); 23 | var links = result.QuerySelectorAll("#web h3.title a"); 24 | 25 | return links.Select((link, index) => 26 | CreateResult((ushort)index, link.Attributes["href"].Value, link.Text)); 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/PickAll/paket.references: -------------------------------------------------------------------------------- 1 | group main 2 | AngleSharp 3 | AngleSharp.Io 4 | SharpX 5 | -------------------------------------------------------------------------------- /tests/PickAll.Specs/.editorconfig: -------------------------------------------------------------------------------- 1 | [*.{csproj,config}] 2 | indent_style = space 3 | indent_size = 2 4 | 5 | # C# files 6 | [*.cs] 7 | 8 | #### Core EditorConfig Options #### 9 | 10 | # Indentation and spacing 11 | indent_size = 4 12 | indent_style = space 13 | tab_width = 4 14 | 15 | # New line preferences 16 | end_of_line = crlf 17 | insert_final_newline = false 18 | 19 | #### .NET Coding Conventions #### 20 | 21 | # Organize usings 22 | dotnet_separate_import_directive_groups = false 23 | dotnet_sort_system_directives_first = true 24 | file_header_template = unset 25 | 26 | # this. and Me. preferences 27 | dotnet_style_qualification_for_event = false 28 | dotnet_style_qualification_for_field = false 29 | dotnet_style_qualification_for_method = false 30 | dotnet_style_qualification_for_property = false 31 | 32 | # Language keywords vs BCL types preferences 33 | dotnet_style_predefined_type_for_locals_parameters_members = true 34 | dotnet_style_predefined_type_for_member_access = true 35 | 36 | # Parentheses preferences 37 | dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity 38 | dotnet_style_parentheses_in_other_binary_operators = always_for_clarity 39 | dotnet_style_parentheses_in_other_operators = never_if_unnecessary 40 | dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity 41 | 42 | # Modifier preferences 43 | dotnet_style_require_accessibility_modifiers = for_non_interface_members 44 | 45 | # Expression-level preferences 46 | dotnet_style_coalesce_expression = true 47 | dotnet_style_collection_initializer = true 48 | dotnet_style_explicit_tuple_names = true 49 | dotnet_style_null_propagation = true 50 | dotnet_style_object_initializer = true 51 | dotnet_style_operator_placement_when_wrapping = beginning_of_line 52 | dotnet_style_prefer_auto_properties = true 53 | dotnet_style_prefer_compound_assignment = true 54 | dotnet_style_prefer_conditional_expression_over_assignment = true 55 | dotnet_style_prefer_conditional_expression_over_return = true 56 | dotnet_style_prefer_inferred_anonymous_type_member_names = true 57 | dotnet_style_prefer_inferred_tuple_names = true 58 | dotnet_style_prefer_is_null_check_over_reference_equality_method = true 59 | dotnet_style_prefer_simplified_boolean_expressions = true 60 | dotnet_style_prefer_simplified_interpolation = true 61 | 62 | # Field preferences 63 | dotnet_style_readonly_field = true 64 | 65 | # Parameter preferences 66 | dotnet_code_quality_unused_parameters = all 67 | 68 | # Suppression preferences 69 | dotnet_remove_unnecessary_suppression_exclusions = none 70 | 71 | #### C# Coding Conventions #### 72 | 73 | # var preferences 74 | csharp_style_var_elsewhere = false 75 | csharp_style_var_for_built_in_types = false 76 | csharp_style_var_when_type_is_apparent = false 77 | 78 | # Expression-bodied members 79 | csharp_style_expression_bodied_accessors = true 80 | csharp_style_expression_bodied_constructors = false 81 | csharp_style_expression_bodied_indexers = true 82 | csharp_style_expression_bodied_lambdas = true 83 | csharp_style_expression_bodied_local_functions = false 84 | csharp_style_expression_bodied_methods = false 85 | csharp_style_expression_bodied_operators = false 86 | csharp_style_expression_bodied_properties = true 87 | 88 | # Pattern matching preferences 89 | csharp_style_pattern_matching_over_as_with_null_check = true 90 | csharp_style_pattern_matching_over_is_with_cast_check = true 91 | csharp_style_prefer_not_pattern = true 92 | csharp_style_prefer_pattern_matching = true 93 | csharp_style_prefer_switch_expression = true 94 | 95 | # Null-checking preferences 96 | csharp_style_conditional_delegate_call = true 97 | 98 | # Modifier preferences 99 | csharp_prefer_static_local_function = true 100 | csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async 101 | 102 | # Code-block preferences 103 | csharp_prefer_braces = true 104 | csharp_prefer_simple_using_statement = true 105 | 106 | # Expression-level preferences 107 | csharp_prefer_simple_default_expression = true 108 | csharp_style_deconstructed_variable_declaration = true 109 | csharp_style_implicit_object_creation_when_type_is_apparent = true 110 | csharp_style_inlined_variable_declaration = true 111 | csharp_style_pattern_local_over_anonymous_function = true 112 | csharp_style_prefer_index_operator = true 113 | csharp_style_prefer_range_operator = true 114 | csharp_style_throw_expression = true 115 | csharp_style_unused_value_assignment_preference = discard_variable 116 | csharp_style_unused_value_expression_statement_preference = discard_variable 117 | 118 | # 'using' directive preferences 119 | csharp_using_directive_placement = outside_namespace 120 | 121 | #### C# Formatting Rules #### 122 | 123 | # New line preferences 124 | csharp_new_line_before_catch = true 125 | csharp_new_line_before_else = true 126 | csharp_new_line_before_finally = true 127 | csharp_new_line_before_members_in_anonymous_types = true 128 | csharp_new_line_before_members_in_object_initializers = true 129 | csharp_new_line_before_open_brace = anonymous_methods,anonymous_types,lambdas,methods,object_collection_array_initializers,properties,types 130 | csharp_new_line_between_query_expression_clauses = true 131 | 132 | # Indentation preferences 133 | csharp_indent_block_contents = true 134 | csharp_indent_braces = false 135 | csharp_indent_case_contents = true 136 | csharp_indent_case_contents_when_block = true 137 | csharp_indent_labels = one_less_than_current 138 | csharp_indent_switch_labels = true 139 | 140 | # Space preferences 141 | csharp_space_after_cast = false 142 | csharp_space_after_colon_in_inheritance_clause = true 143 | csharp_space_after_comma = true 144 | csharp_space_after_dot = false 145 | csharp_space_after_keywords_in_control_flow_statements = true 146 | csharp_space_after_semicolon_in_for_statement = true 147 | csharp_space_around_binary_operators = before_and_after 148 | csharp_space_around_declaration_statements = false 149 | csharp_space_before_colon_in_inheritance_clause = true 150 | csharp_space_before_comma = false 151 | csharp_space_before_dot = false 152 | csharp_space_before_open_square_brackets = false 153 | csharp_space_before_semicolon_in_for_statement = false 154 | csharp_space_between_empty_square_brackets = false 155 | csharp_space_between_method_call_empty_parameter_list_parentheses = false 156 | csharp_space_between_method_call_name_and_opening_parenthesis = false 157 | csharp_space_between_method_call_parameter_list_parentheses = false 158 | csharp_space_between_method_declaration_empty_parameter_list_parentheses = false 159 | csharp_space_between_method_declaration_name_and_open_parenthesis = false 160 | csharp_space_between_method_declaration_parameter_list_parentheses = false 161 | csharp_space_between_parentheses = false 162 | csharp_space_between_square_brackets = false 163 | 164 | # Wrapping preferences 165 | csharp_preserve_single_line_blocks = true 166 | csharp_preserve_single_line_statements = true 167 | 168 | #### Naming styles #### 169 | 170 | # Naming rules 171 | 172 | dotnet_naming_rule.interface_should_be_begins_with_i.severity = suggestion 173 | dotnet_naming_rule.interface_should_be_begins_with_i.symbols = interface 174 | dotnet_naming_rule.interface_should_be_begins_with_i.style = begins_with_i 175 | 176 | dotnet_naming_rule.types_should_be_pascal_case.severity = suggestion 177 | dotnet_naming_rule.types_should_be_pascal_case.symbols = types 178 | dotnet_naming_rule.types_should_be_pascal_case.style = pascal_case 179 | 180 | dotnet_naming_rule.non_field_members_should_be_pascal_case.severity = suggestion 181 | dotnet_naming_rule.non_field_members_should_be_pascal_case.symbols = non_field_members 182 | dotnet_naming_rule.non_field_members_should_be_pascal_case.style = pascal_case 183 | 184 | # Symbol specifications 185 | 186 | dotnet_naming_symbols.interface.applicable_kinds = interface 187 | dotnet_naming_symbols.interface.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected 188 | dotnet_naming_symbols.interface.required_modifiers = 189 | 190 | dotnet_naming_symbols.types.applicable_kinds = class, struct, interface, enum 191 | dotnet_naming_symbols.types.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected 192 | dotnet_naming_symbols.types.required_modifiers = 193 | 194 | dotnet_naming_symbols.non_field_members.applicable_kinds = property, event, method 195 | dotnet_naming_symbols.non_field_members.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected 196 | dotnet_naming_symbols.non_field_members.required_modifiers = 197 | 198 | # Naming styles 199 | 200 | dotnet_naming_style.pascal_case.required_prefix = 201 | dotnet_naming_style.pascal_case.required_suffix = 202 | dotnet_naming_style.pascal_case.word_separator = 203 | dotnet_naming_style.pascal_case.capitalization = pascal_case 204 | 205 | dotnet_naming_style.begins_with_i.required_prefix = I 206 | dotnet_naming_style.begins_with_i.required_suffix = 207 | dotnet_naming_style.begins_with_i.word_separator = 208 | dotnet_naming_style.begins_with_i.capitalization = pascal_case 209 | -------------------------------------------------------------------------------- /tests/PickAll.Specs/Fakes/Arbitrary.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Text; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using Bogus; 6 | using WaffleGenerator; 7 | using AngleSharp; 8 | using AngleSharp.Dom; 9 | using SharpX; 10 | using PickAll; 11 | 12 | static class ResultInfoBuilder 13 | { 14 | public static IEnumerable Generate(string originator, ushort samples) 15 | { 16 | for (ushort index = 0; index <= samples - 1; index++) { 17 | var faker = new Faker() 18 | .RuleFor(o => o.Originator, _ => originator) 19 | .RuleFor(o => o.Index, _ => index) 20 | .RuleFor(o => o.Url, f => f.Internet.UrlWithPath(fileExt: "html")) 21 | .RuleFor(o => o.Description, f => f.WaffleTitle()); 22 | yield return faker.Generate(); 23 | } 24 | } 25 | 26 | public static IEnumerable GenerateRandom(string originator, ushort minSamples, ushort maxSamples) 27 | { 28 | var samples = new CryptoRandom().Next(minSamples, maxSamples); 29 | for (ushort index = 0; index <= samples - 1; index++) { 30 | var faker = new Faker() 31 | .RuleFor(o => o.Originator, _ => originator) 32 | .RuleFor(o => o.Index, _ => index) 33 | .RuleFor(o => o.Url, f => f.Internet.UrlWithPath(fileExt: "html")) 34 | .RuleFor(o => o.Description, f => f.WaffleTitle()); 35 | yield return faker.Generate(); 36 | } 37 | } 38 | 39 | public static IEnumerable GenerateUnique(string originator, ushort samples) 40 | { 41 | var generated = new List(); 42 | for (ushort index = 0; index <= samples - 1; index++) { 43 | var faker = new Faker() 44 | .RuleFor(o => o.Originator, _ => originator) 45 | .RuleFor(o => o.Index, _ => index) 46 | .RuleFor(o => o.Url, f => f.Internet.UrlWithPath(fileExt: "html")) 47 | .RuleFor(o => o.Description, f => f.WaffleTitle()); 48 | var candidate = faker.Generate(); 49 | var searched = from @this in generated 50 | where @this.Url == candidate.Url || @this.Description == candidate.Description 51 | select @this; 52 | if (!searched.Any()) { 53 | generated.Add(candidate); 54 | } 55 | else { 56 | index--; 57 | } 58 | } 59 | return generated; 60 | } 61 | } 62 | 63 | static class WaffleBuilder 64 | { 65 | public static IEnumerable GenerateTitle(int times, Func modifier = null) 66 | { 67 | Func _nullModifier = @string => @string; 68 | var _modifier = modifier ?? _nullModifier; 69 | 70 | for (var i = 0; i < times; i++) { 71 | var title = WaffleEngine.Title(); 72 | yield return _modifier(title); 73 | } 74 | } 75 | 76 | public static IDocument GeneratePage(int paragraphs = 1) 77 | { 78 | var context = BrowsingContext.New(Configuration.Default.WithDefaultLoader()); 79 | return context.OpenAsync(request => request.Content( 80 | WaffleEngine.Html( 81 | paragraphs: paragraphs, 82 | includeHeading: true, 83 | includeHeadAndBody: true))).Result; 84 | } 85 | 86 | public static string GenerateParagraph(int samples = 1, bool small = false) 87 | { 88 | var builder = new StringBuilder(); 89 | for (var i = 0; i < samples; i ++) { 90 | builder.AppendLine($"{Generate()}"); 91 | } 92 | return builder.ToString(); 93 | string Generate() { 94 | var paragraph = small 95 | ? GenerateTitle(1).Single() 96 | : WaffleEngine.Text(paragraphs: 1, includeHeading: true); 97 | return $"

{paragraph}

"; 98 | } 99 | } 100 | 101 | public static string GeneratePageAsString(string body, string title = "") 102 | { 103 | return $@" 104 | 105 | {title} 106 | 107 | 108 | {body} 109 | 110 | 111 | "; 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /tests/PickAll.Specs/Fakes/ArbitrarySearcher.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading.Tasks; 5 | using SharpX; 6 | using PickAll; 7 | 8 | class ArbitrarySearcherSettings 9 | { 10 | public ArbitrarySearcherSettings() 11 | { 12 | AtLeast = Maybe.Nothing(); 13 | } 14 | 15 | public ushort Samples { get; set; } 16 | 17 | public Maybe AtLeast { get; set; } 18 | } 19 | 20 | class ArbitrarySearcher : Searcher 21 | { 22 | readonly ArbitrarySearcherSettings _settings; 23 | 24 | public ArbitrarySearcher(object settings) : base(settings) 25 | { 26 | if (!(Settings is ArbitrarySearcherSettings)) { 27 | throw new NotSupportedException(); 28 | } 29 | _settings = (ArbitrarySearcherSettings)Settings; 30 | } 31 | 32 | public override Task> SearchAsync(string query) 33 | { 34 | return Task.FromResult(_()); IEnumerable _() 35 | { 36 | var originator = Guid.NewGuid().ToString(); 37 | var results = _settings.AtLeast.IsJust() 38 | ? ResultInfoBuilder.GenerateRandom(originator, 39 | _settings.AtLeast.FromJust(@default: 1), _settings.Samples) 40 | : ResultInfoBuilder.Generate(originator, _settings.Samples); 41 | if (Runtime.MaximumResults.HasValue) { 42 | results = results.Take((int)Runtime.MaximumResults.Value); 43 | } 44 | for (ushort i = 0; i < results.Count(); i++) { 45 | var result = results.ElementAt(i); 46 | yield return CreateResult(i, result.Url, result.Description); 47 | } 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /tests/PickAll.Specs/Fakes/Marker.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using PickAll; 4 | 5 | public class MarkerSettings 6 | { 7 | public string Stamp; 8 | } 9 | 10 | public class Marker : PostProcessor 11 | { 12 | readonly MarkerSettings _settings; 13 | 14 | public Marker(object settings) : base(settings) 15 | { 16 | _settings = Settings as MarkerSettings; 17 | if (_settings == null) { 18 | throw new NotImplementedException(); 19 | } 20 | } 21 | 22 | public override bool PublishEvents { get { return true; } } 23 | 24 | public override IEnumerable Process(IEnumerable results) 25 | { 26 | foreach (var result in results) { 27 | var marked = new ResultInfo(result.Originator, result.Index, result.Url, 28 | $"{_settings.Stamp}|{result.Description}", null); 29 | yield return marked; 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /tests/PickAll.Specs/Helpers/ResultInfoExtensions.cs: -------------------------------------------------------------------------------- 1 | using PickAll; 2 | 3 | static class ResultInfoExtensions 4 | { 5 | public static ResultInfo CloneWithIndex(this ResultInfo result, ushort index) 6 | { 7 | return new ResultInfo( 8 | result.Originator, 9 | index, 10 | result.Url, 11 | result.Description, 12 | result.Data); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /tests/PickAll.Specs/Helpers/ResultInfoHelper.cs: -------------------------------------------------------------------------------- 1 | using PickAll; 2 | using Bogus.DataSets; 3 | 4 | static class ResultInfoHelper 5 | { 6 | static readonly Internet _internet = new Internet(); 7 | 8 | public static ResultInfo OnlyDescription(string text) 9 | { 10 | return new ResultInfo( 11 | originator: "helper", 12 | index: 0, 13 | url: _internet.UrlWithPath(fileExt: "php"), 14 | description: text, 15 | data: null); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/PickAll.Specs/Outcomes/FetchedDocumentExtensionsSpecs.cs: -------------------------------------------------------------------------------- 1 | using System.Text; 2 | using System.Linq; 3 | using Xunit; 4 | using FluentAssertions; 5 | using PickAll; 6 | 7 | public class FetchedDocumentExtensionsSpecs 8 | { 9 | [Fact] 10 | public void Should_extract_single_tag() 11 | { 12 | var body = WaffleBuilder.GenerateParagraph(); 13 | var html = WaffleBuilder.GeneratePageAsString(body); 14 | 15 | var sut = new FetchedDocument(Encoding.UTF8.GetBytes(html)); 16 | 17 | var content = sut.ElementSelector("body"); 18 | 19 | content.Trim().Should().NotBeNull() 20 | .And.Be(body.Trim()); 21 | } 22 | 23 | [Fact] 24 | public void Should_extract_multiple_tags() 25 | { 26 | var body = WaffleBuilder.GenerateParagraph(samples: 3, small: true); 27 | var html = WaffleBuilder.GeneratePageAsString(body); 28 | 29 | var sut = new FetchedDocument(Encoding.UTF8.GetBytes(html)); 30 | 31 | var contents = sut.ElementSelectorAll("p"); 32 | 33 | contents.Should().NotBeNullOrEmpty() 34 | .And.HaveCount(3); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /tests/PickAll.Specs/Outcomes/FetchedDocumentSpecs.cs: -------------------------------------------------------------------------------- 1 | using System.Text; 2 | using Xunit; 3 | using FluentAssertions; 4 | using PickAll; 5 | 6 | public class FetchedDocumentSpecs 7 | { 8 | [Fact] 9 | public void An_empty_FetchedDocument_should_be_equal_to_Empty_value() 10 | { 11 | var sut = new FetchedDocument(new byte[] {}); 12 | 13 | sut.Equals(FetchedDocument.Empty).Should().BeTrue(); 14 | } 15 | 16 | [Fact] 17 | public void A_FetchedDocument_should_be_equal_to_an_identical_one() 18 | { 19 | var sut = new FetchedDocument(Encoding.UTF8.GetBytes("foo")); 20 | var other = new FetchedDocument(Encoding.UTF8.GetBytes("foo")); 21 | 22 | sut.Equals(other).Should().BeTrue(); 23 | } 24 | 25 | [Fact] 26 | public void A_FetchedDocument_should_be_equal_to_a_different_one() 27 | { 28 | var sut = new FetchedDocument(Encoding.UTF8.GetBytes("foo")); 29 | var other = new FetchedDocument(Encoding.UTF8.GetBytes("bar")); 30 | 31 | sut.Equals(other).Should().BeFalse(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /tests/PickAll.Specs/Outcomes/FetchingContextSpecs.cs: -------------------------------------------------------------------------------- 1 | using System.Net.Http; 2 | using Xunit; 3 | using FluentAssertions; 4 | using PickAll; 5 | 6 | public class FetchingContextSpecs 7 | { 8 | [Fact] 9 | public async void Should_fetch_a_document() 10 | { 11 | var sut = new FetchingContext(new HttpClient()); 12 | 13 | var document = await sut.FetchAsync("https://google.com/"); 14 | 15 | document.Length.Should().BeGreaterThan(0); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/PickAll.Specs/Outcomes/FuzzyMatchSpecs.cs: -------------------------------------------------------------------------------- 1 | using Xunit; 2 | using FluentAssertions; 3 | using SharpX.Extensions; 4 | using PickAll; 5 | 6 | public class FuzzyMatchSpecs 7 | { 8 | [Fact] 9 | public void Matching_text_with_minimum_distance_of_zero_excludes_other_results() 10 | { 11 | var results = ResultInfoBuilder.GenerateUnique("tests", 10); 12 | var expected = results.Choice(); 13 | 14 | var sut = new FuzzyMatch(new FuzzyMatchSettings { Text = expected.Description }); 15 | var processed = sut.Process(results); 16 | 17 | processed.Should().NotBeNullOrEmpty() 18 | .And.ContainSingle() 19 | .And.ContainEquivalentOf(expected); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /tests/PickAll.Specs/Outcomes/ImproveSpecs.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using Xunit; 3 | using FluentAssertions; 4 | using SharpX.Extensions; 5 | using PickAll; 6 | 7 | public class ImproveSpecs 8 | { 9 | [Fact] 10 | public async void Should_exclude_non_alphanumeric_words() 11 | { 12 | var context = new SearchContext(); 13 | await context.SearchAsync("query"); 14 | 15 | var titles = WaffleBuilder.GenerateTitle(3); 16 | 17 | var sut = new Improve(new ImproveSettings 18 | { 19 | WordCount = (ushort)titles.FlattenOnce().Count() 20 | }); 21 | sut.Context = context; 22 | 23 | var first = titles.First() 24 | .ApplyAt(titles.First().ChoiceOfIndex(), word => word.Mangle()) 25 | .ApplyAt(titles.First().ChoiceOfIndex(word => word.IsAlphanumeric()), word => word.Mangle()); 26 | var second = titles.ElementAt(1) 27 | .ApplyAt(titles.ElementAt(1).ChoiceOfIndex(), word => word.Mangle()); 28 | 29 | var results = new ResultInfo[] { 30 | ResultInfoHelper.OnlyDescription(first), 31 | ResultInfoHelper.OnlyDescription(second), 32 | ResultInfoHelper.OnlyDescription(titles.ElementAt(2)) 33 | }; 34 | 35 | sut.FoldDescriptions(results).Should().NotBeNullOrEmpty() 36 | .And.OnlyContain(word => word.IsAlphanumeric()); 37 | } 38 | 39 | [Fact] 40 | public async void Should_fold_descriptions_excluding_query() 41 | { 42 | var context = new SearchContext() 43 | .With(new ImproveSettings 44 | { 45 | WordCount = 2 46 | }); 47 | await context.SearchAsync("massive repetition"); 48 | 49 | var sut = (Improve)context.Services.First(); 50 | sut.Context = context; 51 | 52 | var titles = WaffleBuilder.GenerateTitle(3, title => title 53 | .Intersperse("massive".Replicate(50, separator: " ")) 54 | .Intersperse("something".Replicate(25, separator: " ")) 55 | .Intersperse("repetition".Replicate(50, separator: " ")) 56 | .Intersperse("hello".Replicate(25, separator: " "))); 57 | 58 | var results = new ResultInfo[] { 59 | ResultInfoHelper.OnlyDescription(titles.First()), 60 | ResultInfoHelper.OnlyDescription(titles.ElementAt(1)), 61 | ResultInfoHelper.OnlyDescription(titles.ElementAt(2)) 62 | }; 63 | 64 | sut.FoldDescriptions(results).Should().NotBeNullOrEmpty() 65 | .And.HaveCount(2) 66 | .And.BeEquivalentTo("something", "hello"); 67 | } 68 | 69 | [Fact] 70 | public async void Should_fold_descriptions_excluding_query_and_noise() 71 | { 72 | var context = new SearchContext() 73 | .With(new ImproveSettings 74 | { 75 | WordCount = 2, 76 | NoiseLength = 3 77 | }); 78 | await context.SearchAsync("massive repetition"); 79 | 80 | var sut = (Improve)context.Services.First(); 81 | sut.Context = context; 82 | 83 | var titles = WaffleBuilder.GenerateTitle(3, title => title 84 | .Intersperse("massive".Replicate(50, separator: " ")) 85 | .Intersperse("catch".Replicate(25, separator: " ")) 86 | .Intersperse("a".Replicate(30, separator: " ")) 87 | .Intersperse("repetition".Replicate(50, separator: " ")) 88 | .Intersperse("word".Replicate(25, separator: " ")) 89 | .Intersperse("of").Replicate(30, separator: " ")); 90 | 91 | var results = new ResultInfo[] { 92 | ResultInfoHelper.OnlyDescription(titles.First()), 93 | ResultInfoHelper.OnlyDescription(titles.ElementAt(1)), 94 | ResultInfoHelper.OnlyDescription(titles.ElementAt(2)) 95 | }; 96 | 97 | sut.FoldDescriptions(results).Should().NotBeNullOrEmpty() 98 | .And.HaveCount(2) 99 | .And.BeEquivalentTo("catch", "word"); 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /tests/PickAll.Specs/Outcomes/OrderSpecs.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using Xunit; 3 | using FluentAssertions; 4 | using PickAll; 5 | 6 | public class OrderSpecs 7 | { 8 | [Fact] 9 | public void Should_ordered_by_index() 10 | { 11 | var results = new List(); 12 | results.AddRange(ResultInfoBuilder.Generate("Choice1", 3)); 13 | results.AddRange(ResultInfoBuilder.Generate("Choice2", 5)); 14 | var sut = new Order(null); 15 | var processed = sut.Process(results); 16 | 17 | processed.Should().NotBeNullOrEmpty() 18 | .And.SatisfyRespectively( 19 | item => item.Index.Should().Be(0), 20 | item => item.Index.Should().Be(0), 21 | item => item.Index.Should().Be(1), 22 | item => item.Index.Should().Be(1), 23 | item => item.Index.Should().Be(2), 24 | item => item.Index.Should().Be(2), 25 | item => item.Index.Should().Be(3), 26 | item => item.Index.Should().Be(4) 27 | ); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /tests/PickAll.Specs/Outcomes/SearchContextExtensionsSpecs.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using Xunit; 4 | using FluentAssertions; 5 | using PickAll; 6 | 7 | public class SearchContextExtensionsSpecs 8 | { 9 | [Fact] 10 | public void Can_add_service_by_name() 11 | { 12 | var sut = new SearchContext() 13 | .With("DuckDuckGo") 14 | .With("Uniqueness"); 15 | 16 | sut.Services.Should().NotBeNullOrEmpty() 17 | .And.HaveCount(2) 18 | .And.SatisfyRespectively( 19 | item => item.Should().BeOfType(), 20 | item => item.Should().BeOfType()); 21 | } 22 | 23 | [Fact] 24 | public void Can_add_service_by_name_ignoring_case() 25 | { 26 | var sut = new SearchContext() 27 | .With("DUCKDUCKgo") 28 | .With("uniQueness"); 29 | 30 | sut.Services.Should().NotBeNullOrEmpty() 31 | .And.HaveCount(2) 32 | .And.SatisfyRespectively( 33 | item => item.Should().BeOfType(), 34 | item => item.Should().BeOfType()); 35 | } 36 | 37 | [Fact] 38 | public void Can_remove_service_by_name_ignoring_case() 39 | { 40 | var sut = new SearchContext() 41 | .With() 42 | .With() 43 | .Without("DUCKDUCKgo") 44 | .Without("uniQueness"); 45 | 46 | sut.Services.Should().BeEmpty(); 47 | } 48 | 49 | [Fact] 50 | public void Can_add_post_processor_service_with_parameters_by_name() 51 | { 52 | var sut = new SearchContext() 53 | .With("FuzzyMatch", new FuzzyMatchSettings { Text = "nothing", MaximumDistance = 10 }); 54 | 55 | sut.Services.Should().NotBeNullOrEmpty() 56 | .And.ContainSingle() 57 | .And.ContainItemsAssignableTo(); 58 | } 59 | 60 | [Fact] 61 | public void Can_add_service_with_generic_or_non_generic_With_method() 62 | { 63 | var sut = new SearchContext() 64 | .With() 65 | .With("DuckDuckGo") 66 | .With() 67 | .With("Order"); 68 | 69 | sut.Services.Should().NotBeNullOrEmpty() 70 | .And.HaveCount(4) 71 | .And.SatisfyRespectively( 72 | item => item.Should().BeOfType(), 73 | item => item.Should().BeOfType(), 74 | item => item.Should().BeOfType(), 75 | item => item.Should().BeOfType()); 76 | } 77 | 78 | [Fact] 79 | public void Can_remove_service_by_name() 80 | { 81 | var sut = new SearchContext() 82 | .With() 83 | .With() 84 | .Without("Yahoo") 85 | .Without("Order"); 86 | 87 | sut.Services.Should().BeEmpty(); 88 | } 89 | 90 | [Fact] 91 | public void Adding_a_custom_searcher_by_name_throws_NotSupportedException() 92 | { 93 | var sut = new SearchContext(); 94 | 95 | Action action = () => sut.With("ArbitrarySearcher", new ArbitrarySearcherSettings()); 96 | 97 | action.Should().ThrowExactly() 98 | .WithMessage("ArbitrarySearcher service not found"); 99 | } 100 | 101 | [Fact] 102 | public void Adding_a_custom_post_processor_by_name_throws_NotSupportedException() 103 | { 104 | var sut = new SearchContext(); 105 | 106 | Action action = () => sut.With("Marker", new MarkerSettings()); 107 | 108 | action.Should().ThrowExactly() 109 | .WithMessage("Marker service not found"); 110 | } 111 | 112 | [Fact] 113 | public void Without_removes_only_first_added_service_of_a_given_type() 114 | { 115 | var sut = new SearchContext() 116 | .With(new ArbitrarySearcherSettings { Samples = 5 }) 117 | .With() 118 | .With(new ArbitrarySearcherSettings { Samples = 3 }) 119 | .With() 120 | .Without(); 121 | 122 | sut.Services.Should().NotBeNullOrEmpty() 123 | .And.HaveCount(3) 124 | .And.SatisfyRespectively( 125 | item => item.Should().BeOfType(), 126 | item => item.Should().BeOfType(), 127 | item => item.Should().BeOfType()); 128 | } 129 | 130 | [Fact] 131 | public async void A_cloned_SearchContext_retains_services_and_maximum_results() 132 | { 133 | var context = new SearchContext(maximumResults: 5) 134 | .With(new ArbitrarySearcherSettings { Samples = 1 }) 135 | .With(new ArbitrarySearcherSettings { Samples = 2 }) 136 | .With(); 137 | await context.SearchAsync("query"); 138 | 139 | var sut = context.Clone(); 140 | 141 | sut.Settings.MaximumResults.Should().NotBeNull() 142 | .And.HaveValue() 143 | .And.Be(5); 144 | sut.Services.Should().NotBeNullOrEmpty() 145 | .And.HaveCount(context.Services.Count()) 146 | .And.BeEquivalentTo(context.Services); 147 | } 148 | 149 | [Fact] 150 | public async void Services_of_cloned_SearchContext_are_unbound_to_original_context() 151 | { 152 | var context = new SearchContext() 153 | .With(new ArbitrarySearcherSettings { Samples = 1 }) 154 | .With(new ArbitrarySearcherSettings { Samples = 2 }) 155 | .With(); 156 | await context.SearchAsync("query"); 157 | 158 | var sut = context.Clone(); 159 | 160 | sut.Services.Cast().Should().NotBeNullOrEmpty() 161 | .And.HaveCount(context.Services.Count()) 162 | .And.OnlyContain(service => service.Context == null); 163 | } 164 | 165 | [Fact] 166 | public void Can_exclude_all_services_by_category() 167 | { 168 | var context = new SearchContext( 169 | typeof(Google), 170 | typeof(Yahoo), 171 | typeof(Order), 172 | typeof(Uniqueness)); 173 | 174 | var sut = context.WithoutAll(); 175 | 176 | sut.Services.Should().NotBeNullOrEmpty() 177 | .And.HaveCount(2) 178 | .And.SatisfyRespectively( 179 | item => item.Should().BeOfType(), 180 | item => item.Should().BeOfType() 181 | ); 182 | 183 | sut = sut.WithoutAll(); 184 | 185 | sut.Services.Should().BeEmpty(); 186 | } 187 | 188 | [Fact] 189 | public void Should_merge_configuration_settings() 190 | { 191 | var expected = new ContextSettings 192 | { EnableRaisingEvents = true, 193 | MaximumResults = 10 }; 194 | 195 | var sut = SearchContext.Default 196 | .WithConfiguration( 197 | new ContextSettings { EnableRaisingEvents = true }, merge: false) 198 | .WithConfiguration( 199 | new ContextSettings { MaximumResults = 10 }, merge: true); 200 | 201 | sut.Settings.Should().Be(expected); 202 | } 203 | } 204 | -------------------------------------------------------------------------------- /tests/PickAll.Specs/Outcomes/SearchContextSpecs.Events.cs: -------------------------------------------------------------------------------- 1 | using Xunit; 2 | using FluentAssertions; 3 | using PickAll; 4 | 5 | public partial class SearchContextSpecs 6 | { 7 | public class Events 8 | { 9 | [Fact] 10 | public async void Should_fire_SearchStart_event() 11 | { 12 | var evidence = string.Empty; 13 | var sut = new SearchContext(new ContextSettings { EnableRaisingEvents = true }) 14 | .With(new ArbitrarySearcherSettings { Samples = 10 }); 15 | sut.SearchBegin += (sender, e) => evidence = e.Query; 16 | 17 | await sut.SearchAsync("query"); 18 | 19 | evidence.Should().Be("query"); 20 | } 21 | 22 | [Fact] 23 | public async void Should_fire_SearchEnd_event() 24 | { 25 | var evidence = false; 26 | var sut = new SearchContext(new ContextSettings { EnableRaisingEvents = true }) 27 | .With(new ArbitrarySearcherSettings { Samples = 10 }); 28 | sut.SearchEnd += (sender, e) => evidence = true; 29 | 30 | await sut.SearchAsync("query"); 31 | 32 | evidence.Should().BeTrue(); 33 | } 34 | 35 | [Fact] 36 | public async void Should_fire_ServiceLoad_event() 37 | { 38 | var evidence = false; 39 | var sut = new SearchContext(new ContextSettings { EnableRaisingEvents = true }) 40 | .With(new ArbitrarySearcherSettings { Samples = 10 }); 41 | sut.ServiceLoad += (sender, e) => evidence = true; 42 | 43 | await sut.SearchAsync("query"); 44 | 45 | evidence.Should().BeTrue(); 46 | } 47 | 48 | [Fact] 49 | public async void Should_fire_ResultCreated_event() 50 | { 51 | var evidence = 0; 52 | var sut = new SearchContext(new ContextSettings { EnableRaisingEvents = true }) 53 | .With(new ArbitrarySearcherSettings { Samples = 10 }); 54 | sut.ResultCreated += (sender, e) => evidence++; 55 | 56 | await sut.SearchAsync("query"); 57 | 58 | evidence.Should().Be(10); 59 | } 60 | 61 | [Fact] 62 | public async void Should_fire_ResultProcessed_event() 63 | { 64 | var evidence = 0; 65 | void sut_ResultProcessed(object sender, ResultHandledEventArgs e) { evidence++; } 66 | 67 | var sut = new SearchContext(new ContextSettings { EnableRaisingEvents = true }) 68 | .With(new ArbitrarySearcherSettings { Samples = 10 }) 69 | .With(new MarkerSettings { Stamp = string.Empty }); 70 | sut.ResultProcessed += sut_ResultProcessed; 71 | 72 | await sut.SearchAsync("query"); 73 | 74 | evidence.Should().Be(10); 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /tests/PickAll.Specs/Outcomes/SearchContextSpecs.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Linq; 3 | using Xunit; 4 | using FluentAssertions; 5 | using SharpX; 6 | using PickAll; 7 | 8 | public partial class SearchContextSpecs 9 | { 10 | [Fact] 11 | public void Can_initialize_SearchContext_using_types() 12 | { 13 | var sut = new SearchContext( 14 | typeof(Google), 15 | typeof(Yahoo), 16 | typeof(Uniqueness), 17 | typeof(Order)); 18 | 19 | sut.Services.Should().NotBeNullOrEmpty() 20 | .And.HaveCount(4) 21 | .And.SatisfyRespectively( 22 | item => item.Should().BeOfType(), 23 | item => item.Should().BeOfType(), 24 | item => item.Should().BeOfType(), 25 | item => item.Should().BeOfType()); 26 | } 27 | 28 | [Fact] 29 | public void Initializing_SearchContext_with_wrong_types_throws_NotSupportedException() 30 | { 31 | Action action = () => new SearchContext( 32 | typeof(Google), 33 | typeof(string), 34 | typeof(Uniqueness), 35 | typeof(int)); 36 | 37 | action.Should().ThrowExactly() 38 | .WithMessage("All services must inherit from T."); 39 | } 40 | 41 | [Fact] 42 | public async void When_none_searcher_is_set_Search_returns_an_empty_collection() 43 | { 44 | var sut = new SearchContext(); 45 | 46 | var results = await sut.SearchAsync("query"); 47 | 48 | results.Should().BeEmpty(); 49 | } 50 | 51 | [Fact] 52 | public async void When_two_searchers_are_set_Search_returns_a_merged_collection() 53 | { 54 | var sut = new SearchContext() 55 | .With(new ArbitrarySearcherSettings { Samples = 8 }) 56 | .With(new ArbitrarySearcherSettings { Samples = 12 }); 57 | var results = await sut.SearchAsync("query"); 58 | 59 | results.Should().NotBeNullOrEmpty() 60 | .And.HaveCount(20); 61 | } 62 | 63 | [Fact] 64 | public async void Search_invokes_services_by_addition_order() 65 | { 66 | var sut = new SearchContext() 67 | .With( 68 | new MarkerSettings { Stamp = "stamp0" }) 69 | .With(new ArbitrarySearcherSettings { Samples = 5 }) 70 | .With(new MarkerSettings { Stamp = "stamp1" }) 71 | .With(new MarkerSettings { Stamp = "stamp2" }); 72 | var results = await sut.SearchAsync("search"); 73 | 74 | results.First().Description.Should().StartWith("stamp2|stamp1|"); 75 | } 76 | 77 | [Fact] 78 | public async void Removed_searcher_does_not_produce_results() 79 | { 80 | var sut = new SearchContext() 81 | .With(new ArbitrarySearcherSettings { Samples = 8 }) 82 | .With(new ArbitrarySearcherSettings { Samples = 10 }) 83 | .Without(); 84 | var results = await sut.SearchAsync("search"); 85 | 86 | results.Should().NotBeNullOrEmpty() 87 | .And.HaveCount(10); 88 | } 89 | 90 | [Fact] 91 | public async void Removed_post_processor_does_not_take_effect() 92 | { 93 | var sut = new SearchContext() 94 | .With(new ArbitrarySearcherSettings { Samples = 5 }) 95 | .With(new MarkerSettings { Stamp = "stamp" }) 96 | .Without(); 97 | var results = await sut.SearchAsync("query"); 98 | 99 | results.Should().NotBeNullOrEmpty() 100 | .And.OnlyContain(x => !x.Description.StartsWith("stamp")); 101 | } 102 | 103 | [Fact] 104 | public async void Context_is_set_in_services() 105 | { 106 | var sut = new SearchContext() 107 | .With(new ArbitrarySearcherSettings { Samples = 1 }) 108 | .With(); 109 | 110 | await sut.SearchAsync("query"); 111 | 112 | sut.Services.Should().NotBeNullOrEmpty() 113 | .And.HaveCount(2) 114 | .And.SatisfyRespectively( 115 | item => ((Searcher)item).Context.Should().NotBeNull(), 116 | item => ((PostProcessor)item).Context.Should().NotBeNull()); 117 | } 118 | 119 | [Fact] 120 | public async void Should_limit_results_if_maximumResults_is_set() 121 | { 122 | var sut = new SearchContext(maximumResults: 10) 123 | .With(new ArbitrarySearcherSettings { Samples = 20 }); 124 | 125 | var results = await sut.SearchAsync("query"); 126 | 127 | results.Should().NotBeNullOrEmpty() 128 | .And.HaveCount(10); 129 | } 130 | 131 | [Fact] 132 | public async void A_searcher_should_limit_results_directly() 133 | { 134 | var sut = new SearchContext(maximumResults: 10) 135 | .With( 136 | new ArbitrarySearcherSettings { Samples = 20, AtLeast = Maybe.Just(15) }); 137 | sut.EnforceMaximumResults = false; 138 | 139 | var results = await sut.SearchAsync("query"); 140 | 141 | results.Should().NotBeNullOrEmpty() 142 | .And.HaveCount(10); 143 | } 144 | 145 | [Fact] 146 | public async void Maximum_results_should_be_partitioned_per_searcher() 147 | { 148 | var sut = new SearchContext(maximumResults: 10) 149 | .With(new ArbitrarySearcherSettings { Samples = 20 }) 150 | .With(new ArbitrarySearcherSettings { Samples = 20 }) 151 | .With(new ArbitrarySearcherSettings { Samples = 20 }) 152 | .With() 153 | .With(); 154 | sut.EnforceMaximumResults = false; 155 | 156 | await sut.SearchAsync("query"); 157 | 158 | sut.Services.Take(3).Cast().Should() 159 | .SatisfyRespectively( 160 | item => item.Runtime.MaximumResults.Should().Be(4), 161 | item => item.Runtime.MaximumResults.Should().Be(3), 162 | item => item.Runtime.MaximumResults.Should().Be(3)); 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /tests/PickAll.Specs/Outcomes/UniquenessSpecs.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using Xunit; 4 | using FluentAssertions; 5 | using SharpX.Extensions; 6 | using PickAll; 7 | 8 | public class UniquenessSpecs 9 | { 10 | [Fact] 11 | public void Should_exclude_duplicate_urls() 12 | { 13 | var results = new List(); 14 | results.AddRange(ResultInfoBuilder.GenerateUnique("random", 10)); 15 | results.Add(results.Choice().CloneWithIndex(0)); 16 | var sut = new Uniqueness(null); 17 | var processed = sut.Process(results); 18 | 19 | processed.Should().NotBeNullOrEmpty() 20 | .And.HaveCount(results.Count() - 1); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /tests/PickAll.Specs/PickAll.Specs.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | net5.0 8 | 9.0 9 | false 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /tests/PickAll.Specs/paket.references: -------------------------------------------------------------------------------- 1 | group specs 2 | Bogus 3 | FluentAssertions 4 | Microsoft.NET.Test.Sdk 5 | WaffleGenerator 6 | WaffleGenerator.Bogus 7 | xunit 8 | xunit.runner.visualstudio 9 | coverlet.collector 10 | SharpX 11 | --------------------------------------------------------------------------------