├── .codecov.yml ├── .editorconfig ├── .github ├── FUNDING.yml ├── dependabot.yml ├── release-drafter.yml └── workflows │ ├── build.yml │ └── release-drafter.yml ├── .gitignore ├── CodeCoverage.runsettings ├── InfinityCrawler.sln ├── License.txt ├── README.md ├── images └── icon.png ├── src ├── Directory.Build.props └── InfinityCrawler │ ├── CrawlLink.cs │ ├── CrawlResult.cs │ ├── CrawlSettings.cs │ ├── CrawledUri.cs │ ├── Crawler.cs │ ├── InfinityCrawler.csproj │ ├── Internal │ ├── CrawlRunner.cs │ └── UriExtensions.cs │ ├── Processing │ ├── Content │ │ ├── CrawlHeaders.cs │ │ ├── DefaultContentProcessor.cs │ │ └── IContentProcessor.cs │ └── Requests │ │ ├── DefaultRequestProcessor.cs │ │ ├── IRequestProcessor.cs │ │ ├── RequestContext.cs │ │ ├── RequestProcessorOptions.cs │ │ └── RequestResult.cs │ └── UriCrawlState.cs └── tests ├── InfinityCrawler.Tests.Benchmarks ├── BasicSiteCrawlBenchmark.cs ├── InfinityCrawler.Tests.Benchmarks.csproj └── Program.cs ├── InfinityCrawler.Tests.TestSite ├── Controllers │ ├── HelperController.cs │ └── RobotsController.cs ├── InfinityCrawler.Tests.TestSite.csproj ├── Resources │ ├── BasicSite │ │ ├── basic-page.html │ │ ├── index.html │ │ ├── looping-links.html │ │ ├── robots-blocked-child.html │ │ ├── robots-blocked.html │ │ └── robots.txt │ ├── DefaultContentProcessor │ │ ├── AbsoluteCanonicalUri.html │ │ ├── BaseHrefCrawlLink.html │ │ ├── CrawlLinkContent.html │ │ ├── MetaNoFollow.html │ │ ├── MetaNoIndex.html │ │ ├── MetaNoIndexNoFollow.html │ │ ├── MetaNone.html │ │ ├── NoCanonicalUri.html │ │ └── RelativeCanonicalUri.html │ ├── DefaultRequestProcessor │ │ └── index.html │ └── EmptySite │ │ └── readme.txt ├── SiteContext.cs ├── Startup.cs ├── TestHttpMessageHandler.cs └── TestSiteManager.cs └── InfinityCrawler.Tests ├── BasicSiteTests.cs ├── ContentProcessorTestBase.cs ├── CrawlerTestBase.cs ├── DefaultContentProcessorTests.cs ├── DefaultRequestProcessorTests.cs ├── ExpectedExceptionPatternAttribute.cs ├── InfinityCrawler.Tests.csproj ├── TestBase.cs └── TestSiteConfiguration.cs /.codecov.yml: -------------------------------------------------------------------------------- 1 | comment: off -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/.editorconfig -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: Turnerj -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/.github/dependabot.yml -------------------------------------------------------------------------------- /.github/release-drafter.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/.github/release-drafter.yml -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/.github/workflows/build.yml -------------------------------------------------------------------------------- /.github/workflows/release-drafter.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/.github/workflows/release-drafter.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/.gitignore -------------------------------------------------------------------------------- /CodeCoverage.runsettings: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/CodeCoverage.runsettings -------------------------------------------------------------------------------- /InfinityCrawler.sln: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/InfinityCrawler.sln -------------------------------------------------------------------------------- /License.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/License.txt -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/README.md -------------------------------------------------------------------------------- /images/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/images/icon.png -------------------------------------------------------------------------------- /src/Directory.Build.props: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/src/Directory.Build.props -------------------------------------------------------------------------------- /src/InfinityCrawler/CrawlLink.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/src/InfinityCrawler/CrawlLink.cs -------------------------------------------------------------------------------- /src/InfinityCrawler/CrawlResult.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/src/InfinityCrawler/CrawlResult.cs -------------------------------------------------------------------------------- /src/InfinityCrawler/CrawlSettings.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/src/InfinityCrawler/CrawlSettings.cs -------------------------------------------------------------------------------- /src/InfinityCrawler/CrawledUri.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/src/InfinityCrawler/CrawledUri.cs -------------------------------------------------------------------------------- /src/InfinityCrawler/Crawler.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/src/InfinityCrawler/Crawler.cs -------------------------------------------------------------------------------- /src/InfinityCrawler/InfinityCrawler.csproj: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/src/InfinityCrawler/InfinityCrawler.csproj -------------------------------------------------------------------------------- /src/InfinityCrawler/Internal/CrawlRunner.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/src/InfinityCrawler/Internal/CrawlRunner.cs -------------------------------------------------------------------------------- /src/InfinityCrawler/Internal/UriExtensions.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/src/InfinityCrawler/Internal/UriExtensions.cs -------------------------------------------------------------------------------- /src/InfinityCrawler/Processing/Content/CrawlHeaders.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/src/InfinityCrawler/Processing/Content/CrawlHeaders.cs -------------------------------------------------------------------------------- /src/InfinityCrawler/Processing/Content/DefaultContentProcessor.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/src/InfinityCrawler/Processing/Content/DefaultContentProcessor.cs -------------------------------------------------------------------------------- /src/InfinityCrawler/Processing/Content/IContentProcessor.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/src/InfinityCrawler/Processing/Content/IContentProcessor.cs -------------------------------------------------------------------------------- /src/InfinityCrawler/Processing/Requests/DefaultRequestProcessor.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/src/InfinityCrawler/Processing/Requests/DefaultRequestProcessor.cs -------------------------------------------------------------------------------- /src/InfinityCrawler/Processing/Requests/IRequestProcessor.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/src/InfinityCrawler/Processing/Requests/IRequestProcessor.cs -------------------------------------------------------------------------------- /src/InfinityCrawler/Processing/Requests/RequestContext.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/src/InfinityCrawler/Processing/Requests/RequestContext.cs -------------------------------------------------------------------------------- /src/InfinityCrawler/Processing/Requests/RequestProcessorOptions.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/src/InfinityCrawler/Processing/Requests/RequestProcessorOptions.cs -------------------------------------------------------------------------------- /src/InfinityCrawler/Processing/Requests/RequestResult.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/src/InfinityCrawler/Processing/Requests/RequestResult.cs -------------------------------------------------------------------------------- /src/InfinityCrawler/UriCrawlState.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/src/InfinityCrawler/UriCrawlState.cs -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.Benchmarks/BasicSiteCrawlBenchmark.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.Benchmarks/BasicSiteCrawlBenchmark.cs -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.Benchmarks/InfinityCrawler.Tests.Benchmarks.csproj: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.Benchmarks/InfinityCrawler.Tests.Benchmarks.csproj -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.Benchmarks/Program.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.Benchmarks/Program.cs -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/Controllers/HelperController.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/Controllers/HelperController.cs -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/Controllers/RobotsController.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/Controllers/RobotsController.cs -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/InfinityCrawler.Tests.TestSite.csproj: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/InfinityCrawler.Tests.TestSite.csproj -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/Resources/BasicSite/basic-page.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/Resources/BasicSite/basic-page.html -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/Resources/BasicSite/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/Resources/BasicSite/index.html -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/Resources/BasicSite/looping-links.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/Resources/BasicSite/looping-links.html -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/Resources/BasicSite/robots-blocked-child.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/Resources/BasicSite/robots-blocked-child.html -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/Resources/BasicSite/robots-blocked.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/Resources/BasicSite/robots-blocked.html -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/Resources/BasicSite/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Disallow: /robots-blocked.html -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/Resources/DefaultContentProcessor/AbsoluteCanonicalUri.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/Resources/DefaultContentProcessor/AbsoluteCanonicalUri.html -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/Resources/DefaultContentProcessor/BaseHrefCrawlLink.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/Resources/DefaultContentProcessor/BaseHrefCrawlLink.html -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/Resources/DefaultContentProcessor/CrawlLinkContent.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/Resources/DefaultContentProcessor/CrawlLinkContent.html -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/Resources/DefaultContentProcessor/MetaNoFollow.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/Resources/DefaultContentProcessor/MetaNoFollow.html -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/Resources/DefaultContentProcessor/MetaNoIndex.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/Resources/DefaultContentProcessor/MetaNoIndex.html -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/Resources/DefaultContentProcessor/MetaNoIndexNoFollow.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/Resources/DefaultContentProcessor/MetaNoIndexNoFollow.html -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/Resources/DefaultContentProcessor/MetaNone.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/Resources/DefaultContentProcessor/MetaNone.html -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/Resources/DefaultContentProcessor/NoCanonicalUri.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/Resources/DefaultContentProcessor/NoCanonicalUri.html -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/Resources/DefaultContentProcessor/RelativeCanonicalUri.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/Resources/DefaultContentProcessor/RelativeCanonicalUri.html -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/Resources/DefaultRequestProcessor/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/Resources/DefaultRequestProcessor/index.html -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/Resources/EmptySite/readme.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/Resources/EmptySite/readme.txt -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/SiteContext.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/SiteContext.cs -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/Startup.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/Startup.cs -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/TestHttpMessageHandler.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/TestHttpMessageHandler.cs -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests.TestSite/TestSiteManager.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests.TestSite/TestSiteManager.cs -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests/BasicSiteTests.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests/BasicSiteTests.cs -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests/ContentProcessorTestBase.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests/ContentProcessorTestBase.cs -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests/CrawlerTestBase.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests/CrawlerTestBase.cs -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests/DefaultContentProcessorTests.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests/DefaultContentProcessorTests.cs -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests/DefaultRequestProcessorTests.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests/DefaultRequestProcessorTests.cs -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests/ExpectedExceptionPatternAttribute.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests/ExpectedExceptionPatternAttribute.cs -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests/InfinityCrawler.Tests.csproj: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests/InfinityCrawler.Tests.csproj -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests/TestBase.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests/TestBase.cs -------------------------------------------------------------------------------- /tests/InfinityCrawler.Tests/TestSiteConfiguration.cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TurnerSoftware/InfinityCrawler/HEAD/tests/InfinityCrawler.Tests/TestSiteConfiguration.cs --------------------------------------------------------------------------------