├── .gitignore ├── .scalafmt.conf ├── .travis.yml ├── LICENSE ├── README.md ├── modules ├── scraper │ └── src │ │ ├── main │ │ └── scala │ │ │ └── com │ │ │ └── marekkadek │ │ │ └── scraper │ │ │ ├── browser.scala │ │ │ ├── dom.scala │ │ │ ├── extractors.scala │ │ │ ├── htmlunit │ │ │ ├── browser.scala │ │ │ └── dom.scala │ │ │ ├── jsoup │ │ │ ├── browser.scala │ │ │ └── dom.scala │ │ │ ├── package.scala │ │ │ └── proxy.scala │ │ └── test │ │ ├── resources │ │ └── building_materials.htm │ │ └── scala │ │ └── com │ │ └── marekkadek │ │ └── scraper │ │ ├── JsoupBrowserSpec.scala │ │ ├── ScraperTest.scala │ │ └── Utils.scala └── scrawler │ └── src │ ├── main │ └── scala │ │ └── com │ │ └── marekkadek │ │ └── scrawler │ │ └── crawlers │ │ └── crawlers.scala │ └── test │ ├── resources │ └── building_materials.htm │ └── scala │ └── com │ └── marekkadek │ └── scrawler │ ├── ScrawlerTest.scala │ ├── SomeTest.scala │ ├── Utils.scala │ ├── crawlers │ ├── BrowserAgnostic.scala │ ├── CompareCrawling.scala │ ├── HttpsLinksInfiniteCrawler.scala │ ├── Instances.scala │ ├── JsoupProxyPickingTest.scala │ ├── ParallelCrawlingSpec.scala │ └── SequentialCrawlingSpec.scala │ └── opencrawling.scala ├── project ├── build.properties ├── pgp.sbt ├── plugins.sbt ├── release.sbt ├── scalafmt.sbt ├── scoverage.sbt └── wartremover.sbt └── version.sbt /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/.gitignore -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/.scalafmt.conf -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/.travis.yml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/README.md -------------------------------------------------------------------------------- /modules/scraper/src/main/scala/com/marekkadek/scraper/browser.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scraper/src/main/scala/com/marekkadek/scraper/browser.scala -------------------------------------------------------------------------------- /modules/scraper/src/main/scala/com/marekkadek/scraper/dom.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scraper/src/main/scala/com/marekkadek/scraper/dom.scala -------------------------------------------------------------------------------- /modules/scraper/src/main/scala/com/marekkadek/scraper/extractors.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scraper/src/main/scala/com/marekkadek/scraper/extractors.scala -------------------------------------------------------------------------------- /modules/scraper/src/main/scala/com/marekkadek/scraper/htmlunit/browser.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scraper/src/main/scala/com/marekkadek/scraper/htmlunit/browser.scala -------------------------------------------------------------------------------- /modules/scraper/src/main/scala/com/marekkadek/scraper/htmlunit/dom.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scraper/src/main/scala/com/marekkadek/scraper/htmlunit/dom.scala -------------------------------------------------------------------------------- /modules/scraper/src/main/scala/com/marekkadek/scraper/jsoup/browser.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scraper/src/main/scala/com/marekkadek/scraper/jsoup/browser.scala -------------------------------------------------------------------------------- /modules/scraper/src/main/scala/com/marekkadek/scraper/jsoup/dom.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scraper/src/main/scala/com/marekkadek/scraper/jsoup/dom.scala -------------------------------------------------------------------------------- /modules/scraper/src/main/scala/com/marekkadek/scraper/package.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scraper/src/main/scala/com/marekkadek/scraper/package.scala -------------------------------------------------------------------------------- /modules/scraper/src/main/scala/com/marekkadek/scraper/proxy.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scraper/src/main/scala/com/marekkadek/scraper/proxy.scala -------------------------------------------------------------------------------- /modules/scraper/src/test/resources/building_materials.htm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scraper/src/test/resources/building_materials.htm -------------------------------------------------------------------------------- /modules/scraper/src/test/scala/com/marekkadek/scraper/JsoupBrowserSpec.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scraper/src/test/scala/com/marekkadek/scraper/JsoupBrowserSpec.scala -------------------------------------------------------------------------------- /modules/scraper/src/test/scala/com/marekkadek/scraper/ScraperTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scraper/src/test/scala/com/marekkadek/scraper/ScraperTest.scala -------------------------------------------------------------------------------- /modules/scraper/src/test/scala/com/marekkadek/scraper/Utils.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scraper/src/test/scala/com/marekkadek/scraper/Utils.scala -------------------------------------------------------------------------------- /modules/scrawler/src/main/scala/com/marekkadek/scrawler/crawlers/crawlers.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scrawler/src/main/scala/com/marekkadek/scrawler/crawlers/crawlers.scala -------------------------------------------------------------------------------- /modules/scrawler/src/test/resources/building_materials.htm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scrawler/src/test/resources/building_materials.htm -------------------------------------------------------------------------------- /modules/scrawler/src/test/scala/com/marekkadek/scrawler/ScrawlerTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scrawler/src/test/scala/com/marekkadek/scrawler/ScrawlerTest.scala -------------------------------------------------------------------------------- /modules/scrawler/src/test/scala/com/marekkadek/scrawler/SomeTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scrawler/src/test/scala/com/marekkadek/scrawler/SomeTest.scala -------------------------------------------------------------------------------- /modules/scrawler/src/test/scala/com/marekkadek/scrawler/Utils.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scrawler/src/test/scala/com/marekkadek/scrawler/Utils.scala -------------------------------------------------------------------------------- /modules/scrawler/src/test/scala/com/marekkadek/scrawler/crawlers/BrowserAgnostic.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scrawler/src/test/scala/com/marekkadek/scrawler/crawlers/BrowserAgnostic.scala -------------------------------------------------------------------------------- /modules/scrawler/src/test/scala/com/marekkadek/scrawler/crawlers/CompareCrawling.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scrawler/src/test/scala/com/marekkadek/scrawler/crawlers/CompareCrawling.scala -------------------------------------------------------------------------------- /modules/scrawler/src/test/scala/com/marekkadek/scrawler/crawlers/HttpsLinksInfiniteCrawler.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scrawler/src/test/scala/com/marekkadek/scrawler/crawlers/HttpsLinksInfiniteCrawler.scala -------------------------------------------------------------------------------- /modules/scrawler/src/test/scala/com/marekkadek/scrawler/crawlers/Instances.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scrawler/src/test/scala/com/marekkadek/scrawler/crawlers/Instances.scala -------------------------------------------------------------------------------- /modules/scrawler/src/test/scala/com/marekkadek/scrawler/crawlers/JsoupProxyPickingTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scrawler/src/test/scala/com/marekkadek/scrawler/crawlers/JsoupProxyPickingTest.scala -------------------------------------------------------------------------------- /modules/scrawler/src/test/scala/com/marekkadek/scrawler/crawlers/ParallelCrawlingSpec.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scrawler/src/test/scala/com/marekkadek/scrawler/crawlers/ParallelCrawlingSpec.scala -------------------------------------------------------------------------------- /modules/scrawler/src/test/scala/com/marekkadek/scrawler/crawlers/SequentialCrawlingSpec.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scrawler/src/test/scala/com/marekkadek/scrawler/crawlers/SequentialCrawlingSpec.scala -------------------------------------------------------------------------------- /modules/scrawler/src/test/scala/com/marekkadek/scrawler/opencrawling.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/modules/scrawler/src/test/scala/com/marekkadek/scrawler/opencrawling.scala -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 0.13.13 2 | -------------------------------------------------------------------------------- /project/pgp.sbt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/project/pgp.sbt -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/project/plugins.sbt -------------------------------------------------------------------------------- /project/release.sbt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/project/release.sbt -------------------------------------------------------------------------------- /project/scalafmt.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.geirsson" %% "sbt-scalafmt" % "0.6.8") 2 | -------------------------------------------------------------------------------- /project/scoverage.sbt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/project/scoverage.sbt -------------------------------------------------------------------------------- /project/wartremover.sbt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KadekM/scrawler/HEAD/project/wartremover.sbt -------------------------------------------------------------------------------- /version.sbt: -------------------------------------------------------------------------------- 1 | version in ThisBuild := "0.0.5-SNAPSHOT" 2 | --------------------------------------------------------------------------------