├── .travis.yml ├── LICENSE ├── README.md ├── example └── src │ └── main │ └── kotlin │ ├── SimpleExample.kt │ └── main.kt ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── settings.gradle └── src ├── main ├── kotlin │ └── io │ │ └── thelandscape │ │ └── krawler │ │ ├── HSQLConnection.kt │ │ ├── crawler │ │ ├── History │ │ │ ├── Dao.kt │ │ │ ├── KrawlHistory.kt │ │ │ └── Model.kt │ │ ├── KrawlConfig.kt │ │ ├── KrawlQueue │ │ │ ├── Dao.kt │ │ │ ├── Model.kt │ │ │ ├── Queue.kt │ │ │ └── ScheduledQueue.kt │ │ ├── Krawler.kt │ │ └── NoopTaskRejector.kt │ │ ├── http │ │ ├── KrawlDocument.kt │ │ ├── KrawlUrl.kt │ │ ├── RedirectHistoryNode.kt │ │ └── Requests.kt │ │ └── robots │ │ ├── RoboMinder.kt │ │ ├── RobotsConfig.kt │ │ └── RobotsTxt.kt └── resources │ └── log4j2.xml └── test └── kotlin └── io └── thelandscape ├── KrawlDocumentTest.kt ├── KrawlHistoryDaoTest.kt ├── KrawlQueueDaoTest.kt ├── KrawlUrlTest.kt ├── KrawlerTest.kt ├── RequestsTest.kt ├── RoboMinderTest.kt └── RobotsTxtTest.kt /.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/.travis.yml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/README.md -------------------------------------------------------------------------------- /example/src/main/kotlin/SimpleExample.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/example/src/main/kotlin/SimpleExample.kt -------------------------------------------------------------------------------- /example/src/main/kotlin/main.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/example/src/main/kotlin/main.kt -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/gradle/wrapper/gradle-wrapper.properties -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/gradlew -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = "krawler" 2 | 3 | include "example" -------------------------------------------------------------------------------- /src/main/kotlin/io/thelandscape/krawler/HSQLConnection.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/main/kotlin/io/thelandscape/krawler/HSQLConnection.kt -------------------------------------------------------------------------------- /src/main/kotlin/io/thelandscape/krawler/crawler/History/Dao.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/main/kotlin/io/thelandscape/krawler/crawler/History/Dao.kt -------------------------------------------------------------------------------- /src/main/kotlin/io/thelandscape/krawler/crawler/History/KrawlHistory.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/main/kotlin/io/thelandscape/krawler/crawler/History/KrawlHistory.kt -------------------------------------------------------------------------------- /src/main/kotlin/io/thelandscape/krawler/crawler/History/Model.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/main/kotlin/io/thelandscape/krawler/crawler/History/Model.kt -------------------------------------------------------------------------------- /src/main/kotlin/io/thelandscape/krawler/crawler/KrawlConfig.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/main/kotlin/io/thelandscape/krawler/crawler/KrawlConfig.kt -------------------------------------------------------------------------------- /src/main/kotlin/io/thelandscape/krawler/crawler/KrawlQueue/Dao.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/main/kotlin/io/thelandscape/krawler/crawler/KrawlQueue/Dao.kt -------------------------------------------------------------------------------- /src/main/kotlin/io/thelandscape/krawler/crawler/KrawlQueue/Model.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/main/kotlin/io/thelandscape/krawler/crawler/KrawlQueue/Model.kt -------------------------------------------------------------------------------- /src/main/kotlin/io/thelandscape/krawler/crawler/KrawlQueue/Queue.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/main/kotlin/io/thelandscape/krawler/crawler/KrawlQueue/Queue.kt -------------------------------------------------------------------------------- /src/main/kotlin/io/thelandscape/krawler/crawler/KrawlQueue/ScheduledQueue.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/main/kotlin/io/thelandscape/krawler/crawler/KrawlQueue/ScheduledQueue.kt -------------------------------------------------------------------------------- /src/main/kotlin/io/thelandscape/krawler/crawler/Krawler.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/main/kotlin/io/thelandscape/krawler/crawler/Krawler.kt -------------------------------------------------------------------------------- /src/main/kotlin/io/thelandscape/krawler/crawler/NoopTaskRejector.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/main/kotlin/io/thelandscape/krawler/crawler/NoopTaskRejector.kt -------------------------------------------------------------------------------- /src/main/kotlin/io/thelandscape/krawler/http/KrawlDocument.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/main/kotlin/io/thelandscape/krawler/http/KrawlDocument.kt -------------------------------------------------------------------------------- /src/main/kotlin/io/thelandscape/krawler/http/KrawlUrl.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/main/kotlin/io/thelandscape/krawler/http/KrawlUrl.kt -------------------------------------------------------------------------------- /src/main/kotlin/io/thelandscape/krawler/http/RedirectHistoryNode.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/main/kotlin/io/thelandscape/krawler/http/RedirectHistoryNode.kt -------------------------------------------------------------------------------- /src/main/kotlin/io/thelandscape/krawler/http/Requests.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/main/kotlin/io/thelandscape/krawler/http/Requests.kt -------------------------------------------------------------------------------- /src/main/kotlin/io/thelandscape/krawler/robots/RoboMinder.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/main/kotlin/io/thelandscape/krawler/robots/RoboMinder.kt -------------------------------------------------------------------------------- /src/main/kotlin/io/thelandscape/krawler/robots/RobotsConfig.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/main/kotlin/io/thelandscape/krawler/robots/RobotsConfig.kt -------------------------------------------------------------------------------- /src/main/kotlin/io/thelandscape/krawler/robots/RobotsTxt.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/main/kotlin/io/thelandscape/krawler/robots/RobotsTxt.kt -------------------------------------------------------------------------------- /src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/main/resources/log4j2.xml -------------------------------------------------------------------------------- /src/test/kotlin/io/thelandscape/KrawlDocumentTest.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/test/kotlin/io/thelandscape/KrawlDocumentTest.kt -------------------------------------------------------------------------------- /src/test/kotlin/io/thelandscape/KrawlHistoryDaoTest.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/test/kotlin/io/thelandscape/KrawlHistoryDaoTest.kt -------------------------------------------------------------------------------- /src/test/kotlin/io/thelandscape/KrawlQueueDaoTest.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/test/kotlin/io/thelandscape/KrawlQueueDaoTest.kt -------------------------------------------------------------------------------- /src/test/kotlin/io/thelandscape/KrawlUrlTest.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/test/kotlin/io/thelandscape/KrawlUrlTest.kt -------------------------------------------------------------------------------- /src/test/kotlin/io/thelandscape/KrawlerTest.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/test/kotlin/io/thelandscape/KrawlerTest.kt -------------------------------------------------------------------------------- /src/test/kotlin/io/thelandscape/RequestsTest.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/test/kotlin/io/thelandscape/RequestsTest.kt -------------------------------------------------------------------------------- /src/test/kotlin/io/thelandscape/RoboMinderTest.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/test/kotlin/io/thelandscape/RoboMinderTest.kt -------------------------------------------------------------------------------- /src/test/kotlin/io/thelandscape/RobotsTxtTest.kt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brianmadden/krawler/HEAD/src/test/kotlin/io/thelandscape/RobotsTxtTest.kt --------------------------------------------------------------------------------