├── .gitignore ├── .travis.yml ├── Dockerfile ├── example ├── gaocegege.com-spider │ ├── build.sbt │ └── src │ │ └── main │ │ └── scala │ │ └── Main.scala └── github-api-spider │ ├── Dockerfile │ ├── build.sbt │ └── src │ └── main │ └── scala │ └── com │ └── gaocegege │ └── scrala │ └── example │ └── SJTUSpider.scala ├── project └── plugins.sbt ├── readme.md └── src └── main ├── resources ├── application.conf └── logback.xml └── scala └── com └── gaocegege └── scrala ├── core ├── common │ ├── request │ │ ├── Request.scala │ │ └── impl │ │ │ └── HttpRequest.scala │ ├── response │ │ ├── Response.scala │ │ └── impl │ │ │ └── HttpResponse.scala │ └── util │ │ ├── Constant.scala │ │ └── ConvertTool.scala ├── downloader │ ├── Downloader.scala │ ├── httpclient │ │ └── DefaultHttpClient.scala │ └── impl │ │ └── HttpDownloader.scala ├── engine │ ├── Engine.scala │ └── manager │ │ ├── DownloadManager.scala │ │ └── impl │ │ └── DefaultDownloadManager.scala ├── middleware │ └── filter │ │ ├── Filter.scala │ │ └── impl │ │ ├── DefaultFilter.scala │ │ └── UrlDupFilter.scala ├── scheduler │ ├── Scheduler.scala │ └── impl │ │ └── DefaultScheduler.scala └── spider │ ├── Spider.scala │ └── impl │ └── DefaultSpider.scala └── example ├── GaocegegeSpider.scala └── SJTUSpider.scala /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/.gitignore -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/.travis.yml -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/Dockerfile -------------------------------------------------------------------------------- /example/gaocegege.com-spider/build.sbt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/example/gaocegege.com-spider/build.sbt -------------------------------------------------------------------------------- /example/gaocegege.com-spider/src/main/scala/Main.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/example/gaocegege.com-spider/src/main/scala/Main.scala -------------------------------------------------------------------------------- /example/github-api-spider/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/example/github-api-spider/Dockerfile -------------------------------------------------------------------------------- /example/github-api-spider/build.sbt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/example/github-api-spider/build.sbt -------------------------------------------------------------------------------- /example/github-api-spider/src/main/scala/com/gaocegege/scrala/example/SJTUSpider.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/example/github-api-spider/src/main/scala/com/gaocegege/scrala/example/SJTUSpider.scala -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/project/plugins.sbt -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/readme.md -------------------------------------------------------------------------------- /src/main/resources/application.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/resources/application.conf -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/resources/logback.xml -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/core/common/request/Request.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/core/common/request/Request.scala -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/core/common/request/impl/HttpRequest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/core/common/request/impl/HttpRequest.scala -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/core/common/response/Response.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/core/common/response/Response.scala -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/core/common/response/impl/HttpResponse.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/core/common/response/impl/HttpResponse.scala -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/core/common/util/Constant.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/core/common/util/Constant.scala -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/core/common/util/ConvertTool.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/core/common/util/ConvertTool.scala -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/core/downloader/Downloader.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/core/downloader/Downloader.scala -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/core/downloader/httpclient/DefaultHttpClient.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/core/downloader/httpclient/DefaultHttpClient.scala -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/core/downloader/impl/HttpDownloader.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/core/downloader/impl/HttpDownloader.scala -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/core/engine/Engine.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/core/engine/Engine.scala -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/core/engine/manager/DownloadManager.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/core/engine/manager/DownloadManager.scala -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/core/engine/manager/impl/DefaultDownloadManager.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/core/engine/manager/impl/DefaultDownloadManager.scala -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/core/middleware/filter/Filter.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/core/middleware/filter/Filter.scala -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/core/middleware/filter/impl/DefaultFilter.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/core/middleware/filter/impl/DefaultFilter.scala -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/core/middleware/filter/impl/UrlDupFilter.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/core/middleware/filter/impl/UrlDupFilter.scala -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/core/scheduler/Scheduler.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/core/scheduler/Scheduler.scala -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/core/scheduler/impl/DefaultScheduler.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/core/scheduler/impl/DefaultScheduler.scala -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/core/spider/Spider.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/core/spider/Spider.scala -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/core/spider/impl/DefaultSpider.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/core/spider/impl/DefaultSpider.scala -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/example/GaocegegeSpider.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/example/GaocegegeSpider.scala -------------------------------------------------------------------------------- /src/main/scala/com/gaocegege/scrala/example/SJTUSpider.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dyweb/scrala/HEAD/src/main/scala/com/gaocegege/scrala/example/SJTUSpider.scala --------------------------------------------------------------------------------