├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── cetty-core ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── jibug │ │ │ └── cetty │ │ │ └── core │ │ │ ├── AnnotationBootstrap.java │ │ │ ├── Bootstrap.java │ │ │ ├── Cetty.java │ │ │ ├── Page.java │ │ │ ├── Payload.java │ │ │ ├── Result.java │ │ │ ├── Seed.java │ │ │ ├── annotation │ │ │ ├── Query.java │ │ │ └── TargetUrl.java │ │ │ ├── concurrent │ │ │ ├── CettyAbortPolicy.java │ │ │ ├── CettyThreadPool.java │ │ │ ├── CountableThreadPool.java │ │ │ ├── NamedThreadFactory.java │ │ │ └── ThreadPerTaskExecutor.java │ │ │ ├── constants │ │ │ └── HttpConstants.java │ │ │ ├── context │ │ │ └── CettyContext.java │ │ │ ├── handler │ │ │ ├── AbstractHandlerContext.java │ │ │ ├── ConsoleReduceHandler.java │ │ │ ├── DefaultHandlerContext.java │ │ │ ├── Handler.java │ │ │ ├── HandlerContext.java │ │ │ ├── HandlerInitializer.java │ │ │ ├── HandlerPipeline.java │ │ │ ├── HttpDownloadHandler.java │ │ │ ├── PageProcessHandler.java │ │ │ ├── ProcessHandler.java │ │ │ ├── ProcessHandlerAdapter.java │ │ │ ├── ReduceHandler.java │ │ │ └── ReduceHandlerAdapter.java │ │ │ ├── model │ │ │ ├── AnnotationConfig.java │ │ │ └── RequestBody.java │ │ │ ├── net │ │ │ ├── AbstractHttpClientGenerator.java │ │ │ ├── AsyncHttpClientGenerator.java │ │ │ ├── HttpClientFactory.java │ │ │ ├── HttpClientGenerator.java │ │ │ ├── Proxy.java │ │ │ ├── ProxyStrategy.java │ │ │ └── SyncHttpClientGenerator.java │ │ │ ├── scheduler │ │ │ ├── DuplicateStrategy.java │ │ │ ├── QueueScheduler.java │ │ │ └── Scheduler.java │ │ │ └── utils │ │ │ └── UrlUtils.java │ └── resources │ │ └── logback.xml │ └── test │ └── java │ └── com │ └── jibug │ └── cetty │ ├── core │ ├── bootstrap │ │ └── BootstrapTest.java │ ├── handler │ │ └── HandlerTest.java │ └── net │ │ ├── AsyncHttpClientTest.java │ │ └── SyncHttpClientTest.java │ └── sample │ ├── kuaidaili │ └── Kuaidaili.java │ └── tianya │ └── Tianya.java ├── cetty-samples ├── .gitignore ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── jibug │ │ └── cetty │ │ └── sample │ │ ├── SampleApplication.java │ │ ├── entity │ │ └── Article.java │ │ ├── handler │ │ ├── BasePageHandler.java │ │ ├── CifnewsPageHandler.java │ │ ├── GuxiaobeiPageHandler.java │ │ └── Waimaob2cPageHandler.java │ │ ├── reduce │ │ └── ArticleReducer.java │ │ ├── runner │ │ └── CrawlerRunner.java │ │ └── service │ │ ├── CrawlerService.java │ │ └── TaskService.java │ └── resources │ ├── application.properties │ └── crawler.json └── pom.xml /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/.gitignore -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/.travis.yml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/README.md -------------------------------------------------------------------------------- /cetty-core/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/pom.xml -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/AnnotationBootstrap.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/AnnotationBootstrap.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/Bootstrap.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/Bootstrap.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/Cetty.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/Cetty.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/Page.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/Page.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/Payload.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/Payload.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/Result.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/Result.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/Seed.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/Seed.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/annotation/Query.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/annotation/Query.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/annotation/TargetUrl.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/annotation/TargetUrl.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/concurrent/CettyAbortPolicy.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/concurrent/CettyAbortPolicy.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/concurrent/CettyThreadPool.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/concurrent/CettyThreadPool.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/concurrent/CountableThreadPool.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/concurrent/CountableThreadPool.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/concurrent/NamedThreadFactory.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/concurrent/NamedThreadFactory.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/concurrent/ThreadPerTaskExecutor.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/concurrent/ThreadPerTaskExecutor.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/constants/HttpConstants.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/constants/HttpConstants.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/context/CettyContext.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/context/CettyContext.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/handler/AbstractHandlerContext.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/handler/AbstractHandlerContext.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/handler/ConsoleReduceHandler.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/handler/ConsoleReduceHandler.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/handler/DefaultHandlerContext.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/handler/DefaultHandlerContext.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/handler/Handler.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/handler/Handler.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/handler/HandlerContext.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/handler/HandlerContext.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/handler/HandlerInitializer.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/handler/HandlerInitializer.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/handler/HandlerPipeline.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/handler/HandlerPipeline.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/handler/HttpDownloadHandler.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/handler/HttpDownloadHandler.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/handler/PageProcessHandler.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/handler/PageProcessHandler.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/handler/ProcessHandler.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/handler/ProcessHandler.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/handler/ProcessHandlerAdapter.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/handler/ProcessHandlerAdapter.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/handler/ReduceHandler.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/handler/ReduceHandler.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/handler/ReduceHandlerAdapter.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/handler/ReduceHandlerAdapter.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/model/AnnotationConfig.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/model/AnnotationConfig.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/model/RequestBody.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/model/RequestBody.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/net/AbstractHttpClientGenerator.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/net/AbstractHttpClientGenerator.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/net/AsyncHttpClientGenerator.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/net/AsyncHttpClientGenerator.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/net/HttpClientFactory.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/net/HttpClientFactory.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/net/HttpClientGenerator.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/net/HttpClientGenerator.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/net/Proxy.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/net/Proxy.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/net/ProxyStrategy.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/net/ProxyStrategy.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/net/SyncHttpClientGenerator.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/net/SyncHttpClientGenerator.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/scheduler/DuplicateStrategy.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/scheduler/DuplicateStrategy.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/scheduler/QueueScheduler.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/scheduler/QueueScheduler.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/scheduler/Scheduler.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/scheduler/Scheduler.java -------------------------------------------------------------------------------- /cetty-core/src/main/java/com/jibug/cetty/core/utils/UrlUtils.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/java/com/jibug/cetty/core/utils/UrlUtils.java -------------------------------------------------------------------------------- /cetty-core/src/main/resources/logback.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/main/resources/logback.xml -------------------------------------------------------------------------------- /cetty-core/src/test/java/com/jibug/cetty/core/bootstrap/BootstrapTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/test/java/com/jibug/cetty/core/bootstrap/BootstrapTest.java -------------------------------------------------------------------------------- /cetty-core/src/test/java/com/jibug/cetty/core/handler/HandlerTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/test/java/com/jibug/cetty/core/handler/HandlerTest.java -------------------------------------------------------------------------------- /cetty-core/src/test/java/com/jibug/cetty/core/net/AsyncHttpClientTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/test/java/com/jibug/cetty/core/net/AsyncHttpClientTest.java -------------------------------------------------------------------------------- /cetty-core/src/test/java/com/jibug/cetty/core/net/SyncHttpClientTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/test/java/com/jibug/cetty/core/net/SyncHttpClientTest.java -------------------------------------------------------------------------------- /cetty-core/src/test/java/com/jibug/cetty/sample/kuaidaili/Kuaidaili.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/test/java/com/jibug/cetty/sample/kuaidaili/Kuaidaili.java -------------------------------------------------------------------------------- /cetty-core/src/test/java/com/jibug/cetty/sample/tianya/Tianya.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-core/src/test/java/com/jibug/cetty/sample/tianya/Tianya.java -------------------------------------------------------------------------------- /cetty-samples/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-samples/.gitignore -------------------------------------------------------------------------------- /cetty-samples/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-samples/pom.xml -------------------------------------------------------------------------------- /cetty-samples/src/main/java/com/jibug/cetty/sample/SampleApplication.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-samples/src/main/java/com/jibug/cetty/sample/SampleApplication.java -------------------------------------------------------------------------------- /cetty-samples/src/main/java/com/jibug/cetty/sample/entity/Article.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-samples/src/main/java/com/jibug/cetty/sample/entity/Article.java -------------------------------------------------------------------------------- /cetty-samples/src/main/java/com/jibug/cetty/sample/handler/BasePageHandler.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-samples/src/main/java/com/jibug/cetty/sample/handler/BasePageHandler.java -------------------------------------------------------------------------------- /cetty-samples/src/main/java/com/jibug/cetty/sample/handler/CifnewsPageHandler.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-samples/src/main/java/com/jibug/cetty/sample/handler/CifnewsPageHandler.java -------------------------------------------------------------------------------- /cetty-samples/src/main/java/com/jibug/cetty/sample/handler/GuxiaobeiPageHandler.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-samples/src/main/java/com/jibug/cetty/sample/handler/GuxiaobeiPageHandler.java -------------------------------------------------------------------------------- /cetty-samples/src/main/java/com/jibug/cetty/sample/handler/Waimaob2cPageHandler.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-samples/src/main/java/com/jibug/cetty/sample/handler/Waimaob2cPageHandler.java -------------------------------------------------------------------------------- /cetty-samples/src/main/java/com/jibug/cetty/sample/reduce/ArticleReducer.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-samples/src/main/java/com/jibug/cetty/sample/reduce/ArticleReducer.java -------------------------------------------------------------------------------- /cetty-samples/src/main/java/com/jibug/cetty/sample/runner/CrawlerRunner.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-samples/src/main/java/com/jibug/cetty/sample/runner/CrawlerRunner.java -------------------------------------------------------------------------------- /cetty-samples/src/main/java/com/jibug/cetty/sample/service/CrawlerService.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-samples/src/main/java/com/jibug/cetty/sample/service/CrawlerService.java -------------------------------------------------------------------------------- /cetty-samples/src/main/java/com/jibug/cetty/sample/service/TaskService.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-samples/src/main/java/com/jibug/cetty/sample/service/TaskService.java -------------------------------------------------------------------------------- /cetty-samples/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /cetty-samples/src/main/resources/crawler.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/cetty-samples/src/main/resources/crawler.json -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/heyingcai/cetty/HEAD/pom.xml --------------------------------------------------------------------------------