├── .classpath ├── .project ├── .settings ├── org.eclipse.core.resources.prefs ├── org.eclipse.jdt.core.prefs └── org.eclipse.m2e.core.prefs ├── README.md ├── pom.xml ├── src ├── main │ ├── java │ │ └── cn │ │ │ └── ivan │ │ │ └── spider │ │ │ ├── Spider.java │ │ │ ├── SpiderWatcher.java │ │ │ ├── UrlJob.java │ │ │ ├── UrlManager.java │ │ │ ├── domain │ │ │ └── Page.java │ │ │ ├── download │ │ │ ├── Downloadable.java │ │ │ └── HttpClientDownable.java │ │ │ ├── process │ │ │ ├── JDProcessable.java │ │ │ └── Processable.java │ │ │ ├── reposity │ │ │ ├── QueueRepository.java │ │ │ ├── RandomQueueRepository.java │ │ │ ├── RandomRedisRepository.java │ │ │ ├── RedisRepository.java │ │ │ └── Repository.java │ │ │ ├── store │ │ │ ├── ConsoleStoreable.java │ │ │ ├── HBaseStoreable.java │ │ │ └── Storeable.java │ │ │ └── utils │ │ │ ├── Config.java │ │ │ ├── DomainUtils.java │ │ │ ├── HBaseUtils.java │ │ │ ├── HtmlUtils.java │ │ │ ├── PageUtils.java │ │ │ ├── RedisUtils.java │ │ │ └── SleepUtils.java │ └── resources │ │ └── log4j.properties └── test │ └── java │ └── cn │ └── ivan │ └── spider │ ├── CuratorTest.java │ └── SpiderTest.java └── target ├── classes ├── META-INF │ ├── MANIFEST.MF │ └── maven │ │ └── Spider │ │ └── spider │ │ ├── pom.properties │ │ └── pom.xml ├── cn │ └── ivan │ │ └── spider │ │ ├── Spider$1.class │ │ ├── Spider.class │ │ ├── SpiderWatcher.class │ │ ├── UrlJob.class │ │ ├── UrlManager.class │ │ ├── domain │ │ └── Page.class │ │ ├── download │ │ ├── Downloadable.class │ │ └── HttpClientDownable.class │ │ ├── process │ │ ├── JDProcessable.class │ │ └── Processable.class │ │ ├── reposity │ │ ├── QueueRepository.class │ │ ├── RandomQueueRepository.class │ │ ├── RandomRedisRepository.class │ │ ├── RedisRepository.class │ │ └── Repository.class │ │ ├── store │ │ ├── ConsoleStoreable.class │ │ ├── HBaseStoreable.class │ │ └── Storeable.class │ │ └── utils │ │ ├── Config.class │ │ ├── DomainUtils.class │ │ ├── HBaseUtils.class │ │ ├── HtmlUtils.class │ │ ├── PageUtils.class │ │ ├── RedisUtils.class │ │ └── SleepUtils.class └── log4j.properties └── test-classes └── cn └── ivan └── spider ├── CuratorTest.class └── SpiderTest.class /.classpath: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/.classpath -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/.project -------------------------------------------------------------------------------- /.settings/org.eclipse.core.resources.prefs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/.settings/org.eclipse.core.resources.prefs -------------------------------------------------------------------------------- /.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/.settings/org.eclipse.jdt.core.prefs -------------------------------------------------------------------------------- /.settings/org.eclipse.m2e.core.prefs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/.settings/org.eclipse.m2e.core.prefs -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/README.md -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/pom.xml -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/Spider.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/Spider.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/SpiderWatcher.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/SpiderWatcher.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/UrlJob.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/UrlJob.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/UrlManager.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/UrlManager.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/domain/Page.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/domain/Page.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/download/Downloadable.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/download/Downloadable.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/download/HttpClientDownable.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/download/HttpClientDownable.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/process/JDProcessable.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/process/JDProcessable.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/process/Processable.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/process/Processable.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/reposity/QueueRepository.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/reposity/QueueRepository.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/reposity/RandomQueueRepository.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/reposity/RandomQueueRepository.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/reposity/RandomRedisRepository.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/reposity/RandomRedisRepository.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/reposity/RedisRepository.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/reposity/RedisRepository.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/reposity/Repository.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/reposity/Repository.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/store/ConsoleStoreable.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/store/ConsoleStoreable.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/store/HBaseStoreable.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/store/HBaseStoreable.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/store/Storeable.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/store/Storeable.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/utils/Config.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/utils/Config.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/utils/DomainUtils.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/utils/DomainUtils.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/utils/HBaseUtils.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/utils/HBaseUtils.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/utils/HtmlUtils.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/utils/HtmlUtils.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/utils/PageUtils.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/utils/PageUtils.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/utils/RedisUtils.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/utils/RedisUtils.java -------------------------------------------------------------------------------- /src/main/java/cn/ivan/spider/utils/SleepUtils.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/java/cn/ivan/spider/utils/SleepUtils.java -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/main/resources/log4j.properties -------------------------------------------------------------------------------- /src/test/java/cn/ivan/spider/CuratorTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/test/java/cn/ivan/spider/CuratorTest.java -------------------------------------------------------------------------------- /src/test/java/cn/ivan/spider/SpiderTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/src/test/java/cn/ivan/spider/SpiderTest.java -------------------------------------------------------------------------------- /target/classes/META-INF/MANIFEST.MF: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/META-INF/MANIFEST.MF -------------------------------------------------------------------------------- /target/classes/META-INF/maven/Spider/spider/pom.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/META-INF/maven/Spider/spider/pom.properties -------------------------------------------------------------------------------- /target/classes/META-INF/maven/Spider/spider/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/META-INF/maven/Spider/spider/pom.xml -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/Spider$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/Spider$1.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/Spider.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/Spider.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/SpiderWatcher.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/SpiderWatcher.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/UrlJob.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/UrlJob.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/UrlManager.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/UrlManager.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/domain/Page.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/domain/Page.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/download/Downloadable.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/download/Downloadable.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/download/HttpClientDownable.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/download/HttpClientDownable.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/process/JDProcessable.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/process/JDProcessable.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/process/Processable.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/process/Processable.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/reposity/QueueRepository.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/reposity/QueueRepository.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/reposity/RandomQueueRepository.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/reposity/RandomQueueRepository.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/reposity/RandomRedisRepository.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/reposity/RandomRedisRepository.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/reposity/RedisRepository.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/reposity/RedisRepository.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/reposity/Repository.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/reposity/Repository.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/store/ConsoleStoreable.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/store/ConsoleStoreable.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/store/HBaseStoreable.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/store/HBaseStoreable.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/store/Storeable.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/store/Storeable.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/utils/Config.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/utils/Config.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/utils/DomainUtils.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/utils/DomainUtils.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/utils/HBaseUtils.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/utils/HBaseUtils.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/utils/HtmlUtils.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/utils/HtmlUtils.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/utils/PageUtils.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/utils/PageUtils.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/utils/RedisUtils.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/utils/RedisUtils.class -------------------------------------------------------------------------------- /target/classes/cn/ivan/spider/utils/SleepUtils.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/cn/ivan/spider/utils/SleepUtils.class -------------------------------------------------------------------------------- /target/classes/log4j.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/classes/log4j.properties -------------------------------------------------------------------------------- /target/test-classes/cn/ivan/spider/CuratorTest.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/test-classes/cn/ivan/spider/CuratorTest.class -------------------------------------------------------------------------------- /target/test-classes/cn/ivan/spider/SpiderTest.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JFanZhao/spider/HEAD/target/test-classes/cn/ivan/spider/SpiderTest.class --------------------------------------------------------------------------------