├── .idea └── vcs.xml ├── README.md ├── pom.xml └── src ├── main ├── java │ ├── database │ │ ├── MyRedis.java │ │ ├── MySQL.java │ │ ├── MySQLDB.java │ │ └── RedisDB.java │ ├── httpbrower │ │ └── HttpRequest.java │ ├── ipproxypool │ │ ├── grabutils │ │ │ ├── MyHttpResponse.java │ │ │ └── URLFecter.java │ │ ├── ipfilter │ │ │ └── IPFilter.java │ │ ├── ipmodel │ │ │ └── IPMessage.java │ │ ├── jobthread │ │ │ ├── CreateIPProxyPool.java │ │ │ └── IPProxyGrabThread.java │ │ ├── operation │ │ │ ├── IPProxyPool.java │ │ │ └── IPProxyPoolThread.java │ │ └── timer │ │ │ ├── MyTimeJob.java │ │ │ └── MyTimer.java │ ├── mainmethod │ │ └── MainMethod.java │ ├── mythread │ │ ├── GoodsDetailsUrlThread.java │ │ ├── TagBasicPageCrawlerThread.java │ │ └── TagBasicPageURLsCacheThread.java │ ├── parse │ │ ├── CommoditySearchPage.java │ │ ├── GoodsPageUrl.java │ │ └── TagsPage.java │ ├── urlbuild │ │ ├── GoodsUrl.java │ │ └── MainClassifyUrl.java │ └── utilclass │ │ ├── BloomFilter.java │ │ ├── MD5.java │ │ └── SerializeUtil.java └── resources │ ├── mysql-config.properties │ └── redis-config.properties └── test └── java ├── MD5压缩算法 └── MD5.java └── ThreadLocalMisunderstand.java /.idea/vcs.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/.idea/vcs.xml -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/README.md -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/pom.xml -------------------------------------------------------------------------------- /src/main/java/database/MyRedis.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/database/MyRedis.java -------------------------------------------------------------------------------- /src/main/java/database/MySQL.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/database/MySQL.java -------------------------------------------------------------------------------- /src/main/java/database/MySQLDB.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/database/MySQLDB.java -------------------------------------------------------------------------------- /src/main/java/database/RedisDB.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/database/RedisDB.java -------------------------------------------------------------------------------- /src/main/java/httpbrower/HttpRequest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/httpbrower/HttpRequest.java -------------------------------------------------------------------------------- /src/main/java/ipproxypool/grabutils/MyHttpResponse.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/ipproxypool/grabutils/MyHttpResponse.java -------------------------------------------------------------------------------- /src/main/java/ipproxypool/grabutils/URLFecter.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/ipproxypool/grabutils/URLFecter.java -------------------------------------------------------------------------------- /src/main/java/ipproxypool/ipfilter/IPFilter.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/ipproxypool/ipfilter/IPFilter.java -------------------------------------------------------------------------------- /src/main/java/ipproxypool/ipmodel/IPMessage.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/ipproxypool/ipmodel/IPMessage.java -------------------------------------------------------------------------------- /src/main/java/ipproxypool/jobthread/CreateIPProxyPool.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/ipproxypool/jobthread/CreateIPProxyPool.java -------------------------------------------------------------------------------- /src/main/java/ipproxypool/jobthread/IPProxyGrabThread.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/ipproxypool/jobthread/IPProxyGrabThread.java -------------------------------------------------------------------------------- /src/main/java/ipproxypool/operation/IPProxyPool.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/ipproxypool/operation/IPProxyPool.java -------------------------------------------------------------------------------- /src/main/java/ipproxypool/operation/IPProxyPoolThread.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/ipproxypool/operation/IPProxyPoolThread.java -------------------------------------------------------------------------------- /src/main/java/ipproxypool/timer/MyTimeJob.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/ipproxypool/timer/MyTimeJob.java -------------------------------------------------------------------------------- /src/main/java/ipproxypool/timer/MyTimer.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/ipproxypool/timer/MyTimer.java -------------------------------------------------------------------------------- /src/main/java/mainmethod/MainMethod.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/mainmethod/MainMethod.java -------------------------------------------------------------------------------- /src/main/java/mythread/GoodsDetailsUrlThread.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/mythread/GoodsDetailsUrlThread.java -------------------------------------------------------------------------------- /src/main/java/mythread/TagBasicPageCrawlerThread.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/mythread/TagBasicPageCrawlerThread.java -------------------------------------------------------------------------------- /src/main/java/mythread/TagBasicPageURLsCacheThread.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/mythread/TagBasicPageURLsCacheThread.java -------------------------------------------------------------------------------- /src/main/java/parse/CommoditySearchPage.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/parse/CommoditySearchPage.java -------------------------------------------------------------------------------- /src/main/java/parse/GoodsPageUrl.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/parse/GoodsPageUrl.java -------------------------------------------------------------------------------- /src/main/java/parse/TagsPage.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/parse/TagsPage.java -------------------------------------------------------------------------------- /src/main/java/urlbuild/GoodsUrl.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/urlbuild/GoodsUrl.java -------------------------------------------------------------------------------- /src/main/java/urlbuild/MainClassifyUrl.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/urlbuild/MainClassifyUrl.java -------------------------------------------------------------------------------- /src/main/java/utilclass/BloomFilter.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/utilclass/BloomFilter.java -------------------------------------------------------------------------------- /src/main/java/utilclass/MD5.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/utilclass/MD5.java -------------------------------------------------------------------------------- /src/main/java/utilclass/SerializeUtil.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/java/utilclass/SerializeUtil.java -------------------------------------------------------------------------------- /src/main/resources/mysql-config.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/resources/mysql-config.properties -------------------------------------------------------------------------------- /src/main/resources/redis-config.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/main/resources/redis-config.properties -------------------------------------------------------------------------------- /src/test/java/MD5压缩算法/MD5.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/test/java/MD5压缩算法/MD5.java -------------------------------------------------------------------------------- /src/test/java/ThreadLocalMisunderstand.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dhengyi/multithreading-crawlers/HEAD/src/test/java/ThreadLocalMisunderstand.java --------------------------------------------------------------------------------