├── .gitignore ├── LICENSE ├── README.md ├── docs ├── contains.png ├── distributeredis.bmp ├── master.png └── worker.png ├── main.go └── src ├── data ├── mongodb │ └── .DS_Store ├── mstarturlname.map └── redis │ └── .DS_Store ├── distribute ├── common.go ├── master.go ├── single.go ├── threadpool.go └── worker.go ├── docs ├── framework.md ├── model.md └── scrawler.md ├── model ├── mongodb.go └── redismq.go ├── scrawler ├── downloader.go ├── pipeline.go ├── request.go ├── scrawler.go ├── sinaLogin.go ├── spiders.go └── utils.go └── segment └── segment.go /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/README.md -------------------------------------------------------------------------------- /docs/contains.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/docs/contains.png -------------------------------------------------------------------------------- /docs/distributeredis.bmp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/docs/distributeredis.bmp -------------------------------------------------------------------------------- /docs/master.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/docs/master.png -------------------------------------------------------------------------------- /docs/worker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/docs/worker.png -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/main.go -------------------------------------------------------------------------------- /src/data/mongodb/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/data/mongodb/.DS_Store -------------------------------------------------------------------------------- /src/data/mstarturlname.map: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/data/mstarturlname.map -------------------------------------------------------------------------------- /src/data/redis/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/data/redis/.DS_Store -------------------------------------------------------------------------------- /src/distribute/common.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/distribute/common.go -------------------------------------------------------------------------------- /src/distribute/master.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/distribute/master.go -------------------------------------------------------------------------------- /src/distribute/single.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/distribute/single.go -------------------------------------------------------------------------------- /src/distribute/threadpool.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/distribute/threadpool.go -------------------------------------------------------------------------------- /src/distribute/worker.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/distribute/worker.go -------------------------------------------------------------------------------- /src/docs/framework.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/docs/framework.md -------------------------------------------------------------------------------- /src/docs/model.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/docs/model.md -------------------------------------------------------------------------------- /src/docs/scrawler.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/docs/scrawler.md -------------------------------------------------------------------------------- /src/model/mongodb.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/model/mongodb.go -------------------------------------------------------------------------------- /src/model/redismq.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/model/redismq.go -------------------------------------------------------------------------------- /src/scrawler/downloader.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/scrawler/downloader.go -------------------------------------------------------------------------------- /src/scrawler/pipeline.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/scrawler/pipeline.go -------------------------------------------------------------------------------- /src/scrawler/request.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/scrawler/request.go -------------------------------------------------------------------------------- /src/scrawler/scrawler.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/scrawler/scrawler.go -------------------------------------------------------------------------------- /src/scrawler/sinaLogin.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/scrawler/sinaLogin.go -------------------------------------------------------------------------------- /src/scrawler/spiders.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/scrawler/spiders.go -------------------------------------------------------------------------------- /src/scrawler/utils.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/scrawler/utils.go -------------------------------------------------------------------------------- /src/segment/segment.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zjucx/DistributedCrawler/HEAD/src/segment/segment.go --------------------------------------------------------------------------------