├── .gitignore ├── LICENSE ├── README.md ├── doc └── manual.md ├── lib ├── .DS_Store ├── __init__.py ├── common │ ├── __init__.py │ ├── common.py │ ├── focus.py │ ├── initializtion.py │ └── logs.py ├── core │ ├── __init__.py │ ├── console.py │ ├── crawl.py │ ├── fetch.py │ ├── rules.py │ ├── scheduling.py │ └── spider.py ├── data │ ├── .DS_Store │ └── allurl.txt ├── server │ ├── __init__.py │ ├── scheduling.py │ └── server.py └── structure │ ├── GlobalData.py │ ├── HtmlData.py │ ├── UrlData.py │ └── __init__.py ├── mspider.py ├── plugins ├── __init__.py └── phantomjs │ ├── .gitignore │ └── .travis.yml └── tools ├── test_crawler.py └── test_fetcher.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/README.md -------------------------------------------------------------------------------- /doc/manual.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/doc/manual.md -------------------------------------------------------------------------------- /lib/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/lib/.DS_Store -------------------------------------------------------------------------------- /lib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/common/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/common/common.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/lib/common/common.py -------------------------------------------------------------------------------- /lib/common/focus.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/lib/common/focus.py -------------------------------------------------------------------------------- /lib/common/initializtion.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/lib/common/initializtion.py -------------------------------------------------------------------------------- /lib/common/logs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/lib/common/logs.py -------------------------------------------------------------------------------- /lib/core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/core/console.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/lib/core/console.py -------------------------------------------------------------------------------- /lib/core/crawl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/lib/core/crawl.py -------------------------------------------------------------------------------- /lib/core/fetch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/lib/core/fetch.py -------------------------------------------------------------------------------- /lib/core/rules.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/lib/core/rules.py -------------------------------------------------------------------------------- /lib/core/scheduling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/lib/core/scheduling.py -------------------------------------------------------------------------------- /lib/core/spider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/lib/core/spider.py -------------------------------------------------------------------------------- /lib/data/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/lib/data/.DS_Store -------------------------------------------------------------------------------- /lib/data/allurl.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/lib/data/allurl.txt -------------------------------------------------------------------------------- /lib/server/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/server/scheduling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/lib/server/scheduling.py -------------------------------------------------------------------------------- /lib/server/server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/lib/server/server.py -------------------------------------------------------------------------------- /lib/structure/GlobalData.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/lib/structure/GlobalData.py -------------------------------------------------------------------------------- /lib/structure/HtmlData.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/lib/structure/HtmlData.py -------------------------------------------------------------------------------- /lib/structure/UrlData.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/lib/structure/UrlData.py -------------------------------------------------------------------------------- /lib/structure/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mspider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/mspider.py -------------------------------------------------------------------------------- /plugins/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/phantomjs/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/plugins/phantomjs/.gitignore -------------------------------------------------------------------------------- /plugins/phantomjs/.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/plugins/phantomjs/.travis.yml -------------------------------------------------------------------------------- /tools/test_crawler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/tools/test_crawler.py -------------------------------------------------------------------------------- /tools/test_fetcher.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/manning23/MSpider/HEAD/tools/test_fetcher.py --------------------------------------------------------------------------------