├── .DS_Store ├── .gitignore ├── LICENSE ├── README.MD ├── _config.yml ├── docs ├── .DS_Store ├── CNAME ├── Gemfile ├── Gemfile.lock ├── LICENSE ├── README.md ├── _config.yml ├── _includes │ ├── disqus.html │ ├── facebook.html │ ├── footer.html │ ├── google-analytics.html │ ├── head.html │ ├── jumbotron.html │ ├── navigation.html │ └── twitter.html ├── _layouts │ ├── default.html │ ├── full.html │ └── page.html ├── _site │ ├── CNAME │ ├── LICENSE │ ├── README.md │ ├── assets │ │ ├── css │ │ │ ├── docs.css │ │ │ └── styles.css │ │ └── js │ │ │ └── docs.min.js │ ├── dataset │ │ └── moviedata_small.tar.gz │ ├── index.html │ └── jekyll-docs-theme.gemspec ├── assets │ ├── css │ │ ├── docs.css │ │ └── styles.scss │ └── js │ │ └── docs.min.js ├── dataset │ └── moviedata_small.tar.gz ├── index.md └── jekyll-docs-theme.gemspec ├── img ├── code-main-fun.png └── douban_data_count.png ├── requirements.txt └── scrapy ├── .DS_Store ├── README.md ├── douban ├── .DS_Store ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-37.pyc │ ├── database.cpython-36.pyc │ ├── database.cpython-37.pyc │ ├── items.cpython-37.pyc │ ├── middlewares.cpython-37.pyc │ ├── pipelines.cpython-37.pyc │ ├── proxylib.cpython-37.pyc │ ├── settings.cpython-37.pyc │ ├── util.cpython-37.pyc │ └── validator.cpython-37.pyc ├── database.py ├── items.py ├── middlewares.py ├── pipelines.py ├── proxy.txt ├── proxylib.py ├── rebuild_pid.py ├── settings.py ├── spiders │ ├── .DS_Store │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-37.pyc │ │ ├── book_comment.cpython-37.pyc │ │ ├── book_meta.cpython-37.pyc │ │ ├── book_subject.cpython-37.pyc │ │ ├── movie_comment.cpython-37.pyc │ │ ├── movie_comment2.cpython-37.pyc │ │ ├── movie_comment3.cpython-37.pyc │ │ ├── movie_meta.cpython-37.pyc │ │ ├── movie_meta2.cpython-37.pyc │ │ ├── movie_subject.cpython-37.pyc │ │ └── person_meta.cpython-37.pyc │ ├── book_comment.py │ ├── book_meta.py │ ├── book_subject.py │ ├── movie_comment.py │ ├── movie_meta.py │ ├── movie_subject.py │ └── person_meta.py ├── update_proxy.py ├── util.py └── validator.py ├── scrapy.cfg ├── sql └── douban.sql ├── start_comment.sh ├── start_meta.sh └── start_person.sh /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/.DS_Store -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/LICENSE -------------------------------------------------------------------------------- /README.MD: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/README.MD -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/_config.yml -------------------------------------------------------------------------------- /docs/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/.DS_Store -------------------------------------------------------------------------------- /docs/CNAME: -------------------------------------------------------------------------------- 1 | moviedata.csuldw.com 2 | -------------------------------------------------------------------------------- /docs/Gemfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/Gemfile -------------------------------------------------------------------------------- /docs/Gemfile.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/Gemfile.lock -------------------------------------------------------------------------------- /docs/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/LICENSE -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/README.md -------------------------------------------------------------------------------- /docs/_config.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/_config.yml -------------------------------------------------------------------------------- /docs/_includes/disqus.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/_includes/disqus.html -------------------------------------------------------------------------------- /docs/_includes/facebook.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/_includes/facebook.html -------------------------------------------------------------------------------- /docs/_includes/footer.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/_includes/footer.html -------------------------------------------------------------------------------- /docs/_includes/google-analytics.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/_includes/google-analytics.html -------------------------------------------------------------------------------- /docs/_includes/head.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/_includes/head.html -------------------------------------------------------------------------------- /docs/_includes/jumbotron.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/_includes/jumbotron.html -------------------------------------------------------------------------------- /docs/_includes/navigation.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/_includes/navigation.html -------------------------------------------------------------------------------- /docs/_includes/twitter.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/_includes/twitter.html -------------------------------------------------------------------------------- /docs/_layouts/default.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/_layouts/default.html -------------------------------------------------------------------------------- /docs/_layouts/full.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/_layouts/full.html -------------------------------------------------------------------------------- /docs/_layouts/page.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/_layouts/page.html -------------------------------------------------------------------------------- /docs/_site/CNAME: -------------------------------------------------------------------------------- 1 | moviedata.csuldw.com 2 | -------------------------------------------------------------------------------- /docs/_site/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/_site/LICENSE -------------------------------------------------------------------------------- /docs/_site/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/_site/README.md -------------------------------------------------------------------------------- /docs/_site/assets/css/docs.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/_site/assets/css/docs.css -------------------------------------------------------------------------------- /docs/_site/assets/css/styles.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/_site/assets/css/styles.css -------------------------------------------------------------------------------- /docs/_site/assets/js/docs.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/_site/assets/js/docs.min.js -------------------------------------------------------------------------------- /docs/_site/dataset/moviedata_small.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/_site/dataset/moviedata_small.tar.gz -------------------------------------------------------------------------------- /docs/_site/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/_site/index.html -------------------------------------------------------------------------------- /docs/_site/jekyll-docs-theme.gemspec: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/_site/jekyll-docs-theme.gemspec -------------------------------------------------------------------------------- /docs/assets/css/docs.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/assets/css/docs.css -------------------------------------------------------------------------------- /docs/assets/css/styles.scss: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/assets/css/styles.scss -------------------------------------------------------------------------------- /docs/assets/js/docs.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/assets/js/docs.min.js -------------------------------------------------------------------------------- /docs/dataset/moviedata_small.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/dataset/moviedata_small.tar.gz -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/index.md -------------------------------------------------------------------------------- /docs/jekyll-docs-theme.gemspec: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/docs/jekyll-docs-theme.gemspec -------------------------------------------------------------------------------- /img/code-main-fun.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/img/code-main-fun.png -------------------------------------------------------------------------------- /img/douban_data_count.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/img/douban_data_count.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/requirements.txt -------------------------------------------------------------------------------- /scrapy/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/.DS_Store -------------------------------------------------------------------------------- /scrapy/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/README.md -------------------------------------------------------------------------------- /scrapy/douban/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/.DS_Store -------------------------------------------------------------------------------- /scrapy/douban/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scrapy/douban/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /scrapy/douban/__pycache__/database.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/__pycache__/database.cpython-36.pyc -------------------------------------------------------------------------------- /scrapy/douban/__pycache__/database.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/__pycache__/database.cpython-37.pyc -------------------------------------------------------------------------------- /scrapy/douban/__pycache__/items.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/__pycache__/items.cpython-37.pyc -------------------------------------------------------------------------------- /scrapy/douban/__pycache__/middlewares.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/__pycache__/middlewares.cpython-37.pyc -------------------------------------------------------------------------------- /scrapy/douban/__pycache__/pipelines.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/__pycache__/pipelines.cpython-37.pyc -------------------------------------------------------------------------------- /scrapy/douban/__pycache__/proxylib.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/__pycache__/proxylib.cpython-37.pyc -------------------------------------------------------------------------------- /scrapy/douban/__pycache__/settings.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/__pycache__/settings.cpython-37.pyc -------------------------------------------------------------------------------- /scrapy/douban/__pycache__/util.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/__pycache__/util.cpython-37.pyc -------------------------------------------------------------------------------- /scrapy/douban/__pycache__/validator.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/__pycache__/validator.cpython-37.pyc -------------------------------------------------------------------------------- /scrapy/douban/database.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/database.py -------------------------------------------------------------------------------- /scrapy/douban/items.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/items.py -------------------------------------------------------------------------------- /scrapy/douban/middlewares.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/middlewares.py -------------------------------------------------------------------------------- /scrapy/douban/pipelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/pipelines.py -------------------------------------------------------------------------------- /scrapy/douban/proxy.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/proxy.txt -------------------------------------------------------------------------------- /scrapy/douban/proxylib.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/proxylib.py -------------------------------------------------------------------------------- /scrapy/douban/rebuild_pid.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/rebuild_pid.py -------------------------------------------------------------------------------- /scrapy/douban/settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/settings.py -------------------------------------------------------------------------------- /scrapy/douban/spiders/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/spiders/.DS_Store -------------------------------------------------------------------------------- /scrapy/douban/spiders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/spiders/__init__.py -------------------------------------------------------------------------------- /scrapy/douban/spiders/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/spiders/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /scrapy/douban/spiders/__pycache__/book_comment.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/spiders/__pycache__/book_comment.cpython-37.pyc -------------------------------------------------------------------------------- /scrapy/douban/spiders/__pycache__/book_meta.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/spiders/__pycache__/book_meta.cpython-37.pyc -------------------------------------------------------------------------------- /scrapy/douban/spiders/__pycache__/book_subject.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/spiders/__pycache__/book_subject.cpython-37.pyc -------------------------------------------------------------------------------- /scrapy/douban/spiders/__pycache__/movie_comment.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/spiders/__pycache__/movie_comment.cpython-37.pyc -------------------------------------------------------------------------------- /scrapy/douban/spiders/__pycache__/movie_comment2.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/spiders/__pycache__/movie_comment2.cpython-37.pyc -------------------------------------------------------------------------------- /scrapy/douban/spiders/__pycache__/movie_comment3.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/spiders/__pycache__/movie_comment3.cpython-37.pyc -------------------------------------------------------------------------------- /scrapy/douban/spiders/__pycache__/movie_meta.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/spiders/__pycache__/movie_meta.cpython-37.pyc -------------------------------------------------------------------------------- /scrapy/douban/spiders/__pycache__/movie_meta2.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/spiders/__pycache__/movie_meta2.cpython-37.pyc -------------------------------------------------------------------------------- /scrapy/douban/spiders/__pycache__/movie_subject.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/spiders/__pycache__/movie_subject.cpython-37.pyc -------------------------------------------------------------------------------- /scrapy/douban/spiders/__pycache__/person_meta.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/spiders/__pycache__/person_meta.cpython-37.pyc -------------------------------------------------------------------------------- /scrapy/douban/spiders/book_comment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/spiders/book_comment.py -------------------------------------------------------------------------------- /scrapy/douban/spiders/book_meta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/spiders/book_meta.py -------------------------------------------------------------------------------- /scrapy/douban/spiders/book_subject.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/spiders/book_subject.py -------------------------------------------------------------------------------- /scrapy/douban/spiders/movie_comment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/spiders/movie_comment.py -------------------------------------------------------------------------------- /scrapy/douban/spiders/movie_meta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/spiders/movie_meta.py -------------------------------------------------------------------------------- /scrapy/douban/spiders/movie_subject.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/spiders/movie_subject.py -------------------------------------------------------------------------------- /scrapy/douban/spiders/person_meta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/spiders/person_meta.py -------------------------------------------------------------------------------- /scrapy/douban/update_proxy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/update_proxy.py -------------------------------------------------------------------------------- /scrapy/douban/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/util.py -------------------------------------------------------------------------------- /scrapy/douban/validator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/douban/validator.py -------------------------------------------------------------------------------- /scrapy/scrapy.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/scrapy.cfg -------------------------------------------------------------------------------- /scrapy/sql/douban.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/sql/douban.sql -------------------------------------------------------------------------------- /scrapy/start_comment.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/start_comment.sh -------------------------------------------------------------------------------- /scrapy/start_meta.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/start_meta.sh -------------------------------------------------------------------------------- /scrapy/start_person.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/csuldw/AntSpider/HEAD/scrapy/start_person.sh --------------------------------------------------------------------------------