├── .gitignore ├── README.md ├── docker ├── docker-compose.yml ├── mysql │ ├── Dockerfile │ └── douban.sql └── scrapyd │ ├── Dockerfile │ └── scrapyd.conf ├── requirements.txt └── scrapy ├── douban ├── __init__.py ├── database.py ├── items.py ├── middlewares.py ├── pipelines.py ├── settings.py ├── spiders │ ├── __init__.py │ ├── book_comment.py │ ├── book_meta.py │ ├── book_subject.py │ ├── movie_comment.py │ ├── movie_meta.py │ └── movie_subject.py ├── util.py └── validator.py └── scrapy.cfg /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/README.md -------------------------------------------------------------------------------- /docker/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/docker/docker-compose.yml -------------------------------------------------------------------------------- /docker/mysql/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mysql:8 2 | 3 | ADD douban.sql /docker-entrypoint-initdb.d 4 | -------------------------------------------------------------------------------- /docker/mysql/douban.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/docker/mysql/douban.sql -------------------------------------------------------------------------------- /docker/scrapyd/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/docker/scrapyd/Dockerfile -------------------------------------------------------------------------------- /docker/scrapyd/scrapyd.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/docker/scrapyd/scrapyd.conf -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/requirements.txt -------------------------------------------------------------------------------- /scrapy/douban/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scrapy/douban/database.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/scrapy/douban/database.py -------------------------------------------------------------------------------- /scrapy/douban/items.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/scrapy/douban/items.py -------------------------------------------------------------------------------- /scrapy/douban/middlewares.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/scrapy/douban/middlewares.py -------------------------------------------------------------------------------- /scrapy/douban/pipelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/scrapy/douban/pipelines.py -------------------------------------------------------------------------------- /scrapy/douban/settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/scrapy/douban/settings.py -------------------------------------------------------------------------------- /scrapy/douban/spiders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/scrapy/douban/spiders/__init__.py -------------------------------------------------------------------------------- /scrapy/douban/spiders/book_comment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/scrapy/douban/spiders/book_comment.py -------------------------------------------------------------------------------- /scrapy/douban/spiders/book_meta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/scrapy/douban/spiders/book_meta.py -------------------------------------------------------------------------------- /scrapy/douban/spiders/book_subject.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/scrapy/douban/spiders/book_subject.py -------------------------------------------------------------------------------- /scrapy/douban/spiders/movie_comment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/scrapy/douban/spiders/movie_comment.py -------------------------------------------------------------------------------- /scrapy/douban/spiders/movie_meta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/scrapy/douban/spiders/movie_meta.py -------------------------------------------------------------------------------- /scrapy/douban/spiders/movie_subject.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/scrapy/douban/spiders/movie_subject.py -------------------------------------------------------------------------------- /scrapy/douban/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/scrapy/douban/util.py -------------------------------------------------------------------------------- /scrapy/douban/validator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/scrapy/douban/validator.py -------------------------------------------------------------------------------- /scrapy/scrapy.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baabaaox/ScrapyDouban/HEAD/scrapy/scrapy.cfg --------------------------------------------------------------------------------