├── .gitignore ├── README.md ├── baidunews ├── baidunews │ ├── __init__.py │ ├── items.py │ ├── middlewares.py │ ├── pipelines.py │ ├── settings.py │ └── spiders │ │ ├── __init__.py │ │ └── news.py ├── main.py └── scrapy.cfg ├── dangdang ├── dangdang │ ├── __init__.py │ ├── items.py │ ├── middlewares.py │ ├── pipelines.py │ ├── settings.py │ └── spiders │ │ ├── __init__.py │ │ └── dd.py ├── main.py └── scrapy.cfg ├── douban ├── douban │ ├── __init__.py │ ├── items.py │ ├── middlewares.py │ ├── pipelines.py │ ├── settings.py │ └── spiders │ │ ├── __init__.py │ │ └── dou.py ├── main.py ├── scrapy.cfg └── ydm │ ├── YDMHTTP.py │ ├── YDMPython3.py │ ├── __init__.py │ ├── yundamaAPI-x64.dll │ └── yundamaAPI.dll ├── examples ├── __init__.py ├── example-1.py ├── example-10.py ├── example-11.py ├── example-12.py ├── example-13.py ├── example-14.py ├── example-15.py ├── example-16.py ├── example-17.py ├── example-18.py ├── example-19.py ├── example-2.py ├── example-20.py ├── example-21.py ├── example-22.py ├── example-23.py ├── example-24.py ├── example-25.py ├── example-26.py ├── example-3.py ├── example-4.py ├── example-5.py ├── example-6.py ├── example-7.py ├── example-8.py └── example-9.py └── jdgoods ├── jdgoods ├── __init__.py ├── items.py ├── middlewares.py ├── pipelines.py ├── settings.py └── spiders │ ├── __init__.py │ └── good.py ├── main.py └── scrapy.cfg /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | *.pyc 3 | *.log 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/README.md -------------------------------------------------------------------------------- /baidunews/baidunews/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /baidunews/baidunews/items.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/baidunews/baidunews/items.py -------------------------------------------------------------------------------- /baidunews/baidunews/middlewares.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/baidunews/baidunews/middlewares.py -------------------------------------------------------------------------------- /baidunews/baidunews/pipelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/baidunews/baidunews/pipelines.py -------------------------------------------------------------------------------- /baidunews/baidunews/settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/baidunews/baidunews/settings.py -------------------------------------------------------------------------------- /baidunews/baidunews/spiders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/baidunews/baidunews/spiders/__init__.py -------------------------------------------------------------------------------- /baidunews/baidunews/spiders/news.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/baidunews/baidunews/spiders/news.py -------------------------------------------------------------------------------- /baidunews/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/baidunews/main.py -------------------------------------------------------------------------------- /baidunews/scrapy.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/baidunews/scrapy.cfg -------------------------------------------------------------------------------- /dangdang/dangdang/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dangdang/dangdang/items.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/dangdang/dangdang/items.py -------------------------------------------------------------------------------- /dangdang/dangdang/middlewares.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/dangdang/dangdang/middlewares.py -------------------------------------------------------------------------------- /dangdang/dangdang/pipelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/dangdang/dangdang/pipelines.py -------------------------------------------------------------------------------- /dangdang/dangdang/settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/dangdang/dangdang/settings.py -------------------------------------------------------------------------------- /dangdang/dangdang/spiders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/dangdang/dangdang/spiders/__init__.py -------------------------------------------------------------------------------- /dangdang/dangdang/spiders/dd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/dangdang/dangdang/spiders/dd.py -------------------------------------------------------------------------------- /dangdang/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/dangdang/main.py -------------------------------------------------------------------------------- /dangdang/scrapy.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/dangdang/scrapy.cfg -------------------------------------------------------------------------------- /douban/douban/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /douban/douban/items.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/douban/douban/items.py -------------------------------------------------------------------------------- /douban/douban/middlewares.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/douban/douban/middlewares.py -------------------------------------------------------------------------------- /douban/douban/pipelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/douban/douban/pipelines.py -------------------------------------------------------------------------------- /douban/douban/settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/douban/douban/settings.py -------------------------------------------------------------------------------- /douban/douban/spiders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/douban/douban/spiders/__init__.py -------------------------------------------------------------------------------- /douban/douban/spiders/dou.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/douban/douban/spiders/dou.py -------------------------------------------------------------------------------- /douban/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/douban/main.py -------------------------------------------------------------------------------- /douban/scrapy.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/douban/scrapy.cfg -------------------------------------------------------------------------------- /douban/ydm/YDMHTTP.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/douban/ydm/YDMHTTP.py -------------------------------------------------------------------------------- /douban/ydm/YDMPython3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/douban/ydm/YDMPython3.py -------------------------------------------------------------------------------- /douban/ydm/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /douban/ydm/yundamaAPI-x64.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/douban/ydm/yundamaAPI-x64.dll -------------------------------------------------------------------------------- /douban/ydm/yundamaAPI.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/douban/ydm/yundamaAPI.dll -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/example-1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-1.py -------------------------------------------------------------------------------- /examples/example-10.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-10.py -------------------------------------------------------------------------------- /examples/example-11.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-11.py -------------------------------------------------------------------------------- /examples/example-12.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-12.py -------------------------------------------------------------------------------- /examples/example-13.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-13.py -------------------------------------------------------------------------------- /examples/example-14.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-14.py -------------------------------------------------------------------------------- /examples/example-15.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-15.py -------------------------------------------------------------------------------- /examples/example-16.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-16.py -------------------------------------------------------------------------------- /examples/example-17.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-17.py -------------------------------------------------------------------------------- /examples/example-18.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-18.py -------------------------------------------------------------------------------- /examples/example-19.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-19.py -------------------------------------------------------------------------------- /examples/example-2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-2.py -------------------------------------------------------------------------------- /examples/example-20.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-20.py -------------------------------------------------------------------------------- /examples/example-21.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-21.py -------------------------------------------------------------------------------- /examples/example-22.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-22.py -------------------------------------------------------------------------------- /examples/example-23.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-23.py -------------------------------------------------------------------------------- /examples/example-24.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-24.py -------------------------------------------------------------------------------- /examples/example-25.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-25.py -------------------------------------------------------------------------------- /examples/example-26.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-26.py -------------------------------------------------------------------------------- /examples/example-3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-3.py -------------------------------------------------------------------------------- /examples/example-4.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-4.py -------------------------------------------------------------------------------- /examples/example-5.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-5.py -------------------------------------------------------------------------------- /examples/example-6.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-6.py -------------------------------------------------------------------------------- /examples/example-7.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-7.py -------------------------------------------------------------------------------- /examples/example-8.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-8.py -------------------------------------------------------------------------------- /examples/example-9.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/examples/example-9.py -------------------------------------------------------------------------------- /jdgoods/jdgoods/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /jdgoods/jdgoods/items.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/jdgoods/jdgoods/items.py -------------------------------------------------------------------------------- /jdgoods/jdgoods/middlewares.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/jdgoods/jdgoods/middlewares.py -------------------------------------------------------------------------------- /jdgoods/jdgoods/pipelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/jdgoods/jdgoods/pipelines.py -------------------------------------------------------------------------------- /jdgoods/jdgoods/settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/jdgoods/jdgoods/settings.py -------------------------------------------------------------------------------- /jdgoods/jdgoods/spiders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/jdgoods/jdgoods/spiders/__init__.py -------------------------------------------------------------------------------- /jdgoods/jdgoods/spiders/good.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/jdgoods/jdgoods/spiders/good.py -------------------------------------------------------------------------------- /jdgoods/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/jdgoods/main.py -------------------------------------------------------------------------------- /jdgoods/scrapy.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gaoyaqiu/python-spider/HEAD/jdgoods/scrapy.cfg --------------------------------------------------------------------------------