├── .gitignore ├── README.md ├── Sina_spider1 ├── Begin.py ├── Sina_spider1 │ ├── __init__.py │ ├── cookies.py │ ├── items.py │ ├── middleware.py │ ├── pipelines.py │ ├── settings.py │ ├── spiders │ │ ├── __init__.py │ │ └── spiders.py │ ├── user_agents.py │ └── yumdama.py └── scrapy.cfg ├── Sina_spider2 ├── Begin.py ├── Sina_spider2 │ ├── __init__.py │ ├── commands │ │ ├── __init__.py │ │ └── crawlall.py │ ├── cookies.py │ ├── items.py │ ├── middleware.py │ ├── pipelines.py │ ├── settings.py │ ├── spiders │ │ ├── __init__.py │ │ ├── informationSpider.py │ │ └── tweetsSpider.py │ ├── user_agents.py │ └── weiboID.py └── scrapy.cfg ├── Sina_spider3 ├── Sina_spider3 │ ├── __init__.py │ ├── cleanRedis │ ├── cookies.py │ ├── items.py │ ├── middleware.py │ ├── pipelines.py │ ├── scrapy_redis │ │ ├── __init__.py │ │ ├── connection.py │ │ ├── dupefilter.py │ │ ├── pipelines.py │ │ ├── queue.py │ │ ├── scheduler.py │ │ ├── spiders.py │ │ └── tests.py │ ├── settings.py │ ├── spiders │ │ ├── __init__.py │ │ └── sinaSpider.py │ ├── user_agents.py │ ├── weiboID.py │ └── yumdama.py ├── launch.py └── scrapy.cfg └── pipelines.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/README.md -------------------------------------------------------------------------------- /Sina_spider1/Begin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider1/Begin.py -------------------------------------------------------------------------------- /Sina_spider1/Sina_spider1/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Sina_spider1/Sina_spider1/cookies.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider1/Sina_spider1/cookies.py -------------------------------------------------------------------------------- /Sina_spider1/Sina_spider1/items.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider1/Sina_spider1/items.py -------------------------------------------------------------------------------- /Sina_spider1/Sina_spider1/middleware.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider1/Sina_spider1/middleware.py -------------------------------------------------------------------------------- /Sina_spider1/Sina_spider1/pipelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider1/Sina_spider1/pipelines.py -------------------------------------------------------------------------------- /Sina_spider1/Sina_spider1/settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider1/Sina_spider1/settings.py -------------------------------------------------------------------------------- /Sina_spider1/Sina_spider1/spiders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider1/Sina_spider1/spiders/__init__.py -------------------------------------------------------------------------------- /Sina_spider1/Sina_spider1/spiders/spiders.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider1/Sina_spider1/spiders/spiders.py -------------------------------------------------------------------------------- /Sina_spider1/Sina_spider1/user_agents.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider1/Sina_spider1/user_agents.py -------------------------------------------------------------------------------- /Sina_spider1/Sina_spider1/yumdama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider1/Sina_spider1/yumdama.py -------------------------------------------------------------------------------- /Sina_spider1/scrapy.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider1/scrapy.cfg -------------------------------------------------------------------------------- /Sina_spider2/Begin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider2/Begin.py -------------------------------------------------------------------------------- /Sina_spider2/Sina_spider2/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Sina_spider2/Sina_spider2/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Sina_spider2/Sina_spider2/commands/crawlall.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider2/Sina_spider2/commands/crawlall.py -------------------------------------------------------------------------------- /Sina_spider2/Sina_spider2/cookies.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider2/Sina_spider2/cookies.py -------------------------------------------------------------------------------- /Sina_spider2/Sina_spider2/items.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider2/Sina_spider2/items.py -------------------------------------------------------------------------------- /Sina_spider2/Sina_spider2/middleware.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider2/Sina_spider2/middleware.py -------------------------------------------------------------------------------- /Sina_spider2/Sina_spider2/pipelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider2/Sina_spider2/pipelines.py -------------------------------------------------------------------------------- /Sina_spider2/Sina_spider2/settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider2/Sina_spider2/settings.py -------------------------------------------------------------------------------- /Sina_spider2/Sina_spider2/spiders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider2/Sina_spider2/spiders/__init__.py -------------------------------------------------------------------------------- /Sina_spider2/Sina_spider2/spiders/informationSpider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider2/Sina_spider2/spiders/informationSpider.py -------------------------------------------------------------------------------- /Sina_spider2/Sina_spider2/spiders/tweetsSpider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider2/Sina_spider2/spiders/tweetsSpider.py -------------------------------------------------------------------------------- /Sina_spider2/Sina_spider2/user_agents.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider2/Sina_spider2/user_agents.py -------------------------------------------------------------------------------- /Sina_spider2/Sina_spider2/weiboID.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider2/Sina_spider2/weiboID.py -------------------------------------------------------------------------------- /Sina_spider2/scrapy.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider2/scrapy.cfg -------------------------------------------------------------------------------- /Sina_spider3/Sina_spider3/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Sina_spider3/Sina_spider3/cleanRedis: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider3/Sina_spider3/cleanRedis -------------------------------------------------------------------------------- /Sina_spider3/Sina_spider3/cookies.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider3/Sina_spider3/cookies.py -------------------------------------------------------------------------------- /Sina_spider3/Sina_spider3/items.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider3/Sina_spider3/items.py -------------------------------------------------------------------------------- /Sina_spider3/Sina_spider3/middleware.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider3/Sina_spider3/middleware.py -------------------------------------------------------------------------------- /Sina_spider3/Sina_spider3/pipelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider3/Sina_spider3/pipelines.py -------------------------------------------------------------------------------- /Sina_spider3/Sina_spider3/scrapy_redis/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Sina_spider3/Sina_spider3/scrapy_redis/connection.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider3/Sina_spider3/scrapy_redis/connection.py -------------------------------------------------------------------------------- /Sina_spider3/Sina_spider3/scrapy_redis/dupefilter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider3/Sina_spider3/scrapy_redis/dupefilter.py -------------------------------------------------------------------------------- /Sina_spider3/Sina_spider3/scrapy_redis/pipelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider3/Sina_spider3/scrapy_redis/pipelines.py -------------------------------------------------------------------------------- /Sina_spider3/Sina_spider3/scrapy_redis/queue.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider3/Sina_spider3/scrapy_redis/queue.py -------------------------------------------------------------------------------- /Sina_spider3/Sina_spider3/scrapy_redis/scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider3/Sina_spider3/scrapy_redis/scheduler.py -------------------------------------------------------------------------------- /Sina_spider3/Sina_spider3/scrapy_redis/spiders.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider3/Sina_spider3/scrapy_redis/spiders.py -------------------------------------------------------------------------------- /Sina_spider3/Sina_spider3/scrapy_redis/tests.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider3/Sina_spider3/scrapy_redis/tests.py -------------------------------------------------------------------------------- /Sina_spider3/Sina_spider3/settings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider3/Sina_spider3/settings.py -------------------------------------------------------------------------------- /Sina_spider3/Sina_spider3/spiders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider3/Sina_spider3/spiders/__init__.py -------------------------------------------------------------------------------- /Sina_spider3/Sina_spider3/spiders/sinaSpider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider3/Sina_spider3/spiders/sinaSpider.py -------------------------------------------------------------------------------- /Sina_spider3/Sina_spider3/user_agents.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider3/Sina_spider3/user_agents.py -------------------------------------------------------------------------------- /Sina_spider3/Sina_spider3/weiboID.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider3/Sina_spider3/weiboID.py -------------------------------------------------------------------------------- /Sina_spider3/Sina_spider3/yumdama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider3/Sina_spider3/yumdama.py -------------------------------------------------------------------------------- /Sina_spider3/launch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider3/launch.py -------------------------------------------------------------------------------- /Sina_spider3/scrapy.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/Sina_spider3/scrapy.cfg -------------------------------------------------------------------------------- /pipelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tangbotony/SinaSpider/HEAD/pipelines.py --------------------------------------------------------------------------------