├── .DS_Store ├── .gitignore ├── Coursera ├── .DS_Store └── coursera │ ├── coursera │ ├── __init__.py │ ├── __init__.pyc │ ├── items.py │ ├── items.pyc │ ├── pipelines.py │ ├── settings.py │ ├── settings.pyc │ └── spiders │ │ ├── __init__.py │ │ ├── __init__.pyc │ │ ├── coursera_spider.py │ │ └── coursera_spider.pyc │ ├── login.html │ └── scrapy.cfg ├── LICENSE ├── README.md ├── cnbeta ├── cnbeta │ ├── __init__.py │ ├── __init__.pyc │ ├── items.py │ ├── items.pyc │ ├── pipelines.py │ ├── settings.py │ ├── settings.pyc │ └── spiders │ │ ├── __init__.py │ │ ├── __init__.pyc │ │ ├── cnbeta_spider.py │ │ └── cnbeta_spider.pyc ├── result.json └── scrapy.cfg ├── douban ├── douban │ ├── __init__.py │ ├── __init__.pyc │ ├── items.py │ ├── items.pyc │ ├── pipelines.py │ ├── settings.py │ ├── settings.pyc │ └── spiders │ │ ├── __init__.py │ │ ├── __init__.pyc │ │ ├── douban_spider.py │ │ └── douban_spider.pyc ├── result.json └── scrapy.cfg ├── tinydeal ├── data.csv ├── data.json ├── data.xml ├── data_total.json ├── scrapy.cfg └── tinydeal │ ├── __init__.py │ ├── items.py │ ├── middlewares.py │ ├── pipelines.py │ ├── settings.py │ └── spiders │ ├── __init__.py │ └── special_offers.py ├── tutorial ├── Books ├── Resources ├── dmoz.json ├── scrapy.cfg └── tutorial │ ├── __init__.py │ ├── __init__.pyc │ ├── items.py │ ├── items.pyc │ ├── pipelines.py │ ├── pipelines.pyc │ ├── settings.py │ ├── settings.pyc │ └── spiders │ ├── __init__.py │ ├── __init__.pyc │ ├── dmoz_spider.py │ └── dmoz_spider.pyc ├── xiaobaihe ├── result.json ├── scrapy.cfg └── xiaobaihe │ ├── __init__.py │ ├── __init__.pyc │ ├── items.py │ ├── items.pyc │ ├── pipelines.py │ ├── settings.py │ ├── settings.pyc │ └── spiders │ ├── __init__.py │ ├── __init__.pyc │ ├── xiaobaohe_spider.py │ └── xiaobaohe_spider.pyc └── zhihu ├── answer.json ├── scrapy.cfg └── zhihu ├── __init__.py ├── __init__.pyc ├── items.py ├── items.pyc ├── pipelines.py ├── settings.py ├── settings.pyc └── spiders ├── __init__.py ├── __init__.pyc ├── zhihu_spider.py └── zhihu_spider.pyc /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/.DS_Store -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | 25 | # PyInstaller 26 | # Usually these files are written by a python script from a template 27 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 28 | *.manifest 29 | *.spec 30 | 31 | # Installer logs 32 | pip-log.txt 33 | pip-delete-this-directory.txt 34 | 35 | # Unit test / coverage reports 36 | htmlcov/ 37 | .tox/ 38 | .coverage 39 | .cache 40 | nosetests.xml 41 | coverage.xml 42 | 43 | # Translations 44 | *.mo 45 | *.pot 46 | 47 | # Django stuff: 48 | *.log 49 | 50 | # Sphinx documentation 51 | docs/_build/ 52 | 53 | # PyBuilder 54 | target/ 55 | 56 | .DS_Store 57 | .AppleDouble 58 | .LSOverride 59 | 60 | # Icon must end with two \r 61 | Icon 62 | 63 | 64 | # Thumbnails 65 | ._* 66 | 67 | # Files that might appear on external disk 68 | .Spotlight-V100 69 | .Trashes 70 | 71 | # Directories potentially created on remote AFP share 72 | .AppleDB 73 | .AppleDesktop 74 | Network Trash Folder 75 | Temporary Items 76 | .apdisk 77 | -------------------------------------------------------------------------------- /Coursera/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/Coursera/.DS_Store -------------------------------------------------------------------------------- /Coursera/coursera/coursera/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/Coursera/coursera/coursera/__init__.py -------------------------------------------------------------------------------- /Coursera/coursera/coursera/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/Coursera/coursera/coursera/__init__.pyc -------------------------------------------------------------------------------- /Coursera/coursera/coursera/items.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your scraped items 4 | # 5 | # See documentation in: 6 | # http://doc.scrapy.org/en/latest/topics/items.html 7 | 8 | from scrapy.item import Item, Field 9 | 10 | 11 | class CourseraItem(Item): 12 | # define the fields for your item here like: 13 | # name = scrapy.Field() 14 | title = Field() 15 | mp4_url = Field() 16 | #pdf_url = Field() 17 | #txt_url = Field() 18 | #srt_url = Field() 19 | -------------------------------------------------------------------------------- /Coursera/coursera/coursera/items.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/Coursera/coursera/coursera/items.pyc -------------------------------------------------------------------------------- /Coursera/coursera/coursera/pipelines.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define your item pipelines here 4 | # 5 | # Don't forget to add your pipeline to the ITEM_PIPELINES setting 6 | # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html 7 | 8 | 9 | class CourseraPipeline(object): 10 | def process_item(self, item, spider): 11 | return item 12 | -------------------------------------------------------------------------------- /Coursera/coursera/coursera/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Scrapy settings for coursera project 4 | # 5 | # For simplicity, this file contains only the most important settings by 6 | # default. All the other settings are documented here: 7 | # 8 | # http://doc.scrapy.org/en/latest/topics/settings.html 9 | # 10 | 11 | BOT_NAME = 'coursera' 12 | 13 | SPIDER_MODULES = ['coursera.spiders'] 14 | NEWSPIDER_MODULE = 'coursera.spiders' 15 | 16 | # Crawl responsibly by identifying yourself (and your website) on the user-agent 17 | #USER_AGENT = 'coursera (+http://www.yourdomain.com)' 18 | -------------------------------------------------------------------------------- /Coursera/coursera/coursera/settings.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/Coursera/coursera/coursera/settings.pyc -------------------------------------------------------------------------------- /Coursera/coursera/coursera/spiders/__init__.py: -------------------------------------------------------------------------------- 1 | # This package will contain the spiders of your Scrapy project 2 | # 3 | # Please refer to the documentation for information on how to create and manage 4 | # your spiders. 5 | -------------------------------------------------------------------------------- /Coursera/coursera/coursera/spiders/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/Coursera/coursera/coursera/spiders/__init__.pyc -------------------------------------------------------------------------------- /Coursera/coursera/coursera/spiders/coursera_spider.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | """ 4 | 一个简单的Python爬虫, 用于抓取coursera网站的下载链接和pdf 5 | 6 | Anthor: Andrew Liu 7 | Version: 0.0.2 8 | Date: 2014-12-15 9 | Language: Python2.7.8 10 | Editor: Sublime Text2 11 | Operate: 具体操作请看README.md介绍 12 | """ 13 | 14 | import scrapy 15 | import random, string 16 | from scrapy.http import Request, FormRequest 17 | from scrapy.selector import Selector 18 | from coursera.items import CourseraItem 19 | 20 | def random_string(length): 21 | """ 22 | 随机生成指定长度的字母和数字序列 23 | """ 24 | return ''.join(random.choice(string.letters + string.digits) for i in xrange(length)) 25 | 26 | class CourseraSipder(scrapy.Spider) : 27 | name = "coursera" 28 | allowed_domains = ["coursera.org"] 29 | start_urls = [ 30 | "https://class.coursera.org/pkuco-001/lecture" 31 | ] 32 | 33 | def make_header(self, response) : 34 | user_agent = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) " 35 | "AppleWebKit/537.36 (KHTML, like Gecko) " 36 | "Chrome/38.0.2125.111 Safari/537.36") 37 | XCSRF2Cookie = 'csrf2_token_%s' % ''.join(random_string(8)) 38 | XCSRF2Token = ''.join(random_string(24)) 39 | XCSRFToken = ''.join(random_string(24)) 40 | cookie = "csrftoken=%s; %s=%s" % (XCSRFToken, XCSRF2Cookie, XCSRF2Token) 41 | headers= { 42 | "Referer": response.url, #对付防盗链设置, 为跳转来源的url 43 | "User-Agent": user_agent, #伪装成浏览器访问 44 | "X-Requested-With": "XMLHttpRequest", 45 | "X-CSRF2-Cookie": XCSRF2Cookie, 46 | "X-CSRF2-Token": XCSRF2Token, 47 | "X-CSRFToken": XCSRFToken, 48 | "Cookie": cookie 49 | } 50 | return headers 51 | 52 | 53 | def start_requests(self): 54 | print 'Preparing login' 55 | return [FormRequest("https://accounts.coursera.org/api/v1/login", 56 | headers = self.make_header(response), 57 | formdata = { 58 | "email": "1095511864@qq.com", 59 | "password": "HUAZANG.55789260", 60 | "webrequest": "true" 61 | }, 62 | callback = self.parse_page 63 | )] 64 | 65 | 66 | def parse_page(self, response): 67 | course = Selector(response) 68 | item = CourseraItem() 69 | item['title'] = course.xpath('//pre/a/text()').extract() 70 | item['mp4_url'] = course.xpath('//pre/text()[2]').extract() 71 | return item 72 | -------------------------------------------------------------------------------- /Coursera/coursera/coursera/spiders/coursera_spider.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/Coursera/coursera/coursera/spiders/coursera_spider.pyc -------------------------------------------------------------------------------- /Coursera/coursera/login.html: -------------------------------------------------------------------------------- 1 | Coursera.org
2 | 16 | -------------------------------------------------------------------------------- /Coursera/coursera/scrapy.cfg: -------------------------------------------------------------------------------- 1 | # Automatically created by: scrapy startproject 2 | # 3 | # For more information about the [deploy] section see: 4 | # http://doc.scrapy.org/en/latest/topics/scrapyd.html 5 | 6 | [settings] 7 | default = coursera.settings 8 | 9 | [deploy] 10 | #url = http://localhost:6800/ 11 | project = coursera 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 [Andrew Liu](http://andrewliu.tk) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 7 | of the Software, and to permit persons to whom the Software is furnished to do 8 | so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | **This repository store some example to learn scrapy better, which include tutorial, zhihu, cnbeta, xiaobaihe, douban, coursera** 4 | 5 | # Requirements 6 | 7 | - Python: 2.7.8 8 | - Scrapy: 0.24 9 | - System: Mac OS X 10.10.1 10 | 11 | 12 | # Install 13 | 14 | ``` 15 | $ git clone git@github.com:Andrew-liu/scrapy_example.git 16 | ``` 17 | 18 | # Usage 19 | 20 | You can use the six scrapy simply, just to do below: 21 | 22 | ``` 23 | $ cd zhuhu #into some one scarpy 24 | $ scrapy crwal zhihu #xiaobaihe or tutorial or cnbeta or xiaobaihe, coursera 25 | ``` 26 | 27 | **If you want to store the result into the format of json, just to do :** 28 | 29 | ``` 30 | $ scrapy crwal zhihu -o filename.json 31 | ``` 32 | 33 | 34 | 35 | # More Detail 36 | 37 | - [Python爬虫(一)](http://www.jianshu.com/p/f76bd2164856) 38 | - [Python爬虫(二)](http://www.jianshu.com/p/c3dbf8294c33) 39 | - [Python爬虫(三)](http://www.jianshu.com/p/e062b3dd110c) 40 | - [Python爬虫(四)](http://www.jianshu.com/p/86b8e78c418a) 41 | - [Python爬虫(五)](http://www.jianshu.com/p/544d406e0875) 42 | - [Python爬虫(六)](http://www.jianshu.com/p/078ad2067419) 43 | - [Python爬虫(七)](http://www.jianshu.com/p/b7f41df6202d) 44 | 45 | 46 | # License 47 | 48 | Copyright (c) 2014 [Andrew Liu](http://andrewliu.tk) 49 | 50 | Licensed under the MIT License 51 | 52 | -------------------------------------------------------------------------------- /cnbeta/cnbeta/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/cnbeta/cnbeta/__init__.py -------------------------------------------------------------------------------- /cnbeta/cnbeta/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/cnbeta/cnbeta/__init__.pyc -------------------------------------------------------------------------------- /cnbeta/cnbeta/items.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your scraped items 4 | # 5 | # See documentation in: 6 | # http://doc.scrapy.org/en/latest/topics/items.html 7 | 8 | from scrapy.item import Item, Field 9 | 10 | 11 | class CnbetaItem(Item): 12 | # define the fields for your item here like: 13 | # name = Field() 14 | title = Field() 15 | url = Field() 16 | -------------------------------------------------------------------------------- /cnbeta/cnbeta/items.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/cnbeta/cnbeta/items.pyc -------------------------------------------------------------------------------- /cnbeta/cnbeta/pipelines.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define your item pipelines here 4 | # 5 | # Don't forget to add your pipeline to the ITEM_PIPELINES setting 6 | # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html 7 | 8 | 9 | class CnbetaPipeline(object): 10 | def process_item(self, item, spider): 11 | return item 12 | -------------------------------------------------------------------------------- /cnbeta/cnbeta/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Scrapy settings for cnbeta project 4 | # 5 | # For simplicity, this file contains only the most important settings by 6 | # default. All the other settings are documented here: 7 | # 8 | # http://doc.scrapy.org/en/latest/topics/settings.html 9 | # 10 | 11 | BOT_NAME = 'cnbeta' 12 | 13 | SPIDER_MODULES = ['cnbeta.spiders'] 14 | NEWSPIDER_MODULE = 'cnbeta.spiders' 15 | 16 | # Crawl responsibly by identifying yourself (and your website) on the user-agent 17 | #USER_AGENT = 'cnbeta (+http://www.yourdomain.com)' 18 | -------------------------------------------------------------------------------- /cnbeta/cnbeta/settings.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/cnbeta/cnbeta/settings.pyc -------------------------------------------------------------------------------- /cnbeta/cnbeta/spiders/__init__.py: -------------------------------------------------------------------------------- 1 | # This package will contain the spiders of your Scrapy project 2 | # 3 | # Please refer to the documentation for information on how to create and manage 4 | # your spiders. 5 | -------------------------------------------------------------------------------- /cnbeta/cnbeta/spiders/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/cnbeta/cnbeta/spiders/__init__.pyc -------------------------------------------------------------------------------- /cnbeta/cnbeta/spiders/cnbeta_spider.py: -------------------------------------------------------------------------------- 1 | from scrapy.contrib.spiders import CrawlSpider, Rule 2 | from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor 3 | from scrapy.selector import Selector 4 | from cnbeta.items import CnbetaItem 5 | 6 | class CBSpider(CrawlSpider): 7 | name = 'cnbeta' 8 | allowed_domains = ['cnbeta.com'] 9 | start_urls = ['http://www.cnbeta.com'] 10 | 11 | rules = ( 12 | Rule(SgmlLinkExtractor(allow = ('/articles/.*\.htm', )), 13 | callback = 'parse_page', follow = True), 14 | ) 15 | 16 | def parse_page(self, response): 17 | item = CnbetaItem() 18 | sel = Selector(response) 19 | item['title'] = sel.xpath('//title/text()').extract() 20 | item['url'] = response.url 21 | return item -------------------------------------------------------------------------------- /cnbeta/cnbeta/spiders/cnbeta_spider.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/cnbeta/cnbeta/spiders/cnbeta_spider.pyc -------------------------------------------------------------------------------- /cnbeta/result.json: -------------------------------------------------------------------------------- 1 | [{"url": "http://www.cnbeta.com/articles/355235.htm", "title": ["\u641c\u72d7\u6d4f\u89c8\u5668\u62a2\u79684.0\u4e0a\u7ebf \u4e3a\u56de\u5bb6\u63d0\u4f9b\u66f4\u591a\u9009\u62e9_Sogou \u641c\u72d7\u6d4f\u89c8\u5668_cnBeta.COM"]}, 2 | {"url": "http://www.cnbeta.com/articles/355221.htm", "title": ["\u8d1d\u6602\u521b\u59cb\u4eba\u53cd\u70ae\u8f70\u683c\u529b\u8463\u660e\u73e0\u201c\u4f60\u624d\u662f\u5c0f\u5077\u201d_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"]}, 3 | {"url": "http://www.cnbeta.com/articles/355207.htm", "title": ["\u5965\u5df4\u9a6c\u5c31\u9ad8\u901a\u53cd\u5784\u65ad\u6848\u5411\u4e2d\u56fd\u65bd\u538b_cnBeta \u4eba\u7269_cnBeta.COM"]}, 4 | {"url": "http://www.cnbeta.com/articles/355211.htm", "title": ["\u536b\u661f\u6570\u636e\u663e\u793a\u5317\u6781\u6d77\u51b0\u91cf\u6574\u4f53\u4ecd\u5448\u51cf\u5c11\u8d8b\u52bf_cnBeta \u79d1\u5b66\u63a2\u7d22_cnBeta.COM"]}, 5 | {"url": "http://www.cnbeta.com/articles/355209.htm", "title": ["\u82f1\u56fd\u516c\u53f8\u53d1\u660e\u667a\u80fd\u624b\u5957 \u6325\u6325\u624b\u5373\u53ef\u4ed8\u8d26_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"]}, 6 | {"url": "http://www.cnbeta.com/articles/355215.htm", "title": ["\u590f\u666e\u9996\u6279\u53ef\u5f2f\u66f2\u6db2\u6676\u5c4f\u6216\u5c06\u7528\u4e8e\u4efb\u5929\u58023DS\u65b0\u673a_SHARP \u590f\u666e_cnBeta.COM"]}, 7 | {"url": "http://www.cnbeta.com/articles/355213.htm", "title": ["\u6c83\u5c14\u6c83\u5c06\u76f4\u63a5\u5728\u4e92\u8054\u7f51\u4e0a\u51fa\u552e\u5176\u6c7d\u8f66_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"]}, 8 | {"url": "http://www.cnbeta.com/articles/355217.htm", "title": ["[\u591a\u56fe]\u8584\u52a8\u4e4b\u4e50 vivo X5Max\u8be6\u7ec6\u8bc4\u6d4b_Cellphones \u624b\u673a_cnBeta.COM"]}, 9 | {"url": "http://www.cnbeta.com/articles/355229.htm", "title": ["[\u89c6\u9891]\u76ae\u5939\u4e2d\u7684\u7535\u6e90\uff1aSlimger\u5361\u7247\u79fb\u52a8\u7535\u6e90_Cellphones \u624b\u673a_cnBeta.COM"]}, 10 | {"url": "http://www.cnbeta.com/articles/213778.htm", "title": ["[\u56fe+\u89c6\u9891]\u82ac\u5170\u516c\u53f8\u63a8\u51fa\u6781\u5149\u5524\u9192\u706f \u80fd\u50cf\u592a\u9633\u822c\u7f13\u7f13\u5347\u8d77_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"]}, 11 | {"url": "http://www.cnbeta.com/articles/355233.htm", "title": ["\u6709\u94b1\u4efb\u6027\u5145\u4fe1\u4ef0 \u5c0f\u7c73Yeelight\u667a\u80fd\u706f\u8bc4\u6d4b_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"]}, 12 | {"url": "http://www.cnbeta.com/articles/355231.htm", "title": ["[\u56fe]\u540e\u671f\u6613\u7ef4\u4fee \u4e9a\u9a6c\u900aEcho\u5bb6\u5ead\u8bed\u97f3\u52a9\u7406\u62c6\u89e3_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"]}, 13 | {"url": "http://www.cnbeta.com/articles/355227.htm", "title": ["[\u89c6\u9891]\u8ba9\u623f\u95f4\u56db\u5b63\u5982\u6625\uff1aSunn Light\u63a8\u51fa\u201c\u5fae\u578b\u667a\u80fd\u5c0f\u592a\u9633\u201d_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"]}, 14 | {"url": "http://www.cnbeta.com/articles/355239.htm", "title": ["\u4fc4\u7f57\u65af\u8fde\u9501\u5e97\u4f7f\u7528\u65e0\u4eba\u673a\u9012\u9001\u62ab\u8428 30\u5206\u949f\u5185\u9001\u8fbe_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"]}, 15 | {"url": "http://www.cnbeta.com/articles/354227.htm", "title": ["\u5168\u65b0Windows 10\u63a7\u5236\u9762\u677f\u754c\u9762\u6cc4\u9732 \u7ecf\u5178\u754c\u9762\u5b8c\u5168\u6d88\u5931_Windows \u6b21\u4e16\u4ee3_cnBeta.COM"]}, 16 | {"url": "http://www.cnbeta.com/articles/353895.htm", "title": ["[\u89c2\u70b9]\u5c0f\u7c73\uff0cHiFi \u7834\u5c40\u8005\uff1f_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"]}, 17 | {"url": "http://www.cnbeta.com/articles/354437.htm", "title": ["\u96f7\u519b\u56de\u5e94\u8463\u660e\u73e0\u545b\u58f0\uff1a\u611f\u89c9\u8463\u603b\u597d\u50cf\u8ba4\u8f93\u4e86\u4f3c\u7684_cnBeta \u4eba\u7269_cnBeta.COM"]}, 18 | {"url": "http://www.cnbeta.com/articles/355237.htm", "title": ["[\u56fe]\u65b0\u578b\u8bbe\u8ba1\u8ba9\u57c3\u535a\u62c9\u201c\u6708\u7403\u670d\u201d\u5b89\u5168\u53c8\u51c9\u723d_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"]}, 19 | {"url": "http://www.cnbeta.com/articles/355219.htm", "title": ["\u7d22\u5c3c\u9ad8\u7ba1\uff1aPS4\u8fd8\u672a\u53d1\u529b \u597d\u620f\u5728\u540e\u9762_SONY \u7d22\u5c3c_cnBeta.COM"]}, 20 | {"url": "http://www.cnbeta.com/articles/333479.htm", "title": ["[\u4e0b\u8f7d]Windows 10\u6280\u672f\u9884\u89c8\u7248\u672c\u5df2\u53d1\u5e03 \u73b0\u5df2\u53ef\u52a0\u5165\u6d4b\u8bd5\u8ba1\u5212_Windows \u6b21\u4e16\u4ee3_cnBeta.COM"]}, 21 | {"url": "http://www.cnbeta.com/articles/354817.htm", "title": ["\u9b45\u65cf\u518d\u5356\u5173\u5b50\uff1a\u9b45\u84dd\u552e\u4ef7\u5343\u5143\u4ee5\u4e0a?_Meizu \u9b45\u65cf_cnBeta.COM"]}, 22 | {"url": "http://www.cnbeta.com/articles/324431.htm", "title": ["\u536b\u661f\u7167\u7247\u663e\u793a\u5317\u6781\u51b0\u76d6\u4e0d\u51cf\u53cd\u589e:\u4e24\u5e74\u591a\u51fa43%_cnBeta \u79d1\u5b66\u63a2\u7d22_cnBeta.COM"]}, 23 | {"url": "http://www.cnbeta.com/articles/335357.htm", "title": ["[\u9884\u5b9a\u5165\u53e3]\u82f9\u679c\u4e2d\u56fd\u5728\u7ebf\u5546\u5e97\u5f00\u59cb\u9884\u552eiPhone 6/6 Plus_Apple iPhone_cnBeta.COM"]}, 24 | {"url": "http://www.cnbeta.com/articles/354467.htm", "title": ["\u4f4e\u4ef7\u9ad8\u80fd \u5c0f\u7c73\u7a7a\u6c14\u51c0\u5316\u5668\u8bc4\u6d4b_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"]}, 25 | {"url": "http://www.cnbeta.com/articles/338665.htm", "title": ["\u5185\u90e8\u7206\u6599\u9b45\u65cf\u963f\u91cc\u6216\u8054\u5408\u63a8MX4\u5b9a\u5236\u673a_Meizu \u9b45\u65cf_cnBeta.COM"]}, 26 | {"url": "http://www.cnbeta.com/articles/339269.htm", "title": ["\u76d8\u53e4\u8d8a\u72f1 for iOS 8 \u6b63\u5f0f\u53d1\u5e03 \u6682\u4e0d\u63d0\u4f9bCydia_Apple iPhone_cnBeta.COM"]}, 27 | {"url": "http://www.cnbeta.com/articles/339707.htm", "title": ["\u82f9\u679c\u5728\u7ebf\u5546\u5e97iPad/Mac\u56db\u6b3e\u65b0\u54c1\u5df2\u7ecf\u5f00\u59cb\u9500\u552e_Apple \u82f9\u679c_cnBeta.COM"]}, 28 | {"url": "http://www.cnbeta.com/articles/340427.htm", "title": ["Outlook for Mac 16\u72ec\u5bb6\u9884\u89c8:\u5168\u65b0\u73b0\u4ee3\u5316\u754c\u9762 \u4e91\u534f\u540c \u591a\u5e73\u53f0\u7edf\u4e00\u4f53\u9a8c_Microsoft Office for Mac_cnBeta.COM"]}, 29 | {"url": "http://www.cnbeta.com/articles/337521.htm", "title": ["[\u56fe\u6587\u76f4\u64ad]\u82f9\u679ciPad Air/mini\u53caMac\u65b0\u54c1\u53d1\u5e03\u4f1a_Apple iPad_cnBeta.COM"]}, 30 | {"url": "http://www.cnbeta.com/articles/338773.htm", "title": ["\u4ee3\u53f7\u201c\u957f\u5c3e\u9ed1\u989a\u7334\u201d\uff1aUbuntu 15.04\u4ee3\u53f7\u786e\u5b9a\u4e3aVivid Vervet_Ubuntu_cnBeta.COM"]}, 31 | {"url": "http://www.cnbeta.com/articles/353203.htm", "title": ["\u4ec0\u4e48\u4ec7\u4ec0\u4e48\u6028\uff1a\u7f51\u6613\u70ae\u8f70\u964c\u964c\u80cc\u540e\u7684\u6545\u4e8b_NetEase \u7f51\u6613_cnBeta.COM"]}, 32 | {"url": "http://www.cnbeta.com/articles/340837.htm", "title": ["Office for Mac 16 \u8def\u7ebf\u56fe+\u5404\u7ec4\u4ef6\u72ec\u5bb6\u622a\u56fe\u66dd\u5149:2015\u5e74\u4e0a\u534a\u5e74\u63a8\u51fa_Microsoft Office for Mac_cnBeta.COM"]}, 33 | {"url": "http://www.cnbeta.com/articles/353883.htm", "title": ["[\u89c2\u70b9]\u5c0f\u7c73\u5370\u5ea6\u906d\u7981\uff1a\u8fd9\u4e00\u5e55\u4e3a\u4f55\u672a\u5728\u4e2d\u56fd\u53d1\u751f\uff1f_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"]}, 34 | {"url": "http://www.cnbeta.com/articles/354907.htm", "title": ["90\u540e\u7814\u7a76\u751f\u53d1\u660e3D\u6253\u5370\u629b\u5149\u673a \u4e13\u5229\u88ab200\u4e07\u4e70\u8d70_cnBeta \u4eba\u7269_cnBeta.COM"]}, 35 | {"url": "http://www.cnbeta.com/articles/354883.htm", "title": ["[\u89c6\u9891]2014\u5e74\u6211\u4eec\u5728\u8c37\u6b4c\u4e0a\u90fd\u641c\u7d22\u4e86\u4ec0\u4e48\uff1f_Google / \u8c37\u6b4c_cnBeta.COM"]}, 36 | {"url": "http://www.cnbeta.com/articles/353333.htm", "title": ["\u5b57\u5e55\u7ec4\u5173\u95ed \u8bd1\u5236\u7247\u662f\u5426\u5377\u571f\u91cd\u6765\uff1f_\u89c6\u9891\u7f51\u7ad9 / \u5a92\u4f53\u64ad\u653e\u5668_cnBeta.COM"]}, 37 | {"url": "http://www.cnbeta.com/articles/353595.htm", "title": ["[\u56fe]\u4f60\u89c9\u5f97\u662f\u5426\u503c\uff1a\u4e24\u53f0\u5168\u65b0iPhone 2G $30,000_Apple iPhone_cnBeta.COM"]}, 38 | {"url": "http://www.cnbeta.com/articles/353953.htm", "title": ["\u5c0f\u7c73\u5370\u5ea6\u906d\u7981\u53ea\u662f\u524d\u594f\uff1f\u82f9\u679c\u6b32\u5411\u5c0f\u7c73\u52a8\u5200\uff1f_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"]}, 39 | {"url": "http://www.cnbeta.com/articles/354205.htm", "title": ["\u4e2d\u56fd\u8054\u901a\u56de\u5e94\u201c\u516b\u5927\u5751\u201d\uff1a\u5c06\u9650\u65f6\u7ed9\u4e88\u6ee1\u610f\u89e3\u51b3_China Unicom \u4e2d\u56fd\u8054\u901a_cnBeta.COM"]}, 40 | {"url": "http://www.cnbeta.com/articles/355243.htm", "title": ["\u82f1\u5a92\uff1a\u65af\u8bfa\u767b\u4e8b\u4ef6\u540e\u201c\u4eba\u4eec\u66f4\u6ce8\u91cd\u4fdd\u62a4\u9690\u79c1\u201d_cnBeta \u4eba\u7269_cnBeta.COM"]}, 41 | {"url": "http://www.cnbeta.com/articles/355245.htm", "title": ["\u5fae\u535a10\u79d2\u5185\u5b9e\u65f6\u63a8\u9001\u533a\u57df\u5730\u9707\u901f\u62a5 \u672a\u6765\u6216\u53ef\u53d1\u5e03\u9884\u8b66\u4fe1\u606f_Web2.0 - Microblogging \u5fae\u535a_cnBeta.COM"]}, 42 | {"url": "http://www.cnbeta.com/articles/355249.htm", "title": ["\u53ea\u5199\u5b58\u50a8\u5668\u7684\u771f\u5b9e\u6545\u4e8b_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"]}, 43 | {"url": "http://www.cnbeta.com/articles/355241.htm", "title": ["[\u56fe]\u82f1\u56fd\u667a\u80fd\u8d29\u5356\u673a\u53ef\u9762\u90e8\u8bc6\u522b \u4ee5\u51b3\u5b9a\u662f\u5426\u552e\u51fa\u5546\u54c1_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"]}, 44 | {"url": "http://www.cnbeta.com/articles/355263.htm", "title": ["\u4e00\u52a010000mAh\u805a\u5408\u7269\u79fb\u52a8\u7535\u6e90\u53d1\u5e03\uff1a89\u5143_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"]}, 45 | {"url": "http://www.cnbeta.com/articles/355251.htm", "title": ["\u674e\u5f66\u5b8f\uff1a\u767e\u5ea6Uber\u6218\u7565\u5408\u4f5c\u5177\u6709\u91cc\u7a0b\u7891\u610f\u4e49_Baidu \u767e\u5ea6_cnBeta.COM"]}, 46 | {"url": "http://www.cnbeta.com/articles/355265.htm", "title": ["\u7537\u5b50\u65e0\u4eba\u673a\u7a7a\u6295\u5962\u4f88\u54c1\u6c42\u7231 \u906d\u5973\u751f\u6325\u62d6\u628a\u62d2\u7edd_cnBeta \u4eba\u7269_cnBeta.COM"]}, 47 | {"url": "http://www.cnbeta.com/articles/355253.htm", "title": ["\u4e00\u52a0\u624b\u673a\u5728\u5370\u5ea6\u88ab\u7981\uff1a\u56e0\u906d\u5f53\u5730\u5382\u5546\u8d77\u8bc9_Cellphones \u624b\u673a_cnBeta.COM"]}, 48 | {"url": "http://www.cnbeta.com/articles/355255.htm", "title": ["\u7537\u5b50\u4e00\u5929\u5728\u5730\u94c1\u5185\u50778\u90e8\u624b\u673a \u79f0\u4e0d\u5077\u5bf9\u4e0d\u8d77\u81ea\u5df1_cnBeta \u89c6\u70b9\u89c2\u5bdf_cnBeta.COM"]}, 49 | {"url": "http://www.cnbeta.com/articles/355257.htm", "title": ["\u4e2d\u56fd\u8d85\u5f3a\u8d85\u77ed\u6fc0\u5149\u5668\u5b9e\u73b01\u5343\u4e07\u4ebf\u74e6\u8f93\u51fa \u56fd\u9645\u9996\u6b21_cnBeta \u79d1\u5b66\u63a2\u7d22_cnBeta.COM"]}, 50 | {"url": "http://www.cnbeta.com/articles/355259.htm", "title": ["\u71c3\u6599\u8017\u5c3d - \u201c\u91d1\u661f\u5feb\u8f66\u201d\u7ec8\u8fce\u6765\u5bff\u7ec8\u6b63\u5bdd\u4e00\u523b_cnBeta \u79d1\u5b66\u63a2\u7d22_cnBeta.COM"]}, 51 | {"url": "http://www.cnbeta.com/articles/355269.htm", "title": ["\u51e0\u5e74\u540e\u6211\u4eec\u7b14\u8bb0\u672c\u7535\u8111\u5c06\u91c7\u7528\u7684\u51e0\u5927\u6280\u672f_Netbook \u4e0a\u7f51\u672c / \u5e73\u677f / Ultrabook_cnBeta.COM"]}, 52 | {"url": "http://www.cnbeta.com/articles/355271.htm", "title": ["\u90e8\u5206\u79fb\u52a8\u7535\u6e90\u5bb9\u91cf\u865a\u6807 \u91d1\u5c5e\u5916\u58f3\u4ea7\u54c1\u66f4\u5b89\u5168_cnBeta \u89c6\u70b9\u89c2\u5bdf_cnBeta.COM"]}, 53 | {"url": "http://www.cnbeta.com/articles/355273.htm", "title": ["\u53ef\u518d\u751f\u706b\u7bad\uff1aSpaceX\u518d\u4e00\u6b21\u6539\u53d8\u822a\u5929_cnBeta \u79d1\u5b66\u63a2\u7d22_cnBeta.COM"]}, 54 | {"url": "http://www.cnbeta.com/articles/355275.htm", "title": ["Google Drive\u5f00\u59cb\u652f\u6301ODF\u6587\u6863\u683c\u5f0f_Google Drive_cnBeta.COM"]}, 55 | {"url": "http://www.cnbeta.com/articles/355267.htm", "title": ["\u7535\u4fe1\u8054\u901aFDD\u8bd5\u9a8c\u518d\u6269\u5bb9\uff1a\u5747\u589e\u81f356\u4e2a\u57ce\u5e02_\u901a\u4fe1\u6280\u672f - 4G/LTE_cnBeta.COM"]}, 56 | {"url": "http://www.cnbeta.com/articles/355279.htm", "title": ["Android QQ 5.3.1 \u6b63\u5f0f\u7248\u53d1\u5e03_Tencent \u817e\u8baf QQ_cnBeta.COM"]}, 57 | {"url": "http://www.cnbeta.com/articles/355283.htm", "title": ["\u817e\u8baf QQ 6.7.13458 \u6b63\u5f0f\u53d1\u5e03_Tencent \u817e\u8baf QQ_cnBeta.COM"]}, 58 | {"url": "http://www.cnbeta.com/articles/355285.htm", "title": ["\u79fb\u52a8\u7535\u6e9060\u6279\u6b21\u4ea7\u54c1\u65e0\u4e00\u5408\u683c \u4e09\u661f\u5c0f\u7c73\u7b49\u4e0a\u9ed1\u699c_\u8b66\u544a!_cnBeta.COM"]}, 59 | {"url": "http://www.cnbeta.com/articles/355289.htm", "title": ["\u5c0f\u5a1c\u6765\u4e86\uff01\u56fd\u884cLumia Denim\u66f4\u65b0\u6b63\u5f0f\u63a8\u9001_Windows Phone_cnBeta.COM"]}, 60 | {"url": "http://www.cnbeta.com/articles/355287.htm", "title": ["\u7d22\u5c3c\u516c\u53f8\u53d1\u660e\u53ef\u7a7f\u6234\u5f0f\u667a\u80fd\u773c\u955c_SONY \u7d22\u5c3c_cnBeta.COM"]}, 61 | {"url": "http://www.cnbeta.com/articles/355277.htm", "title": ["\u82f9\u679c\u53c8\u83b7\u591a\u9879\u4e13\u5229 \u865a\u62df3D\u4e92\u52a8\u6280\u672f\u662f\u4eae\u70b9_3D / IMAX / \u865a\u62df\u73b0\u5b9e_cnBeta.COM"]}, 62 | {"url": "http://www.cnbeta.com/articles/355281.htm", "title": ["\u7f8e\u56fd\u516d\u4ee3\u673a\u53d1\u52a8\u673aADVENT\u6d4b\u8bd5\u753b\u9762_cnBeta \u79d1\u5b66\u63a2\u7d22_cnBeta.COM"]}, 63 | {"url": "http://www.cnbeta.com/articles/352737.htm", "title": ["[\u591a\u56fe]\u4f20\u9b45\u65cf\u5343\u5143\u673a\u5b9a\u540d\u9b45\u84ddNote \u66f4\u591a\u8c0d\u7167\u51fa\u73b0_Meizu \u9b45\u65cf_cnBeta.COM"]}, 64 | {"url": "http://www.cnbeta.com/articles/352919.htm", "title": ["[\u56fe]\u5c0f\u7c73\u7a7a\u6c14\u51c0\u5316\u5668\u4eae\u76f8 \u552e\uffe5899 16\u65e5\u5f00\u653e\u8d2d\u4e70_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"]}, 65 | {"url": "http://www.cnbeta.com/articles/355291.htm", "title": ["\u667a\u80fd\u624b\u673a\u5e94\u6025\u7535\u6e90\u4f17\u7b79 \u91c7\u7528AA\u5e72\u7535\u6c60_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"]}, 66 | {"url": "http://www.cnbeta.com/articles/337751.htm", "title": ["\u4e2d\u533b\u628a\u8109\u9a8c\u5b55\u51c6\u4e0d\u51c6\uff1f_cnBeta \u79d1\u5b66\u63a2\u7d22_cnBeta.COM"]}, 67 | {"url": "http://www.cnbeta.com/articles/352965.htm", "title": ["\u4e9a\u9a6c\u900a\u9519\u5bc4Xbox\u514d\u8d39\u9001\u4e2d\u56fd\u5973\u5b69_\u7535\u5b50\u5546\u52a1 - B2C / B2B_cnBeta.COM"]}, 68 | {"url": "http://www.cnbeta.com/articles/337167.htm", "title": ["\u8c37\u6b4c\u53d1\u591a\u6b3eNexus\u65b0\u54c1 Android Lollipop\u6b63\u5f0f\u4eae\u76f8_Google Android_cnBeta.COM"]}, 69 | {"url": "http://www.cnbeta.com/articles/353461.htm", "title": ["\u5df4\u6155\u8fbe\u9996\u5ea6\u56de\u5e94\u5c0f\u7c73\u7a7a\u6c14\u51c0\u5316\u5668\u6284\u88ad\u95e8 \u79f0\u6b63\u8c03\u67e5_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"]}, 70 | {"url": "http://www.cnbeta.com/articles/353665.htm", "title": ["\u5c0f\u7c73\u56de\u5e94\u51c0\u5316\u5668\u6284\u88ad\uff1a\u5bf9\u5df4\u6155\u8fbe\u58f0\u660e\u6df1\u8868\u9057\u61be_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"]}, 71 | {"url": "http://www.cnbeta.com/articles/353343.htm", "title": ["[\u89c6\u9891]\u7537\u5b50\u4e0e\u8ba1\u7b97\u5668\u5973\u53cb\u751c\u8a00\u871c\u8bed1\u5206\u949f_cnBeta \u4eba\u7269_cnBeta.COM"]}, 72 | {"url": "http://www.cnbeta.com/articles/353329.htm", "title": ["YouTube\u5e74\u5ea6\u5341\u5927\u89c6\u9891\uff1a\u5e93\u514b\u5f2f\u4e86iPhone 6_Google YouTube_cnBeta.COM"]}, 73 | {"url": "http://www.cnbeta.com/articles/336613.htm", "title": ["\u7ecf\u5178\uff01\u5468\u661f\u9a70\u300a\u5927\u8bdd\u897f\u6e38\u300b20\u5e74\u540e\u91cd\u65b0\u4e0a\u6620_Media \u5168\u7403\u5a92\u4f53_cnBeta.COM"]}, 74 | {"url": "http://www.cnbeta.com/articles/338817.htm", "title": ["[\u89c6\u9891+\u56fe\u6587\u76f4\u64ad]\u9b45\u65cf\u963f\u91cc\u5df4\u5df4\u6218\u7565\u5408\u4f5c\u53d1\u5e03\u4f1a_Meizu \u9b45\u65cf_cnBeta.COM"]}, 75 | {"url": "http://www.cnbeta.com/articles/353943.htm", "title": ["\u6dd8\u5b9d\u53cc12\u8d85\u5e02\u534a\u4ef7\u81f4\u7cfb\u7edf\u762b\u75ea_\u7535\u5b50\u5546\u52a1 - B2C / B2B_cnBeta.COM"]}, 76 | {"url": "http://www.cnbeta.com/articles/354313.htm", "title": ["[\u89c6\u9891]\u8fd8\u80fd\u66f4\u903c\u771f\u70b9\u5417 Nvidia\u9006\u5929\u7269\u7406\u6548\u679c\u6f14\u793a_NVIDIA_cnBeta.COM"]}, 77 | {"url": "http://www.cnbeta.com/articles/354819.htm", "title": ["\u9a6c\u4e91\uff1a\u66fe\u7ecf\u53bb\u80af\u5fb7\u57fa\u9762\u8bd5 25\u4e2a\u4eba\u5c31\u6211\u6ca1\u88ab\u5f55\u7528_cnBeta \u4eba\u7269 - \u9a6c\u4e91_cnBeta.COM"]}, 78 | {"url": "http://www.cnbeta.com/articles/355097.htm", "title": ["\u652f\u4ed8\u5b9d\u5237\u5730\u94c1\uff1f\u5b83\u771f\u7684\u6765\u4e86_IT\u4e0e\u4ea4\u901a_cnBeta.COM"]}, 79 | {"url": "http://www.cnbeta.com/articles/352357.htm", "title": ["\u9b45\u65cf\u5343\u5143\u65b0\u673a\u66dd\u5149\uff1a\u5f88\u6709\u7231_Meizu \u9b45\u65cf_cnBeta.COM"]}, 80 | {"url": "http://www.cnbeta.com/articles/354315.htm", "title": ["\u8463\u660e\u73e0\u70ae\u8f70\u5c0f\u7c73\u7f8e\u7684:\u4e24\u4e2a\u9a97\u5b50\u5728\u4e00\u8d77\u90a3\u5c31\u662f\u5c0f\u5077\u96c6\u56e2_cnBeta \u4eba\u7269_cnBeta.COM"]}, 81 | {"url": "http://www.cnbeta.com/articles/354731.htm", "title": ["Windows 10\uff1aAero Glass\u4e3b\u9898\u5c06\u56de\u5f52 \u53ea\u8981\u6709\u8db3\u591f\u6295\u7968_Windows \u4e2a\u6027\u5316_cnBeta.COM"]}, 82 | {"url": "http://www.cnbeta.com/articles/355247.htm", "title": ["\u4eba\u4eba\u5f71\u89c6\uff1a\u4f60\u82e5\u89c1\u6211 \u5fc5\u662f\u5c71\u5be8_\u89c6\u9891\u7f51\u7ad9 / \u5a92\u4f53\u64ad\u653e\u5668_cnBeta.COM"]}, 83 | {"url": "http://www.cnbeta.com/articles/353759.htm", "title": ["\u5c0f\u7c73\u5370\u5ea6\u5b98\u7f51\u53d1\u51fa\u516c\u5f00\u4fe1 \u6b63\u5f0f\u5ba3\u5e03\u5728\u8be5\u56fd\u505c\u552e\u4ea7\u54c1_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"]}, 84 | {"url": "http://www.cnbeta.com/articles/354939.htm", "title": ["\u534e\u4e3a\u8363\u8000\u7545\u73a94X \u79fb\u52a8/\u8054\u901a/\u5168\u7f51\u901a\u91d1\u8272\u7248\u53d1\u5e03_Huawei \u534e\u4e3a_cnBeta.COM"]}, 85 | {"url": "http://www.cnbeta.com/articles/354909.htm", "title": ["\u5168\u91d1\u5c5e\u4e00\u4f53\u673a\u8eab \u534e\u4e3a\u63a8\u51fa\u8363\u8000T1\u5e73\u677f_Huawei \u534e\u4e3a_cnBeta.COM"]}, 86 | {"url": "http://www.cnbeta.com/articles/354937.htm", "title": ["\u8363\u80006 Plus\u53d1\u5e03\uff1a\u914d\u6d77\u601d\u516b\u6838\u5904\u7406\u5668 \u540e\u7f6e\u53cc800\u4e07\u6444\u50cf\u5934_Huawei \u534e\u4e3a_cnBeta.COM"]}, 87 | {"url": "http://www.cnbeta.com/articles/355145.htm", "title": ["\u4e03\u9879\u4e0d\u53ef\u601d\u8bae\u79d1\u5b66\u5947\u601d\uff1a\u5b87\u5b99\u6839\u672c\u4e0d\u5e94\u8be5\u5b58\u5728_cnBeta \u79d1\u5b66\u63a2\u7d22_cnBeta.COM"]}, 88 | {"url": "http://www.cnbeta.com/articles/354723.htm", "title": ["\u300a\u51b0\u96ea\u5947\u7f18\u300b\u5bfc\u6f14\uff1a\u975e\u5e38\u62b1\u6b49\u8ba9\u4f60\u4eec\u8fd8\u5728\u542c\u5b69\u5b50\u4eec\u5531\u300aLet It Go\u300b_\u89c6\u9891\u7f51\u7ad9 / \u5a92\u4f53\u64ad\u653e\u5668_cnBeta.COM"]}, 89 | {"url": "http://www.cnbeta.com/articles/352449.htm", "title": ["\u9b45\u65cf4.6\u5bf8\u65b0\u673a\u518d\u66dd\u5149 \u84dd\u8272Home\u952e_Meizu \u9b45\u65cf_cnBeta.COM"]}, 90 | {"url": "http://www.cnbeta.com/articles/351693.htm", "title": ["\u4fc4\u7f57\u65af\u4e00\u540d\u8bae\u5458\u63d0\u8bae \u7acb\u6cd5\u7981\u6b62\u6240\u6709\u8bae\u5458\u7528\u82f9\u679c\u4ea7\u54c1_Apple \u82f9\u679c_cnBeta.COM"]}, 91 | {"url": "http://www.cnbeta.com/articles/354337.htm", "title": ["\u8c37\u6b4c\u3001\u5fae\u8f6f\u548cAdobe\u64a4\u79bb\u4fc4\u7f57\u65af_Google / \u8c37\u6b4c_cnBeta.COM"]}, 92 | {"url": "http://www.cnbeta.com/articles/355019.htm", "title": ["[\u53d1\u5e03\u4f1a\u56de\u987e]\u53cc\u773c\u770b\u4e16\u754c \u534e\u4e3a\u8363\u80006 Plus 1999\u5143\u8d77_cnBeta \u73b0\u573a\u62a5\u9053_cnBeta.COM"]}, 93 | {"url": "http://www.cnbeta.com/articles/354841.htm", "title": ["\u8def\u900f\u793e\u8bc4\u5c0f\u7c73: \u6bdb\u5229\u7387\u624d1.8%\uff0c\u8bc1\u660e\u6284\u88ad\u82f9\u679c\u8d5a\u4e0d\u5230\u94b1_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"]}, 94 | {"url": "http://www.cnbeta.com/articles/354895.htm", "title": ["\u5c0f\u7c73\u771f\u5b9e\u80a1\u6743\u67b6\u6784\u906d\u610f\u5916\u66dd\u5149\uff1a\u5c0f\u7c73\u4e0e\u5c0f\u7c73\u79d1\u6280\u201c\u5173\u7cfb\u7279\u522b\u201d_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"]}, 95 | {"url": "http://www.cnbeta.com/articles/354897.htm", "title": ["\u5c0f\u7c73\u624b\u673a\u5728\u5370\u5ea6\u6682\u65f6\u6062\u590d\u9500\u552e \u660e\u5e741\u67088\u65e5\u540e\u5c06\u518d\u6b21\u88ab\u7981_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"]}, 96 | {"url": "http://www.cnbeta.com/articles/354975.htm", "title": ["\u5362\u5e03\u5927\u5e45\u8d2c\u503c \u82f9\u679c\u6682\u505c\u4fc4\u7f57\u65af\u5728\u7ebf\u5546\u5e97\u9500\u552e_Apple \u82f9\u679c_cnBeta.COM"]}, 97 | {"url": "http://www.cnbeta.com/articles/354949.htm", "title": ["\u5362\u5e03\u52a0\u901f\u8d2c\u503c\uff1a\u4fc4\u7f57\u65afiPhone 6\u5168\u7403\u6700\u4fbf\u5b9c_Apple iPhone_cnBeta.COM"]}, 98 | {"url": "http://www.cnbeta.com/articles/354985.htm", "title": ["\u7231\u7acb\u4fe1\u56de\u5e94\u5c0f\u7c73\u5370\u5ea6\u4e34\u65f6\u89e3\u7981:1\u67088\u65e5\u5c0f\u7c73\u5fc5\u5c06\u8d25\u8bc9_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"]}, 99 | {"url": "http://www.cnbeta.com/articles/353441.htm", "title": ["\u9b45\u84dd\u624b\u673a\u5b98\u65b9\u5fae\u535a\u4e0a\u7ebf \u9080\u8bf7\u51fd\u5df2\u53d1\u51fa_Meizu \u9b45\u65cf_cnBeta.COM"]}][[{"url": "http://www.cnbeta.com/articles/355235.htm", "title": "\u641c\u72d7\u6d4f\u89c8\u5668\u62a2\u79684.0\u4e0a\u7ebf \u4e3a\u56de\u5bb6\u63d0\u4f9b\u66f4\u591a\u9009\u62e9_Sogou \u641c\u72d7\u6d4f\u89c8\u5668_cnBeta.COM"}, 100 | {"url": "http://www.cnbeta.com/articles/339269.htm", "title": "\u76d8\u53e4\u8d8a\u72f1 for iOS 8 \u6b63\u5f0f\u53d1\u5e03 \u6682\u4e0d\u63d0\u4f9bCydia_Apple iPhone_cnBeta.COM"}, 101 | {"url": "http://www.cnbeta.com/articles/354467.htm", "title": "\u4f4e\u4ef7\u9ad8\u80fd \u5c0f\u7c73\u7a7a\u6c14\u51c0\u5316\u5668\u8bc4\u6d4b_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"}, 102 | {"url": "http://www.cnbeta.com/articles/340427.htm", "title": "Outlook for Mac 16\u72ec\u5bb6\u9884\u89c8:\u5168\u65b0\u73b0\u4ee3\u5316\u754c\u9762 \u4e91\u534f\u540c \u591a\u5e73\u53f0\u7edf\u4e00\u4f53\u9a8c_Microsoft Office for Mac_cnBeta.COM"}, 103 | {"url": "http://www.cnbeta.com/articles/340837.htm", "title": "Office for Mac 16 \u8def\u7ebf\u56fe+\u5404\u7ec4\u4ef6\u72ec\u5bb6\u622a\u56fe\u66dd\u5149:2015\u5e74\u4e0a\u534a\u5e74\u63a8\u51fa_Microsoft Office for Mac_cnBeta.COM"}, 104 | {"url": "http://www.cnbeta.com/articles/337521.htm", "title": "[\u56fe\u6587\u76f4\u64ad]\u82f9\u679ciPad Air/mini\u53caMac\u65b0\u54c1\u53d1\u5e03\u4f1a_Apple iPad_cnBeta.COM"}, 105 | {"url": "http://www.cnbeta.com/articles/338665.htm", "title": "\u5185\u90e8\u7206\u6599\u9b45\u65cf\u963f\u91cc\u6216\u8054\u5408\u63a8MX4\u5b9a\u5236\u673a_Meizu \u9b45\u65cf_cnBeta.COM"}, 106 | {"url": "http://www.cnbeta.com/articles/333479.htm", "title": "[\u4e0b\u8f7d]Windows 10\u6280\u672f\u9884\u89c8\u7248\u672c\u5df2\u53d1\u5e03 \u73b0\u5df2\u53ef\u52a0\u5165\u6d4b\u8bd5\u8ba1\u5212_Windows \u6b21\u4e16\u4ee3_cnBeta.COM"}, 107 | {"url": "http://www.cnbeta.com/articles/339707.htm", "title": "\u82f9\u679c\u5728\u7ebf\u5546\u5e97iPad/Mac\u56db\u6b3e\u65b0\u54c1\u5df2\u7ecf\u5f00\u59cb\u9500\u552e_Apple \u82f9\u679c_cnBeta.COM"}, 108 | {"url": "http://www.cnbeta.com/articles/338773.htm", "title": "\u4ee3\u53f7\u201c\u957f\u5c3e\u9ed1\u989a\u7334\u201d\uff1aUbuntu 15.04\u4ee3\u53f7\u786e\u5b9a\u4e3aVivid Vervet_Ubuntu_cnBeta.COM"}, 109 | {"url": "http://www.cnbeta.com/articles/353203.htm", "title": "\u4ec0\u4e48\u4ec7\u4ec0\u4e48\u6028\uff1a\u7f51\u6613\u70ae\u8f70\u964c\u964c\u80cc\u540e\u7684\u6545\u4e8b_NetEase \u7f51\u6613_cnBeta.COM"}, 110 | {"url": "http://www.cnbeta.com/articles/335357.htm", "title": "[\u9884\u5b9a\u5165\u53e3]\u82f9\u679c\u4e2d\u56fd\u5728\u7ebf\u5546\u5e97\u5f00\u59cb\u9884\u552eiPhone 6/6 Plus_Apple iPhone_cnBeta.COM"}, 111 | {"url": "http://www.cnbeta.com/articles/353333.htm", "title": "\u5b57\u5e55\u7ec4\u5173\u95ed \u8bd1\u5236\u7247\u662f\u5426\u5377\u571f\u91cd\u6765\uff1f_\u89c6\u9891\u7f51\u7ad9 / \u5a92\u4f53\u64ad\u653e\u5668_cnBeta.COM"}, 112 | {"url": "http://www.cnbeta.com/articles/354227.htm", "title": "\u5168\u65b0Windows 10\u63a7\u5236\u9762\u677f\u754c\u9762\u6cc4\u9732 \u7ecf\u5178\u754c\u9762\u5b8c\u5168\u6d88\u5931_Windows \u6b21\u4e16\u4ee3_cnBeta.COM"}, 113 | {"url": "http://www.cnbeta.com/articles/353595.htm", "title": "[\u56fe]\u4f60\u89c9\u5f97\u662f\u5426\u503c\uff1a\u4e24\u53f0\u5168\u65b0iPhone 2G $30,000_Apple iPhone_cnBeta.COM"}, 114 | {"url": "http://www.cnbeta.com/articles/353895.htm", "title": "[\u89c2\u70b9]\u5c0f\u7c73\uff0cHiFi \u7834\u5c40\u8005\uff1f_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"}, 115 | {"url": "http://www.cnbeta.com/articles/353953.htm", "title": "\u5c0f\u7c73\u5370\u5ea6\u906d\u7981\u53ea\u662f\u524d\u594f\uff1f\u82f9\u679c\u6b32\u5411\u5c0f\u7c73\u52a8\u5200\uff1f_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"}, 116 | {"url": "http://www.cnbeta.com/articles/354437.htm", "title": "\u96f7\u519b\u56de\u5e94\u8463\u660e\u73e0\u545b\u58f0\uff1a\u611f\u89c9\u8463\u603b\u597d\u50cf\u8ba4\u8f93\u4e86\u4f3c\u7684_cnBeta \u4eba\u7269_cnBeta.COM"}, 117 | {"url": "http://www.cnbeta.com/articles/354205.htm", "title": "\u4e2d\u56fd\u8054\u901a\u56de\u5e94\u201c\u516b\u5927\u5751\u201d\uff1a\u5c06\u9650\u65f6\u7ed9\u4e88\u6ee1\u610f\u89e3\u51b3_China Unicom \u4e2d\u56fd\u8054\u901a_cnBeta.COM"}, 118 | {"url": "http://www.cnbeta.com/articles/353883.htm", "title": "[\u89c2\u70b9]\u5c0f\u7c73\u5370\u5ea6\u906d\u7981\uff1a\u8fd9\u4e00\u5e55\u4e3a\u4f55\u672a\u5728\u4e2d\u56fd\u53d1\u751f\uff1f_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"}, 119 | {"url": "http://www.cnbeta.com/articles/354883.htm", "title": "[\u89c6\u9891]2014\u5e74\u6211\u4eec\u5728\u8c37\u6b4c\u4e0a\u90fd\u641c\u7d22\u4e86\u4ec0\u4e48\uff1f_Google / \u8c37\u6b4c_cnBeta.COM"}, 120 | {"url": "http://www.cnbeta.com/articles/355207.htm", "title": "\u5965\u5df4\u9a6c\u5c31\u9ad8\u901a\u53cd\u5784\u65ad\u6848\u5411\u4e2d\u56fd\u65bd\u538b_cnBeta \u4eba\u7269_cnBeta.COM"}, 121 | {"url": "http://www.cnbeta.com/articles/355211.htm", "title": "\u536b\u661f\u6570\u636e\u663e\u793a\u5317\u6781\u6d77\u51b0\u91cf\u6574\u4f53\u4ecd\u5448\u51cf\u5c11\u8d8b\u52bf_cnBeta \u79d1\u5b66\u63a2\u7d22_cnBeta.COM"}, 122 | {"url": "http://www.cnbeta.com/articles/354817.htm", "title": "\u9b45\u65cf\u518d\u5356\u5173\u5b50\uff1a\u9b45\u84dd\u552e\u4ef7\u5343\u5143\u4ee5\u4e0a?_Meizu \u9b45\u65cf_cnBeta.COM"}, 123 | {"url": "http://www.cnbeta.com/articles/355215.htm", "title": "\u590f\u666e\u9996\u6279\u53ef\u5f2f\u66f2\u6db2\u6676\u5c4f\u6216\u5c06\u7528\u4e8e\u4efb\u5929\u58023DS\u65b0\u673a_SHARP \u590f\u666e_cnBeta.COM"}, 124 | {"url": "http://www.cnbeta.com/articles/354907.htm", "title": "90\u540e\u7814\u7a76\u751f\u53d1\u660e3D\u6253\u5370\u629b\u5149\u673a \u4e13\u5229\u88ab200\u4e07\u4e70\u8d70_cnBeta \u4eba\u7269_cnBeta.COM"}, 125 | {"url": "http://www.cnbeta.com/articles/355213.htm", "title": "\u6c83\u5c14\u6c83\u5c06\u76f4\u63a5\u5728\u4e92\u8054\u7f51\u4e0a\u51fa\u552e\u5176\u6c7d\u8f66_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"}, 126 | {"url": "http://www.cnbeta.com/articles/355217.htm", "title": "[\u591a\u56fe]\u8584\u52a8\u4e4b\u4e50 vivo X5Max\u8be6\u7ec6\u8bc4\u6d4b_Cellphones \u624b\u673a_cnBeta.COM"}, 127 | {"url": "http://www.cnbeta.com/articles/355209.htm", "title": "\u82f1\u56fd\u516c\u53f8\u53d1\u660e\u667a\u80fd\u624b\u5957 \u6325\u6325\u624b\u5373\u53ef\u4ed8\u8d26_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"}, 128 | {"url": "http://www.cnbeta.com/articles/355219.htm", "title": "\u7d22\u5c3c\u9ad8\u7ba1\uff1aPS4\u8fd8\u672a\u53d1\u529b \u597d\u620f\u5728\u540e\u9762_SONY \u7d22\u5c3c_cnBeta.COM"}, 129 | {"url": "http://www.cnbeta.com/articles/355227.htm", "title": "[\u89c6\u9891]\u8ba9\u623f\u95f4\u56db\u5b63\u5982\u6625\uff1aSunn Light\u63a8\u51fa\u201c\u5fae\u578b\u667a\u80fd\u5c0f\u592a\u9633\u201d_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"}, 130 | {"url": "http://www.cnbeta.com/articles/355221.htm", "title": "\u8d1d\u6602\u521b\u59cb\u4eba\u53cd\u70ae\u8f70\u683c\u529b\u8463\u660e\u73e0\u201c\u4f60\u624d\u662f\u5c0f\u5077\u201d_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"}, 131 | {"url": "http://www.cnbeta.com/articles/355231.htm", "title": "[\u56fe]\u540e\u671f\u6613\u7ef4\u4fee \u4e9a\u9a6c\u900aEcho\u5bb6\u5ead\u8bed\u97f3\u52a9\u7406\u62c6\u89e3_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"}, 132 | {"url": "http://www.cnbeta.com/articles/355237.htm", "title": "[\u56fe]\u65b0\u578b\u8bbe\u8ba1\u8ba9\u57c3\u535a\u62c9\u201c\u6708\u7403\u670d\u201d\u5b89\u5168\u53c8\u51c9\u723d_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"}, 133 | {"url": "http://www.cnbeta.com/articles/355239.htm", "title": "\u4fc4\u7f57\u65af\u8fde\u9501\u5e97\u4f7f\u7528\u65e0\u4eba\u673a\u9012\u9001\u62ab\u8428 30\u5206\u949f\u5185\u9001\u8fbe_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"}, 134 | {"url": "http://www.cnbeta.com/articles/355241.htm", "title": "[\u56fe]\u82f1\u56fd\u667a\u80fd\u8d29\u5356\u673a\u53ef\u9762\u90e8\u8bc6\u522b \u4ee5\u51b3\u5b9a\u662f\u5426\u552e\u51fa\u5546\u54c1_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"}, 135 | {"url": "http://www.cnbeta.com/articles/355233.htm", "title": "\u6709\u94b1\u4efb\u6027\u5145\u4fe1\u4ef0 \u5c0f\u7c73Yeelight\u667a\u80fd\u706f\u8bc4\u6d4b_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"}, 136 | {"url": "http://www.cnbeta.com/articles/213778.htm", "title": "[\u56fe+\u89c6\u9891]\u82ac\u5170\u516c\u53f8\u63a8\u51fa\u6781\u5149\u5524\u9192\u706f \u80fd\u50cf\u592a\u9633\u822c\u7f13\u7f13\u5347\u8d77_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"}, 137 | {"url": "http://www.cnbeta.com/articles/355243.htm", "title": "\u82f1\u5a92\uff1a\u65af\u8bfa\u767b\u4e8b\u4ef6\u540e\u201c\u4eba\u4eec\u66f4\u6ce8\u91cd\u4fdd\u62a4\u9690\u79c1\u201d_cnBeta \u4eba\u7269_cnBeta.COM"}, 138 | {"url": "http://www.cnbeta.com/articles/355229.htm", "title": "[\u89c6\u9891]\u76ae\u5939\u4e2d\u7684\u7535\u6e90\uff1aSlimger\u5361\u7247\u79fb\u52a8\u7535\u6e90_Cellphones \u624b\u673a_cnBeta.COM"}, 139 | {"url": "http://www.cnbeta.com/articles/355245.htm", "title": "\u5fae\u535a10\u79d2\u5185\u5b9e\u65f6\u63a8\u9001\u533a\u57df\u5730\u9707\u901f\u62a5 \u672a\u6765\u6216\u53ef\u53d1\u5e03\u9884\u8b66\u4fe1\u606f_Web2.0 - Microblogging \u5fae\u535a_cnBeta.COM"}, 140 | {"url": "http://www.cnbeta.com/articles/355249.htm", "title": "\u53ea\u5199\u5b58\u50a8\u5668\u7684\u771f\u5b9e\u6545\u4e8b_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"}, 141 | {"url": "http://www.cnbeta.com/articles/324431.htm", "title": "\u536b\u661f\u7167\u7247\u663e\u793a\u5317\u6781\u51b0\u76d6\u4e0d\u51cf\u53cd\u589e:\u4e24\u5e74\u591a\u51fa43%_cnBeta \u79d1\u5b66\u63a2\u7d22_cnBeta.COM"}, 142 | {"url": "http://www.cnbeta.com/articles/355251.htm", "title": "\u674e\u5f66\u5b8f\uff1a\u767e\u5ea6Uber\u6218\u7565\u5408\u4f5c\u5177\u6709\u91cc\u7a0b\u7891\u610f\u4e49_Baidu \u767e\u5ea6_cnBeta.COM"}, 143 | {"url": "http://www.cnbeta.com/articles/355253.htm", "title": "\u4e00\u52a0\u624b\u673a\u5728\u5370\u5ea6\u88ab\u7981\uff1a\u56e0\u906d\u5f53\u5730\u5382\u5546\u8d77\u8bc9_Cellphones \u624b\u673a_cnBeta.COM"}, 144 | {"url": "http://www.cnbeta.com/articles/355255.htm", "title": "\u7537\u5b50\u4e00\u5929\u5728\u5730\u94c1\u5185\u50778\u90e8\u624b\u673a \u79f0\u4e0d\u5077\u5bf9\u4e0d\u8d77\u81ea\u5df1_cnBeta \u89c6\u70b9\u89c2\u5bdf_cnBeta.COM"}, 145 | {"url": "http://www.cnbeta.com/articles/355259.htm", "title": "\u71c3\u6599\u8017\u5c3d - \u201c\u91d1\u661f\u5feb\u8f66\u201d\u7ec8\u8fce\u6765\u5bff\u7ec8\u6b63\u5bdd\u4e00\u523b_cnBeta \u79d1\u5b66\u63a2\u7d22_cnBeta.COM"}, 146 | {"url": "http://www.cnbeta.com/articles/355263.htm", "title": "\u4e00\u52a010000mAh\u805a\u5408\u7269\u79fb\u52a8\u7535\u6e90\u53d1\u5e03\uff1a89\u5143_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"}, 147 | {"url": "http://www.cnbeta.com/articles/355267.htm", "title": "\u7535\u4fe1\u8054\u901aFDD\u8bd5\u9a8c\u518d\u6269\u5bb9\uff1a\u5747\u589e\u81f356\u4e2a\u57ce\u5e02_\u901a\u4fe1\u6280\u672f - 4G/LTE_cnBeta.COM"}, 148 | {"url": "http://www.cnbeta.com/articles/355257.htm", "title": "\u4e2d\u56fd\u8d85\u5f3a\u8d85\u77ed\u6fc0\u5149\u5668\u5b9e\u73b01\u5343\u4e07\u4ebf\u74e6\u8f93\u51fa \u56fd\u9645\u9996\u6b21_cnBeta \u79d1\u5b66\u63a2\u7d22_cnBeta.COM"}, 149 | {"url": "http://www.cnbeta.com/articles/355273.htm", "title": "\u53ef\u518d\u751f\u706b\u7bad\uff1aSpaceX\u518d\u4e00\u6b21\u6539\u53d8\u822a\u5929_cnBeta \u79d1\u5b66\u63a2\u7d22_cnBeta.COM"}, 150 | {"url": "http://www.cnbeta.com/articles/355269.htm", "title": "\u51e0\u5e74\u540e\u6211\u4eec\u7b14\u8bb0\u672c\u7535\u8111\u5c06\u91c7\u7528\u7684\u51e0\u5927\u6280\u672f_Netbook \u4e0a\u7f51\u672c / \u5e73\u677f / Ultrabook_cnBeta.COM"}, 151 | {"url": "http://www.cnbeta.com/articles/355275.htm", "title": "Google Drive\u5f00\u59cb\u652f\u6301ODF\u6587\u6863\u683c\u5f0f_Google Drive_cnBeta.COM"}, 152 | {"url": "http://www.cnbeta.com/articles/355277.htm", "title": "\u82f9\u679c\u53c8\u83b7\u591a\u9879\u4e13\u5229 \u865a\u62df3D\u4e92\u52a8\u6280\u672f\u662f\u4eae\u70b9_3D / IMAX / \u865a\u62df\u73b0\u5b9e_cnBeta.COM"}, 153 | {"url": "http://www.cnbeta.com/articles/355279.htm", "title": "Android QQ 5.3.1 \u6b63\u5f0f\u7248\u53d1\u5e03_Tencent \u817e\u8baf QQ_cnBeta.COM"}, 154 | {"url": "http://www.cnbeta.com/articles/355265.htm", "title": "\u7537\u5b50\u65e0\u4eba\u673a\u7a7a\u6295\u5962\u4f88\u54c1\u6c42\u7231 \u906d\u5973\u751f\u6325\u62d6\u628a\u62d2\u7edd_cnBeta \u4eba\u7269_cnBeta.COM"}, 155 | {"url": "http://www.cnbeta.com/articles/355283.htm", "title": "\u817e\u8baf QQ 6.7.13458 \u6b63\u5f0f\u53d1\u5e03_Tencent \u817e\u8baf QQ_cnBeta.COM"}, 156 | {"url": "http://www.cnbeta.com/articles/355285.htm", "title": "\u79fb\u52a8\u7535\u6e9060\u6279\u6b21\u4ea7\u54c1\u65e0\u4e00\u5408\u683c \u4e09\u661f\u5c0f\u7c73\u7b49\u4e0a\u9ed1\u699c_\u8b66\u544a!_cnBeta.COM"}, 157 | {"url": "http://www.cnbeta.com/articles/355289.htm", "title": "\u5c0f\u5a1c\u6765\u4e86\uff01\u56fd\u884cLumia Denim\u66f4\u65b0\u6b63\u5f0f\u63a8\u9001_Windows Phone_cnBeta.COM"}, 158 | {"url": "http://www.cnbeta.com/articles/355291.htm", "title": "\u667a\u80fd\u624b\u673a\u5e94\u6025\u7535\u6e90\u4f17\u7b79 \u91c7\u7528AA\u5e72\u7535\u6c60_cnBeta \u786c\u4ef6\u65b0\u95fb_cnBeta.COM"}, 159 | {"url": "http://www.cnbeta.com/articles/355287.htm", "title": "\u7d22\u5c3c\u516c\u53f8\u53d1\u660e\u53ef\u7a7f\u6234\u5f0f\u667a\u80fd\u773c\u955c_SONY \u7d22\u5c3c_cnBeta.COM"}, 160 | {"url": "http://www.cnbeta.com/articles/355281.htm", "title": "\u7f8e\u56fd\u516d\u4ee3\u673a\u53d1\u52a8\u673aADVENT\u6d4b\u8bd5\u753b\u9762_cnBeta \u79d1\u5b66\u63a2\u7d22_cnBeta.COM"}, 161 | {"url": "http://www.cnbeta.com/articles/352737.htm", "title": "[\u591a\u56fe]\u4f20\u9b45\u65cf\u5343\u5143\u673a\u5b9a\u540d\u9b45\u84ddNote \u66f4\u591a\u8c0d\u7167\u51fa\u73b0_Meizu \u9b45\u65cf_cnBeta.COM"}, 162 | {"url": "http://www.cnbeta.com/articles/353461.htm", "title": "\u5df4\u6155\u8fbe\u9996\u5ea6\u56de\u5e94\u5c0f\u7c73\u7a7a\u6c14\u51c0\u5316\u5668\u6284\u88ad\u95e8 \u79f0\u6b63\u8c03\u67e5_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"}, 163 | {"url": "http://www.cnbeta.com/articles/353665.htm", "title": "\u5c0f\u7c73\u56de\u5e94\u51c0\u5316\u5668\u6284\u88ad\uff1a\u5bf9\u5df4\u6155\u8fbe\u58f0\u660e\u6df1\u8868\u9057\u61be_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"}, 164 | {"url": "http://www.cnbeta.com/articles/352919.htm", "title": "[\u56fe]\u5c0f\u7c73\u7a7a\u6c14\u51c0\u5316\u5668\u4eae\u76f8 \u552e\uffe5899 16\u65e5\u5f00\u653e\u8d2d\u4e70_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"}, 165 | {"url": "http://www.cnbeta.com/articles/353759.htm", "title": "\u5c0f\u7c73\u5370\u5ea6\u5b98\u7f51\u53d1\u51fa\u516c\u5f00\u4fe1 \u6b63\u5f0f\u5ba3\u5e03\u5728\u8be5\u56fd\u505c\u552e\u4ea7\u54c1_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"}, 166 | {"url": "http://www.cnbeta.com/articles/337167.htm", "title": "\u8c37\u6b4c\u53d1\u591a\u6b3eNexus\u65b0\u54c1 Android Lollipop\u6b63\u5f0f\u4eae\u76f8_Google Android_cnBeta.COM"}, 167 | {"url": "http://www.cnbeta.com/articles/353343.htm", "title": "[\u89c6\u9891]\u7537\u5b50\u4e0e\u8ba1\u7b97\u5668\u5973\u53cb\u751c\u8a00\u871c\u8bed1\u5206\u949f_cnBeta \u4eba\u7269_cnBeta.COM"}, 168 | {"url": "http://www.cnbeta.com/articles/353329.htm", "title": "YouTube\u5e74\u5ea6\u5341\u5927\u89c6\u9891\uff1a\u5e93\u514b\u5f2f\u4e86iPhone 6_Google YouTube_cnBeta.COM"}, 169 | {"url": "http://www.cnbeta.com/articles/353441.htm", "title": "\u9b45\u84dd\u624b\u673a\u5b98\u65b9\u5fae\u535a\u4e0a\u7ebf \u9080\u8bf7\u51fd\u5df2\u53d1\u51fa_Meizu \u9b45\u65cf_cnBeta.COM"}, 170 | {"url": "http://www.cnbeta.com/articles/338817.htm", "title": "[\u89c6\u9891+\u56fe\u6587\u76f4\u64ad]\u9b45\u65cf\u963f\u91cc\u5df4\u5df4\u6218\u7565\u5408\u4f5c\u53d1\u5e03\u4f1a_Meizu \u9b45\u65cf_cnBeta.COM"}, 171 | {"url": "http://www.cnbeta.com/articles/353943.htm", "title": "\u6dd8\u5b9d\u53cc12\u8d85\u5e02\u534a\u4ef7\u81f4\u7cfb\u7edf\u762b\u75ea_\u7535\u5b50\u5546\u52a1 - B2C / B2B_cnBeta.COM"}, 172 | {"url": "http://www.cnbeta.com/articles/352965.htm", "title": "\u4e9a\u9a6c\u900a\u9519\u5bc4Xbox\u514d\u8d39\u9001\u4e2d\u56fd\u5973\u5b69_\u7535\u5b50\u5546\u52a1 - B2C / B2B_cnBeta.COM"}, 173 | {"url": "http://www.cnbeta.com/articles/354313.htm", "title": "[\u89c6\u9891]\u8fd8\u80fd\u66f4\u903c\u771f\u70b9\u5417 Nvidia\u9006\u5929\u7269\u7406\u6548\u679c\u6f14\u793a_NVIDIA_cnBeta.COM"}, 174 | {"url": "http://www.cnbeta.com/articles/337751.htm", "title": "\u4e2d\u533b\u628a\u8109\u9a8c\u5b55\u51c6\u4e0d\u51c6\uff1f_cnBeta \u79d1\u5b66\u63a2\u7d22_cnBeta.COM"}, 175 | {"url": "http://www.cnbeta.com/articles/354315.htm", "title": "\u8463\u660e\u73e0\u70ae\u8f70\u5c0f\u7c73\u7f8e\u7684:\u4e24\u4e2a\u9a97\u5b50\u5728\u4e00\u8d77\u90a3\u5c31\u662f\u5c0f\u5077\u96c6\u56e2_cnBeta \u4eba\u7269_cnBeta.COM"}, 176 | {"url": "http://www.cnbeta.com/articles/336613.htm", "title": "\u7ecf\u5178\uff01\u5468\u661f\u9a70\u300a\u5927\u8bdd\u897f\u6e38\u300b20\u5e74\u540e\u91cd\u65b0\u4e0a\u6620_Media \u5168\u7403\u5a92\u4f53_cnBeta.COM"}, 177 | {"url": "http://www.cnbeta.com/articles/355097.htm", "title": "\u652f\u4ed8\u5b9d\u5237\u5730\u94c1\uff1f\u5b83\u771f\u7684\u6765\u4e86_IT\u4e0e\u4ea4\u901a_cnBeta.COM"}, 178 | {"url": "http://www.cnbeta.com/articles/354819.htm", "title": "\u9a6c\u4e91\uff1a\u66fe\u7ecf\u53bb\u80af\u5fb7\u57fa\u9762\u8bd5 25\u4e2a\u4eba\u5c31\u6211\u6ca1\u88ab\u5f55\u7528_cnBeta \u4eba\u7269 - \u9a6c\u4e91_cnBeta.COM"}, 179 | {"url": "http://www.cnbeta.com/articles/355247.htm", "title": "\u4eba\u4eba\u5f71\u89c6\uff1a\u4f60\u82e5\u89c1\u6211 \u5fc5\u662f\u5c71\u5be8_\u89c6\u9891\u7f51\u7ad9 / \u5a92\u4f53\u64ad\u653e\u5668_cnBeta.COM"}, 180 | {"url": "http://www.cnbeta.com/articles/352449.htm", "title": "\u9b45\u65cf4.6\u5bf8\u65b0\u673a\u518d\u66dd\u5149 \u84dd\u8272Home\u952e_Meizu \u9b45\u65cf_cnBeta.COM"}, 181 | {"url": "http://www.cnbeta.com/articles/354731.htm", "title": "Windows 10\uff1aAero Glass\u4e3b\u9898\u5c06\u56de\u5f52 \u53ea\u8981\u6709\u8db3\u591f\u6295\u7968_Windows \u4e2a\u6027\u5316_cnBeta.COM"}, 182 | {"url": "http://www.cnbeta.com/articles/352357.htm", "title": "\u9b45\u65cf\u5343\u5143\u65b0\u673a\u66dd\u5149\uff1a\u5f88\u6709\u7231_Meizu \u9b45\u65cf_cnBeta.COM"}, 183 | {"url": "http://www.cnbeta.com/articles/354723.htm", "title": "\u300a\u51b0\u96ea\u5947\u7f18\u300b\u5bfc\u6f14\uff1a\u975e\u5e38\u62b1\u6b49\u8ba9\u4f60\u4eec\u8fd8\u5728\u542c\u5b69\u5b50\u4eec\u5531\u300aLet It Go\u300b_\u89c6\u9891\u7f51\u7ad9 / \u5a92\u4f53\u64ad\u653e\u5668_cnBeta.COM"}, 184 | {"url": "http://www.cnbeta.com/articles/354939.htm", "title": "\u534e\u4e3a\u8363\u8000\u7545\u73a94X \u79fb\u52a8/\u8054\u901a/\u5168\u7f51\u901a\u91d1\u8272\u7248\u53d1\u5e03_Huawei \u534e\u4e3a_cnBeta.COM"}, 185 | {"url": "http://www.cnbeta.com/articles/355145.htm", "title": "\u4e03\u9879\u4e0d\u53ef\u601d\u8bae\u79d1\u5b66\u5947\u601d\uff1a\u5b87\u5b99\u6839\u672c\u4e0d\u5e94\u8be5\u5b58\u5728_cnBeta \u79d1\u5b66\u63a2\u7d22_cnBeta.COM"}, 186 | {"url": "http://www.cnbeta.com/articles/354909.htm", "title": "\u5168\u91d1\u5c5e\u4e00\u4f53\u673a\u8eab \u534e\u4e3a\u63a8\u51fa\u8363\u8000T1\u5e73\u677f_Huawei \u534e\u4e3a_cnBeta.COM"}, 187 | {"url": "http://www.cnbeta.com/articles/354337.htm", "title": "\u8c37\u6b4c\u3001\u5fae\u8f6f\u548cAdobe\u64a4\u79bb\u4fc4\u7f57\u65af_Google / \u8c37\u6b4c_cnBeta.COM"}, 188 | {"url": "http://www.cnbeta.com/articles/354937.htm", "title": "\u8363\u80006 Plus\u53d1\u5e03\uff1a\u914d\u6d77\u601d\u516b\u6838\u5904\u7406\u5668 \u540e\u7f6e\u53cc800\u4e07\u6444\u50cf\u5934_Huawei \u534e\u4e3a_cnBeta.COM"}, 189 | {"url": "http://www.cnbeta.com/articles/355019.htm", "title": "[\u53d1\u5e03\u4f1a\u56de\u987e]\u53cc\u773c\u770b\u4e16\u754c \u534e\u4e3a\u8363\u80006 Plus 1999\u5143\u8d77_cnBeta \u73b0\u573a\u62a5\u9053_cnBeta.COM"}, 190 | {"url": "http://www.cnbeta.com/articles/351693.htm", "title": "\u4fc4\u7f57\u65af\u4e00\u540d\u8bae\u5458\u63d0\u8bae \u7acb\u6cd5\u7981\u6b62\u6240\u6709\u8bae\u5458\u7528\u82f9\u679c\u4ea7\u54c1_Apple \u82f9\u679c_cnBeta.COM"}, 191 | {"url": "http://www.cnbeta.com/articles/354841.htm", "title": "\u8def\u900f\u793e\u8bc4\u5c0f\u7c73: \u6bdb\u5229\u7387\u624d1.8%\uff0c\u8bc1\u660e\u6284\u88ad\u82f9\u679c\u8d5a\u4e0d\u5230\u94b1_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"}, 192 | {"url": "http://www.cnbeta.com/articles/354975.htm", "title": "\u5362\u5e03\u5927\u5e45\u8d2c\u503c \u82f9\u679c\u6682\u505c\u4fc4\u7f57\u65af\u5728\u7ebf\u5546\u5e97\u9500\u552e_Apple \u82f9\u679c_cnBeta.COM"}, 193 | {"url": "http://www.cnbeta.com/articles/354895.htm", "title": "\u5c0f\u7c73\u771f\u5b9e\u80a1\u6743\u67b6\u6784\u906d\u610f\u5916\u66dd\u5149\uff1a\u5c0f\u7c73\u4e0e\u5c0f\u7c73\u79d1\u6280\u201c\u5173\u7cfb\u7279\u522b\u201d_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"}, 194 | {"url": "http://www.cnbeta.com/articles/354897.htm", "title": "\u5c0f\u7c73\u624b\u673a\u5728\u5370\u5ea6\u6682\u65f6\u6062\u590d\u9500\u552e \u660e\u5e741\u67088\u65e5\u540e\u5c06\u518d\u6b21\u88ab\u7981_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"}, 195 | {"url": "http://www.cnbeta.com/articles/354949.htm", "title": "\u5362\u5e03\u52a0\u901f\u8d2c\u503c\uff1a\u4fc4\u7f57\u65afiPhone 6\u5168\u7403\u6700\u4fbf\u5b9c_Apple iPhone_cnBeta.COM"}, 196 | {"url": "http://www.cnbeta.com/articles/354985.htm", "title": "\u7231\u7acb\u4fe1\u56de\u5e94\u5c0f\u7c73\u5370\u5ea6\u4e34\u65f6\u89e3\u7981:1\u67088\u65e5\u5c0f\u7c73\u5fc5\u5c06\u8d25\u8bc9_Xiaomi \u5c0f\u7c73\u79d1\u6280_cnBeta.COM"}, 197 | {"url": "http://www.cnbeta.com/articles/355271.htm", "title": "\u90e8\u5206\u79fb\u52a8\u7535\u6e90\u5bb9\u91cf\u865a\u6807 \u91d1\u5c5e\u5916\u58f3\u4ea7\u54c1\u66f4\u5b89\u5168_cnBeta \u89c6\u70b9\u89c2\u5bdf_cnBeta.COM"}] -------------------------------------------------------------------------------- /cnbeta/scrapy.cfg: -------------------------------------------------------------------------------- 1 | # Automatically created by: scrapy startproject 2 | # 3 | # For more information about the [deploy] section see: 4 | # http://doc.scrapy.org/en/latest/topics/scrapyd.html 5 | 6 | [settings] 7 | default = cnbeta.settings 8 | 9 | [deploy] 10 | #url = http://localhost:6800/ 11 | project = cnbeta 12 | -------------------------------------------------------------------------------- /douban/douban/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/douban/douban/__init__.py -------------------------------------------------------------------------------- /douban/douban/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/douban/douban/__init__.pyc -------------------------------------------------------------------------------- /douban/douban/items.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your scraped items 4 | # 5 | # See documentation in: 6 | # http://doc.scrapy.org/en/latest/topics/items.html 7 | 8 | from scrapy.item import Item, Field 9 | 10 | 11 | class DoubanItem(Item): 12 | # define the fields for your item here like: 13 | # name = scrapy.Field() 14 | name = Field() 15 | description = Field() 16 | url = Field() 17 | -------------------------------------------------------------------------------- /douban/douban/items.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/douban/douban/items.pyc -------------------------------------------------------------------------------- /douban/douban/pipelines.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define your item pipelines here 4 | # 5 | # Don't forget to add your pipeline to the ITEM_PIPELINES setting 6 | # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html 7 | 8 | 9 | class DoubanPipeline(object): 10 | def process_item(self, item, spider): 11 | return item 12 | -------------------------------------------------------------------------------- /douban/douban/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Scrapy settings for douban project 4 | # 5 | # For simplicity, this file contains only the most important settings by 6 | # default. All the other settings are documented here: 7 | # 8 | # http://doc.scrapy.org/en/latest/topics/settings.html 9 | # 10 | 11 | BOT_NAME = 'douban' 12 | 13 | SPIDER_MODULES = ['douban.spiders'] 14 | NEWSPIDER_MODULE = 'douban.spiders' 15 | 16 | # Crawl responsibly by identifying yourself (and your website) on the user-agent 17 | #USER_AGENT = 'douban (+http://www.yourdomain.com)' 18 | -------------------------------------------------------------------------------- /douban/douban/settings.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/douban/douban/settings.pyc -------------------------------------------------------------------------------- /douban/douban/spiders/__init__.py: -------------------------------------------------------------------------------- 1 | # This package will contain the spiders of your Scrapy project 2 | # 3 | # Please refer to the documentation for information on how to create and manage 4 | # your spiders. 5 | -------------------------------------------------------------------------------- /douban/douban/spiders/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/douban/douban/spiders/__init__.pyc -------------------------------------------------------------------------------- /douban/douban/spiders/douban_spider.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | """ 4 | 一个简单的Python 爬虫, 用于抓取豆瓣电影Top前250的电影的名称描述等 5 | 6 | Anthor: Andrew Liu 7 | Version: 0.0.3 8 | Date: 2014-12-17 9 | Language: Python2.7.8 10 | Editor: Sublime Text2 11 | Operate: 具体操作请看README.md介绍 12 | """ 13 | 14 | from scrapy.contrib.spiders import CrawlSpider, Rule 15 | from scrapy.selector import Selector 16 | from douban.items import DoubanItem 17 | from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor 18 | # import sys 19 | # reload(sys) 20 | # sys.setdefaultencoding('utf8') 21 | 22 | class DoubanSpider(CrawlSpider) : 23 | 24 | name = "douban" 25 | allowed_domains = ["movie.douban.com"] 26 | start_urls = ["http://movie.douban.com/top250"] 27 | rules = ( 28 | #将所有符合正则表达式的url加入到抓取列表中 29 | Rule(SgmlLinkExtractor(allow = (r'http://movie\.douban\.com/top250\?start=\d+&filter=&type=',))), 30 | #将所有符合正则表达式的url请求后下载网页代码, 形成response后调用自定义回调函数 31 | Rule(SgmlLinkExtractor(allow = (r'http://movie\.douban\.com/subject/\d+', )), callback = 'parse_page', follow = True), 32 | ) 33 | 34 | def parse_page(self, response) : 35 | sel = Selector(response) 36 | item = DoubanItem() 37 | item['name'] = sel.xpath('//h1/span[@property="v:itemreviewed"]/text()').extract() 38 | item['description'] = sel.xpath('//div/span[@property="v:summary"]/text()').extract() 39 | item['url'] = response.url 40 | return item 41 | -------------------------------------------------------------------------------- /douban/douban/spiders/douban_spider.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/douban/douban/spiders/douban_spider.pyc -------------------------------------------------------------------------------- /douban/scrapy.cfg: -------------------------------------------------------------------------------- 1 | # Automatically created by: scrapy startproject 2 | # 3 | # For more information about the [deploy] section see: 4 | # http://doc.scrapy.org/en/latest/topics/scrapyd.html 5 | 6 | [settings] 7 | default = douban.settings 8 | 9 | [deploy] 10 | #url = http://localhost:6800/ 11 | project = douban 12 | -------------------------------------------------------------------------------- /tinydeal/data.csv: -------------------------------------------------------------------------------- 1 | title,url,discounted_price,original_price 2 | 50 x Medical Disposable Face Mouth Mask 95% Filtration CE FDA HHI-582255,https://www.tinydeal.com/50-x-medical-disposable-face-mouth-mask-95-filtration-ce-fda-p-200376.html,$17.50,$33.88 3 | Transparent Anti Droplet Dust-proof Medical Face Shield Face Cover Mask HHI-581539,https://www.tinydeal.com/transparent-anti-droplet-dust-proof-medical-face-shield-face-cover-mask-p-200336.html,$2.79,$3.44 4 | 10 x Transparent Anti Droplet Dust-proof Medical Face Shield Face Cover Mask KB-581546,https://www.tinydeal.com/10-x-transparent-anti-droplet-dust-proof-medical-face-shield-face-cover-mask-p-200338.html,$13.66,$24.33 5 | Mini Airtight Sealer Machine for Plastic Bag - Assorted Color HLI-48626,https://www.tinydeal.com/mini-airtight-sealer-machine-for-plastic-bag-assorted-color-p-39392.html,$2.27,$5.79 6 | GM320 Infrared Thermometer Non-Contact Laser Gun IR Thermometer EDT-528767,https://www.tinydeal.com/gm320-infrared-thermometer-non-contact-laser-gun-ir-thermometer-p-165306.html,$16.77,$18.72 7 | TWS F9 Bluetooth Earphone Touch Wireless Headphones 8D Stereo Sports Earbuds EEP-582910,https://www.tinydeal.com/tws-f9-bluetooth-earphone-touch-wireless-headphones-8d-stereo-sports-earbuds-p-200780.html,$11.93,$23.74 8 | Infrared Non-Contact Forehead Thermometer for Fever Check HHIHE-580934,https://www.tinydeal.com/infrared-non-contact-forehead-thermometer-for-fever-check-p-200328.html,$43.48,$71.04 9 | Smart Bracelet Watch Fitness Bracelet Pedometer Heart Rate Monitor Waterproof E-583049,https://www.tinydeal.com/smart-bracelet-watch-fitness-bracelet-pedometer-heart-rate-monitor-waterproof-p-200803.html,$11.31,$14.70 10 | 3 in 1 Cable Nylon-Coating Fast Charging 8Pin Type-C Micro-USB EPA-530685,https://www.tinydeal.com/3-in-1-cable-nylon-coating-fast-charging-8pin-type-micro-usb-p-165980.html,$2.50,$4.32 11 | 1/2/5/10 pcs Snore Stopper Silicone Nose Clip Sleeping Device BKH-531636,https://www.tinydeal.com/12510-pcs-snore-stopper-silicone-nose-clip-sleeping-device-p-166268.html,$1.22,$3.66 12 | Light Controlled Square LED Wall Night Lamp Bedroom Night Light HLT-506114,https://www.tinydeal.com/light-controlled-square-led-wall-night-lamp-bedroom-night-light-p-158312.html,$2.29,$5.24 13 | Portable Mini LCD Digital Tyre Tire Pressure Gauge Tester CTL-27481,https://www.tinydeal.com/portable-mini-lcd-digital-tyre-tire-pressure-gauge-tester-p-23322.html,$3.35,$3.88 14 | DUBERY Vintage Sunglasses Polarized Men's Sun Glasses For Men S-580871,https://www.tinydeal.com/dubery-vintage-sunglasses-polarized-mens-sun-glasses-for-men-p-200631.html,$6.25,$8.12 15 | Silicone Nipple Cover Stickers Adhesive Bra Reusable Invisible Nipple Cover HHI-550767,https://www.tinydeal.com/silicone-nipple-cover-stickers-adhesive-bra-reusable-invisible-nipple-cover-p-175493.html,$1.97,$2.37 16 | 1 Pair of Silicone Magnetic Losing Weight Toe Rings HBI-13397,https://www.tinydeal.com/1-pair-of-silicone-magnetic-losing-weight-toe-rings-p-12727.html,$1.05,$3.22 17 | Xiaomi Mijia iHealth Thermometer LED Non Contact Digital Infrared Forehead Body RTH-575855,https://www.tinydeal.com/xiaomi-mijia-ihealth-thermometer-led-non-contact-digital-infrared-forehead-body-p-200865.html,$28.26,$36.74 18 | 5 LED Water Resistant USB Rechargeable LED Bike Tail Light RTH-555898,https://www.tinydeal.com/5-led-water-resistant-usb-rechargeable-led-bike-tail-light-p-177336.html,$2.91,$5.24 19 | Teeth Whitening Pen Tooth Gel Whitener Bleach HBI-113105,https://www.tinydeal.com/teeth-whitening-pen-tooth-gel-whitener-bleach-p-66709.html,$2.12,$5.88 20 | Android Robot Shaped Micro USB to USB 2.0 Host OTG Adapter ECACR-487310,https://www.tinydeal.com/android-robot-shaped-micro-usb-to-usb-20-host-otg-adapter-p-157418.html,$1.03,$2.70 21 | Mini Video Capture Card USB 2.0 HDMI Video Grabber Record Box for PS4 Game DVD HHIHE-583000,https://www.tinydeal.com/mini-video-capture-card-usb-20-hdmi-video-grabber-record-box-for-ps4-game-dvd-p-200736.html,$9.69,$21.51 22 | XIAOMI MIJIA Microhoo Mini Air Conditioner Fan Personal Portable USB Air Cooler HHIHE-582526,https://www.tinydeal.com/xiaomi-mijia-microhoo-mini-air-conditioner-fan-personal-portable-usb-air-cooler-p-200948.html,$60.02,$72.02 23 | Funlife 12pcs / set Green Crystal Three-dimensional Wall Stickers Mirror HHI-562368,https://www.tinydeal.com/funlife-12pcs-set-green-crystal-three-dimensional-wall-stickers-mirror-p-179148.html,$2.26,$6.79 24 | IP67 P8 Smart Watch Wristband Men Women Monitor Smartwatch Tracker For Phone E-580457,https://www.tinydeal.com/ip67-p8-smart-watch-wristband-men-women-monitor-smartwatch-tracker-for-phone-p-200649.html,$18.52,$36.34 25 | Detachable Baseball Protective Clear Face mask Cap Anti-Splash Hat S-581830,https://www.tinydeal.com/detachable-baseball-protective-clear-face-mask-cap-anti-splash-hat-p-200362.html,$3.06,$3.69 26 | T10 Bluetooth Headsets Wireless Earbuds 5.0 TWS Earphone EEP-581913,https://www.tinydeal.com/t10-bluetooth-headsets-wireless-earbuds-50-tws-earphone-p-200802.html,$13.21,$25.40 27 | Fsat Quick Charge 3.0 4.0 QC3.0 Fast Charging Tablet Wall Adapter HHITH-582324,https://www.tinydeal.com/fsat-quick-charge-30-40-qc30-fast-charging-tablet-wall-adapter-p-201049.html,$3.70,$6.67 28 | Multifunction USB 2.0 M2/ TF Card Reader w/ Lanyard CCR-85173,https://www.tinydeal.com/multifunction-usb-20-m2-tf-card-reader-w-lanyard-p-57252.html,$1.55,$1.66 29 | 0.25W Intelligent Cabinet Sensor Lights HHI-546845,https://www.tinydeal.com/025w-intelligent-cabinet-sensor-lights-p-174242.html,$1.93,$5.22 30 | 8mm Intelligent Thermometer Temperature Measuring Ring HHI-566384,https://www.tinydeal.com/8mm-intelligent-thermometer-temperature-measuring-ring-p-181149.html,$1.48,$6.57 31 | 3.5mm Stereo Woven Fiber Cloth Line Headset EDT-556403,https://www.tinydeal.com/35mm-stereo-woven-fiber-cloth-line-headset-p-178024.html,$1.54,$4.85 32 | 90*30cm Ice Towel Instant Cooling Towel Heat Relief Reusable Chill Cool Towel HHI-549301,https://www.tinydeal.com/9030cm-ice-towel-instant-cooling-towel-heat-relief-reusable-chill-cool-towel-p-174936.html,$2.17,$2.60 33 | DUBERY Polarized Sunglasses Men's Driving Shades Male Sun Glasses S-580864,https://www.tinydeal.com/dubery-polarized-sunglasses-mens-driving-shades-male-sun-glasses-p-200630.html,$5.83,$15.03 34 | DC 12V 45cm 45-SMD 1210 LED Flexible Waterproof Car Strip Light RLT-274929,https://www.tinydeal.com/dc-12v-45cm-45-smd-1210-led-flexible-waterproof-car-strip-light-p-118388.html,$1.27,$2.28 35 | 2.4GHz 1600dpi Wireless Optical Mouse Mice CMS-11833-C3,https://www.tinydeal.com/24ghz-1600dpi-wireless-optical-mouse-mice-p-168444.html,$3.54,$6.79 36 | Baellerry Men Wallets PU Leather Solid Luxury Slim Short Credit Card Holder HHI-582256,https://www.tinydeal.com/baellerry-men-wallets-pu-leather-solid-luxury-slim-short-credit-card-holder-p-200737.html,$3.68,$20.26 37 | Car Paint Scratches Repair Pen Brush Waterproof Paint Marker Pen Car Maintain RTH-571469,https://www.tinydeal.com/car-paint-scratches-repair-pen-brush-waterproof-paint-marker-pen-car-maintain-p-182211.html,$1.57,$4.82 38 | Anti-droplets Hat Detachable Design Clear Facial Mask Outdoor Protector Sun Hat EEP-583095,https://www.tinydeal.com/anti-droplets-hat-detachable-design-clear-facial-mask-outdoor-protector-sun-hat-p-200783.html,$6.94,$10.95 39 | S03 Smart Watch Waterproof Activity Tracker Smart Band Bracelet E-582866,https://www.tinydeal.com/s03-smart-watch-waterproof-activity-tracker-smart-band-bracelet-p-200781.html,$11.81,$23.74 40 | Universal 6'' Phone Lanyard Necklace & Wrist Strap Card Bag Holder w Lanyard HHITH-543753,https://www.tinydeal.com/universal-6-phone-lanyard-necklace-wrist-strap-card-bag-holder-w-lanyard-p-172964.html,$2.05,$3.06 41 | Hook Clip-on Rear Bike Seat Light Warning Lamp STH-548943,https://www.tinydeal.com/hook-clip-on-rear-bike-seat-light-warning-lamp-p-174917.html,$2.59,$14.35 42 | NAVIFORCE NF9161 Men's Watches Top Luxury Brand Analog Stainless Steel WWT-580848,https://www.tinydeal.com/naviforce-nf9161-mens-watches-top-luxury-brand-analog-stainless-steel-p-200814.html,$18.58,$33.44 43 | Stainless Steel Vacuum Sealed Red Wine Bottle Stopper Spout HKI-507455,https://www.tinydeal.com/stainless-steel-vacuum-sealed-red-wine-bottle-stopper-spout-p-158461.html,$2.56,$2.91 44 | Xiaomi Turok Steinhardt TS Brand Polarized Stainless Mirror Lenses Sunglasses S-582106,https://www.tinydeal.com/xiaomi-turok-steinhardt-ts-brand-polarized-stainless-mirror-lenses-sunglasses-p-200936.html,$19.99,$33.98 45 | Outdoor Lights Solar Powered LED Wall Lamp Porch Lights Sensor Motion Sensor HSI-582757,https://www.tinydeal.com/outdoor-lights-solar-powered-led-wall-lamp-porch-lights-sensor-motion-sensor-p-201016.html,$8.07,$14.53 46 | 2in1 Electric Mosquito Swatter Dispeller /Mosquito Killer Lamp USB Charging HHIHE-582871,https://www.tinydeal.com/2in1-electric-mosquito-swatter-dispeller-mosquito-killer-lamp-usb-charging-p-200982.html,$27.45,$49.40 47 | Xiaomi 3D Men Electric Shaver Razor Enchen BlackStone3 IPX7 Waterproof HHI-580937,https://www.tinydeal.com/xiaomi-3d-men-electric-shaver-razor-enchen-blackstone3-ipx7-waterproof-p-200797.html,$23.99,$28.79 48 | Waterproof Reusable Rain Shoes Covers Rubber Slip-Resistant Rain Boot Overshoes S-557900,https://www.tinydeal.com/waterproof-reusable-rain-shoes-covers-rubber-slip-resistant-rain-boot-overshoes-p-179875.html,$3.61,$6.79 49 | 1/2 pcs Mini Camera SQ11 Night Vision FOV140 1080P DV Video Recorder ECM-533708,https://www.tinydeal.com/12-pcs-mini-camera-sq11-night-vision-fov140-1080p-dv-video-recorder-p-168604.html,$7.53,$12.04 50 | Men Shoulder Bag iPad Large PU Leather Crossbody Business Travel Handbag HHI-583042,https://www.tinydeal.com/men-shoulder-bag-ipad-large-pu-leather-crossbody-business-travel-handbag-p-200788.html,$18.89,$36.54 51 | Neoprene Short Beach Socks Diving Socks Non-Slip Antiskid Scuba Dive Boots SGD-548443,https://www.tinydeal.com/neoprene-short-beach-socks-diving-socks-non-slip-antiskid-scuba-dive-boots-p-174657.html,$3.37,$3.76 52 | 2PCs Wristbands Sport Sweatband Hand Band Sweat Wrist Support Brace Wrist Guards S-549945,https://www.tinydeal.com/2pcs-wristbands-sport-sweatband-hand-band-sweat-wrist-support-brace-wrist-guards-p-175242.html,$2.04,$3.68 53 | Silicone Stretch Lids Food Saver Covers Set – 6pcs HKI-550666,https://www.tinydeal.com/silicone-stretch-lids-food-saver-covers-set-–-6pcs-p-175450.html,$4.86,$6.79 54 | Smart Whistle LED Key Finder Ring Chain Mini Light HHI-521830,https://www.tinydeal.com/smart-whistle-led-key-finder-ring-chain-mini-light-p-162491.html,$1.77,$1.93 55 | X96Q Android 10.0 TV Box Quad Core Allwinner H313 4K TV Box 2GB RAM 16GB ROM E-581778,https://www.tinydeal.com/x96q-android-100-tv-box-quad-core-allwinner-h313-4k-tv-box-2gb-ram-16gb-rom-p-201107.html,$30.79,$43.10 56 | Fascia Gun Muscle Massage Gun Fitness Decompose Lactic Acid Relief Pain Relax HHI-581942,https://www.tinydeal.com/fascia-gun-muscle-massage-gun-fitness-decompose-lactic-acid-relief-pain-relax-p-200720.html,$48.45,$87.21 57 | Compression Socks for Man Woman STH-563907,https://www.tinydeal.com/compression-socks-for-man-woman-p-180225.html,$2.72,$6.38 58 | -------------------------------------------------------------------------------- /tinydeal/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "title": "50 x Medical Disposable Face Mouth Mask 95% Filtration CE FDA HHI-582255", 4 | "url": "https://www.tinydeal.com/50-x-medical-disposable-face-mouth-mask-95-filtration-ce-fda-p-200376.html", 5 | "discounted_price": "$17.50", 6 | "original_price": "$33.88 " 7 | }, 8 | { 9 | "title": "Transparent Anti Droplet Dust-proof Medical Face Shield Face Cover Mask HHI-581539", 10 | "url": "https://www.tinydeal.com/transparent-anti-droplet-dust-proof-medical-face-shield-face-cover-mask-p-200336.html", 11 | "discounted_price": "$2.79", 12 | "original_price": "$3.44 " 13 | }, 14 | { 15 | "title": "10 x Transparent Anti Droplet Dust-proof Medical Face Shield Face Cover Mask KB-581546", 16 | "url": "https://www.tinydeal.com/10-x-transparent-anti-droplet-dust-proof-medical-face-shield-face-cover-mask-p-200338.html", 17 | "discounted_price": "$13.66", 18 | "original_price": "$24.33 " 19 | }, 20 | { 21 | "title": "Mini Airtight Sealer Machine for Plastic Bag - Assorted Color HLI-48626", 22 | "url": "https://www.tinydeal.com/mini-airtight-sealer-machine-for-plastic-bag-assorted-color-p-39392.html", 23 | "discounted_price": "$2.27", 24 | "original_price": "$5.79 " 25 | }, 26 | { 27 | "title": "GM320 Infrared Thermometer Non-Contact Laser Gun IR Thermometer EDT-528767", 28 | "url": "https://www.tinydeal.com/gm320-infrared-thermometer-non-contact-laser-gun-ir-thermometer-p-165306.html", 29 | "discounted_price": "$16.77", 30 | "original_price": "$18.72 " 31 | }, 32 | { 33 | "title": "TWS F9 Bluetooth Earphone Touch Wireless Headphones 8D Stereo Sports Earbuds EEP-582910", 34 | "url": "https://www.tinydeal.com/tws-f9-bluetooth-earphone-touch-wireless-headphones-8d-stereo-sports-earbuds-p-200780.html", 35 | "discounted_price": "$11.93", 36 | "original_price": "$23.74 " 37 | }, 38 | { 39 | "title": "Infrared Non-Contact Forehead Thermometer for Fever Check HHIHE-580934", 40 | "url": "https://www.tinydeal.com/infrared-non-contact-forehead-thermometer-for-fever-check-p-200328.html", 41 | "discounted_price": "$43.48", 42 | "original_price": "$71.04 " 43 | }, 44 | { 45 | "title": "Smart Bracelet Watch Fitness Bracelet Pedometer Heart Rate Monitor Waterproof E-583049", 46 | "url": "https://www.tinydeal.com/smart-bracelet-watch-fitness-bracelet-pedometer-heart-rate-monitor-waterproof-p-200803.html", 47 | "discounted_price": "$11.31", 48 | "original_price": "$14.70 " 49 | }, 50 | { 51 | "title": "3 in 1 Cable Nylon-Coating Fast Charging 8Pin Type-C Micro-USB EPA-530685", 52 | "url": "https://www.tinydeal.com/3-in-1-cable-nylon-coating-fast-charging-8pin-type-micro-usb-p-165980.html", 53 | "discounted_price": "$2.50", 54 | "original_price": "$4.32 " 55 | }, 56 | { 57 | "title": "1/2/5/10 pcs Snore Stopper Silicone Nose Clip Sleeping Device BKH-531636", 58 | "url": "https://www.tinydeal.com/12510-pcs-snore-stopper-silicone-nose-clip-sleeping-device-p-166268.html", 59 | "discounted_price": "$1.22", 60 | "original_price": "$3.66 " 61 | }, 62 | { 63 | "title": "Light Controlled Square LED Wall Night Lamp Bedroom Night Light HLT-506114", 64 | "url": "https://www.tinydeal.com/light-controlled-square-led-wall-night-lamp-bedroom-night-light-p-158312.html", 65 | "discounted_price": "$2.29", 66 | "original_price": "$5.24 " 67 | }, 68 | { 69 | "title": "Portable Mini LCD Digital Tyre Tire Pressure Gauge Tester CTL-27481", 70 | "url": "https://www.tinydeal.com/portable-mini-lcd-digital-tyre-tire-pressure-gauge-tester-p-23322.html", 71 | "discounted_price": "$3.35", 72 | "original_price": "$3.88 " 73 | }, 74 | { 75 | "title": "DUBERY Vintage Sunglasses Polarized Men's Sun Glasses For Men S-580871", 76 | "url": "https://www.tinydeal.com/dubery-vintage-sunglasses-polarized-mens-sun-glasses-for-men-p-200631.html", 77 | "discounted_price": "$6.25", 78 | "original_price": "$8.12 " 79 | }, 80 | { 81 | "title": "Silicone Nipple Cover Stickers Adhesive Bra Reusable Invisible Nipple Cover HHI-550767", 82 | "url": "https://www.tinydeal.com/silicone-nipple-cover-stickers-adhesive-bra-reusable-invisible-nipple-cover-p-175493.html", 83 | "discounted_price": "$1.97", 84 | "original_price": "$2.37 " 85 | }, 86 | { 87 | "title": "1 Pair of Silicone Magnetic Losing Weight Toe Rings HBI-13397", 88 | "url": "https://www.tinydeal.com/1-pair-of-silicone-magnetic-losing-weight-toe-rings-p-12727.html", 89 | "discounted_price": "$1.05", 90 | "original_price": "$3.22 " 91 | }, 92 | { 93 | "title": "Xiaomi Mijia iHealth Thermometer LED Non Contact Digital Infrared Forehead Body RTH-575855", 94 | "url": "https://www.tinydeal.com/xiaomi-mijia-ihealth-thermometer-led-non-contact-digital-infrared-forehead-body-p-200865.html", 95 | "discounted_price": "$28.26", 96 | "original_price": "$36.74 " 97 | }, 98 | { 99 | "title": "5 LED Water Resistant USB Rechargeable LED Bike Tail Light RTH-555898", 100 | "url": "https://www.tinydeal.com/5-led-water-resistant-usb-rechargeable-led-bike-tail-light-p-177336.html", 101 | "discounted_price": "$2.91", 102 | "original_price": "$5.24 " 103 | }, 104 | { 105 | "title": "Teeth Whitening Pen Tooth Gel Whitener Bleach HBI-113105", 106 | "url": "https://www.tinydeal.com/teeth-whitening-pen-tooth-gel-whitener-bleach-p-66709.html", 107 | "discounted_price": "$2.12", 108 | "original_price": "$5.88 " 109 | }, 110 | { 111 | "title": "Android Robot Shaped Micro USB to USB 2.0 Host OTG Adapter ECACR-487310", 112 | "url": "https://www.tinydeal.com/android-robot-shaped-micro-usb-to-usb-20-host-otg-adapter-p-157418.html", 113 | "discounted_price": "$1.03", 114 | "original_price": "$2.70 " 115 | }, 116 | { 117 | "title": "Mini Video Capture Card USB 2.0 HDMI Video Grabber Record Box for PS4 Game DVD HHIHE-583000", 118 | "url": "https://www.tinydeal.com/mini-video-capture-card-usb-20-hdmi-video-grabber-record-box-for-ps4-game-dvd-p-200736.html", 119 | "discounted_price": "$9.69", 120 | "original_price": "$21.51 " 121 | }, 122 | { 123 | "title": "XIAOMI MIJIA Microhoo Mini Air Conditioner Fan Personal Portable USB Air Cooler HHIHE-582526", 124 | "url": "https://www.tinydeal.com/xiaomi-mijia-microhoo-mini-air-conditioner-fan-personal-portable-usb-air-cooler-p-200948.html", 125 | "discounted_price": "$60.02", 126 | "original_price": "$72.02 " 127 | }, 128 | { 129 | "title": "Funlife 12pcs / set Green Crystal Three-dimensional Wall Stickers Mirror HHI-562368", 130 | "url": "https://www.tinydeal.com/funlife-12pcs-set-green-crystal-three-dimensional-wall-stickers-mirror-p-179148.html", 131 | "discounted_price": "$2.26", 132 | "original_price": "$6.79 " 133 | }, 134 | { 135 | "title": "IP67 P8 Smart Watch Wristband Men Women Monitor Smartwatch Tracker For Phone E-580457", 136 | "url": "https://www.tinydeal.com/ip67-p8-smart-watch-wristband-men-women-monitor-smartwatch-tracker-for-phone-p-200649.html", 137 | "discounted_price": "$18.52", 138 | "original_price": "$36.34 " 139 | }, 140 | { 141 | "title": "Detachable Baseball Protective Clear Face mask Cap Anti-Splash Hat S-581830", 142 | "url": "https://www.tinydeal.com/detachable-baseball-protective-clear-face-mask-cap-anti-splash-hat-p-200362.html", 143 | "discounted_price": "$3.06", 144 | "original_price": "$3.69 " 145 | }, 146 | { 147 | "title": "T10 Bluetooth Headsets Wireless Earbuds 5.0 TWS Earphone EEP-581913", 148 | "url": "https://www.tinydeal.com/t10-bluetooth-headsets-wireless-earbuds-50-tws-earphone-p-200802.html", 149 | "discounted_price": "$13.21", 150 | "original_price": "$25.40 " 151 | }, 152 | { 153 | "title": "Fsat Quick Charge 3.0 4.0 QC3.0 Fast Charging Tablet Wall Adapter HHITH-582324", 154 | "url": "https://www.tinydeal.com/fsat-quick-charge-30-40-qc30-fast-charging-tablet-wall-adapter-p-201049.html", 155 | "discounted_price": "$3.70", 156 | "original_price": "$6.67 " 157 | }, 158 | { 159 | "title": "Multifunction USB 2.0 M2/ TF Card Reader w/ Lanyard CCR-85173", 160 | "url": "https://www.tinydeal.com/multifunction-usb-20-m2-tf-card-reader-w-lanyard-p-57252.html", 161 | "discounted_price": "$1.55", 162 | "original_price": "$1.66 " 163 | }, 164 | { 165 | "title": "0.25W Intelligent Cabinet Sensor Lights HHI-546845", 166 | "url": "https://www.tinydeal.com/025w-intelligent-cabinet-sensor-lights-p-174242.html", 167 | "discounted_price": "$1.93", 168 | "original_price": "$5.22 " 169 | }, 170 | { 171 | "title": "8mm Intelligent Thermometer Temperature Measuring Ring HHI-566384", 172 | "url": "https://www.tinydeal.com/8mm-intelligent-thermometer-temperature-measuring-ring-p-181149.html", 173 | "discounted_price": "$1.48", 174 | "original_price": "$6.57 " 175 | }, 176 | { 177 | "title": "3.5mm Stereo Woven Fiber Cloth Line Headset EDT-556403", 178 | "url": "https://www.tinydeal.com/35mm-stereo-woven-fiber-cloth-line-headset-p-178024.html", 179 | "discounted_price": "$1.54", 180 | "original_price": "$4.85 " 181 | }, 182 | { 183 | "title": "90*30cm Ice Towel Instant Cooling Towel Heat Relief Reusable Chill Cool Towel HHI-549301", 184 | "url": "https://www.tinydeal.com/9030cm-ice-towel-instant-cooling-towel-heat-relief-reusable-chill-cool-towel-p-174936.html", 185 | "discounted_price": "$2.17", 186 | "original_price": "$2.60 " 187 | }, 188 | { 189 | "title": "DUBERY Polarized Sunglasses Men's Driving Shades Male Sun Glasses S-580864", 190 | "url": "https://www.tinydeal.com/dubery-polarized-sunglasses-mens-driving-shades-male-sun-glasses-p-200630.html", 191 | "discounted_price": "$5.83", 192 | "original_price": "$15.03 " 193 | }, 194 | { 195 | "title": "DC 12V 45cm 45-SMD 1210 LED Flexible Waterproof Car Strip Light RLT-274929", 196 | "url": "https://www.tinydeal.com/dc-12v-45cm-45-smd-1210-led-flexible-waterproof-car-strip-light-p-118388.html", 197 | "discounted_price": "$1.27", 198 | "original_price": "$2.28 " 199 | }, 200 | { 201 | "title": "2.4GHz 1600dpi Wireless Optical Mouse Mice CMS-11833-C3", 202 | "url": "https://www.tinydeal.com/24ghz-1600dpi-wireless-optical-mouse-mice-p-168444.html", 203 | "discounted_price": "$3.54", 204 | "original_price": "$6.79 " 205 | }, 206 | { 207 | "title": "Baellerry Men Wallets PU Leather Solid Luxury Slim Short Credit Card Holder HHI-582256", 208 | "url": "https://www.tinydeal.com/baellerry-men-wallets-pu-leather-solid-luxury-slim-short-credit-card-holder-p-200737.html", 209 | "discounted_price": "$3.68", 210 | "original_price": "$20.26 " 211 | }, 212 | { 213 | "title": "Car Paint Scratches Repair Pen Brush Waterproof Paint Marker Pen Car Maintain RTH-571469", 214 | "url": "https://www.tinydeal.com/car-paint-scratches-repair-pen-brush-waterproof-paint-marker-pen-car-maintain-p-182211.html", 215 | "discounted_price": "$1.57", 216 | "original_price": "$4.82 " 217 | }, 218 | { 219 | "title": "Anti-droplets Hat Detachable Design Clear Facial Mask Outdoor Protector Sun Hat EEP-583095", 220 | "url": "https://www.tinydeal.com/anti-droplets-hat-detachable-design-clear-facial-mask-outdoor-protector-sun-hat-p-200783.html", 221 | "discounted_price": "$6.94", 222 | "original_price": "$10.95 " 223 | }, 224 | { 225 | "title": "S03 Smart Watch Waterproof Activity Tracker Smart Band Bracelet E-582866", 226 | "url": "https://www.tinydeal.com/s03-smart-watch-waterproof-activity-tracker-smart-band-bracelet-p-200781.html", 227 | "discounted_price": "$11.81", 228 | "original_price": "$23.74 " 229 | }, 230 | { 231 | "title": "Universal 6'' Phone Lanyard Necklace & Wrist Strap Card Bag Holder w Lanyard HHITH-543753", 232 | "url": "https://www.tinydeal.com/universal-6-phone-lanyard-necklace-wrist-strap-card-bag-holder-w-lanyard-p-172964.html", 233 | "discounted_price": "$2.05", 234 | "original_price": "$3.06 " 235 | }, 236 | { 237 | "title": "Hook Clip-on Rear Bike Seat Light Warning Lamp STH-548943", 238 | "url": "https://www.tinydeal.com/hook-clip-on-rear-bike-seat-light-warning-lamp-p-174917.html", 239 | "discounted_price": "$2.59", 240 | "original_price": "$14.35 " 241 | }, 242 | { 243 | "title": "NAVIFORCE NF9161 Men's Watches Top Luxury Brand Analog Stainless Steel WWT-580848", 244 | "url": "https://www.tinydeal.com/naviforce-nf9161-mens-watches-top-luxury-brand-analog-stainless-steel-p-200814.html", 245 | "discounted_price": "$18.58", 246 | "original_price": "$33.44 " 247 | }, 248 | { 249 | "title": "Stainless Steel Vacuum Sealed Red Wine Bottle Stopper Spout HKI-507455", 250 | "url": "https://www.tinydeal.com/stainless-steel-vacuum-sealed-red-wine-bottle-stopper-spout-p-158461.html", 251 | "discounted_price": "$2.56", 252 | "original_price": "$2.91 " 253 | }, 254 | { 255 | "title": "Xiaomi Turok Steinhardt TS Brand Polarized Stainless Mirror Lenses Sunglasses S-582106", 256 | "url": "https://www.tinydeal.com/xiaomi-turok-steinhardt-ts-brand-polarized-stainless-mirror-lenses-sunglasses-p-200936.html", 257 | "discounted_price": "$19.99", 258 | "original_price": "$33.98 " 259 | }, 260 | { 261 | "title": "Outdoor Lights Solar Powered LED Wall Lamp Porch Lights Sensor Motion Sensor HSI-582757", 262 | "url": "https://www.tinydeal.com/outdoor-lights-solar-powered-led-wall-lamp-porch-lights-sensor-motion-sensor-p-201016.html", 263 | "discounted_price": "$8.07", 264 | "original_price": "$14.53 " 265 | }, 266 | { 267 | "title": "2in1 Electric Mosquito Swatter Dispeller /Mosquito Killer Lamp USB Charging HHIHE-582871", 268 | "url": "https://www.tinydeal.com/2in1-electric-mosquito-swatter-dispeller-mosquito-killer-lamp-usb-charging-p-200982.html", 269 | "discounted_price": "$27.45", 270 | "original_price": "$49.40 " 271 | }, 272 | { 273 | "title": "Xiaomi 3D Men Electric Shaver Razor Enchen BlackStone3 IPX7 Waterproof HHI-580937", 274 | "url": "https://www.tinydeal.com/xiaomi-3d-men-electric-shaver-razor-enchen-blackstone3-ipx7-waterproof-p-200797.html", 275 | "discounted_price": "$23.99", 276 | "original_price": "$28.79 " 277 | }, 278 | { 279 | "title": "Waterproof Reusable Rain Shoes Covers Rubber Slip-Resistant Rain Boot Overshoes S-557900", 280 | "url": "https://www.tinydeal.com/waterproof-reusable-rain-shoes-covers-rubber-slip-resistant-rain-boot-overshoes-p-179875.html", 281 | "discounted_price": "$3.61", 282 | "original_price": "$6.79 " 283 | }, 284 | { 285 | "title": "1/2 pcs Mini Camera SQ11 Night Vision FOV140 1080P DV Video Recorder ECM-533708", 286 | "url": "https://www.tinydeal.com/12-pcs-mini-camera-sq11-night-vision-fov140-1080p-dv-video-recorder-p-168604.html", 287 | "discounted_price": "$7.53", 288 | "original_price": "$12.04 " 289 | }, 290 | { 291 | "title": "Men Shoulder Bag iPad Large PU Leather Crossbody Business Travel Handbag HHI-583042", 292 | "url": "https://www.tinydeal.com/men-shoulder-bag-ipad-large-pu-leather-crossbody-business-travel-handbag-p-200788.html", 293 | "discounted_price": "$18.89", 294 | "original_price": "$36.54 " 295 | }, 296 | { 297 | "title": "Neoprene Short Beach Socks Diving Socks Non-Slip Antiskid Scuba Dive Boots SGD-548443", 298 | "url": "https://www.tinydeal.com/neoprene-short-beach-socks-diving-socks-non-slip-antiskid-scuba-dive-boots-p-174657.html", 299 | "discounted_price": "$3.37", 300 | "original_price": "$3.76 " 301 | }, 302 | { 303 | "title": "2PCs Wristbands Sport Sweatband Hand Band Sweat Wrist Support Brace Wrist Guards S-549945", 304 | "url": "https://www.tinydeal.com/2pcs-wristbands-sport-sweatband-hand-band-sweat-wrist-support-brace-wrist-guards-p-175242.html", 305 | "discounted_price": "$2.04", 306 | "original_price": "$3.68 " 307 | }, 308 | { 309 | "title": "Silicone Stretch Lids Food Saver Covers Set – 6pcs HKI-550666", 310 | "url": "https://www.tinydeal.com/silicone-stretch-lids-food-saver-covers-set-–-6pcs-p-175450.html", 311 | "discounted_price": "$4.86", 312 | "original_price": "$6.79 " 313 | }, 314 | { 315 | "title": "Smart Whistle LED Key Finder Ring Chain Mini Light HHI-521830", 316 | "url": "https://www.tinydeal.com/smart-whistle-led-key-finder-ring-chain-mini-light-p-162491.html", 317 | "discounted_price": "$1.77", 318 | "original_price": "$1.93 " 319 | }, 320 | { 321 | "title": "X96Q Android 10.0 TV Box Quad Core Allwinner H313 4K TV Box 2GB RAM 16GB ROM E-581778", 322 | "url": "https://www.tinydeal.com/x96q-android-100-tv-box-quad-core-allwinner-h313-4k-tv-box-2gb-ram-16gb-rom-p-201107.html", 323 | "discounted_price": "$30.79", 324 | "original_price": "$43.10 " 325 | }, 326 | { 327 | "title": "Fascia Gun Muscle Massage Gun Fitness Decompose Lactic Acid Relief Pain Relax HHI-581942", 328 | "url": "https://www.tinydeal.com/fascia-gun-muscle-massage-gun-fitness-decompose-lactic-acid-relief-pain-relax-p-200720.html", 329 | "discounted_price": "$48.45", 330 | "original_price": "$87.21 " 331 | }, 332 | { 333 | "title": "Compression Socks for Man Woman STH-563907", 334 | "url": "https://www.tinydeal.com/compression-socks-for-man-woman-p-180225.html", 335 | "discounted_price": "$2.72", 336 | "original_price": "$6.38 " 337 | } 338 | ] -------------------------------------------------------------------------------- /tinydeal/data.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 50 x Medical Disposable Face Mouth Mask 95% Filtration CE FDA HHI-582255https://www.tinydeal.com/50-x-medical-disposable-face-mouth-mask-95-filtration-ce-fda-p-200376.html$17.50$33.88 4 | Transparent Anti Droplet Dust-proof Medical Face Shield Face Cover Mask HHI-581539https://www.tinydeal.com/transparent-anti-droplet-dust-proof-medical-face-shield-face-cover-mask-p-200336.html$2.79$3.44 5 | 10 x Transparent Anti Droplet Dust-proof Medical Face Shield Face Cover Mask KB-581546https://www.tinydeal.com/10-x-transparent-anti-droplet-dust-proof-medical-face-shield-face-cover-mask-p-200338.html$13.66$24.33 6 | Mini Airtight Sealer Machine for Plastic Bag - Assorted Color HLI-48626https://www.tinydeal.com/mini-airtight-sealer-machine-for-plastic-bag-assorted-color-p-39392.html$2.27$5.79 7 | GM320 Infrared Thermometer Non-Contact Laser Gun IR Thermometer EDT-528767https://www.tinydeal.com/gm320-infrared-thermometer-non-contact-laser-gun-ir-thermometer-p-165306.html$16.77$18.72 8 | TWS F9 Bluetooth Earphone Touch Wireless Headphones 8D Stereo Sports Earbuds EEP-582910https://www.tinydeal.com/tws-f9-bluetooth-earphone-touch-wireless-headphones-8d-stereo-sports-earbuds-p-200780.html$11.93$23.74 9 | Infrared Non-Contact Forehead Thermometer for Fever Check HHIHE-580934https://www.tinydeal.com/infrared-non-contact-forehead-thermometer-for-fever-check-p-200328.html$43.48$71.04 10 | Smart Bracelet Watch Fitness Bracelet Pedometer Heart Rate Monitor Waterproof E-583049https://www.tinydeal.com/smart-bracelet-watch-fitness-bracelet-pedometer-heart-rate-monitor-waterproof-p-200803.html$11.31$14.70 11 | 3 in 1 Cable Nylon-Coating Fast Charging 8Pin Type-C Micro-USB EPA-530685https://www.tinydeal.com/3-in-1-cable-nylon-coating-fast-charging-8pin-type-micro-usb-p-165980.html$2.50$4.32 12 | 1/2/5/10 pcs Snore Stopper Silicone Nose Clip Sleeping Device BKH-531636https://www.tinydeal.com/12510-pcs-snore-stopper-silicone-nose-clip-sleeping-device-p-166268.html$1.22$3.66 13 | Light Controlled Square LED Wall Night Lamp Bedroom Night Light HLT-506114https://www.tinydeal.com/light-controlled-square-led-wall-night-lamp-bedroom-night-light-p-158312.html$2.29$5.24 14 | Portable Mini LCD Digital Tyre Tire Pressure Gauge Tester CTL-27481https://www.tinydeal.com/portable-mini-lcd-digital-tyre-tire-pressure-gauge-tester-p-23322.html$3.35$3.88 15 | DUBERY Vintage Sunglasses Polarized Men's Sun Glasses For Men S-580871https://www.tinydeal.com/dubery-vintage-sunglasses-polarized-mens-sun-glasses-for-men-p-200631.html$6.25$8.12 16 | Silicone Nipple Cover Stickers Adhesive Bra Reusable Invisible Nipple Cover HHI-550767https://www.tinydeal.com/silicone-nipple-cover-stickers-adhesive-bra-reusable-invisible-nipple-cover-p-175493.html$1.97$2.37 17 | 1 Pair of Silicone Magnetic Losing Weight Toe Rings HBI-13397https://www.tinydeal.com/1-pair-of-silicone-magnetic-losing-weight-toe-rings-p-12727.html$1.05$3.22 18 | Xiaomi Mijia iHealth Thermometer LED Non Contact Digital Infrared Forehead Body RTH-575855https://www.tinydeal.com/xiaomi-mijia-ihealth-thermometer-led-non-contact-digital-infrared-forehead-body-p-200865.html$28.26$36.74 19 | 5 LED Water Resistant USB Rechargeable LED Bike Tail Light RTH-555898https://www.tinydeal.com/5-led-water-resistant-usb-rechargeable-led-bike-tail-light-p-177336.html$2.91$5.24 20 | Teeth Whitening Pen Tooth Gel Whitener Bleach HBI-113105https://www.tinydeal.com/teeth-whitening-pen-tooth-gel-whitener-bleach-p-66709.html$2.12$5.88 21 | Android Robot Shaped Micro USB to USB 2.0 Host OTG Adapter ECACR-487310https://www.tinydeal.com/android-robot-shaped-micro-usb-to-usb-20-host-otg-adapter-p-157418.html$1.03$2.70 22 | Mini Video Capture Card USB 2.0 HDMI Video Grabber Record Box for PS4 Game DVD HHIHE-583000https://www.tinydeal.com/mini-video-capture-card-usb-20-hdmi-video-grabber-record-box-for-ps4-game-dvd-p-200736.html$9.69$21.51 23 | XIAOMI MIJIA Microhoo Mini Air Conditioner Fan Personal Portable USB Air Cooler HHIHE-582526https://www.tinydeal.com/xiaomi-mijia-microhoo-mini-air-conditioner-fan-personal-portable-usb-air-cooler-p-200948.html$60.02$72.02 24 | Funlife 12pcs / set Green Crystal Three-dimensional Wall Stickers Mirror HHI-562368https://www.tinydeal.com/funlife-12pcs-set-green-crystal-three-dimensional-wall-stickers-mirror-p-179148.html$2.26$6.79 25 | IP67 P8 Smart Watch Wristband Men Women Monitor Smartwatch Tracker For Phone E-580457https://www.tinydeal.com/ip67-p8-smart-watch-wristband-men-women-monitor-smartwatch-tracker-for-phone-p-200649.html$18.52$36.34 26 | Detachable Baseball Protective Clear Face mask Cap Anti-Splash Hat S-581830https://www.tinydeal.com/detachable-baseball-protective-clear-face-mask-cap-anti-splash-hat-p-200362.html$3.06$3.69 27 | T10 Bluetooth Headsets Wireless Earbuds 5.0 TWS Earphone EEP-581913https://www.tinydeal.com/t10-bluetooth-headsets-wireless-earbuds-50-tws-earphone-p-200802.html$13.21$25.40 28 | Fsat Quick Charge 3.0 4.0 QC3.0 Fast Charging Tablet Wall Adapter HHITH-582324https://www.tinydeal.com/fsat-quick-charge-30-40-qc30-fast-charging-tablet-wall-adapter-p-201049.html$3.70$6.67 29 | Multifunction USB 2.0 M2/ TF Card Reader w/ Lanyard CCR-85173https://www.tinydeal.com/multifunction-usb-20-m2-tf-card-reader-w-lanyard-p-57252.html$1.55$1.66 30 | 0.25W Intelligent Cabinet Sensor Lights HHI-546845https://www.tinydeal.com/025w-intelligent-cabinet-sensor-lights-p-174242.html$1.93$5.22 31 | 8mm Intelligent Thermometer Temperature Measuring Ring HHI-566384https://www.tinydeal.com/8mm-intelligent-thermometer-temperature-measuring-ring-p-181149.html$1.48$6.57 32 | 3.5mm Stereo Woven Fiber Cloth Line Headset EDT-556403https://www.tinydeal.com/35mm-stereo-woven-fiber-cloth-line-headset-p-178024.html$1.54$4.85 33 | 90*30cm Ice Towel Instant Cooling Towel Heat Relief Reusable Chill Cool Towel HHI-549301https://www.tinydeal.com/9030cm-ice-towel-instant-cooling-towel-heat-relief-reusable-chill-cool-towel-p-174936.html$2.17$2.60 34 | DUBERY Polarized Sunglasses Men's Driving Shades Male Sun Glasses S-580864https://www.tinydeal.com/dubery-polarized-sunglasses-mens-driving-shades-male-sun-glasses-p-200630.html$5.83$15.03 35 | DC 12V 45cm 45-SMD 1210 LED Flexible Waterproof Car Strip Light RLT-274929https://www.tinydeal.com/dc-12v-45cm-45-smd-1210-led-flexible-waterproof-car-strip-light-p-118388.html$1.27$2.28 36 | 2.4GHz 1600dpi Wireless Optical Mouse Mice CMS-11833-C3https://www.tinydeal.com/24ghz-1600dpi-wireless-optical-mouse-mice-p-168444.html$3.54$6.79 37 | Baellerry Men Wallets PU Leather Solid Luxury Slim Short Credit Card Holder HHI-582256https://www.tinydeal.com/baellerry-men-wallets-pu-leather-solid-luxury-slim-short-credit-card-holder-p-200737.html$3.68$20.26 38 | Car Paint Scratches Repair Pen Brush Waterproof Paint Marker Pen Car Maintain RTH-571469https://www.tinydeal.com/car-paint-scratches-repair-pen-brush-waterproof-paint-marker-pen-car-maintain-p-182211.html$1.57$4.82 39 | Anti-droplets Hat Detachable Design Clear Facial Mask Outdoor Protector Sun Hat EEP-583095https://www.tinydeal.com/anti-droplets-hat-detachable-design-clear-facial-mask-outdoor-protector-sun-hat-p-200783.html$6.94$10.95 40 | S03 Smart Watch Waterproof Activity Tracker Smart Band Bracelet E-582866https://www.tinydeal.com/s03-smart-watch-waterproof-activity-tracker-smart-band-bracelet-p-200781.html$11.81$23.74 41 | Universal 6'' Phone Lanyard Necklace & Wrist Strap Card Bag Holder w Lanyard HHITH-543753https://www.tinydeal.com/universal-6-phone-lanyard-necklace-wrist-strap-card-bag-holder-w-lanyard-p-172964.html$2.05$3.06 42 | Hook Clip-on Rear Bike Seat Light Warning Lamp STH-548943https://www.tinydeal.com/hook-clip-on-rear-bike-seat-light-warning-lamp-p-174917.html$2.59$14.35 43 | NAVIFORCE NF9161 Men's Watches Top Luxury Brand Analog Stainless Steel WWT-580848https://www.tinydeal.com/naviforce-nf9161-mens-watches-top-luxury-brand-analog-stainless-steel-p-200814.html$18.58$33.44 44 | Stainless Steel Vacuum Sealed Red Wine Bottle Stopper Spout HKI-507455https://www.tinydeal.com/stainless-steel-vacuum-sealed-red-wine-bottle-stopper-spout-p-158461.html$2.56$2.91 45 | Xiaomi Turok Steinhardt TS Brand Polarized Stainless Mirror Lenses Sunglasses S-582106https://www.tinydeal.com/xiaomi-turok-steinhardt-ts-brand-polarized-stainless-mirror-lenses-sunglasses-p-200936.html$19.99$33.98 46 | Outdoor Lights Solar Powered LED Wall Lamp Porch Lights Sensor Motion Sensor HSI-582757https://www.tinydeal.com/outdoor-lights-solar-powered-led-wall-lamp-porch-lights-sensor-motion-sensor-p-201016.html$8.07$14.53 47 | 2in1 Electric Mosquito Swatter Dispeller /Mosquito Killer Lamp USB Charging HHIHE-582871https://www.tinydeal.com/2in1-electric-mosquito-swatter-dispeller-mosquito-killer-lamp-usb-charging-p-200982.html$27.45$49.40 48 | Xiaomi 3D Men Electric Shaver Razor Enchen BlackStone3 IPX7 Waterproof HHI-580937https://www.tinydeal.com/xiaomi-3d-men-electric-shaver-razor-enchen-blackstone3-ipx7-waterproof-p-200797.html$23.99$28.79 49 | Waterproof Reusable Rain Shoes Covers Rubber Slip-Resistant Rain Boot Overshoes S-557900https://www.tinydeal.com/waterproof-reusable-rain-shoes-covers-rubber-slip-resistant-rain-boot-overshoes-p-179875.html$3.61$6.79 50 | 1/2 pcs Mini Camera SQ11 Night Vision FOV140 1080P DV Video Recorder ECM-533708https://www.tinydeal.com/12-pcs-mini-camera-sq11-night-vision-fov140-1080p-dv-video-recorder-p-168604.html$7.53$12.04 51 | Men Shoulder Bag iPad Large PU Leather Crossbody Business Travel Handbag HHI-583042https://www.tinydeal.com/men-shoulder-bag-ipad-large-pu-leather-crossbody-business-travel-handbag-p-200788.html$18.89$36.54 52 | Neoprene Short Beach Socks Diving Socks Non-Slip Antiskid Scuba Dive Boots SGD-548443https://www.tinydeal.com/neoprene-short-beach-socks-diving-socks-non-slip-antiskid-scuba-dive-boots-p-174657.html$3.37$3.76 53 | 2PCs Wristbands Sport Sweatband Hand Band Sweat Wrist Support Brace Wrist Guards S-549945https://www.tinydeal.com/2pcs-wristbands-sport-sweatband-hand-band-sweat-wrist-support-brace-wrist-guards-p-175242.html$2.04$3.68 54 | Silicone Stretch Lids Food Saver Covers Set – 6pcs HKI-550666https://www.tinydeal.com/silicone-stretch-lids-food-saver-covers-set-–-6pcs-p-175450.html$4.86$6.79 55 | Smart Whistle LED Key Finder Ring Chain Mini Light HHI-521830https://www.tinydeal.com/smart-whistle-led-key-finder-ring-chain-mini-light-p-162491.html$1.77$1.93 56 | X96Q Android 10.0 TV Box Quad Core Allwinner H313 4K TV Box 2GB RAM 16GB ROM E-581778https://www.tinydeal.com/x96q-android-100-tv-box-quad-core-allwinner-h313-4k-tv-box-2gb-ram-16gb-rom-p-201107.html$30.79$43.10 57 | Fascia Gun Muscle Massage Gun Fitness Decompose Lactic Acid Relief Pain Relax HHI-581942https://www.tinydeal.com/fascia-gun-muscle-massage-gun-fitness-decompose-lactic-acid-relief-pain-relax-p-200720.html$48.45$87.21 58 | Compression Socks for Man Woman STH-563907https://www.tinydeal.com/compression-socks-for-man-woman-p-180225.html$2.72$6.38 59 | -------------------------------------------------------------------------------- /tinydeal/data_total.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "title": "50 x Medical Disposable Face Mouth Mask 95% Filtration CE FDA HHI-582255", 4 | "url": "https://www.tinydeal.com/50-x-medical-disposable-face-mouth-mask-95-filtration-ce-fda-p-200376.html", 5 | "discounted_price": "$17.50", 6 | "original_price": "$33.88 " 7 | }, 8 | { 9 | "title": "Transparent Anti Droplet Dust-proof Medical Face Shield Face Cover Mask HHI-581539", 10 | "url": "https://www.tinydeal.com/transparent-anti-droplet-dust-proof-medical-face-shield-face-cover-mask-p-200336.html", 11 | "discounted_price": "$2.79", 12 | "original_price": "$3.44 " 13 | }, 14 | { 15 | "title": "10 x Transparent Anti Droplet Dust-proof Medical Face Shield Face Cover Mask KB-581546", 16 | "url": "https://www.tinydeal.com/10-x-transparent-anti-droplet-dust-proof-medical-face-shield-face-cover-mask-p-200338.html", 17 | "discounted_price": "$13.66", 18 | "original_price": "$24.33 " 19 | }, 20 | { 21 | "title": "Mini Airtight Sealer Machine for Plastic Bag - Assorted Color HLI-48626", 22 | "url": "https://www.tinydeal.com/mini-airtight-sealer-machine-for-plastic-bag-assorted-color-p-39392.html", 23 | "discounted_price": "$2.27", 24 | "original_price": "$5.79 " 25 | }, 26 | { 27 | "title": "GM320 Infrared Thermometer Non-Contact Laser Gun IR Thermometer EDT-528767", 28 | "url": "https://www.tinydeal.com/gm320-infrared-thermometer-non-contact-laser-gun-ir-thermometer-p-165306.html", 29 | "discounted_price": "$16.77", 30 | "original_price": "$18.72 " 31 | }, 32 | { 33 | "title": "TWS F9 Bluetooth Earphone Touch Wireless Headphones 8D Stereo Sports Earbuds EEP-582910", 34 | "url": "https://www.tinydeal.com/tws-f9-bluetooth-earphone-touch-wireless-headphones-8d-stereo-sports-earbuds-p-200780.html", 35 | "discounted_price": "$11.93", 36 | "original_price": "$23.74 " 37 | }, 38 | { 39 | "title": "Infrared Non-Contact Forehead Thermometer for Fever Check HHIHE-580934", 40 | "url": "https://www.tinydeal.com/infrared-non-contact-forehead-thermometer-for-fever-check-p-200328.html", 41 | "discounted_price": "$43.48", 42 | "original_price": "$71.04 " 43 | }, 44 | { 45 | "title": "Smart Bracelet Watch Fitness Bracelet Pedometer Heart Rate Monitor Waterproof E-583049", 46 | "url": "https://www.tinydeal.com/smart-bracelet-watch-fitness-bracelet-pedometer-heart-rate-monitor-waterproof-p-200803.html", 47 | "discounted_price": "$11.31", 48 | "original_price": "$14.70 " 49 | }, 50 | { 51 | "title": "3 in 1 Cable Nylon-Coating Fast Charging 8Pin Type-C Micro-USB EPA-530685", 52 | "url": "https://www.tinydeal.com/3-in-1-cable-nylon-coating-fast-charging-8pin-type-micro-usb-p-165980.html", 53 | "discounted_price": "$2.50", 54 | "original_price": "$4.32 " 55 | }, 56 | { 57 | "title": "1/2/5/10 pcs Snore Stopper Silicone Nose Clip Sleeping Device BKH-531636", 58 | "url": "https://www.tinydeal.com/12510-pcs-snore-stopper-silicone-nose-clip-sleeping-device-p-166268.html", 59 | "discounted_price": "$1.22", 60 | "original_price": "$3.66 " 61 | }, 62 | { 63 | "title": "Light Controlled Square LED Wall Night Lamp Bedroom Night Light HLT-506114", 64 | "url": "https://www.tinydeal.com/light-controlled-square-led-wall-night-lamp-bedroom-night-light-p-158312.html", 65 | "discounted_price": "$2.29", 66 | "original_price": "$5.24 " 67 | }, 68 | { 69 | "title": "Portable Mini LCD Digital Tyre Tire Pressure Gauge Tester CTL-27481", 70 | "url": "https://www.tinydeal.com/portable-mini-lcd-digital-tyre-tire-pressure-gauge-tester-p-23322.html", 71 | "discounted_price": "$3.35", 72 | "original_price": "$3.88 " 73 | }, 74 | { 75 | "title": "DUBERY Vintage Sunglasses Polarized Men's Sun Glasses For Men S-580871", 76 | "url": "https://www.tinydeal.com/dubery-vintage-sunglasses-polarized-mens-sun-glasses-for-men-p-200631.html", 77 | "discounted_price": "$6.25", 78 | "original_price": "$8.12 " 79 | }, 80 | { 81 | "title": "Silicone Nipple Cover Stickers Adhesive Bra Reusable Invisible Nipple Cover HHI-550767", 82 | "url": "https://www.tinydeal.com/silicone-nipple-cover-stickers-adhesive-bra-reusable-invisible-nipple-cover-p-175493.html", 83 | "discounted_price": "$1.97", 84 | "original_price": "$2.37 " 85 | }, 86 | { 87 | "title": "1 Pair of Silicone Magnetic Losing Weight Toe Rings HBI-13397", 88 | "url": "https://www.tinydeal.com/1-pair-of-silicone-magnetic-losing-weight-toe-rings-p-12727.html", 89 | "discounted_price": "$1.05", 90 | "original_price": "$3.22 " 91 | }, 92 | { 93 | "title": "Xiaomi Mijia iHealth Thermometer LED Non Contact Digital Infrared Forehead Body RTH-575855", 94 | "url": "https://www.tinydeal.com/xiaomi-mijia-ihealth-thermometer-led-non-contact-digital-infrared-forehead-body-p-200865.html", 95 | "discounted_price": "$28.26", 96 | "original_price": "$36.74 " 97 | }, 98 | { 99 | "title": "5 LED Water Resistant USB Rechargeable LED Bike Tail Light RTH-555898", 100 | "url": "https://www.tinydeal.com/5-led-water-resistant-usb-rechargeable-led-bike-tail-light-p-177336.html", 101 | "discounted_price": "$2.91", 102 | "original_price": "$5.24 " 103 | }, 104 | { 105 | "title": "Teeth Whitening Pen Tooth Gel Whitener Bleach HBI-113105", 106 | "url": "https://www.tinydeal.com/teeth-whitening-pen-tooth-gel-whitener-bleach-p-66709.html", 107 | "discounted_price": "$2.12", 108 | "original_price": "$5.88 " 109 | }, 110 | { 111 | "title": "Android Robot Shaped Micro USB to USB 2.0 Host OTG Adapter ECACR-487310", 112 | "url": "https://www.tinydeal.com/android-robot-shaped-micro-usb-to-usb-20-host-otg-adapter-p-157418.html", 113 | "discounted_price": "$1.03", 114 | "original_price": "$2.70 " 115 | }, 116 | { 117 | "title": "Mini Video Capture Card USB 2.0 HDMI Video Grabber Record Box for PS4 Game DVD HHIHE-583000", 118 | "url": "https://www.tinydeal.com/mini-video-capture-card-usb-20-hdmi-video-grabber-record-box-for-ps4-game-dvd-p-200736.html", 119 | "discounted_price": "$9.69", 120 | "original_price": "$21.51 " 121 | }, 122 | { 123 | "title": "XIAOMI MIJIA Microhoo Mini Air Conditioner Fan Personal Portable USB Air Cooler HHIHE-582526", 124 | "url": "https://www.tinydeal.com/xiaomi-mijia-microhoo-mini-air-conditioner-fan-personal-portable-usb-air-cooler-p-200948.html", 125 | "discounted_price": "$60.02", 126 | "original_price": "$72.02 " 127 | }, 128 | { 129 | "title": "Funlife 12pcs / set Green Crystal Three-dimensional Wall Stickers Mirror HHI-562368", 130 | "url": "https://www.tinydeal.com/funlife-12pcs-set-green-crystal-three-dimensional-wall-stickers-mirror-p-179148.html", 131 | "discounted_price": "$2.26", 132 | "original_price": "$6.79 " 133 | }, 134 | { 135 | "title": "IP67 P8 Smart Watch Wristband Men Women Monitor Smartwatch Tracker For Phone E-580457", 136 | "url": "https://www.tinydeal.com/ip67-p8-smart-watch-wristband-men-women-monitor-smartwatch-tracker-for-phone-p-200649.html", 137 | "discounted_price": "$18.52", 138 | "original_price": "$36.34 " 139 | }, 140 | { 141 | "title": "Detachable Baseball Protective Clear Face mask Cap Anti-Splash Hat S-581830", 142 | "url": "https://www.tinydeal.com/detachable-baseball-protective-clear-face-mask-cap-anti-splash-hat-p-200362.html", 143 | "discounted_price": "$3.06", 144 | "original_price": "$3.69 " 145 | }, 146 | { 147 | "title": "T10 Bluetooth Headsets Wireless Earbuds 5.0 TWS Earphone EEP-581913", 148 | "url": "https://www.tinydeal.com/t10-bluetooth-headsets-wireless-earbuds-50-tws-earphone-p-200802.html", 149 | "discounted_price": "$13.21", 150 | "original_price": "$25.40 " 151 | }, 152 | { 153 | "title": "Fsat Quick Charge 3.0 4.0 QC3.0 Fast Charging Tablet Wall Adapter HHITH-582324", 154 | "url": "https://www.tinydeal.com/fsat-quick-charge-30-40-qc30-fast-charging-tablet-wall-adapter-p-201049.html", 155 | "discounted_price": "$3.70", 156 | "original_price": "$6.67 " 157 | }, 158 | { 159 | "title": "Multifunction USB 2.0 M2/ TF Card Reader w/ Lanyard CCR-85173", 160 | "url": "https://www.tinydeal.com/multifunction-usb-20-m2-tf-card-reader-w-lanyard-p-57252.html", 161 | "discounted_price": "$1.55", 162 | "original_price": "$1.66 " 163 | }, 164 | { 165 | "title": "0.25W Intelligent Cabinet Sensor Lights HHI-546845", 166 | "url": "https://www.tinydeal.com/025w-intelligent-cabinet-sensor-lights-p-174242.html", 167 | "discounted_price": "$1.93", 168 | "original_price": "$5.22 " 169 | }, 170 | { 171 | "title": "8mm Intelligent Thermometer Temperature Measuring Ring HHI-566384", 172 | "url": "https://www.tinydeal.com/8mm-intelligent-thermometer-temperature-measuring-ring-p-181149.html", 173 | "discounted_price": "$1.48", 174 | "original_price": "$6.57 " 175 | }, 176 | { 177 | "title": "3.5mm Stereo Woven Fiber Cloth Line Headset EDT-556403", 178 | "url": "https://www.tinydeal.com/35mm-stereo-woven-fiber-cloth-line-headset-p-178024.html", 179 | "discounted_price": "$1.54", 180 | "original_price": "$4.85 " 181 | }, 182 | { 183 | "title": "90*30cm Ice Towel Instant Cooling Towel Heat Relief Reusable Chill Cool Towel HHI-549301", 184 | "url": "https://www.tinydeal.com/9030cm-ice-towel-instant-cooling-towel-heat-relief-reusable-chill-cool-towel-p-174936.html", 185 | "discounted_price": "$2.17", 186 | "original_price": "$2.60 " 187 | }, 188 | { 189 | "title": "DUBERY Polarized Sunglasses Men's Driving Shades Male Sun Glasses S-580864", 190 | "url": "https://www.tinydeal.com/dubery-polarized-sunglasses-mens-driving-shades-male-sun-glasses-p-200630.html", 191 | "discounted_price": "$5.83", 192 | "original_price": "$15.03 " 193 | }, 194 | { 195 | "title": "DC 12V 45cm 45-SMD 1210 LED Flexible Waterproof Car Strip Light RLT-274929", 196 | "url": "https://www.tinydeal.com/dc-12v-45cm-45-smd-1210-led-flexible-waterproof-car-strip-light-p-118388.html", 197 | "discounted_price": "$1.27", 198 | "original_price": "$2.28 " 199 | }, 200 | { 201 | "title": "2.4GHz 1600dpi Wireless Optical Mouse Mice CMS-11833-C3", 202 | "url": "https://www.tinydeal.com/24ghz-1600dpi-wireless-optical-mouse-mice-p-168444.html", 203 | "discounted_price": "$3.54", 204 | "original_price": "$6.79 " 205 | }, 206 | { 207 | "title": "Baellerry Men Wallets PU Leather Solid Luxury Slim Short Credit Card Holder HHI-582256", 208 | "url": "https://www.tinydeal.com/baellerry-men-wallets-pu-leather-solid-luxury-slim-short-credit-card-holder-p-200737.html", 209 | "discounted_price": "$3.68", 210 | "original_price": "$20.26 " 211 | }, 212 | { 213 | "title": "Car Paint Scratches Repair Pen Brush Waterproof Paint Marker Pen Car Maintain RTH-571469", 214 | "url": "https://www.tinydeal.com/car-paint-scratches-repair-pen-brush-waterproof-paint-marker-pen-car-maintain-p-182211.html", 215 | "discounted_price": "$1.57", 216 | "original_price": "$4.82 " 217 | }, 218 | { 219 | "title": "Anti-droplets Hat Detachable Design Clear Facial Mask Outdoor Protector Sun Hat EEP-583095", 220 | "url": "https://www.tinydeal.com/anti-droplets-hat-detachable-design-clear-facial-mask-outdoor-protector-sun-hat-p-200783.html", 221 | "discounted_price": "$6.94", 222 | "original_price": "$10.95 " 223 | }, 224 | { 225 | "title": "S03 Smart Watch Waterproof Activity Tracker Smart Band Bracelet E-582866", 226 | "url": "https://www.tinydeal.com/s03-smart-watch-waterproof-activity-tracker-smart-band-bracelet-p-200781.html", 227 | "discounted_price": "$11.81", 228 | "original_price": "$23.74 " 229 | }, 230 | { 231 | "title": "Universal 6'' Phone Lanyard Necklace & Wrist Strap Card Bag Holder w Lanyard HHITH-543753", 232 | "url": "https://www.tinydeal.com/universal-6-phone-lanyard-necklace-wrist-strap-card-bag-holder-w-lanyard-p-172964.html", 233 | "discounted_price": "$2.05", 234 | "original_price": "$3.06 " 235 | }, 236 | { 237 | "title": "Hook Clip-on Rear Bike Seat Light Warning Lamp STH-548943", 238 | "url": "https://www.tinydeal.com/hook-clip-on-rear-bike-seat-light-warning-lamp-p-174917.html", 239 | "discounted_price": "$2.59", 240 | "original_price": "$14.35 " 241 | }, 242 | { 243 | "title": "NAVIFORCE NF9161 Men's Watches Top Luxury Brand Analog Stainless Steel WWT-580848", 244 | "url": "https://www.tinydeal.com/naviforce-nf9161-mens-watches-top-luxury-brand-analog-stainless-steel-p-200814.html", 245 | "discounted_price": "$18.58", 246 | "original_price": "$33.44 " 247 | }, 248 | { 249 | "title": "Stainless Steel Vacuum Sealed Red Wine Bottle Stopper Spout HKI-507455", 250 | "url": "https://www.tinydeal.com/stainless-steel-vacuum-sealed-red-wine-bottle-stopper-spout-p-158461.html", 251 | "discounted_price": "$2.56", 252 | "original_price": "$2.91 " 253 | }, 254 | { 255 | "title": "Xiaomi Turok Steinhardt TS Brand Polarized Stainless Mirror Lenses Sunglasses S-582106", 256 | "url": "https://www.tinydeal.com/xiaomi-turok-steinhardt-ts-brand-polarized-stainless-mirror-lenses-sunglasses-p-200936.html", 257 | "discounted_price": "$19.99", 258 | "original_price": "$33.98 " 259 | }, 260 | { 261 | "title": "Outdoor Lights Solar Powered LED Wall Lamp Porch Lights Sensor Motion Sensor HSI-582757", 262 | "url": "https://www.tinydeal.com/outdoor-lights-solar-powered-led-wall-lamp-porch-lights-sensor-motion-sensor-p-201016.html", 263 | "discounted_price": "$8.07", 264 | "original_price": "$14.53 " 265 | }, 266 | { 267 | "title": "2in1 Electric Mosquito Swatter Dispeller /Mosquito Killer Lamp USB Charging HHIHE-582871", 268 | "url": "https://www.tinydeal.com/2in1-electric-mosquito-swatter-dispeller-mosquito-killer-lamp-usb-charging-p-200982.html", 269 | "discounted_price": "$27.45", 270 | "original_price": "$49.40 " 271 | }, 272 | { 273 | "title": "Xiaomi 3D Men Electric Shaver Razor Enchen BlackStone3 IPX7 Waterproof HHI-580937", 274 | "url": "https://www.tinydeal.com/xiaomi-3d-men-electric-shaver-razor-enchen-blackstone3-ipx7-waterproof-p-200797.html", 275 | "discounted_price": "$23.99", 276 | "original_price": "$28.79 " 277 | }, 278 | { 279 | "title": "Waterproof Reusable Rain Shoes Covers Rubber Slip-Resistant Rain Boot Overshoes S-557900", 280 | "url": "https://www.tinydeal.com/waterproof-reusable-rain-shoes-covers-rubber-slip-resistant-rain-boot-overshoes-p-179875.html", 281 | "discounted_price": "$3.61", 282 | "original_price": "$6.79 " 283 | }, 284 | { 285 | "title": "1/2 pcs Mini Camera SQ11 Night Vision FOV140 1080P DV Video Recorder ECM-533708", 286 | "url": "https://www.tinydeal.com/12-pcs-mini-camera-sq11-night-vision-fov140-1080p-dv-video-recorder-p-168604.html", 287 | "discounted_price": "$7.53", 288 | "original_price": "$12.04 " 289 | }, 290 | { 291 | "title": "Men Shoulder Bag iPad Large PU Leather Crossbody Business Travel Handbag HHI-583042", 292 | "url": "https://www.tinydeal.com/men-shoulder-bag-ipad-large-pu-leather-crossbody-business-travel-handbag-p-200788.html", 293 | "discounted_price": "$18.89", 294 | "original_price": "$36.54 " 295 | }, 296 | { 297 | "title": "Neoprene Short Beach Socks Diving Socks Non-Slip Antiskid Scuba Dive Boots SGD-548443", 298 | "url": "https://www.tinydeal.com/neoprene-short-beach-socks-diving-socks-non-slip-antiskid-scuba-dive-boots-p-174657.html", 299 | "discounted_price": "$3.37", 300 | "original_price": "$3.76 " 301 | }, 302 | { 303 | "title": "2PCs Wristbands Sport Sweatband Hand Band Sweat Wrist Support Brace Wrist Guards S-549945", 304 | "url": "https://www.tinydeal.com/2pcs-wristbands-sport-sweatband-hand-band-sweat-wrist-support-brace-wrist-guards-p-175242.html", 305 | "discounted_price": "$2.04", 306 | "original_price": "$3.68 " 307 | }, 308 | { 309 | "title": "Silicone Stretch Lids Food Saver Covers Set – 6pcs HKI-550666", 310 | "url": "https://www.tinydeal.com/silicone-stretch-lids-food-saver-covers-set-–-6pcs-p-175450.html", 311 | "discounted_price": "$4.86", 312 | "original_price": "$6.79 " 313 | }, 314 | { 315 | "title": "Smart Whistle LED Key Finder Ring Chain Mini Light HHI-521830", 316 | "url": "https://www.tinydeal.com/smart-whistle-led-key-finder-ring-chain-mini-light-p-162491.html", 317 | "discounted_price": "$1.77", 318 | "original_price": "$1.93 " 319 | }, 320 | { 321 | "title": "X96Q Android 10.0 TV Box Quad Core Allwinner H313 4K TV Box 2GB RAM 16GB ROM E-581778", 322 | "url": "https://www.tinydeal.com/x96q-android-100-tv-box-quad-core-allwinner-h313-4k-tv-box-2gb-ram-16gb-rom-p-201107.html", 323 | "discounted_price": "$30.79", 324 | "original_price": "$43.10 " 325 | }, 326 | { 327 | "title": "Fascia Gun Muscle Massage Gun Fitness Decompose Lactic Acid Relief Pain Relax HHI-581942", 328 | "url": "https://www.tinydeal.com/fascia-gun-muscle-massage-gun-fitness-decompose-lactic-acid-relief-pain-relax-p-200720.html", 329 | "discounted_price": "$48.45", 330 | "original_price": "$87.21 " 331 | }, 332 | { 333 | "title": "Compression Socks for Man Woman STH-563907", 334 | "url": "https://www.tinydeal.com/compression-socks-for-man-woman-p-180225.html", 335 | "discounted_price": "$2.72", 336 | "original_price": "$6.38 " 337 | } 338 | ] -------------------------------------------------------------------------------- /tinydeal/scrapy.cfg: -------------------------------------------------------------------------------- 1 | # Automatically created by: scrapy startproject 2 | # 3 | # For more information about the [deploy] section see: 4 | # https://scrapyd.readthedocs.io/en/latest/deploy.html 5 | 6 | [settings] 7 | default = tinydeal.settings 8 | 9 | [deploy] 10 | #url = http://localhost:6800/ 11 | project = tinydeal 12 | -------------------------------------------------------------------------------- /tinydeal/tinydeal/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/tinydeal/tinydeal/__init__.py -------------------------------------------------------------------------------- /tinydeal/tinydeal/items.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your scraped items 4 | # 5 | # See documentation in: 6 | # https://doc.scrapy.org/en/latest/topics/items.html 7 | 8 | import scrapy 9 | 10 | 11 | class TinydealItem(scrapy.Item): 12 | # define the fields for your item here like: 13 | # name = scrapy.Field() 14 | pass 15 | -------------------------------------------------------------------------------- /tinydeal/tinydeal/middlewares.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your spider middleware 4 | # 5 | # See documentation in: 6 | # https://doc.scrapy.org/en/latest/topics/spider-middleware.html 7 | 8 | from scrapy import signals 9 | 10 | 11 | class TinydealSpiderMiddleware(object): 12 | # Not all methods need to be defined. If a method is not defined, 13 | # scrapy acts as if the spider middleware does not modify the 14 | # passed objects. 15 | 16 | @classmethod 17 | def from_crawler(cls, crawler): 18 | # This method is used by Scrapy to create your spiders. 19 | s = cls() 20 | crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) 21 | return s 22 | 23 | def process_spider_input(self, response, spider): 24 | # Called for each response that goes through the spider 25 | # middleware and into the spider. 26 | 27 | # Should return None or raise an exception. 28 | return None 29 | 30 | def process_spider_output(self, response, result, spider): 31 | # Called with the results returned from the Spider, after 32 | # it has processed the response. 33 | 34 | # Must return an iterable of Request, dict or Item objects. 35 | for i in result: 36 | yield i 37 | 38 | def process_spider_exception(self, response, exception, spider): 39 | # Called when a spider or process_spider_input() method 40 | # (from other spider middleware) raises an exception. 41 | 42 | # Should return either None or an iterable of Response, dict 43 | # or Item objects. 44 | pass 45 | 46 | def process_start_requests(self, start_requests, spider): 47 | # Called with the start requests of the spider, and works 48 | # similarly to the process_spider_output() method, except 49 | # that it doesn’t have a response associated. 50 | 51 | # Must return only requests (not items). 52 | for r in start_requests: 53 | yield r 54 | 55 | def spider_opened(self, spider): 56 | spider.logger.info('Spider opened: %s' % spider.name) 57 | 58 | 59 | class TinydealDownloaderMiddleware(object): 60 | # Not all methods need to be defined. If a method is not defined, 61 | # scrapy acts as if the downloader middleware does not modify the 62 | # passed objects. 63 | 64 | @classmethod 65 | def from_crawler(cls, crawler): 66 | # This method is used by Scrapy to create your spiders. 67 | s = cls() 68 | crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) 69 | return s 70 | 71 | def process_request(self, request, spider): 72 | # Called for each request that goes through the downloader 73 | # middleware. 74 | 75 | # Must either: 76 | # - return None: continue processing this request 77 | # - or return a Response object 78 | # - or return a Request object 79 | # - or raise IgnoreRequest: process_exception() methods of 80 | # installed downloader middleware will be called 81 | return None 82 | 83 | def process_response(self, request, response, spider): 84 | # Called with the response returned from the downloader. 85 | 86 | # Must either; 87 | # - return a Response object 88 | # - return a Request object 89 | # - or raise IgnoreRequest 90 | return response 91 | 92 | def process_exception(self, request, exception, spider): 93 | # Called when a download handler or a process_request() 94 | # (from other downloader middleware) raises an exception. 95 | 96 | # Must either: 97 | # - return None: continue processing this exception 98 | # - return a Response object: stops process_exception() chain 99 | # - return a Request object: stops process_exception() chain 100 | pass 101 | 102 | def spider_opened(self, spider): 103 | spider.logger.info('Spider opened: %s' % spider.name) 104 | -------------------------------------------------------------------------------- /tinydeal/tinydeal/pipelines.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define your item pipelines here 4 | # 5 | # Don't forget to add your pipeline to the ITEM_PIPELINES setting 6 | # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html 7 | 8 | 9 | class TinydealPipeline(object): 10 | def process_item(self, item, spider): 11 | return item 12 | -------------------------------------------------------------------------------- /tinydeal/tinydeal/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Scrapy settings for tinydeal project 4 | # 5 | # For simplicity, this file contains only settings considered important or 6 | # commonly used. You can find more settings consulting the documentation: 7 | # 8 | # https://doc.scrapy.org/en/latest/topics/settings.html 9 | # https://doc.scrapy.org/en/latest/topics/downloader-middleware.html 10 | # https://doc.scrapy.org/en/latest/topics/spider-middleware.html 11 | 12 | BOT_NAME = 'tinydeal' 13 | 14 | SPIDER_MODULES = ['tinydeal.spiders'] 15 | NEWSPIDER_MODULE = 'tinydeal.spiders' 16 | 17 | 18 | # Crawl responsibly by identifying yourself (and your website) on the user-agent 19 | #USER_AGENT = 'tinydeal (+http://www.yourdomain.com)' 20 | 21 | # Obey robots.txt rules 22 | ROBOTSTXT_OBEY = True 23 | 24 | # Configure maximum concurrent requests performed by Scrapy (default: 16) 25 | #CONCURRENT_REQUESTS = 32 26 | 27 | # Configure a delay for requests for the same website (default: 0) 28 | # See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay 29 | # See also autothrottle settings and docs 30 | #DOWNLOAD_DELAY = 3 31 | # The download delay setting will honor only one of: 32 | #CONCURRENT_REQUESTS_PER_DOMAIN = 16 33 | #CONCURRENT_REQUESTS_PER_IP = 16 34 | 35 | # Disable cookies (enabled by default) 36 | #COOKIES_ENABLED = False 37 | 38 | # Disable Telnet Console (enabled by default) 39 | #TELNETCONSOLE_ENABLED = False 40 | 41 | # Override the default request headers: 42 | # DEFAULT_REQUEST_HEADERS = { 43 | # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 44 | # 'Accept-Language': 'en', 45 | # } 46 | 47 | # Enable or disable spider middlewares 48 | # See https://doc.scrapy.org/en/latest/topics/spider-middleware.html 49 | # SPIDER_MIDDLEWARES = { 50 | # 'tinydeal.middlewares.TinydealSpiderMiddleware': 543, 51 | # } 52 | 53 | # Enable or disable downloader middlewares 54 | # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html 55 | # DOWNLOADER_MIDDLEWARES = { 56 | # 'tinydeal.middlewares.TinydealDownloaderMiddleware': 543, 57 | # } 58 | 59 | # Enable or disable extensions 60 | # See https://doc.scrapy.org/en/latest/topics/extensions.html 61 | # EXTENSIONS = { 62 | # 'scrapy.extensions.telnet.TelnetConsole': None, 63 | # } 64 | 65 | # Configure item pipelines 66 | # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html 67 | # ITEM_PIPELINES = { 68 | # 'tinydeal.pipelines.TinydealPipeline': 300, 69 | # } 70 | 71 | # Enable and configure the AutoThrottle extension (disabled by default) 72 | # See https://doc.scrapy.org/en/latest/topics/autothrottle.html 73 | #AUTOTHROTTLE_ENABLED = True 74 | # The initial download delay 75 | #AUTOTHROTTLE_START_DELAY = 5 76 | # The maximum download delay to be set in case of high latencies 77 | #AUTOTHROTTLE_MAX_DELAY = 60 78 | # The average number of requests Scrapy should be sending in parallel to 79 | # each remote server 80 | #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 81 | # Enable showing throttling stats for every response received: 82 | #AUTOTHROTTLE_DEBUG = False 83 | 84 | # Enable and configure HTTP caching (disabled by default) 85 | # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings 86 | #HTTPCACHE_ENABLED = True 87 | #HTTPCACHE_EXPIRATION_SECS = 0 88 | #HTTPCACHE_DIR = 'httpcache' 89 | #HTTPCACHE_IGNORE_HTTP_CODES = [] 90 | #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' 91 | 92 | 93 | FEED_EXPORT_ENCODING = 'utf-8' 94 | -------------------------------------------------------------------------------- /tinydeal/tinydeal/spiders/__init__.py: -------------------------------------------------------------------------------- 1 | # This package will contain the spiders of your Scrapy project 2 | # 3 | # Please refer to the documentation for information on how to create and manage 4 | # your spiders. 5 | -------------------------------------------------------------------------------- /tinydeal/tinydeal/spiders/special_offers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import scrapy 3 | 4 | 5 | class SpecialOffersSpider(scrapy.Spider): 6 | name = 'special_offers' 7 | allowed_domains = ['www.tinydeal.com'] 8 | start_urls = ['https://www.tinydeal.com/specials.html'] 9 | 10 | def parse(self, response): 11 | for product in response.xpath("//ul[@class='productlisting-ul']/div/li"): 12 | yield{ 13 | 'title': product.xpath(".//a[@class='p_box_title']/text()").get(), 14 | 'url': response.urljoin(product.xpath(".//a[@class='p_box_title']/@href").get()), 15 | 'discounted_price': product.xpath(".//div[@class='p_box_price']/span[1]/text()").get(), 16 | 'original_price': product.xpath(".//div[@class='p_box_price']/span[2]/text()").get() 17 | } 18 | next_page = response.xpath("//a[@class='nextPage']/@href").get() 19 | if next_page: 20 | yield scrapy.response(url=next_page, callback=self.parse) 21 | -------------------------------------------------------------------------------- /tutorial/Books: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | DMOZ - Computers: Programming: Languages: Python: Books 37 | 38 | 55 | 56 |
57 | 58 | 61 | 62 |
63 | 64 |
65 |
66 | 67 | 68 |
69 | In Partnership with AOL 70 |
71 |
72 |
73 | 74 |
75 | 76 | 77 | 78 | 79 | 80 | 81 |
82 | 83 |
84 | about dmoz | 85 | dmoz blog | 86 | 87 | suggest URL | 88 | 89 | 90 | update listing | 91 | 92 | become an editor | 93 | 94 | report abuse/spam | 95 | help 96 |
97 | 98 |
99 |
100 |
101 | 102 | 103 | 110 | 111 |
112 | 113 |
114 |
115 | 116 | 117 | 118 |
119 | 129 |
130 | 131 |
132 | 133 | 134 | 135 |
136 | See also: 137 | 150 |
151 | 152 |
153 | This category in other languages: 154 | 167 |
168 | 169 | 170 |
171 |
    172 | 173 |
  • 174 | 175 | Core Python Programming 176 | 177 | - By Wesley J. Chun; Prentice Hall PTR, 2001, ISBN 0130260363. For experienced developers to improve extant skills; professional level examples. Starts by introducing syntax, objects, error handling, functions, classes, built-ins. [Prentice Hall] 178 | 179 |
    [!]
    180 |
    • 181 | 182 |
  • 183 | 184 | Data Structures and Algorithms with Object-Oriented Design Patterns in Python 185 | 186 | - The primary goal of this book is to promote object-oriented design using Python and to illustrate the use of the emerging object-oriented design patterns. 187 | A secondary goal of the book is to present mathematical tools just in time. Analysis techniques and proofs are presented as needed and in the proper context. 188 | 189 |
    [!]
    190 |
    • 191 | 192 |
  • 193 | 194 | Dive Into Python 3 195 | 196 | - By Mark Pilgrim, Guide to Python 3 and its differences from Python 2. Each chapter starts with a real code sample and explains it fully. Has a comprehensive appendix of all the syntactic and semantic changes in Python 3 197 | 198 | 199 | 200 |
    [!]
    201 |
    • 202 | 203 |
  • 204 | 205 | Foundations of Python Network Programming 206 | 207 | - This book covers a wide range of topics. From raw TCP and UDP to encryption with TSL, and then to HTTP, SMTP, POP, IMAP, and ssh. It gives you a good understanding of each field and how to do everything on the network with Python. 208 | 209 |
    [!]
    210 |
    • 211 | 212 |
  • 213 | 214 | Free Python books 215 | 216 | - Free Python books and tutorials. 217 | 218 |
    [!]
    219 |
    • 220 | 221 |
  • 222 | 223 | FreeTechBooks: Python Scripting Language 224 | 225 | - Annotated list of free online books on Python scripting language. Topics range from beginner to advanced. 226 | 227 |
    [!]
    228 |
    • 229 | 230 |
  • 231 | 232 | How to Think Like a Computer Scientist: Learning with Python 233 | 234 | - By Allen B. Downey, Jeffrey Elkner, Chris Meyers; Green Tea Press, 2002, ISBN 0971677506. Teaches general principles of programming, via Python as subject language. Thorough, in-depth approach to many basic and intermediate programming topics. Full text online and downloads: HTML, PDF, PS, LaTeX. [Free, Green Tea Press] 235 | 236 |
    [!]
    237 |
    • 238 | 239 |
  • 240 | 241 | An Introduction to Python 242 | 243 | - By Guido van Rossum, Fred L. Drake, Jr.; Network Theory Ltd., 2003, ISBN 0954161769. Printed edition of official tutorial, for v2.x, from Python.org. [Network Theory, online] 244 | 245 |
    [!]
    246 |
    • 247 | 248 |
  • 249 | 250 | Learn to Program Using Python 251 | 252 | - Book by Alan Gauld with full text online. Introduction for those learning programming basics: terminology, concepts, methods to write code. Assumes no prior knowledge but basic computer skills. 253 | 254 |
    [!]
    255 |
    • 256 | 257 |
  • 258 | 259 | Making Use of Python 260 | 261 | - By Rashi Gupta; John Wiley and Sons, 2002, ISBN 0471219754. Covers language basics, use for CGI scripting, GUI development, network programming; shows why it is one of more sophisticated of popular scripting languages. [Wiley] 262 | 263 |
    [!]
    264 |
    • 265 | 266 |
  • 267 | 268 | Practical Python 269 | 270 | - By Magnus Lie Hetland; Apress LP, 2002, ISBN 1590590066. Readable guide to ideas most vital to new users, from basics common to high level languages, to more specific aspects, to a series of 10 ever more complex programs. [Apress] 271 | 272 |
    [!]
    273 |
    • 274 | 275 |
  • 276 | 277 | Pro Python System Administration 278 | 279 | - By Rytis Sileika, ISBN13: 978-1-4302-2605-5, Uses real-world system administration examples like manage devices with SNMP and SOAP, build a distributed monitoring system, manage web applications and parse complex log files, monitor and manage MySQL databases. 280 | 281 |
    [!]
    282 |
    • 283 | 284 |
  • 285 | 286 | Programming in Python 3 (Second Edition) 287 | 288 | - A Complete Introduction to the Python 3. 289 | 290 |
    [!]
    291 |
    • 292 | 293 |
  • 294 | 295 | Python 2.1 Bible 296 | 297 | - By Dave Brueck, Stephen Tanner; John Wiley and Sons, 2001, ISBN 0764548077. Full coverage, clear explanations, hands-on examples, full language reference; shows step by step how to use components, assemble them, form full-featured programs. [John Wiley and Sons] 298 | 299 |
    [!]
    300 |
    • 301 | 302 |
  • 303 | 304 | Python 3 Object Oriented Programming 305 | 306 | - A step-by-step tutorial for OOP in Python 3, including discussion and examples of abstraction, encapsulation, information hiding, and raise, handle, define, and manipulate exceptions. 307 | 308 |
    [!]
    309 |
    • 310 | 311 |
  • 312 | 313 | Python Language Reference Manual 314 | 315 | - By Guido van Rossum, Fred L. Drake, Jr.; Network Theory Ltd., 2003, ISBN 0954161785. Printed edition of official language reference, for v2.x, from Python.org, describes syntax, built-in datatypes. [Network Theory, online] 316 | 317 |
    [!]
    318 |
    • 319 | 320 |
  • 321 | 322 | Python Programming Patterns 323 | 324 | - By Thomas W. Christopher; Prentice Hall PTR, 2002, ISBN 0130409561. Shows how to write large programs, introduces powerful design patterns that deliver high levels of robustness, scalability, reuse. 325 | 326 |
    [!]
    327 |
    • 328 | 329 |
  • 330 | 331 | Python Programming with the Java Class Libraries: A Tutorial for Building Web and Enterprise Applications with Jython 332 | 333 | - By Richard Hightower; Addison-Wesley, 2002, 0201616165. Begins with Python basics, many exercises, interactive sessions. Shows programming novices concepts and practical methods. Shows programming experts Python's abilities and ways to interface with Java APIs. [publisher website] 334 | 335 |
    [!]
    336 |
    • 337 | 338 |
  • 339 | 340 | Python: Visual QuickStart Guide 341 | 342 | - By Chris Fehily; Peachpit Press, 2002, ISBN 0201748843. Task-based, step-by-step visual reference guide, many screen shots, for courses in digital graphics; Web design, scripting, development; multimedia, page layout, office tools, operating systems. [Prentice Hall] 343 | 344 |
    [!]
    345 |
    • 346 | 347 |
  • 348 | 349 | Sams Teach Yourself Python in 24 Hours 350 | 351 | - By Ivan Van Laningham; Sams Publishing, 2000, ISBN 0672317354. Split into 24 hands-on, 1 hour lessons; steps needed to learn topic: syntax, language features, OO design and programming, GUIs (Tkinter), system administration, CGI. [Sams Publishing] 352 | 353 |
    [!]
    354 |
    • 355 | 356 |
  • 357 | 358 | Text Processing in Python 359 | 360 | - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.] 361 | 362 |
    [!]
    363 |
    • 364 | 365 |
  • 366 | 367 | XML Processing with Python 368 | 369 | - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR] 370 | 371 |
    [!]
    372 |
    • 373 | 374 |
375 |
376 | 377 | 378 | 379 | 380 |
381 | 382 | 395 | 396 | 417 | 418 |
419 | 420 |
421 | 422 | 423 | 424 | Volunteer to edit this category. 425 | 426 | 427 | 428 | 429 |
430 | 431 | 432 | 433 | 434 |
435 | [Book Mozilla] 436 |
437 |
438 |
439 | 440 | 441 | Copyright © 1998-2014 AOL Inc. 442 | 443 | Terms of Use 444 |
445 |
446 | Visit our sister sites  mozilla.org | MusicMoz | Wikipedia

447 |
Last update: 448 | 449 | May 12, 2014 at 12:44:10 UTC 450 | 451 | - edit
452 |
453 |
454 |
455 | 458 |
459 | 460 | 461 | 462 | -------------------------------------------------------------------------------- /tutorial/Resources: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | DMOZ - Computers: Programming: Languages: Python: Resources 37 | 38 | 55 | 56 |
57 | 58 | 61 | 62 |
63 | 64 |
65 |
66 | 67 | 68 |
69 | In Partnership with AOL 70 |
71 |
72 |
73 | 74 |
75 | 76 | 77 | 78 | 79 | 80 | 81 |
82 | 83 |
84 | about dmoz | 85 | dmoz blog | 86 | 87 | suggest URL | 88 | 89 | 90 | update listing | 91 | 92 | become an editor | 93 | 94 | report abuse/spam | 95 | help 96 |
97 | 98 |
99 |
100 |
101 | 102 | 103 | 110 | 111 |
112 | 113 |
114 |
115 | 116 |
117 | 127 |
128 | 129 |
130 | 131 | 132 | 133 |
134 | See also: 135 | 143 |
144 | 145 | 146 |
147 |
    148 | 149 |
  • 150 | 151 | eff-bot's Daily Python URL 152 | 153 | - Contains links to assorted resources from the Python universe, compiled by PythonWare. 154 | 155 |
    [!]
    156 |
    • 157 | 158 |
  • 159 | 160 | Free Python and Zope Hosting Directory 161 | 162 | - A directory of free Python and Zope hosting providers, with reviews and ratings. 163 | 164 |
    [!]
    165 |
    • 166 | 167 |
  • 168 | 169 | O'Reilly Python Center 170 | 171 | - Features Python books, resources, news and articles. 172 | 173 |
    [!]
    174 |
    • 175 | 176 |
  • 177 | 178 | Python Developer's Guide 179 | 180 | - Resources for reporting bugs, accessing the Python source tree with CVS and taking part in the development of Python. 181 | 182 |
    [!]
    183 |
    • 184 | 185 |
  • 186 | 187 | Social Bug 188 | 189 | - Scripts, examples and news about Python programming for the Windows platform. 190 | 191 |
    [!]
    192 |
    • 193 | 194 |
195 |
196 | 197 | 198 | 199 | 200 |
201 | 202 | 223 | 224 |
225 | 226 |
227 | 228 | 229 | 230 | Volunteer to edit this category. 231 | 232 | 233 | 234 | 235 |
236 | 237 | 238 | 239 | 240 |
241 | 242 |
243 |
244 |
245 | 246 | 247 | Copyright © 1998-2014 AOL Inc. 248 | 249 | Terms of Use 250 |
251 |
252 | Visit our sister sites  mozilla.org | MusicMoz | Wikipedia

253 |
Last update: 254 | 255 | May 12, 2014 at 12:44:22 UTC 256 | 257 | - edit
258 |
259 |
260 |
261 | 264 |
265 | 266 | 267 | 268 | -------------------------------------------------------------------------------- /tutorial/dmoz.json: -------------------------------------------------------------------------------- 1 | [{"url": ["http://www.pythonware.com/daily/"], "name": ["eff-bot's Daily Python URL"], "description": ["- Contains links to assorted resources from the Python universe, compiled by PythonWare.\r"]}, 2 | {"url": ["http://www.oinko.net/freepython/"], "name": ["Free Python and Zope Hosting Directory"], "description": ["- A directory of free Python and Zope hosting providers, with reviews and ratings.\r"]}, 3 | {"url": ["http://oreilly.com/python/"], "name": ["O'Reilly Python Center"], "description": ["- Features Python books, resources, news and articles.\r"]}, 4 | {"url": ["https://www.python.org/dev/"], "name": ["Python Developer's Guide"], "description": ["- Resources for reporting bugs, accessing the Python source tree with CVS and taking part in the development of Python.\r"]}, 5 | {"url": ["http://win32com.goermezer.de/"], "name": ["Social Bug"], "description": ["- Scripts, examples and news about Python programming for the Windows platform.\r"]}, 6 | {"url": ["http://www.pearsonhighered.com/educator/academic/product/0,,0130260363,00%2Ben-USS_01DBC.html"], "name": ["Core Python Programming"], "description": ["- By Wesley J. Chun; Prentice Hall PTR, 2001, ISBN 0130260363. For experienced developers to improve extant skills; professional level examples. Starts by introducing syntax, objects, error handling, functions, classes, built-ins. [Prentice Hall]\r"]}, 7 | {"url": ["http://www.brpreiss.com/books/opus7/html/book.html"], "name": ["Data Structures and Algorithms with Object-Oriented Design Patterns in Python"], "description": ["- The primary goal of this book is to promote object-oriented design using Python and to illustrate the use of the emerging object-oriented design patterns.\r"]}, 8 | {"url": ["http://www.diveintopython.net/"], "name": ["Dive Into Python 3"], "description": ["- By Mark Pilgrim, Guide to Python 3 and its differences from Python 2. Each chapter starts with a real code sample and explains it fully. Has a comprehensive appendix of all the syntactic and semantic changes in Python 3\r"]}, 9 | {"url": ["http://rhodesmill.org/brandon/2011/foundations-of-python-network-programming/"], "name": ["Foundations of Python Network Programming"], "description": ["- This book covers a wide range of topics. From raw TCP and UDP to encryption with TSL, and then to HTTP, SMTP, POP, IMAP, and ssh. It gives you a good understanding of each field and how to do everything on the network with Python.\r"]}, 10 | {"url": ["http://www.techbooksforfree.com/perlpython.shtml"], "name": ["Free Python books"], "description": ["- Free Python books and tutorials.\r"]}, 11 | {"url": ["http://www.freetechbooks.com/python-f6.html"], "name": ["FreeTechBooks: Python Scripting Language"], "description": ["- Annotated list of free online books on Python scripting language. Topics range from beginner to advanced.\r"]}, 12 | {"url": ["http://greenteapress.com/thinkpython/"], "name": ["How to Think Like a Computer Scientist: Learning with Python"], "description": ["- By Allen B. Downey, Jeffrey Elkner, Chris Meyers; Green Tea Press, 2002, ISBN 0971677506. Teaches general principles of programming, via Python as subject language. Thorough, in-depth approach to many basic and intermediate programming topics. Full text online and downloads: HTML, PDF, PS, LaTeX. [Free, Green Tea Press]\r"]}, 13 | {"url": ["http://www.network-theory.co.uk/python/intro/"], "name": ["An Introduction to Python"], "description": ["- By Guido van Rossum, Fred L. Drake, Jr.; Network Theory Ltd., 2003, ISBN 0954161769. Printed edition of official tutorial, for v2.x, from Python.org. [Network Theory, online]\r"]}, 14 | {"url": ["http://www.freenetpages.co.uk/hp/alan.gauld/"], "name": ["Learn to Program Using Python"], "description": ["- Book by Alan Gauld with full text online. Introduction for those learning programming basics: terminology, concepts, methods to write code. Assumes no prior knowledge but basic computer skills.\r"]}, 15 | {"url": ["http://www.wiley.com/WileyCDA/WileyTitle/productCd-0471219754.html"], "name": ["Making Use of Python"], "description": ["- By Rashi Gupta; John Wiley and Sons, 2002, ISBN 0471219754. Covers language basics, use for CGI scripting, GUI development, network programming; shows why it is one of more sophisticated of popular scripting languages. [Wiley]\r"]}, 16 | {"url": ["http://hetland.org/writing/practical-python/"], "name": ["Practical Python"], "description": ["- By Magnus Lie Hetland; Apress LP, 2002, ISBN 1590590066. Readable guide to ideas most vital to new users, from basics common to high level languages, to more specific aspects, to a series of 10 ever more complex programs. [Apress]\r"]}, 17 | {"url": ["http://sysadminpy.com/"], "name": ["Pro Python System Administration"], "description": ["- By Rytis Sileika, ISBN13: 978-1-4302-2605-5, Uses real-world system administration examples like manage devices with SNMP and SOAP, build a distributed monitoring system, manage web applications and parse complex log files, monitor and manage MySQL databases.\r"]}, 18 | {"url": ["http://www.qtrac.eu/py3book.html"], "name": ["Programming in Python 3 (Second Edition)"], "description": ["- A Complete Introduction to the Python 3.\r"]}, 19 | {"url": ["http://www.wiley.com/WileyCDA/WileyTitle/productCd-0764548077.html"], "name": ["Python 2.1 Bible"], "description": ["- By Dave Brueck, Stephen Tanner; John Wiley and Sons, 2001, ISBN 0764548077. Full coverage, clear explanations, hands-on examples, full language reference; shows step by step how to use components, assemble them, form full-featured programs. [John Wiley and Sons]\r"]}, 20 | {"url": ["https://www.packtpub.com/python-3-object-oriented-programming/book"], "name": ["Python 3 Object Oriented Programming"], "description": ["- A step-by-step tutorial for OOP in Python 3, including discussion and examples of abstraction, encapsulation, information hiding, and raise, handle, define, and manipulate exceptions.\r"]}, 21 | {"url": ["http://www.network-theory.co.uk/python/language/"], "name": ["Python Language Reference Manual"], "description": ["- By Guido van Rossum, Fred L. Drake, Jr.; Network Theory Ltd., 2003, ISBN 0954161785. Printed edition of official language reference, for v2.x, from Python.org, describes syntax, built-in datatypes. [Network Theory, online]\r"]}, 22 | {"url": ["http://www.pearsonhighered.com/educator/academic/product/0,,0130409561,00%2Ben-USS_01DBC.html"], "name": ["Python Programming Patterns"], "description": ["- By Thomas W. Christopher; Prentice Hall PTR, 2002, ISBN 0130409561. Shows how to write large programs, introduces powerful design patterns that deliver high levels of robustness, scalability, reuse.\r"]}, 23 | {"url": ["http://www.informit.com/store/product.aspx?isbn=0201616165&redir=1"], "name": ["Python Programming with the Java Class Libraries: A Tutorial for Building Web and Enterprise Applications with Jython"], "description": ["- By Richard Hightower; Addison-Wesley, 2002, 0201616165. Begins with Python basics, many exercises, interactive sessions. Shows programming novices concepts and practical methods. Shows programming experts Python's abilities and ways to interface with Java APIs. [publisher website]\r"]}, 24 | {"url": ["http://www.pearsonhighered.com/educator/academic/product/0,,0201748843,00%2Ben-USS_01DBC.html"], "name": ["Python: Visual QuickStart Guide"], "description": ["- By Chris Fehily; Peachpit Press, 2002, ISBN 0201748843. Task-based, step-by-step visual reference guide, many screen shots, for courses in digital graphics; Web design, scripting, development; multimedia, page layout, office tools, operating systems. [Prentice Hall]\r"]}, 25 | {"url": ["http://www.informit.com/store/product.aspx?isbn=0672317354"], "name": ["Sams Teach Yourself Python in 24 Hours"], "description": ["- By Ivan Van Laningham; Sams Publishing, 2000, ISBN 0672317354. Split into 24 hands-on, 1 hour lessons; steps needed to learn topic: syntax, language features, OO design and programming, GUIs (Tkinter), system administration, CGI. [Sams Publishing]\r"]}, 26 | {"url": ["http://gnosis.cx/TPiP/"], "name": ["Text Processing in Python"], "description": ["- By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.]\r"]}, 27 | {"url": ["http://www.informit.com/store/product.aspx?isbn=0130211192"], "name": ["XML Processing with Python"], "description": ["- By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\r"]}] -------------------------------------------------------------------------------- /tutorial/scrapy.cfg: -------------------------------------------------------------------------------- 1 | # Automatically created by: scrapy startproject 2 | # 3 | # For more information about the [deploy] section see: 4 | # http://doc.scrapy.org/en/latest/topics/scrapyd.html 5 | 6 | [settings] 7 | default = tutorial.settings 8 | 9 | [deploy] 10 | #url = http://localhost:6800/ 11 | project = tutorial 12 | -------------------------------------------------------------------------------- /tutorial/tutorial/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/tutorial/tutorial/__init__.py -------------------------------------------------------------------------------- /tutorial/tutorial/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/tutorial/tutorial/__init__.pyc -------------------------------------------------------------------------------- /tutorial/tutorial/items.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your scraped items 4 | # 5 | # See documentation in: 6 | # http://doc.scrapy.org/en/latest/topics/items.html 7 | from scrapy.item import Item, Field 8 | 9 | 10 | class DmozItem(Item): 11 | # define the fields for your item here like: 12 | name = Field() 13 | description = Field() 14 | url = Field() 15 | 16 | -------------------------------------------------------------------------------- /tutorial/tutorial/items.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/tutorial/tutorial/items.pyc -------------------------------------------------------------------------------- /tutorial/tutorial/pipelines.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define your item pipelines here 4 | # 5 | # Don't forget to add your pipeline to the ITEM_PIPELINES setting 6 | # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html 7 | from scrapy.exceptions import DropItem 8 | 9 | class TutorialPipeline(object): 10 | """A pipeline for filtering out items which contain certain words in their 11 | description""" 12 | 13 | # put all words in lowercase 14 | words_to_filter = ['politics', 'religion'] 15 | 16 | def process_item(self, item, spider): 17 | for word in self.words_to_filter: 18 | if word in unicode(item['description']).lower(): 19 | raise DropItem("Contains forbidden word: %s" % word) 20 | else: 21 | return item -------------------------------------------------------------------------------- /tutorial/tutorial/pipelines.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/tutorial/tutorial/pipelines.pyc -------------------------------------------------------------------------------- /tutorial/tutorial/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Scrapy settings for tutorial project 4 | # 5 | # For simplicity, this file contains only the most important settings by 6 | # default. All the other settings are documented here: 7 | # 8 | # http://doc.scrapy.org/en/latest/topics/settings.html 9 | # 10 | 11 | BOT_NAME = 'tutorial' 12 | 13 | SPIDER_MODULES = ['tutorial.spiders'] 14 | NEWSPIDER_MODULE = 'tutorial.spiders' 15 | ITEM_PIPELINES = {'tutorial.pipelines.TutorialPipeline': 200} 16 | # Crawl responsibly by identifying yourself (and your website) on the user-agent 17 | #USER_AGENT = 'tutorial (+http://www.yourdomain.com)' 18 | -------------------------------------------------------------------------------- /tutorial/tutorial/settings.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/tutorial/tutorial/settings.pyc -------------------------------------------------------------------------------- /tutorial/tutorial/spiders/__init__.py: -------------------------------------------------------------------------------- 1 | # This package will contain the spiders of your Scrapy project 2 | # 3 | # Please refer to the documentation for information on how to create and manage 4 | # your spiders. 5 | -------------------------------------------------------------------------------- /tutorial/tutorial/spiders/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/tutorial/tutorial/spiders/__init__.pyc -------------------------------------------------------------------------------- /tutorial/tutorial/spiders/dmoz_spider.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | 4 | from scrapy.spider import Spider 5 | from scrapy.selector import Selector 6 | from tutorial.items import DmozItem 7 | 8 | 9 | class DmozSpider(Spider): 10 | name = "dmoz" 11 | allowed_domains = ["dmoz.org"] 12 | start_urls = [ 13 | "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", 14 | "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/", 15 | ] 16 | 17 | def parse(self, response): 18 | """ 19 | The lines below is a spider contract. For more info see: 20 | http://doc.scrapy.org/en/latest/topics/contracts.html 21 | @url http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/ 22 | @scrapes name 23 | """ 24 | sel = Selector(response) 25 | sites = sel.xpath('//ul[@class="directory-url"]/li') 26 | items = [] 27 | 28 | for site in sites: 29 | item = DmozItem() 30 | item['name'] = site.xpath('a/text()').extract() 31 | item['url'] = site.xpath('a/@href').extract() 32 | item['description'] = site.xpath('text()').re('-\s[^\n]*\\r') 33 | items.append(item) 34 | 35 | return items -------------------------------------------------------------------------------- /tutorial/tutorial/spiders/dmoz_spider.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/tutorial/tutorial/spiders/dmoz_spider.pyc -------------------------------------------------------------------------------- /xiaobaihe/scrapy.cfg: -------------------------------------------------------------------------------- 1 | # Automatically created by: scrapy startproject 2 | # 3 | # For more information about the [deploy] section see: 4 | # http://doc.scrapy.org/en/latest/topics/scrapyd.html 5 | 6 | [settings] 7 | default = xiaobaihe.settings 8 | 9 | [deploy] 10 | #url = http://localhost:6800/ 11 | project = xiaobaihe 12 | -------------------------------------------------------------------------------- /xiaobaihe/xiaobaihe/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/xiaobaihe/xiaobaihe/__init__.py -------------------------------------------------------------------------------- /xiaobaihe/xiaobaihe/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/xiaobaihe/xiaobaihe/__init__.pyc -------------------------------------------------------------------------------- /xiaobaihe/xiaobaihe/items.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your scraped items 4 | # 5 | # See documentation in: 6 | # http://doc.scrapy.org/en/latest/topics/items.html 7 | 8 | from scrapy.item import Item, Field 9 | 10 | 11 | class XiaobaiheItem(Item): 12 | # define the fields for your item here like: 13 | # name = scrapy.Field() 14 | username = Field() 15 | text = Field() 16 | -------------------------------------------------------------------------------- /xiaobaihe/xiaobaihe/items.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/xiaobaihe/xiaobaihe/items.pyc -------------------------------------------------------------------------------- /xiaobaihe/xiaobaihe/pipelines.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define your item pipelines here 4 | # 5 | # Don't forget to add your pipeline to the ITEM_PIPELINES setting 6 | # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html 7 | 8 | 9 | class XiaobaihePipeline(object): 10 | def process_item(self, item, spider): 11 | return item 12 | -------------------------------------------------------------------------------- /xiaobaihe/xiaobaihe/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Scrapy settings for xiaobaihe project 4 | # 5 | # For simplicity, this file contains only the most important settings by 6 | # default. All the other settings are documented here: 7 | # 8 | # http://doc.scrapy.org/en/latest/topics/settings.html 9 | # 10 | 11 | BOT_NAME = 'xiaobaihe' 12 | 13 | SPIDER_MODULES = ['xiaobaihe.spiders'] 14 | NEWSPIDER_MODULE = 'xiaobaihe.spiders' 15 | 16 | # Crawl responsibly by identifying yourself (and your website) on the user-agent 17 | #USER_AGENT = 'xiaobaihe (+http://www.yourdomain.com)' 18 | -------------------------------------------------------------------------------- /xiaobaihe/xiaobaihe/settings.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/xiaobaihe/xiaobaihe/settings.pyc -------------------------------------------------------------------------------- /xiaobaihe/xiaobaihe/spiders/__init__.py: -------------------------------------------------------------------------------- 1 | # This package will contain the spiders of your Scrapy project 2 | # 3 | # Please refer to the documentation for information on how to create and manage 4 | # your spiders. 5 | -------------------------------------------------------------------------------- /xiaobaihe/xiaobaihe/spiders/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/xiaobaihe/xiaobaihe/spiders/__init__.pyc -------------------------------------------------------------------------------- /xiaobaihe/xiaobaihe/spiders/xiaobaohe_spider.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | 4 | from scrapy.contrib.spiders import CrawlSpider, Rule 5 | from scrapy.selector import Selector 6 | from xiaobaihe.items import XiaobaiheItem 7 | from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor 8 | import sys 9 | reload(sys) 10 | sys.setdefaultencoding('utf8') 11 | 12 | class XiaoBaiHeSpider(CrawlSpider) : 13 | 14 | name = "xiaobaihe" 15 | allowed_domains = ["bbs.nju.edu.cn"] 16 | start_urls = ["http://bbs.nju.edu.cn/bbstdoc?board=D_Computer"] 17 | rules = ( 18 | #对上一页网页加入url列表 19 | Rule(SgmlLinkExtractor(allow = ('http://bbs\.nju\.edu\.cn/bbstdoc\?board=D_Computer&start=\d+')), follow = True), 20 | #将读取的网页进行分析 21 | Rule(SgmlLinkExtractor(allow = ('http://bbs\.nju\.edu\.cn/bbstcon\?board=D_Computer&file=M\.\d+\.A')), callback = 'parse_page') 22 | ) 23 | 24 | def parse_page(self, response) : 25 | sel = Selector(response) 26 | item = XiaobaiheItem() 27 | item['username'] = sel.xpath('//table/tr/td/a/text()').extract()[2] 28 | item['text'] = sel.xpath("//textarea/text()").extract()[0] 29 | return item 30 | 31 | -------------------------------------------------------------------------------- /xiaobaihe/xiaobaihe/spiders/xiaobaohe_spider.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/xiaobaihe/xiaobaihe/spiders/xiaobaohe_spider.pyc -------------------------------------------------------------------------------- /zhihu/scrapy.cfg: -------------------------------------------------------------------------------- 1 | # Automatically created by: scrapy startproject 2 | # 3 | # For more information about the [deploy] section see: 4 | # http://doc.scrapy.org/en/latest/topics/scrapyd.html 5 | 6 | [settings] 7 | default = zhihu.settings 8 | 9 | [deploy] 10 | #url = http://localhost:6800/ 11 | project = zhihu 12 | -------------------------------------------------------------------------------- /zhihu/zhihu/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/zhihu/zhihu/__init__.py -------------------------------------------------------------------------------- /zhihu/zhihu/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/zhihu/zhihu/__init__.pyc -------------------------------------------------------------------------------- /zhihu/zhihu/items.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your scraped items 4 | # 5 | # See documentation in: 6 | # http://doc.scrapy.org/en/latest/topics/items.html 7 | 8 | from scrapy.item import Item, Field 9 | 10 | 11 | class ZhihuItem(Item): 12 | # define the fields for your item here like: 13 | # name = scrapy.Field() 14 | url = Field() #保存抓取问题的url 15 | title = Field() #抓取问题的标题 16 | description = Field() #抓取问题的描述 17 | answer = Field() #抓取问题的答案 18 | name = Field() #个人用户的名称 19 | 20 | -------------------------------------------------------------------------------- /zhihu/zhihu/items.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/zhihu/zhihu/items.pyc -------------------------------------------------------------------------------- /zhihu/zhihu/pipelines.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define your item pipelines here 4 | # 5 | # Don't forget to add your pipeline to the ITEM_PIPELINES setting 6 | # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html 7 | 8 | 9 | class ZhihuPipeline(object): 10 | def process_item(self, item, spider): 11 | return item 12 | -------------------------------------------------------------------------------- /zhihu/zhihu/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Scrapy settings for zhihu project 4 | # 5 | # For simplicity, this file contains only the most important settings by 6 | # default. All the other settings are documented here: 7 | # 8 | # http://doc.scrapy.org/en/latest/topics/settings.html 9 | # 10 | 11 | BOT_NAME = 'zhihu' 12 | 13 | SPIDER_MODULES = ['zhihu.spiders'] 14 | NEWSPIDER_MODULE = 'zhihu.spiders' 15 | DOWNLOAD_DELAY = 0.25 16 | 17 | # Crawl responsibly by identifying yourself (and your website) on the user-agent 18 | #USER_AGENT = 'zhihu (+http://www.yourdomain.com)' 19 | -------------------------------------------------------------------------------- /zhihu/zhihu/settings.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/zhihu/zhihu/settings.pyc -------------------------------------------------------------------------------- /zhihu/zhihu/spiders/__init__.py: -------------------------------------------------------------------------------- 1 | # This package will contain the spiders of your Scrapy project 2 | # 3 | # Please refer to the documentation for information on how to create and manage 4 | # your spiders. 5 | -------------------------------------------------------------------------------- /zhihu/zhihu/spiders/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/zhihu/zhihu/spiders/__init__.pyc -------------------------------------------------------------------------------- /zhihu/zhihu/spiders/zhihu_spider.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | """ 4 | 一个简单的Python爬虫, 用于抓取coursera网站的下载链接和pdf 5 | 6 | Anthor: Andrew Liu 7 | Version: 0.0.2 8 | Date: 2014-12-15 9 | Language: Python2.7.8 10 | Editor: Sublime Text2 11 | Operate: 具体操作请看README.md介绍 12 | """ 13 | from scrapy.contrib.spiders import CrawlSpider, Rule 14 | from scrapy.selector import Selector 15 | from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor 16 | from scrapy.http import Request, FormRequest 17 | from zhihu.items import ZhihuItem 18 | 19 | 20 | 21 | class ZhihuSipder(CrawlSpider) : 22 | name = "zhihu" 23 | allowed_domains = ["www.zhihu.com"] 24 | start_urls = [ 25 | "http://www.zhihu.com" 26 | ] 27 | rules = ( 28 | Rule(SgmlLinkExtractor(allow = ('/question/\d+#.*?', )), callback = 'parse_page', follow = True), 29 | Rule(SgmlLinkExtractor(allow = ('/question/\d+', )), callback = 'parse_page', follow = True), 30 | ) 31 | headers = { 32 | "Accept": "*/*", 33 | "Accept-Encoding": "gzip,deflate", 34 | "Accept-Language": "en-US,en;q=0.8,zh-TW;q=0.6,zh;q=0.4", 35 | "Connection": "keep-alive", 36 | "Content-Type":" application/x-www-form-urlencoded; charset=UTF-8", 37 | "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36", 38 | "Referer": "http://www.zhihu.com/" 39 | } 40 | 41 | #重写了爬虫类的方法, 实现了自定义请求, 运行成功后会调用callback回调函数 42 | def start_requests(self): 43 | return [Request("https://www.zhihu.com/login", meta = {'cookiejar' : 1}, callback = self.post_login)] 44 | 45 | #FormRequeset出问题了 46 | def post_login(self, response): 47 | print 'Preparing login' 48 | #下面这句话用于抓取请求网页后返回网页中的_xsrf字段的文字, 用于成功提交表单 49 | xsrf = Selector(response).xpath('//input[@name="_xsrf"]/@value').extract()[0] 50 | print xsrf 51 | #FormRequeset.from_response是Scrapy提供的一个函数, 用于post表单 52 | #登陆成功后, 会调用after_login回调函数 53 | return [FormRequest.from_response(response, #"http://www.zhihu.com/login", 54 | meta = {'cookiejar' : response.meta['cookiejar']}, 55 | headers = self.headers, 56 | formdata = { 57 | '_xsrf': xsrf, 58 | 'email': '123456', 59 | 'password': '123456' 60 | }, 61 | callback = self.after_login, 62 | dont_filter = True 63 | )] 64 | 65 | def after_login(self, response) : 66 | for url in self.start_urls : 67 | yield self.make_requests_from_url(url) 68 | 69 | def parse_page(self, response): 70 | problem = Selector(response) 71 | item = ZhihuItem() 72 | item['url'] = response.url 73 | item['name'] = problem.xpath('//span[@class="name"]/text()').extract() 74 | print item['name'] 75 | item['title'] = problem.xpath('//h2[@class="zm-item-title zm-editable-content"]/text()').extract() 76 | item['description'] = problem.xpath('//div[@class="zm-editable-content"]/text()').extract() 77 | item['answer']= problem.xpath('//div[@class=" zm-editable-content clearfix"]/text()').extract() 78 | return item 79 | -------------------------------------------------------------------------------- /zhihu/zhihu/spiders/zhihu_spider.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Andrew-liu/scrapy_example/6fba398bb4a6f40e1a907611d165b495d5d693a1/zhihu/zhihu/spiders/zhihu_spider.pyc --------------------------------------------------------------------------------