├── .gitattributes ├── .idea ├── ProductAnalysis.iml ├── inspectionProfiles │ └── profiles_settings.xml ├── markdown-navigator.xml ├── markdown-navigator │ └── profiles_settings.xml ├── misc.xml ├── modules.xml └── vcs.xml ├── ProductAnalysis ├── __init__.py ├── settings.py ├── urls.py └── wsgi.py ├── README.md ├── app ├── ProductData │ ├── __init__.py │ ├── config.py │ ├── items.py │ ├── log_helper.py │ ├── middlewares.py │ ├── pipelines.py │ ├── redis_helper.py │ ├── run.py │ ├── settings.py │ ├── spiders │ │ ├── __init__.py │ │ └── zol.py │ └── sql_hepler.py ├── __init__.py ├── admin.py ├── apps.py ├── migrations │ ├── 0001_initial.py │ └── __init__.py ├── models.py ├── scrapy.cfg ├── search_indexes.py ├── templatetags │ ├── __init__.py │ └── product_tags.py ├── tests.py └── views.py ├── doc ├── django1.8中文文档.pdf ├── django之查询.md ├── python必备知识点.md ├── scrapy知识汇总.md └── 数据分析之pandas.md ├── hepler ├── __init__.py ├── config.py ├── file_hepler.py ├── log_helper.py ├── make_bokeh_hepler.py ├── make_gif_helper.py ├── make_plot_helper.py └── redis_helper.py ├── manage.py ├── requirements.txt ├── screen ├── Scrapy流程.png ├── index.png └── search.png ├── static ├── css │ ├── bokeh-0.12.6.min.css │ ├── bootstrap.min.css │ ├── font-awesome.min.css │ ├── light-bootstrap-dashboard.css │ ├── loading.css │ └── main.css ├── file │ └── product_info.json ├── fonts │ ├── FontAwesome.otf │ ├── fontawesome-webfont.eot │ ├── fontawesome-webfont.svg │ ├── fontawesome-webfont.ttf │ └── fontawesome-webfont.woff ├── gif │ ├── loading.gif │ └── loading1.gif ├── img │ ├── bg-1.jpg │ ├── bg-2.jpg │ ├── load.gif │ ├── logo.png │ └── sidebar.jpg ├── js │ ├── bokeh-0.12.6.min.js │ ├── bootstrap.min.js │ ├── jquery-1.11.1.min.js │ ├── jquery.backstretch.min.js │ ├── jquery.simple-text-rotator.min.js │ ├── loading.js │ ├── main.js │ ├── search.js │ └── wow.min.js └── upload │ ├── no_good_comments.png │ ├── no_hot.png │ └── no_overview.png └── templates ├── analysis.html ├── base.html ├── index.html └── search ├── indexes └── app │ └── productinfo_text.txt └── search.html /.gitattributes: -------------------------------------------------------------------------------- 1 | *.js linguist-language=Python 2 | *.css linguist-language=Python 3 | *.html linguist-language=Python 4 | -------------------------------------------------------------------------------- /.idea/ProductAnalysis.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 26 | 27 | 28 | 30 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | -------------------------------------------------------------------------------- /.idea/markdown-navigator.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 33 | 34 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /.idea/markdown-navigator/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /ProductAnalysis/__init__.py: -------------------------------------------------------------------------------- 1 | import pymysql 2 | pymysql.install_as_MySQLdb() -------------------------------------------------------------------------------- /ProductAnalysis/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for ProductAnalysis project. 3 | 4 | Generated by 'django-admin startproject' using Django 1.10.6. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.10/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/1.10/ref/settings/ 11 | """ 12 | 13 | import os 14 | 15 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 16 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 17 | 18 | 19 | # Quick-start development settings - unsuitable for production 20 | # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ 21 | 22 | # SECURITY WARNING: keep the secret key used in production secret! 23 | SECRET_KEY = 'gq(b4zs=z+ckpyfol7*qbzm60$2i)j3jg66mf5f0%1w=d4$vsr' 24 | 25 | # SECURITY WARNING: don't run with debug turned on in production! 26 | DEBUG = True 27 | 28 | ALLOWED_HOSTS = ["127.0.0.1","192.168.0.123"] 29 | 30 | 31 | # Application definition 32 | 33 | INSTALLED_APPS = [ 34 | 'django.contrib.admin', 35 | 'django.contrib.auth', 36 | 'django.contrib.contenttypes', 37 | 'django.contrib.sessions', 38 | 'django.contrib.messages', 39 | 'django.contrib.staticfiles', 40 | 'app', 41 | 'haystack', 42 | ] 43 | 44 | MIDDLEWARE = [ 45 | 'django.middleware.security.SecurityMiddleware', 46 | 'django.contrib.sessions.middleware.SessionMiddleware', 47 | 'django.middleware.common.CommonMiddleware', 48 | 'django.middleware.csrf.CsrfViewMiddleware', 49 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 50 | 'django.contrib.messages.middleware.MessageMiddleware', 51 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 52 | ] 53 | 54 | ROOT_URLCONF = 'ProductAnalysis.urls' 55 | 56 | TEMPLATES = [ 57 | { 58 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 59 | 'DIRS': [os.path.join(BASE_DIR, 'templates')] 60 | , 61 | 'APP_DIRS': True, 62 | 'OPTIONS': { 63 | 'context_processors': [ 64 | 'django.template.context_processors.debug', 65 | 'django.template.context_processors.request', 66 | 'django.contrib.auth.context_processors.auth', 67 | 'django.contrib.messages.context_processors.messages', 68 | ], 69 | }, 70 | }, 71 | ] 72 | 73 | WSGI_APPLICATION = 'ProductAnalysis.wsgi.application' 74 | 75 | 76 | # Database 77 | # https://docs.djangoproject.com/en/1.10/ref/settings/#databases 78 | 79 | DATABASES = { 80 | 'default': { 81 | 'ENGINE': 'django.db.backends.mysql', 82 | 'NAME': 'productanalysis', 83 | 'USER': 'root', 84 | 'PASSWORD': '', 85 | 'HOST': '', 86 | 'PORT': '', 87 | } 88 | } 89 | 90 | 91 | # Password validation 92 | # https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators 93 | 94 | AUTH_PASSWORD_VALIDATORS = [ 95 | { 96 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 97 | }, 98 | { 99 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 100 | }, 101 | { 102 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 103 | }, 104 | { 105 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 106 | }, 107 | ] 108 | 109 | 110 | # Internationalization 111 | # https://docs.djangoproject.com/en/1.10/topics/i18n/ 112 | 113 | LANGUAGE_CODE = 'zh-Hans' 114 | 115 | TIME_ZONE = 'Asia/Shanghai' 116 | 117 | USE_I18N = True 118 | 119 | USE_L10N = True 120 | 121 | USE_TZ = True 122 | 123 | 124 | # Static files (CSS, JavaScript, Images) 125 | # https://docs.djangoproject.com/en/1.10/howto/static-files/ 126 | 127 | # 添加haystack配置 128 | HAYSTACK_CONNECTIONS = { 129 | 'default': { 130 | 'ENGINE': 'haystack.backends.whoosh_cn_backend.WhooshEngine', 131 | 'PATH': os.path.join(BASE_DIR, 'whoosh_index'), 132 | } 133 | } 134 | HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor' 135 | 136 | 137 | STATIC_URL = '/static/' 138 | STATICFILES_DIRS=( 139 | os.path.join(BASE_DIR,'static'), 140 | ) 141 | -------------------------------------------------------------------------------- /ProductAnalysis/urls.py: -------------------------------------------------------------------------------- 1 | """ProductAnalysis URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/1.10/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.conf.urls import url, include 14 | 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) 15 | """ 16 | from django.conf.urls import url,include 17 | from django.contrib import admin 18 | from app import views 19 | 20 | urlpatterns = [ 21 | url(r'^admin/', admin.site.urls), 22 | url(r'^$',views.index), 23 | url(r'^search/', include('haystack.urls')), 24 | url(r'^make_plot/',views.make_plot), 25 | url(r'^get_analysis_res/',views.make_bokeh), 26 | ] 27 | -------------------------------------------------------------------------------- /ProductAnalysis/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for ProductAnalysis project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.10/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ProductAnalysis.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ProductAnalysis 2 | 3 | 项目说明: 4 | 5 | 主要是用来练习数据抓取和数据分析,并通过前端页面来显示分析结果,目前只抓取zol中的手机,笔记本,数码相机 6 | 7 | 相关知识点放在doc目录,更详细的还是要去官方文档查看 8 | 9 | 请用IE或Chrome浏览器打开,本人不是做前端的,好像别的浏览器打开有问题 10 | 11 | 使用步骤: 12 | 13 | 1.先mysql数据创建数据库名为:productanalysis,数据库编码一定要设置成utf-8 14 | 15 | 2.同步数据库:python manage.py makemigrations 和 python manage.py migrate 16 | 17 | 3.进入app/ProductData,执行:run.py 18 | 19 | 等待执行结束就可以了 20 | 21 | 用到技术点: 22 | 23 | 1.通过django-haystack来实现全文搜索 24 | 25 | 2.直接通过scrapy-redis来实现去重 26 | 27 | 3.前端数据分析展示通过bokeh来实现 28 | 29 | 4.数据分析用的是pandas 30 | 31 | 5.通过redis进行缓存处理 32 | 33 | ## 首页 34 | ![](./screen/index.png) 35 | ## 分析结果页 36 | ![](./screen/search.png) 37 | -------------------------------------------------------------------------------- /app/ProductData/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/app/ProductData/__init__.py -------------------------------------------------------------------------------- /app/ProductData/config.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | productanalysis_db_config = { 4 | 'host': 'localhost', 5 | 'port': 3306, 6 | 'user': 'root', 7 | 'password': '', 8 | 'charset': 'utf8', 9 | } 10 | 11 | productanalysis_db = 'productanalysis' 12 | 13 | product_info = 'product_info' 14 | 15 | 16 | # 请求的header 17 | headers = { 18 | "Host": "detail.zol.com.cn", 19 | "Connection": "keep-alive", 20 | "Pragma": "no-cache", 21 | "Cache-Control": "no-cache", 22 | "Upgrade-Insecure-Requests": 1, 23 | "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36", 24 | "Accept": "text/html, application/xhtml + xml, application/xml;q = 0.9, image/webp, * / *;q = 0.8", 25 | "Accept-Encoding": "gzip, deflate, sdch", 26 | "Accept-Language": "zh-CN,zh;q=0.8", 27 | } 28 | 29 | # redis 30 | redis_pass = '' 31 | redis_host = '127.0.0.1' 32 | redis_port = '6379' 33 | 34 | REDIS_PRODUCT_INFO_EXPIRES_SECONDES = 86400 # 设置存储过期时间,单位为秒 -------------------------------------------------------------------------------- /app/ProductData/items.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your scraped items 4 | # 5 | # See documentation in: 6 | # http://doc.scrapy.org/en/latest/topics/items.html 7 | 8 | import scrapy 9 | 10 | 11 | class ProductdataItem(scrapy.Item): 12 | # define the fields for your item here like: 13 | # name = scrapy.Field() 14 | """ 15 | p_url 16 | p_title 17 | p_c_score 18 | p_img 19 | p_prices 20 | p_c_all_nums 21 | p_comments 22 | p_c_times 23 | p_id 24 | p_price_trend 25 | """ 26 | 27 | p_url = scrapy.Field() 28 | p_title = scrapy.Field() 29 | p_c_score = scrapy.Field() 30 | p_img = scrapy.Field() 31 | p_id = scrapy.Field() 32 | p_prices = scrapy.Field() 33 | p_c_all_nums = scrapy.Field() 34 | p_comments = scrapy.Field() 35 | p_c_times = scrapy.Field() 36 | p_price_trend = scrapy.Field() 37 | 38 | -------------------------------------------------------------------------------- /app/ProductData/log_helper.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | import logging 4 | import datetime 5 | 6 | def log(msg,level = logging.DEBUG): 7 | """ 8 | 自定义输出日志 9 | :param msg: 日志信息 10 | :param level: 日志等级 11 | :return: 12 | """ 13 | logging.log(level,msg) 14 | 15 | print("log时间[%s]-log等级[%s]-log信息[%s]" % (datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),level,msg)) 16 | -------------------------------------------------------------------------------- /app/ProductData/middlewares.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your spider middleware 4 | # 5 | # See documentation in: 6 | # http://doc.scrapy.org/en/latest/topics/spider-middleware.html 7 | 8 | from scrapy import signals 9 | from .settings import PROXIES 10 | import random 11 | 12 | class RandomProxy(object): 13 | """ 14 | 设置代理IP 15 | """ 16 | def process_request(self,request,spider): 17 | proxy = random.choice(PROXIES) 18 | request.meta['proxy'] = "http://" + proxy['ip_port'] 19 | 20 | -------------------------------------------------------------------------------- /app/ProductData/pipelines.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define your item pipelines here 4 | # 5 | # Don't forget to add your pipeline to the ITEM_PIPELINES setting 6 | # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html 7 | from . import redis_helper,config 8 | from .sql_hepler import SqlHelper 9 | p_sql = SqlHelper() 10 | 11 | class ProductdataPipeline(object): 12 | 13 | def process_item(self, item, spider): 14 | """ 15 | 将数据保存到mysql数据库 16 | :param item: 17 | :param spider: 18 | :return: 19 | """ 20 | item = dict(item) 21 | product_info = { 22 | 'p_url': item['p_url'], 23 | 'p_title': item['p_title'], 24 | 'p_img': item['p_img'], 25 | 'p_id': item['p_id'], 26 | 'p_prices': item['p_prices'], 27 | 'p_c_score': item['p_c_score'], 28 | 'p_comments': item['p_comments'], 29 | 'p_c_all_nums': item['p_c_all_nums'], 30 | 'p_c_time': item['p_c_times'], 31 | 'p_price_trend': item['p_price_trend'], 32 | } 33 | p_sql.insert_json(product_info, config.product_info) 34 | 35 | return item -------------------------------------------------------------------------------- /app/ProductData/redis_helper.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | import redis 4 | from . import config 5 | import json 6 | 7 | p_redis = redis.StrictRedis(host=config.redis_host, port=config.redis_port) 8 | 9 | def save_to_redis(item): 10 | """ 11 | 保存数据的redis 12 | :param data: 需要保存的数据 13 | :return: 14 | """ 15 | p_redis.setex(item['p_id'],config.REDIS_PRODUCT_INFO_EXPIRES_SECONDES,json.dumps(item)) -------------------------------------------------------------------------------- /app/ProductData/run.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | from scrapy import cmdline 4 | 5 | 6 | def cmd(): 7 | cmdline.execute("scrapy crawl zol".split()) 8 | 9 | if __name__ == '__main__': 10 | cmd() -------------------------------------------------------------------------------- /app/ProductData/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Scrapy settings for ProductData project 4 | # 5 | # For simplicity, this file contains only settings considered important or 6 | # commonly used. You can find more settings consulting the documentation: 7 | # 8 | # http://doc.scrapy.org/en/latest/topics/settings.html 9 | # http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html 10 | # http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html 11 | 12 | BOT_NAME = 'ProductData' 13 | 14 | SPIDER_MODULES = ['ProductData.spiders'] 15 | NEWSPIDER_MODULE = 'ProductData.spiders' 16 | 17 | # 使用了scrapy-redis里的去重组件，不使用scrapy默认的去重 18 | DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter" 19 | # 使用了scrapy-redis里的调度器组件，不实用scrapy默认的调度器 20 | SCHEDULER = "scrapy_redis.scheduler.Scheduler" 21 | # 使用队列形式 22 | SCHEDULER_QUEUE_CLASS = "scrapy_redis.queue.SpiderQueue" 23 | # 允许暂停，redis请求记录不丢失 24 | SCHEDULER_PERSIST = True 25 | 26 | # Crawl responsibly by identifying yourself (and your website) on the user-agent 27 | #USER_AGENT = 'ProductData (+http://www.yourdomain.com)' 28 | 29 | # Obey robots.txt rules 30 | ROBOTSTXT_OBEY = False 31 | 32 | # Configure maximum concurrent requests performed by Scrapy (default: 16) 33 | #CONCURRENT_REQUESTS = 32 34 | 35 | # Configure a delay for requests for the same website (default: 0) 36 | # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay 37 | # See also autothrottle settings and docs 38 | # DOWNLOAD_DELAY = 2 39 | # The download delay setting will honor only one of: 40 | #CONCURRENT_REQUESTS_PER_DOMAIN = 16 41 | #CONCURRENT_REQUESTS_PER_IP = 16 42 | 43 | # Disable cookies (enabled by default) 44 | COOKIES_ENABLED = False 45 | 46 | # Disable Telnet Console (enabled by default) 47 | #TELNETCONSOLE_ENABLED = False 48 | 49 | # Override the default request headers: 50 | #DEFAULT_REQUEST_HEADERS = { 51 | # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 52 | # 'Accept-Language': 'en', 53 | #} 54 | 55 | # Enable or disable spider middlewares 56 | # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html 57 | #SPIDER_MIDDLEWARES = { 58 | # 'ProductData.middlewares.ProductdataSpiderMiddleware': 543, 59 | #} 60 | 61 | # Enable or disable downloader middlewares 62 | # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html 63 | # DOWNLOADER_MIDDLEWARES = { 64 | # 'ProductData.middlewares.RandomProxy': 100, 65 | # } 66 | 67 | # Enable or disable extensions 68 | # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html 69 | #EXTENSIONS = { 70 | # 'scrapy.extensions.telnet.TelnetConsole': None, 71 | #} 72 | 73 | # Configure item pipelines 74 | # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html 75 | ITEM_PIPELINES = { 76 | 'ProductData.pipelines.ProductdataPipeline': 300, 77 | } 78 | 79 | # Enable and configure the AutoThrottle extension (disabled by default) 80 | # See http://doc.scrapy.org/en/latest/topics/autothrottle.html 81 | #AUTOTHROTTLE_ENABLED = True 82 | # The initial download delay 83 | #AUTOTHROTTLE_START_DELAY = 5 84 | # The maximum download delay to be set in case of high latencies 85 | #AUTOTHROTTLE_MAX_DELAY = 60 86 | # The average number of requests Scrapy should be sending in parallel to 87 | # each remote server 88 | #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 89 | # Enable showing throttling stats for every response received: 90 | #AUTOTHROTTLE_DEBUG = False 91 | 92 | # Enable and configure HTTP caching (disabled by default) 93 | # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings 94 | #HTTPCACHE_ENABLED = True 95 | #HTTPCACHE_EXPIRATION_SECS = 0 96 | #HTTPCACHE_DIR = 'httpcache' 97 | #HTTPCACHE_IGNORE_HTTP_CODES = [] 98 | #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' 99 | 100 | REDIS_HOST = '127.0.0.1' 101 | REDIS_PORT = 6379 102 | 103 | # 代理IP 104 | PROXIES = [ 105 | {'ip_port': '119.29.126.115:80'}, 106 | {'ip_port': '36.97.145.29:9797'}, 107 | {'ip_port': '222.52.142.242:8080'}, 108 | {'ip_port': '120.132.71.212:80'}, 109 | {'ip_port': '111.155.116.235:8123'}, 110 | {'ip_port': '222.33.192.238:8118'}, 111 | {'ip_port': '60.174.237.43:9999'}, 112 | {'ip_port': '183.56.177.130:808'}, 113 | {'ip_port': '111.155.124.73 8123'}, 114 | ] 115 | 116 | -------------------------------------------------------------------------------- /app/ProductData/spiders/__init__.py: -------------------------------------------------------------------------------- 1 | # This package will contain the spiders of your Scrapy project 2 | # 3 | # Please refer to the documentation for information on how to create and manage 4 | # your spiders. 5 | -------------------------------------------------------------------------------- /app/ProductData/spiders/zol.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import logging 3 | import re 4 | 5 | import scrapy 6 | from scrapy import Request 7 | # 使用redis去重 8 | from scrapy.dupefilters import RFPDupeFilter 9 | 10 | from .. import config,log_helper 11 | from ..items import ProductdataItem 12 | 13 | class ZolSpider(scrapy.Spider): 14 | name = 'zol' 15 | allowed_domains = ['detail.zol.com.cn'] 16 | base_url = "http://detail.zol.com.cn" 17 | urls = ["/notebook_index/subcate16_0_list_1_0_1_2_0_1.html","/cell_phone_index/subcate57_0_list_1_0_1_2_0_1.html","/digital_camera_index/subcate15_0_list_1_0_1_2_0_1.html"] 18 | 19 | def start_requests(self): 20 | 21 | for url in self.urls: 22 | yield Request( 23 | url=self.base_url + url, 24 | headers=config.headers, 25 | method='GET', 26 | callback=self.get_product_list, 27 | ) 28 | 29 | def get_product_list(self,response): 30 | """ 31 | 获取每个产品类别下的所有产品 32 | :param response: 33 | :return: 34 | """ 35 | p_urls = response.xpath("//div[@class='pic-mode-box']/ul/li/a/@href").extract() 36 | p_titles = response.xpath("//div[@class='pic-mode-box']/ul/li/h3/a/text()").extract() 37 | p_scores = response.xpath("//div[@class='comment-row']/span[@class='score']/text()").extract() 38 | 39 | # 执行分页 40 | p_pages = response.xpath("//div[@class='pagebar']/a/@href").extract() 41 | 42 | try: 43 | p_nums = len(p_urls) 44 | for p_url in p_urls: 45 | # p_url = p_urls[i] 46 | res = re.findall(r'(\d+)', p_url) 47 | if len(res) > 1: 48 | p_id = res[-2] 49 | else: 50 | p_id = res[-1] 51 | 52 | product_info = { 53 | 'p_url':self.base_url + p_url, 54 | 'p_id':p_id 55 | } 56 | 57 | yield Request( 58 | url=self.base_url + p_url, 59 | headers=config.headers, 60 | method='GET', 61 | meta=product_info, 62 | callback=self.get_product_info, 63 | ) 64 | 65 | for p in p_pages: 66 | # 分页回调 67 | yield Request( 68 | url=self.base_url + p, 69 | headers=config.headers, 70 | method='GET', 71 | callback=self.get_product_list, 72 | ) 73 | 74 | except Exception as e: 75 | 76 | log_helper.log(e, logging.WARNING) 77 | 78 | def get_product_info(self,response): 79 | """ 80 | 获取每个产品信息 81 | :param response: 82 | :return: 83 | """ 84 | p_title = "".join(response.xpath("//div[@class='page-title clearfix']/h1/text()").extract()) 85 | p_c_score = "".join(response.xpath("//div[@class='product-comment']/div/div/div/strong/text()").extract()) 86 | comment_url = "".join(response.xpath("//ul[@class='nav']/li/a[@class='ol-comment']/@href").extract()) 87 | p_img = "".join(response.xpath("//div[@class='bigpic']/a/img/@src").extract()) 88 | p_prices = response.xpath("//div[@class='product-merchant-price clearfix']/ul/li/strong/a/text() | //div[@class='product-merchant-price clearfix']/ul/li/span/a/text()").extract() 89 | 90 | # 参考价 91 | p_price_retain = ",".join(response.xpath("//div[@class='product-price-info']/div/span/b[@class='price-type price-retain']/text()").extract()) 92 | # 格式 ['[["7.08","7.09","7.10","7.11","7.14"],[2999,2999,2999,2999,2999],2999,2999]'] 93 | 94 | p_price_trend = response.xpath("//div[@class='product-price-info']/div/span/b[@class='price-type price-retain']/@chart-data").extract() 95 | 96 | if p_price_trend: 97 | p_price_trend = p_price_trend[0] 98 | 99 | response.meta['p_img'] = p_img 100 | response.meta['p_title'] = p_title 101 | response.meta['p_c_score'] = p_c_score 102 | response.meta['p_prices'] = p_prices if p_prices else p_price_retain 103 | response.meta['p_price_trend'] = p_price_trend 104 | 105 | yield Request( 106 | url=self.base_url + comment_url, 107 | headers=config.headers, 108 | method='GET', 109 | meta=response.meta, 110 | callback=self.get_product_comment, 111 | ) 112 | 113 | def get_product_comment(self,response): 114 | """ 115 | 获取产品评论信息 116 | :param response: 117 | :return: 118 | """ 119 | p_c_all_nums = response.xpath("//div[@class='good-words']/ul/li/a/text() | //div[@class='good-words']/ul/li/a/span/text()").extract() 120 | # 所有评论 121 | p_comments = response.xpath("//div[@class='comment-tabs']/div/label/text() | //div[@class='comment-tabs']/div/label/em/text()").extract() 122 | 123 | # 获取评论日期 这里只获取第一页 获取最新评论日期 来判断这个手机当前的活跃度 124 | p_c_times = response.xpath("//div[@class='comments-list-content']/div/span[@class='date']/text()").extract() 125 | 126 | """ 127 | p_url 128 | p_title 129 | p_c_score 130 | p_img 131 | p_prices 132 | p_c_all_nums 133 | p_comments 134 | p_c_times 135 | p_id 136 | p_price_trend 137 | """ 138 | product_info = response.meta 139 | 140 | item = ProductdataItem() 141 | item['p_url'] = product_info['p_url'] 142 | item['p_title'] = product_info['p_title'] 143 | item['p_c_score'] = product_info['p_c_score'] 144 | item['p_prices'] = "".join(product_info['p_prices']) if product_info['p_prices'] else "暂无报价" 145 | item['p_img'] = product_info['p_img'] 146 | item['p_id'] = product_info['p_id'] 147 | item['p_c_all_nums'] = "".join(p_c_all_nums) 148 | item['p_comments'] = "".join(p_comments).replace(" ","").replace("\r\n","") 149 | item['p_c_times'] = ",".join(p_c_times) 150 | item['p_price_trend'] = product_info['p_price_trend'] 151 | 152 | yield item 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | -------------------------------------------------------------------------------- /app/ProductData/sql_hepler.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | import logging 4 | from . import config 5 | from . import log_helper 6 | import pymysql 7 | 8 | class SqlHelper(object): 9 | def __init__(self): 10 | self.conn = pymysql.connect(**config.productanalysis_db_config) 11 | self.cursor = self.conn.cursor() 12 | self.conn.select_db(config.productanalysis_db) 13 | 14 | def insert_json(self,data={},table_name = None): 15 | """ 16 | 以json数据插入 17 | :param data: 18 | :param table_name: 19 | :return: 20 | """ 21 | try: 22 | keys = [] 23 | vals = [] 24 | for k,v in data.items(): 25 | keys.append(k) 26 | vals.append(v) 27 | 28 | val_str = ','.join(['%s']*len(vals)) 29 | key_str = ','.join(keys) 30 | 31 | sql = 'insert ignore into {table} ({keys}) values({values})'.\ 32 | format(keys=key_str,values=val_str,table=table_name) 33 | 34 | self.cursor.execute(sql,tuple(vals)) 35 | self.conn.commit() 36 | 37 | return self.cursor.lastrowid 38 | 39 | except Exception as e: 40 | log_helper.log(e,logging.WARNING) 41 | return -1 42 | 43 | def query_one(self,command,cursor_type = 'tuple'): 44 | try: 45 | cursor = None 46 | if cursor_type == 'dict': 47 | cursor = self.conn.cursor(pymysql.cursors.DictCursor) 48 | else: 49 | cursor = self.cursor 50 | 51 | cursor.execute(command) 52 | data = cursor.fetchone() 53 | self.conn.commit() 54 | 55 | return data 56 | except Exception as e: 57 | log_helper.log(e,logging.WARNING) 58 | return None 59 | 60 | def close(self): 61 | """ 62 | 关闭操作 63 | :return: 64 | """ 65 | self.cursor.close() 66 | self.conn.close() -------------------------------------------------------------------------------- /app/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/app/__init__.py -------------------------------------------------------------------------------- /app/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from . import models 3 | # Register your models here. 4 | 5 | class ProductInfoAdmin(admin.ModelAdmin): 6 | list_display = ('p_title','p_c_score','p_id') 7 | search_fields = ('p_id','p_title') 8 | 9 | admin.site.register(models.ProductInfo,ProductInfoAdmin) 10 | 11 | -------------------------------------------------------------------------------- /app/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class AppConfig(AppConfig): 5 | name = 'app' 6 | -------------------------------------------------------------------------------- /app/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.6 on 2017-07-17 05:27 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | initial = True 11 | 12 | dependencies = [ 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name='ProductInfo', 18 | fields=[ 19 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 20 | ('p_url', models.CharField(max_length=256, verbose_name='产品url')), 21 | ('p_title', models.CharField(max_length=64, verbose_name='产品名称')), 22 | ('p_img', models.CharField(max_length=256, verbose_name='产品图片')), 23 | ('p_id', models.IntegerField(verbose_name='产品ID')), 24 | ('p_prices', models.CharField(blank=True, max_length=128, null=True, verbose_name='商品价格')), 25 | ('p_c_score', models.FloatField(blank=True, null=True, verbose_name='产品总评分')), 26 | ('p_comments', models.CharField(blank=True, max_length=128, null=True, verbose_name='产品评价')), 27 | ('p_c_all_nums', models.CharField(blank=True, max_length=256, null=True, verbose_name='概况点评内容')), 28 | ('p_c_time', models.CharField(blank=True, max_length=256, null=True, verbose_name='产品评论日期')), 29 | ('p_price_trend', models.CharField(blank=True, max_length=128, null=True, verbose_name='价格趋势')), 30 | ], 31 | options={ 32 | 'verbose_name_plural': '产品信息', 33 | 'db_table': 'product_info', 34 | }, 35 | ), 36 | ] 37 | -------------------------------------------------------------------------------- /app/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/app/migrations/__init__.py -------------------------------------------------------------------------------- /app/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | # Create your models here. 4 | class ProductInfo(models.Model): 5 | """ 6 | 产品信息 7 | """ 8 | p_url = models.CharField(max_length=256,name='p_url',verbose_name='产品url') 9 | p_title = models.CharField(max_length=64,name='p_title',verbose_name='产品名称') 10 | p_img = models.CharField(max_length=256,name='p_img',verbose_name='产品图片') 11 | p_id = models.IntegerField(name='p_id',verbose_name='产品ID') 12 | 13 | p_prices= models.CharField(max_length=128,name='p_prices',verbose_name='商品价格', null=True,blank=True) 14 | 15 | p_c_score = models.FloatField(name='p_c_score', verbose_name='产品总评分', null=True,blank=True) 16 | p_comments = models.CharField(max_length=128,name='p_comments', verbose_name='产品评价', null=True,blank=True) 17 | 18 | p_c_all_nums = models.CharField(max_length=256,name='p_c_all_nums',verbose_name='概况点评内容',null=True,blank=True) 19 | 20 | p_c_time = models.CharField(max_length=256, name='p_c_time', verbose_name='产品评论日期', null=True, blank=True) 21 | 22 | p_price_trend = models.CharField(max_length=128,name='p_price_trend',verbose_name='价格趋势',null=True,blank=True) 23 | 24 | class Meta: 25 | db_table = 'product_info' 26 | verbose_name_plural = '产品信息' 27 | 28 | def __str__(self): 29 | 30 | return self.p_title 31 | -------------------------------------------------------------------------------- /app/scrapy.cfg: -------------------------------------------------------------------------------- 1 | # Automatically created by: scrapy startproject 2 | # 3 | # For more information about the [deploy] section see: 4 | # https://scrapyd.readthedocs.org/en/latest/deploy.html 5 | 6 | [settings] 7 | default = ProductData.settings 8 | 9 | [deploy] 10 | #url = http://localhost:6800/ 11 | project = ProductData 12 | -------------------------------------------------------------------------------- /app/search_indexes.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | from haystack import indexes 4 | from app.models import ProductInfo 5 | 6 | class ProductInfoSearch(indexes.SearchIndex,indexes.Indexable): 7 | """ 8 | 全文搜索 9 | """ 10 | text = indexes.CharField(document=True,use_template=True) 11 | 12 | def get_model(self): 13 | 14 | return ProductInfo 15 | 16 | def index_queryset(self, using=None): 17 | 18 | return self.get_model().objects.all() -------------------------------------------------------------------------------- /app/templatetags/__init__.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | -------------------------------------------------------------------------------- /app/templatetags/product_tags.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | from django.utils.safestring import mark_safe 4 | from django import template 5 | register = template.Library() 6 | 7 | @register.simple_tag 8 | def get_search_result_list(product_list): 9 | """ 10 | 获取搜索产品列表 11 | :param product_list: 12 | :return: 13 | """ 14 | res = "" 15 | if product_list: 16 | for index in range(len(product_list)): 17 | if index == 0: 18 | res = '
  • %s
    • ' % (product_list[index].object.p_id,product_list[index].object.p_title) 19 | else: 20 | res += '
  • %s
    • ' % (product_list[index].object.p_id,product_list[index].object.p_title) 21 | else: 22 | res = '
  • 未找到相应的结果' 23 | 24 | return mark_safe(res) 25 | 26 | -------------------------------------------------------------------------------- /app/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | 5 | def test(): 6 | 7 | pass -------------------------------------------------------------------------------- /app/views.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import render 2 | from django.http import JsonResponse 3 | import json 4 | from hepler import make_bokeh_hepler,make_plot_helper 5 | from app.models import ProductInfo 6 | from multiprocessing import Pool 7 | from hepler import redis_helper, log_helper 8 | import logging 9 | 10 | 11 | def index(request): 12 | return render(request, 'index.html') 13 | 14 | 15 | # def make_plot(request): 16 | # """ 17 | # 生成分析图片 18 | # :param request: 19 | # :return: 20 | # """ 21 | # # 通过进程池来存储图片 22 | # pool = Pool(3) 23 | # 24 | # p_id = int(request.POST.get('p_id')) 25 | # 26 | # images = [] 27 | # result = { 28 | # 'status': 1, 29 | # } 30 | # """ 31 | # { 32 | # p_id:{ 33 | # "p_url": 34 | # "p_title": 35 | # "p_img": 36 | # "p_price": 37 | # "p_c_score": 38 | # "p_analysis_imgs":[] 39 | # } 40 | # } 41 | # 42 | # """ 43 | # # 先从redis里获取数据,如果数据不存在再从mysql数据获取 44 | # try: 45 | # res = redis_helper.p_redis.get(p_id) 46 | # except Exception as e: 47 | # res = None 48 | # if res: 49 | # p_info = json.loads(res.decode('utf8'),encoding='utf8') 50 | # else: 51 | # try: 52 | # product_info = ProductInfo.objects.get(p_id=p_id) 53 | # p_url = product_info.p_url 54 | # p_title = product_info.p_title 55 | # p_img = product_info.p_img 56 | # p_prices = product_info.p_prices 57 | # p_c_score = product_info.p_c_score 58 | # if not p_c_score: 59 | # p_c_score = "暂无评分" 60 | # p_info = { 61 | # 'p_id': p_id, 62 | # 'p_url': p_url, 63 | # 'p_title': p_title, 64 | # 'p_img': p_img, 65 | # 'p_price': p_prices, 66 | # 'p_c_score': p_c_score 67 | # } 68 | # images.append(pool.apply_async(make_plot_helper.make_comment_plot, args=(product_info.p_comments, p_id))) 69 | # images.append(pool.apply_async(make_plot_helper.make_overview_plot, args=(product_info.p_c_all_nums, p_id))) 70 | # images.append(pool.apply_async(make_plot_helper.make_hot_plot, args=(product_info.p_c_time, p_id))) 71 | # pool.close() 72 | # pool.join() 73 | # 74 | # p_info['p_analysis_imgs'] = ["http://127.0.0.1:8080/" + img.get() for img in images] 75 | # 76 | # # 将数据保存到redis 77 | # redis_helper.save_to_redis(p_info) 78 | # 79 | # except Exception as e: 80 | # result['error_msg'] = "暂无此产品" 81 | # result['status'] = 0 82 | # p_info = {} 83 | # log_helper.log(e, logging.WARNING) 84 | # 85 | # result['data'] = p_info 86 | # 87 | # return JsonResponse(result) 88 | 89 | def make_bokeh(request): 90 | p_id = int(request.GET.get('p_id')) 91 | result = { 92 | 'status': 1, 93 | } 94 | # 先从redis里获取数据,如果数据不存在再从mysql数据获取 95 | try: 96 | res = redis_helper.p_redis.get(p_id) 97 | except Exception as e: 98 | res = None 99 | if res: 100 | p_info = json.loads(res.decode('utf8'), encoding='utf8') 101 | else: 102 | try: 103 | product_info = ProductInfo.objects.get(p_id=p_id) 104 | p_url = product_info.p_url 105 | p_title = product_info.p_title 106 | p_img = product_info.p_img 107 | p_prices = product_info.p_prices 108 | p_c_score = product_info.p_c_score 109 | if not p_c_score: 110 | p_c_score = "暂无评分" 111 | p_info = { 112 | 'p_id': p_id, 113 | 'p_url': p_url, 114 | 'p_title': p_title, 115 | 'p_img': p_img, 116 | 'p_price': p_prices, 117 | 'p_c_score': p_c_score 118 | } 119 | cb_script, cb_div = make_bokeh_hepler.make_commment_bokeh(product_info.p_comments) 120 | ob_script, ob_div = make_bokeh_hepler.make_overview_bokeh(product_info.p_c_all_nums) 121 | hb_script, hb_div = make_bokeh_hepler.make_hot_bokeh(product_info.p_c_time) 122 | 123 | p_info['p_analysis_imgs'] = [[cb_script, cb_div],[ob_script,ob_div],[hb_script,hb_div]] 124 | # 将数据保存到redis 125 | redis_helper.save_to_redis(p_info) 126 | 127 | except Exception as e: 128 | result['error_msg'] = "暂无此产品" 129 | result['status'] = 0 130 | p_info = {} 131 | log_helper.log(e, logging.WARNING) 132 | 133 | result['data'] = p_info 134 | 135 | return JsonResponse(result) 136 | -------------------------------------------------------------------------------- /doc/django1.8中文文档.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/doc/django1.8中文文档.pdf -------------------------------------------------------------------------------- /doc/django之查询.md: -------------------------------------------------------------------------------- 1 | ## 开篇说明 2 | 3 | 个人感觉django是一个很大的框架,通过几篇文章就想把django搞清,这难度有点大,这里,我只挑我认为最重要的也就是经常用到的----模型层中的查询,中文文档我也上传了,需要的可以下载 4 | 5 | ## QuerySet 6 | 一个对象的集合,可以包含多个过滤器,过滤器等价于mysql中的where和limit,而queryset等价于select,一般情况下,都看到一个模型类.objects这个返回的就是一个queryset对 7 | 象,而这个objects就是一个Manager,Manager实际就是数据库查询操作的一个接口 8 | 9 | 特点:QuerySet是惰性的,QuerySet创建不涉及到任何数据库操作,除非这个QuerySet被调用 10 | 11 | > models类: 12 | 13 | class UserInfo(models.Model): 14 | """ 15 | 用户信息 16 | """ 17 | username = models.CharField(max_length=32,verbose_name="用户名",unique=True) # unique 设置字段唯一 18 | gender_type = ( 19 | (0,"保密"), 20 | (1,"男"), 21 | (2,"女") 22 | ) 23 | gender = models.IntegerField(choices=gender_type,verbose_name="性别",default=0,help_text="请选择您的性别") 24 | 25 | class Meta: 26 | db_table = "user_info" # 设置数据库表名 27 | verbose_name_plural = "用户表" 28 | 29 | def __str__(self): 30 | 31 | return self.username 32 | 33 | class CardInfo(models.Model): 34 | """ 35 | 帖子信息 36 | """ 37 | title = models.CharField(max_length=32,verbose_name="帖子标题") 38 | content = models.TextField() 39 | ui_id = models.ForeignKey("UserInfo",verbose_name="用户") 40 | ci_id = models.ForeignKey("CircleInfo",verbose_name="圈子") 41 | create_time = models.DateTimeField(verbose_name="帖子创建日期",auto_now_add=True) 42 | 43 | def __str__(self): 44 | 45 | return "%s/%s" % (self.title,self.create_time) 46 | 47 | class Meta: 48 | db_table = "card_info" 49 | verbose_name_plural = "帖子信息表" 50 | ordering = ["create_time"] # 根据创建日期来排序 51 | 52 | 53 | class CircleInfo(models.Model): 54 | """ 55 | 圈子信息 56 | """ 57 | circle_name = models.CharField(max_length=32,verbose_name="圈子标题") 58 | ui_id = models.ManyToManyField("UserInfo",verbose_name="用户") 59 | 60 | class Meta: 61 | db_table = "circle_info" 62 | verbose_name_plural = "圈子信息表" 63 | 64 | def __str__(self): 65 | 66 | return "id=%s,circle_name=%s" % (self.id,self.circle_name) 67 | 68 | 69 | > 粗看objects 70 | 71 | >>> from app.models import * 72 | >>> obj = CircleInfo.objects 73 | >>> obj 74 | # 这里可以看出objects实际就是一个Manager 75 | 76 | 通过dir(obj)就可以查看里面的属性了,比如: 77 | ['aggregate', 'all', 'annotate', 'auto_created', 'bulk_create', 'check', 'complex_filter', 'contribute_to_class', 'count', 'create', 78 | 'creation_counter', 'dates', 'datetimes','db', 'db_manager', 'deconstruct', 'defer', 'distinct', 'earliest', 'exclude', 'exists', 'extra', 79 | 'filter', 'first', 'from_queryset', 'get', 'get_or_create', 'get_queryset', 'in_bulk', 'iterator', 'last', 'latest', 'model', 'name', 80 | 'none', 'only', 'order_by', 'prefetch_related', 'raw', 'reverse', 'select_for_update', 'select_related', 'update', 'update_or_create', 'use_in_migrations', 81 | 'using', 'values', 'values_list'] 82 | 83 | > 常用属性 84 | 85 | > all():获取一个表中所有的对象,返回的是一个QuerySet对象 86 | >>> obj.all() 87 | , , ]> 88 | > filter():过滤器,通过条件来删选自己想要的 89 | >>> obj.filter(circle_name="爱美剪发") # 过滤出circle_name为爱美剪发的圈子 90 | ]> 91 | 92 | > order_by():默认是升序,如果想要降序的话,加上负号 93 | >>> obj.order_by("id") 94 | , , ]> 95 | >>> obj.order_by("-id") 96 | , , ]> 97 | 98 | > 关于双下划线 99 | 100 | __gt:大于 101 | __gte:大于等于 102 | __lt:小于 103 | __lte:小于等于 104 | __in:在...内 105 | __isnull:是否null 106 | __contains:包含,大小写敏感 107 | __icontains:包含,大小写不敏感 108 | __range:范围 109 | __startswith:以什么开头 110 | __endswith:以什么结尾 111 | 112 | __字段名:用来反向查询 113 | 114 | >>> obj.all() 115 | , , , , ]> 116 | >>> obj.filter(id__lt=3) 117 | , ]> 118 | >>> obj.filter(id__in=[2,3,5]) 119 | , , ]> 120 | 121 | 反向查询的使用 122 | 比如现在想知道当前圈子有哪些帖子: 123 | >>> CircleInfo.objects.filter(id=2).values("cardinfo__title") 124 | 125 | 查看帖子数: 126 | >>> CircleInfo.objects.filter(id=2).values("cardinfo__title").count() 127 | 2 128 | 129 | > Q对象:主要是用来构造搜索条件的,对于filter来说,条件都是AND,如果想实现OR关系,这个时候,就可以通过Q来实现,多个Q对象可以通过|和&来连接,如果想取反,就在Q前面加~ 130 | 131 | 比如现在想查询圈子名为"爱美剪发"或者"造型视觉"下的帖子: 132 | >>> obj.filter(Q(circle_name="爱美剪发") | Q(circle_name="造型视觉")).values("cardinfo__title") 133 | 134 | 如果多个Q之间用","隔开就变成了AND关系,比如现在查询名字为Lily的并且并且在造型视觉发的帖子 135 | >>> CardInfo.objects.filter(Q(ui_id__username="Lily")) 136 | , ]> 137 | >>> CardInfo.objects.filter(Q(ui_id__username="Lily"),Q(ci_id__circle_name="造型视觉")) 138 | ]> 139 | 140 | > 反向关联:*_set,_set前面关联的models类名小写,而这个_set是属于被关联models类的属性,比如,现在每个CardInfo都要关联到一个CircleInfo,那么现在通过反向关联,说着有点绕,简单的说就是,ForeignKey或者ManyToManyField字段 141 | 在那个models下,那么_set前面就是这个models的小写,通过这个这段指向的那个models对象来调用 142 | 143 | 比如: 144 | >>> circle_info = CircleInfo.objects.get(id=2) 145 | >>> circle_info.cardinfo_set.all() 146 | , , ]> 147 | >>> user_info = UserInfo.objects.get(id=1) 148 | >>> user_info.circleinfo_set.all() 149 | , ]> 150 | 151 | 152 | > 其他常用API: 153 | 154 | values():迭代返回的是一个字典 155 | 156 | 如: 157 | >>> obj.values() 158 | 159 | >>> for o in obj.values(): 160 | ... type(o) 161 | ... 162 | 163 | 164 | 165 | 166 | 167 | 168 | select_related():有的时候在查询数据的时候,会用到关联表数据,而这些数据在后面的也会用到,这个时候就可以通过这个函数进行关联查询,这种查询虽然会影响到性能的损耗,但是在接下来的应用都将不会操作数据库 169 | 170 | 如: 171 | >>> cardinfo = CardInfo.objects.select_related("ci_id").get(id=2) 172 | >>> cardinfo.ci_id 173 | 174 | 175 | aggregate():聚合查询,返回的是一个字典 176 | 177 | 如: 178 | >>> CircleInfo.objects.filter(id=2).aggregate(Count("cardinfo")) 179 | {'cardinfo__count': 3} 180 | 181 | 182 | > 序列化之JSON,一般情况,希望查出的数据就能直接进行数据传输,而不需要再次封装,这个时候可以直接序列化 183 | 184 | 如:序列化帖子的所有信息 185 | 186 | >>> from django.core import serializers 187 | >>> import json 188 | >>> card_info = json.loads(serializers.serialize('json',CardInfo.objects.all()),encoding='utf8') 189 | >>> card_info 190 | [{'model': 'app.cardinfo', 'pk': 1, 'fields': {'title': '什么发型好看', 'content': '想剪个头发,但是不知道哪里剪得好看', 'ui_id': 1, 'ci_id': 3, 'create_time': '2017-07-21T01:15:32.852Z'}}, {'model': 'app.cardinfo', 'pk': 2, 'fiel 191 | ds': {'title': '美发学院活动', 'content': '今天美发学院做活动,凡是到场的将送本公司精美礼品一份', 'ui_id': 4, 'ci_id': 1, 'create_time': '2017-07-21T01:17:31.075Z'}}, {'model': 'app.cardinfo', 'pk': 3, 'fields': {'title': '杀马特' 192 | , 'content': '现在做个杀马特怎么样', 'ui_id': 3, 'ci_id': 2, 'create_time': '2017-07-21T01:18:12.834Z'}}, {'model': 'app.cardinfo', 'pk': 4, 'fields': {'title': '锅盖头', 'content': '每次看到那些锅盖头的帅锅,都感觉好别扭', 'ui_id 193 | ': 5, 'ci_id': 1, 'create_time': '2017-07-21T01:19:43.964Z'}}, {'model': 'app.cardinfo', 'pk': 5, 'fields': {'title': '脸型是圆', 'content': '圆脸适合什么样的发型呢?', 'ui_id': 2, 'ci_id': 2, 'create_time': '2017-07-21T05:29:43.7 194 | 99Z'}}, {'model': 'app.cardinfo', 'pk': 6, 'fields': {'title': '超卷发好看吗', 'content': '每次看到那些很卷的发型,第一感觉就是方便面,这种发型真的好看吗?为什么会有这么多人喜欢呢?', 'ui_id': 5, 'ci_id': 2, 'create_time': '2017-07-2 195 | 1T06:19:17.451Z'}}] 196 | 197 | 如果只想序列化一部分字段,可以通过fields来指定 198 | 199 | >>> card_info = json.loads(serializers.serialize('json',CardInfo.objects.all(),fields=('title','content','create_time')),encoding='utf8') 200 | >>> card_info 201 | [{'model': 'app.cardinfo', 'pk': 1, 'fields': {'title': '什么发型好看', 'content': '想剪个头发,但是不知道哪里剪得好看', 'create_time': '2017-07-21T01:15:32.852Z'}}, {'model': 'app.cardinfo', 'pk': 2, 'fields': {'title': '美发学院 202 | 活动', 'content': '今天美发学院做活动,凡是到场的将送本公司精美礼品一份', 'create_time': '2017-07-21T01:17:31.075Z'}}, {'model': 'app.cardinfo', 'pk': 3, 'fields': {'title': '杀马特', 'content': '现在做个杀马特怎么样', 'create_tim 203 | e': '2017-07-21T01:18:12.834Z'}}, {'model': 'app.cardinfo', 'pk': 4, 'fields': {'title': '锅盖头', 'content': '每次看到那些锅盖头的帅锅,都感觉好别扭', 'create_time': '2017-07-21T01:19:43.964Z'}}, {'model': 'app.cardinfo', 'pk': 5 204 | , 'fields': {'title': '脸型是圆', 'content': '圆脸适合什么样的发型呢?', 'create_time': '2017-07-21T05:29:43.799Z'}}, {'model': 'app.cardinfo', 'pk': 6, 'fields': {'title': '超卷发好看吗', 'content': '每次看到那些很卷的发型,第一感 205 | 觉就是方便面,这种发型真的好看吗?为什么会有这么多人喜欢呢?', 'create_time': '2017-07-21T06:19:17.451Z'}}] 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | -------------------------------------------------------------------------------- /doc/python必备知识点.md: -------------------------------------------------------------------------------- 1 | ## 常用的数据类型 2 | 3 | ### 字符串 列表 字典 集合 4 | > str:常用用到的函数有: 5 | 6 | > join():字符串拼接,一般情况是用来拼接list,tuple,比如将数据保存到mysql数据库的时候,mysql数据库是不支持list和tuple的类型的 7 | 用法: 8 | >>> l = ['Jack','Jan','Jason'] 9 | >>> ','.join(l) 10 | 'Jack,Jan,Jason' 11 | > replace():替换,比如抓取一条新闻的时间部分,拿到的数据是: 12 | res = '\n 2017-07-20 09:57:48\u3000来源: ',通过replace对数据进行处理 13 | >>> res.replace(" ","").replace("\n","").replace("\u3000","") 14 | '2017-07-2009:57:48来源:' 15 | > strip():去空格 16 | res ='\n （原标题：中关村部分企业家到雄安新区调研）\n ' 17 | >>> res.strip() 18 | '（原标题：中关村部分企业家到雄安新区调研）' 19 | > split():切割,比如从数据库查到用户名为username = 'Jack,Jan,Jason',需要以列表的形式返回给前台 20 | >>> username 21 | 'Jack,Jan,Jason' 22 | >>> username.split(',') 23 | ['Jack', 'Jan', 'Jason'] 24 | 25 | > list:常用函数有: 26 | 27 | 插入元素:分别有三个函数append,extend,insert,现在有一个列表l = ['A','B','C'] 28 | > append():尾部追加元素 29 | >>> l = ['A','B','C'] 30 | >>> l.append('D') 31 | >>> l 32 | ['A', 'B', 'C', 'D'] 33 | 34 | > extend():以迭代的方式,添加元素 35 | >>> l = ['A','B','C'] 36 | >>> l.extend('F') 37 | >>> l 38 | ['A', 'B', 'C', 'F'] 39 | 40 | > insert():指定位置添加元素 41 | >>> l = ['A','B','C'] 42 | >>> l.insert(1,'D') 43 | >>> l 44 | ['A', 'D', 'B', 'C'] 45 | 46 | 这三种插入的方式,我感觉最大的区别就是效率问题,就list来说,它在数据结构中属于顺序表,顺序表的插入元素特点是:尾部添加,时间复杂度为O(1), 47 | 而以其他形式添加,时间复杂度为O(n),所以append的效率相对来说要高,一般情况用到的也是append 48 | 49 | 删除元素:也是有三个函数,clear,pop,remove,同样l = ['A','B','C'] 50 | > clear():删除所有元素,一般很少用到 51 | > pop():默认删除最后一个元素,也可以通过索引来指定,返回的是被删除的元素 52 | >>> l = ['A','B','C'] 53 | >>> l.pop() 54 | 'C' 55 | > remove():删除被选中的元素 56 | >>> l = ['A','B','C'] 57 | >>> l.remove('B') 58 | >>> l 59 | ['A', 'C'] 60 | 61 | 以顺序表的特点来看,尾部删除元素时间复杂度为O(1),其他位置删除为O(n),所以通过pop()默认删除的话,效率要高 62 | 63 | > sort():排序,比如抓取到几条帖子的评论数分为comment = [400,23,89,100,55],默认排序是从小到大,如果加上reverse=True,则相反也可以 64 | 通过key来自己指定规则, 65 | >>> comment = [400,23,89,100,55] 66 | >>> comment.sort() 67 | >>> comment 68 | [23, 55, 89, 100, 400] 69 | 70 | 关于内置函数sorted(),与sort的区别就是,原函数并没有改变 71 | >>> comment = [400,23,89,100,55] 72 | >>> c = sorted(comment) 73 | >>> c 74 | [23, 55, 89, 100, 400] 75 | >>> comment 76 | [400, 23, 89, 100, 55] 77 | 78 | > dict:感觉字典比较随意,创建,获取等都比较直观 79 | 80 | 比如: 81 | >>> p_info = {"p_id":1116725,"p_title":"联想小新"} 82 | >>> type(p_info) 83 | 84 | >>> p_info["p_id"] # 直接获取 85 | 1116725 86 | >>> p_info.get("p_id") # 通过get来获取 87 | 1116725 88 | 一般情况用来做数据封装之后,进行数据传输 89 | 90 | > set:最大的特点就是去重,一般情况,也是用来去重,比如,在获取的评论日期中,有可能一天有很多评论,但是我们的需求是只要知道这一天有没有评论就行了,并不需要 91 | 知道评论数,dt = ['2015-06-12','2015-06-23','2015-03-22','2015-06-23'] 92 | 93 | >>> dt = ['2015-06-12','2015-06-23','2015-03-22','2015-06-23'] 94 | >>> set(dt) 95 | {'2015-06-12', '2015-03-22', '2015-06-23'} 96 | 还有一些情况,用来两个数据之间的运算 97 | a = "手机便宜,性价比高",b='性价比高,像素也好' 98 | >>> a = "手机便宜,性价比高" 99 | >>> b='性价比高,像素也好' 100 | >>> set(a) & set(b) # 交集 101 | {'价', '高', '性', '比', ','} 102 | >>> set(a) | set(b) # 并集 103 | {'机', '也', '好', ',', '手', '宜', '像', '价', '便', '性', '高', '比', '素'} 104 | >>> set(a) - set(b) # 差集 105 | {'手', '机', '宜', '便'} 106 | >>> set(a) ^ set(b) # 交叉补集 107 | {'也', '机', '好', '手', '宜', '像', '便', '素'} 108 | 109 | ## 内置函数 110 | 111 | > map():主要用来修改列表里的元素,和列表解析差不多,在效率上也差不多,dt = ['2015-06-12','2015-06-23','2015-03-22','2015-06-23'],想把里面的字符串时期 112 | 113 | 格式化成日期格式 114 | >>> list(map(lambda x:datetime.datetime.strptime(x,'%Y-%m-%d'),dt)) 115 | [datetime.datetime(2015, 6, 12, 0, 0), datetime.datetime(2015, 6, 23, 0, 0), datetime.datetime(2015, 3, 22, 0, 0), datetime.datetime(2015, 6, 23, 0, 0)] 116 | >>> [datetime.datetime.strptime(d,'%Y-%m-%d') for d in dt] 117 | [datetime.datetime(2015, 6, 12, 0, 0), datetime.datetime(2015, 6, 23, 0, 0), datetime.datetime(2015, 3, 22, 0, 0), datetime.datetime(2015, 6, 23, 0, 0)] 118 | 就个人而言,我感觉,列表解析更加直观,简洁 119 | 120 | > zip():合并列表,将对应的元素合并成一个元组,合并之后,通过*来解压 121 | 122 | >>> mall = ["京东商城","天猫商城","zol商城"] 123 | >>> price = ["1599","1499","1549"] 124 | >>> z = zip(mall,price) 125 | >>> list(z) 126 | [('京东商城', '1599'), ('天猫商城', '1499'), ('zol商城', '1549')] 127 | 128 | ## 装饰器 129 | 130 | > 一般情况都是用来做登录验证 131 | 132 | >>> def auth(func): 133 | ... def wrapper(request,*args,**kwargs): 134 | ... if request.session.has_key('username'): 135 | ... result = func(request,*args,**kwargs) 136 | ... else: 137 | ... result = redirect('/login') 138 | ... return result 139 | ... return wrapper 140 | 141 | >>> @auth 142 | ... def cart(request): 143 | ... return render(reeuqet,'cart.html') 144 | 145 | 146 | 注:以上只是我常用到的,有的在项目中也会用到,需要知道更全面的,就去查看官方文档了 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | -------------------------------------------------------------------------------- /doc/scrapy知识汇总.md: -------------------------------------------------------------------------------- 1 | # 官方文档:https://doc.scrapy.org/en/master/topics/architecture.html 2 | 3 | ## 数据抓取流程 4 | ![](../screen/Scrapy流程.png) 5 | 6 | ### 各个组件的功能: 7 | 8 | > 1.Engine:引擎,图中它是位于中间的位置,所以它的主要功能就是用来提供Spider,ItemPipelines,Scheduler,Downloader之间通信的渠道 9 | 10 | > 2.Scheduler:调度器,负责接收Engine发送过来的Request,按照一定的形式进行排列,并将Request返回给引擎 11 | 12 | > 3.Downloader:下载器,负责下载Engine里所有的Request,然后将获取到的Response交给Engine 13 | 14 | > 4.Spider:负责发送Request和接收Response,对Response里的数据进行提取,分析,如果有需要跟进的URL,再次将这个Request交给Engine 15 | 16 | > 5.Item Pipeline:管道,负责处理Spider中数据,并对这些数据进行处理,比如将数据存储到数据库 17 | 18 | > 6.Middlewares:中间件,包括Downloader Middlewares 和 Spider Middewares,有的时候需要对一些请求进行处理,比如添加请求头,设置动态代理,那么这个任务就可以交给Downloader Middlewares来完成,同时,当Downloader下载 19 | 完成将Response返回给Engine的时候,这个时候Downloader Middlewares也可以对Response进行一定的处理;对于Spider Middlewares,如果需要在Spider和Engine之间的的Request和Response进行扩展的话,可以通过Spider Middewares来实现 20 | 21 | ### 流程 22 | > 对于流程的话,就是按照上图那样进行的,我感觉需要解释的就是第八步,第八步那里为什么会有两个指向箭头呢?当一个Spider完成了第六步之后,也就是接收到了 23 | Response,可以说,这个时候完成了一次请求,这个时候有两种情况就是,这个Response处理完的结果一种是需要Item Pipeline处理数据,一种还有就是里面有需要跟进的URL,比如分页,所以就把需要进行数据处理的交给Item Pipeline,需要跟进的URL交给Scheduler,由Scheduler将这个URL加入队列进行处理 24 | 25 | ### 项目的创建 26 | > scrapy startproject autohomeProduct 27 | 进入spider所在的目录: 28 | 29 | 如果创建的是Spider类:scrapy genspider autohome "autohome.com.cn",这样就创建了一个spider,如: 30 | import scrapy 31 | class AutohomeSpider(scrapy.Spider): 32 | name = 'autohome' 33 | allowed_domains = ['autohome.com.cn'] 34 | start_urls = ['http://autohome.com.cn/'] 35 | 36 | def parse(self, response): 37 | pass 38 | 39 | 如果创建的是CrawlSpiders类:scrapy genspider -t crawl autohome2 autohome.com.cn,如: 40 | 41 | from scrapy.linkextractors import LinkExtractor 42 | from scrapy.spiders import CrawlSpider, Rule 43 | 44 | 45 | class Autohome2Spider(CrawlSpider): 46 | name = 'autohome2' 47 | allowed_domains = ['autohome.com.cn'] 48 | start_urls = ['http://autohome.com.cn/'] 49 | 50 | rules = ( 51 | Rule(LinkExtractor(allow=r'Items/'), callback='parse_item', follow=True), 52 | ) 53 | 54 | def parse_item(self, response): 55 | i = {} 56 | #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract() 57 | #i['name'] = response.xpath('//div[@id="name"]').extract() 58 | #i['description'] = response.xpath('//div[@id="description"]').extract() 59 | return i 60 | 61 | 这样项目就创建完成了 62 | 63 | ## 关于Item 64 | 先从Item讲起,就是因为,我们要抓取数据,必须清楚自己需要的是什么数据,比如现在我们需要的数据如下: 65 | 66 | import scrapy 67 | 68 | class AutohomeproductItem(scrapy.Item): 69 | # 车名 70 | title = scrapy.Field() 71 | # 车价 72 | price = scrapy.Field() 73 | # 车图片 74 | pic = scrapy.Field() 75 | # 车评分 76 | grade = scrapy.Field() 77 | # 油耗 78 | fuel_consumption = scrapy.Field() 79 | 80 | 创建Item的方式,可以通过创建对象的方式,也可以通过字典的形式 81 | 82 | 创建对象的方式: 83 | >>> from autohomeProduct.items import AutohomeproductItem 84 | >>> ahpitem = AutohomeproductItem() 85 | >>> ahpitem['title'] = '宝马' 86 | >>> ahpitem['price'] = '56万' 87 | >>> ahpitem['pic'] = '1.png' 88 | >>> ahpitem['grade'] = '8.6分' 89 | >>> ahpitem['fuel_consumption'] = '12.6L' 90 | >>> ahpitem 91 | {'fuel_consumption': '12.6L', 92 | 'grade': '8.6分', 93 | 'pic': '1.png', 94 | 'price': '56万', 95 | 'title': '宝马'} 96 | 97 | 字典的方式: 98 | >>> autoinfo = {'fuel_consumption': '12.6L', 99 | ... 'grade': '8.6分', 100 | ... 'pic': '1.png', 101 | ... 'price': '56万', 102 | ... 'title': '宝马'} 103 | >>> item = AutohomeproductItem(autoinfo) 104 | >>> item 105 | {'fuel_consumption': '12.6L', 106 | 'grade': '8.6分', 107 | 'pic': '1.png', 108 | 'price': '56万', 109 | 'title': '宝马'} 110 | 111 | ## 关于Spider和CrawlSpider 112 | 主要就是用来定义需要爬取的网站,还有就是需要爬取的规则,并根据获取的网页内容提取需要的数据 113 | 114 | 执行规则:先调用Spider的__init__方法,初始化name和start_urls,在不重写start_requests的情况下,又start_requests读取start_urls里面的内容,而start_requests实际调用的是make_requests_from_url将url交给Request去处 115 | 理,以parse为回调函数返回,在parse里面就有我们需要的网页信息Response,然后通过选择器来提取需要的内容 116 | 117 | 相关函数: 118 | start_requests():返回的是一个迭代器对象,所以一般情况将其实现生成器,如果我们我的请求需要提交参数,添加请求头或者进行数据的传递的时候,可以重写这个方法 119 | 120 | make_requests_from_url():接收一个url参数,并返回一个Request对象,未被重写的情况下,默认parse作为回调函数 121 | 122 | parse():处理返回的Response信息 123 | 124 | closed():当spider关闭的时候,该函数被调用 125 | 126 | > CrawlSpider:除了具有Spider的功能外,还提供了一些规则来继续跟进 127 | 128 | rules:包含一个或多个Rule对象的元组,这个Rule对象定义的就是需要跟进的规则,还有就是对返回信息的处理,在Rule对象,有个参数需要值得要注意一下,就是,当我们在爬取数据的时候,有的网站可能需要跟进的链接进行变动,这个时候, 129 | 可以通过process_links参数对提取的链接进行处理,一般情况下都是一个回调函数 130 | 131 | > 其他的Spider: 132 | XMLFeedSpider:主要是用来分析XML源的 133 | CSVFeedSpider:顾名思义也就是CSV源,和XMLFeedSpider有点相似,只不过它是按行遍历,XMLFeedSpider是按节点遍历 134 | SitemapSpider:通过Sitemaps来发现需要爬取的URL,比如(官网的示例): 135 | from scrapy.spiders import SitemapSpider 136 | 137 | class MySpider(SitemapSpider): 138 | sitemap_urls = ['http://www.example.com/sitemap.xml'] 139 | sitemap_rules = [ 140 | ('/product/', 'parse_product'), 141 | ('/category/', 'parse_category'), 142 | ] 143 | 144 | def parse_product(self, response): 145 | pass # ... scrape product ... 146 | 147 | def parse_category(self, response): 148 | pass # ... scrape category ... 149 | 150 | ## 关于选择器 151 | 除了常用的xpath之外,还有就css,同时还支持re和set匹配规则,用到的就去官网查看吧-----Selector 152 | 153 | ## 关于Item Pipeline 154 | 当我们在Spider的回调函数yield一个item的时候,数据就会传递到Item Pipeline,在这里对每个Item进行处理 155 | 156 | > 必须要实现的方法: 157 | process_item():如果需要pipeline继续处理item,必须将item返回,当抛出异常,这个item将会被丢弃 158 | 参数:item-->被抓取的item 159 | spider-->抓取该item的spider 160 | 161 | > 可以实现的方法: 162 | open_spider():当spider被开启的时候,就会调用,比如需要将数据保存到数据的时候,可以在这里进行数据库的初始化工作以及数据库的连接操作 163 | 164 | close_spider():当spider关闭的时候,会调用,比如,在进行数据保存完成之后,可以在这里执行数据库关闭操作 165 | 166 | ## 关于Downloader Middleware 167 | 介于request和response之间,主要用来全局修改request和response 168 | 169 | > 常用函数: 170 | process_request():每个request通过下载中间件时,该方法都会被调用 171 | 172 | 关于返回值: 173 | 1.如果返回的是None,将会继续处理该request 174 | 2.如果返回的是Response对象,scrapy将不会调用其他的process_request或者process_exception方法,然后将返回该response 175 | 3.如果返回Request对象,scrapy则停止调用process_request方法,并重新调度返回的request 176 | 4.如果抛出异常,测process_exception会被调用 177 | 178 | 关于参数: 179 | request:需要处理的request 180 | spider:request对应的spider 181 | 182 | process_response(): 183 | 关于返回值: 184 | 1.如果返回的是一个Response,该response会被下一个中间件继续执行 185 | 2.如果返回一个Request,中间件链会停止,返回的request会被重新调度下载 186 | 3.如果抛出异常,则会调用request的errback 187 | 188 | 关于内置的中间件可以查看文档,比如CookiesMiddleware,HttpProxyMiddleware等 189 | 190 | ## 关于Spider Middleware 191 | 可以在这里添加代码来处理发送给Spider的response及spider产生的item和request 192 | 193 | > 常用函数: 194 | process_spider_input():当response通过spider中间件时,这个方法会被调用,处理该response 195 | 196 | 关于返回值: 197 | 1.如果返回None,scrapy将继续处理该response 198 | 2.如果抛出异常,则会调用request的errback,errback指向的是process_spider_output()来处理,如果这个方法也抛出异常,则会调用process_spider_exception() 199 | 200 | process_spider_output():处理response返回result时,该方法会被调用 201 | 202 | process_start_requests():以spider启动的request为参数被调用 203 | 204 | ## 关于Requests 和 Responses 205 | 206 | > Request对象: 207 | 有关参数: 208 | url:请求的url 209 | callback:请求返回的Response,由哪个函数处理 210 | method:指定请求方式 211 | headers:添加请求头信息 212 | meta:不同请求之间数据传递 213 | encoding:默认utf8 214 | cookies:请求的cookies 215 | dont_filter:表明该请求不由调度器过滤,默认就是False 216 | errback:指定处理错误函数 217 | 218 | > Response对象: 219 | 有关参数: 220 | url:响应的url 221 | status:响应状态码 222 | headers:响应头 223 | request:生成此响应的请求头 224 | 225 | ## 关于settings 226 | 关于settings的配置,可以查看官方文档 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | -------------------------------------------------------------------------------- /doc/数据分析之pandas.md: -------------------------------------------------------------------------------- 1 | ## NumPy基础 2 | 3 | > 创建ndarray的常用函数: 4 | 5 | array():将输入数据(列表,元组,数组或其他序列类型)直接转换为ndarray 6 | >>> import numpy as np 7 | # 创建一维数组 8 | >>> np.array([2,4,5,9]) 9 | array([2, 4, 5, 9]) 10 | 11 | # 创建二维数组 12 | >>> np.array([[3,5],[8,9]]) 13 | array([[3, 5], 14 | [8, 9]]) 15 | 16 | arange():类似range函数 17 | >>> np.arange(6) 18 | array([0, 1, 2, 3, 4, 5]) 19 | 20 | reshape():改变数组的维度 21 | >>> n1 = np.arange(16) 22 | >>> n1 23 | array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) 24 | >>> n1.shape # 查看数组的维度 25 | (16,) 26 | >>> n2 = n1.reshape(4,4) # 四行四列 27 | >>> n2 28 | array([[ 0, 1, 2, 3], 29 | [ 4, 5, 6, 7], 30 | [ 8, 9, 10, 11], 31 | [12, 13, 14, 15]]) 32 | >>> n2.shape 33 | (4, 4) 34 | 35 | > 索引:主要用来获取值 36 | 37 | 一维数组的获取: 38 | >>> n3 = np.arange(16,step=2) 39 | >>> n3 40 | array([ 0, 2, 4, 6, 8, 10, 12, 14]) 41 | >>> n3[3] # 获取特定索引下的值 42 | 6 43 | >>> n3[4:] # 切片的形式,索引为4到最后 44 | array([ 8, 10, 12, 14]) 45 | >>> n3[:4] 46 | array([0, 2, 4, 6]) 47 | >>> n3[2:6] # 索引2到6之间,不包含6 48 | array([ 4, 6, 8, 10]) 49 | 50 | 二维数组的获取: 51 | >>> n2 52 | array([[ 0, 1, 2, 3], 53 | [ 4, 5, 6, 7], 54 | [ 8, 9, 10, 11], 55 | [12, 13, 14, 15]]) 56 | >>> n2[1:2] # 在没有逗号分隔的情况,默认按行 57 | array([[4, 5, 6, 7]]) 58 | >>> n2[:,2:] # 逗号分隔后面是按列取 59 | array([[ 2, 3], 60 | [ 6, 7], 61 | [10, 11], 62 | [14, 15]]) 63 | 64 | 通过数组的形式获取值: 65 | >>> n2 66 | array([[ 0, 1, 2, 3], 67 | [ 4, 5, 6, 7], 68 | [ 8, 9, 10, 11], 69 | [12, 13, 14, 15]]) 70 | 71 | >>> n2[[1,3],[2,3]] 72 | array([ 6, 15]) 73 | 74 | bool索引: 75 | >>> n4 = n2 >= 10 76 | >>> n4 77 | array([[False, False, False, False], 78 | [False, False, False, False], 79 | [False, False, True, True], 80 | [ True, True, True, True]], dtype=bool) 81 | 82 | >>> n2[n4] # 通过布尔索引来获取值 83 | array([10, 11, 12, 13, 14, 15]) 84 | 85 | ## pandas 86 | 87 | ### 两大数据结构 88 | 89 | > Series:可以看成是一维数组加上索引组成的对象 90 | 91 | >>> import pandas as pd 92 | >>> from pandas import Series,DataFrame 93 | >>> s = Series(np.arange(5),index=['a','b','c','d','e']) # 通过index来指定索引 94 | >>> s 95 | a 0 96 | b 1 97 | c 2 98 | d 3 99 | e 4 100 | dtype: int32 101 | >>> s.index 102 | Index(['a', 'b', 'c', 'd', 'e'], dtype='object') 103 | >>> s.values 104 | array([0, 1, 2, 3, 4]) 105 | 106 | 字典的形式创建Series,字典的key默认变成索引 107 | >>> p_info = {"p_id":120364,"p_title":"雷神","p_price":12800} 108 | >>> s_p = Series(p_info) 109 | >>> s_p 110 | p_id 120364 111 | p_price 12800 112 | p_title 雷神 113 | dtype: object 114 | 115 | 116 | 117 | > DataFrame:是一个表格型的数据结构对象,所以它既有列索引,也有行索引,也可以看成是由Series组成的字典 118 | 119 | 创建方式一般是直接传入一个字典 120 | >>> p_infos = {"p_title":["雷神","机械师","战神"], 121 | ... "p_price":[12288,13000,8999], 122 | ... "p_score":[7.8,6.8,8.9]} 123 | >>> d_p = DataFrame(p_infos) 124 | >>> d_p 125 | p_price p_score p_title 126 | 0 12288 7.8 雷神 127 | 1 13000 6.8 机械师 128 | 2 8999 8.9 战神 129 | 130 | 排序 131 | 按照索引排序 132 | >>> d_p.sort_index(ascending=False) 133 | p_price p_score p_title 134 | 2 8999 8.9 战神 135 | 1 13000 6.8 机械师 136 | 0 12288 7.8 雷神 137 | 按照某一列排序 138 | >>> d_p.sort_values(by="p_score") 139 | p_price p_score p_title 140 | 1 13000 6.8 机械师 141 | 0 12288 7.8 雷神 142 | 2 8999 8.9 战神 143 | 144 | 145 | 数据的获取 146 | >>> d_p.iloc[1] 147 | p_price 13000 148 | p_score 6.8 149 | p_title 机械师 150 | Name: 1, dtype: object 151 | 152 | >>> d_p.loc[:,"p_title"] 153 | 0 雷神 154 | 1 机械师 155 | 2 战神 156 | Name: p_title, dtype: object 157 | 158 | > 关于数据丢失的处理 159 | 160 | >>> p_infos2 = {"p_title":["雷神","机械师","战神","飞行堡垒"], 161 | ... "p_score":[7.8,6.8,8.9,6.7], 162 | ... "p_price":[12288,13000,8999,6799]} 163 | >>> d_p2 = DataFrame(p_infos2) 164 | >>> d_p2 165 | p_price p_score p_title 166 | 0 12288 7.8 雷神 167 | 1 13000 6.8 机械师 168 | 2 8999 8.9 战神 169 | 3 6799 6.7 飞行堡垒 170 | 171 | d_p2.loc[3,"p_score"] = np.nan 172 | >>> d_p2 173 | p_price p_score p_title 174 | 0 12288 7.8 雷神 175 | 1 13000 6.8 机械师 176 | 2 8999 8.9 战神 177 | 3 6799 NaN 飞行堡垒 178 | 179 | 直接将缺失数据丢掉 180 | >>> d_p2.dropna() 181 | p_price p_score p_title 182 | 0 12288 7.8 雷神 183 | 1 13000 6.8 机械师 184 | 2 8999 8.9 战神 185 | 186 | 填充的方式 187 | >>> d_p2.fillna(10) 188 | p_price p_score p_title 189 | 0 12288 7.8 雷神 190 | 1 13000 6.8 机械师 191 | 2 8999 8.9 战神 192 | 3 6799 10.0 飞行堡垒 193 | 194 | 195 | > 统计 196 | 197 | 求平均值: 198 | >>> d_p2.mean() 199 | p_price 10271.500000 200 | p_score 7.833333 201 | 202 | 求和: 203 | >>> d_p2.sum() 204 | p_price 41086 205 | p_score 23.5 206 | p_title 雷神机械师战神飞行堡垒 207 | 208 | > 函数的应用 209 | 210 | >>> d = d_p2.loc[:,"p_price"] 211 | >>> d 212 | 0 12288 213 | 1 13000 214 | 2 8999 215 | 3 6799 216 | Name: p_price, dtype: int64 217 | 218 | >>> d.apply(lambda x : float(x)) 219 | 0 12288.0 220 | 1 13000.0 221 | 2 8999.0 222 | 3 6799.0 223 | Name: p_price, dtype: float64 224 | 225 | 226 | 更多应用查看官方文档:http://pandas.pydata.org/ 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | -------------------------------------------------------------------------------- /hepler/__init__.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding:utf-8 -*- -------------------------------------------------------------------------------- /hepler/config.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | 4 | # redis 5 | redis_pass = '' 6 | redis_host = '127.0.0.1' 7 | redis_port = '6379' 8 | 9 | 10 | REDIS_PRODUCT_INFO_EXPIRES_SECONDES = 43200 # 设置存储过期时间,单位为秒 -------------------------------------------------------------------------------- /hepler/file_hepler.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | 4 | def get_image_path(file_name): 5 | """ 6 | 获取图片地址 7 | :param file_name: 8 | :return: 9 | """ 10 | return "static/upload/%s" % file_name -------------------------------------------------------------------------------- /hepler/log_helper.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | import logging 4 | import datetime 5 | 6 | def log(msg,level = logging.DEBUG): 7 | """ 8 | 自定义输出日志 9 | :param msg: 日志信息 10 | :param level: 日志等级 11 | :return: 12 | """ 13 | logging.log(level,msg) 14 | 15 | print("log时间[%s]-log等级[%s]-log信息[%s]" % (datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),level,msg)) 16 | -------------------------------------------------------------------------------- /hepler/make_bokeh_hepler.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | from bokeh.plotting import figure 4 | from bokeh.embed import components 5 | from bokeh.resources import CDN 6 | from bokeh.charts import Bar,Line 7 | from hepler import file_hepler 8 | import re 9 | import pandas as pd 10 | import datetime 11 | 12 | TOOLS = "save" 13 | 14 | def make_commment_bokeh(data): 15 | if data: 16 | temps = "".join(data).replace(" ", "").replace("\r\n", "") 17 | values = re.findall(r'(\d+)', temps) 18 | c_values = [int(value) for value in values] 19 | c_keys = re.findall('[\u4e00-\u9fa5]+', temps) # 20 | s = pd.Series(c_values, index=c_keys, name='好评率') 21 | s = s[3:6] 22 | s_sum = s.sum() 23 | s = s.apply(lambda x: x / s_sum * 100) 24 | factors = list(s.index) 25 | x = s.values 26 | dot = figure(title="好评率(单位:%)", tools=TOOLS, toolbar_location=None, 27 | y_range=factors, x_range=[0, 100],width=400, height=400) 28 | dot.segment(0, factors, x, factors, line_width=2, line_color="green", ) 29 | dot.circle(x, factors, size=15, fill_color="orange", line_color="green", line_width=3, ) 30 | script, div = components(dot, CDN) 31 | 32 | return [script,div] 33 | else: 34 | return [0,file_hepler.get_image_path("no_good_comments.png")] 35 | 36 | def make_overview_bokeh(data): 37 | if data: 38 | temps = "".join(data) 39 | values = re.findall(r'(\d+)', temps) 40 | c_values = [int(value) for value in values] 41 | c_keys = re.findall('[\u4e00-\u9fa5]+', temps) 42 | s = pd.Series(c_values, index=c_keys) 43 | data = s 44 | # 创建一个新的含有标题和轴标签的窗口在线窗口 45 | p = Bar(data,title="总览图", ylabel='关键字数量', width=400, height=400,legend=None,tools="") 46 | script, div = components(p, CDN) 47 | 48 | return [script, div] 49 | else: 50 | return [0,file_hepler.get_image_path("no_overview.png")] 51 | 52 | def make_hot_bokeh(data): 53 | from bokeh.models import BoxAnnotation 54 | if data: 55 | data = data.split(",") 56 | dt1 = datetime.datetime.now() 57 | temp = list(set(data)) 58 | sub_dates = [(dt1 - datetime.datetime.strptime(dt, '%Y-%m-%d')).days for dt in temp] 59 | 60 | s = pd.Series(sub_dates, [datetime.datetime.strptime(dt, '%Y-%m-%d') for dt in temp]) 61 | s = s.sort_index(ascending=False) 62 | 63 | p = figure(x_axis_type="datetime", tools="",title="热度", x_axis_label='日期', y_axis_label='天数',width=400, height=400) 64 | 65 | p.line(s.index,s.values, legend="距离当前天数", line_width=2) 66 | 67 | script, div = components(p, CDN) 68 | 69 | return [script, div] 70 | else: 71 | return [0,file_hepler.get_image_path("no_hot.png")] 72 | -------------------------------------------------------------------------------- /hepler/make_gif_helper.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | # api:http://matplotlib.org/api/animation_api.html 4 | # imagemagick环境配置:http://www.hankcs.com/ml/using-matplotlib-and-imagemagick-to-realize-the-algorithm-visualization-and-gif-export.html 5 | import numpy as np 6 | 7 | import matplotlib.pyplot as plt 8 | import matplotlib.patches as patches 9 | import matplotlib.path as path 10 | import matplotlib.animation as animation 11 | fig, ax = plt.subplots() 12 | 13 | # histogram our data with numpy 14 | data = np.random.randn(1000) 15 | n, bins = np.histogram(data, 100) 16 | 17 | # get the corners of the rectangles for the histogram 18 | left = np.array(bins[:-1]) 19 | right = np.array(bins[1:]) 20 | bottom = np.zeros(len(left)) 21 | top = bottom + n 22 | nrects = len(left) 23 | 24 | # here comes the tricky part -- we have to set up the vertex and path 25 | # codes arrays using moveto, lineto and closepoly 26 | 27 | # for each rect: 1 for the MOVETO, 3 for the LINETO, 1 for the 28 | # CLOSEPOLY; the vert for the closepoly is ignored but we still need 29 | # it to keep the codes aligned with the vertices 30 | nverts = nrects*(1 + 3 + 1) 31 | verts = np.zeros((nverts, 2)) 32 | codes = np.ones(nverts, int) * path.Path.LINETO 33 | codes[0::5] = path.Path.MOVETO 34 | codes[4::5] = path.Path.CLOSEPOLY 35 | verts[0::5, 0] = left 36 | verts[0::5, 1] = bottom 37 | verts[1::5, 0] = left 38 | verts[1::5, 1] = top 39 | verts[2::5, 0] = right 40 | verts[2::5, 1] = top 41 | verts[3::5, 0] = right 42 | verts[3::5, 1] = bottom 43 | 44 | barpath = path.Path(verts, codes) 45 | patch = patches.PathPatch( 46 | barpath, facecolor='#787878', edgecolor='#787878', alpha=1) 47 | ax.add_patch(patch) 48 | 49 | ax.set_xlim(left[0], right[-1]) 50 | ax.set_ylim(bottom.min(), top.max()) 51 | 52 | 53 | def animate(i): 54 | # simulate new data coming in 55 | data = np.random.randn(1000) 56 | n, bins = np.histogram(data, 100) 57 | top = bottom + n 58 | verts[1::5, 1] = top 59 | verts[2::5, 1] = top 60 | return [patch, ] 61 | 62 | ani = animation.FuncAnimation(fig, animate, 100, repeat=False, blit=True) 63 | ani.save('loading1.gif',fps=2, writer='imagemagick') -------------------------------------------------------------------------------- /hepler/make_plot_helper.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | import re 4 | import matplotlib.pyplot as plt 5 | from matplotlib.pylab import mpl 6 | import datetime 7 | import pandas as pd 8 | from hepler import redis_helper,file_hepler 9 | from bokeh.plotting import figure,show,output_file 10 | 11 | mpl.rcParams['font.sans-serif'] = ['SimHei'] # 显示中文 12 | mpl.rcParams['font.serif'] = ['SimHei'] # 显示中文 13 | mpl.rcParams['axes.unicode_minus'] = False # 负号'-'正常显示 14 | import json 15 | 16 | 17 | def make_comment_plot(data,p_id): 18 | """ 19 | 生成好评率的饼图 20 | :param datas: 21 | :return: 22 | """ 23 | if data: 24 | temps = "".join(data).replace(" ", "").replace("\r\n", "") 25 | values = re.findall(r'(\d+)', temps) 26 | c_values = [int(value) for value in values] 27 | c_keys = re.findall('[\u4e00-\u9fa5]+', temps) 28 | print(c_keys) 29 | s = pd.Series(c_values, index=c_keys,name='好评率') 30 | s = s[3:6] 31 | s_sum = s.sum() 32 | s = s.apply(lambda x: x / s_sum) 33 | s.plot.pie(autopct='%0.2f%%', fontsize=8, colors=['g', 'y', 'r']) 34 | plt.savefig("static/upload/%s_c.png" % p_id,dpi=90) 35 | plt.close() 36 | 37 | return file_hepler.get_image_path("%s_c.png" % p_id) 38 | else: 39 | return file_hepler.get_image_path("no_good_comments.png") 40 | 41 | def make_overview_plot(data,p_id): 42 | """ 43 | 评价概览 44 | :param datas: 45 | :return: 46 | """ 47 | if data: 48 | temps = "".join(data) 49 | values = re.findall(r'(\d+)', temps) 50 | c_values = [int(value) for value in values] 51 | c_keys = re.findall('[\u4e00-\u9fa5]+', temps) 52 | s = pd.Series(c_values, index=c_keys) 53 | s.plot.bar(figsize=(6, 8), fontsize=8) 54 | 55 | plt.savefig("static/upload/%s_o.png" % p_id,dpi=90) 56 | plt.close() 57 | return file_hepler.get_image_path("%s_o.png" % p_id) 58 | else: 59 | return file_hepler.get_image_path("no_overview.png") 60 | 61 | def make_hot_plot(data,p_id): 62 | """ 63 | 根据评价最近评价来判断这个产品是否是热门 64 | :param datas: 65 | :return: 66 | """ 67 | if data: 68 | data = data.split(",") 69 | dt1 = datetime.datetime.now() 70 | temp = list(set(data)) 71 | sub_dates = [(dt1 - datetime.datetime.strptime(dt, '%Y-%m-%d')).days for dt in temp] 72 | actives = [] 73 | for d in sub_dates: 74 | if d <= 360: 75 | actives.append(100) 76 | elif 360 < d <= 500: 77 | actives.append(60) 78 | else: 79 | actives.append(0) 80 | 81 | date_dict = { 82 | 'sub_date': sub_dates, 83 | 'active': actives 84 | } 85 | df = pd.DataFrame(date_dict, index=temp) 86 | df.plot(subplots=True) 87 | plt.savefig("static/upload/%s_h.png" % p_id,dpi=90) 88 | return file_hepler.get_image_path("%s_h.png" % p_id) 89 | else: 90 | return file_hepler.get_image_path("no_hot.png") 91 | 92 | 93 | -------------------------------------------------------------------------------- /hepler/redis_helper.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | import redis 4 | from . import config 5 | import json 6 | 7 | p_redis = redis.StrictRedis(host=config.redis_host, port=config.redis_port) 8 | 9 | def save_to_redis(item): 10 | """ 11 | 保存数据的redis 12 | :param data: 需要保存的数据 13 | :return: 14 | """ 15 | p_redis.setex(item['p_id'],config.REDIS_PRODUCT_INFO_EXPIRES_SECONDES,json.dumps(item)) -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ProductAnalysis.settings") 7 | try: 8 | from django.core.management import execute_from_command_line 9 | except ImportError: 10 | # The above import may fail for some other reason. Ensure that the 11 | # issue is really that Django is missing to avoid masking other 12 | # exceptions on Python 2. 13 | try: 14 | import django 15 | except ImportError: 16 | raise ImportError( 17 | "Couldn't import Django. Are you sure it's installed and " 18 | "available on your PYTHONPATH environment variable? Did you " 19 | "forget to activate a virtual environment?" 20 | ) 21 | raise 22 | execute_from_command_line(sys.argv) 23 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | asn1crypto==0.22.0 2 | attrs==17.2.0 3 | Automat==0.6.0 4 | bkcharts==0.2 5 | bokeh==0.12.6 6 | certifi==2017.4.17 7 | cffi==1.10.0 8 | chardet==3.0.4 9 | constantly==15.1.0 10 | cryptography==1.9 11 | cssselect==1.0.1 12 | cycler==0.10.0 13 | Django==1.10.6 14 | django-haystack==2.6.1 15 | gunicorn==19.7.1 16 | hyperlink==17.2.1 17 | idna==2.5 18 | incremental==17.5.0 19 | jieba==0.38 20 | Jinja2==2.9.6 21 | lxml==3.8.0 22 | MarkupSafe==1.0 23 | matplotlib==2.0.2 24 | numpy==1.13.1 25 | pandas==0.20.3 26 | parsel==1.2.0 27 | pkg-resources==0.0.0 28 | psycopg2==2.7.1 29 | pyasn1==0.2.3 30 | pyasn1-modules==0.0.9 31 | pycparser==2.18 32 | PyDispatcher==2.0.5 33 | PyMySQL==0.7.11 34 | pyOpenSSL==17.1.0 35 | pyparsing==2.2.0 36 | python-dateutil==2.6.1 37 | pytz==2017.2 38 | PyYAML==3.12 39 | queuelib==1.4.2 40 | redis==2.10.5 41 | requests==2.18.1 42 | Scrapy==1.4.0 43 | scrapy-redis==0.6.8 44 | service-identity==17.0.0 45 | six==1.10.0 46 | tornado==4.5.1 47 | Twisted==17.5.0 48 | urllib3==1.21.1 49 | uWSGI==2.0.15 50 | w3lib==1.17.0 51 | Whoosh==2.7.4 52 | zope.interface==4.4.2 53 | -------------------------------------------------------------------------------- /screen/Scrapy流程.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/screen/Scrapy流程.png -------------------------------------------------------------------------------- /screen/index.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/screen/index.png -------------------------------------------------------------------------------- /screen/search.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/screen/search.png -------------------------------------------------------------------------------- /static/css/font-awesome.min.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Font Awesome 4.2.0 by @davegandy - http://fontawesome.io - @fontawesome 3 | * License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License) 4 | */@font-face{font-family:'FontAwesome';src:url('../fonts/fontawesome-webfont.eot?v=4.2.0');src:url('../fonts/fontawesome-webfont.eot?#iefix&v=4.2.0') format('embedded-opentype'),url('../fonts/fontawesome-webfont.woff?v=4.2.0') format('woff'),url('../fonts/fontawesome-webfont.ttf?v=4.2.0') format('truetype'),url('../fonts/fontawesome-webfont.svg?v=4.2.0#fontawesomeregular') format('svg');font-weight:normal;font-style:normal}.fa{display:inline-block;font:normal normal normal 14px/1 FontAwesome;font-size:inherit;text-rendering:auto;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.fa-lg{font-size:1.33333333em;line-height:.75em;vertical-align:-15%}.fa-2x{font-size:2em}.fa-3x{font-size:3em}.fa-4x{font-size:4em}.fa-5x{font-size:5em}.fa-fw{width:1.28571429em;text-align:center}.fa-ul{padding-left:0;margin-left:2.14285714em;list-style-type:none}.fa-ul>li{position:relative}.fa-li{position:absolute;left:-2.14285714em;width:2.14285714em;top:.14285714em;text-align:center}.fa-li.fa-lg{left:-1.85714286em}.fa-border{padding:.2em .25em .15em;border:solid .08em #eee;border-radius:.1em}.pull-right{float:right}.pull-left{float:left}.fa.pull-left{margin-right:.3em}.fa.pull-right{margin-left:.3em}.fa-spin{-webkit-animation:fa-spin 2s infinite linear;animation:fa-spin 2s infinite linear}@-webkit-keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}@keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}.fa-rotate-90{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=1);-webkit-transform:rotate(90deg);-ms-transform:rotate(90deg);transform:rotate(90deg)}.fa-rotate-180{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=2);-webkit-transform:rotate(180deg);-ms-transform:rotate(180deg);transform:rotate(180deg)}.fa-rotate-270{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=3);-webkit-transform:rotate(270deg);-ms-transform:rotate(270deg);transform:rotate(270deg)}.fa-flip-horizontal{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1);-webkit-transform:scale(-1, 1);-ms-transform:scale(-1, 1);transform:scale(-1, 1)}.fa-flip-vertical{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1);-webkit-transform:scale(1, -1);-ms-transform:scale(1, -1);transform:scale(1, -1)}:root .fa-rotate-90,:root .fa-rotate-180,:root .fa-rotate-270,:root .fa-flip-horizontal,:root .fa-flip-vertical{filter:none}.fa-stack{position:relative;display:inline-block;width:2em;height:2em;line-height:2em;vertical-align:middle}.fa-stack-1x,.fa-stack-2x{position:absolute;left:0;width:100%;text-align:center}.fa-stack-1x{line-height:inherit}.fa-stack-2x{font-size:2em}.fa-inverse{color:#fff}.fa-glass:before{content:"\f000"}.fa-music:before{content:"\f001"}.fa-search:before{content:"\f002"}.fa-envelope-o:before{content:"\f003"}.fa-heart:before{content:"\f004"}.fa-star:before{content:"\f005"}.fa-star-o:before{content:"\f006"}.fa-user:before{content:"\f007"}.fa-film:before{content:"\f008"}.fa-th-large:before{content:"\f009"}.fa-th:before{content:"\f00a"}.fa-th-list:before{content:"\f00b"}.fa-check:before{content:"\f00c"}.fa-remove:before,.fa-close:before,.fa-times:before{content:"\f00d"}.fa-search-plus:before{content:"\f00e"}.fa-search-minus:before{content:"\f010"}.fa-power-off:before{content:"\f011"}.fa-signal:before{content:"\f012"}.fa-gear:before,.fa-cog:before{content:"\f013"}.fa-trash-o:before{content:"\f014"}.fa-home:before{content:"\f015"}.fa-file-o:before{content:"\f016"}.fa-clock-o:before{content:"\f017"}.fa-road:before{content:"\f018"}.fa-download:before{content:"\f019"}.fa-arrow-circle-o-down:before{content:"\f01a"}.fa-arrow-circle-o-up:before{content:"\f01b"}.fa-inbox:before{content:"\f01c"}.fa-play-circle-o:before{content:"\f01d"}.fa-rotate-right:before,.fa-repeat:before{content:"\f01e"}.fa-refresh:before{content:"\f021"}.fa-list-alt:before{content:"\f022"}.fa-lock:before{content:"\f023"}.fa-flag:before{content:"\f024"}.fa-headphones:before{content:"\f025"}.fa-volume-off:before{content:"\f026"}.fa-volume-down:before{content:"\f027"}.fa-volume-up:before{content:"\f028"}.fa-qrcode:before{content:"\f029"}.fa-barcode:before{content:"\f02a"}.fa-tag:before{content:"\f02b"}.fa-tags:before{content:"\f02c"}.fa-book:before{content:"\f02d"}.fa-bookmark:before{content:"\f02e"}.fa-print:before{content:"\f02f"}.fa-camera:before{content:"\f030"}.fa-font:before{content:"\f031"}.fa-bold:before{content:"\f032"}.fa-italic:before{content:"\f033"}.fa-text-height:before{content:"\f034"}.fa-text-width:before{content:"\f035"}.fa-align-left:before{content:"\f036"}.fa-align-center:before{content:"\f037"}.fa-align-right:before{content:"\f038"}.fa-align-justify:before{content:"\f039"}.fa-list:before{content:"\f03a"}.fa-dedent:before,.fa-outdent:before{content:"\f03b"}.fa-indent:before{content:"\f03c"}.fa-video-camera:before{content:"\f03d"}.fa-photo:before,.fa-image:before,.fa-picture-o:before{content:"\f03e"}.fa-pencil:before{content:"\f040"}.fa-map-marker:before{content:"\f041"}.fa-adjust:before{content:"\f042"}.fa-tint:before{content:"\f043"}.fa-edit:before,.fa-pencil-square-o:before{content:"\f044"}.fa-share-square-o:before{content:"\f045"}.fa-check-square-o:before{content:"\f046"}.fa-arrows:before{content:"\f047"}.fa-step-backward:before{content:"\f048"}.fa-fast-backward:before{content:"\f049"}.fa-backward:before{content:"\f04a"}.fa-play:before{content:"\f04b"}.fa-pause:before{content:"\f04c"}.fa-stop:before{content:"\f04d"}.fa-forward:before{content:"\f04e"}.fa-fast-forward:before{content:"\f050"}.fa-step-forward:before{content:"\f051"}.fa-eject:before{content:"\f052"}.fa-chevron-left:before{content:"\f053"}.fa-chevron-right:before{content:"\f054"}.fa-plus-circle:before{content:"\f055"}.fa-minus-circle:before{content:"\f056"}.fa-times-circle:before{content:"\f057"}.fa-check-circle:before{content:"\f058"}.fa-question-circle:before{content:"\f059"}.fa-info-circle:before{content:"\f05a"}.fa-crosshairs:before{content:"\f05b"}.fa-times-circle-o:before{content:"\f05c"}.fa-check-circle-o:before{content:"\f05d"}.fa-ban:before{content:"\f05e"}.fa-arrow-left:before{content:"\f060"}.fa-arrow-right:before{content:"\f061"}.fa-arrow-up:before{content:"\f062"}.fa-arrow-down:before{content:"\f063"}.fa-mail-forward:before,.fa-share:before{content:"\f064"}.fa-expand:before{content:"\f065"}.fa-compress:before{content:"\f066"}.fa-plus:before{content:"\f067"}.fa-minus:before{content:"\f068"}.fa-asterisk:before{content:"\f069"}.fa-exclamation-circle:before{content:"\f06a"}.fa-gift:before{content:"\f06b"}.fa-leaf:before{content:"\f06c"}.fa-fire:before{content:"\f06d"}.fa-eye:before{content:"\f06e"}.fa-eye-slash:before{content:"\f070"}.fa-warning:before,.fa-exclamation-triangle:before{content:"\f071"}.fa-plane:before{content:"\f072"}.fa-calendar:before{content:"\f073"}.fa-random:before{content:"\f074"}.fa-comment:before{content:"\f075"}.fa-magnet:before{content:"\f076"}.fa-chevron-up:before{content:"\f077"}.fa-chevron-down:before{content:"\f078"}.fa-retweet:before{content:"\f079"}.fa-shopping-cart:before{content:"\f07a"}.fa-folder:before{content:"\f07b"}.fa-folder-open:before{content:"\f07c"}.fa-arrows-v:before{content:"\f07d"}.fa-arrows-h:before{content:"\f07e"}.fa-bar-chart-o:before,.fa-bar-chart:before{content:"\f080"}.fa-twitter-square:before{content:"\f081"}.fa-facebook-square:before{content:"\f082"}.fa-camera-retro:before{content:"\f083"}.fa-key:before{content:"\f084"}.fa-gears:before,.fa-cogs:before{content:"\f085"}.fa-comments:before{content:"\f086"}.fa-thumbs-o-up:before{content:"\f087"}.fa-thumbs-o-down:before{content:"\f088"}.fa-star-half:before{content:"\f089"}.fa-heart-o:before{content:"\f08a"}.fa-sign-out:before{content:"\f08b"}.fa-linkedin-square:before{content:"\f08c"}.fa-thumb-tack:before{content:"\f08d"}.fa-external-link:before{content:"\f08e"}.fa-sign-in:before{content:"\f090"}.fa-trophy:before{content:"\f091"}.fa-github-square:before{content:"\f092"}.fa-upload:before{content:"\f093"}.fa-lemon-o:before{content:"\f094"}.fa-phone:before{content:"\f095"}.fa-square-o:before{content:"\f096"}.fa-bookmark-o:before{content:"\f097"}.fa-phone-square:before{content:"\f098"}.fa-twitter:before{content:"\f099"}.fa-facebook:before{content:"\f09a"}.fa-github:before{content:"\f09b"}.fa-unlock:before{content:"\f09c"}.fa-credit-card:before{content:"\f09d"}.fa-rss:before{content:"\f09e"}.fa-hdd-o:before{content:"\f0a0"}.fa-bullhorn:before{content:"\f0a1"}.fa-bell:before{content:"\f0f3"}.fa-certificate:before{content:"\f0a3"}.fa-hand-o-right:before{content:"\f0a4"}.fa-hand-o-left:before{content:"\f0a5"}.fa-hand-o-up:before{content:"\f0a6"}.fa-hand-o-down:before{content:"\f0a7"}.fa-arrow-circle-left:before{content:"\f0a8"}.fa-arrow-circle-right:before{content:"\f0a9"}.fa-arrow-circle-up:before{content:"\f0aa"}.fa-arrow-circle-down:before{content:"\f0ab"}.fa-globe:before{content:"\f0ac"}.fa-wrench:before{content:"\f0ad"}.fa-tasks:before{content:"\f0ae"}.fa-filter:before{content:"\f0b0"}.fa-briefcase:before{content:"\f0b1"}.fa-arrows-alt:before{content:"\f0b2"}.fa-group:before,.fa-users:before{content:"\f0c0"}.fa-chain:before,.fa-link:before{content:"\f0c1"}.fa-cloud:before{content:"\f0c2"}.fa-flask:before{content:"\f0c3"}.fa-cut:before,.fa-scissors:before{content:"\f0c4"}.fa-copy:before,.fa-files-o:before{content:"\f0c5"}.fa-paperclip:before{content:"\f0c6"}.fa-save:before,.fa-floppy-o:before{content:"\f0c7"}.fa-square:before{content:"\f0c8"}.fa-navicon:before,.fa-reorder:before,.fa-bars:before{content:"\f0c9"}.fa-list-ul:before{content:"\f0ca"}.fa-list-ol:before{content:"\f0cb"}.fa-strikethrough:before{content:"\f0cc"}.fa-underline:before{content:"\f0cd"}.fa-table:before{content:"\f0ce"}.fa-magic:before{content:"\f0d0"}.fa-truck:before{content:"\f0d1"}.fa-pinterest:before{content:"\f0d2"}.fa-pinterest-square:before{content:"\f0d3"}.fa-google-plus-square:before{content:"\f0d4"}.fa-google-plus:before{content:"\f0d5"}.fa-money:before{content:"\f0d6"}.fa-caret-down:before{content:"\f0d7"}.fa-caret-up:before{content:"\f0d8"}.fa-caret-left:before{content:"\f0d9"}.fa-caret-right:before{content:"\f0da"}.fa-columns:before{content:"\f0db"}.fa-unsorted:before,.fa-sort:before{content:"\f0dc"}.fa-sort-down:before,.fa-sort-desc:before{content:"\f0dd"}.fa-sort-up:before,.fa-sort-asc:before{content:"\f0de"}.fa-envelope:before{content:"\f0e0"}.fa-linkedin:before{content:"\f0e1"}.fa-rotate-left:before,.fa-undo:before{content:"\f0e2"}.fa-legal:before,.fa-gavel:before{content:"\f0e3"}.fa-dashboard:before,.fa-tachometer:before{content:"\f0e4"}.fa-comment-o:before{content:"\f0e5"}.fa-comments-o:before{content:"\f0e6"}.fa-flash:before,.fa-bolt:before{content:"\f0e7"}.fa-sitemap:before{content:"\f0e8"}.fa-umbrella:before{content:"\f0e9"}.fa-paste:before,.fa-clipboard:before{content:"\f0ea"}.fa-lightbulb-o:before{content:"\f0eb"}.fa-exchange:before{content:"\f0ec"}.fa-cloud-download:before{content:"\f0ed"}.fa-cloud-upload:before{content:"\f0ee"}.fa-user-md:before{content:"\f0f0"}.fa-stethoscope:before{content:"\f0f1"}.fa-suitcase:before{content:"\f0f2"}.fa-bell-o:before{content:"\f0a2"}.fa-coffee:before{content:"\f0f4"}.fa-cutlery:before{content:"\f0f5"}.fa-file-text-o:before{content:"\f0f6"}.fa-building-o:before{content:"\f0f7"}.fa-hospital-o:before{content:"\f0f8"}.fa-ambulance:before{content:"\f0f9"}.fa-medkit:before{content:"\f0fa"}.fa-fighter-jet:before{content:"\f0fb"}.fa-beer:before{content:"\f0fc"}.fa-h-square:before{content:"\f0fd"}.fa-plus-square:before{content:"\f0fe"}.fa-angle-double-left:before{content:"\f100"}.fa-angle-double-right:before{content:"\f101"}.fa-angle-double-up:before{content:"\f102"}.fa-angle-double-down:before{content:"\f103"}.fa-angle-left:before{content:"\f104"}.fa-angle-right:before{content:"\f105"}.fa-angle-up:before{content:"\f106"}.fa-angle-down:before{content:"\f107"}.fa-desktop:before{content:"\f108"}.fa-laptop:before{content:"\f109"}.fa-tablet:before{content:"\f10a"}.fa-mobile-phone:before,.fa-mobile:before{content:"\f10b"}.fa-circle-o:before{content:"\f10c"}.fa-quote-left:before{content:"\f10d"}.fa-quote-right:before{content:"\f10e"}.fa-spinner:before{content:"\f110"}.fa-circle:before{content:"\f111"}.fa-mail-reply:before,.fa-reply:before{content:"\f112"}.fa-github-alt:before{content:"\f113"}.fa-folder-o:before{content:"\f114"}.fa-folder-open-o:before{content:"\f115"}.fa-smile-o:before{content:"\f118"}.fa-frown-o:before{content:"\f119"}.fa-meh-o:before{content:"\f11a"}.fa-gamepad:before{content:"\f11b"}.fa-keyboard-o:before{content:"\f11c"}.fa-flag-o:before{content:"\f11d"}.fa-flag-checkered:before{content:"\f11e"}.fa-terminal:before{content:"\f120"}.fa-code:before{content:"\f121"}.fa-mail-reply-all:before,.fa-reply-all:before{content:"\f122"}.fa-star-half-empty:before,.fa-star-half-full:before,.fa-star-half-o:before{content:"\f123"}.fa-location-arrow:before{content:"\f124"}.fa-crop:before{content:"\f125"}.fa-code-fork:before{content:"\f126"}.fa-unlink:before,.fa-chain-broken:before{content:"\f127"}.fa-question:before{content:"\f128"}.fa-info:before{content:"\f129"}.fa-exclamation:before{content:"\f12a"}.fa-superscript:before{content:"\f12b"}.fa-subscript:before{content:"\f12c"}.fa-eraser:before{content:"\f12d"}.fa-puzzle-piece:before{content:"\f12e"}.fa-microphone:before{content:"\f130"}.fa-microphone-slash:before{content:"\f131"}.fa-shield:before{content:"\f132"}.fa-calendar-o:before{content:"\f133"}.fa-fire-extinguisher:before{content:"\f134"}.fa-rocket:before{content:"\f135"}.fa-maxcdn:before{content:"\f136"}.fa-chevron-circle-left:before{content:"\f137"}.fa-chevron-circle-right:before{content:"\f138"}.fa-chevron-circle-up:before{content:"\f139"}.fa-chevron-circle-down:before{content:"\f13a"}.fa-html5:before{content:"\f13b"}.fa-css3:before{content:"\f13c"}.fa-anchor:before{content:"\f13d"}.fa-unlock-alt:before{content:"\f13e"}.fa-bullseye:before{content:"\f140"}.fa-ellipsis-h:before{content:"\f141"}.fa-ellipsis-v:before{content:"\f142"}.fa-rss-square:before{content:"\f143"}.fa-play-circle:before{content:"\f144"}.fa-ticket:before{content:"\f145"}.fa-minus-square:before{content:"\f146"}.fa-minus-square-o:before{content:"\f147"}.fa-level-up:before{content:"\f148"}.fa-level-down:before{content:"\f149"}.fa-check-square:before{content:"\f14a"}.fa-pencil-square:before{content:"\f14b"}.fa-external-link-square:before{content:"\f14c"}.fa-share-square:before{content:"\f14d"}.fa-compass:before{content:"\f14e"}.fa-toggle-down:before,.fa-caret-square-o-down:before{content:"\f150"}.fa-toggle-up:before,.fa-caret-square-o-up:before{content:"\f151"}.fa-toggle-right:before,.fa-caret-square-o-right:before{content:"\f152"}.fa-euro:before,.fa-eur:before{content:"\f153"}.fa-gbp:before{content:"\f154"}.fa-dollar:before,.fa-usd:before{content:"\f155"}.fa-rupee:before,.fa-inr:before{content:"\f156"}.fa-cny:before,.fa-rmb:before,.fa-yen:before,.fa-jpy:before{content:"\f157"}.fa-ruble:before,.fa-rouble:before,.fa-rub:before{content:"\f158"}.fa-won:before,.fa-krw:before{content:"\f159"}.fa-bitcoin:before,.fa-btc:before{content:"\f15a"}.fa-file:before{content:"\f15b"}.fa-file-text:before{content:"\f15c"}.fa-sort-alpha-asc:before{content:"\f15d"}.fa-sort-alpha-desc:before{content:"\f15e"}.fa-sort-amount-asc:before{content:"\f160"}.fa-sort-amount-desc:before{content:"\f161"}.fa-sort-numeric-asc:before{content:"\f162"}.fa-sort-numeric-desc:before{content:"\f163"}.fa-thumbs-up:before{content:"\f164"}.fa-thumbs-down:before{content:"\f165"}.fa-youtube-square:before{content:"\f166"}.fa-youtube:before{content:"\f167"}.fa-xing:before{content:"\f168"}.fa-xing-square:before{content:"\f169"}.fa-youtube-play:before{content:"\f16a"}.fa-dropbox:before{content:"\f16b"}.fa-stack-overflow:before{content:"\f16c"}.fa-instagram:before{content:"\f16d"}.fa-flickr:before{content:"\f16e"}.fa-adn:before{content:"\f170"}.fa-bitbucket:before{content:"\f171"}.fa-bitbucket-square:before{content:"\f172"}.fa-tumblr:before{content:"\f173"}.fa-tumblr-square:before{content:"\f174"}.fa-long-arrow-down:before{content:"\f175"}.fa-long-arrow-up:before{content:"\f176"}.fa-long-arrow-left:before{content:"\f177"}.fa-long-arrow-right:before{content:"\f178"}.fa-apple:before{content:"\f179"}.fa-windows:before{content:"\f17a"}.fa-android:before{content:"\f17b"}.fa-linux:before{content:"\f17c"}.fa-dribbble:before{content:"\f17d"}.fa-skype:before{content:"\f17e"}.fa-foursquare:before{content:"\f180"}.fa-trello:before{content:"\f181"}.fa-female:before{content:"\f182"}.fa-male:before{content:"\f183"}.fa-gittip:before{content:"\f184"}.fa-sun-o:before{content:"\f185"}.fa-moon-o:before{content:"\f186"}.fa-archive:before{content:"\f187"}.fa-bug:before{content:"\f188"}.fa-vk:before{content:"\f189"}.fa-weibo:before{content:"\f18a"}.fa-renren:before{content:"\f18b"}.fa-pagelines:before{content:"\f18c"}.fa-stack-exchange:before{content:"\f18d"}.fa-arrow-circle-o-right:before{content:"\f18e"}.fa-arrow-circle-o-left:before{content:"\f190"}.fa-toggle-left:before,.fa-caret-square-o-left:before{content:"\f191"}.fa-dot-circle-o:before{content:"\f192"}.fa-wheelchair:before{content:"\f193"}.fa-vimeo-square:before{content:"\f194"}.fa-turkish-lira:before,.fa-try:before{content:"\f195"}.fa-plus-square-o:before{content:"\f196"}.fa-space-shuttle:before{content:"\f197"}.fa-slack:before{content:"\f198"}.fa-envelope-square:before{content:"\f199"}.fa-wordpress:before{content:"\f19a"}.fa-openid:before{content:"\f19b"}.fa-institution:before,.fa-bank:before,.fa-university:before{content:"\f19c"}.fa-mortar-board:before,.fa-graduation-cap:before{content:"\f19d"}.fa-yahoo:before{content:"\f19e"}.fa-google:before{content:"\f1a0"}.fa-reddit:before{content:"\f1a1"}.fa-reddit-square:before{content:"\f1a2"}.fa-stumbleupon-circle:before{content:"\f1a3"}.fa-stumbleupon:before{content:"\f1a4"}.fa-delicious:before{content:"\f1a5"}.fa-digg:before{content:"\f1a6"}.fa-pied-piper:before{content:"\f1a7"}.fa-pied-piper-alt:before{content:"\f1a8"}.fa-drupal:before{content:"\f1a9"}.fa-joomla:before{content:"\f1aa"}.fa-language:before{content:"\f1ab"}.fa-fax:before{content:"\f1ac"}.fa-building:before{content:"\f1ad"}.fa-child:before{content:"\f1ae"}.fa-paw:before{content:"\f1b0"}.fa-spoon:before{content:"\f1b1"}.fa-cube:before{content:"\f1b2"}.fa-cubes:before{content:"\f1b3"}.fa-behance:before{content:"\f1b4"}.fa-behance-square:before{content:"\f1b5"}.fa-steam:before{content:"\f1b6"}.fa-steam-square:before{content:"\f1b7"}.fa-recycle:before{content:"\f1b8"}.fa-automobile:before,.fa-car:before{content:"\f1b9"}.fa-cab:before,.fa-taxi:before{content:"\f1ba"}.fa-tree:before{content:"\f1bb"}.fa-spotify:before{content:"\f1bc"}.fa-deviantart:before{content:"\f1bd"}.fa-soundcloud:before{content:"\f1be"}.fa-database:before{content:"\f1c0"}.fa-file-pdf-o:before{content:"\f1c1"}.fa-file-word-o:before{content:"\f1c2"}.fa-file-excel-o:before{content:"\f1c3"}.fa-file-powerpoint-o:before{content:"\f1c4"}.fa-file-photo-o:before,.fa-file-picture-o:before,.fa-file-image-o:before{content:"\f1c5"}.fa-file-zip-o:before,.fa-file-archive-o:before{content:"\f1c6"}.fa-file-sound-o:before,.fa-file-audio-o:before{content:"\f1c7"}.fa-file-movie-o:before,.fa-file-video-o:before{content:"\f1c8"}.fa-file-code-o:before{content:"\f1c9"}.fa-vine:before{content:"\f1ca"}.fa-codepen:before{content:"\f1cb"}.fa-jsfiddle:before{content:"\f1cc"}.fa-life-bouy:before,.fa-life-buoy:before,.fa-life-saver:before,.fa-support:before,.fa-life-ring:before{content:"\f1cd"}.fa-circle-o-notch:before{content:"\f1ce"}.fa-ra:before,.fa-rebel:before{content:"\f1d0"}.fa-ge:before,.fa-empire:before{content:"\f1d1"}.fa-git-square:before{content:"\f1d2"}.fa-git:before{content:"\f1d3"}.fa-hacker-news:before{content:"\f1d4"}.fa-tencent-weibo:before{content:"\f1d5"}.fa-qq:before{content:"\f1d6"}.fa-wechat:before,.fa-weixin:before{content:"\f1d7"}.fa-send:before,.fa-paper-plane:before{content:"\f1d8"}.fa-send-o:before,.fa-paper-plane-o:before{content:"\f1d9"}.fa-history:before{content:"\f1da"}.fa-circle-thin:before{content:"\f1db"}.fa-header:before{content:"\f1dc"}.fa-paragraph:before{content:"\f1dd"}.fa-sliders:before{content:"\f1de"}.fa-share-alt:before{content:"\f1e0"}.fa-share-alt-square:before{content:"\f1e1"}.fa-bomb:before{content:"\f1e2"}.fa-soccer-ball-o:before,.fa-futbol-o:before{content:"\f1e3"}.fa-tty:before{content:"\f1e4"}.fa-binoculars:before{content:"\f1e5"}.fa-plug:before{content:"\f1e6"}.fa-slideshare:before{content:"\f1e7"}.fa-twitch:before{content:"\f1e8"}.fa-yelp:before{content:"\f1e9"}.fa-newspaper-o:before{content:"\f1ea"}.fa-wifi:before{content:"\f1eb"}.fa-calculator:before{content:"\f1ec"}.fa-paypal:before{content:"\f1ed"}.fa-google-wallet:before{content:"\f1ee"}.fa-cc-visa:before{content:"\f1f0"}.fa-cc-mastercard:before{content:"\f1f1"}.fa-cc-discover:before{content:"\f1f2"}.fa-cc-amex:before{content:"\f1f3"}.fa-cc-paypal:before{content:"\f1f4"}.fa-cc-stripe:before{content:"\f1f5"}.fa-bell-slash:before{content:"\f1f6"}.fa-bell-slash-o:before{content:"\f1f7"}.fa-trash:before{content:"\f1f8"}.fa-copyright:before{content:"\f1f9"}.fa-at:before{content:"\f1fa"}.fa-eyedropper:before{content:"\f1fb"}.fa-paint-brush:before{content:"\f1fc"}.fa-birthday-cake:before{content:"\f1fd"}.fa-area-chart:before{content:"\f1fe"}.fa-pie-chart:before{content:"\f200"}.fa-line-chart:before{content:"\f201"}.fa-lastfm:before{content:"\f202"}.fa-lastfm-square:before{content:"\f203"}.fa-toggle-off:before{content:"\f204"}.fa-toggle-on:before{content:"\f205"}.fa-bicycle:before{content:"\f206"}.fa-bus:before{content:"\f207"}.fa-ioxhost:before{content:"\f208"}.fa-angellist:before{content:"\f209"}.fa-cc:before{content:"\f20a"}.fa-shekel:before,.fa-sheqel:before,.fa-ils:before{content:"\f20b"}.fa-meanpath:before{content:"\f20c"} -------------------------------------------------------------------------------- /static/css/loading.css: -------------------------------------------------------------------------------- 1 | .ui-loading{position:absolute;left:0;top:0;z-index: 9999;} 2 | .ui-loading .ui-loading-mask{ position:absolute;top:0;left:0;background-color: #000;opacity: .2;z-index: 1} 3 | .ui-loading i{height: 90px;width:90px; display: block;background: url('/static/gif/loading1.gif') no-repeat center center;background-size:100% 100%;position: absolute;z-index: 2} 4 | -------------------------------------------------------------------------------- /static/css/main.css: -------------------------------------------------------------------------------- 1 | 2 | /* ------------------------------------------------------------------------------ 3 | Typography 4 | -------------------------------------------------------------------------------*/ 5 | 6 | 7 | @import url(http://fonts.googleapis.com/css?family=Source%20Sans%20Pro:300,400,500,600,700); 8 | /* ------------------------------------------------------------------------------ 9 | Global Styles 10 | -------------------------------------------------------------------------------*/ 11 | 12 | 13 | body { 14 | font-family: "Source Sans Pro"; 15 | font-weight: 300; 16 | font-size:13px; 17 | letter-spacing:1px; 18 | line-height:1.8em; 19 | } 20 | p { 21 | color: #888; 22 | } 23 | h1, h2, h3, h5, h6 { 24 | font-weight: 400; 25 | } 26 | h4 { 27 | font-weight: 300; 28 | font-size: 15px; 29 | 30 | } 31 | a { 32 | color: #ff5335; 33 | } 34 | a:hover { 35 | color: #d92709; 36 | text-decoration: none; 37 | } 38 | a:focus { 39 | outline: none; 40 | } 41 | img { 42 | max-width: 100%; 43 | height: auto; 44 | } 45 | ul, li { 46 | list-style: none; 47 | padding: 0; 48 | margin: 0; 49 | } 50 | /* ------------------------------------------------------------------------------ 51 | main 52 | -------------------------------------------------------------------------------*/ 53 | 54 | .main { 55 | background-size: cover; 56 | padding: 3em 0 4.5em; 57 | position: relative !important; 58 | color: #fff; 59 | } 60 | .overlay { 61 | position: absolute; 62 | width: 100%; 63 | height: 100%; 64 | background: rgba(27, 32, 38, 0.5); 65 | top: 0; 66 | } 67 | .logo { 68 | text-align: center; 69 | margin-bottom: 2.25em; 70 | } 71 | .main h1 { 72 | font-size: 36px; 73 | font-weight: 700; 74 | letter-spacing:7px; 75 | text-transform:uppercase; 76 | } 77 | .welcome-message { 78 | padding-top: 1.5em; 79 | } 80 | .rotate { 81 | text-shadow: none !important; 82 | } 83 | .copyrights{ 84 | text-indent:-9999px; 85 | height:0; 86 | line-height:0; 87 | font-size:0; 88 | overflow:hidden; 89 | } 90 | 91 | /* ------------------------------------------------------------------------------ 92 | Search 93 | -------------------------------------------------------------------------------*/ 94 | 95 | .form-control::-webkit-input-placeholder { 96 | color: #727272; 97 | font-weight: 300; 98 | } 99 | .form-control:-moz-placeholder { 100 | color: #727272; 101 | font-weight: 300; 102 | } 103 | .form-control::-moz-placeholder { 104 | color: #727272; 105 | font-weight: 300; 106 | } 107 | .form-control:-ms-input-placeholder { 108 | color: #727272; 109 | font-weight: 300; 110 | } 111 | .sub-form { 112 | padding-top: 1.5em; 113 | } 114 | .center-block { 115 | float: none; 116 | } 117 | .form-control { 118 | background-color: #fff; 119 | border: 1px solid #fff; 120 | box-shadow: none; 121 | height: 50px; 122 | font-weight: 300; 123 | } 124 | .form-control:focus { 125 | border-color: #fff; 126 | outline: 0; 127 | box-shadow: none; 128 | } 129 | .input-group-btn>.btn:hover, .input-group-btn>.btn:focus, .input-group-btn>.btn:active { 130 | z-index: 1; 131 | } 132 | .btn-default { 133 | color: #fff; 134 | background-color: #ff5335; 135 | border-color: #ff5335; 136 | -webkit-transition: all .5s ease-out; 137 | transition: all .5s ease-out; 138 | padding: 12px 1.5em; 139 | height: 50px; 140 | width: 100px; 141 | font-size: 18px; 142 | } 143 | .btn-default:hover, .btn-default:focus, .btn-default:active { 144 | color: #fff; 145 | background-color: #d92709; 146 | border-color: #d92709; 147 | } 148 | .btn:active:focus, .btn-default:focus, .btn:focus, .btn-default:active { 149 | outline: none; 150 | } 151 | #mc-notification { 152 | margin: .75em 0 0; 153 | color: #FFF; 154 | } 155 | 156 | .center-block { 157 | float: none; 158 | } 159 | 160 | /* ------------------------------------------------------------------------------ 161 | sidebar 162 | -------------------------------------------------------------------------------*/ 163 | .sidebar .nav > li.active-pro{ 164 | position: absolute; 165 | width: 100%; 166 | bottom: 10px; 167 | } 168 | .sidebar .nav > li.active-pro a{ 169 | background: rgba(255, 255, 255, 0.14); 170 | opacity: 1; 171 | color: #FFFFFF; 172 | } 173 | 174 | .p-detail{ 175 | float: left; 176 | } 177 | 178 | /* ------------------------------------------------------------------------------ 179 | Small devices (tablets, 768px and up) 180 | -------------------------------------------------------------------------------*/ 181 | 182 | @media (min-width: 768px) { 183 | .main { 184 | padding: 3em 0 7.5em; 185 | } 186 | .logo { 187 | text-align: justify; 188 | margin-bottom: 0; 189 | } 190 | .welcome-message { 191 | padding-top: 6em; 192 | } 193 | .social { 194 | float: right; 195 | } 196 | ul.countdown { 197 | padding-top: 6em; 198 | } 199 | ul.countdown li span { 200 | font-size: 27px; 201 | } 202 | ul.countdown li.seperator { 203 | font-size: 27px; 204 | padding: 0 20px; 205 | } 206 | } 207 | 208 | /* ------------------------------------------------------------------------------ 209 | Medium devices (desktops, 992px and up) 210 | -------------------------------------------------------------------------------*/ 211 | 212 | @media (min-width: 992px) { 213 | .main { 214 | padding: 3em 0 16em; 215 | position: fixed !important; 216 | top: 0; 217 | z-index: 1; 218 | width: 100%; 219 | } 220 | .features, .twitter-feed, .contact, .site-footer { 221 | position: relative; 222 | z-index: 2; 223 | } 224 | .welcome-message { 225 | padding-top: 9em; 226 | } 227 | .section-spacing { 228 | padding: 6em 0; 229 | } 230 | .features.section-spacing { 231 | padding: 6em 0 3em; 232 | margin-top: 46.8em; 233 | } 234 | .product-features { 235 | margin-bottom: 3.75em; 236 | } 237 | .features.section-spacing, .contact { 238 | background: #fff; 239 | } 240 | .contact-form { 241 | margin-top: 0; 242 | } 243 | .animated { 244 | -webkit-animation-duration: 1s; 245 | animation-duration: 1s; 246 | -webkit-animation-fill-mode: both; 247 | animation-fill-mode: both; 248 | } 249 | @-webkit-keyframes fadeInUp { 250 | 0% { 251 | opacity:0; 252 | -webkit-transform:translate3d(0, 100%, 0); 253 | transform:translate3d(0, 100%, 0); 254 | } 255 | 100% { 256 | opacity:1; 257 | -webkit-transform:none; 258 | transform:none; 259 | } 260 | } 261 | @keyframes fadeInUp { 262 | 0% { 263 | opacity:0; 264 | -webkit-transform:translate3d(0, 100%, 0); 265 | -ms-transform:translate3d(0, 100%, 0); 266 | transform:translate3d(0, 100%, 0); 267 | } 268 | 100% { 269 | opacity:1; 270 | -webkit-transform:none; 271 | -ms-transform:none; 272 | transform:none; 273 | } 274 | } 275 | .fadeInUp { 276 | -webkit-animation-name: fadeInUp; 277 | animation-name: fadeInUp; 278 | } 279 | } 280 | 281 | /* ------------------------------------------------------------------------------ 282 | small devices 283 | -------------------------------------------------------------------------------*/ 284 | 285 | 286 | 287 | @media (max-width: 320px) { 288 | ul.countdown li span { 289 | font-size: 21px; 290 | } 291 | ul.countdown li.seperator { 292 | font-size: 25.629px; 293 | vertical-align: top; 294 | padding: 0 2px; 295 | } 296 | ul.countdown li p { 297 | font-size: 12.642px; 298 | } 299 | } 300 | 301 | .bk-plot-layout,.bk-plot-wrapper,.bk-canvas-wrapper,.bk-canvas{ 302 | width: 100% !important; 303 | } 304 | .bk-root .bk-toolbar-right{ 305 | display: none !important; 306 | } 307 | 308 | -------------------------------------------------------------------------------- /static/fonts/FontAwesome.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/static/fonts/FontAwesome.otf -------------------------------------------------------------------------------- /static/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/static/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /static/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/static/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /static/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/static/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /static/gif/loading.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/static/gif/loading.gif -------------------------------------------------------------------------------- /static/gif/loading1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/static/gif/loading1.gif -------------------------------------------------------------------------------- /static/img/bg-1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/static/img/bg-1.jpg -------------------------------------------------------------------------------- /static/img/bg-2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/static/img/bg-2.jpg -------------------------------------------------------------------------------- /static/img/load.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/static/img/load.gif -------------------------------------------------------------------------------- /static/img/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/static/img/logo.png -------------------------------------------------------------------------------- /static/img/sidebar.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/static/img/sidebar.jpg -------------------------------------------------------------------------------- /static/js/bootstrap.min.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * Bootstrap v3.3.5 (http://getbootstrap.com) 3 | * Copyright 2011-2015 Twitter, Inc. 4 | * Licensed under the MIT license 5 | */ 6 | if("undefined"==typeof jQuery)throw new Error("Bootstrap's JavaScript requires jQuery");+function(a){"use strict";var b=a.fn.jquery.split(" ")[0].split(".");if(b[0]<2&&b[1]<9||1==b[0]&&9==b[1]&&b[2]<1)throw new Error("Bootstrap's JavaScript requires jQuery version 1.9.1 or higher")}(jQuery),+function(a){"use strict";function b(){var a=document.createElement("bootstrap"),b={WebkitTransition:"webkitTransitionEnd",MozTransition:"transitionend",OTransition:"oTransitionEnd otransitionend",transition:"transitionend"};for(var c in b)if(void 0!==a.style[c])return{end:b[c]};return!1}a.fn.emulateTransitionEnd=function(b){var c=!1,d=this;a(this).one("bsTransitionEnd",function(){c=!0});var e=function(){c||a(d).trigger(a.support.transition.end)};return setTimeout(e,b),this},a(function(){a.support.transition=b(),a.support.transition&&(a.event.special.bsTransitionEnd={bindType:a.support.transition.end,delegateType:a.support.transition.end,handle:function(b){return a(b.target).is(this)?b.handleObj.handler.apply(this,arguments):void 0}})})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var c=a(this),e=c.data("bs.alert");e||c.data("bs.alert",e=new d(this)),"string"==typeof b&&e[b].call(c)})}var c='[data-dismiss="alert"]',d=function(b){a(b).on("click",c,this.close)};d.VERSION="3.3.5",d.TRANSITION_DURATION=150,d.prototype.close=function(b){function c(){g.detach().trigger("closed.bs.alert").remove()}var e=a(this),f=e.attr("data-target");f||(f=e.attr("href"),f=f&&f.replace(/.*(?=#[^\s]*$)/,""));var g=a(f);b&&b.preventDefault(),g.length||(g=e.closest(".alert")),g.trigger(b=a.Event("close.bs.alert")),b.isDefaultPrevented()||(g.removeClass("in"),a.support.transition&&g.hasClass("fade")?g.one("bsTransitionEnd",c).emulateTransitionEnd(d.TRANSITION_DURATION):c())};var e=a.fn.alert;a.fn.alert=b,a.fn.alert.Constructor=d,a.fn.alert.noConflict=function(){return a.fn.alert=e,this},a(document).on("click.bs.alert.data-api",c,d.prototype.close)}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.button"),f="object"==typeof b&&b;e||d.data("bs.button",e=new c(this,f)),"toggle"==b?e.toggle():b&&e.setState(b)})}var c=function(b,d){this.$element=a(b),this.options=a.extend({},c.DEFAULTS,d),this.isLoading=!1};c.VERSION="3.3.5",c.DEFAULTS={loadingText:"loading..."},c.prototype.setState=function(b){var c="disabled",d=this.$element,e=d.is("input")?"val":"html",f=d.data();b+="Text",null==f.resetText&&d.data("resetText",d[e]()),setTimeout(a.proxy(function(){d[e](null==f[b]?this.options[b]:f[b]),"loadingText"==b?(this.isLoading=!0,d.addClass(c).attr(c,c)):this.isLoading&&(this.isLoading=!1,d.removeClass(c).removeAttr(c))},this),0)},c.prototype.toggle=function(){var a=!0,b=this.$element.closest('[data-toggle="buttons"]');if(b.length){var c=this.$element.find("input");"radio"==c.prop("type")?(c.prop("checked")&&(a=!1),b.find(".active").removeClass("active"),this.$element.addClass("active")):"checkbox"==c.prop("type")&&(c.prop("checked")!==this.$element.hasClass("active")&&(a=!1),this.$element.toggleClass("active")),c.prop("checked",this.$element.hasClass("active")),a&&c.trigger("change")}else this.$element.attr("aria-pressed",!this.$element.hasClass("active")),this.$element.toggleClass("active")};var d=a.fn.button;a.fn.button=b,a.fn.button.Constructor=c,a.fn.button.noConflict=function(){return a.fn.button=d,this},a(document).on("click.bs.button.data-api",'[data-toggle^="button"]',function(c){var d=a(c.target);d.hasClass("btn")||(d=d.closest(".btn")),b.call(d,"toggle"),a(c.target).is('input[type="radio"]')||a(c.target).is('input[type="checkbox"]')||c.preventDefault()}).on("focus.bs.button.data-api blur.bs.button.data-api",'[data-toggle^="button"]',function(b){a(b.target).closest(".btn").toggleClass("focus",/^focus(in)?$/.test(b.type))})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.carousel"),f=a.extend({},c.DEFAULTS,d.data(),"object"==typeof b&&b),g="string"==typeof b?b:f.slide;e||d.data("bs.carousel",e=new c(this,f)),"number"==typeof b?e.to(b):g?e[g]():f.interval&&e.pause().cycle()})}var c=function(b,c){this.$element=a(b),this.$indicators=this.$element.find(".carousel-indicators"),this.options=c,this.paused=null,this.sliding=null,this.interval=null,this.$active=null,this.$items=null,this.options.keyboard&&this.$element.on("keydown.bs.carousel",a.proxy(this.keydown,this)),"hover"==this.options.pause&&!("ontouchstart"in document.documentElement)&&this.$element.on("mouseenter.bs.carousel",a.proxy(this.pause,this)).on("mouseleave.bs.carousel",a.proxy(this.cycle,this))};c.VERSION="3.3.5",c.TRANSITION_DURATION=600,c.DEFAULTS={interval:5e3,pause:"hover",wrap:!0,keyboard:!0},c.prototype.keydown=function(a){if(!/input|textarea/i.test(a.target.tagName)){switch(a.which){case 37:this.prev();break;case 39:this.next();break;default:return}a.preventDefault()}},c.prototype.cycle=function(b){return b||(this.paused=!1),this.interval&&clearInterval(this.interval),this.options.interval&&!this.paused&&(this.interval=setInterval(a.proxy(this.next,this),this.options.interval)),this},c.prototype.getItemIndex=function(a){return this.$items=a.parent().children(".item"),this.$items.index(a||this.$active)},c.prototype.getItemForDirection=function(a,b){var c=this.getItemIndex(b),d="prev"==a&&0===c||"next"==a&&c==this.$items.length-1;if(d&&!this.options.wrap)return b;var e="prev"==a?-1:1,f=(c+e)%this.$items.length;return this.$items.eq(f)},c.prototype.to=function(a){var b=this,c=this.getItemIndex(this.$active=this.$element.find(".item.active"));return a>this.$items.length-1||0>a?void 0:this.sliding?this.$element.one("slid.bs.carousel",function(){b.to(a)}):c==a?this.pause().cycle():this.slide(a>c?"next":"prev",this.$items.eq(a))},c.prototype.pause=function(b){return b||(this.paused=!0),this.$element.find(".next, .prev").length&&a.support.transition&&(this.$element.trigger(a.support.transition.end),this.cycle(!0)),this.interval=clearInterval(this.interval),this},c.prototype.next=function(){return this.sliding?void 0:this.slide("next")},c.prototype.prev=function(){return this.sliding?void 0:this.slide("prev")},c.prototype.slide=function(b,d){var e=this.$element.find(".item.active"),f=d||this.getItemForDirection(b,e),g=this.interval,h="next"==b?"left":"right",i=this;if(f.hasClass("active"))return this.sliding=!1;var j=f[0],k=a.Event("slide.bs.carousel",{relatedTarget:j,direction:h});if(this.$element.trigger(k),!k.isDefaultPrevented()){if(this.sliding=!0,g&&this.pause(),this.$indicators.length){this.$indicators.find(".active").removeClass("active");var l=a(this.$indicators.children()[this.getItemIndex(f)]);l&&l.addClass("active")}var m=a.Event("slid.bs.carousel",{relatedTarget:j,direction:h});return a.support.transition&&this.$element.hasClass("slide")?(f.addClass(b),f[0].offsetWidth,e.addClass(h),f.addClass(h),e.one("bsTransitionEnd",function(){f.removeClass([b,h].join(" ")).addClass("active"),e.removeClass(["active",h].join(" ")),i.sliding=!1,setTimeout(function(){i.$element.trigger(m)},0)}).emulateTransitionEnd(c.TRANSITION_DURATION)):(e.removeClass("active"),f.addClass("active"),this.sliding=!1,this.$element.trigger(m)),g&&this.cycle(),this}};var d=a.fn.carousel;a.fn.carousel=b,a.fn.carousel.Constructor=c,a.fn.carousel.noConflict=function(){return a.fn.carousel=d,this};var e=function(c){var d,e=a(this),f=a(e.attr("data-target")||(d=e.attr("href"))&&d.replace(/.*(?=#[^\s]+$)/,""));if(f.hasClass("carousel")){var g=a.extend({},f.data(),e.data()),h=e.attr("data-slide-to");h&&(g.interval=!1),b.call(f,g),h&&f.data("bs.carousel").to(h),c.preventDefault()}};a(document).on("click.bs.carousel.data-api","[data-slide]",e).on("click.bs.carousel.data-api","[data-slide-to]",e),a(window).on("load",function(){a('[data-ride="carousel"]').each(function(){var c=a(this);b.call(c,c.data())})})}(jQuery),+function(a){"use strict";function b(b){var c,d=b.attr("data-target")||(c=b.attr("href"))&&c.replace(/.*(?=#[^\s]+$)/,"");return a(d)}function c(b){return this.each(function(){var c=a(this),e=c.data("bs.collapse"),f=a.extend({},d.DEFAULTS,c.data(),"object"==typeof b&&b);!e&&f.toggle&&/show|hide/.test(b)&&(f.toggle=!1),e||c.data("bs.collapse",e=new d(this,f)),"string"==typeof b&&e[b]()})}var d=function(b,c){this.$element=a(b),this.options=a.extend({},d.DEFAULTS,c),this.$trigger=a('[data-toggle="collapse"][href="#'+b.id+'"],[data-toggle="collapse"][data-target="#'+b.id+'"]'),this.transitioning=null,this.options.parent?this.$parent=this.getParent():this.addAriaAndCollapsedClass(this.$element,this.$trigger),this.options.toggle&&this.toggle()};d.VERSION="3.3.5",d.TRANSITION_DURATION=350,d.DEFAULTS={toggle:!0},d.prototype.dimension=function(){var a=this.$element.hasClass("width");return a?"width":"height"},d.prototype.show=function(){if(!this.transitioning&&!this.$element.hasClass("in")){var b,e=this.$parent&&this.$parent.children(".panel").children(".in, .collapsing");if(!(e&&e.length&&(b=e.data("bs.collapse"),b&&b.transitioning))){var f=a.Event("show.bs.collapse");if(this.$element.trigger(f),!f.isDefaultPrevented()){e&&e.length&&(c.call(e,"hide"),b||e.data("bs.collapse",null));var g=this.dimension();this.$element.removeClass("collapse").addClass("collapsing")[g](0).attr("aria-expanded",!0),this.$trigger.removeClass("collapsed").attr("aria-expanded",!0),this.transitioning=1;var h=function(){this.$element.removeClass("collapsing").addClass("collapse in")[g](""),this.transitioning=0,this.$element.trigger("shown.bs.collapse")};if(!a.support.transition)return h.call(this);var i=a.camelCase(["scroll",g].join("-"));this.$element.one("bsTransitionEnd",a.proxy(h,this)).emulateTransitionEnd(d.TRANSITION_DURATION)[g](this.$element[0][i])}}}},d.prototype.hide=function(){if(!this.transitioning&&this.$element.hasClass("in")){var b=a.Event("hide.bs.collapse");if(this.$element.trigger(b),!b.isDefaultPrevented()){var c=this.dimension();this.$element[c](this.$element[c]())[0].offsetHeight,this.$element.addClass("collapsing").removeClass("collapse in").attr("aria-expanded",!1),this.$trigger.addClass("collapsed").attr("aria-expanded",!1),this.transitioning=1;var e=function(){this.transitioning=0,this.$element.removeClass("collapsing").addClass("collapse").trigger("hidden.bs.collapse")};return a.support.transition?void this.$element[c](0).one("bsTransitionEnd",a.proxy(e,this)).emulateTransitionEnd(d.TRANSITION_DURATION):e.call(this)}}},d.prototype.toggle=function(){this[this.$element.hasClass("in")?"hide":"show"]()},d.prototype.getParent=function(){return a(this.options.parent).find('[data-toggle="collapse"][data-parent="'+this.options.parent+'"]').each(a.proxy(function(c,d){var e=a(d);this.addAriaAndCollapsedClass(b(e),e)},this)).end()},d.prototype.addAriaAndCollapsedClass=function(a,b){var c=a.hasClass("in");a.attr("aria-expanded",c),b.toggleClass("collapsed",!c).attr("aria-expanded",c)};var e=a.fn.collapse;a.fn.collapse=c,a.fn.collapse.Constructor=d,a.fn.collapse.noConflict=function(){return a.fn.collapse=e,this},a(document).on("click.bs.collapse.data-api",'[data-toggle="collapse"]',function(d){var e=a(this);e.attr("data-target")||d.preventDefault();var f=b(e),g=f.data("bs.collapse"),h=g?"toggle":e.data();c.call(f,h)})}(jQuery),+function(a){"use strict";function b(b){var c=b.attr("data-target");c||(c=b.attr("href"),c=c&&/#[A-Za-z]/.test(c)&&c.replace(/.*(?=#[^\s]*$)/,""));var d=c&&a(c);return d&&d.length?d:b.parent()}function c(c){c&&3===c.which||(a(e).remove(),a(f).each(function(){var d=a(this),e=b(d),f={relatedTarget:this};e.hasClass("open")&&(c&&"click"==c.type&&/input|textarea/i.test(c.target.tagName)&&a.contains(e[0],c.target)||(e.trigger(c=a.Event("hide.bs.dropdown",f)),c.isDefaultPrevented()||(d.attr("aria-expanded","false"),e.removeClass("open").trigger("hidden.bs.dropdown",f))))}))}function d(b){return this.each(function(){var c=a(this),d=c.data("bs.dropdown");d||c.data("bs.dropdown",d=new g(this)),"string"==typeof b&&d[b].call(c)})}var e=".dropdown-backdrop",f='[data-toggle="dropdown"]',g=function(b){a(b).on("click.bs.dropdown",this.toggle)};g.VERSION="3.3.5",g.prototype.toggle=function(d){var e=a(this);if(!e.is(".disabled, :disabled")){var f=b(e),g=f.hasClass("open");if(c(),!g){"ontouchstart"in document.documentElement&&!f.closest(".navbar-nav").length&&a(document.createElement("div")).addClass("dropdown-backdrop").insertAfter(a(this)).on("click",c);var h={relatedTarget:this};if(f.trigger(d=a.Event("show.bs.dropdown",h)),d.isDefaultPrevented())return;e.trigger("focus").attr("aria-expanded","true"),f.toggleClass("open").trigger("shown.bs.dropdown",h)}return!1}},g.prototype.keydown=function(c){if(/(38|40|27|32)/.test(c.which)&&!/input|textarea/i.test(c.target.tagName)){var d=a(this);if(c.preventDefault(),c.stopPropagation(),!d.is(".disabled, :disabled")){var e=b(d),g=e.hasClass("open");if(!g&&27!=c.which||g&&27==c.which)return 27==c.which&&e.find(f).trigger("focus"),d.trigger("click");var h=" li:not(.disabled):visible a",i=e.find(".dropdown-menu"+h);if(i.length){var j=i.index(c.target);38==c.which&&j>0&&j--,40==c.which&&jdocument.documentElement.clientHeight;this.$element.css({paddingLeft:!this.bodyIsOverflowing&&a?this.scrollbarWidth:"",paddingRight:this.bodyIsOverflowing&&!a?this.scrollbarWidth:""})},c.prototype.resetAdjustments=function(){this.$element.css({paddingLeft:"",paddingRight:""})},c.prototype.checkScrollbar=function(){var a=window.innerWidth;if(!a){var b=document.documentElement.getBoundingClientRect();a=b.right-Math.abs(b.left)}this.bodyIsOverflowing=document.body.clientWidth
    ',trigger:"hover focus",title:"",delay:0,html:!1,container:!1,viewport:{selector:"body",padding:0}},c.prototype.init=function(b,c,d){if(this.enabled=!0,this.type=b,this.$element=a(c),this.options=this.getOptions(d),this.$viewport=this.options.viewport&&a(a.isFunction(this.options.viewport)?this.options.viewport.call(this,this.$element):this.options.viewport.selector||this.options.viewport),this.inState={click:!1,hover:!1,focus:!1},this.$element[0]instanceof document.constructor&&!this.options.selector)throw new Error("`selector` option must be specified when initializing "+this.type+" on the window.document object!");for(var e=this.options.trigger.split(" "),f=e.length;f--;){var g=e[f];if("click"==g)this.$element.on("click."+this.type,this.options.selector,a.proxy(this.toggle,this));else if("manual"!=g){var h="hover"==g?"mouseenter":"focusin",i="hover"==g?"mouseleave":"focusout";this.$element.on(h+"."+this.type,this.options.selector,a.proxy(this.enter,this)),this.$element.on(i+"."+this.type,this.options.selector,a.proxy(this.leave,this))}}this.options.selector?this._options=a.extend({},this.options,{trigger:"manual",selector:""}):this.fixTitle()},c.prototype.getDefaults=function(){return c.DEFAULTS},c.prototype.getOptions=function(b){return b=a.extend({},this.getDefaults(),this.$element.data(),b),b.delay&&"number"==typeof b.delay&&(b.delay={show:b.delay,hide:b.delay}),b},c.prototype.getDelegateOptions=function(){var b={},c=this.getDefaults();return this._options&&a.each(this._options,function(a,d){c[a]!=d&&(b[a]=d)}),b},c.prototype.enter=function(b){var c=b instanceof this.constructor?b:a(b.currentTarget).data("bs."+this.type);return c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c)),b instanceof a.Event&&(c.inState["focusin"==b.type?"focus":"hover"]=!0),c.tip().hasClass("in")||"in"==c.hoverState?void(c.hoverState="in"):(clearTimeout(c.timeout),c.hoverState="in",c.options.delay&&c.options.delay.show?void(c.timeout=setTimeout(function(){"in"==c.hoverState&&c.show()},c.options.delay.show)):c.show())},c.prototype.isInStateTrue=function(){for(var a in this.inState)if(this.inState[a])return!0;return!1},c.prototype.leave=function(b){var c=b instanceof this.constructor?b:a(b.currentTarget).data("bs."+this.type);return c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c)),b instanceof a.Event&&(c.inState["focusout"==b.type?"focus":"hover"]=!1),c.isInStateTrue()?void 0:(clearTimeout(c.timeout),c.hoverState="out",c.options.delay&&c.options.delay.hide?void(c.timeout=setTimeout(function(){"out"==c.hoverState&&c.hide()},c.options.delay.hide)):c.hide())},c.prototype.show=function(){var b=a.Event("show.bs."+this.type);if(this.hasContent()&&this.enabled){this.$element.trigger(b);var d=a.contains(this.$element[0].ownerDocument.documentElement,this.$element[0]);if(b.isDefaultPrevented()||!d)return;var e=this,f=this.tip(),g=this.getUID(this.type);this.setContent(),f.attr("id",g),this.$element.attr("aria-describedby",g),this.options.animation&&f.addClass("fade");var h="function"==typeof this.options.placement?this.options.placement.call(this,f[0],this.$element[0]):this.options.placement,i=/\s?auto?\s?/i,j=i.test(h);j&&(h=h.replace(i,"")||"top"),f.detach().css({top:0,left:0,display:"block"}).addClass(h).data("bs."+this.type,this),this.options.container?f.appendTo(this.options.container):f.insertAfter(this.$element),this.$element.trigger("inserted.bs."+this.type);var k=this.getPosition(),l=f[0].offsetWidth,m=f[0].offsetHeight;if(j){var n=h,o=this.getPosition(this.$viewport);h="bottom"==h&&k.bottom+m>o.bottom?"top":"top"==h&&k.top-mo.width?"left":"left"==h&&k.left-lg.top+g.height&&(e.top=g.top+g.height-i)}else{var j=b.left-f,k=b.left+f+c;jg.right&&(e.left=g.left+g.width-k)}return e},c.prototype.getTitle=function(){var a,b=this.$element,c=this.options;return a=b.attr("data-original-title")||("function"==typeof c.title?c.title.call(b[0]):c.title)},c.prototype.getUID=function(a){do a+=~~(1e6*Math.random());while(document.getElementById(a));return a},c.prototype.tip=function(){if(!this.$tip&&(this.$tip=a(this.options.template),1!=this.$tip.length))throw new Error(this.type+" `template` option must consist of exactly 1 top-level element!");return this.$tip},c.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".tooltip-arrow")},c.prototype.enable=function(){this.enabled=!0},c.prototype.disable=function(){this.enabled=!1},c.prototype.toggleEnabled=function(){this.enabled=!this.enabled},c.prototype.toggle=function(b){var c=this;b&&(c=a(b.currentTarget).data("bs."+this.type),c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c))),b?(c.inState.click=!c.inState.click,c.isInStateTrue()?c.enter(c):c.leave(c)):c.tip().hasClass("in")?c.leave(c):c.enter(c)},c.prototype.destroy=function(){var a=this;clearTimeout(this.timeout),this.hide(function(){a.$element.off("."+a.type).removeData("bs."+a.type),a.$tip&&a.$tip.detach(),a.$tip=null,a.$arrow=null,a.$viewport=null})};var d=a.fn.tooltip;a.fn.tooltip=b,a.fn.tooltip.Constructor=c,a.fn.tooltip.noConflict=function(){return a.fn.tooltip=d,this}}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.popover"),f="object"==typeof b&&b;(e||!/destroy|hide/.test(b))&&(e||d.data("bs.popover",e=new c(this,f)),"string"==typeof b&&e[b]())})}var c=function(a,b){this.init("popover",a,b)};if(!a.fn.tooltip)throw new Error("Popover requires tooltip.js");c.VERSION="3.3.5",c.DEFAULTS=a.extend({},a.fn.tooltip.Constructor.DEFAULTS,{placement:"right",trigger:"click",content:"",template:''}),c.prototype=a.extend({},a.fn.tooltip.Constructor.prototype),c.prototype.constructor=c,c.prototype.getDefaults=function(){return c.DEFAULTS},c.prototype.setContent=function(){var a=this.tip(),b=this.getTitle(),c=this.getContent();a.find(".popover-title")[this.options.html?"html":"text"](b),a.find(".popover-content").children().detach().end()[this.options.html?"string"==typeof c?"html":"append":"text"](c),a.removeClass("fade top bottom left right in"),a.find(".popover-title").html()||a.find(".popover-title").hide()},c.prototype.hasContent=function(){return this.getTitle()||this.getContent()},c.prototype.getContent=function(){var a=this.$element,b=this.options;return a.attr("data-content")||("function"==typeof b.content?b.content.call(a[0]):b.content)},c.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".arrow")};var d=a.fn.popover;a.fn.popover=b,a.fn.popover.Constructor=c,a.fn.popover.noConflict=function(){return a.fn.popover=d,this}}(jQuery),+function(a){"use strict";function b(c,d){this.$body=a(document.body),this.$scrollElement=a(a(c).is(document.body)?window:c),this.options=a.extend({},b.DEFAULTS,d),this.selector=(this.options.target||"")+" .nav li > a",this.offsets=[],this.targets=[],this.activeTarget=null,this.scrollHeight=0,this.$scrollElement.on("scroll.bs.scrollspy",a.proxy(this.process,this)),this.refresh(),this.process()}function c(c){return this.each(function(){var d=a(this),e=d.data("bs.scrollspy"),f="object"==typeof c&&c;e||d.data("bs.scrollspy",e=new b(this,f)),"string"==typeof c&&e[c]()})}b.VERSION="3.3.5",b.DEFAULTS={offset:10},b.prototype.getScrollHeight=function(){return this.$scrollElement[0].scrollHeight||Math.max(this.$body[0].scrollHeight,document.documentElement.scrollHeight)},b.prototype.refresh=function(){var b=this,c="offset",d=0;this.offsets=[],this.targets=[],this.scrollHeight=this.getScrollHeight(),a.isWindow(this.$scrollElement[0])||(c="position",d=this.$scrollElement.scrollTop()),this.$body.find(this.selector).map(function(){var b=a(this),e=b.data("target")||b.attr("href"),f=/^#./.test(e)&&a(e);return f&&f.length&&f.is(":visible")&&[[f[c]().top+d,e]]||null}).sort(function(a,b){return a[0]-b[0]}).each(function(){b.offsets.push(this[0]),b.targets.push(this[1])})},b.prototype.process=function(){var a,b=this.$scrollElement.scrollTop()+this.options.offset,c=this.getScrollHeight(),d=this.options.offset+c-this.$scrollElement.height(),e=this.offsets,f=this.targets,g=this.activeTarget;if(this.scrollHeight!=c&&this.refresh(),b>=d)return g!=(a=f[f.length-1])&&this.activate(a);if(g&&b=e[a]&&(void 0===e[a+1]||b .dropdown-menu > .active").removeClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!1),b.addClass("active").find('[data-toggle="tab"]').attr("aria-expanded",!0),h?(b[0].offsetWidth,b.addClass("in")):b.removeClass("fade"),b.parent(".dropdown-menu").length&&b.closest("li.dropdown").addClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!0),e&&e()}var g=d.find("> .active"),h=e&&a.support.transition&&(g.length&&g.hasClass("fade")||!!d.find("> .fade").length);g.length&&h?g.one("bsTransitionEnd",f).emulateTransitionEnd(c.TRANSITION_DURATION):f(),g.removeClass("in")};var d=a.fn.tab;a.fn.tab=b,a.fn.tab.Constructor=c,a.fn.tab.noConflict=function(){return a.fn.tab=d,this};var e=function(c){c.preventDefault(),b.call(a(this),"show")};a(document).on("click.bs.tab.data-api",'[data-toggle="tab"]',e).on("click.bs.tab.data-api",'[data-toggle="pill"]',e)}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.affix"),f="object"==typeof b&&b;e||d.data("bs.affix",e=new c(this,f)),"string"==typeof b&&e[b]()})}var c=function(b,d){this.options=a.extend({},c.DEFAULTS,d),this.$target=a(this.options.target).on("scroll.bs.affix.data-api",a.proxy(this.checkPosition,this)).on("click.bs.affix.data-api",a.proxy(this.checkPositionWithEventLoop,this)),this.$element=a(b),this.affixed=null,this.unpin=null,this.pinnedOffset=null,this.checkPosition()};c.VERSION="3.3.5",c.RESET="affix affix-top affix-bottom",c.DEFAULTS={offset:0,target:window},c.prototype.getState=function(a,b,c,d){var e=this.$target.scrollTop(),f=this.$element.offset(),g=this.$target.height();if(null!=c&&"top"==this.affixed)return c>e?"top":!1;if("bottom"==this.affixed)return null!=c?e+this.unpin<=f.top?!1:"bottom":a-d>=e+g?!1:"bottom";var h=null==this.affixed,i=h?e:f.top,j=h?g:b;return null!=c&&c>=e?"top":null!=d&&i+j>=a-d?"bottom":!1},c.prototype.getPinnedOffset=function(){if(this.pinnedOffset)return this.pinnedOffset;this.$element.removeClass(c.RESET).addClass("affix");var a=this.$target.scrollTop(),b=this.$element.offset();return this.pinnedOffset=b.top-a},c.prototype.checkPositionWithEventLoop=function(){setTimeout(a.proxy(this.checkPosition,this),1)},c.prototype.checkPosition=function(){if(this.$element.is(":visible")){var b=this.$element.height(),d=this.options.offset,e=d.top,f=d.bottom,g=Math.max(a(document).height(),a(document.body).height());"object"!=typeof d&&(f=e=d),"function"==typeof e&&(e=d.top(this.$element)),"function"==typeof f&&(f=d.bottom(this.$element));var h=this.getState(g,b,e,f);if(this.affixed!=h){null!=this.unpin&&this.$element.css("top","");var i="affix"+(h?"-"+h:""),j=a.Event(i+".bs.affix");if(this.$element.trigger(j),j.isDefaultPrevented())return;this.affixed=h,this.unpin="bottom"==h?this.getPinnedOffset():null,this.$element.removeClass(c.RESET).addClass(i).trigger(i.replace("affix","affixed")+".bs.affix")}"bottom"==h&&this.$element.offset({top:g-b-f})}};var d=a.fn.affix;a.fn.affix=b,a.fn.affix.Constructor=c,a.fn.affix.noConflict=function(){return a.fn.affix=d,this},a(window).on("load",function(){a('[data-spy="affix"]').each(function(){var c=a(this),d=c.data();d.offset=d.offset||{},null!=d.offsetBottom&&(d.offset.bottom=d.offsetBottom),null!=d.offsetTop&&(d.offset.top=d.offsetTop),b.call(c,d)})})}(jQuery); -------------------------------------------------------------------------------- /static/js/jquery.backstretch.min.js: -------------------------------------------------------------------------------- 1 | /*! Backstretch - v2.0.4 - 2013-06-19 2 | * http://srobbin.com/jquery-plugins/backstretch/ 3 | * Copyright (c) 2013 Scott Robbin; Licensed MIT */ 4 | (function(a,d,p){a.fn.backstretch=function(c,b){(c===p||0===c.length)&&a.error("No images were supplied for Backstretch");0===a(d).scrollTop()&&d.scrollTo(0,0);return this.each(function(){var d=a(this),g=d.data("backstretch");if(g){if("string"==typeof c&&"function"==typeof g[c]){g[c](b);return}b=a.extend(g.options,b);g.destroy(!0)}g=new q(this,c,b);d.data("backstretch",g)})};a.backstretch=function(c,b){return a("body").backstretch(c,b).data("backstretch")};a.expr[":"].backstretch=function(c){return a(c).data("backstretch")!==p};a.fn.backstretch.defaults={centeredX:!0,centeredY:!0,duration:5E3,fade:0};var r={left:0,top:0,overflow:"hidden",margin:0,padding:0,height:"100%",width:"100%",zIndex:-999999},s={position:"absolute",display:"none",margin:0,padding:0,border:"none",width:"auto",height:"auto",maxHeight:"none",maxWidth:"none",zIndex:-999999},q=function(c,b,e){this.options=a.extend({},a.fn.backstretch.defaults,e||{});this.images=a.isArray(b)?b:[b];a.each(this.images,function(){a("")[0].src=this});this.isBody=c===document.body;this.$container=a(c);this.$root=this.isBody?l?a(d):a(document):this.$container;c=this.$container.children(".backstretch").first();this.$wrap=c.length?c:a('
    ').css(r).appendTo(this.$container);this.isBody||(c=this.$container.css("position"),b=this.$container.css("zIndex"),this.$container.css({position:"static"===c?"relative":c,zIndex:"auto"===b?0:b,background:"none"}),this.$wrap.css({zIndex:-999998}));this.$wrap.css({position:this.isBody&&l?"fixed":"absolute"});this.index=0;this.show(this.index);a(d).on("resize.backstretch",a.proxy(this.resize,this)).on("orientationchange.backstretch",a.proxy(function(){this.isBody&&0===d.pageYOffset&&(d.scrollTo(0,1),this.resize())},this))};q.prototype={resize:function(){try{var a={left:0,top:0},b=this.isBody?this.$root.width():this.$root.innerWidth(),e=b,g=this.isBody?d.innerHeight?d.innerHeight:this.$root.height():this.$root.innerHeight(),j=e/this.$img.data("ratio"),f;j>=g?(f=(j-g)/2,this.options.centeredY&&(a.top="-"+f+"px")):(j=g,e=j*this.$img.data("ratio"),f=(e-b)/2,this.options.centeredX&&(a.left="-"+f+"px"));this.$wrap.css({width:b,height:g}).find("img:not(.deleteable)").css({width:e,height:j}).css(a)}catch(h){}return this},show:function(c){if(!(Math.abs(c)>this.images.length-1)){var b=this,e=b.$wrap.find("img").addClass("deleteable"),d={relatedTarget:b.$container[0]};b.$container.trigger(a.Event("backstretch.before",d),[b,c]);this.index=c;clearInterval(b.interval);b.$img=a("").css(s).bind("load",function(f){var h=this.width||a(f.target).width();f=this.height||a(f.target).height();a(this).data("ratio",h/f);a(this).fadeIn(b.options.speed||b.options.fade,function(){e.remove();b.paused||b.cycle();a(["after","show"]).each(function(){b.$container.trigger(a.Event("backstretch."+this,d),[b,c])})});b.resize()}).appendTo(b.$wrap);b.$img.attr("src",b.images[c]);return b}},next:function(){return this.show(this.indexe||d.operamini&&"[object OperaMini]"==={}.toString.call(d.operamini)||n&&7458>t||-1e||h&&6>h||"palmGetResource"in d&&e&&534>e||-1=k)})(jQuery,window); -------------------------------------------------------------------------------- /static/js/jquery.simple-text-rotator.min.js: -------------------------------------------------------------------------------- 1 | !function(e){var t={animation:"dissolve",separator:",",speed:2e3};e.fx.step.textShadowBlur=function(t){e(t.elem).prop("textShadowBlur",t.now).css({textShadow:"0 0 "+Math.floor(t.now)+"px black"})};e.fn.textrotator=function(n){var r=e.extend({},t,n);return this.each(function(){var t=e(this);var n=[];e.each(t.text().split(r.separator),function(e,t){n.push(t)});t.text(n[0]);var i=function(){switch(r.animation){case"dissolve":t.animate({textShadowBlur:20,opacity:0},500,function(){s=e.inArray(t.text(),n);if(s+1==n.length)s=-1;t.text(n[s+1]).animate({textShadowBlur:0,opacity:1},500)});break;case"flip":if(t.find(".back").length>0){t.html(t.find(".back").html())}var i=t.text();var s=e.inArray(i,n);if(s+1==n.length)s=-1;t.html("");e(""+i+"").appendTo(t);e(""+n[s+1]+"").appendTo(t);t.wrapInner("").find(".rotating").hide().addClass("flip").show().css({"-webkit-transform":" rotateY(-180deg)","-moz-transform":" rotateY(-180deg)","-o-transform":" rotateY(-180deg)",transform:" rotateY(-180deg)"});break;case"flipUp":if(t.find(".back").length>0){t.html(t.find(".back").html())}var i=t.text();var s=e.inArray(i,n);if(s+1==n.length)s=-1;t.html("");e(""+i+"").appendTo(t);e(""+n[s+1]+"").appendTo(t);t.wrapInner("").find(".rotating").hide().addClass("flip up").show().css({"-webkit-transform":" rotateX(-180deg)","-moz-transform":" rotateX(-180deg)","-o-transform":" rotateX(-180deg)",transform:" rotateX(-180deg)"});break;case"flipCube":if(t.find(".back").length>0){t.html(t.find(".back").html())}var i=t.text();var s=e.inArray(i,n);if(s+1==n.length)s=-1;t.html("");e(""+i+"").appendTo(t);e(""+n[s+1]+"").appendTo(t);t.wrapInner("").find(".rotating").hide().addClass("flip cube").show().css({"-webkit-transform":" rotateY(180deg)","-moz-transform":" rotateY(180deg)","-o-transform":" rotateY(180deg)",transform:" rotateY(180deg)"});break;case"flipCubeUp":if(t.find(".back").length>0){t.html(t.find(".back").html())}var i=t.text();var s=e.inArray(i,n);if(s+1==n.length)s=-1;t.html("");e(""+i+"").appendTo(t);e(""+n[s+1]+"").appendTo(t);t.wrapInner("").find(".rotating").hide().addClass("flip cube up").show().css({"-webkit-transform":" rotateX(180deg)","-moz-transform":" rotateX(180deg)","-o-transform":" rotateX(180deg)",transform:" rotateX(180deg)"});break;case"spin":if(t.find(".rotating").length>0){t.html(t.find(".rotating").html())}s=e.inArray(t.text(),n);if(s+1==n.length)s=-1;t.wrapInner("").find(".rotating").hide().text(n[s+1]).show().css({"-webkit-transform":" rotate(0) scale(1)","-moz-transform":"rotate(0) scale(1)","-o-transform":"rotate(0) scale(1)",transform:"rotate(0) scale(1)"});break;case"fade":t.fadeOut(r.speed,function(){s=e.inArray(t.text(),n);if(s+1==n.length)s=-1;t.text(n[s+1]).fadeIn(r.speed)});break}};setInterval(i,r.speed)})}}(window.jQuery) -------------------------------------------------------------------------------- /static/js/loading.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Created with Sublime Text 3. 3 | * license: http://www.lovewebgames.com/jsmodule/index.html 4 | * github: https://github.com/tianxiangbing/loading 5 | * User: 田想兵 6 | * Date: 2015-08-05 7 | * Time: 11:27:55 8 | * Contact: 55342775@qq.com 9 | * desc:请尽量使用github上的代码，会修复一些问题，关注https://github.com/tianxiangbing/loading 10 | */ 11 | ; 12 | (function (root, factory) { 13 | //amd 14 | if (typeof define === 'function' && define.amd) { 15 | define(['$'], factory); 16 | } else if (typeof exports === 'object') { //umd 17 | module.exports = factory(); 18 | } else { 19 | root.Loading = factory(window.Zepto || window.jQuery || $); 20 | } 21 | })(window, function ($) { 22 | var Loading = function () { }; 23 | Loading.prototype = { 24 | loadingTpl: '
    ', 25 | stop: function () { 26 | var content = $(this.target); 27 | this.loading.remove(); 28 | this.loading = null; 29 | }, 30 | start: function () { 31 | var _this = this; 32 | var loading = this.loading; 33 | if (!loading) { 34 | loading = $(_this.loadingTpl); 35 | $('body').append(loading); 36 | } 37 | this.loading = loading; 38 | //console.log(cw,ch) 39 | this.setPosition(); 40 | }, 41 | setPosition: function () { 42 | var _this = this; 43 | var loading = this.loading; 44 | var target = _this.target; 45 | var content = $(target); 46 | var ch = $(content).outerHeight(); 47 | var cw = $(content).outerWidth(); 48 | if ($(target)[0].tagName == "HTML") { 49 | ch = Math.max($(target).height(), $(window).height()); 50 | cw = Math.max($(target).width(), $(window).width()); 51 | } 52 | if(loading != null){ 53 | loading.height(ch).width(cw); 54 | loading.find('div').height(ch).width(cw); 55 | if (ch < 100) { 56 | loading.find('i').height(ch).width(ch); 57 | } 58 | var offset = $(content).offset(); 59 | loading.css({ 60 | top: offset.top, 61 | left: offset.left 62 | }); 63 | var icon = loading.find('i'); 64 | var h = ch, 65 | w = cw, 66 | top = 0, 67 | left = 0; 68 | if ($(target)[0].tagName == "HTML") { 69 | h = $(window).height(); 70 | w = $(window).width(); 71 | top = (h - icon.height()) / 2 + $(window).scrollTop(); 72 | left = (w - icon.width()) / 2 + $(window).scrollLeft(); 73 | } else { 74 | top = (h - icon.height()) / 2; 75 | left = (w - icon.width()) / 2; 76 | } 77 | icon.css({ 78 | top: top, 79 | left: left 80 | }) 81 | }; 82 | 83 | 84 | 85 | }, 86 | init: function (settings) { 87 | settings = settings || {}; 88 | this.loadingTpl = settings.loadingTpl || this.loadingTpl; 89 | this.target = settings.target || 'html'; 90 | this.bindEvent(); 91 | }, 92 | bindEvent: function () { 93 | var _this = this; 94 | $(this.target).on('stop', function () { 95 | _this.stop(); 96 | }); 97 | $(window).on('resize', function () { 98 | _this.setPosition(); 99 | }); 100 | } 101 | } 102 | return Loading; 103 | }); -------------------------------------------------------------------------------- /static/js/main.js: -------------------------------------------------------------------------------- 1 | $(function () { 2 | "use strict"; 3 | 4 | 5 | /* ========================================================================== 6 | Background Slideshow images 7 | ========================================================================== */ 8 | 9 | $(".main").backstretch([ 10 | "/static/img/bg-1.jpg", 11 | "/static/img/bg-2.jpg" 12 | 13 | ], { 14 | fade: 750, 15 | duration: 4000 16 | }); 17 | 18 | 19 | /* ========================================================================== 20 | On Scroll animation 21 | ========================================================================== */ 22 | 23 | if ($(window).width() > 992) { 24 | new WOW().init(); 25 | } 26 | ; 27 | 28 | 29 | /* ========================================================================== 30 | Fade On Scroll 31 | ========================================================================== */ 32 | 33 | 34 | if ($(window).width() > 992) { 35 | 36 | $(window).on('scroll', function () { 37 | $('.main').css('opacity', function () { 38 | return 1 - ($(window).scrollTop() / $(this).outerHeight()); 39 | }); 40 | }); 41 | } 42 | ; 43 | 44 | 45 | $(".rotate").textrotator({ 46 | animation: "dissolve", 47 | separator: ",", 48 | speed: 2500 49 | }); 50 | 51 | 52 | }); 53 | -------------------------------------------------------------------------------- /static/js/search.js: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /static/js/wow.min.js: -------------------------------------------------------------------------------- 1 | /*! WOW - v1.0.1 - 2014-08-15 2 | * Copyright (c) 2014 Matthieu Aussaguel; Licensed MIT */(function(){var a,b,c,d=function(a,b){return function(){return a.apply(b,arguments)}},e=[].indexOf||function(a){for(var b=0,c=this.length;c>b;b++)if(b in this&&this[b]===a)return b;return-1};b=function(){function a(){}return a.prototype.extend=function(a,b){var c,d;for(c in b)d=b[c],null==a[c]&&(a[c]=d);return a},a.prototype.isMobile=function(a){return/Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(a)},a}(),c=this.WeakMap||this.MozWeakMap||(c=function(){function a(){this.keys=[],this.values=[]}return a.prototype.get=function(a){var b,c,d,e,f;for(f=this.keys,b=d=0,e=f.length;e>d;b=++d)if(c=f[b],c===a)return this.values[b]},a.prototype.set=function(a,b){var c,d,e,f,g;for(g=this.keys,c=e=0,f=g.length;f>e;c=++e)if(d=g[c],d===a)return void(this.values[c]=b);return this.keys.push(a),this.values.push(b)},a}()),a=this.MutationObserver||this.WebkitMutationObserver||this.MozMutationObserver||(a=function(){function a(){console.warn("MutationObserver is not supported by your browser."),console.warn("WOW.js cannot detect dom mutations, please call .sync() after loading new content.")}return a.notSupported=!0,a.prototype.observe=function(){},a}()),this.WOW=function(){function f(a){null==a&&(a={}),this.scrollCallback=d(this.scrollCallback,this),this.scrollHandler=d(this.scrollHandler,this),this.start=d(this.start,this),this.scrolled=!0,this.config=this.util().extend(a,this.defaults),this.animationNameCache=new c}return f.prototype.defaults={boxClass:"wow",animateClass:"animated",offset:0,mobile:!0,live:!0},f.prototype.init=function(){var a;return this.element=window.document.documentElement,"interactive"===(a=document.readyState)||"complete"===a?this.start():document.addEventListener("DOMContentLoaded",this.start),this.finished=[]},f.prototype.start=function(){var b,c,d,e;if(this.stopped=!1,this.boxes=function(){var a,c,d,e;for(d=this.element.querySelectorAll("."+this.config.boxClass),e=[],a=0,c=d.length;c>a;a++)b=d[a],e.push(b);return e}.call(this),this.all=function(){var a,c,d,e;for(d=this.boxes,e=[],a=0,c=d.length;c>a;a++)b=d[a],e.push(b);return e}.call(this),this.boxes.length)if(this.disabled())this.resetStyle();else{for(e=this.boxes,c=0,d=e.length;d>c;c++)b=e[c],this.applyStyle(b,!0);window.addEventListener("scroll",this.scrollHandler,!1),window.addEventListener("resize",this.scrollHandler,!1),this.interval=setInterval(this.scrollCallback,50)}return this.config.live?new a(function(a){return function(b){var c,d,e,f,g;for(g=[],e=0,f=b.length;f>e;e++)d=b[e],g.push(function(){var a,b,e,f;for(e=d.addedNodes||[],f=[],a=0,b=e.length;b>a;a++)c=e[a],f.push(this.doSync(c));return f}.call(a));return g}}(this)).observe(document.body,{childList:!0,subtree:!0}):void 0},f.prototype.stop=function(){return this.stopped=!0,window.removeEventListener("scroll",this.scrollHandler,!1),window.removeEventListener("resize",this.scrollHandler,!1),null!=this.interval?clearInterval(this.interval):void 0},f.prototype.sync=function(){return a.notSupported?this.doSync(this.element):void 0},f.prototype.doSync=function(a){var b,c,d,f,g;if(!this.stopped){if(null==a&&(a=this.element),1!==a.nodeType)return;for(a=a.parentNode||a,f=a.querySelectorAll("."+this.config.boxClass),g=[],c=0,d=f.length;d>c;c++)b=f[c],e.call(this.all,b)<0?(this.applyStyle(b,!0),this.boxes.push(b),this.all.push(b),g.push(this.scrolled=!0)):g.push(void 0);return g}},f.prototype.show=function(a){return this.applyStyle(a),a.className=""+a.className+" "+this.config.animateClass},f.prototype.applyStyle=function(a,b){var c,d,e;return d=a.getAttribute("data-wow-duration"),c=a.getAttribute("data-wow-delay"),e=a.getAttribute("data-wow-iteration"),this.animate(function(f){return function(){return f.customStyle(a,b,d,c,e)}}(this))},f.prototype.animate=function(){return"requestAnimationFrame"in window?function(a){return window.requestAnimationFrame(a)}:function(a){return a()}}(),f.prototype.resetStyle=function(){var a,b,c,d,e;for(d=this.boxes,e=[],b=0,c=d.length;c>b;b++)a=d[b],e.push(a.setAttribute("style","visibility: visible;"));return e},f.prototype.customStyle=function(a,b,c,d,e){return b&&this.cacheAnimationName(a),a.style.visibility=b?"hidden":"visible",c&&this.vendorSet(a.style,{animationDuration:c}),d&&this.vendorSet(a.style,{animationDelay:d}),e&&this.vendorSet(a.style,{animationIterationCount:e}),this.vendorSet(a.style,{animationName:b?"none":this.cachedAnimationName(a)}),a},f.prototype.vendors=["moz","webkit"],f.prototype.vendorSet=function(a,b){var c,d,e,f;f=[];for(c in b)d=b[c],a[""+c]=d,f.push(function(){var b,f,g,h;for(g=this.vendors,h=[],b=0,f=g.length;f>b;b++)e=g[b],h.push(a[""+e+c.charAt(0).toUpperCase()+c.substr(1)]=d);return h}.call(this));return f},f.prototype.vendorCSS=function(a,b){var c,d,e,f,g,h;for(d=window.getComputedStyle(a),c=d.getPropertyCSSValue(b),h=this.vendors,f=0,g=h.length;g>f;f++)e=h[f],c=c||d.getPropertyCSSValue("-"+e+"-"+b);return c},f.prototype.animationName=function(a){var b;try{b=this.vendorCSS(a,"animation-name").cssText}catch(c){b=window.getComputedStyle(a).getPropertyValue("animation-name")}return"none"===b?"":b},f.prototype.cacheAnimationName=function(a){return this.animationNameCache.set(a,this.animationName(a))},f.prototype.cachedAnimationName=function(a){return this.animationNameCache.get(a)},f.prototype.scrollHandler=function(){return this.scrolled=!0},f.prototype.scrollCallback=function(){var a;return!this.scrolled||(this.scrolled=!1,this.boxes=function(){var b,c,d,e;for(d=this.boxes,e=[],b=0,c=d.length;c>b;b++)a=d[b],a&&(this.isVisible(a)?this.show(a):e.push(a));return e}.call(this),this.boxes.length||this.config.live)?void 0:this.stop()},f.prototype.offsetTop=function(a){for(var b;void 0===a.offsetTop;)a=a.parentNode;for(b=a.offsetTop;a=a.offsetParent;)b+=a.offsetTop;return b},f.prototype.isVisible=function(a){var b,c,d,e,f;return c=a.getAttribute("data-wow-offset")||this.config.offset,f=window.pageYOffset,e=f+Math.min(this.element.clientHeight,innerHeight)-c,d=this.offsetTop(a),b=d+a.clientHeight,e>=d&&b>=f},f.prototype.util=function(){return null!=this._util?this._util:this._util=new b},f.prototype.disabled=function(){return!this.config.mobile&&this.util().isMobile(navigator.userAgent)},f}()}).call(this); -------------------------------------------------------------------------------- /static/upload/no_good_comments.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/static/upload/no_good_comments.png -------------------------------------------------------------------------------- /static/upload/no_hot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/static/upload/no_hot.png -------------------------------------------------------------------------------- /static/upload/no_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jasonhy/ProductAnalysis/b1015890dd6b64cd7476deb155cf26501859fe7a/static/upload/no_overview.png -------------------------------------------------------------------------------- /templates/analysis.html: -------------------------------------------------------------------------------- 1 | 16 | 17 |
    18 |
    19 |
    20 |
    21 |
    22 |
    23 |
    24 | 25 | 26 | 27 |
    28 |
    29 |

    30 |

    31 |

    32 |
    33 |
    34 | 35 |
    36 | 37 |
    38 |
    39 |

    评价总览:分析评价关键字所占的数量,纵坐标表示数量

    40 |
    41 |
    42 | 43 |
    44 |
    45 |
    46 | 47 |
    48 |
    49 |
    50 |

    好评率:根据好中差的数量了进行统计

    51 |
    52 |
    53 | 54 |
    55 |
    56 | 57 |
    58 |
    59 |

    热度:按照最近的评价日期来分析当前产品是否还是热点话题

    60 |
    61 |
    62 | 63 |
    64 |
    65 | 66 |
    67 |
    68 |
    69 |
    70 | -------------------------------------------------------------------------------- /templates/base.html: -------------------------------------------------------------------------------- 1 | {% load static from staticfiles%} 2 | 3 | 4 | 5 | 6 | 7 | 8 | {% block title %} 9 | 菜鸟产品分析平台 10 | {% endblock %} 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | {% block extra %} 23 | 24 | {% endblock %} 25 | 26 | 27 | 28 | 29 | {% block body %} 30 | 31 | {% endblock %} 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | {% block js %} 42 | 43 | {% endblock %} 44 | 45 | -------------------------------------------------------------------------------- /templates/index.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | {% load static from staticfiles %} 3 | 4 | {% block body %} 5 |
    6 |
    7 |
    8 |
    9 |
    10 | 11 | 12 | 13 |
    14 | 15 |
    16 |
    17 |
    18 | 19 | 20 |
    21 |

    欢迎来到菜鸟产品分析平台

    22 |
    23 | 24 | 25 | 26 |
    27 |
    28 |
    29 |
    30 |
    31 | 32 | 33 | 34 | 35 |
    36 |
    37 |

    38 |
    39 |
    40 |
    41 | 42 | 43 |
    44 |
    45 |
    46 | 47 |
    48 | 49 | 50 | {% endblock %} -------------------------------------------------------------------------------- /templates/search/indexes/app/productinfo_text.txt: -------------------------------------------------------------------------------- 1 | {{ object.p_title }} 2 | {{ object.p_id }} -------------------------------------------------------------------------------- /templates/search/search.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | {% load static from staticfiles %} 3 | {% load product_tags %} 4 | 5 | {% block title %} 6 | 搜索结果 7 | {% endblock %} 8 | 9 | 10 | {% block body %} 11 |
    12 | 13 | {# #} 25 |
    26 | 27 |
    28 | 33 | 34 |
      35 | {% if query %} 36 | 37 | {% get_search_result_list page.object_list %} 38 | 39 | {% endif %} 40 | 41 |
    42 | 43 | {% if query %} 44 | {% if page.has_previous or page.has_next %} 45 |
    46 | {% if page.has_previous %} 47 | {% endif %} 上一页 48 | {% if page.has_previous %}{% endif %} 49 | | 50 | {% if page.has_next %} 51 | {% endif %}下一页 52 | {% if page.has_next %}{% endif %} 53 |
    54 | {% endif %} 55 | 56 | {% endif %} 57 |
    58 |
    59 |
    60 | {% include 'analysis.html' %} 61 |
    62 |
    63 | 64 | {% endblock %} 65 | 66 | 67 | {% block js %} 68 | 159 | {% endblock %} --------------------------------------------------------------------------------