下
23 | for car in response.xpath("//div[@class='list-infoBox']"):
24 | item = GuaziItem()
25 | item['name'] = car.xpath(".//a/@title").extract()
26 |
27 | item['city'] = car.xpath(".//p[@class='fc-gray']/span[1]/text()").extract()
28 |
29 | item['time'] = car.xpath(".//p[@class='fc-gray']/span[2]/text()").extract()
30 |
31 | car_mile = car.xpath(".//p[@class='fc-gray']/text()").extract()
32 | #去掉文字前后的空格
33 | item['mile'] = ''.join(car_mile).strip()
34 |
35 | car_price = car.xpath(".//p[@class='priType-s']/span[1]/i/text()").extract()
36 | item['price'] = ''.join(car_price).strip()
37 |
38 | items.append(item)
39 | return items
40 |
41 |
--------------------------------------------------------------------------------
/guazi/guazi_mysite/templates/guazi.html:
--------------------------------------------------------------------------------
1 | {% extends 'base.html' %}
2 | {% block content %}
3 |
Sources of Guazi.com
4 |
11 |
All Results
12 |
13 |
14 | | name |
15 | city |
16 | time |
17 | mile |
18 | price |
19 |
20 | {% for guazi in guazi_list.object_list %}
21 |
22 | | {{ guazi.name }} |
23 | {{ guazi.city }} |
24 | {{ guazi.time }} |
25 | {{ guazi.mile }} |
26 | {{guazi.price }} |
27 | {% endfor %}
28 |
29 |
30 |
48 | {% endblock %}
49 |
--------------------------------------------------------------------------------
/guazi/guazi/pipelines.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | from twisted.enterprise import adbapi
4 | import MySQLdb
5 | import MySQLdb.cursors
6 | #from scrapy import signals
7 | #import json
8 | #import codecs
9 |
10 | class GuaziPipeline(object):
11 | def __init__(self):
12 | self.dbpool = adbapi.ConnectionPool('MySQLdb',
13 | db = 'guazi',
14 | user = 'root',
15 | passwd = '',
16 | cursorclass = MySQLdb.cursors.DictCursor,
17 | charset = 'utf8',
18 | use_unicode = True
19 | )
20 |
21 | def process_item(self, item, spider):
22 | query = self.dbpool.runInteraction(self._conditional_insert, item)
23 | return item
24 | #将数据写入或者更新
25 | def _conditional_insert(self, tx, item):
26 | name = self._get_name(item)
27 | #获得数据
28 | tx.execute("""
29 | select 1 from guazi_guazicar where name = %s
30 | """, (name, ))
31 | ret = tx.fetchone()
32 |
33 | if ret:
34 | #更新
35 | sql_up = 'update guazi_guazicar set city = %s, time = %s, mile = %s, price = %s where name = %s'
36 | tx.execute(sql_up, (item['city'][0], item['time'][0], item['mile'], item['price'], item['name'][0]))
37 | else:
38 | #写入
39 | sql_in = 'insert into guazi_guazicar values(%s, %s, %s, %s, %s, %s)'
40 | tx.execute(sql_in, ('', item['name'][0], item['city'][0], item['time'][0], item['mile'], item['price']))
41 |
42 | def _get_name(self, item):
43 | return item['name'][0]
44 |
45 | #sql = 'insert into guazicar values(%s, %s, %s, %s, %s, %s)'
46 | #tx.execute(sql, ('', item['name'][0], item['city'][0], item['time'][0], item['mile'], item['price']))
47 | #将数据保存为json格式
48 | #class JsonGuaziPipeline(object):
49 | # def __init__(self):
50 | # self.file = codecs.open('guazi.json', 'w', encoding='utf-8')
51 | # def process_item(self, item, spider):
52 | # line = json.dumps(dict(item), ensure_ascii=False) + "\n"
53 | # self.file.write(line)
54 | # return item
55 | # def spider_closed(self, spider):
56 | # self.file.close()
--------------------------------------------------------------------------------
/guazi/guazi_mysite/guazi/views.py:
--------------------------------------------------------------------------------
1 | # coding:utf-8
2 | from django.shortcuts import render
3 | from guazi.models import GuaziCar
4 | from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger, InvalidPage
5 | from django.db.models import Q
6 |
7 | # Create your views here.
8 |
9 | def list_guazi(request):
10 | after_range_num = 5
11 | before_range_num = 4
12 | page_size = 20
13 | Guazi_list = GuaziCar.objects.all().order_by('-id')
14 | paginator = Paginator(Guazi_list, page_size)
15 | try:
16 | page = int(request.GET.get('page','1'))
17 | if page < 1:
18 | page=1
19 | except ValueError:
20 | page=1
21 | try:
22 | carlist = paginator.page(page)
23 | except (EmptyPage,InvalidPage,PageNotAnInteger):
24 | carlist = paginator.page(1)
25 | if page >= after_range_num:
26 | page_range = list(paginator.page_range)[page-after_range_num:page+before_range_num]
27 | else:
28 | page_range = list(paginator.page_range)[0:int(page)+before_range_num]
29 | return render(request, 'guazi.html', {'guazi_list': carlist, 'page_range': page_range})
30 |
31 | def search(request):
32 | query = request.GET.get('q','')
33 | page_size = 20
34 | after_range_num = 5
35 | before_range_num = 4
36 | if query:
37 | qset = (
38 | Q(name__icontains=query) |
39 | Q(city__icontains=query) |
40 | Q(time__icontains=query) |
41 | Q(mile__icontains=query) |
42 | Q(price__icontains=query)
43 | )
44 | result = GuaziCar.objects.filter(qset).distinct()
45 | paginator = Paginator(result, page_size)
46 | try:
47 | page = int(request.GET.get('page', '1'))
48 | if page < 1:
49 | page = 1
50 | except ValueError:
51 | page = 1
52 | try:
53 | contacts = paginator.page(page)
54 | except (EmptyPage,InvalidPage,PageNotAnInteger):
55 | contacts = paginator.page(paginator.num_pages)
56 | if page >= after_range_num:
57 | page_range = list(paginator.page_range)[page-after_range_num:page+before_range_num]
58 | else:
59 | page_range = list(paginator.page_range)[0:int(page)+before_range_num]
60 | return render(request, 'search.html', {'result': contacts, 'query': query,
61 | 'page_range': page_range })
62 | else:
63 | result = []
64 | return render(request, 'search.html', {})
65 |
--------------------------------------------------------------------------------
/guazi/guazi/settings.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | # Scrapy settings for guazi project
4 | #
5 | # For simplicity, this file contains only settings considered important or
6 | # commonly used. You can find more settings consulting the documentation:
7 | #
8 | # http://doc.scrapy.org/en/latest/topics/settings.html
9 | # http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
10 | # http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
11 |
12 | BOT_NAME = 'guazi'
13 |
14 | SPIDER_MODULES = ['guazi.spiders']
15 | NEWSPIDER_MODULE = 'guazi.spiders'
16 |
17 |
18 | # Crawl responsibly by identifying yourself (and your website) on the user-agent
19 | #USER_AGENT = 'guazi (+http://www.yourdomain.com)'
20 |
21 | # Configure maximum concurrent requests performed by Scrapy (default: 16)
22 | #CONCURRENT_REQUESTS=32
23 |
24 | # Configure a delay for requests for the same website (default: 0)
25 | # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
26 | # See also autothrottle settings and docs
27 | #DOWNLOAD_DELAY=3
28 | # The download delay setting will honor only one of:
29 | #CONCURRENT_REQUESTS_PER_DOMAIN=16
30 | #CONCURRENT_REQUESTS_PER_IP=16
31 |
32 | # Disable cookies (enabled by default)
33 | #COOKIES_ENABLED=False
34 |
35 | # Disable Telnet Console (enabled by default)
36 | #TELNETCONSOLE_ENABLED=False
37 |
38 | # Override the default request headers:
39 | #DEFAULT_REQUEST_HEADERS = {
40 | # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
41 | # 'Accept-Language': 'en',
42 | #}
43 |
44 | # Enable or disable spider middlewares
45 | # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
46 | #SPIDER_MIDDLEWARES = {
47 | # 'guazi.middlewares.MyCustomSpiderMiddleware': 543,
48 | #}
49 |
50 | # Enable or disable downloader middlewares
51 | # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
52 | #DOWNLOADER_MIDDLEWARES = {
53 | # 'guazi.middlewares.MyCustomDownloaderMiddleware': 543,
54 | #}
55 |
56 | # Enable or disable extensions
57 | # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
58 | #EXTENSIONS = {
59 | # 'scrapy.telnet.TelnetConsole': None,
60 | #}
61 |
62 | # Configure item pipelines
63 | # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
64 | ITEM_PIPELINES = {
65 | 'guazi.pipelines.GuaziPipeline': 300,
66 | }
67 |
68 | # Enable and configure the AutoThrottle extension (disabled by default)
69 | # See http://doc.scrapy.org/en/latest/topics/autothrottle.html
70 | # NOTE: AutoThrottle will honour the standard settings for concurrency and delay
71 | #AUTOTHROTTLE_ENABLED=True
72 | # The initial download delay
73 | #AUTOTHROTTLE_START_DELAY=5
74 | # The maximum download delay to be set in case of high latencies
75 | #AUTOTHROTTLE_MAX_DELAY=60
76 | # Enable showing throttling stats for every response received:
77 | #AUTOTHROTTLE_DEBUG=False
78 |
79 | # Enable and configure HTTP caching (disabled by default)
80 | # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
81 | #HTTPCACHE_ENABLED=True
82 | #HTTPCACHE_EXPIRATION_SECS=0
83 | #HTTPCACHE_DIR='httpcache'
84 | #HTTPCACHE_IGNORE_HTTP_CODES=[]
85 | #HTTPCACHE_STORAGE='scrapy.extensions.httpcache.FilesystemCacheStorage'
86 |
--------------------------------------------------------------------------------
/guazi/guazi_mysite/templates/search.html:
--------------------------------------------------------------------------------
1 | {% extends 'base.html' %}
2 | {% block content %}
3 | {% block title %}
4 |
search
5 | {% endblock %}
6 |
Guazi.com
7 |
8 |
18 |
19 |
24 |
25 | {% if query %}
26 |
Results for "{{ query }}":
27 | {% endif %}
28 | {% if result %}
29 |
30 |
31 |
32 | | name |
33 | city |
34 | time |
35 | mile |
36 | price |
37 |
38 | {% for guazicar in result %}
39 |
40 | | {{ guazicar.name }} |
41 | {{ guazicar.city }} |
42 | {{ guazicar.time }} |
43 | {{ guazicar.mile }} |
44 | {{ guazicar.price }} |
45 |
46 | {% endfor %}
47 |
48 |
66 |
67 | {% else %}
68 |
没有记录
69 | {% endif %}
70 |
71 | {% endblock %}
72 |
73 |
--------------------------------------------------------------------------------
/guazi/guazi_mysite/guazi_mysite/settings.py:
--------------------------------------------------------------------------------
1 | """
2 | Django settings for guazi_mysite project.
3 |
4 | Generated by 'django-admin startproject' using Django 1.9.
5 |
6 | For more information on this file, see
7 | https://docs.djangoproject.com/en/1.9/topics/settings/
8 |
9 | For the full list of settings and their values, see
10 | https://docs.djangoproject.com/en/1.9/ref/settings/
11 | """
12 |
13 | import os
14 |
15 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
16 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
17 |
18 |
19 | # Quick-start development settings - unsuitable for production
20 | # See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/
21 |
22 | # SECURITY WARNING: keep the secret key used in production secret!
23 | SECRET_KEY = 'd4@15r&a_wht8=@#)x2&!x3c06_c5t4%rd%3i%2zff6zirhwp7'
24 |
25 | # SECURITY WARNING: don't run with debug turned on in production!
26 | DEBUG = True
27 |
28 | ALLOWED_HOSTS = []
29 |
30 |
31 | # Application definition
32 |
33 | INSTALLED_APPS = [
34 | 'django.contrib.admin',
35 | 'django.contrib.auth',
36 | 'django.contrib.contenttypes',
37 | 'django.contrib.sessions',
38 | 'django.contrib.messages',
39 | 'django.contrib.staticfiles',
40 | 'guazi',
41 | ]
42 |
43 | MIDDLEWARE_CLASSES = [
44 | 'django.middleware.security.SecurityMiddleware',
45 | 'django.contrib.sessions.middleware.SessionMiddleware',
46 | 'django.middleware.common.CommonMiddleware',
47 | 'django.middleware.csrf.CsrfViewMiddleware',
48 | 'django.contrib.auth.middleware.AuthenticationMiddleware',
49 | 'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
50 | 'django.contrib.messages.middleware.MessageMiddleware',
51 | 'django.middleware.clickjacking.XFrameOptionsMiddleware',
52 | ]
53 |
54 | ROOT_URLCONF = 'guazi_mysite.urls'
55 |
56 | TEMPLATES = [
57 | {
58 | 'BACKEND': 'django.template.backends.django.DjangoTemplates',
59 | 'DIRS': [os.path.join(BASE_DIR, 'templates')],
60 | 'APP_DIRS': True,
61 | 'OPTIONS': {
62 | 'context_processors': [
63 | 'django.template.context_processors.debug',
64 | 'django.template.context_processors.request',
65 | 'django.contrib.auth.context_processors.auth',
66 | 'django.contrib.messages.context_processors.messages',
67 | ],
68 | },
69 | },
70 | ]
71 |
72 | WSGI_APPLICATION = 'guazi_mysite.wsgi.application'
73 |
74 |
75 | # Database
76 | # https://docs.djangoproject.com/en/1.9/ref/settings/#databases
77 |
78 | DATABASES = {
79 | 'default': {
80 | 'ENGINE': 'django.db.backends.mysql',
81 | 'NAME': 'guazi',
82 | 'USER': 'root',
83 | 'PASSWORD': '',
84 | 'HOST': '127.0.0.1',
85 | 'PORT': '3306',
86 | }
87 | }
88 |
89 |
90 | # Password validation
91 | # https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators
92 |
93 | AUTH_PASSWORD_VALIDATORS = [
94 | {
95 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
96 | },
97 | {
98 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
99 | },
100 | {
101 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
102 | },
103 | {
104 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
105 | },
106 | ]
107 |
108 |
109 | # Internationalization
110 | # https://docs.djangoproject.com/en/1.9/topics/i18n/
111 |
112 | LANGUAGE_CODE = 'en-us'
113 |
114 | TIME_ZONE = 'UTC'
115 |
116 | USE_I18N = True
117 |
118 | USE_L10N = True
119 |
120 | USE_TZ = True
121 |
122 |
123 | # Static files (CSS, JavaScript, Images)
124 | # https://docs.djangoproject.com/en/1.9/howto/static-files/
125 |
126 | STATIC_URL = '/static/'
127 | STATICFILES_DIRS = (
128 | os.path.join(BASE_DIR, "static"),
129 | )
130 |
--------------------------------------------------------------------------------