├── qrcode.jpg
├── README.md
├── .gitignore
├── data_viewer.py
├── lianjia_crawler.py
└── LICENSE
/qrcode.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Python3Spiders/LianJiaSpider/HEAD/qrcode.jpg
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 | # 项目简介
10 |
11 | 一个基于分页、线程池、代理池的链家网快速爬虫项目,速度可达 10000 条/5 分钟,严禁将所得数据商用!
12 |
13 | 同时对数据进行了清洗、分析、可视化。
14 |
15 | 欢迎提 issue,共同改进本项目!
16 |
17 | # 作者简介
18 |
19 |
20 | |作者|[inspurer](https://inspurer.github.io/2018/06/07/%E6%9C%88%E5%B0%8F%E6%B0%B4%E9%95%BF%E7%9A%84%E7%94%B1%E6%9D%A5/#more)|
21 | |:---:|:---:|
22 | |QQ交流群|[861016679](https://jq.qq.com/?_wv=1027&k=5Js6sKS)|
23 | |个人博客|[https://inspurer.github.io/](https://inspurer.github.io/)|
24 |
25 |
26 | 更多精彩请关注公众号,微信扫描下方二维码或者在微信内搜索 **微信公众号:月小水长(ID:inspurer)**;
27 |
28 |
29 |
30 |
31 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | *.egg-info/
24 | .installed.cfg
25 | *.egg
26 | MANIFEST
27 |
28 | # PyInstaller
29 | # Usually these files are written by a python script from a template
30 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
31 | *.manifest
32 | *.spec
33 |
34 | # Installer logs
35 | pip-log.txt
36 | pip-delete-this-directory.txt
37 |
38 | # Unit test / coverage reports
39 | htmlcov/
40 | .tox/
41 | .coverage
42 | .coverage.*
43 | .cache
44 | nosetests.xml
45 | coverage.xml
46 | *.cover
47 | .hypothesis/
48 | .pytest_cache/
49 |
50 | # Translations
51 | *.mo
52 | *.pot
53 |
54 | # Django stuff:
55 | *.log
56 | local_settings.py
57 | db.sqlite3
58 |
59 | # Flask stuff:
60 | instance/
61 | .webassets-cache
62 |
63 | # Scrapy stuff:
64 | .scrapy
65 |
66 | # Sphinx documentation
67 | docs/_build/
68 |
69 | # PyBuilder
70 | target/
71 |
72 | # Jupyter Notebook
73 | .ipynb_checkpoints
74 |
75 | # pyenv
76 | .python-version
77 |
78 | # celery beat schedule file
79 | celerybeat-schedule
80 |
81 | # SageMath parsed files
82 | *.sage.py
83 |
84 | # Environments
85 | .env
86 | .venv
87 | env/
88 | venv/
89 | ENV/
90 | env.bak/
91 | venv.bak/
92 |
93 | # Spyder project settings
94 | .spyderproject
95 | .spyproject
96 |
97 | # Rope project settings
98 | .ropeproject
99 |
100 | # mkdocs documentation
101 | /site
102 |
103 | # mypy
104 | .mypy_cache/
105 |
--------------------------------------------------------------------------------
/data_viewer.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # author: inspurer(月小水长)
3 | # pc_type lenovo
4 | # create_date: 2019/2/27
5 | # file_name: data_viewer.py
6 | # github https://github.com/inspurer
7 | # qq_mail 2391527690@qq.com
8 |
9 | import json
10 | import matplotlib.pyplot as plt
11 | import matplotlib
12 | # 设置中文字体和负号正常显示
13 | matplotlib.rcParams['font.sans-serif'] = ['SimHei']
14 | matplotlib.rcParams['axes.unicode_minus'] = False
15 | def load_data():
16 | global data,dic_data
17 | with open("xm.json","r",encoding="utf-8") as f:
18 | str = f.read()
19 | data = json.loads(str)
20 | dic_data = dict()
21 | for i in range(len(data)):
22 | per_price = (int)(data[i].get("price"))
23 | region = data[i].get("area")
24 | # print(per_price,region)
25 | try:
26 | dic_data[region].append(per_price)
27 | except:
28 | dic_data[region] = [per_price]
29 | print(dic_data)
30 |
31 | def split_data():
32 | global region_data
33 | region_data = dict()
34 | for region in dic_data.keys():
35 | # 最大值、最小值、平均值
36 | region_data[region] = {"max":dic_data[region][0],"min":dic_data[region][0],"average":0}
37 | for per_price in dic_data[region]:
38 | if per_price > region_data[region]["max"]:
39 | region_data[region]["max"] = per_price
40 | if per_price < region_data[region]["min"]:
41 | region_data[region]["min"] = per_price
42 | region_data[region]["average"] += per_price
43 | region_data[region]["average"] /= len(dic_data[region])
44 | # 保留两位小数
45 | region_data[region]["average"] = round(region_data[region]["average"],2)
46 | print(region_data)
47 |
48 | def data_viewer():
49 | label_list = region_data.keys() # 横坐标刻度显示值
50 | max = []
51 | min = []
52 | average = []
53 | for label in label_list:
54 | max.append(region_data[label].get("max"))
55 | min.append(region_data[label].get("min"))
56 | average.append(region_data[label].get("average"))
57 | x = range(len(max))
58 | """
59 | 绘制条形图
60 | left: 长条形中点横坐标
61 | height: 长条形高度
62 | width: 长条形宽度,默认值0
63 | .8
64 | label: 为后面设置legend准备
65 | """
66 | rects1 = plt.bar(x=x, height=max, width=0.25, alpha=0.8, color='red', label="最大值")
67 | rects2 = plt.bar(x=[i + 0.25 for i in x], height=average, width=0.25, color='green', label="平均值")
68 | rects3 = plt.bar(x=[i + 0.5 for i in x], height=min, width=0.25, color='blue', label="最小值")
69 | #plt.ylim(0, 50) # y轴取值范围
70 | plt.ylabel("房价/元")
71 | """
72 | 设置x轴刻度显示值
73 | 参数一:中点坐标
74 | 参数二:显示值
75 | """
76 | plt.xticks([index + 0.25 for index in x], label_list)
77 | plt.xlabel("区")
78 | plt.title("厦门二手房价格分析图")
79 | plt.legend() # 设置题注 # 编辑文本
80 | for rect in rects1:
81 | height = rect.get_height()
82 | plt.text(rect.get_x() + rect.get_width() / 2, height+1, str(height), ha="center", va="bottom")
83 | for rect in rects2:
84 | height = rect.get_height()
85 | plt.text(rect.get_x() + rect.get_width() / 2, height + 1, str(height), ha="center", va="bottom")
86 | for rect in rects3:
87 | height = rect.get_height()
88 | plt.text(rect.get_x() + rect.get_width() / 2, height + 1, str(height), ha="center", va="bottom")
89 | plt.show()
90 |
91 | def main():
92 | load_data()
93 | split_data()
94 | data_viewer()
95 |
96 | if __name__ == '__main__':
97 | main()
98 |
--------------------------------------------------------------------------------
/lianjia_crawler.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # author: inspurer(月小水长)
3 | # pc_type lenovo
4 | # create_date: 2019/2/27
5 | # file_name: lianjia_crawler.py
6 | # github https://github.com/inspurer
7 | # qq_mail 2391527690@qq.com
8 |
9 | import requests
10 |
11 | from concurrent.futures import ThreadPoolExecutor
12 |
13 | from pyquery import PyQuery as pq
14 |
15 | import json
16 |
17 | import threading
18 |
19 |
20 | import time
21 |
22 |
23 | def get_list_page_url(city):
24 |
25 | start_url = "https://{}.lianjia.com/ershoufang".format(city)
26 | headers = {
27 | 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
28 | }
29 | try:
30 | response = requests.get(start_url, headers=headers)
31 | # print(response.status_code, response.text)
32 | doc = pq(response.text)
33 | total_num = int(doc(".resultDes .total span").text())
34 | total_page = total_num // 30 + 1
35 | # 只能访问到前一百页
36 | if total_page > 100:
37 | total_page = 100
38 |
39 | page_url_list = list()
40 |
41 | for i in range(total_page):
42 | url = start_url + "/pg" + str(i + 1) + "/"
43 | page_url_list.append(url)
44 | #print(url)
45 | return page_url_list
46 |
47 | except:
48 | print("获取总套数出错,请确认起始URL是否正确")
49 | return None
50 |
51 |
52 | detail_list = list()
53 |
54 | # 需要先在本地开启代理池
55 | # 代理池仓库: https://github.com/Python3WebSpider/ProxyPool
56 | def get_valid_ip():
57 | url = "http://localhost:5000/get"
58 | try:
59 | ip = requests.get(url).text
60 | return ip
61 | except:
62 | print("请先运行代理池")
63 |
64 |
65 | def get_detail_page_url(page_url):
66 | global detail_list
67 | headers = {
68 | 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
69 | 'Referer': 'https://bj.lianjia.com/ershoufang'
70 | }
71 |
72 | try:
73 | response = requests.get(page_url,headers=headers,timeout=3)
74 | doc = pq(response.text)
75 | # broswer.get(page_url)
76 | # print(page_url)
77 | # doc = pq(broswer.page_source)
78 | i = 0
79 | detail_urls = list()
80 | for item in doc(".sellListContent li").items():
81 | i += 1
82 | if i == 31:
83 | break
84 | child_item = item(".noresultRecommend")
85 | if child_item == None:
86 | i -= 1
87 | detail_url = child_item.attr("href")
88 | detail_urls.append(detail_url)
89 | return detail_urls
90 | except:
91 | print("获取列表页" + page_url + "出错")
92 |
93 | lock = threading.Lock()
94 |
95 | def detail_page_parser(res):
96 | global detail_list
97 | detail_urls = res.result()
98 | if not detail_urls:
99 | print("detail url 为空")
100 | return None
101 | headers = {
102 | 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
103 | 'Referer': 'https://bj.lianjia.com/ershoufang'
104 | }
105 | for detail_url in detail_urls:
106 | try:
107 | response = requests.get(url=detail_url, headers=headers,timeout=3)
108 | #print(response.status_code)
109 | detail_dict = dict()
110 | doc = pq(response.text)
111 | unit_price = doc(".unitPriceValue").text()
112 | unit_price = unit_price[0:unit_price.index("元")]
113 | title = doc("h1").text()
114 | area = doc(".areaName .info a").eq(0).text().strip()
115 | url = detail_url
116 | detail_dict["title"] = title
117 | detail_dict["area"] = area
118 | detail_dict["price"] = unit_price
119 | detail_dict["url"] = url
120 |
121 | detail_list.append(detail_dict)
122 |
123 | print(unit_price, title, area)
124 |
125 | except:
126 | print("获取详情页出错,换ip重试")
127 | proxies = {
128 | "http": "http://" + get_valid_ip(),
129 | }
130 | try:
131 | response = requests.get(url=detail_url, headers=headers, proxies=proxies)
132 | #print(response.status_code)
133 | detail_dict = dict()
134 | doc = pq(response.text)
135 | unit_price = doc(".unitPriceValue").text()
136 | unit_price = unit_price[0:unit_price.index("元")]
137 | title = doc("h1").text()
138 | area = doc(".areaName .info a").eq(0).text().strip()
139 | url = detail_url
140 | # 已下架的还会爬取,但是没有价格
141 | if len(unit_price)>0:
142 | detail_dict["title"] = title
143 | detail_dict["area"] = area
144 | detail_dict["price"] = unit_price
145 | detail_dict["url"] = url
146 |
147 | detail_list.append(detail_dict)
148 |
149 | print(unit_price, title, area)
150 | except:
151 | print("重试失败...")
152 |
153 |
154 | def save_data(data,filename):
155 | with open(filename+".json", 'w', encoding="utf-8") as f:
156 | f.write(json.dumps(data, indent=2, ensure_ascii=False))
157 |
158 | def main():
159 | # cq,cs,nj,dl,wh,cc
160 | city_list = ['nj']
161 | for city in city_list:
162 | page_url_list = get_list_page_url(city)
163 |
164 | # pool = threadpool.ThreadPool(20)
165 | # requests = threadpool.makeRequests(page_and_detail_parser, page_url_list)
166 | # [pool.putRequest(req) for req in requests]
167 | # pool.wait()
168 |
169 | p = ThreadPoolExecutor(30)
170 |
171 | for page_url in page_url_list:
172 | p.submit(get_detail_page_url, page_url).add_done_callback(detail_page_parser)
173 | # 这里的回调函数拿到的是一个对象。
174 | # 先把返回的res得到一个结果。即在前面加上一个res.result(),这个结果就是get_detail_page_url的返回
175 | p.shutdown()
176 |
177 | print(detail_list)
178 |
179 | save_data(detail_list, city)
180 |
181 | detail_list.clear()
182 |
183 | if __name__ == '__main__':
184 | old = time.time()
185 | main()
186 | new = time.time()
187 | delta_time = new - old
188 | print("程序共运行{}s".format(delta_time))
189 |
190 |
191 |
192 |
193 |
194 |
195 |
196 |
197 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright [yyyy] [name of copyright owner]
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------