├── .gitignore ├── LICENSE ├── README.md ├── data_viewer.py ├── lianjia_crawler.py └── qrcode.jpg /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 |

3 | 4 | 5 | 6 | 7 |

8 | 9 | # 项目简介 10 | 11 | 一个基于分页、线程池、代理池的链家网快速爬虫项目，速度可达 10000 条/5 分钟，严禁将所得数据商用！ 12 | 13 | 同时对数据进行了清洗、分析、可视化。 14 | 15 | 欢迎提 issue,共同改进本项目！ 16 | 17 | # 作者简介 18 | 19 | 20 | |作者|[inspurer](https://inspurer.github.io/2018/06/07/%E6%9C%88%E5%B0%8F%E6%B0%B4%E9%95%BF%E7%9A%84%E7%94%B1%E6%9D%A5/#more)| 21 | |:---:|:---:| 22 | |QQ交流群|[861016679](https://jq.qq.com/?_wv=1027&k=5Js6sKS)| 23 | |个人博客|[https://inspurer.github.io/](https://inspurer.github.io/)| 24 | 25 | 26 | 更多精彩请关注公众号，微信扫描下方二维码或者在微信内搜索 **微信公众号：月小水长（ID:inspurer)**； 27 | 28 |

29 | 30 |

31 | -------------------------------------------------------------------------------- /data_viewer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # author: inspurer(月小水长) 3 | # pc_type lenovo 4 | # create_date: 2019/2/27 5 | # file_name: data_viewer.py 6 | # github https://github.com/inspurer 7 | # qq_mail 2391527690@qq.com 8 | 9 | import json 10 | import matplotlib.pyplot as plt 11 | import matplotlib 12 | # 设置中文字体和负号正常显示 13 | matplotlib.rcParams['font.sans-serif'] = ['SimHei'] 14 | matplotlib.rcParams['axes.unicode_minus'] = False 15 | def load_data(): 16 | global data,dic_data 17 | with open("xm.json","r",encoding="utf-8") as f: 18 | str = f.read() 19 | data = json.loads(str) 20 | dic_data = dict() 21 | for i in range(len(data)): 22 | per_price = (int)(data[i].get("price")) 23 | region = data[i].get("area") 24 | # print(per_price,region) 25 | try: 26 | dic_data[region].append(per_price) 27 | except: 28 | dic_data[region] = [per_price] 29 | print(dic_data) 30 | 31 | def split_data(): 32 | global region_data 33 | region_data = dict() 34 | for region in dic_data.keys(): 35 | # 最大值、最小值、平均值 36 | region_data[region] = {"max":dic_data[region][0],"min":dic_data[region][0],"average":0} 37 | for per_price in dic_data[region]: 38 | if per_price > region_data[region]["max"]: 39 | region_data[region]["max"] = per_price 40 | if per_price < region_data[region]["min"]: 41 | region_data[region]["min"] = per_price 42 | region_data[region]["average"] += per_price 43 | region_data[region]["average"] /= len(dic_data[region]) 44 | # 保留两位小数 45 | region_data[region]["average"] = round(region_data[region]["average"],2) 46 | print(region_data) 47 | 48 | def data_viewer(): 49 | label_list = region_data.keys() # 横坐标刻度显示值 50 | max = [] 51 | min = [] 52 | average = [] 53 | for label in label_list: 54 | max.append(region_data[label].get("max")) 55 | min.append(region_data[label].get("min")) 56 | average.append(region_data[label].get("average")) 57 | x = range(len(max)) 58 | """ 59 | 绘制条形图 60 | left: 长条形中点横坐标 61 | height: 长条形高度 62 | width: 长条形宽度，默认值0 63 | .8 64 | label: 为后面设置legend准备 65 | """ 66 | rects1 = plt.bar(x=x, height=max, width=0.25, alpha=0.8, color='red', label="最大值") 67 | rects2 = plt.bar(x=[i + 0.25 for i in x], height=average, width=0.25, color='green', label="平均值") 68 | rects3 = plt.bar(x=[i + 0.5 for i in x], height=min, width=0.25, color='blue', label="最小值") 69 | #plt.ylim(0, 50) # y轴取值范围 70 | plt.ylabel("房价/元") 71 | """ 72 | 设置x轴刻度显示值 73 | 参数一：中点坐标 74 | 参数二：显示值 75 | """ 76 | plt.xticks([index + 0.25 for index in x], label_list) 77 | plt.xlabel("区") 78 | plt.title("厦门二手房价格分析图") 79 | plt.legend() # 设置题注 # 编辑文本 80 | for rect in rects1: 81 | height = rect.get_height() 82 | plt.text(rect.get_x() + rect.get_width() / 2, height+1, str(height), ha="center", va="bottom") 83 | for rect in rects2: 84 | height = rect.get_height() 85 | plt.text(rect.get_x() + rect.get_width() / 2, height + 1, str(height), ha="center", va="bottom") 86 | for rect in rects3: 87 | height = rect.get_height() 88 | plt.text(rect.get_x() + rect.get_width() / 2, height + 1, str(height), ha="center", va="bottom") 89 | plt.show() 90 | 91 | def main(): 92 | load_data() 93 | split_data() 94 | data_viewer() 95 | 96 | if __name__ == '__main__': 97 | main() 98 | -------------------------------------------------------------------------------- /lianjia_crawler.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # author: inspurer(月小水长) 3 | # pc_type lenovo 4 | # create_date: 2019/2/27 5 | # file_name: lianjia_crawler.py 6 | # github https://github.com/inspurer 7 | # qq_mail 2391527690@qq.com 8 | 9 | import requests 10 | 11 | from concurrent.futures import ThreadPoolExecutor 12 | 13 | from pyquery import PyQuery as pq 14 | 15 | import json 16 | 17 | import threading 18 | 19 | 20 | import time 21 | 22 | 23 | def get_list_page_url(city): 24 | 25 | start_url = "https://{}.lianjia.com/ershoufang".format(city) 26 | headers = { 27 | 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36', 28 | } 29 | try: 30 | response = requests.get(start_url, headers=headers) 31 | # print(response.status_code, response.text) 32 | doc = pq(response.text) 33 | total_num = int(doc(".resultDes .total span").text()) 34 | total_page = total_num // 30 + 1 35 | # 只能访问到前一百页 36 | if total_page > 100: 37 | total_page = 100 38 | 39 | page_url_list = list() 40 | 41 | for i in range(total_page): 42 | url = start_url + "/pg" + str(i + 1) + "/" 43 | page_url_list.append(url) 44 | #print(url) 45 | return page_url_list 46 | 47 | except: 48 | print("获取总套数出错,请确认起始URL是否正确") 49 | return None 50 | 51 | 52 | detail_list = list() 53 | 54 | # 需要先在本地开启代理池 55 | # 代理池仓库: https://github.com/Python3WebSpider/ProxyPool 56 | def get_valid_ip(): 57 | url = "http://localhost:5000/get" 58 | try: 59 | ip = requests.get(url).text 60 | return ip 61 | except: 62 | print("请先运行代理池") 63 | 64 | 65 | def get_detail_page_url(page_url): 66 | global detail_list 67 | headers = { 68 | 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36', 69 | 'Referer': 'https://bj.lianjia.com/ershoufang' 70 | } 71 | 72 | try: 73 | response = requests.get(page_url,headers=headers,timeout=3) 74 | doc = pq(response.text) 75 | # broswer.get(page_url) 76 | # print(page_url) 77 | # doc = pq(broswer.page_source) 78 | i = 0 79 | detail_urls = list() 80 | for item in doc(".sellListContent li").items(): 81 | i += 1 82 | if i == 31: 83 | break 84 | child_item = item(".noresultRecommend") 85 | if child_item == None: 86 | i -= 1 87 | detail_url = child_item.attr("href") 88 | detail_urls.append(detail_url) 89 | return detail_urls 90 | except: 91 | print("获取列表页" + page_url + "出错") 92 | 93 | lock = threading.Lock() 94 | 95 | def detail_page_parser(res): 96 | global detail_list 97 | detail_urls = res.result() 98 | if not detail_urls: 99 | print("detail url 为空") 100 | return None 101 | headers = { 102 | 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36', 103 | 'Referer': 'https://bj.lianjia.com/ershoufang' 104 | } 105 | for detail_url in detail_urls: 106 | try: 107 | response = requests.get(url=detail_url, headers=headers,timeout=3) 108 | #print(response.status_code) 109 | detail_dict = dict() 110 | doc = pq(response.text) 111 | unit_price = doc(".unitPriceValue").text() 112 | unit_price = unit_price[0:unit_price.index("元")] 113 | title = doc("h1").text() 114 | area = doc(".areaName .info a").eq(0).text().strip() 115 | url = detail_url 116 | detail_dict["title"] = title 117 | detail_dict["area"] = area 118 | detail_dict["price"] = unit_price 119 | detail_dict["url"] = url 120 | 121 | detail_list.append(detail_dict) 122 | 123 | print(unit_price, title, area) 124 | 125 | except: 126 | print("获取详情页出错,换ip重试") 127 | proxies = { 128 | "http": "http://" + get_valid_ip(), 129 | } 130 | try: 131 | response = requests.get(url=detail_url, headers=headers, proxies=proxies) 132 | #print(response.status_code) 133 | detail_dict = dict() 134 | doc = pq(response.text) 135 | unit_price = doc(".unitPriceValue").text() 136 | unit_price = unit_price[0:unit_price.index("元")] 137 | title = doc("h1").text() 138 | area = doc(".areaName .info a").eq(0).text().strip() 139 | url = detail_url 140 | # 已下架的还会爬取，但是没有价格 141 | if len(unit_price)>0: 142 | detail_dict["title"] = title 143 | detail_dict["area"] = area 144 | detail_dict["price"] = unit_price 145 | detail_dict["url"] = url 146 | 147 | detail_list.append(detail_dict) 148 | 149 | print(unit_price, title, area) 150 | except: 151 | print("重试失败...") 152 | 153 | 154 | def save_data(data,filename): 155 | with open(filename+".json", 'w', encoding="utf-8") as f: 156 | f.write(json.dumps(data, indent=2, ensure_ascii=False)) 157 | 158 | def main(): 159 | # cq,cs,nj,dl,wh,cc 160 | city_list = ['nj'] 161 | for city in city_list: 162 | page_url_list = get_list_page_url(city) 163 | 164 | # pool = threadpool.ThreadPool(20) 165 | # requests = threadpool.makeRequests(page_and_detail_parser, page_url_list) 166 | # [pool.putRequest(req) for req in requests] 167 | # pool.wait() 168 | 169 | p = ThreadPoolExecutor(30) 170 | 171 | for page_url in page_url_list: 172 | p.submit(get_detail_page_url, page_url).add_done_callback(detail_page_parser) 173 | # 这里的回调函数拿到的是一个对象。 174 | # 先把返回的res得到一个结果。即在前面加上一个res.result(),这个结果就是get_detail_page_url的返回 175 | p.shutdown() 176 | 177 | print(detail_list) 178 | 179 | save_data(detail_list, city) 180 | 181 | detail_list.clear() 182 | 183 | if __name__ == '__main__': 184 | old = time.time() 185 | main() 186 | new = time.time() 187 | delta_time = new - old 188 | print("程序共运行{}s".format(delta_time)) 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | -------------------------------------------------------------------------------- /qrcode.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Python3Spiders/LianJiaSpider/0dc2d859e9fa3615f97a639e5b399a0287e00d2c/qrcode.jpg --------------------------------------------------------------------------------