├── 1.png ├── 10.png ├── 11.png ├── 12.png ├── 13.png ├── 14.png ├── 15.png ├── 16.png ├── 17.png ├── 18.png ├── 19.png ├── 2.png ├── 20.png ├── 3.png ├── 4.png ├── 5.png ├── 6.png ├── 7.png ├── 8.png ├── 9.png ├── JedisUtil.java ├── README.md ├── ReduceByKeySortRddDemo.scala ├── crawler.py ├── drawio.png ├── 大屏.png └── 无标题.png /1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/1.png -------------------------------------------------------------------------------- /10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/10.png -------------------------------------------------------------------------------- /11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/11.png -------------------------------------------------------------------------------- /12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/12.png -------------------------------------------------------------------------------- /13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/13.png -------------------------------------------------------------------------------- /14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/14.png -------------------------------------------------------------------------------- /15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/15.png -------------------------------------------------------------------------------- /16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/16.png -------------------------------------------------------------------------------- /17.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/17.png -------------------------------------------------------------------------------- /18.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/18.png -------------------------------------------------------------------------------- /19.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/19.png -------------------------------------------------------------------------------- /2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/2.png -------------------------------------------------------------------------------- /20.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/20.png -------------------------------------------------------------------------------- /3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/3.png -------------------------------------------------------------------------------- /4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/4.png -------------------------------------------------------------------------------- /5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/5.png -------------------------------------------------------------------------------- /6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/6.png -------------------------------------------------------------------------------- /7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/7.png -------------------------------------------------------------------------------- /8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/8.png -------------------------------------------------------------------------------- /9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/9.png -------------------------------------------------------------------------------- /JedisUtil.java: -------------------------------------------------------------------------------- 1 | package com.bigdata.storm.kafka.util; 2 | 3 | import redis.clients.jedis.Jedis; 4 | import redis.clients.jedis.JedisPool; 5 | import redis.clients.jedis.JedisPoolConfig; 6 | 7 | /** 8 | * @program: storm-kafka-api-demo 9 | * @description: redis工具类 10 | * @author: 小毕 11 | * @company: 清华大学深圳研究生院 12 | * @create: 2019-08-22 17:23 13 | */ 14 | public class JedisUtil { 15 | 16 | /*redis连接池*/ 17 | private static JedisPool pool; 18 | 19 | /** 20 | *@Description: 返回redis连接池 21 | *@Param: 22 | *@return: 23 | *@Author: 小毕 24 | *@date: 2019/8/22 0022 25 | */ 26 | public static JedisPool getPool(){ 27 | if(pool==null){ 28 | //创建jedis连接池配置 29 | JedisPoolConfig jedisPoolConfig = new JedisPoolConfig(); 30 | //最大连接数 31 | jedisPoolConfig.setMaxTotal(20); 32 | //最大空闲连接 33 | jedisPoolConfig.setMaxIdle(5); 34 | pool=new JedisPool(jedisPoolConfig,"node03.hadoop.com",6379,3000); 35 | } 36 | return pool; 37 | } 38 | 39 | public static Jedis getConnection(){ 40 | return getPool().getResource(); 41 | } 42 | 43 | /* public static void main(String[] args) { 44 | //System.out.println(getPool()); 45 | //System.out.println(getConnection().set("hello","world")); 46 | }*/ 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 计算机毕业设计之Python+Spark汽车推荐系统 汽车可视化 汽车数据分析 汽车大数据 汽车推荐app 汽车小程序 大数据毕业设计 2 | 3 | ## 要求 4 | ### 源码有偿一套(论文 PPT 源码+sql脚本) 5 | ### 6 | ### 加好友前帮忙start一下，并备注github有偿获取源码 7 | ### 我的QQ号是2877135669 或者 1679232425 8 | ### 加qq好友说明（被部分 网友整得心力交瘁）： 9 | 1.加好友务必按照格式备注 10 | 2.避免浪费各自的时间 11 | 12 | https://www.bilibili.com/video/BV1cb4y1n7Yy?spm_id_from=333.999.0.0 13 | 14 | ## 开发技术 15 | Hadoop、HDFS、Spark、SpringBoot、echarts、Spark、Python、MySQL、Vue.js 16 | 17 | ## 创新点 18 | 大数据架构、爬虫、数据可视化、短信、百度AI身份证识别、阿里云短信接口、支付宝沙箱支付 19 | 20 | app+web 21 | 22 | 23 | 24 | ## 补充说明 25 | 适合大数据毕业设计、数据分析、爬虫类计算机毕业设计 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | ## 功能 34 | 35 | ![](drawio.png) 36 | 37 | 38 | 39 | 40 | 41 | 最近移动端的沙箱支付不太稳定，经常报订单不存在，可以不管，直接多点几次，仍然可以支付的。 42 | 43 | \## Vue spark 懂车帝汽车大数据大屏 44 | 45 | \### 技术 46 | 47 | \+ 基于spark java API 48 | 49 | \### 实现分析功能 50 | 51 | \+ 基础情况：预约流程、汽车经纪人、用户统计 ---OK 52 | 53 | \+ 登录情况：APP登录和PC端登录分开统计 ---OK 54 | 55 | \+ 预约试驾情况： 统计 ---OK 56 | 57 | \+ 收藏新闻情况 ---OK 58 | 59 | \+ 用户按照省份来源排行统计、柱状图 ---OK 60 | 61 | \+ 用户的来源构成饼图 ---OK 62 | 63 | \+ 用户订单排行榜（金主排行榜） ---OK 64 | 65 | \+ 登录趋势折线图（按照日期） ---OK 66 | 67 | \+ 汽车按照类型（轿车、SUV、MPV） 和品牌（宝马、奔驰、奥迪等） 做一个环状组合饼图 ---OK 68 | 69 | \+ 懂车弟，汽车外观、内饰、科技、动力、操控六维雷达图 ---OK 70 | 71 | 72 | 73 | ## 开发技术 74 | 75 | spark、hadoop、springboot、vue.js、python、百度AI平台、支付宝沙箱支付、阿里云短信接口、vant 76 | 77 | 78 | 79 | ## 特色 80 | 81 | app/小程序+web双端 82 | 83 | 大屏可视化(用spark分析) 84 | 85 | 短信注册 86 | 87 | 在线支付 88 | 89 | 身份证自动上传识别 90 | 91 | 92 | 93 | # 运行截图 94 | 95 | ![](大屏.png) 96 | 97 | ![](无标题.png) 98 | 99 | ![](1.png) 100 | 101 | ![2](2.png) 102 | 103 | ![3](3.png) 104 | 105 | ![4](4.png) 106 | 107 | ![5](5.png) 108 | 109 | ![6](6.png) 110 | 111 | ![7](7.png) 112 | 113 | ![8](8.png) 114 | 115 | ![9](9.png) 116 | 117 | ![10](10.png) 118 | 119 | ![11](11.png) 120 | 121 | ![12](12.png) 122 | 123 | ![13](13.png) 124 | 125 | ![14](14.png) 126 | 127 | ![15](15.png) 128 | 129 | ![16](16.png) 130 | 131 | ![17](17.png) 132 | 133 | ![18](18.png) 134 | 135 | ![19](19.png) 136 | 137 | ![20](20.png) 138 | 139 | # 运行视频(B站) 140 | 141 | https://www.bilibili.com/video/BV1cb4y1n7Yy?spm_id_from=333.999.0.0 142 | 143 | 144 | 145 | 146 | 147 | -------------------------------------------------------------------------------- /ReduceByKeySortRddDemo.scala: -------------------------------------------------------------------------------- 1 | package com.bigdata.spark.reducebykey_sort 2 | 3 | import org.apache.spark.{SparkConf, SparkContext} 4 | 5 | /** 6 | * @program: spark-api-demo 7 | * @description: 类作用描述 8 | * @author: 小毕 9 | * @company: 清华大学深圳研究生院 10 | * @create: 2019-09-02 18:00 11 | */ 12 | object ReduceByKeySortRddDemo { 13 | 14 | def main(args: Array[String]): Unit = { 15 | val conf=new SparkConf() 16 | .setAppName("MapFilterApp") 17 | .setMaster("local") 18 | val sc=new SparkContext(conf) 19 | val rdd1=sc.parallelize(List(("tom", 1), ("jerry", 3), ("kitty", 2), ("shuke", 1))) 20 | val rdd2=sc.parallelize(List(("jerry", 2), ("tom", 3), ("shuke", 2), ("kitty", 5))) 21 | val rdd3=rdd1.union(rdd2) 22 | //按key进行聚合 23 | val rdd4=rdd3.reduceByKey(_+_) 24 | rdd4.collect.foreach(println(_)) 25 | //按value的降序排序 26 | val rdd5=rdd4.map(t=>(t._2,t._1)).sortByKey(false).map(t=>(t._2,t._1)) 27 | rdd5.collect.foreach(println) 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /crawler.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | from bs4 import BeautifulSoup 3 | import requests 4 | import sys 5 | import random 6 | import pymysql 7 | links = [] 8 | datas = [] 9 | hea = { 10 | 'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36' 11 | } 12 | urls =[ 13 | "https://www.chinanews.com/china.shtml", #国内 14 | "https://www.chinanews.com/society.shtml", #社会 15 | "https://www.chinanews.com/compatriot.shtml",#港澳 16 | "https://www.chinanews.com/wenhua.shtml",#文化 17 | "https://www.chinanews.com/world.shtml",#国际 18 | "https://www.chinanews.com/cj/gd.shtml",#财经 19 | "https://www.chinanews.com/sports.shtml",#体育 20 | "https://www.chinanews.com/huaren.shtml" #华人 21 | ] 22 | # 打开数据库连接 23 | db = pymysql.connect(host='127.0.0.1', user='root', password='123456', port=3396, db='news_recommendation_system') 24 | # 使用cursor()方法获取操作游标 25 | cursor = db.cursor() 26 | 27 | def main(): 28 | #reload(sys) 29 | #sys.setdefaultencoding("utf-8") 30 | #baseurl = 'https://www.chinanews.com/taiwan.shtml' # 要爬取的网页链接 31 | baseurl = 'https://www.chinanews.com/taiwan.shtml' # 要爬取的网页链接 32 | # deleteDate() 33 | # 1.爬取主网页获取各个链接 34 | getLink(baseurl) 35 | # 2.根据链接爬取内部信息并且保存数据到数据库 36 | getInformationAndSave() 37 | # 3.关闭数据库 38 | db.close() 39 | 40 | def getInformationAndSave(): 41 | for link in links: 42 | data = [] 43 | url = "https://www.chinanews.com" + link[1] 44 | cur_html = requests.get(url, headers=hea) 45 | cur_html.encoding = "utf8" 46 | soup = BeautifulSoup(cur_html.text, 'html.parser') 47 | # 获取时间 48 | title = soup.find('h1') 49 | title = title.text.strip() 50 | # 获取时间和来源 51 | tr = soup.find('div', class_='left-t').text.split() 52 | time = tr[0] + tr[1] 53 | recourse = tr[2] 54 | # 获取内容 55 | cont = soup.find('div', class_="left_zw") 56 | content = cont.text.strip() 57 | print(link[0] + "---" + title + "---" + time + "---" + recourse + "---" + url) 58 | saveDate(title,content,time,recourse,url) 59 | 60 | def deleteDate(): 61 | sql = "DELETE FROM news " 62 | try: 63 | # 执行SQL语句 64 | cursor.execute(sql) 65 | # 提交修改 66 | db.commit() 67 | except: 68 | # 发生错误时回滚 69 | db.rollback() 70 | 71 | def saveDate(title,content,time,recourse,url): 72 | try: 73 | cursor.execute("INSERT INTO news(news_title, news_content, type_id, news_creatTime, news_recourse,news_link) VALUES ('%s', '%s', '%s', '%s', '%s' ,'%s')" % \ 74 | (title, content, random.randint(1,8), time, recourse,url)) 75 | db.commit() 76 | print("执行成功") 77 | except: 78 | db.rollback() 79 | print("执行失败") 80 | 81 | def getLink(baseurl): 82 | html = requests.get(baseurl, headers=hea) 83 | html.encoding = 'utf8' 84 | soup = BeautifulSoup(html.text, 'html.parser') 85 | for item in soup.select('div.content_list > ul > li'): 86 | # 对不符合的数据进行清洗 87 | if (item.a == None): 88 | continue 89 | data = [] 90 | type = item.div.text[1:3] # 类型 91 | link = item.div.next_sibling.next_sibling.a['href'] 92 | data.append(type) 93 | data.append(link) 94 | links.append(data) 95 | 96 | if __name__ == '__main__': 97 | main() 98 | 99 | 100 | -------------------------------------------------------------------------------- /drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/drawio.png -------------------------------------------------------------------------------- /大屏.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/大屏.png -------------------------------------------------------------------------------- /无标题.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bysj2022NB/car_spark_python_nb2022/f4617153153eb230575005b0290605b60f6f01d2/无标题.png --------------------------------------------------------------------------------