├── README.md ├── back ├── .gitignore ├── Dockerfile ├── data │ └── sample_movielens_ratings.txt ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── haust │ │ │ └── back │ │ │ ├── BackApplication.java │ │ │ ├── config │ │ │ ├── MyBatisPlusConfig.java │ │ │ ├── SwaggerConfig.java │ │ │ └── WebConfig.java │ │ │ ├── controller │ │ │ ├── CommentController.java │ │ │ ├── NewController.java │ │ │ ├── NewRecommend.java │ │ │ ├── NewstatusController.java │ │ │ └── UserController.java │ │ │ ├── entity │ │ │ ├── Comment.java │ │ │ ├── New.java │ │ │ ├── Newstatus.java │ │ │ ├── User.java │ │ │ └── Useroperationnew.java │ │ │ ├── interceptor │ │ │ └── LoginInterceptor.java │ │ │ ├── mapper │ │ │ ├── CommentMapper.java │ │ │ ├── NewMapper.java │ │ │ ├── NewstatusMapper.java │ │ │ ├── UserMapper.java │ │ │ └── UseroperationnewMapper.java │ │ │ ├── service │ │ │ ├── CommentService.java │ │ │ ├── NewService.java │ │ │ ├── NewstatusService.java │ │ │ ├── UserService.java │ │ │ └── impl │ │ │ │ ├── CommentServiceImpl.java │ │ │ │ ├── NewServiceImpl.java │ │ │ │ ├── NewstatusServiceImpl.java │ │ │ │ └── UserServiceImpl.java │ │ │ └── util │ │ │ └── ReadFile.java │ └── resources │ │ ├── application.properties │ │ ├── log4j.properties │ │ └── static │ │ └── index.html │ └── test │ └── java │ └── com │ └── haust │ └── back │ └── BackApplicationTests.java ├── front ├── .gitignore ├── .vscode │ └── extensions.json ├── README.md ├── index.html ├── package.json ├── pnpm-lock.yaml ├── public │ └── vite.svg ├── src │ ├── App.vue │ ├── api │ │ ├── comments.js │ │ ├── news.js │ │ ├── newstatus.js │ │ └── user.js │ ├── assets │ │ ├── Logo.png │ │ ├── avatar.jpg │ │ └── vue.svg │ ├── components │ │ ├── Calender.vue │ │ ├── Comment.Vue │ │ ├── Reply.vue │ │ ├── edit.vue │ │ ├── hotnews.vue │ │ ├── news.vue │ │ ├── newstatus.vue │ │ ├── person.vue │ │ ├── public.vue │ │ ├── singalnew.vue │ │ └── tabs.vue │ ├── main.js │ ├── pages │ │ ├── 404.vue │ │ ├── Edit.vue │ │ ├── NewDate.vue │ │ ├── Public.vue │ │ ├── about.vue │ │ ├── index.vue │ │ └── login.vue │ ├── router │ │ └── index.js │ ├── style.css │ ├── util │ │ └── axios.js │ └── views │ │ ├── China.vue │ │ ├── Culture.vue │ │ ├── Huaren.vue │ │ ├── Life.vue │ │ ├── NewDetail.vue │ │ ├── Recommend.vue │ │ ├── Sport.vue │ │ ├── datenew.vue │ │ ├── finance.vue │ │ ├── picture.vue │ │ ├── society.vue │ │ ├── technology.vue │ │ └── video.vue └── vite.config.js ├── newrecommend.sql ├── recommender ├── .idea │ ├── codeStyles │ │ ├── Project.xml │ │ └── codeStyleConfig.xml │ ├── compiler.xml │ ├── encodings.xml │ ├── jarRepositories.xml │ ├── misc.xml │ ├── modules.xml │ ├── uiDesigner.xml │ ├── vcs.xml │ └── workspace.xml ├── alsMoudle │ ├── data │ │ ├── product │ │ │ ├── ._SUCCESS.crc │ │ │ ├── .part-00000-05b11672-a325-428f-826e-fcc8ea268fbc-c000.snappy.parquet.crc │ │ │ ├── _SUCCESS │ │ │ └── part-00000-05b11672-a325-428f-826e-fcc8ea268fbc-c000.snappy.parquet │ │ └── user │ │ │ ├── ._SUCCESS.crc │ │ │ ├── .part-00000-0bf4a94e-a5af-4231-92fc-65f85011387a-c000.snappy.parquet.crc │ │ │ ├── _SUCCESS │ │ │ └── part-00000-0bf4a94e-a5af-4231-92fc-65f85011387a-c000.snappy.parquet │ └── metadata │ │ ├── ._SUCCESS.crc │ │ ├── .part-00000.crc │ │ ├── _SUCCESS │ │ └── part-00000 ├── als_rating_test.txt ├── pom.xml ├── recommender.iml ├── src │ └── main │ │ ├── resources │ │ └── log4j.properties │ │ └── scala │ │ ├── Main.scala │ │ ├── Test.scala │ │ └── Train.scala └── target │ └── classes │ ├── Main$$anonfun$1.class │ ├── Main$$anonfun$2.class │ ├── Main$$anonfun$main$1.class │ ├── Main$.class │ ├── Main.class │ ├── Test$$anonfun$main$1.class │ ├── Test$$anonfun$main$2$$anonfun$apply$1.class │ ├── Test$$anonfun$main$2.class │ ├── Test$$anonfun$main$3$$anonfun$apply$2.class │ ├── Test$$anonfun$main$3.class │ ├── Test$.class │ ├── Test.class │ ├── Train$$anonfun$1.class │ ├── Train$$anonfun$2.class │ ├── Train$$anonfun$dirDel$1.class │ ├── Train$$anonfun$main$1.class │ ├── Train$$anonfun$main$2.class │ ├── Train$$anonfun$main$3$$anonfun$apply$1.class │ ├── Train$$anonfun$main$3.class │ ├── Train$$anonfun$main$4$$anonfun$apply$2.class │ ├── Train$$anonfun$main$4.class │ ├── Train$.class │ ├── Train.class │ └── log4j.properties └── spider ├── .idea ├── .gitignore ├── dataSources.xml ├── inspectionProfiles │ └── profiles_settings.xml ├── misc.xml ├── modules.xml ├── spider.iml └── vcs.xml ├── cnew_data.xlsx ├── cnew_url.txt ├── cnew_url1.txt └── main.py /README.md: -------------------------------------------------------------------------------- 1 | # 基于协同过滤算法的新闻推荐系统 2 | # 技术栈 3 | 1. 后端使用SpringBoot+Mybatis-Plus框架 4 | 2. 前端使用Vue.js+Element-Plus+Windi CSS框架 5 | 3. 数据爬虫使用Python+Scrapy框架 6 | 4. 大数据推荐功能使用Scala+Spark实现协同过滤算法,采用余弦相似度计算。 7 | 5. 使用Docker+Maven打包部署服务器 8 | ## Tolist 9 | 微服务重构见另一个分支[branch- new](https://github.com/koyangyang/NewsRecommend/tree/new) 10 | 1. 后端使用SpringCloud重构 11 | 2. 微服务使用nacos + feign 12 | -------------------------------------------------------------------------------- /back/.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | target/ 3 | !.mvn/wrapper/maven-wrapper.jar 4 | !**/src/main/**/target/ 5 | !**/src/test/**/target/ 6 | 7 | ### STS ### 8 | .apt_generated 9 | .classpath 10 | .factorypath 11 | .project 12 | .settings 13 | .springBeans 14 | .sts4-cache 15 | 16 | ### IntelliJ IDEA ### 17 | .idea 18 | *.iws 19 | *.iml 20 | *.ipr 21 | 22 | ### NetBeans ### 23 | /nbproject/private/ 24 | /nbbuild/ 25 | /dist/ 26 | /nbdist/ 27 | /.nb-gradle/ 28 | build/ 29 | !**/src/main/**/build/ 30 | !**/src/test/**/build/ 31 | 32 | ### VS Code ### 33 | .vscode/ 34 | -------------------------------------------------------------------------------- /back/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-jdk-alpine 2 | #发布到网上时只会把jar包和Dockerfile发布上去 3 | COPY ./target/*.jar /app.jar 4 | #地址映射 5 | CMD ["--server.port=8080"] 6 | #对外暴露端口 7 | EXPOSE 8080 8 | #执行命令 9 | ENTRYPOINT ["java","-jar","/app.jar"] 10 | -------------------------------------------------------------------------------- /back/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | com.haust 6 | back 7 | 0.0.1-SNAPSHOT 8 | back 9 | back 10 | 11 | 1.8 12 | UTF-8 13 | UTF-8 14 | 2.7.6 15 | 16 | 17 | 18 | org.springframework.boot 19 | spring-boot-starter-web 20 | 21 | 22 | ch.qos.logback 23 | logback-classic 24 | 25 | 26 | 27 | 28 | 29 | org.springframework.boot 30 | spring-boot-starter-test 31 | test 32 | 33 | 34 | io.springfox 35 | springfox-swagger2 36 | 2.9.2 37 | 38 | 39 | io.springfox 40 | springfox-swagger-ui 41 | 2.9.2 42 | 43 | 44 | 45 | com.baomidou 46 | mybatis-plus-boot-starter 47 | 3.4.2 48 | 49 | 50 | mysql 51 | mysql-connector-java 52 | 5.1.47 53 | 54 | 55 | com.alibaba 56 | druid-spring-boot-starter 57 | 1.1.20 58 | 59 | 60 | org.projectlombok 61 | lombok 62 | 1.18.4 63 | provided 64 | 65 | 66 | 67 | 68 | org.apache.mahout 69 | mahout-mr 70 | 0.12.2 71 | 72 | 73 | javax.servlet 74 | servlet-api 75 | 76 | 77 | org.slf4j 78 | slf4j-api 79 | 80 | 81 | org.slf4j 82 | slf4j-jcl 83 | 84 | 85 | org.apache.lucene 86 | lucene-core 87 | 88 | 89 | org.apache.lucene 90 | lucene-analyzers-common 91 | 92 | 93 | log4j 94 | log4j 95 | 96 | 97 | org.slf4j 98 | slf4j-log4j12 99 | 100 | 101 | jersey-client 102 | com.sun.jersey 103 | 104 | 105 | jersey-core 106 | com.sun.jersey 107 | 108 | 109 | jersey-apache-client4 110 | com.sun.jersey.contribs 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | org.springframework.boot 120 | spring-boot-dependencies 121 | ${spring-boot.version} 122 | pom 123 | import 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | org.apache.maven.plugins 132 | maven-compiler-plugin 133 | 3.8.1 134 | 135 | 1.8 136 | 1.8 137 | UTF-8 138 | 139 | 140 | 141 | org.springframework.boot 142 | spring-boot-maven-plugin 143 | ${spring-boot.version} 144 | 145 | com.haust.back.BackApplication 146 | 147 | 148 | 149 | 150 | repackage 151 | 152 | repackage 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/BackApplication.java: -------------------------------------------------------------------------------- 1 | package com.haust.back; 2 | 3 | import org.mybatis.spring.annotation.MapperScan; 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.SpringBootApplication; 6 | 7 | @SpringBootApplication 8 | @MapperScan("com.haust.back.mapper") 9 | public class BackApplication { 10 | 11 | public static void main(String[] args) { 12 | SpringApplication.run(BackApplication.class, args); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/config/MyBatisPlusConfig.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.config; 2 | 3 | import com.baomidou.mybatisplus.annotation.DbType; 4 | import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor; 5 | import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor; 6 | import org.springframework.context.annotation.Bean; 7 | import org.springframework.context.annotation.Configuration; 8 | 9 | @Configuration 10 | public class MyBatisPlusConfig { 11 | @Bean 12 | public MybatisPlusInterceptor paginationInterceptor() { 13 | MybatisPlusInterceptor interceptor = new MybatisPlusInterceptor(); 14 | PaginationInnerInterceptor paginationInnerInterceptor = new PaginationInnerInterceptor(DbType.MYSQL); 15 | interceptor.addInnerInterceptor(paginationInnerInterceptor); 16 | return interceptor; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/config/SwaggerConfig.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.config; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import springfox.documentation.builders.ApiInfoBuilder; 6 | import springfox.documentation.builders.PathSelectors; 7 | import springfox.documentation.builders.RequestHandlerSelectors; 8 | import springfox.documentation.service.ApiInfo; 9 | import springfox.documentation.spi.DocumentationType; 10 | import springfox.documentation.spring.web.plugins.Docket; 11 | import springfox.documentation.swagger2.annotations.EnableSwagger2; 12 | 13 | @Configuration 14 | @EnableSwagger2 15 | public class SwaggerConfig { 16 | @Bean 17 | public Docket createRestApi() { 18 | return new Docket(DocumentationType.SWAGGER_2) 19 | .apiInfo(apiInfo()) 20 | .select() 21 | .apis(RequestHandlerSelectors.basePackage("com")) //这个是重点 22 | .paths(PathSelectors.any()) 23 | .build(); 24 | } 25 | private ApiInfo apiInfo() { 26 | return new ApiInfoBuilder() 27 | .title("接口文档标题") 28 | .description("文档内容描述") 29 | .version("1.0") 30 | .build(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/config/WebConfig.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.config; 2 | 3 | import com.haust.back.interceptor.LoginInterceptor; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.web.servlet.config.annotation.InterceptorRegistry; 6 | import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; 7 | @Configuration 8 | public class WebConfig implements WebMvcConfigurer { 9 | @Override 10 | public void addInterceptors(InterceptorRegistry registry) 11 | { 12 | registry.addInterceptor(new LoginInterceptor()); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/controller/CommentController.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.controller; 2 | 3 | import com.baomidou.mybatisplus.core.metadata.IPage; 4 | import com.haust.back.entity.Comment; 5 | import com.haust.back.service.CommentService; 6 | import io.swagger.annotations.ApiOperation; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.web.bind.annotation.*; 9 | 10 | @RestController 11 | public class CommentController { 12 | @Autowired 13 | private CommentService commentService; 14 | @ApiOperation("添加一条评论,需传入评论内容") 15 | @PostMapping("/comment/insert") 16 | public Comment insertComment(Comment comment) 17 | { 18 | return commentService.insertComment(comment); 19 | } 20 | @ApiOperation("分页查询指定nid新闻的所有评论,需传入新闻ID,默认每次查询5条数据") 21 | @GetMapping("/comment/{nid}/{id}/{size}") 22 | public IPage getCommentByid(@PathVariable Integer nid,@PathVariable int id,@PathVariable int size) 23 | { 24 | return commentService.getCommentById(nid,id,size); 25 | } 26 | @ApiOperation("查询指定nid新闻的点赞数量,需传入新闻ID") 27 | @GetMapping("/comment/like/{nid}") 28 | public Integer getlike(@PathVariable Integer nid) 29 | { 30 | return commentService.getLike(nid); 31 | } 32 | @ApiOperation("查询指定nid新闻的点踩数量,需传入新闻ID") 33 | @GetMapping("/comment/gdislike/{nid}") 34 | public Integer getdislike(@PathVariable Integer nid) 35 | { 36 | return commentService.getDislike(nid); 37 | } 38 | @ApiOperation("增加指定nid新闻的点赞数量,需传入新闻ID") 39 | @GetMapping("/comment/plike/{nid}") 40 | public Integer postlike(@PathVariable Integer nid) 41 | { 42 | return commentService.postLike(nid); 43 | } 44 | @ApiOperation("增加指定nid新闻的点踩数量,需传入新闻ID") 45 | @GetMapping("/comment/pdislike/{nid}") 46 | public Integer postdislike(@PathVariable Integer nid) 47 | { 48 | return commentService.postDislike(nid); 49 | } 50 | } 51 | 52 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/controller/NewController.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.controller; 2 | 3 | import com.baomidou.mybatisplus.core.metadata.IPage; 4 | import com.haust.back.entity.New; 5 | import com.haust.back.service.NewService; 6 | import io.swagger.annotations.ApiOperation; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.web.bind.annotation.*; 9 | 10 | @RestController 11 | public class NewController { 12 | @Autowired 13 | private NewService newService; 14 | @ApiOperation("对所有的新闻进行分页查询,默认每次查询5条数据") 15 | @GetMapping("/news/findbypage/{id}/{size}") 16 | public IPage getNewsPage(@PathVariable int id,@PathVariable int size) 17 | { 18 | return newService.getNewsPage(id,size); 19 | } 20 | @ApiOperation("对不同分类的新闻进行分页查询,需传入新闻分类id,默认每次查询5条数据") 21 | @GetMapping("/news/{cid}/{id}/{size}") 22 | public IPage getNewsPageByCid(@PathVariable int cid,@PathVariable int id,@PathVariable int size) 23 | { 24 | return newService.getNewsPageByCid(cid,id,size); 25 | } 26 | @ApiOperation("对不同日期的新闻进行分页查询,需传入新闻日期,默认每次查询5条数据") 27 | @GetMapping("/news/pagebydate/{date}/{id}/{size}") 28 | public IPage getNewsPageByDate(@PathVariable String date,@PathVariable int id,@PathVariable int size) 29 | { 30 | return newService.getNewsPageByDate(date,id,size); 31 | } 32 | @ApiOperation("查询指定新闻的详细内容,需传入新闻id") 33 | @GetMapping("/newdetail/{id}") 34 | public New getNewsDetailById(@PathVariable int id) 35 | { 36 | return newService.getNewsDetailById(id); 37 | } 38 | @ApiOperation("添加一条新闻,需传入新闻的详细内容") 39 | @PostMapping("/news/insert") 40 | public New insertNews(New news) 41 | { 42 | return newService.insertNews(news); 43 | } 44 | @ApiOperation("更新一条新闻信息,根据传入的新闻id进行匹配,需传入更新新闻的详细内容") 45 | @PutMapping("/news/update") 46 | public New updateNew(New news) 47 | { 48 | return newService.updateNew(news); 49 | } 50 | 51 | @ApiOperation("删除一条新闻信息,需传入新闻id进行匹配") 52 | @DeleteMapping("/new/{id}") 53 | public int deleteNewById(@PathVariable Integer id) 54 | { 55 | return newService.deleteNewById(id); 56 | } 57 | 58 | } 59 | 60 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/controller/NewRecommend.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.controller; 2 | 3 | import java.util.Arrays; 4 | import java.util.Collection; 5 | import java.util.List; 6 | import java.util.Map; 7 | import java.util.stream.Collectors; 8 | 9 | import org.apache.mahout.cf.taste.common.TasteException; 10 | import org.apache.mahout.cf.taste.impl.common.FastByIDMap; 11 | import org.apache.mahout.cf.taste.impl.model.GenericDataModel; 12 | import org.apache.mahout.cf.taste.impl.model.GenericPreference; 13 | import org.apache.mahout.cf.taste.impl.model.GenericUserPreferenceArray; 14 | import org.apache.mahout.cf.taste.impl.neighborhood.NearestNUserNeighborhood; 15 | import org.apache.mahout.cf.taste.impl.recommender.GenericUserBasedRecommender; 16 | import org.apache.mahout.cf.taste.impl.similarity.UncenteredCosineSimilarity; 17 | import org.apache.mahout.cf.taste.model.DataModel; 18 | import org.apache.mahout.cf.taste.model.PreferenceArray; 19 | import org.apache.mahout.cf.taste.neighborhood.UserNeighborhood; 20 | import org.apache.mahout.cf.taste.recommender.RecommendedItem; 21 | import org.apache.mahout.cf.taste.recommender.Recommender; 22 | import org.apache.mahout.cf.taste.similarity.UserSimilarity; 23 | import org.springframework.beans.factory.annotation.Autowired; 24 | import org.springframework.web.bind.annotation.GetMapping; 25 | import org.springframework.web.bind.annotation.PathVariable; 26 | import org.springframework.web.bind.annotation.RestController; 27 | 28 | import com.haust.back.entity.New; 29 | import com.haust.back.entity.Useroperationnew; 30 | import com.haust.back.mapper.NewMapper; 31 | import com.haust.back.mapper.UseroperationnewMapper; 32 | 33 | import io.swagger.annotations.ApiOperation; 34 | 35 | @RestController 36 | public class NewRecommend { 37 | // 注入用户操作新闻表的Mapper 38 | @Autowired 39 | private UseroperationnewMapper useroperationnewMapper; 40 | // 注入新闻表的Mapper 41 | @Autowired 42 | private NewMapper newMapper; 43 | 44 | @ApiOperation("基于协同过滤算法,对用户ID为{ userId }的指定用户推荐{ size }个新闻") 45 | @GetMapping("/recommend/{userId}/{size}") 46 | public List recommend(@PathVariable Integer userId, @PathVariable Integer size) throws TasteException { 47 | // 获取所有用户的操作新闻 48 | List userList = useroperationnewMapper.getAllUserPreference(); 49 | // 创建数据模型 50 | DataModel dataModel = this.createDataModel(userList); 51 | // 获取用户相似程度,使用余弦相似度 52 | UserSimilarity similarity = new UncenteredCosineSimilarity(dataModel); 53 | // 获取用户邻居 54 | UserNeighborhood userNeighborhood = new NearestNUserNeighborhood(2, similarity, dataModel); 55 | // 构建推荐器 56 | Recommender recommender = new GenericUserBasedRecommender(dataModel, userNeighborhood, similarity); 57 | // 推荐{ size }条新闻 58 | List recommendedItems = recommender.recommend(userId, size); 59 | // 获取计算新闻得分 60 | List itemScores = recommendedItems.stream().map(RecommendedItem::getValue).collect(Collectors.toList()); 61 | // 获取新闻ID 62 | List itemIds = recommendedItems.stream().map(RecommendedItem::getItemID).collect(Collectors.toList()); 63 | for (int i = 0; i < itemIds.size(); i++) { 64 | System.out.println("推荐新闻ID:" + itemIds.get(i) + " 推荐新闻分数:" + itemScores.get(i)); 65 | } 66 | if (itemIds.size() == 0) { 67 | System.out.println("数据不足无法产生推荐"); 68 | return null; 69 | } else 70 | // 根据新闻ID查询新闻的详细内容并返回 71 | { 72 | return newMapper.selectBatchIds(itemIds); 73 | } 74 | } 75 | 76 | private DataModel createDataModel(List Useroperationnews) { 77 | // 创建数据模型 78 | FastByIDMap fastByIdMap = new FastByIDMap<>(); 79 | // 根据用户ID分组 80 | Map> map = 81 | Useroperationnews.stream().collect(Collectors.groupingBy(Useroperationnew::getUserId)); 82 | Collection> list = map.values(); 83 | // 遍历分组后的数据 84 | for (List userPreferences : list) { 85 | // 创建用户偏好数组 86 | GenericPreference[] array = new GenericPreference[userPreferences.size()]; 87 | // 遍历用户偏好 88 | for (int i = 0; i < userPreferences.size(); i++) { 89 | Useroperationnew userPreference = userPreferences.get(i); 90 | GenericPreference item = new GenericPreference(userPreference.getUserId(), userPreference.getNewId(), 91 | userPreference.getValue()); 92 | array[i] = item; 93 | } 94 | // 将用户偏好数组放入数据模型中 95 | fastByIdMap.put(array[0].getUserID(), new GenericUserPreferenceArray(Arrays.asList(array))); 96 | } 97 | // 返回数据模型 98 | return new GenericDataModel(fastByIdMap); 99 | } 100 | 101 | @ApiOperation("获取所有用户的推荐新闻") 102 | @GetMapping("/recommend/hot") 103 | public List getHotNews() { 104 | // 获取所有用户的操作新闻,得到热门新闻 105 | List hotsnews = useroperationnewMapper.getHotNews(); 106 | // 获取新闻ID 107 | List itemIds = hotsnews.stream().map(Useroperationnew::getNewId).collect(Collectors.toList()); 108 | // 获取新闻得分 109 | if (itemIds.size() == 0) { 110 | System.out.println("数据不足无法产生推荐"); 111 | return null; 112 | } else { 113 | for (int i = 0; i < itemIds.size(); i++) { 114 | System.out.println("排名第" + (i + 1) + "的新闻ID是:" + itemIds.get(i)); 115 | } 116 | } 117 | // 根据新闻ID查询新闻的详细内容并返回 118 | return newMapper.selectBatchIds(itemIds); 119 | } 120 | } -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/controller/NewstatusController.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.controller; 2 | 3 | import com.haust.back.entity.Newstatus; 4 | import com.haust.back.service.NewstatusService; 5 | import io.swagger.annotations.ApiOperation; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.web.bind.annotation.*; 8 | 9 | @RestController 10 | public class NewstatusController { 11 | @Autowired 12 | private NewstatusService newstatusService; 13 | 14 | @ApiOperation("查询指定新闻内容的行为数据,需传入新闻ID") 15 | @GetMapping("/newstatus/status/{nid}") 16 | public Newstatus getstatusbynid(@PathVariable Integer nid) { 17 | return newstatusService.getStatusByNid(nid); 18 | } 19 | 20 | @ApiOperation("查询指定nid新闻的点赞数量,需传入新闻ID") 21 | @GetMapping("/newstatus/like/{nid}") 22 | public Integer getlike(@PathVariable Integer nid) { 23 | return newstatusService.getLike(nid); 24 | } 25 | 26 | @ApiOperation("查询指定nid新闻的点踩数量,需传入新闻ID") 27 | @GetMapping("/newstatus/gdislike/{nid}") 28 | public Integer getdislike(@PathVariable Integer nid) { 29 | return newstatusService.getDislike(nid); 30 | } 31 | 32 | @ApiOperation("增加指定nid新闻的点赞数量,需传入新闻ID") 33 | @PostMapping("/newstatus/like") 34 | public Newstatus postlike(Newstatus newstatus) 35 | { 36 | return newstatusService.postLike(newstatus); 37 | } 38 | 39 | @ApiOperation("增加指定nid新闻的点踩数量,需传入新闻ID") 40 | @PostMapping("/newstatus/dislike") 41 | public Newstatus postdislike(Newstatus newstatus) { 42 | return newstatusService.postDislike(newstatus); 43 | } 44 | 45 | @ApiOperation("增加指定nid新闻的转发数量,需传入新闻ID") 46 | @PostMapping("/newstatus/forward") 47 | public Newstatus postforward(Newstatus newstatus) { 48 | return newstatusService.postForward(newstatus); 49 | } 50 | } 51 | 52 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/controller/UserController.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.controller; 2 | 3 | import com.baomidou.mybatisplus.core.metadata.IPage; 4 | import com.haust.back.entity.User; 5 | import com.haust.back.service.UserService; 6 | import io.swagger.annotations.ApiOperation; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.web.bind.annotation.*; 9 | 10 | @RestController 11 | public class UserController { 12 | @Autowired 13 | private UserService userService; 14 | @ApiOperation("登录接口,需传入用户账号和密码") 15 | @PostMapping("/user/login") 16 | public User login(@RequestBody User user) { 17 | return userService.login(user); 18 | } 19 | 20 | @ApiOperation("对所有的用户信息进行分页查询,默认每次查询5条数据") 21 | @GetMapping("/user/{id}/{size}") 22 | public IPage getUserPage(@PathVariable int id, @PathVariable int size) { 23 | return userService.getUserPage(id,size); 24 | } 25 | 26 | @ApiOperation("查询指定用户的详细内容,需传入用户id") 27 | @GetMapping("/user/{id}") 28 | public User getUserDetailById(@PathVariable Integer id) { 29 | return userService.getUserDetailById(id); 30 | } 31 | 32 | @ApiOperation("添加一条用户,需传入用户的详细内容") 33 | @PostMapping("/user/insert") 34 | public User insertUser(User user) { 35 | return userService.insertUser(user); 36 | } 37 | 38 | @ApiOperation("更新一条用户信息,根据传入的用户id进行匹配,需传入更新用户的详细内容") 39 | @PutMapping("/user/update") 40 | public User updateUser(User user) { 41 | return userService.updateUser(user); 42 | } 43 | 44 | @ApiOperation("删除一条用户信息,需传入用户id进行匹配") 45 | @DeleteMapping("/user/{id}") 46 | public int deleteById(@PathVariable Integer id) { 47 | return userService.deleteById(id); 48 | } 49 | 50 | } 51 | 52 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/entity/Comment.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.entity; 2 | 3 | import java.sql.Timestamp; 4 | 5 | import com.baomidou.mybatisplus.annotation.IdType; 6 | import com.baomidou.mybatisplus.annotation.TableId; 7 | 8 | import lombok.Data; 9 | 10 | @Data 11 | public class Comment { 12 | @TableId(type = IdType.AUTO) 13 | private Integer cId; 14 | private Integer userId; 15 | private Integer newId; 16 | private String cCnt; 17 | private Timestamp cDate; 18 | private String userName; 19 | } 20 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/entity/New.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.entity; 2 | 3 | import com.baomidou.mybatisplus.annotation.IdType; 4 | import com.baomidou.mybatisplus.annotation.TableId; 5 | import lombok.Data; 6 | 7 | @Data 8 | public class New { 9 | @TableId(type = IdType.AUTO) 10 | private Integer newId; 11 | private Integer newCid; 12 | private String newCategory; 13 | private String newCover; 14 | private String newTitle; 15 | private String newCnt; 16 | private String newUrl; 17 | private String newDate; 18 | } 19 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/entity/Newstatus.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.entity; 2 | 3 | import lombok.Data; 4 | 5 | @Data 6 | public class Newstatus { 7 | private Integer newId; 8 | private Integer newLike; 9 | private Integer newDislike; 10 | private Integer newForward; 11 | } 12 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/entity/User.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.entity; 2 | 3 | import java.util.Date; 4 | 5 | import com.baomidou.mybatisplus.annotation.IdType; 6 | import com.baomidou.mybatisplus.annotation.TableId; 7 | import com.fasterxml.jackson.annotation.JsonFormat; 8 | 9 | import lombok.Data; 10 | 11 | @Data 12 | public class User { 13 | @TableId(type = IdType.AUTO) 14 | private Integer userId; 15 | private String userName; 16 | private String userPwd; 17 | private String userUrl; 18 | private String userCnt; 19 | @JsonFormat(pattern="yyyy-MM-dd") 20 | private Date userBirthday; 21 | private Integer userGender; 22 | private String userPhone; 23 | private String userEmail; 24 | } 25 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/entity/Useroperationnew.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.entity; 2 | 3 | import com.baomidou.mybatisplus.annotation.IdType; 4 | import com.baomidou.mybatisplus.annotation.TableField; 5 | import com.baomidou.mybatisplus.annotation.TableId; 6 | import lombok.Data; 7 | 8 | import java.io.Serializable; 9 | 10 | @Data 11 | public class Useroperationnew implements Serializable { 12 | @TableId(type = IdType.AUTO) 13 | private Integer id; 14 | private Integer userId; 15 | private Integer newId; 16 | private Integer operationType; 17 | @TableField(exist = false) 18 | private Integer value; 19 | } 20 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/interceptor/LoginInterceptor.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.interceptor; 2 | 3 | import org.springframework.web.servlet.HandlerInterceptor; 4 | 5 | import javax.servlet.http.HttpServletRequest; 6 | import javax.servlet.http.HttpServletResponse; 7 | 8 | public class LoginInterceptor implements HandlerInterceptor { 9 | @Override 10 | public boolean preHandle(HttpServletRequest request, HttpServletResponse response,Object handler) 11 | { 12 | System.out.println( 13 | "拦截器拦截请求:"+request.getMethod()+" 请求路径:"+request.getRequestURI()+" 请求ip:"+request.getRemoteAddr()+" 请求主机名:" 14 | +request.getLocalName()+"请求认证通过" 15 | ); 16 | return true; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/mapper/CommentMapper.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.mapper; 2 | 3 | import org.apache.ibatis.annotations.Mapper; 4 | 5 | import com.baomidou.mybatisplus.core.mapper.BaseMapper; 6 | import com.haust.back.entity.Comment; 7 | 8 | @Mapper 9 | public interface CommentMapper extends BaseMapper { 10 | // @Select("SELECT comment.c_id,comment.user_id,comment.new_id,comment.c_cnt,c_date,user.user_name FROM comment,user where new_id = #{nid} AND comment.user_id= user.user_id") 11 | // public List getCommentById(int nid); 12 | } 13 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/mapper/NewMapper.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.mapper; 2 | 3 | import com.baomidou.mybatisplus.core.mapper.BaseMapper; 4 | import com.haust.back.entity.New; 5 | import org.apache.ibatis.annotations.Mapper; 6 | 7 | @Mapper 8 | public interface NewMapper extends BaseMapper { 9 | } 10 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/mapper/NewstatusMapper.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.mapper; 2 | 3 | import com.baomidou.mybatisplus.core.mapper.BaseMapper; 4 | import com.haust.back.entity.Newstatus; 5 | import org.apache.ibatis.annotations.Mapper; 6 | 7 | @Mapper 8 | public interface NewstatusMapper extends BaseMapper{ 9 | } 10 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/mapper/UserMapper.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.mapper; 2 | 3 | import org.apache.ibatis.annotations.Mapper; 4 | 5 | import com.baomidou.mybatisplus.core.mapper.BaseMapper; 6 | import com.haust.back.entity.User; 7 | 8 | 9 | @Mapper 10 | public interface UserMapper extends BaseMapper { 11 | } 12 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/mapper/UseroperationnewMapper.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.mapper; 2 | 3 | import java.util.List; 4 | 5 | import org.apache.ibatis.annotations.Mapper; 6 | import org.apache.ibatis.annotations.Select; 7 | 8 | import com.baomidou.mybatisplus.core.mapper.BaseMapper; 9 | import com.haust.back.entity.Useroperationnew; 10 | @Mapper 11 | public interface UseroperationnewMapper extends BaseMapper{ 12 | //查询并计算返回不同用户的历史行为数据对每条新闻的value值 13 | @Select("SELECT user_id,new_id,SUM( CASE operation_type WHEN 0 THEN 3 WHEN 1 THEN 3 WHEN 2 THEN 5 ELSE 0 END ) AS value FROM useroperationnew GROUP BY user_id,new_id ORDER BY user_id") 14 | public List getAllUserPreference(); 15 | //查询并计算返回value值排名前20的新闻 16 | @Select("SELECT user_id,new_id,SUM( CASE operation_type WHEN 0 THEN 3 WHEN 1 THEN 3 WHEN 2 THEN 5 ELSE 0 END ) AS value FROM useroperationnew GROUP BY user_id,new_id ORDER BY value DESC LIMIT 0,20") 17 | public List getHotNews(); 18 | } 19 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/service/CommentService.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.service; 2 | 3 | import com.baomidou.mybatisplus.core.metadata.IPage; 4 | import com.haust.back.entity.Comment; 5 | 6 | public interface CommentService { 7 | Comment insertComment(Comment comment); 8 | IPage getCommentById(Integer nid, int id, int size); 9 | Integer getLike(Integer nid); 10 | Integer getDislike(Integer nid); 11 | Integer postLike(Integer nid); 12 | Integer postDislike(Integer nid); 13 | } 14 | 15 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/service/NewService.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.service; 2 | 3 | import com.baomidou.mybatisplus.core.metadata.IPage; 4 | import com.haust.back.entity.New; 5 | 6 | public interface NewService { 7 | IPage getNewsPage(int id, int size); 8 | IPage getNewsPageByCid(int cid, int id, int size); 9 | IPage getNewsPageByDate(String date, int id, int size); 10 | New getNewsDetailById(int id); 11 | New insertNews(New news); 12 | New updateNew(New news); 13 | int deleteNewById(Integer id); 14 | } 15 | 16 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/service/NewstatusService.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.service; 2 | 3 | import com.haust.back.entity.Newstatus; 4 | 5 | public interface NewstatusService { 6 | Newstatus getStatusByNid(Integer nid); 7 | Integer getLike(Integer nid); 8 | Integer getDislike(Integer nid); 9 | Newstatus postLike(Newstatus newstatus); 10 | Newstatus postDislike(Newstatus newstatus); 11 | Newstatus postForward(Newstatus newstatus); 12 | } 13 | 14 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/service/UserService.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.service; 2 | 3 | import com.baomidou.mybatisplus.core.metadata.IPage; 4 | import com.haust.back.entity.User; 5 | 6 | public interface UserService { 7 | User login(User user); 8 | IPage getUserPage(int id, int size); 9 | User getUserDetailById(Integer id); 10 | User insertUser(User user); 11 | User updateUser(User user); 12 | int deleteById(Integer id); 13 | } 14 | 15 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/service/impl/CommentServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.service.impl; 2 | 3 | import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; 4 | import com.baomidou.mybatisplus.core.metadata.IPage; 5 | import com.baomidou.mybatisplus.extension.plugins.pagination.Page; 6 | import com.haust.back.entity.Comment; 7 | import com.haust.back.mapper.CommentMapper; 8 | import com.haust.back.service.CommentService; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.stereotype.Service; 11 | 12 | import java.sql.Timestamp; 13 | import java.util.Date; 14 | 15 | @Service 16 | public class CommentServiceImpl implements CommentService { 17 | @Autowired 18 | private CommentMapper commentMapper; 19 | 20 | @Override 21 | public Comment insertComment(Comment comment) { 22 | Date date = new Date(); 23 | Timestamp t = new Timestamp(date.getTime()); 24 | comment.setCDate(t); 25 | commentMapper.insert(comment); 26 | return comment; 27 | } 28 | 29 | @Override 30 | public IPage getCommentById(Integer nid, int id, int size) { 31 | Page page = new Page<>(id, size); 32 | return commentMapper.selectPage(page, new QueryWrapper().eq("new_id", nid)); 33 | } 34 | 35 | @Override 36 | public Integer getLike(Integer nid) { 37 | return commentMapper.selectCount(new QueryWrapper().eq("new_id", nid)); 38 | } 39 | 40 | @Override 41 | public Integer getDislike(Integer nid) { 42 | return commentMapper.selectCount(new QueryWrapper().eq("new_id", nid)); 43 | } 44 | 45 | @Override 46 | public Integer postLike(Integer nid) { 47 | return commentMapper.selectCount(new QueryWrapper().eq("new_id", nid)); 48 | } 49 | 50 | @Override 51 | public Integer postDislike(Integer nid) { 52 | return commentMapper.selectCount(new QueryWrapper().eq("new_id", nid)); 53 | } 54 | } 55 | 56 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/service/impl/NewServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.service.impl; 2 | 3 | import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; 4 | import com.baomidou.mybatisplus.core.metadata.IPage; 5 | import com.baomidou.mybatisplus.extension.plugins.pagination.Page; 6 | import com.haust.back.entity.New; 7 | import com.haust.back.mapper.NewMapper; 8 | import com.haust.back.service.NewService; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.stereotype.Service; 11 | 12 | @Service 13 | public class NewServiceImpl implements NewService { 14 | @Autowired 15 | private NewMapper newMapper; 16 | 17 | @Override 18 | public IPage getNewsPage(int id, int size) { 19 | Page page = new Page<>(id, size); 20 | return newMapper.selectPage(page, null); 21 | } 22 | 23 | @Override 24 | public IPage getNewsPageByCid(int cid, int id, int size) { 25 | Page page = new Page<>(id, size); 26 | return newMapper.selectPage(page, new QueryWrapper().eq("new_cid", cid)); 27 | } 28 | 29 | @Override 30 | public IPage getNewsPageByDate(String date, int id, int size) { 31 | Page page = new Page<>(id, size); 32 | return newMapper.selectPage(page, new QueryWrapper().eq("new_date", date)); 33 | } 34 | 35 | @Override 36 | public New getNewsDetailById(int id) { 37 | return newMapper.selectById(id); 38 | } 39 | 40 | @Override 41 | public New insertNews(New news) { 42 | newMapper.insert(news); 43 | return news; 44 | } 45 | 46 | @Override 47 | public New updateNew(New news) { 48 | int num = newMapper.update(news, new QueryWrapper().eq("new_id", news.getNewId())); 49 | return num == 1 ? news : null; 50 | } 51 | 52 | @Override 53 | public int deleteNewById(Integer id) { 54 | return newMapper.deleteById(id); 55 | } 56 | } 57 | 58 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/service/impl/NewstatusServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.service.impl; 2 | 3 | import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; 4 | import com.haust.back.entity.Newstatus; 5 | import com.haust.back.mapper.NewstatusMapper; 6 | import com.haust.back.service.NewstatusService; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.stereotype.Service; 9 | 10 | @Service 11 | public class NewstatusServiceImpl implements NewstatusService { 12 | @Autowired 13 | private NewstatusMapper newstatusMapper; 14 | 15 | @Override 16 | public Newstatus getStatusByNid(Integer nid) { 17 | return newstatusMapper.selectOne(new QueryWrapper().eq("new_id", nid)); 18 | } 19 | 20 | @Override 21 | public Integer getLike(Integer nid) { 22 | return newstatusMapper.selectCount(new QueryWrapper().eq("new_id", nid)); 23 | } 24 | 25 | @Override 26 | public Integer getDislike(Integer nid) { 27 | return newstatusMapper.selectCount(new QueryWrapper().eq("new_id", nid)); 28 | } 29 | 30 | @Override 31 | public Newstatus postLike(Newstatus newstatus) { 32 | newstatus.setNewLike(newstatus.getNewLike() + 1); 33 | newstatusMapper.update(newstatus, new QueryWrapper().eq("new_id", newstatus.getNewId())); 34 | return newstatus; 35 | } 36 | 37 | @Override 38 | public Newstatus postDislike(Newstatus newstatus) { 39 | newstatus.setNewDislike(newstatus.getNewDislike() + 1); 40 | newstatusMapper.update(newstatus, new QueryWrapper().eq("new_id", newstatus.getNewId())); 41 | return newstatus; 42 | } 43 | 44 | @Override 45 | public Newstatus postForward(Newstatus newstatus) { 46 | newstatus.setNewForward(newstatus.getNewForward() + 1); 47 | newstatusMapper.update(newstatus, new QueryWrapper().eq("new_id", newstatus.getNewId())); 48 | return newstatus; 49 | } 50 | } 51 | 52 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/service/impl/UserServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.service.impl; 2 | 3 | import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; 4 | import com.baomidou.mybatisplus.core.metadata.IPage; 5 | import com.baomidou.mybatisplus.extension.plugins.pagination.Page; 6 | import com.haust.back.entity.User; 7 | import com.haust.back.mapper.UserMapper; 8 | import com.haust.back.service.UserService; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.stereotype.Service; 11 | 12 | @Service 13 | public class UserServiceImpl implements UserService { 14 | @Autowired 15 | private UserMapper userMapper; 16 | 17 | @Override 18 | public User login(User user) { 19 | return userMapper 20 | .selectOne(new QueryWrapper() 21 | .eq("user_name", user.getUserName()) 22 | .eq("user_pwd", user.getUserPwd())); 23 | } 24 | 25 | @Override 26 | public IPage getUserPage(int id, int size) { 27 | Page page = new Page<>(id, size); 28 | return userMapper.selectPage(page, null); 29 | } 30 | 31 | @Override 32 | public User getUserDetailById(Integer id) { 33 | return userMapper.selectById(id); 34 | } 35 | 36 | @Override 37 | public User insertUser(User user) { 38 | User nuser = userMapper.selectOne(new QueryWrapper().eq("user_name", user.getUserName())); 39 | if (nuser != null) 40 | return null; 41 | else { 42 | userMapper.insert(user); 43 | return user; 44 | } 45 | } 46 | 47 | @Override 48 | public User updateUser(User user) { 49 | int num = userMapper.update(user, new QueryWrapper().eq("user_id", user.getUserId())); 50 | return num == 1 ? user : null; 51 | } 52 | 53 | @Override 54 | public int deleteById(Integer id) { 55 | return userMapper.deleteById(id); 56 | } 57 | } 58 | 59 | -------------------------------------------------------------------------------- /back/src/main/java/com/haust/back/util/ReadFile.java: -------------------------------------------------------------------------------- 1 | package com.haust.back.util; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.io.InputStreamReader; 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | 10 | public class ReadFile { 11 | public List readTxtFile(String filePath, String encoding) throws IOException { 12 | List res = new ArrayList(); 13 | InputStream is = null; 14 | InputStreamReader isr = null; 15 | is = this.getClass().getClassLoader().getResourceAsStream(filePath); 16 | isr = new InputStreamReader(is); 17 | BufferedReader bufferedReader = new BufferedReader(isr); 18 | String lineTxt = null; 19 | while ((lineTxt = bufferedReader.readLine()) != null) { 20 | res.add(lineTxt); 21 | } 22 | return res; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /back/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | server.port=8080 2 | 3 | spring.mvc.pathmatch.matching-strategy=ant_path_matcher 4 | 5 | spring.datasource.type=com.alibaba.druid.pool.DruidDataSource 6 | spring.datasource.driver-class-name=com.mysql.jdbc.Driver 7 | spring.datasource.url=jdbc:mysql://localhost:3306/News?useSSL=false&useUnicode=true&characterEncoding=utf-8&serverTimezone=UTC 8 | spring.datasource.username=root 9 | spring.datasource.password=010619HYy 10 | #mybatis-plus.configuration.log-impl=org.apache.ibatis.logging.stdout.StdOutImpl -------------------------------------------------------------------------------- /back/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=error 2 | #log4j.rootLogger=info, stdout 3 | #log4j.appender.stdout=org.apache.log4j.ConsoleAppender 4 | #log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 5 | #log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %5p --- [%50t] %-80c(line:%5L) : %m%n 6 | -------------------------------------------------------------------------------- /back/src/main/resources/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |

SpringBoot后端服务已经启动

4 |

SpringBoot backend service is running ......

5 | 6 | -------------------------------------------------------------------------------- /back/src/test/java/com/haust/back/BackApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.haust.back; 2 | 3 | import java.sql.Timestamp; 4 | import java.util.Date; 5 | import java.util.List; 6 | 7 | import org.junit.jupiter.api.Test; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.boot.test.context.SpringBootTest; 10 | 11 | import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; 12 | import com.haust.back.entity.Comment; 13 | import com.haust.back.entity.Newstatus; 14 | import com.haust.back.entity.User; 15 | import com.haust.back.entity.Useroperationnew; 16 | import com.haust.back.mapper.*; 17 | 18 | @SpringBootTest 19 | class BackApplicationTests { 20 | @Autowired 21 | private UserMapper userMapper; 22 | @Autowired 23 | private NewMapper newMapper; 24 | @Autowired 25 | private UseroperationnewMapper useroperationnewMapper; 26 | @Autowired 27 | private CommentMapper commentMapper; 28 | @Autowired 29 | private NewstatusMapper newstatusMapper; 30 | @Test 31 | public void test() { 32 | List users = userMapper.selectList(null); 33 | users.forEach(System.out::println); 34 | } 35 | @Test 36 | public void GetHotNewsTest() { 37 | List hotsnews = useroperationnewMapper.getHotNews(); 38 | hotsnews.forEach(System.out::println); 39 | } 40 | @Test 41 | public void GetCommentTest() { 42 | List comments = commentMapper.selectList(null); 43 | System.out.println(comments); 44 | } 45 | @Test 46 | public void InsertCommentTest() { 47 | Comment comment = new Comment(); 48 | comment.setUserId(1); 49 | comment.setNewId(1); 50 | comment.setCCnt("测试评论"); 51 | java.util.Date date = new Date();//获得当前时间 52 | Timestamp t = new Timestamp(date.getTime());//将时间转换成Timestamp类型,这样便可以存入到Mysql数据库中 53 | comment.setCDate(t); 54 | commentMapper.insert(comment); 55 | System.out.println(comment); 56 | } 57 | @Test 58 | public void CommentTest() 59 | { 60 | List comment=commentMapper.selectList(null); 61 | System.out.println(comment); 62 | } 63 | @Test 64 | public void GetStatusTest() 65 | { 66 | System.out.println(newstatusMapper.selectOne(new QueryWrapper().eq("new_id", 1))); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /front/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | -------------------------------------------------------------------------------- /front/.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": ["Vue.volar", "Vue.vscode-typescript-vue-plugin"] 3 | } 4 | -------------------------------------------------------------------------------- /front/README.md: -------------------------------------------------------------------------------- 1 | # Vue 3 + Vite 2 | 3 | This template should help get you started developing with Vue 3 in Vite. The template uses Vue 3 ` 12 | 13 | 14 | -------------------------------------------------------------------------------- /front/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "front", 3 | "private": true, 4 | "version": "0.0.0", 5 | "type": "module", 6 | "scripts": { 7 | "dev": "vite", 8 | "build": "vite build", 9 | "preview": "vite preview" 10 | }, 11 | "dependencies": { 12 | "@kangc/v-md-editor": "^2.3.15", 13 | "axios": "^1.3.5", 14 | "element-plus": "^2.3.3", 15 | "highlight.js": "^11.7.0", 16 | "prismjs": "^1.29.0", 17 | "vue": "^3.2.47", 18 | "vue-router": "4" 19 | }, 20 | "devDependencies": { 21 | "@vitejs/plugin-vue": "^4.1.0", 22 | "vite": "^4.2.0", 23 | "vite-plugin-windicss": "^1.8.10", 24 | "windicss": "^3.5.6" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /front/public/vite.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /front/src/App.vue: -------------------------------------------------------------------------------- 1 | 3 | 4 | 7 | 8 | 11 | -------------------------------------------------------------------------------- /front/src/api/comments.js: -------------------------------------------------------------------------------- 1 | import axios from '~/util/axios' 2 | export function getcomments(nid,id,size) { 3 | return axios.get('/comment/'+nid+'/'+id+'/'+size) 4 | } 5 | export function getcommentlike(nid) { 6 | return axios.get('/comment/like'+nid) 7 | } 8 | export function getcommentdislike(nid) { 9 | return axios.get('/comment/dislike'+nid) 10 | } 11 | export function postcommentlike(nid) { 12 | return axios.get('/comment/plike'+nid) 13 | } 14 | export function postcommentdislike(nid) { 15 | return axios.get('/comment/pdislike'+nid) 16 | } 17 | export function insertComment(userId,newId,cCnt,uname) { 18 | let formData = new FormData();//可以将form表单直接放入构造器,自动封装 19 | formData.append("userId", userId); 20 | formData.append("newId", newId); 21 | formData.append("cCnt", cCnt); 22 | formData.append("userName",uname) 23 | return axios.post('/comment/insert', formData) 24 | } -------------------------------------------------------------------------------- /front/src/api/news.js: -------------------------------------------------------------------------------- 1 | import axios from '~/util/axios' 2 | export function query(id, size) { 3 | return axios.get('/news/findbypage/' + id + '/' + size) 4 | } 5 | //根据新闻分类id查询新闻 6 | export function querybyid(id, size, newsid) { 7 | return axios.get('/news/' + newsid + '/' + id + '/' + size) 8 | } 9 | //根据新闻日期查新闻 10 | export function querybydate(id, size, newsdate) { 11 | return axios.get('/news/pagebydate/' + newsdate + '/' + id + '/' + size) 12 | } 13 | //根据新闻id查新闻详细信息 14 | export function querybynid(id) { 15 | return axios.get('/newdetail/' + id) 16 | } 17 | //插入一条新闻 18 | export function addnew(newCategory, newTitle, newCnt, newCover, newDate) { 19 | var catorgy = {'国际': 1, '财经': 2,'体育':3,'文化':4,'国内':5,'军事':6,'美食':7,'社会':8,'生活':9,'华人':10} 20 | let formData = new FormData();//可以将form表单直接放入构造器,自动封装 21 | formData.append("newCategory", newCategory); 22 | formData.append("newTitle", newTitle); 23 | formData.append("newCid", catorgy[newCategory]); 24 | formData.append("newCover", newCover); 25 | formData.append("newDate", newDate); 26 | formData.append("newCnt", newCnt); 27 | return axios.post('/news/insert', formData) 28 | } 29 | 30 | //根据用户id查看系统推荐新闻 31 | export function getrecommendnews(id,size) { 32 | return axios.get('/recommend/' + id+'/'+size) 33 | } 34 | 35 | //获取系统的热门新闻 36 | export function gethotnews() { 37 | return axios.get('/recommend/hot') 38 | } -------------------------------------------------------------------------------- /front/src/api/newstatus.js: -------------------------------------------------------------------------------- 1 | import axios from '~/util/axios' 2 | export function getnewstatus(nid) { 3 | return axios.get('/newstatus/status/' + nid) 4 | } 5 | export function addlike(nid,like,dislike,forward) { 6 | let formData = new FormData(); 7 | formData.append('newId', nid); 8 | formData.append('newLike', like); 9 | formData.append('newDislike', dislike); 10 | formData.append('newForward', forward); 11 | return axios.post('/newstatus/like',formData) 12 | } 13 | export function adddislike(nid,like,dislike,forward) { 14 | let formData = new FormData(); 15 | formData.append('newId', nid); 16 | formData.append('newLike', like); 17 | formData.append('newDislike', dislike); 18 | formData.append('newForward', forward); 19 | return axios.post('/newstatus/dislike',formData) 20 | } 21 | export function addforward(nid,like,dislike,forward) { 22 | let formData = new FormData(); 23 | formData.append('newId', nid); 24 | formData.append('newLike', like); 25 | formData.append('newDislike', dislike); 26 | formData.append('newForward', forward); 27 | return axios.post('/newstatus/forward',formData) 28 | } -------------------------------------------------------------------------------- /front/src/api/user.js: -------------------------------------------------------------------------------- 1 | import axios from '~/util/axios' 2 | export function login(userName,userPwd) { 3 | return axios.post('/user/login', { 4 | userName, 5 | userPwd 6 | }) 7 | } 8 | export function adduser(userName,userPwd) { 9 | let formData = new FormData(); 10 | formData.append("userName", userName); 11 | formData.append("userPwd", userPwd); 12 | return axios.post('/user/insert', formData) 13 | } 14 | export function updateuser(formData) { 15 | //可以将form表单直接放入构造器,自动封装 16 | return axios.put('/user/update', formData) 17 | } -------------------------------------------------------------------------------- /front/src/assets/Logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/front/src/assets/Logo.png -------------------------------------------------------------------------------- /front/src/assets/avatar.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/front/src/assets/avatar.jpg -------------------------------------------------------------------------------- /front/src/assets/vue.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /front/src/components/Calender.vue: -------------------------------------------------------------------------------- 1 | 11 | 18 | 23 | -------------------------------------------------------------------------------- /front/src/components/Comment.Vue: -------------------------------------------------------------------------------- 1 | 12 | 53 | 54 | -------------------------------------------------------------------------------- /front/src/components/Reply.vue: -------------------------------------------------------------------------------- 1 | 31 | 32 | 69 | -------------------------------------------------------------------------------- /front/src/components/edit.vue: -------------------------------------------------------------------------------- 1 | 57 | 58 | 153 | -------------------------------------------------------------------------------- /front/src/components/hotnews.vue: -------------------------------------------------------------------------------- 1 | 24 | 25 | 51 | 71 | -------------------------------------------------------------------------------- /front/src/components/news.vue: -------------------------------------------------------------------------------- 1 | 23 | 62 | 63 | -------------------------------------------------------------------------------- /front/src/components/newstatus.vue: -------------------------------------------------------------------------------- 1 | 39 | 40 | 100 | -------------------------------------------------------------------------------- /front/src/components/person.vue: -------------------------------------------------------------------------------- 1 | 38 | 39 | 85 | -------------------------------------------------------------------------------- /front/src/components/public.vue: -------------------------------------------------------------------------------- 1 | 38 | 39 | -------------------------------------------------------------------------------- /front/src/components/singalnew.vue: -------------------------------------------------------------------------------- 1 | 21 | 47 | 48 | -------------------------------------------------------------------------------- /front/src/components/tabs.vue: -------------------------------------------------------------------------------- 1 | 44 | 45 | 66 | -------------------------------------------------------------------------------- /front/src/main.js: -------------------------------------------------------------------------------- 1 | import { createApp } from 'vue' 2 | import ElementPlus from 'element-plus' 3 | import 'element-plus/dist/index.css' 4 | import App from './App.vue' 5 | import router from './router' 6 | import VueMarkdownEditor from '@kangc/v-md-editor'; 7 | import '@kangc/v-md-editor/lib/style/base-editor.css'; 8 | import vuepressTheme from '@kangc/v-md-editor/lib/theme/vuepress.js'; 9 | import '@kangc/v-md-editor/lib/theme/style/vuepress.css'; 10 | import VMdPreview from '@kangc/v-md-editor/lib/preview'; 11 | import '@kangc/v-md-editor/lib/style/preview.css'; 12 | import githubTheme from '@kangc/v-md-editor/lib/theme/github.js'; 13 | import '@kangc/v-md-editor/lib/theme/style/github.css'; 14 | 15 | // highlightjs 16 | import hljs from 'highlight.js'; 17 | 18 | VMdPreview.use(githubTheme, { 19 | Hljs: hljs, 20 | }); 21 | import Prism from 'prismjs'; 22 | 23 | VueMarkdownEditor.use(vuepressTheme, { 24 | Prism, 25 | }); 26 | 27 | const app = createApp(App) 28 | app.config.globalProperties.$avatorurl = 'https://img-oss.langs.ink/2023/04/21/64426202c6c66.webp' 29 | app.config.globalProperties.$uid = 1 30 | app.config.globalProperties.$nid = 1 31 | app.config.globalProperties.$uname = "admin" 32 | app.use(router) 33 | app.use(ElementPlus) 34 | import 'virtual:windi.css' 35 | app.use(VueMarkdownEditor); 36 | app.use(VMdPreview); 37 | app.mount('#app') 38 | -------------------------------------------------------------------------------- /front/src/pages/404.vue: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /front/src/pages/Edit.vue: -------------------------------------------------------------------------------- 1 | 4 | 67 | 68 | 93 | -------------------------------------------------------------------------------- /front/src/pages/NewDate.vue: -------------------------------------------------------------------------------- 1 | 4 | 64 | 65 | 89 | -------------------------------------------------------------------------------- /front/src/pages/Public.vue: -------------------------------------------------------------------------------- 1 | 4 | 67 | 68 | 93 | -------------------------------------------------------------------------------- /front/src/pages/about.vue: -------------------------------------------------------------------------------- 1 | 4 | 67 | 68 | -------------------------------------------------------------------------------- /front/src/pages/index.vue: -------------------------------------------------------------------------------- 1 | 6 | 85 | 86 | 118 | -------------------------------------------------------------------------------- /front/src/pages/login.vue: -------------------------------------------------------------------------------- 1 | 68 | 69 | 136 | -------------------------------------------------------------------------------- /front/src/router/index.js: -------------------------------------------------------------------------------- 1 | import { 2 | createRouter, 3 | createWebHistory 4 | } from 'vue-router' 5 | 6 | import Index from '~/pages/index.vue' 7 | import Login from '~/pages/login.vue' 8 | import NotFound from '~/pages/404.vue' 9 | import About from '~/pages/about.vue' 10 | import Public from '~/pages/Public.vue' 11 | import Edit from '~/pages/edit.vue' 12 | import NewDate from '~/pages/NewDate.vue' 13 | import NewDetail from '~/views/NewDetail.vue' 14 | const routes = [ 15 | { 16 | path: '/', 17 | component: Login 18 | }, 19 | { 20 | path: '/index', 21 | component: Index 22 | }, 23 | { 24 | path: '/about', 25 | component: About 26 | }, { 27 | path: '/public', 28 | component: Public 29 | }, 30 | { 31 | path: '/edit', 32 | component: Edit 33 | }, 34 | { 35 | path: '/newdate/:date', 36 | component: NewDate, 37 | props: true 38 | }, 39 | { 40 | path: '/newsdetail/:id', 41 | component: NewDetail, 42 | props: true 43 | }, 44 | { 45 | path: '/:pathMatch(.*)*', 46 | name: 'NotFound', 47 | component: NotFound 48 | } 49 | ] 50 | 51 | const router = createRouter({ 52 | history: createWebHistory(), 53 | routes: routes 54 | }) 55 | export default router 56 | -------------------------------------------------------------------------------- /front/src/style.css: -------------------------------------------------------------------------------- 1 | :root { 2 | font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif; 3 | line-height: 1.5; 4 | font-weight: 400; 5 | 6 | color-scheme: light dark; 7 | color: rgba(255, 255, 255, 0.87); 8 | background-color: #242424; 9 | 10 | font-synthesis: none; 11 | text-rendering: optimizeLegibility; 12 | -webkit-font-smoothing: antialiased; 13 | -moz-osx-font-smoothing: grayscale; 14 | -webkit-text-size-adjust: 100%; 15 | } 16 | 17 | a { 18 | font-weight: 500; 19 | color: #646cff; 20 | text-decoration: inherit; 21 | } 22 | a:hover { 23 | color: #535bf2; 24 | } 25 | 26 | a { 27 | font-weight: 500; 28 | color: #646cff; 29 | text-decoration: inherit; 30 | } 31 | a:hover { 32 | color: #535bf2; 33 | } 34 | 35 | body { 36 | margin: 0; 37 | display: flex; 38 | place-items: center; 39 | min-width: 320px; 40 | min-height: 100vh; 41 | } 42 | 43 | h1 { 44 | font-size: 3.2em; 45 | line-height: 1.1; 46 | } 47 | 48 | button { 49 | border-radius: 8px; 50 | border: 1px solid transparent; 51 | padding: 0.6em 1.2em; 52 | font-size: 1em; 53 | font-weight: 500; 54 | font-family: inherit; 55 | background-color: #1a1a1a; 56 | cursor: pointer; 57 | transition: border-color 0.25s; 58 | } 59 | button:hover { 60 | border-color: #646cff; 61 | } 62 | button:focus, 63 | button:focus-visible { 64 | outline: 4px auto -webkit-focus-ring-color; 65 | } 66 | 67 | .card { 68 | padding: 2em; 69 | } 70 | 71 | #app { 72 | max-width: 1280px; 73 | margin: 0 auto; 74 | padding: 2rem; 75 | text-align: center; 76 | } 77 | 78 | @media (prefers-color-scheme: light) { 79 | :root { 80 | color: #213547; 81 | background-color: #ffffff; 82 | } 83 | a:hover { 84 | color: #747bff; 85 | } 86 | button { 87 | background-color: #f9f9f9; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /front/src/util/axios.js: -------------------------------------------------------------------------------- 1 | import axios from 'axios' 2 | 3 | const service = axios.create({ 4 | baseURL: "/api", 5 | timeout: 5000 6 | }) 7 | export default service -------------------------------------------------------------------------------- /front/src/views/China.vue: -------------------------------------------------------------------------------- 1 | 20 | 59 | 60 | -------------------------------------------------------------------------------- /front/src/views/Culture.vue: -------------------------------------------------------------------------------- 1 | 20 | 59 | 60 | -------------------------------------------------------------------------------- /front/src/views/Huaren.vue: -------------------------------------------------------------------------------- 1 | 20 | 62 | 63 | -------------------------------------------------------------------------------- /front/src/views/Life.vue: -------------------------------------------------------------------------------- 1 | 20 | 62 | 63 | -------------------------------------------------------------------------------- /front/src/views/NewDetail.vue: -------------------------------------------------------------------------------- 1 | 7 | 74 | 75 | 100 | -------------------------------------------------------------------------------- /front/src/views/Recommend.vue: -------------------------------------------------------------------------------- 1 | 20 | 59 | 60 | -------------------------------------------------------------------------------- /front/src/views/Sport.vue: -------------------------------------------------------------------------------- 1 | 20 | 62 | 63 | -------------------------------------------------------------------------------- /front/src/views/datenew.vue: -------------------------------------------------------------------------------- 1 | 20 | 59 | 60 | -------------------------------------------------------------------------------- /front/src/views/finance.vue: -------------------------------------------------------------------------------- 1 | 20 | 62 | 63 | -------------------------------------------------------------------------------- /front/src/views/picture.vue: -------------------------------------------------------------------------------- 1 | 20 | 62 | 63 | -------------------------------------------------------------------------------- /front/src/views/society.vue: -------------------------------------------------------------------------------- 1 | 20 | 62 | 63 | -------------------------------------------------------------------------------- /front/src/views/technology.vue: -------------------------------------------------------------------------------- 1 | 20 | 62 | 63 | -------------------------------------------------------------------------------- /front/src/views/video.vue: -------------------------------------------------------------------------------- 1 | 20 | 61 | 62 | -------------------------------------------------------------------------------- /front/vite.config.js: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vite' 2 | import vue from '@vitejs/plugin-vue' 3 | import WindiCSS from 'vite-plugin-windicss' 4 | import path from "path" 5 | 6 | 7 | export default defineConfig({ 8 | resolve: { 9 | alias:{ 10 | "~": path.resolve(__dirname, "src"), 11 | } 12 | }, 13 | server: { 14 | proxy: { 15 | '/api': { 16 | target: 'http://localhost:8080', 17 | changeOrigin: true, 18 | rewrite: (path) => path.replace(/^\/api/, '') 19 | } 20 | }}, 21 | plugins: [vue(),WindiCSS()] 22 | }) 23 | -------------------------------------------------------------------------------- /recommender/.idea/codeStyles/Project.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | -------------------------------------------------------------------------------- /recommender/.idea/codeStyles/codeStyleConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | -------------------------------------------------------------------------------- /recommender/.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /recommender/.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /recommender/.idea/jarRepositories.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 9 | 10 | 14 | 15 | 19 | 20 | -------------------------------------------------------------------------------- /recommender/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /recommender/.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /recommender/.idea/uiDesigner.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | -------------------------------------------------------------------------------- /recommender/.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /recommender/.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 65 | 66 | 68 | 69 | 71 | 72 | 79 | 80 | 81 | 84 | 85 | 87 | 88 | 89 | 90 | 91 | 92 | 95 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 133 | 134 | 135 | 136 | 137 | 143 | 144 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 1684322203182 168 | 183 | 184 | 185 | 186 | 188 | -------------------------------------------------------------------------------- /recommender/alsMoudle/data/product/._SUCCESS.crc: -------------------------------------------------------------------------------- 1 | crc -------------------------------------------------------------------------------- /recommender/alsMoudle/data/product/.part-00000-05b11672-a325-428f-826e-fcc8ea268fbc-c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/alsMoudle/data/product/.part-00000-05b11672-a325-428f-826e-fcc8ea268fbc-c000.snappy.parquet.crc -------------------------------------------------------------------------------- /recommender/alsMoudle/data/product/_SUCCESS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/alsMoudle/data/product/_SUCCESS -------------------------------------------------------------------------------- /recommender/alsMoudle/data/product/part-00000-05b11672-a325-428f-826e-fcc8ea268fbc-c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/alsMoudle/data/product/part-00000-05b11672-a325-428f-826e-fcc8ea268fbc-c000.snappy.parquet -------------------------------------------------------------------------------- /recommender/alsMoudle/data/user/._SUCCESS.crc: -------------------------------------------------------------------------------- 1 | crc -------------------------------------------------------------------------------- /recommender/alsMoudle/data/user/.part-00000-0bf4a94e-a5af-4231-92fc-65f85011387a-c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/alsMoudle/data/user/.part-00000-0bf4a94e-a5af-4231-92fc-65f85011387a-c000.snappy.parquet.crc -------------------------------------------------------------------------------- /recommender/alsMoudle/data/user/_SUCCESS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/alsMoudle/data/user/_SUCCESS -------------------------------------------------------------------------------- /recommender/alsMoudle/data/user/part-00000-0bf4a94e-a5af-4231-92fc-65f85011387a-c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/alsMoudle/data/user/part-00000-0bf4a94e-a5af-4231-92fc-65f85011387a-c000.snappy.parquet -------------------------------------------------------------------------------- /recommender/alsMoudle/metadata/._SUCCESS.crc: -------------------------------------------------------------------------------- 1 | crc -------------------------------------------------------------------------------- /recommender/alsMoudle/metadata/.part-00000.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/alsMoudle/metadata/.part-00000.crc -------------------------------------------------------------------------------- /recommender/alsMoudle/metadata/_SUCCESS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/alsMoudle/metadata/_SUCCESS -------------------------------------------------------------------------------- /recommender/alsMoudle/metadata/part-00000: -------------------------------------------------------------------------------- 1 | {"class":"org.apache.spark.mllib.recommendation.MatrixFactorizationModel","version":"1.0","rank":4} 2 | -------------------------------------------------------------------------------- /recommender/als_rating_test.txt: -------------------------------------------------------------------------------- 1 | 101#1001#4.0 2 | 101#1002#2.5 3 | 101#1004#3.0 4 | 101#1007#1.5 5 | 101#1010#4.0 6 | 101#1016#3.5 7 | 101#1022#4.0 8 | 102#1002#2.5 9 | 102#1003#1.0 10 | 102#1004#3.5 11 | 102#1006#2.0 12 | 102#1009#2.5 13 | 102#1011#4.0 14 | 102#1013#3.5 15 | 102#1015#4.0 16 | 102#1017#4.5 -------------------------------------------------------------------------------- /recommender/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | groupId 8 | recommender 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 8 13 | 8 14 | UTF-8 15 | 2.11.0 16 | 2.4.5 17 | 18 | 19 | 20 | 21 | 22 | org.apache.spark 23 | spark-core_2.12 24 | ${spark.version} 25 | 26 | 27 | org.apache.spark 28 | spark-sql_2.12 29 | ${spark.version} 30 | 31 | 32 | org.apache.spark 33 | spark-mllib_2.12 34 | ${spark.version} 35 | 36 | 37 | 38 | org.apache.hadoop 39 | hadoop-client 40 | 2.7.5 41 | 42 | 43 | 44 | org.apache.hadoop 45 | hadoop-hdfs 46 | 2.7.3 47 | 48 | 49 | org.apache.hadoop 50 | hadoop-common 51 | 2.7.3 52 | 53 | 54 | mysql 55 | mysql-connector-java 56 | 5.1.47 57 | 58 | 59 | -------------------------------------------------------------------------------- /recommender/recommender.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /recommender/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootCategory=ERROR -------------------------------------------------------------------------------- /recommender/src/main/scala/Main.scala: -------------------------------------------------------------------------------- 1 | import org.apache.spark.mllib.recommendation.Rating 2 | 3 | import java.sql.{Connection, DriverManager} 4 | import org.apache.spark.rdd.{JdbcRDD, RDD} 5 | import org.apache.spark.{SparkConf, SparkContext} 6 | object Main { 7 | def main(args: Array[String]): Unit = { 8 | // 定义配置文件 9 | val conf = new SparkConf().setAppName("mysql").setMaster("local") 10 | // 通过配置文件实例spark上下文对象 11 | val sc = new SparkContext(conf) 12 | // 定义连接器 13 | def getCon(): Connection = { 14 | DriverManager.getConnection("jdbc:mysql://localhost:3306/News?useSSL=false&useUnicode=true&characterEncoding=utf-8&serverTimezone=UTC", "root", "010619HYy") 15 | } 16 | //定义SQL 用来读取数据 username, birthday, sex, address 17 | var sql = "SELECT user_id,new_id,SUM( CASE operation_type WHEN 0 THEN 1 WHEN 1 THEN ? WHEN 2 THEN ? ELSE 0 END ) AS value FROM useroperationnew GROUP BY user_id,new_id;" 18 | // 实例 JdbcRDD 进行读取数据 19 | val ratings = new JdbcRDD(sc, getCon, sql, 3,5,3,re => { 20 | Rating(re.getInt(1).toInt,re.getInt(2).toInt,re.getInt(3).toDouble) 21 | }).collect() 22 | // 结果收集 23 | var data = sc.parallelize(ratings) 24 | data.foreach(println) 25 | } 26 | } -------------------------------------------------------------------------------- /recommender/src/main/scala/Test.scala: -------------------------------------------------------------------------------- 1 | import org.apache.log4j.{Level, Logger} 2 | import org.apache.spark.{SparkConf, SparkContext} 3 | import org.apache.spark.mllib.recommendation.MatrixFactorizationModel 4 | 5 | object Test { 6 | //屏蔽不必要的日志显示在终端上 7 | Logger.getLogger("org").setLevel(Level.ERROR) 8 | 9 | def main(args: Array[String]): Unit = { 10 | //给用户推荐 11 | val conf = new SparkConf().setMaster("local").setAppName("recommend_test") 12 | val sc = new SparkContext(conf) 13 | 14 | val myModelPath = "alsMoudle" 15 | val model = MatrixFactorizationModel.load(sc, myModelPath) 16 | 17 | val recommendProducts = model.recommendProducts(1, 10) 18 | for (r <- recommendProducts) { 19 | println("给用户id:"+r.user+" 推荐新闻id为 "+r.product) 20 | } 21 | val userfeatures = model.userFeatures 22 | println("用户特征矩阵") 23 | println("user_id\\feature f1 f2 f3 f4 ") 24 | for (uf <- userfeatures) { 25 | print("用户id:" + uf._1 + " : ") 26 | for (value <- uf._2) { 27 | print(value + " ") 28 | } 29 | println() 30 | } 31 | val itemfeatures = model.productFeatures 32 | println("新闻特征矩阵") 33 | println("new_id\\feature f1 f2 f3 f4 ") 34 | for (uf <- itemfeatures) { 35 | print("新闻id:" + uf._1 + " : ") 36 | for (value <- uf._2) { 37 | print(value + " ") 38 | } 39 | println() 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /recommender/src/main/scala/Train.scala: -------------------------------------------------------------------------------- 1 | import java.io.File 2 | import org.apache.log4j.{Level, Logger} 3 | import org.apache.spark.{SparkConf, SparkContext} 4 | import org.apache.spark.mllib.recommendation.{ALS, Rating} 5 | import org.apache.spark.rdd.{JdbcRDD, RDD} 6 | 7 | import java.sql.{Connection, DriverManager} 8 | 9 | object Train { 10 | //屏蔽不必要的日志显示在终端上 11 | Logger.getLogger("org").setLevel(Level.WARN) 12 | 13 | def main(args: Array[String]): Unit = { 14 | // 配置Spark 15 | val conf = new SparkConf().setMaster("local").setAppName("alsMoudleTrain") 16 | val sc = new SparkContext(conf) 17 | // 定义模型 18 | val myModelPath = "alsMoudle" 19 | // 获取数据 20 | def getCon(): Connection = { 21 | DriverManager.getConnection("jdbc:mysql://localhost:3306/News?useSSL=false&useUnicode=true&characterEncoding=utf-8&serverTimezone=UTC", "root", "010619HYy") 22 | } 23 | //定义SQL 用来读取数据 user_id,new_id,value 24 | var sql = "SELECT user_id,new_id,SUM( CASE operation_type " + 25 | "WHEN 0 THEN 1 WHEN 1 THEN ? WHEN 2 THEN ? ELSE 0 END ) " + 26 | "AS value FROM useroperationnew GROUP BY user_id,new_id;" 27 | // 实例 JdbcRDD 进行读取Mysql数据 28 | val data = new JdbcRDD(sc, getCon, sql, 3, 5, 3, re => { 29 | Rating(re.getInt(1).toInt, re.getInt(2).toInt, re.getInt(3).toDouble) 30 | }).collect() 31 | var ratings = sc.parallelize(data) 32 | ratings.filter(x => x.user == 101).foreach(println) 33 | // 训练模型 34 | val rank = 4 35 | val numIterations = 10 36 | val model = ALS.train(ratings, rank, numIterations, 0.01) 37 | val recommendProducts = model.recommendProducts(1, 10) 38 | val userfeatures =model.userFeatures 39 | println("打印用户特征矩阵") 40 | for (uf <- userfeatures) { 41 | print("用户id:"+uf._1 + " : ") 42 | for (value <- uf._2) { 43 | print(value + " ") 44 | } 45 | println() 46 | } 47 | val itemfeatures =model.productFeatures 48 | println("打印新闻特征矩阵") 49 | for (uf <- itemfeatures) { 50 | print("新闻id:"+uf._1 + " : ") 51 | for (value <- uf._2) { 52 | print(value + " ") 53 | } 54 | println() 55 | } 56 | val path: File = new File(myModelPath) 57 | dirDel(path) //删除原模型保存的文件,不删除新模型保存会报错 58 | //保存模型 59 | model.save(sc, myModelPath) 60 | sc.stop() 61 | } 62 | 63 | //删除模型目录和文件 64 | def dirDel(path: File) { 65 | if (!path.exists()) 66 | return 67 | else if (path.isFile) { 68 | path.delete() 69 | return 70 | } 71 | val file: Array[File] = path.listFiles() 72 | for (d <- file) { 73 | dirDel(d) 74 | } 75 | path.delete() 76 | } 77 | } -------------------------------------------------------------------------------- /recommender/target/classes/Main$$anonfun$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Main$$anonfun$1.class -------------------------------------------------------------------------------- /recommender/target/classes/Main$$anonfun$2.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Main$$anonfun$2.class -------------------------------------------------------------------------------- /recommender/target/classes/Main$$anonfun$main$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Main$$anonfun$main$1.class -------------------------------------------------------------------------------- /recommender/target/classes/Main$.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Main$.class -------------------------------------------------------------------------------- /recommender/target/classes/Main.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Main.class -------------------------------------------------------------------------------- /recommender/target/classes/Test$$anonfun$main$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Test$$anonfun$main$1.class -------------------------------------------------------------------------------- /recommender/target/classes/Test$$anonfun$main$2$$anonfun$apply$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Test$$anonfun$main$2$$anonfun$apply$1.class -------------------------------------------------------------------------------- /recommender/target/classes/Test$$anonfun$main$2.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Test$$anonfun$main$2.class -------------------------------------------------------------------------------- /recommender/target/classes/Test$$anonfun$main$3$$anonfun$apply$2.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Test$$anonfun$main$3$$anonfun$apply$2.class -------------------------------------------------------------------------------- /recommender/target/classes/Test$$anonfun$main$3.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Test$$anonfun$main$3.class -------------------------------------------------------------------------------- /recommender/target/classes/Test$.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Test$.class -------------------------------------------------------------------------------- /recommender/target/classes/Test.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Test.class -------------------------------------------------------------------------------- /recommender/target/classes/Train$$anonfun$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Train$$anonfun$1.class -------------------------------------------------------------------------------- /recommender/target/classes/Train$$anonfun$2.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Train$$anonfun$2.class -------------------------------------------------------------------------------- /recommender/target/classes/Train$$anonfun$dirDel$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Train$$anonfun$dirDel$1.class -------------------------------------------------------------------------------- /recommender/target/classes/Train$$anonfun$main$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Train$$anonfun$main$1.class -------------------------------------------------------------------------------- /recommender/target/classes/Train$$anonfun$main$2.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Train$$anonfun$main$2.class -------------------------------------------------------------------------------- /recommender/target/classes/Train$$anonfun$main$3$$anonfun$apply$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Train$$anonfun$main$3$$anonfun$apply$1.class -------------------------------------------------------------------------------- /recommender/target/classes/Train$$anonfun$main$3.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Train$$anonfun$main$3.class -------------------------------------------------------------------------------- /recommender/target/classes/Train$$anonfun$main$4$$anonfun$apply$2.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Train$$anonfun$main$4$$anonfun$apply$2.class -------------------------------------------------------------------------------- /recommender/target/classes/Train$$anonfun$main$4.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Train$$anonfun$main$4.class -------------------------------------------------------------------------------- /recommender/target/classes/Train$.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Train$.class -------------------------------------------------------------------------------- /recommender/target/classes/Train.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/recommender/target/classes/Train.class -------------------------------------------------------------------------------- /recommender/target/classes/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootCategory=ERROR -------------------------------------------------------------------------------- /spider/.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Editor-based HTTP Client requests 5 | /httpRequests/ 6 | # Datasource local storage ignored files 7 | /dataSources/ 8 | /dataSources.local.xml 9 | -------------------------------------------------------------------------------- /spider/.idea/dataSources.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | mysql.8 6 | true 7 | com.mysql.cj.jdbc.Driver 8 | jdbc:mysql://47.120.36.144:3306 9 | $ProjectFileDir$ 10 | 11 | 12 | -------------------------------------------------------------------------------- /spider/.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | -------------------------------------------------------------------------------- /spider/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /spider/.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /spider/.idea/spider.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /spider/.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /spider/cnew_data.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ccoz/NewsRecommend/c7a4857a70ce072542f3e86685f059b8bed169b1/spider/cnew_data.xlsx -------------------------------------------------------------------------------- /spider/cnew_url.txt: -------------------------------------------------------------------------------- 1 | https://www.chinanews.com.cn/scroll-news/2023/0406/news.shtml 2 | https://www.chinanews.com.cn/scroll-news/2023/0407/news.shtml 3 | https://www.chinanews.com.cn/scroll-news/2023/0408/news.shtml 4 | https://www.chinanews.com.cn/scroll-news/2023/0409/news.shtml 5 | https://www.chinanews.com.cn/scroll-news/2023/0410/news.shtml 6 | -------------------------------------------------------------------------------- /spider/cnew_url1.txt: -------------------------------------------------------------------------------- 1 | https://www.chinanews.com.cn/tp/hd2011/2023/04-06/1063679.shtmlhttps://www.chinanews.com.cn//gn/2023/04-06/9985429.shtmlhttps://www.chinanews.com.cn//gn/2023/04-06/9985376.shtmlhttps://www.chinanews.com.cn//sh/2023/04-06/9985347.shtmlhttps://www.chinanews.com.cn//gn/2023/04-06/9985297.shtmlhttps://www.chinanews.com.cn//gn/2023/04-06/9985270.shtmlhttps://www.chinanews.com.cn//life/2023/04-06/9985247.shtmlhttps://www.chinanews.com.cn//gn/2023/04-06/9985230.shtmlhttps://www.chinanews.com.cn//life/2023/04-06/9985173.shtml -------------------------------------------------------------------------------- /spider/main.py: -------------------------------------------------------------------------------- 1 | import openpyxl 2 | import requests 3 | import pymysql 4 | from lxml import etree 5 | from tqdm import tqdm 6 | 7 | headers = { 8 | 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36' 9 | } 10 | 11 | 12 | # 组合日期链接 13 | def cnew_url(): 14 | f = open(r'C:/Code/NewsRecommend/spider/cnew_url.txt', 'w', encoding='utf8') 15 | for i in range(6, 11): 16 | if i < 10: 17 | url = 'https://www.chinanews.com.cn/scroll-news/2023/040' + str(i) + '/news.shtml' 18 | else: 19 | url = 'https://www.chinanews.com.cn/scroll-news/2023/04' + str(i) + '/news.shtml' 20 | f.write(url + '\n') 21 | f.close() 22 | 23 | 24 | # step 1.2 写入数据库 25 | def saveToDB(new_cid, new_category, new_title, new_cnt, new_url, new_date): 26 | # mysql连接配置 27 | cur = pymysql.connect(user='root', 28 | password="010619HYy", 29 | host='localhost', 30 | database='News') 31 | cursor = cur.cursor() # 游标 32 | # print(new_cid+new_category+new_title+new_cnt+new_url+new_date) 33 | sql = f"insert into news (new_cid,new_category,new_title,new_cnt,new_url,new_date) values ('{new_cid}','{new_category}','{new_title}','{new_cnt}','{new_url}','{new_date}')" 34 | # sql插入语句 35 | # 操作捕捉异常,如果没有异常则继续执行,如若有则抛出异常 36 | try: 37 | cursor.execute(sql) # 执行sql语句 38 | cur.commit() # 提交至数据库 39 | except Exception as e: 40 | print(e) 41 | print(sql) 42 | 43 | 44 | def cnew_data(): 45 | f = open(r'C:/Code/NewsRecommend/spider/cnew_url.txt', encoding='utf8') # 读取上面已经组合好的链接 46 | l = openpyxl.load_workbook(r'C:/Code/NewsRecommend/spider/cnew_data.xlsx') 47 | sheet = l.active 48 | m = open(r'C:/Code/NewsRecommend/spider/cnew_url1.txt', 'a', encoding='utf8') # 保存报错的链接 49 | x = 1 # 从Excel的第几行开始写入 50 | for i in f: 51 | lj1 = [] 52 | # 发起请求,获取页面里面的新闻链接 53 | req = requests.get(i.replace('\n', ''), headers=headers) 54 | # 设置网页编码,不设置会乱码 55 | req.encoding = 'utf8' 56 | ht = etree.HTML(req.text) 57 | # 获取分类的数据还有正文链接 58 | fl = ht.xpath("//div[@class='dd_lm']/a/text()") 59 | lj = ht.xpath("//div[@class='dd_bt']/a/@href") 60 | # 链接有两种格式,分别组合成可以用的 61 | for j in lj: 62 | if j[:5] == '//www': 63 | lj1.append('https:' + j) 64 | else: 65 | lj1.append('https://www.chinanews.com.cn/' + j) 66 | n = 0 67 | for k in tqdm(lj1): 68 | try: 69 | data = [] 70 | reqs = requests.get(k, headers=headers, timeout=10) 71 | reqs.encoding = 'utf8' 72 | ht1 = etree.HTML(reqs.text) 73 | bt = ht1.xpath("//h1[@class='content_left_title']/text()") # 标题 74 | if bt: 75 | data.append([fl[n]]) 76 | data.append(ht1.xpath("//h1[@class='content_left_title']/text()")) # 标题 77 | data.append(ht1.xpath("//div[@class='left_zw']/p/text()")) # 简介 78 | data.append([lj1[n]]) 79 | else: 80 | data.append([fl[n]]) 81 | data.append(ht1.xpath("//div[@class='content_title']/div[@class='title']/text()")) 82 | data.append(ht1.xpath("//div[@class='content_desc']/p/text()")) # 简介 83 | data.append([lj1[n]]) 84 | 85 | catorgy = {'国际': 1, '财经': 2, '体育': 3, '文化': 4, '国内': 5, '视频': 6, '图片': 7, '社会': 8, 86 | '生活': 9, '华人': 10} 87 | data[2][0] = data[2][0].strip() 88 | # print(data) 89 | list = data[3][0].split('/') 90 | if data[2][0] != "": 91 | if 10 >= catorgy[data[0][0]] >= 1: 92 | 93 | saveToDB(catorgy[data[0][0]], data[0][0], data[1][0], data[2][0], data[3][0], 94 | "2023-" + list[-2]) 95 | else: 96 | saveToDB(0, data[0][0], data[1][0], data[2][0], data[3][0], "2023-" + list[-2]) 97 | for y in range(len(data)): 98 | sheet.cell(x, y + 1).value = '\n'.join(data[y]) 99 | x += 1 100 | n += 1 101 | except Exception as arr: 102 | m.write(lj1[n]) 103 | continue 104 | l.save(r'C:/Code/NewsRecommend/spider/cnew_data.xlsx') 105 | f.close() 106 | m.close() 107 | 108 | 109 | if __name__ == '__main__': 110 | cnew_url() 111 | cnew_data() 112 | --------------------------------------------------------------------------------