├── .idea ├── .name ├── encodings.xml ├── compiler.xml ├── misc.xml └── uiDesigner.xml ├── logs ├── myLog.log.2018-06-11.0.gz ├── myLog.log.2018-06-11.1.gz ├── myLog.log.2018-06-11.2.gz ├── myLog.log.2018-06-11.3.gz ├── myLog.log.2018-06-11.4.gz ├── myLog.log.2018-06-11.5.gz ├── myLog.log.2018-06-11.6.gz ├── myLog.log.2018-06-11.7.gz └── myLog.log.2018-06-11.8.gz ├── src ├── main │ ├── java │ │ └── com │ │ │ └── netMusic │ │ │ ├── service │ │ │ ├── IBaseService.java │ │ │ ├── ISingerService.java │ │ │ ├── ISongService.java │ │ │ ├── IRecordService.java │ │ │ ├── ISingerSongService.java │ │ │ └── impl │ │ │ │ ├── SingerService.java │ │ │ │ ├── RecordService.java │ │ │ │ ├── SingerSongService.java │ │ │ │ ├── SongService.java │ │ │ │ └── XunMengService.java │ │ │ ├── entity │ │ │ ├── SingerSong.java │ │ │ ├── SingerSongKey.java │ │ │ ├── custom │ │ │ │ ├── Msg.java │ │ │ │ ├── Result.java │ │ │ │ ├── Artist.java │ │ │ │ ├── Artists.java │ │ │ │ ├── Album.java │ │ │ │ └── Songs.java │ │ │ ├── Msg.java │ │ │ ├── Result.java │ │ │ ├── Singer.java │ │ │ ├── custom2 │ │ │ │ ├── Artists.java │ │ │ │ ├── Album.java │ │ │ │ ├── Privilege.java │ │ │ │ └── SongMsg.java │ │ │ ├── Song.java │ │ │ ├── Record.java │ │ │ ├── SingerSongExample.java │ │ │ └── SingerExample.java │ │ │ ├── controller │ │ │ ├── HelloController.java │ │ │ └── XunMengController.java │ │ │ ├── DemoApplication.java │ │ │ ├── utils │ │ │ ├── BaseUtil.java │ │ │ ├── CharacterUtils.java │ │ │ ├── FileControlUtil.java │ │ │ ├── HttpClientUtil.java │ │ │ └── XunMengUtil.java │ │ │ ├── dao │ │ │ ├── SongMapper.java │ │ │ ├── RecordMapper.java │ │ │ ├── SingerMapper.java │ │ │ └── SingerSongMapper.java │ │ │ ├── thread │ │ │ ├── XunMengRunnable.java │ │ │ ├── SpiderSingerRunnable.java │ │ │ ├── SpiderRecordRunnable.java │ │ │ └── SpiderSongRunnnable.java │ │ │ ├── Demo │ │ │ ├── NetMusicDemo.java │ │ │ ├── HtmlUnitDemo.java │ │ │ ├── HttpClientDemo.java │ │ │ ├── SimulationLoginDemo.java │ │ │ ├── SingerGrabDemo.java │ │ │ ├── DocumentDemo.java │ │ │ ├── HttpClientUtil.java │ │ │ └── HttpClientDemo2.java │ │ │ ├── listener │ │ │ └── SaveDataListener.java │ │ │ └── spider │ │ │ └── NetMusicGrab.java │ └── resources │ │ ├── application.properties │ │ ├── generator │ │ └── generatorConfig.xml │ │ └── mapper │ │ ├── SingerSongMapper.xml │ │ ├── SingerMapper.xml │ │ ├── SongMapper.xml │ │ └── RecordMapper.xml └── test │ └── java │ ├── TestGson.java │ └── TestRunnable.java ├── pom.xml └── README.md /.idea/.name: -------------------------------------------------------------------------------- 1 | ReptilianDemo -------------------------------------------------------------------------------- /logs/myLog.log.2018-06-11.0.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yijiajia/ReptilianDemo/HEAD/logs/myLog.log.2018-06-11.0.gz -------------------------------------------------------------------------------- /logs/myLog.log.2018-06-11.1.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yijiajia/ReptilianDemo/HEAD/logs/myLog.log.2018-06-11.1.gz -------------------------------------------------------------------------------- /logs/myLog.log.2018-06-11.2.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yijiajia/ReptilianDemo/HEAD/logs/myLog.log.2018-06-11.2.gz -------------------------------------------------------------------------------- /logs/myLog.log.2018-06-11.3.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yijiajia/ReptilianDemo/HEAD/logs/myLog.log.2018-06-11.3.gz -------------------------------------------------------------------------------- /logs/myLog.log.2018-06-11.4.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yijiajia/ReptilianDemo/HEAD/logs/myLog.log.2018-06-11.4.gz -------------------------------------------------------------------------------- /logs/myLog.log.2018-06-11.5.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yijiajia/ReptilianDemo/HEAD/logs/myLog.log.2018-06-11.5.gz -------------------------------------------------------------------------------- /logs/myLog.log.2018-06-11.6.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yijiajia/ReptilianDemo/HEAD/logs/myLog.log.2018-06-11.6.gz -------------------------------------------------------------------------------- /logs/myLog.log.2018-06-11.7.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yijiajia/ReptilianDemo/HEAD/logs/myLog.log.2018-06-11.7.gz -------------------------------------------------------------------------------- /logs/myLog.log.2018-06-11.8.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yijiajia/ReptilianDemo/HEAD/logs/myLog.log.2018-06-11.8.gz -------------------------------------------------------------------------------- /.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/service/IBaseService.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.service; 2 | 3 | 4 | public interface IBaseService { 5 | 6 | void insertEntity(T entity); 7 | 8 | void updateEntityById(T entity); 9 | 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/service/ISingerService.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.service; 2 | 3 | import com.netMusic.entity.Singer; 4 | 5 | public interface ISingerService extends IBaseService { 6 | 7 | 8 | Singer selectEntityById(Integer id); 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/service/ISongService.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.service; 2 | 3 | import com.netMusic.entity.Song; 4 | 5 | public interface ISongService extends IBaseService{ 6 | 7 | public Song selectEntityById(Long id); 8 | 9 | 10 | 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/service/IRecordService.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.service; 2 | 3 | import com.netMusic.entity.Record; 4 | 5 | public interface IRecordService extends IBaseService{ 6 | 7 | public Record selectEntityById(Integer id); 8 | 9 | 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/entity/SingerSong.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.entity; 2 | 3 | public class SingerSong extends SingerSongKey { 4 | private String intro; 5 | 6 | public String getIntro() { 7 | return intro; 8 | } 9 | 10 | public void setIntro(String intro) { 11 | this.intro = intro; 12 | } 13 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/service/ISingerSongService.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.service; 2 | 3 | import com.netMusic.entity.SingerSong; 4 | import com.netMusic.entity.SingerSongKey; 5 | 6 | public interface ISingerSongService extends IBaseService { 7 | 8 | public SingerSong selectEntityById(SingerSongKey singerSongKey); 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/controller/HelloController.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.controller; 2 | import org.springframework.web.bind.annotation.RequestMapping; 3 | import org.springframework.web.bind.annotation.RestController; 4 | 5 | @RestController 6 | public class HelloController { 7 | 8 | 9 | @RequestMapping("/hello") 10 | public String hello(){ 11 | 12 | return "hello"; 13 | } 14 | 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.datasource.driver-class-name=com.mysql.jdbc.Driver 2 | spring.datasource.url=jdbc:mysql://localhost:3306/netmusic 3 | spring.datasource.username=root 4 | spring.datasource.password=root 5 | 6 | spring.jpa.hibernate.ddl-auto=update 7 | spring.jpa.show-sql=true 8 | spring.jpa.open-in-view=true 9 | 10 | 11 | mybatis.mapper-locations=classpath*:mapper/*.xml 12 | logging.level.mapper=debug 13 | logging.file=logs/myLog.log -------------------------------------------------------------------------------- /src/main/java/com/netMusic/entity/SingerSongKey.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.entity; 2 | 3 | public class SingerSongKey { 4 | private Integer singerId; 5 | 6 | private Integer songId; 7 | 8 | public Integer getSingerId() { 9 | return singerId; 10 | } 11 | 12 | public void setSingerId(Integer singerId) { 13 | this.singerId = singerId; 14 | } 15 | 16 | public Integer getSongId() { 17 | return songId; 18 | } 19 | 20 | public void setSongId(Integer songId) { 21 | this.songId = songId; 22 | } 23 | } -------------------------------------------------------------------------------- /.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/entity/custom/Msg.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 bejson.com 3 | */ 4 | package com.netMusic.entity.custom; 5 | 6 | /** 7 | * Auto-generated: 2018-06-06 20:59:46 8 | * 9 | * @author bejson.com (i@bejson.com) 10 | * @website http://www.bejson.com/java2pojo/ 11 | */ 12 | public class Msg { 13 | 14 | private Result result; 15 | private int code; 16 | public void setResult(Result result) { 17 | this.result = result; 18 | } 19 | public Result getResult() { 20 | return result; 21 | } 22 | 23 | public void setCode(int code) { 24 | this.code = code; 25 | } 26 | public int getCode() { 27 | return code; 28 | } 29 | 30 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/entity/Msg.java: -------------------------------------------------------------------------------- 1 | 2 | package com.netMusic.entity; 3 | 4 | /** 5 | * Auto-generated: 2018-06-06 14:1:37 6 | */ 7 | public class Msg { 8 | 9 | private Result result; 10 | private int code; 11 | public void setResult(Result result) { 12 | this.result = result; 13 | } 14 | public Result getResult() { 15 | return result; 16 | } 17 | 18 | public void setCode(int code) { 19 | this.code = code; 20 | } 21 | public int getCode() { 22 | return code; 23 | } 24 | 25 | @Override 26 | public String toString() { 27 | return "Msg{" + 28 | "result=" + result + 29 | ", code=" + code + 30 | '}'; 31 | } 32 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/entity/custom/Result.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 bejson.com 3 | */ 4 | package com.netMusic.entity.custom; 5 | import java.util.List; 6 | 7 | /** 8 | * Auto-generated: 2018-06-06 20:59:46 9 | * 10 | * @author bejson.com (i@bejson.com) 11 | * @website http://www.bejson.com/java2pojo/ 12 | */ 13 | public class Result { 14 | 15 | private List songs; 16 | private int songCount; 17 | public void setSongs(List songs) { 18 | this.songs = songs; 19 | } 20 | public List getSongs() { 21 | return songs; 22 | } 23 | 24 | public void setSongCount(int songCount) { 25 | this.songCount = songCount; 26 | } 27 | public int getSongCount() { 28 | return songCount; 29 | } 30 | 31 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/DemoApplication.java: -------------------------------------------------------------------------------- 1 | package com.netMusic; 2 | 3 | import org.mybatis.spring.annotation.MapperScan; 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.SpringBootApplication; 6 | import org.springframework.boot.web.servlet.ServletComponentScan; 7 | import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; 8 | import org.springframework.context.annotation.ComponentScan; 9 | 10 | @SpringBootApplication 11 | @MapperScan("com.netMusic.dao") 12 | @ServletComponentScan 13 | @ComponentScan 14 | public class DemoApplication extends SpringBootServletInitializer { 15 | 16 | public static void main(String[] args) { 17 | SpringApplication.run(DemoApplication.class, args); 18 | System.out.println("end-----------------"); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/entity/Result.java: -------------------------------------------------------------------------------- 1 | 2 | package com.netMusic.entity; 3 | import java.util.List; 4 | 5 | /** 6 | * Auto-generated: 2018-06-06 14:1:37 7 | * 8 | */ 9 | public class Result { 10 | 11 | private List songs; 12 | private int songCount; 13 | public void setSongs(List songs) { 14 | this.songs = songs; 15 | } 16 | public List getSongs() { 17 | return songs; 18 | } 19 | 20 | public void setSongCount(int songCount) { 21 | this.songCount = songCount; 22 | } 23 | public int getSongCount() { 24 | return songCount; 25 | } 26 | 27 | @Override 28 | public String toString() { 29 | return "Result{" + 30 | "songs=" + songs + 31 | ", songCount=" + songCount + 32 | '}'; 33 | } 34 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/entity/Singer.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.entity; 2 | 3 | public class Singer { 4 | private Integer id; 5 | 6 | private String name; 7 | 8 | private String intro; 9 | 10 | private String picUrl; 11 | 12 | public Integer getId() { 13 | return id; 14 | } 15 | 16 | public void setId(Integer id) { 17 | this.id = id; 18 | } 19 | 20 | public String getName() { 21 | return name; 22 | } 23 | 24 | public void setName(String name) { 25 | this.name = name; 26 | } 27 | 28 | public String getIntro() { 29 | return intro; 30 | } 31 | 32 | public void setIntro(String intro) { 33 | this.intro = intro; 34 | } 35 | 36 | public String getPicUrl() { 37 | return picUrl; 38 | } 39 | 40 | public void setPicUrl(String picUrl) { 41 | this.picUrl = picUrl; 42 | } 43 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/utils/BaseUtil.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.utils; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | public class BaseUtil { 7 | /** 8 | * 生成(min,max)区间的n个不重复数字 9 | * @param min 10 | * @param max 11 | * @param n 12 | * @return 13 | */ 14 | public static List getRandomNumber(int min, int max, int n){ 15 | 16 | if(n>(max-min+1) || max list = new ArrayList<>(); 20 | int count = 0; 21 | while(count < n ){ 22 | int num = (int) ((Math.random()*(max-min))+min); 23 | 24 | if(!list.contains(num)){ 25 | list.add(num); 26 | count++; 27 | }else{ 28 | count--; 29 | } 30 | 31 | } 32 | 33 | return list; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/dao/SongMapper.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.dao; 2 | 3 | import com.netMusic.entity.Song; 4 | import com.netMusic.entity.SongExample; 5 | import java.util.List; 6 | import org.apache.ibatis.annotations.Param; 7 | 8 | public interface SongMapper { 9 | int countByExample(SongExample example); 10 | 11 | int deleteByExample(SongExample example); 12 | 13 | int deleteByPrimaryKey(Long id); 14 | 15 | int insert(Song record); 16 | 17 | int insertSelective(Song record); 18 | 19 | List selectByExample(SongExample example); 20 | 21 | Song selectByPrimaryKey(Long id); 22 | 23 | int updateByExampleSelective(@Param("record") Song record, @Param("example") SongExample example); 24 | 25 | int updateByExample(@Param("record") Song record, @Param("example") SongExample example); 26 | 27 | int updateByPrimaryKeySelective(Song record); 28 | 29 | int updateByPrimaryKey(Song record); 30 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/entity/custom2/Artists.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 bejson.com 3 | */ 4 | package com.netMusic.entity.custom2; 5 | import java.util.List; 6 | 7 | /** 8 | * Auto-generated: 2018-06-10 20:8:39 9 | * 10 | * @author bejson.com (i@bejson.com) 11 | * @website http://www.bejson.com/java2pojo/ 12 | */ 13 | public class Artists { 14 | 15 | private int id; 16 | private String name; 17 | private List alia; 18 | public void setId(int id) { 19 | this.id = id; 20 | } 21 | public int getId() { 22 | return id; 23 | } 24 | 25 | public void setName(String name) { 26 | this.name = name; 27 | } 28 | public String getName() { 29 | return name; 30 | } 31 | 32 | public void setAlia(List alia) { 33 | this.alia = alia; 34 | } 35 | public List getAlia() { 36 | return alia; 37 | } 38 | 39 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/dao/RecordMapper.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.dao; 2 | 3 | import com.netMusic.entity.Record; 4 | import com.netMusic.entity.RecordExample; 5 | import java.util.List; 6 | import org.apache.ibatis.annotations.Param; 7 | 8 | public interface RecordMapper { 9 | int countByExample(RecordExample example); 10 | 11 | int deleteByExample(RecordExample example); 12 | 13 | int deleteByPrimaryKey(Integer id); 14 | 15 | int insert(Record record); 16 | 17 | int insertSelective(Record record); 18 | 19 | List selectByExample(RecordExample example); 20 | 21 | Record selectByPrimaryKey(Integer id); 22 | 23 | int updateByExampleSelective(@Param("record") Record record, @Param("example") RecordExample example); 24 | 25 | int updateByExample(@Param("record") Record record, @Param("example") RecordExample example); 26 | 27 | int updateByPrimaryKeySelective(Record record); 28 | 29 | int updateByPrimaryKey(Record record); 30 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/dao/SingerMapper.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.dao; 2 | 3 | import com.netMusic.entity.Singer; 4 | import com.netMusic.entity.SingerExample; 5 | import java.util.List; 6 | import org.apache.ibatis.annotations.Param; 7 | 8 | public interface SingerMapper { 9 | int countByExample(SingerExample example); 10 | 11 | int deleteByExample(SingerExample example); 12 | 13 | int deleteByPrimaryKey(Integer id); 14 | 15 | int insert(Singer record); 16 | 17 | int insertSelective(Singer record); 18 | 19 | List selectByExample(SingerExample example); 20 | 21 | Singer selectByPrimaryKey(Integer id); 22 | 23 | int updateByExampleSelective(@Param("record") Singer record, @Param("example") SingerExample example); 24 | 25 | int updateByExample(@Param("record") Singer record, @Param("example") SingerExample example); 26 | 27 | int updateByPrimaryKeySelective(Singer record); 28 | 29 | int updateByPrimaryKey(Singer record); 30 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/service/impl/SingerService.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.service.impl; 2 | 3 | import com.netMusic.dao.SingerMapper; 4 | import com.netMusic.entity.Singer; 5 | import com.netMusic.service.ISingerService; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.stereotype.Service; 8 | 9 | @Service 10 | public class SingerService implements ISingerService { 11 | 12 | @Autowired 13 | private SingerMapper singerMapper; 14 | 15 | @Override 16 | public void insertEntity(Singer entity) { 17 | if(entity!=null && entity.getId()!=null){ 18 | singerMapper.insert(entity); 19 | } 20 | } 21 | 22 | 23 | public Singer selectEntityById(Integer id) { 24 | 25 | if(id!=null){ 26 | return singerMapper.selectByPrimaryKey(id); 27 | } 28 | 29 | return null; 30 | } 31 | 32 | @Override 33 | public void updateEntityById(Singer entity) { 34 | 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/test/java/TestGson.java: -------------------------------------------------------------------------------- 1 | import com.google.gson.Gson; 2 | import com.google.gson.reflect.TypeToken; 3 | 4 | import java.lang.reflect.Type; 5 | import java.util.Map; 6 | 7 | class Result{ 8 | private String result; 9 | 10 | public String getResult() { 11 | return result; 12 | } 13 | 14 | public void setResult(String result) { 15 | this.result = result; 16 | } 17 | 18 | @Override 19 | public String toString() { 20 | return "Result{" + 21 | "result='" + result + '\'' + 22 | '}'; 23 | } 24 | } 25 | 26 | /** 27 | * 测试gson是否可以只转化特定的类 28 | */ 29 | public class TestGson { 30 | 31 | public static void main(String[] args){ 32 | String json = "{'result':'success','code':200,'test':'test'}"; 33 | Gson gson = new Gson(); 34 | Type type = new TypeToken() {}.getType(); 35 | Result result = gson.fromJson(json,type); 36 | System.out.println(result.getResult()); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/service/impl/RecordService.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.service.impl; 2 | 3 | import com.netMusic.dao.RecordMapper; 4 | import com.netMusic.entity.Record; 5 | import com.netMusic.service.IRecordService; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.stereotype.Service; 8 | 9 | @Service 10 | public class RecordService implements IRecordService { 11 | 12 | @Autowired 13 | private RecordMapper recordMapper; 14 | 15 | @Override 16 | public void insertEntity(Record entity) { 17 | if(entity!=null && entity.getId()!=null){ 18 | recordMapper.insert(entity); 19 | } 20 | 21 | } 22 | 23 | 24 | @Override 25 | public void updateEntityById(Record entity) { 26 | 27 | } 28 | 29 | @Override 30 | public Record selectEntityById(Integer id) { 31 | if(id!=null){ 32 | return recordMapper.selectByPrimaryKey(id); 33 | } 34 | 35 | return null; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/service/impl/SingerSongService.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.service.impl; 2 | 3 | import com.netMusic.dao.SingerSongMapper; 4 | import com.netMusic.entity.SingerSong; 5 | import com.netMusic.entity.SingerSongKey; 6 | import com.netMusic.service.ISingerSongService; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.stereotype.Service; 9 | 10 | @Service 11 | public class SingerSongService implements ISingerSongService { 12 | 13 | @Autowired 14 | private SingerSongMapper singerSongMapper; 15 | 16 | @Override 17 | public void insertEntity(SingerSong entity) { 18 | if(entity!=null){ 19 | singerSongMapper.insert(entity); 20 | } 21 | } 22 | 23 | 24 | @Override 25 | public SingerSong selectEntityById(SingerSongKey singerSongKey) { 26 | if(singerSongKey!=null){ 27 | return singerSongMapper.selectBySingerId(singerSongKey); 28 | } 29 | 30 | return null; 31 | } 32 | 33 | @Override 34 | public void updateEntityById(SingerSong entity) { 35 | 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/dao/SingerSongMapper.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.dao; 2 | 3 | import com.netMusic.entity.SingerSong; 4 | import com.netMusic.entity.SingerSongExample; 5 | import com.netMusic.entity.SingerSongKey; 6 | import org.apache.ibatis.annotations.Param; 7 | 8 | import java.util.List; 9 | 10 | public interface SingerSongMapper { 11 | int countByExample(SingerSongExample example); 12 | 13 | int deleteByExample(SingerSongExample example); 14 | 15 | int deleteByPrimaryKey(SingerSongKey key); 16 | 17 | int insert(SingerSong record); 18 | 19 | int insertSelective(SingerSong record); 20 | 21 | List selectByExample(SingerSongExample example); 22 | 23 | SingerSong selectByPrimaryKey(SingerSongKey key); 24 | 25 | int updateByExampleSelective(@Param("record") SingerSong record, @Param("example") SingerSongExample example); 26 | 27 | int updateByExample(@Param("record") SingerSong record, @Param("example") SingerSongExample example); 28 | 29 | int updateByPrimaryKeySelective(SingerSong record); 30 | 31 | int updateByPrimaryKey(SingerSong record); 32 | 33 | SingerSong selectBySingerId(SingerSongKey singerSongKey); 34 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/utils/CharacterUtils.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.utils; 2 | 3 | import java.util.Random; 4 | import java.util.regex.Matcher; 5 | import java.util.regex.Pattern; 6 | 7 | /** 8 | * 字符工具类 9 | */ 10 | public class CharacterUtils { 11 | 12 | 13 | public static String getRandomString(int length) { 14 | String str = "zxcvbnmlkjhgfdsaqwertyuiopQWERTYUIOPASDFGHJKLZXCVBNM"; 15 | Random random = new Random(); 16 | StringBuffer sb = new StringBuffer(); 17 | for (int i = 0; i < length; ++i) { 18 | int number = random.nextInt(52); 19 | sb.append(str.charAt(number)); 20 | } 21 | return sb.toString(); 22 | } 23 | 24 | public static boolean isContainChinese(String str){ 25 | Pattern pattern = Pattern.compile("[\u4e00-\u9fa5]"); 26 | Matcher m = pattern.matcher(str); 27 | if(m.find()){ 28 | return true; 29 | } 30 | return false; 31 | } 32 | 33 | 34 | public static boolean isContainNumber(String str){ 35 | Pattern pattern = Pattern.compile("[0-9]"); 36 | Matcher m = pattern.matcher(str); 37 | if(m.find()){ 38 | return true; 39 | } 40 | return false; 41 | } 42 | 43 | 44 | // public static void main(String[] args){ 45 | // System.out.println(getRandomString(2)); 46 | // } 47 | 48 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/thread/XunMengRunnable.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.thread; 2 | 3 | import com.netMusic.service.impl.XunMengService; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | import java.util.Date; 8 | 9 | public class XunMengRunnable implements Runnable { 10 | 11 | private XunMengService xunMengService; 12 | 13 | private final static Logger logger = LoggerFactory.getLogger(XunMengRunnable.class); 14 | 15 | private String name; 16 | 17 | public XunMengRunnable(String name, XunMengService xunMengService) { 18 | this.name = name; 19 | this.xunMengService = xunMengService; 20 | } 21 | 22 | @Override 23 | public void run() { 24 | String songName = name; 25 | synchronized (songName){ 26 | try { 27 | Thread.sleep(100); 28 | } catch (InterruptedException e) { 29 | e.printStackTrace(); 30 | } 31 | 32 | logger.info(new Date()+":"+name+"..........线程开启"); 33 | System.out.println("---------------"); 34 | xunMengService.saveAllBySongName(songName); 35 | logger.info(new Date()+":"+name+"....。。。..线程结束"); 36 | } 37 | 38 | 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/service/impl/SongService.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.service.impl; 2 | 3 | import com.netMusic.dao.SongMapper; 4 | import com.netMusic.entity.Song; 5 | import com.netMusic.service.ISongService; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.stereotype.Service; 8 | 9 | @Service 10 | public class SongService implements ISongService { 11 | 12 | @Autowired 13 | private SongMapper songMapper; 14 | 15 | @Override 16 | public void insertEntity(Song song) { 17 | 18 | if(song!=null && song.getId()!=null){ 19 | if(song.getMp3url()==null){ 20 | song.setMp3url("http://music.163.com/song/media/outer/url?id="+song.getId()); 21 | } 22 | songMapper.insert(song); 23 | } 24 | 25 | } 26 | 27 | 28 | public Song selectEntityById(Long id) { 29 | 30 | if(id!=null){ 31 | Song song = songMapper.selectByPrimaryKey(id); 32 | return song; 33 | } 34 | return null; 35 | } 36 | 37 | @Override 38 | public void updateEntityById(Song song) { 39 | if(song!=null && song.getId()!=null){ 40 | songMapper.updateByPrimaryKey(song); 41 | } 42 | 43 | } 44 | 45 | 46 | 47 | 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/controller/XunMengController.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.controller; 2 | 3 | import com.netMusic.service.impl.XunMengService; 4 | import org.springframework.beans.factory.annotation.Autowired; 5 | import org.springframework.web.bind.annotation.GetMapping; 6 | import org.springframework.web.bind.annotation.PostMapping; 7 | import org.springframework.web.bind.annotation.RequestMapping; 8 | import org.springframework.web.bind.annotation.RestController; 9 | 10 | @RestController 11 | @RequestMapping("/base") 12 | public class XunMengController { 13 | 14 | @Autowired 15 | private XunMengService xunMengService; 16 | 17 | @PostMapping(value = "/saveAll") 18 | public String saveAll(String songName){ 19 | 20 | if(songName!=null){ 21 | xunMengService.saveAllBySongName(songName); 22 | return "success"; 23 | }else{ 24 | return "error"; 25 | } 26 | 27 | } 28 | 29 | 30 | @GetMapping(value = "/saveAllByName") 31 | public String saveAllByName(){ 32 | 33 | String test = "abcdefghijklnmopqrstuvwxyz"; 34 | for(int i=0;i0){ 19 | ticketCount--; 20 | System.out.println(Thread.currentThread().getName()+"售票一张,余票数为："+ticketCount); 21 | } 22 | } 23 | 24 | @Override 25 | public void run() { 26 | 27 | synchronized (ticketCount){ 28 | while(ticketCount>0){ 29 | saleTicket(); 30 | // System.out.println(Thread.currentThread().getName()+"标志位："+ticketCount); 31 | } 32 | } 33 | 34 | 35 | } 36 | 37 | 38 | public static void main(String[] args){ 39 | 40 | System.out.println(Thread.currentThread().getName()); 41 | // for(int i=0;i<5;i++){//启动5个线程，每个线程有3张票可售 42 | // new Thread(new TestRunnable(3),"售票线程"+(i+1)).start(); 43 | // } 44 | 45 | for(int i=0;i<5;i++){//启动5个线程，此时共享一个变量，即总共10张票 46 | new Thread(new TestRunnable(),"售票线程"+(i+1)).start(); 47 | } 48 | 49 | 50 | 51 | } 52 | 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/Demo/NetMusicDemo.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.Demo; 2 | 3 | import org.apache.http.client.HttpClient; 4 | import org.apache.http.client.methods.HttpGet; 5 | import org.apache.http.impl.client.HttpClients; 6 | 7 | import java.util.List; 8 | 9 | /** 10 | * 爬取网易云热门推荐页面 11 | * Request URL: 12 | * https://music.163.com/discover/playlist/?order=hot&cat=%E5%85%A8%E9%83%A8&limit=35&offset=70 13 | * 其中，cat的值由"全部"转码而成 14 | * 拿到网页，需对html进行解析，采用Jsoup解析 15 | */ 16 | public class NetMusicDemo { 17 | 18 | /** 19 | * 模拟Get请求 20 | * @param url：请求Url 21 | * @param paramsList:参数列表 22 | */ 23 | public static void doGet(String url, List paramsList){ 24 | 25 | HttpClient httpClient = null; 26 | HttpGet httpGet = null; 27 | String result = null; 28 | 29 | try{ 30 | httpClient = HttpClients.createDefault(); 31 | httpGet = new HttpGet(url); 32 | //设置请求头 33 | httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"); 34 | httpGet.setHeader("Referer","https://music.163.com/"); 35 | httpGet.setHeader("Host","music.163.com"); 36 | //设置请求参数 37 | 38 | 39 | 40 | }catch (Exception e){ 41 | e.printStackTrace(); 42 | } 43 | 44 | 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/utils/FileControlUtil.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.utils; 2 | 3 | import java.io.*; 4 | 5 | /** 6 | * 文件控制类 7 | */ 8 | public class FileControlUtil { 9 | 10 | /** 11 | * 读取文本内容 12 | * fileName:文本路径 13 | */ 14 | public static void readFileContent(String fileName){ 15 | 16 | try { 17 | InputStream is = new FileInputStream(fileName); 18 | Reader reader = new InputStreamReader(is); 19 | int tempchar = 0; 20 | while((tempchar=reader.read())!=-1){ 21 | if (((char) tempchar) != '\r') { 22 | System.out.print((char) tempchar); 23 | } 24 | } 25 | 26 | 27 | // byte[] b = new byte[100000]; 28 | // int i = 0; 29 | // int index=0; 30 | // 31 | // while((i=is.read())!=-1){ 32 | // b[index++]=(byte)i; 33 | // } 34 | // 35 | // System.out.println(new String(b)); 36 | is.close(); 37 | reader.close(); 38 | } catch (FileNotFoundException e) { 39 | e.printStackTrace(); 40 | } catch (IOException e){ 41 | e.printStackTrace(); 42 | } 43 | 44 | } 45 | 46 | 47 | public static void main(String[] args){ 48 | 49 | String fileName = "D:\\Code\\idea\\project\\Reptilian\\test.txt"; 50 | readFileContent(fileName); 51 | 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/Demo/HtmlUnitDemo.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.Demo; 2 | 3 | import com.gargoylesoftware.htmlunit.WebClient; 4 | import com.gargoylesoftware.htmlunit.WebRequest; 5 | import com.gargoylesoftware.htmlunit.html.HtmlPage; 6 | import org.jsoup.Jsoup; 7 | import org.jsoup.nodes.Document; 8 | 9 | import java.io.IOException; 10 | import java.net.URL; 11 | 12 | /** 13 | * document 测试 14 | * 15 | */ 16 | public class HtmlUnitDemo { 17 | 18 | public static void main(String[] args){ 19 | String url = "https://www.zhihu.com/explore/recommendations"; 20 | String userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"; 21 | try { 22 | 23 | URL link = new URL(url); 24 | // String url = "https://www.baidu.com"; 25 | //拿到网页 26 | WebClient webClient = new WebClient(); 27 | webClient.getOptions().setJavaScriptEnabled(false); 28 | webClient.getOptions().setCssEnabled(false); 29 | WebRequest webRequest = new WebRequest(link); 30 | webRequest.setAdditionalHeader("User-Agent",userAgent); 31 | 32 | final HtmlPage page = webClient.getPage(webRequest); 33 | Document document = Jsoup.parse(page.asXml()); 34 | System.out.println(document); 35 | 36 | 37 | 38 | } catch (IOException e) { 39 | e.printStackTrace(); 40 | } 41 | 42 | } 43 | 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/entity/custom2/Album.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 bejson.com 3 | */ 4 | package com.netMusic.entity.custom2; 5 | import java.util.List; 6 | 7 | /** 8 | * Auto-generated: 2018-06-10 20:8:39 9 | * 10 | * @author bejson.com (i@bejson.com) 11 | * @website http://www.bejson.com/java2pojo/ 12 | */ 13 | public class Album { 14 | 15 | private long id; 16 | private String name; 17 | private String picUrl; 18 | private String pic_str; 19 | private long pic; 20 | private List alia; 21 | public void setId(long id) { 22 | this.id = id; 23 | } 24 | public long getId() { 25 | return id; 26 | } 27 | 28 | public void setName(String name) { 29 | this.name = name; 30 | } 31 | public String getName() { 32 | return name; 33 | } 34 | 35 | public void setPicUrl(String picUrl) { 36 | this.picUrl = picUrl; 37 | } 38 | public String getPicUrl() { 39 | return picUrl; 40 | } 41 | 42 | public void setPic_str(String pic_str) { 43 | this.pic_str = pic_str; 44 | } 45 | public String getPic_str() { 46 | return pic_str; 47 | } 48 | 49 | public void setPic(long pic) { 50 | this.pic = pic; 51 | } 52 | public long getPic() { 53 | return pic; 54 | } 55 | 56 | public void setAlia(List alia) { 57 | this.alia = alia; 58 | } 59 | public List getAlia() { 60 | return alia; 61 | } 62 | 63 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/entity/Song.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.entity; 2 | 3 | public class Song { 4 | private Long id; 5 | 6 | private String commentThreadId; 7 | 8 | private String mp3url; 9 | 10 | private String name; 11 | 12 | private Integer recordId; 13 | 14 | public Long getId() { 15 | return id; 16 | } 17 | 18 | public void setId(Long id) { 19 | this.id = id; 20 | } 21 | 22 | public String getCommentThreadId() { 23 | return commentThreadId; 24 | } 25 | 26 | public void setCommentThreadId(String commentThreadId) { 27 | this.commentThreadId = commentThreadId; 28 | } 29 | 30 | public String getMp3url() { 31 | return mp3url; 32 | } 33 | 34 | public void setMp3url(String mp3url) { 35 | this.mp3url = mp3url; 36 | } 37 | 38 | public String getName() { 39 | return name; 40 | } 41 | 42 | public void setName(String name) { 43 | this.name = name; 44 | } 45 | 46 | public Integer getRecordId() { 47 | return recordId; 48 | } 49 | 50 | public void setRecordId(Integer recordId) { 51 | this.recordId = recordId; 52 | } 53 | 54 | @Override 55 | public String toString() { 56 | return "Song{" + 57 | "id=" + id + 58 | ", commentThreadId='" + commentThreadId + '\'' + 59 | ", mp3url='" + mp3url + '\'' + 60 | ", name='" + name + '\'' + 61 | ", recordId=" + recordId + 62 | '}'; 63 | } 64 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/thread/SpiderSingerRunnable.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.thread; 2 | 3 | import com.netMusic.entity.Singer; 4 | import com.netMusic.service.ISingerService; 5 | import com.netMusic.spider.NetMusicGrab; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.util.Date; 10 | import java.util.List; 11 | 12 | public class SpiderSingerRunnable implements Runnable { 13 | 14 | private ISingerService singerService; 15 | private List listId; 16 | private final static Logger logger = LoggerFactory.getLogger(SpiderSingerRunnable.class); 17 | 18 | public SpiderSingerRunnable(ISingerService singerService, List listId) { 19 | this.singerService = singerService; 20 | this.listId = listId; 21 | } 22 | 23 | @Override 24 | public void run() { 25 | logger.info(new Date()+Thread.currentThread().getName()+":..........线程开启"); 26 | System.out.println("---------------"); 27 | //插入音乐 28 | insertSinger(); 29 | logger.info(new Date()+Thread.currentThread().getName()+":....。。。..线程结束"); 30 | 31 | } 32 | 33 | private void insertSinger() { 34 | if(listId!=null){ 35 | for(Integer id:listId){ 36 | String url = "https://music.163.com/artist/desc?id="+id; 37 | String charest = "utf-8"; 38 | Singer singer = NetMusicGrab.getSinger(url,charest); 39 | if(singer!=null){ 40 | Singer singer1 = singerService.selectEntityById(id); 41 | if(singer1==null){ 42 | singer.setId(id); 43 | singerService.insertEntity(singer); 44 | } 45 | } 46 | } 47 | } 48 | 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/resources/generator/generatorConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 |
33 |
34 |
35 |
36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/entity/Record.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.entity; 2 | 3 | import java.util.Date; 4 | 5 | public class Record { 6 | private Integer id; 7 | 8 | private String commentThreadId; 9 | 10 | private String company; 11 | 12 | private String intro; 13 | 14 | private String name; 15 | 16 | private String picUrl; 17 | 18 | private Integer singerId; 19 | 20 | private Date publishTime; 21 | 22 | public Integer getId() { 23 | return id; 24 | } 25 | 26 | public void setId(Integer id) { 27 | this.id = id; 28 | } 29 | 30 | public String getCommentThreadId() { 31 | return commentThreadId; 32 | } 33 | 34 | public void setCommentThreadId(String commentThreadId) { 35 | this.commentThreadId = commentThreadId; 36 | } 37 | 38 | public String getCompany() { 39 | return company; 40 | } 41 | 42 | public void setCompany(String company) { 43 | this.company = company; 44 | } 45 | 46 | public String getIntro() { 47 | return intro; 48 | } 49 | 50 | public void setIntro(String intro) { 51 | this.intro = intro; 52 | } 53 | 54 | public String getName() { 55 | return name; 56 | } 57 | 58 | public void setName(String name) { 59 | this.name = name; 60 | } 61 | 62 | public String getPicUrl() { 63 | return picUrl; 64 | } 65 | 66 | public void setPicUrl(String picUrl) { 67 | this.picUrl = picUrl; 68 | } 69 | 70 | public Integer getSingerId() { 71 | return singerId; 72 | } 73 | 74 | public void setSingerId(Integer singerId) { 75 | this.singerId = singerId; 76 | } 77 | 78 | public Date getPublishTime() { 79 | return publishTime; 80 | } 81 | 82 | public void setPublishTime(Date publishTime) { 83 | this.publishTime = publishTime; 84 | } 85 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/Demo/HttpClientDemo.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.Demo; 2 | 3 | 4 | import org.apache.http.Header; 5 | import org.apache.http.HttpEntity; 6 | import org.apache.http.client.methods.CloseableHttpResponse; 7 | import org.apache.http.client.methods.HttpGet; 8 | import org.apache.http.impl.client.CloseableHttpClient; 9 | import org.apache.http.impl.client.HttpClients; 10 | import org.apache.http.util.EntityUtils; 11 | 12 | import java.io.IOException; 13 | 14 | public class HttpClientDemo { 15 | 16 | public static void main(String[] args){ 17 | 18 | //模拟客户端 19 | CloseableHttpClient httpClient = HttpClients.createDefault(); 20 | //模拟输入url请求 21 | HttpGet httpGet = new HttpGet("http://www.baidu.com"); 22 | //添加头部信息模拟浏览器访问 23 | httpGet.setHeader("Accept", "text/html,application/xhtml+xml," + 24 | "application/xml;q=0.9,image/webp,*/*;q=0.8"); 25 | httpGet.setHeader("Accept-Encoding", "gzip, deflate, sdch, br"); 26 | httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.8"); 27 | httpGet.setHeader("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36" + 28 | " (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36"); 29 | try { 30 | // 客户端执行get请求返回响应 31 | CloseableHttpResponse response = httpClient.execute(httpGet); 32 | HttpEntity httpEntity = response.getEntity(); 33 | String strResult = EntityUtils.toString(httpEntity); 34 | System.out.println(strResult); 35 | System.out.println("excuting request:"+httpGet.getURI()); 36 | 37 | System.out.println("-------------"); 38 | System.out.println(response.getStatusLine().toString()); 39 | Header[] headers = response.getAllHeaders(); 40 | for (Header h:headers) { 41 | System.out.println(h.getName()+":"+h.getName()); 42 | } 43 | } catch (IOException e) { 44 | e.printStackTrace(); 45 | } 46 | 47 | 48 | } 49 | 50 | 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/thread/SpiderRecordRunnable.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.thread; 2 | 3 | import com.netMusic.entity.Record; 4 | import com.netMusic.service.IRecordService; 5 | import com.netMusic.spider.NetMusicGrab; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.util.Date; 10 | import java.util.List; 11 | 12 | public class SpiderRecordRunnable implements Runnable { 13 | 14 | private IRecordService recordService; 15 | private List listId; 16 | private final static Logger logger = LoggerFactory.getLogger(SpiderRecordRunnable.class); 17 | 18 | public SpiderRecordRunnable(IRecordService recordService, List listId) { 19 | this.recordService = recordService; 20 | this.listId = listId; 21 | } 22 | 23 | 24 | @Override 25 | public void run() { 26 | logger.info(new Date()+Thread.currentThread().getName()+":..........线程开启"); 27 | System.out.println("---------------"); 28 | //插入音乐 29 | insertRecord(); 30 | logger.info(new Date()+Thread.currentThread().getName()+":....。。。..线程结束"); 31 | } 32 | 33 | /** 34 | * 插入专辑 35 | */ 36 | private void insertRecord() { 37 | 38 | if(listId!=null){ 39 | for(Integer singerId:listId){ 40 | String url = "https://music.163.com/artist/album?id="+singerId+"&limit=100&offset=0"; 41 | String charest = "utf-8"; 42 | List list = NetMusicGrab.getRecordList(url,charest); 43 | if(list!=null && list.size()>0){ 44 | for(Record record:list){ 45 | if(record!=null && record.getId()!=null){ 46 | Record record1 = recordService.selectEntityById(record.getId()); 47 | if (record1 == null) { 48 | record.setSingerId(singerId); 49 | recordService.insertEntity(record); 50 | } 51 | 52 | } 53 | } 54 | 55 | } 56 | 57 | 58 | } 59 | } 60 | 61 | 62 | 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/Demo/SimulationLoginDemo.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.Demo; 2 | 3 | import org.apache.http.Header; 4 | import org.apache.http.NameValuePair; 5 | import org.apache.http.client.ClientProtocolException; 6 | import org.apache.http.client.entity.UrlEncodedFormEntity; 7 | import org.apache.http.client.methods.CloseableHttpResponse; 8 | import org.apache.http.client.methods.HttpPost; 9 | import org.apache.http.impl.client.CloseableHttpClient; 10 | import org.apache.http.impl.client.HttpClients; 11 | import org.apache.http.message.BasicNameValuePair; 12 | 13 | import java.io.IOException; 14 | import java.io.UnsupportedEncodingException; 15 | import java.util.ArrayList; 16 | import java.util.List; 17 | 18 | /** 19 | * HttpClient模拟登录 20 | */ 21 | public class SimulationLoginDemo { 22 | 23 | public static void main(String[] args){ 24 | 25 | //创建客户端 26 | CloseableHttpClient httpClient = HttpClients.createDefault(); 27 | //创建Post请求 28 | HttpPost httpPost = new HttpPost("https://www.douban.com/"); 29 | 30 | //创建参数列表 31 | List nvps = new ArrayList<>(); 32 | nvps.add(new BasicNameValuePair("domain", "renren.com")); 33 | nvps.add(new BasicNameValuePair("isplogin", "true")); 34 | nvps.add(new BasicNameValuePair("submit", "登录")); 35 | nvps.add(new BasicNameValuePair("email", "")); 36 | nvps.add(new BasicNameValuePair("passwd", "")); 37 | 38 | //向对方服务器发送Post请求 39 | try { 40 | //将参数进行封装，提交到服务器端 41 | httpPost.setEntity(new UrlEncodedFormEntity(nvps, "UTF8")); 42 | CloseableHttpResponse httpResponse = httpClient.execute(httpPost); 43 | 44 | //如果模拟登录成功 45 | if(httpResponse.getStatusLine().getStatusCode() == 200) { 46 | Header[] headers = httpResponse.getAllHeaders(); 47 | for (Header header : headers) { 48 | System.out.println(header.getName() + ": " + header.getValue()); 49 | } 50 | } 51 | } catch (UnsupportedEncodingException e) { 52 | e.printStackTrace(); 53 | } catch (ClientProtocolException e) { 54 | e.printStackTrace(); 55 | } catch (IOException e) { 56 | e.printStackTrace(); 57 | } finally { 58 | httpPost.abort(); //释放资源 59 | } 60 | 61 | 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/entity/custom/Artist.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 bejson.com 3 | */ 4 | package com.netMusic.entity.custom; 5 | import java.util.List; 6 | 7 | /** 8 | * Auto-generated: 2018-06-06 20:59:46 9 | * 10 | * @author bejson.com (i@bejson.com) 11 | * @website http://www.bejson.com/java2pojo/ 12 | */ 13 | public class Artist { 14 | 15 | private String name; 16 | private int id; 17 | private int picId; 18 | private int img1v1Id; 19 | private String briefDesc; 20 | private String picUrl; 21 | private String img1v1Url; 22 | private int albumSize; 23 | private List alias; 24 | private String trans; 25 | private int musicSize; 26 | public void setName(String name) { 27 | this.name = name; 28 | } 29 | public String getName() { 30 | return name; 31 | } 32 | 33 | public void setId(int id) { 34 | this.id = id; 35 | } 36 | public int getId() { 37 | return id; 38 | } 39 | 40 | public void setPicId(int picId) { 41 | this.picId = picId; 42 | } 43 | public int getPicId() { 44 | return picId; 45 | } 46 | 47 | public void setImg1v1Id(int img1v1Id) { 48 | this.img1v1Id = img1v1Id; 49 | } 50 | public int getImg1v1Id() { 51 | return img1v1Id; 52 | } 53 | 54 | public void setBriefDesc(String briefDesc) { 55 | this.briefDesc = briefDesc; 56 | } 57 | public String getBriefDesc() { 58 | return briefDesc; 59 | } 60 | 61 | public void setPicUrl(String picUrl) { 62 | this.picUrl = picUrl; 63 | } 64 | public String getPicUrl() { 65 | return picUrl; 66 | } 67 | 68 | public void setImg1v1Url(String img1v1Url) { 69 | this.img1v1Url = img1v1Url; 70 | } 71 | public String getImg1v1Url() { 72 | return img1v1Url; 73 | } 74 | 75 | public void setAlbumSize(int albumSize) { 76 | this.albumSize = albumSize; 77 | } 78 | public int getAlbumSize() { 79 | return albumSize; 80 | } 81 | 82 | public void setAlias(List alias) { 83 | this.alias = alias; 84 | } 85 | public List getAlias() { 86 | return alias; 87 | } 88 | 89 | public void setTrans(String trans) { 90 | this.trans = trans; 91 | } 92 | public String getTrans() { 93 | return trans; 94 | } 95 | 96 | public void setMusicSize(int musicSize) { 97 | this.musicSize = musicSize; 98 | } 99 | public int getMusicSize() { 100 | return musicSize; 101 | } 102 | 103 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/entity/custom/Artists.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 bejson.com 3 | */ 4 | package com.netMusic.entity.custom; 5 | import java.util.List; 6 | 7 | /** 8 | * Auto-generated: 2018-06-06 20:59:46 9 | * 10 | * @author bejson.com (i@bejson.com) 11 | * @website http://www.bejson.com/java2pojo/ 12 | */ 13 | public class Artists { 14 | 15 | private String name; 16 | private int id; 17 | private Long picId; 18 | private int img1v1Id; 19 | private String briefDesc; 20 | private String picUrl; 21 | private String img1v1Url; 22 | private int albumSize; 23 | private List alias; 24 | private String trans; 25 | private int musicSize; 26 | public void setName(String name) { 27 | this.name = name; 28 | } 29 | public String getName() { 30 | return name; 31 | } 32 | 33 | public void setId(int id) { 34 | this.id = id; 35 | } 36 | public int getId() { 37 | return id; 38 | } 39 | 40 | public Long getPicId() { 41 | return picId; 42 | } 43 | 44 | public void setPicId(Long picId) { 45 | this.picId = picId; 46 | } 47 | 48 | public void setImg1v1Id(int img1v1Id) { 49 | this.img1v1Id = img1v1Id; 50 | } 51 | public int getImg1v1Id() { 52 | return img1v1Id; 53 | } 54 | 55 | public void setBriefDesc(String briefDesc) { 56 | this.briefDesc = briefDesc; 57 | } 58 | public String getBriefDesc() { 59 | return briefDesc; 60 | } 61 | 62 | public void setPicUrl(String picUrl) { 63 | this.picUrl = picUrl; 64 | } 65 | public String getPicUrl() { 66 | return picUrl; 67 | } 68 | 69 | public void setImg1v1Url(String img1v1Url) { 70 | this.img1v1Url = img1v1Url; 71 | } 72 | public String getImg1v1Url() { 73 | return img1v1Url; 74 | } 75 | 76 | public void setAlbumSize(int albumSize) { 77 | this.albumSize = albumSize; 78 | } 79 | public int getAlbumSize() { 80 | return albumSize; 81 | } 82 | 83 | public void setAlias(List alias) { 84 | this.alias = alias; 85 | } 86 | public List getAlias() { 87 | return alias; 88 | } 89 | 90 | public void setTrans(String trans) { 91 | this.trans = trans; 92 | } 93 | public String getTrans() { 94 | return trans; 95 | } 96 | 97 | public void setMusicSize(int musicSize) { 98 | this.musicSize = musicSize; 99 | } 100 | public int getMusicSize() { 101 | return musicSize; 102 | } 103 | 104 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/Demo/SingerGrabDemo.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.Demo; 2 | 3 | import com.google.gson.Gson; 4 | import com.google.gson.reflect.TypeToken; 5 | import com.netMusic.entity.Song; 6 | import com.netMusic.entity.custom2.SongMsg; 7 | import org.apache.http.HttpEntity; 8 | import org.apache.http.HttpResponse; 9 | import org.apache.http.client.HttpClient; 10 | import org.apache.http.client.methods.HttpGet; 11 | import org.apache.http.impl.client.HttpClients; 12 | import org.apache.http.util.EntityUtils; 13 | import org.jsoup.Jsoup; 14 | import org.jsoup.nodes.Document; 15 | import org.jsoup.select.Elements; 16 | 17 | import java.lang.reflect.Type; 18 | import java.util.List; 19 | 20 | /** 21 | * 抓取网易云歌手歌曲信息 22 | */ 23 | public class SingerGrabDemo { 24 | 25 | 26 | public static String doGet(String url){ 27 | 28 | HttpClient httpClient = null; 29 | HttpGet httpGet = null; 30 | String result = null; 31 | try { 32 | httpClient = HttpClients.createDefault(); 33 | httpGet = new HttpGet(url); 34 | //设置请求头 35 | httpGet.setHeader("Host","music.163.com"); 36 | httpGet.setHeader("Referer","https://music.163.com/"); 37 | httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"); 38 | 39 | HttpResponse httpResponse = httpClient.execute(httpGet); 40 | if(httpResponse!=null){ 41 | HttpEntity httpEntity = httpResponse.getEntity(); 42 | if(httpEntity!=null){ 43 | result = EntityUtils.toString(httpEntity); 44 | } 45 | } 46 | 47 | 48 | }catch (Exception e){ 49 | e.printStackTrace(); 50 | } 51 | 52 | return result; 53 | 54 | } 55 | 56 | 57 | 58 | 59 | public static void main(String[] args){ 60 | 61 | String result = doGet("https://music.163.com/artist?id=2116"); 62 | // System.out.println(result); 63 | Document document = Jsoup.parse(result); 64 | Elements elements = document.select("#song-list-pre-data"); 65 | System.out.println("json数据如下"); 66 | String resJson = elements.text(); 67 | System.out.println(resJson); 68 | Gson gson = new Gson(); 69 | Type listType = new TypeToken>(){}.getType(); 70 | List list = gson.fromJson(resJson,listType); 71 | for(SongMsg songMsg:list){ 72 | Song song = new Song(); 73 | song.setId(songMsg.getId()); 74 | song.setName(songMsg.getName()); 75 | song.setCommentThreadId(songMsg.getCommentThreadId()); 76 | song.setRecordId((int)songMsg.getAlbum().getId()); 77 | System.out.println(song.toString()); 78 | } 79 | 80 | 81 | 82 | } 83 | 84 | 85 | 86 | 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/entity/custom2/Privilege.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 bejson.com 3 | */ 4 | package com.netMusic.entity.custom2; 5 | 6 | /** 7 | * Auto-generated: 2018-06-10 20:8:39 8 | * 9 | * @author bejson.com (i@bejson.com) 10 | * @website http://www.bejson.com/java2pojo/ 11 | */ 12 | public class Privilege { 13 | 14 | private long id; 15 | private int fee; 16 | private int payed; 17 | private int st; 18 | private long pl; 19 | private int dl; 20 | private int sp; 21 | private int cp; 22 | private int subp; 23 | private boolean cs; 24 | private long maxbr; 25 | private long fl; 26 | private boolean toast; 27 | private int flag; 28 | private boolean preSell; 29 | public void setId(long id) { 30 | this.id = id; 31 | } 32 | public long getId() { 33 | return id; 34 | } 35 | 36 | public void setFee(int fee) { 37 | this.fee = fee; 38 | } 39 | public int getFee() { 40 | return fee; 41 | } 42 | 43 | public void setPayed(int payed) { 44 | this.payed = payed; 45 | } 46 | public int getPayed() { 47 | return payed; 48 | } 49 | 50 | public void setSt(int st) { 51 | this.st = st; 52 | } 53 | public int getSt() { 54 | return st; 55 | } 56 | 57 | public void setPl(long pl) { 58 | this.pl = pl; 59 | } 60 | public long getPl() { 61 | return pl; 62 | } 63 | 64 | public void setDl(int dl) { 65 | this.dl = dl; 66 | } 67 | public int getDl() { 68 | return dl; 69 | } 70 | 71 | public void setSp(int sp) { 72 | this.sp = sp; 73 | } 74 | public int getSp() { 75 | return sp; 76 | } 77 | 78 | public void setCp(int cp) { 79 | this.cp = cp; 80 | } 81 | public int getCp() { 82 | return cp; 83 | } 84 | 85 | public void setSubp(int subp) { 86 | this.subp = subp; 87 | } 88 | public int getSubp() { 89 | return subp; 90 | } 91 | 92 | public void setCs(boolean cs) { 93 | this.cs = cs; 94 | } 95 | public boolean getCs() { 96 | return cs; 97 | } 98 | 99 | public void setMaxbr(long maxbr) { 100 | this.maxbr = maxbr; 101 | } 102 | public long getMaxbr() { 103 | return maxbr; 104 | } 105 | 106 | public void setFl(long fl) { 107 | this.fl = fl; 108 | } 109 | public long getFl() { 110 | return fl; 111 | } 112 | 113 | public void setToast(boolean toast) { 114 | this.toast = toast; 115 | } 116 | public boolean getToast() { 117 | return toast; 118 | } 119 | 120 | public void setFlag(int flag) { 121 | this.flag = flag; 122 | } 123 | public int getFlag() { 124 | return flag; 125 | } 126 | 127 | public void setPreSell(boolean preSell) { 128 | this.preSell = preSell; 129 | } 130 | public boolean getPreSell() { 131 | return preSell; 132 | } 133 | 134 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/listener/SaveDataListener.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.listener; 2 | 3 | import com.netMusic.service.IRecordService; 4 | import com.netMusic.service.ISingerService; 5 | import com.netMusic.service.ISingerSongService; 6 | import com.netMusic.service.ISongService; 7 | import com.netMusic.service.impl.SongService; 8 | import com.netMusic.thread.SpiderRecordRunnable; 9 | import com.netMusic.thread.SpiderSingerRunnable; 10 | import com.netMusic.thread.SpiderSongRunnnable; 11 | import com.netMusic.utils.BaseUtil; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | import org.springframework.beans.factory.annotation.Autowired; 15 | import org.springframework.stereotype.Component; 16 | import org.springframework.web.context.support.WebApplicationContextUtils; 17 | 18 | import javax.servlet.ServletContextEvent; 19 | import javax.servlet.ServletContextListener; 20 | import javax.servlet.annotation.WebListener; 21 | import java.util.Date; 22 | import java.util.List; 23 | 24 | //获取spring容器内的上下文，拿到BaseService的Bean 25 | // baseService = WebApplicationContextUtils 26 | // .getWebApplicationContext(servletContextEvent.getServletContext()).getBean(XunMengService.class); 27 | 28 | 29 | //启动添加数据的线程 30 | // for(int i=0;i<20;i++){ 31 | // for(int j=1;j<3;j++){ 32 | // XunMengRunnable xunMengRunnable = new XunMengRunnable(CharacterUtils.getRandomString(j),xunMengService); 33 | // Thread thread = new Thread(xunMengRunnable); 34 | // thread.start(); 35 | // } 36 | // 37 | // } 38 | @WebListener 39 | @Component 40 | public class SaveDataListener implements ServletContextListener { 41 | 42 | // @Autowired 43 | // private XunMengService xunMengService; 44 | 45 | @Autowired 46 | private ISongService songService; 47 | @Autowired 48 | private ISingerSongService singerSongService; 49 | @Autowired 50 | private ISingerService singerService; 51 | @Autowired 52 | private IRecordService recordService; 53 | 54 | private final static Logger logger = LoggerFactory.getLogger(SaveDataListener.class); 55 | 56 | @Override 57 | public void contextInitialized(ServletContextEvent servletContextEvent) { 58 | logger.info(new Date()+"自定义监听器启动。。。"); 59 | System.out.println("自定义监听器启动。。。"); 60 | 61 | //获取spring容器内的上下文，拿到BaseService的Bean 62 | songService = WebApplicationContextUtils 63 | .getWebApplicationContext(servletContextEvent.getServletContext()).getBean(SongService.class); 64 | 65 | 66 | for(int i=0;i<10;i++){ 67 | List listId = BaseUtil.getRandomNumber(1000,10000,1000); 68 | //song线程 69 | SpiderSongRunnnable spiderSongRunnnable = new SpiderSongRunnnable(songService, 70 | singerSongService,listId); 71 | 72 | Thread songThread = new Thread(spiderSongRunnnable); 73 | songThread.start(); 74 | //singer线程 75 | logger.info(new Date()+"singer线程开启-------"); 76 | SpiderSingerRunnable spiderSingerRunnable = new SpiderSingerRunnable(singerService,listId); 77 | Thread singerThread = new Thread(spiderSingerRunnable); 78 | singerThread.start(); 79 | //record线程 80 | logger.info("record线程开启-------"); 81 | SpiderRecordRunnable spiderRecordRunnable = new SpiderRecordRunnable(recordService,listId); 82 | Thread recordThread = new Thread(spiderRecordRunnable); 83 | recordThread.start(); 84 | } 85 | 86 | } 87 | 88 | 89 | @Override 90 | public void contextDestroyed(ServletContextEvent servletContextEvent) { 91 | logger.info("自定义监听器销毁。。。"); 92 | System.out.println("自定义监听器销毁。。。"); 93 | 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/utils/HttpClientUtil.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.utils; 2 | 3 | import org.apache.http.Header; 4 | import org.apache.http.HttpEntity; 5 | import org.apache.http.HttpResponse; 6 | import org.apache.http.NameValuePair; 7 | import org.apache.http.client.HttpClient; 8 | import org.apache.http.client.entity.UrlEncodedFormEntity; 9 | import org.apache.http.client.methods.HttpGet; 10 | import org.apache.http.client.methods.HttpPost; 11 | import org.apache.http.impl.client.HttpClients; 12 | import org.apache.http.message.BasicNameValuePair; 13 | import org.apache.http.util.EntityUtils; 14 | import org.springframework.stereotype.Component; 15 | 16 | import java.util.*; 17 | 18 | /** 19 | * HttpClient工具类 20 | */ 21 | @Component 22 | public class HttpClientUtil { 23 | 24 | 25 | /** 26 | * 模拟post请求，获取返回数据 27 | * @param url 28 | * @param map 29 | * @param charest 30 | * @param headerList 31 | * @return 32 | */ 33 | public static String doPost(String url, Map map, String charest, List
headerList) { 34 | 35 | HttpClient httpClient = null; 36 | HttpPost httpPost = null; 37 | String result = null; 38 | try { 39 | httpClient = HttpClients.createDefault(); 40 | httpPost = new HttpPost(url); 41 | //设置请求头，反反盗链 42 | for(Header header:headerList){ 43 | // httpPost.setHeader(header.getName(),header.getValue()); 44 | httpPost.setHeader(header); 45 | } 46 | //设置参数 47 | List list = new ArrayList<>(); 48 | Iterator iterator = map.entrySet().iterator(); 49 | while (iterator.hasNext()) { 50 | Map.Entry elem = (Map.Entry) iterator.next(); 51 | list.add(new BasicNameValuePair(elem.getKey(), (String) elem.getValue())); 52 | } 53 | if (list.size() > 0) { 54 | //封装实体 55 | UrlEncodedFormEntity entity = new UrlEncodedFormEntity(list, charest); 56 | httpPost.setEntity(entity); 57 | } 58 | //拿到返回结果 59 | HttpResponse response = httpClient.execute(httpPost); 60 | if (response != null) { 61 | HttpEntity resEntity = response.getEntity(); 62 | if (resEntity != null) { 63 | result = EntityUtils.toString(resEntity, charest); 64 | } 65 | } 66 | 67 | } catch (Exception e) { 68 | e.printStackTrace(); 69 | } 70 | 71 | return result; 72 | 73 | } 74 | 75 | 76 | /** 77 | * 模拟get请求 78 | * @param url 79 | * @param headerList 80 | * @param charest 81 | * @return 82 | */ 83 | public static String doGet(String url,List
headerList,String charest){ 84 | 85 | HttpClient httpClient = null; 86 | HttpGet httpGet = null; 87 | String result = null; 88 | try { 89 | httpClient = HttpClients.createDefault(); 90 | httpGet = new HttpGet(url); 91 | //设置请求头 92 | for(Header header:headerList){ 93 | httpGet.setHeader(header); 94 | } 95 | 96 | HttpResponse httpResponse = httpClient.execute(httpGet); 97 | if(httpResponse!=null){ 98 | HttpEntity httpEntity = httpResponse.getEntity(); 99 | if(httpEntity!=null){ 100 | result = EntityUtils.toString(httpEntity,charest); 101 | } 102 | } 103 | 104 | 105 | }catch (Exception e){ 106 | e.printStackTrace(); 107 | } 108 | 109 | return result; 110 | 111 | } 112 | 113 | 114 | 115 | 116 | 117 | 118 | } 119 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/thread/SpiderSongRunnnable.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.thread; 2 | 3 | import com.netMusic.entity.SingerSong; 4 | import com.netMusic.entity.Song; 5 | import com.netMusic.entity.custom2.Artists; 6 | import com.netMusic.entity.custom2.SongMsg; 7 | import com.netMusic.service.ISingerSongService; 8 | import com.netMusic.service.ISongService; 9 | import com.netMusic.spider.NetMusicGrab; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import java.util.ArrayList; 14 | import java.util.Date; 15 | import java.util.List; 16 | 17 | public class SpiderSongRunnnable implements Runnable { 18 | 19 | private ISongService songService; 20 | private ISingerSongService singerSongService; 21 | private List listId; 22 | private final static Logger logger = LoggerFactory.getLogger(SpiderSongRunnnable.class); 23 | 24 | public SpiderSongRunnnable(ISongService songService, ISingerSongService singerSongService, List listId){ 25 | this.songService = songService; 26 | this.singerSongService = singerSongService; 27 | this.listId = listId; 28 | } 29 | 30 | 31 | @Override 32 | public void run() { 33 | logger.info(new Date()+Thread.currentThread().getName()+":..........线程开启"); 34 | System.out.println("---------------"); 35 | //插入音乐 36 | insertSong(); 37 | logger.info(new Date()+Thread.currentThread().getName()+":....。。。..线程结束"); 38 | 39 | 40 | } 41 | 42 | /** 43 | * 保存音乐与其singer_song中间表 44 | */ 45 | private void insertSong() { 46 | if(listId!=null){ 47 | for(Integer id:listId){ 48 | String url = "https://music.163.com/artist?id="+id; 49 | String charest = "utf-8"; 50 | List list = NetMusicGrab.getSongList(url,charest); 51 | if(list!=null && list.size()>0){ 52 | for (SongMsg songMsg : list) { 53 | Song song = new Song(); 54 | song.setId(songMsg.getId()); 55 | song.setName(songMsg.getName()); 56 | song.setCommentThreadId(songMsg.getCommentThreadId()); 57 | song.setRecordId((int) songMsg.getAlbum().getId()); 58 | logger.info(song.toString()); 59 | Song song1 = songService.selectEntityById(song.getId()); 60 | if(song1==null){ 61 | songService.insertEntity(song); 62 | } 63 | //插入中间表 64 | List artistsList = songMsg.getArtists(); 65 | List singerSongList = new ArrayList<>(); 66 | String intro = ""; 67 | if(artistsList!=null && artistsList.size()>0){ 68 | for(Artists artists:artistsList){ 69 | SingerSong singerSong = new SingerSong(); 70 | singerSong.setSingerId(artists.getId()); 71 | singerSong.setSongId((int)songMsg.getId()); 72 | intro+=artists.getName()+"/"; 73 | SingerSong singerSong1 = singerSongService.selectEntityById(singerSong); 74 | if(singerSong1==null){ 75 | singerSongList.add(singerSong); 76 | // singerSongService.insertEntity(singerSong); 77 | } 78 | } 79 | } 80 | if(singerSongList.size()>0){ 81 | for(SingerSong singerSong:singerSongList){ 82 | singerSong.setIntro(intro); 83 | SingerSong singerSong1 = singerSongService.selectEntityById(singerSong); 84 | if(singerSong1==null){ 85 | singerSongService.insertEntity(singerSong); 86 | } 87 | } 88 | } 89 | 90 | 91 | 92 | } 93 | } 94 | } 95 | } 96 | } 97 | 98 | 99 | } 100 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/entity/custom2/SongMsg.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 bejson.com 3 | */ 4 | package com.netMusic.entity.custom2; 5 | import java.util.List; 6 | 7 | /** 8 | * Auto-generated: 2018-06-10 20:8:39 9 | * 10 | * @author bejson.com (i@bejson.com) 11 | * @website http://www.bejson.com/java2pojo/ 12 | */ 13 | public class SongMsg { 14 | 15 | private int publishTime; 16 | private int fee; 17 | private int ftype; 18 | private int no; 19 | private List alias; 20 | private Privilege privilege; 21 | private int djid; 22 | private Album album; 23 | private List artists; 24 | private int score; 25 | private int copyrightId; 26 | private long mvid; 27 | // private String transNames; 28 | private String commentThreadId; 29 | private int type; 30 | private long duration; 31 | private int status; 32 | private String name; 33 | private long id; 34 | public void setPublishTime(int publishTime) { 35 | this.publishTime = publishTime; 36 | } 37 | public int getPublishTime() { 38 | return publishTime; 39 | } 40 | 41 | public void setFee(int fee) { 42 | this.fee = fee; 43 | } 44 | public int getFee() { 45 | return fee; 46 | } 47 | 48 | public void setFtype(int ftype) { 49 | this.ftype = ftype; 50 | } 51 | public int getFtype() { 52 | return ftype; 53 | } 54 | 55 | public void setNo(int no) { 56 | this.no = no; 57 | } 58 | public int getNo() { 59 | return no; 60 | } 61 | 62 | public void setAlias(List alias) { 63 | this.alias = alias; 64 | } 65 | public List getAlias() { 66 | return alias; 67 | } 68 | 69 | public void setPrivilege(Privilege privilege) { 70 | this.privilege = privilege; 71 | } 72 | public Privilege getPrivilege() { 73 | return privilege; 74 | } 75 | 76 | public void setDjid(int djid) { 77 | this.djid = djid; 78 | } 79 | public int getDjid() { 80 | return djid; 81 | } 82 | 83 | public void setAlbum(Album album) { 84 | this.album = album; 85 | } 86 | public Album getAlbum() { 87 | return album; 88 | } 89 | 90 | public void setArtists(List artists) { 91 | this.artists = artists; 92 | } 93 | public List getArtists() { 94 | return artists; 95 | } 96 | 97 | public void setScore(int score) { 98 | this.score = score; 99 | } 100 | public int getScore() { 101 | return score; 102 | } 103 | 104 | public void setCopyrightId(int copyrightId) { 105 | this.copyrightId = copyrightId; 106 | } 107 | public int getCopyrightId() { 108 | return copyrightId; 109 | } 110 | 111 | public void setMvid(long mvid) { 112 | this.mvid = mvid; 113 | } 114 | public long getMvid() { 115 | return mvid; 116 | } 117 | 118 | // public void setTransNames(String transNames) { 119 | // this.transNames = transNames; 120 | // } 121 | // public String getTransNames() { 122 | // return transNames; 123 | // } 124 | 125 | public void setCommentThreadId(String commentThreadId) { 126 | this.commentThreadId = commentThreadId; 127 | } 128 | public String getCommentThreadId() { 129 | return commentThreadId; 130 | } 131 | 132 | public void setType(int type) { 133 | this.type = type; 134 | } 135 | public int getType() { 136 | return type; 137 | } 138 | 139 | public void setDuration(long duration) { 140 | this.duration = duration; 141 | } 142 | public long getDuration() { 143 | return duration; 144 | } 145 | 146 | public void setStatus(int status) { 147 | this.status = status; 148 | } 149 | public int getStatus() { 150 | return status; 151 | } 152 | 153 | public void setName(String name) { 154 | this.name = name; 155 | } 156 | public String getName() { 157 | return name; 158 | } 159 | 160 | public void setId(long id) { 161 | this.id = id; 162 | } 163 | public long getId() { 164 | return id; 165 | } 166 | 167 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/utils/XunMengUtil.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.utils; 2 | 3 | import com.google.gson.Gson; 4 | import com.netMusic.entity.*; 5 | import com.netMusic.entity.custom.Album; 6 | import com.netMusic.entity.custom.Artists; 7 | import com.netMusic.entity.custom.Msg; 8 | import com.netMusic.entity.custom.Songs; 9 | import org.apache.http.Header; 10 | import org.apache.http.message.BasicHeader; 11 | 12 | import java.util.ArrayList; 13 | import java.util.HashMap; 14 | import java.util.List; 15 | import java.util.Map; 16 | 17 | /** 18 | * 网址：https://www.xunmzy.com/tool/fm/ 19 | * 该网站的请求工具（目前该网站貌似已崩溃，不能使用） 20 | */ 21 | public class XunMengUtil { 22 | 23 | 24 | public static Msg spiderMusic(String url,String offset,String songName){ 25 | 26 | Map map = new HashMap<>(); 27 | String charest = "utf-8"; 28 | //post请求的参数 29 | map.put("types","CloudMusic"); 30 | map.put("SongName",songName); 31 | //偏移量数据 32 | map.put("OffSet",offset); 33 | //增加请求头信息 34 | List
headerList = new ArrayList<>(); 35 | headerList.add(new BasicHeader("origin","https://www.xunmzy.com")); 36 | headerList.add(new BasicHeader("referer","https://www.xunmzy.com/tool/fm/")); 37 | headerList.add(new BasicHeader("user-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64)" + 38 | " AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36")); 39 | //模拟post请求 40 | String resJson = HttpClientUtil.doPost(url,map,charest,headerList); 41 | Gson gson = new Gson(); 42 | Msg msg = null; 43 | if(resJson!=null && !resJson.contains("html")){ 44 | msg = gson.fromJson(resJson,Msg.class); 45 | } 46 | 47 | return msg; 48 | 49 | 50 | 51 | } 52 | 53 | 54 | 55 | public static List> getResultList(String songName){ 56 | 57 | String url = "https://www.xunmzy.com/tool/fm/music/index.php"; 58 | List> listAll = new ArrayList<>(); 59 | //一次爬600首 60 | for(int i=0;i<20;i++){ 61 | Msg msg = spiderMusic(url,i*30+"",songName); 62 | System.out.println(msg); 63 | //要判空 64 | if(msg!=null){ 65 | List list = msg.getResult().getSongs(); 66 | listAll.add(list); 67 | } 68 | } 69 | return listAll; 70 | 71 | 72 | 73 | } 74 | 75 | public static Singer saveSinger(Songs songs) { 76 | Singer singer = new Singer(); 77 | List artists = songs.getArtists(); 78 | for(Artists artist:artists){ 79 | singer.setId(artist.getId()); 80 | singer.setName(artist.getName()); 81 | singer.setPicUrl(artist.getPicUrl()); 82 | 83 | } 84 | return singer; 85 | } 86 | 87 | public static Record saveRecord(Songs songs) { 88 | Record record = new Record(); 89 | Album album = songs.getAlbum(); 90 | record.setId(album.getId()); 91 | record.setCommentThreadId(album.getCommentThreadId()); 92 | record.setCompany(album.getCompany()); 93 | record.setIntro(album.getDescription()); 94 | record.setName(album.getName()); 95 | record.setPicUrl(album.getPicUrl()); 96 | record.setSingerId(album.getArtist().getId()); 97 | 98 | return record; 99 | } 100 | 101 | ////用自定义javaBean保存数据 102 | public static Song saveSong(Songs songs) { 103 | Song song = new Song(); 104 | song.setId(songs.getId()); 105 | song.setName(songs.getName()); 106 | song.setCommentThreadId(songs.getCommentThreadId()); 107 | song.setRecordId(songs.getAlbum().getId()); 108 | song.setMp3url(songs.getMp3Url()); 109 | 110 | return song; 111 | 112 | } 113 | 114 | //保存key 115 | public static List saveSingerSong(Songs songs){ 116 | 117 | List singerSongList = new ArrayList<>(); 118 | List artistsList = songs.getArtists(); 119 | if(artistsList!=null){ 120 | for(Artists artists:artistsList){ 121 | SingerSong key = new SingerSong(); 122 | key.setSingerId(artists.getId()); 123 | key.setSongId((int)songs.getId()); 124 | singerSongList.add(key); 125 | } 126 | } 127 | 128 | 129 | return singerSongList; 130 | 131 | } 132 | 133 | 134 | 135 | } 136 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/service/impl/XunMengService.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.service.impl; 2 | 3 | import com.netMusic.dao.RecordMapper; 4 | import com.netMusic.dao.SingerMapper; 5 | import com.netMusic.dao.SingerSongMapper; 6 | import com.netMusic.dao.SongMapper; 7 | import com.netMusic.entity.*; 8 | import com.netMusic.entity.custom.Songs; 9 | import com.netMusic.utils.XunMengUtil; 10 | import org.springframework.beans.factory.annotation.Autowired; 11 | import org.springframework.stereotype.Service; 12 | 13 | import java.util.List; 14 | 15 | @Service 16 | public class XunMengService { 17 | 18 | @Autowired 19 | private RecordMapper recordMapper; 20 | @Autowired 21 | private SingerMapper singerMapper; 22 | @Autowired 23 | private SongMapper songMapper; 24 | @Autowired 25 | private SingerSongMapper singerSongMapper; 26 | 27 | 28 | public void saveAllBySongName(String songName) { 29 | 30 | if (!"".equals(songName)) { 31 | 32 | List> listAll = XunMengUtil.getResultList(songName); 33 | 34 | if (listAll != null) { 35 | for (List list : listAll) { 36 | if (list != null) { 37 | for (Songs songs : list) { 38 | //用自定义javaBean保存数据 39 | Song song = XunMengUtil.saveSong(songs); 40 | //保存唱片 41 | Record record = XunMengUtil.saveRecord(songs); 42 | //保存歌手 43 | Singer singer = XunMengUtil.saveSinger(songs); 44 | //保存中间键 45 | List singerSongList = XunMengUtil.saveSingerSong(songs); 46 | if (song != null) { 47 | Song song1 = songMapper.selectByPrimaryKey(song.getId()); 48 | if (song1 == null) { 49 | songMapper.insert(song); 50 | //保存中间健 51 | if (singerSongList != null) { 52 | for (SingerSong singerSong : singerSongList) { 53 | if (singerSong != null && singerSong.getSongId() == song.getId().intValue()) { 54 | //保存singerId 55 | SingerSongKey singerSongKey1 = singerSongMapper.selectBySingerId(singerSong); 56 | if (singerSongKey1 == null) { 57 | singerSongMapper.insert(singerSong); 58 | } 59 | } 60 | } 61 | } 62 | 63 | } 64 | } 65 | 66 | if (singer != null) { 67 | Singer singer1 = singerMapper.selectByPrimaryKey(singer.getId()); 68 | if (singer1 == null) { 69 | singerMapper.insert(singer); 70 | //保存中间健 71 | if (singerSongList != null) { 72 | for (SingerSongKey singerSongKey : singerSongList) { 73 | if (singerSongKey != null && singerSongKey.getSingerId() == singer.getId().intValue()) { 74 | //保存songId 75 | SingerSong singerSong1= singerSongMapper.selectBySingerId(singerSongKey); 76 | if (singerSong1 == null) { 77 | singerSongMapper.insert(singerSong1); 78 | } 79 | } 80 | } 81 | } 82 | } 83 | } 84 | if (record != null) { 85 | Record record1 = recordMapper.selectByPrimaryKey(record.getId()); 86 | if (record1 == null) { 87 | recordMapper.insert(record); 88 | } 89 | } 90 | 91 | } 92 | } 93 | } 94 | 95 | } 96 | 97 | } 98 | 99 | } 100 | 101 | 102 | } 103 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/entity/custom/Album.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 bejson.com 3 | */ 4 | package com.netMusic.entity.custom; 5 | import java.util.List; 6 | 7 | /** 8 | * Auto-generated: 2018-06-06 20:59:46 9 | * 10 | * @author bejson.com (i@bejson.com) 11 | * @website http://www.bejson.com/java2pojo/ 12 | */ 13 | public class Album { 14 | 15 | private String name; 16 | private int id; 17 | private String type; 18 | private int size; 19 | private long picId; 20 | private String blurPicUrl; 21 | private int companyId; 22 | private long pic; 23 | private String picUrl; 24 | private long publishTime; 25 | private String description; 26 | private String tags; 27 | private String company; 28 | private String briefDesc; 29 | private Artist artist; 30 | private List songs; 31 | private List alias; 32 | private int status; 33 | private int copyrightId; 34 | private String commentThreadId; 35 | private List artists; 36 | private String picId_str; 37 | public void setName(String name) { 38 | this.name = name; 39 | } 40 | public String getName() { 41 | return name; 42 | } 43 | 44 | public void setId(int id) { 45 | this.id = id; 46 | } 47 | public int getId() { 48 | return id; 49 | } 50 | 51 | public void setType(String type) { 52 | this.type = type; 53 | } 54 | public String getType() { 55 | return type; 56 | } 57 | 58 | public void setSize(int size) { 59 | this.size = size; 60 | } 61 | public int getSize() { 62 | return size; 63 | } 64 | 65 | public void setPicId(long picId) { 66 | this.picId = picId; 67 | } 68 | public long getPicId() { 69 | return picId; 70 | } 71 | 72 | public void setBlurPicUrl(String blurPicUrl) { 73 | this.blurPicUrl = blurPicUrl; 74 | } 75 | public String getBlurPicUrl() { 76 | return blurPicUrl; 77 | } 78 | 79 | public void setCompanyId(int companyId) { 80 | this.companyId = companyId; 81 | } 82 | public int getCompanyId() { 83 | return companyId; 84 | } 85 | 86 | public void setPic(long pic) { 87 | this.pic = pic; 88 | } 89 | public long getPic() { 90 | return pic; 91 | } 92 | 93 | public void setPicUrl(String picUrl) { 94 | this.picUrl = picUrl; 95 | } 96 | public String getPicUrl() { 97 | return picUrl; 98 | } 99 | 100 | public void setPublishTime(long publishTime) { 101 | this.publishTime = publishTime; 102 | } 103 | public long getPublishTime() { 104 | return publishTime; 105 | } 106 | 107 | public void setDescription(String description) { 108 | this.description = description; 109 | } 110 | public String getDescription() { 111 | return description; 112 | } 113 | 114 | public void setTags(String tags) { 115 | this.tags = tags; 116 | } 117 | public String getTags() { 118 | return tags; 119 | } 120 | 121 | public void setCompany(String company) { 122 | this.company = company; 123 | } 124 | public String getCompany() { 125 | return company; 126 | } 127 | 128 | public void setBriefDesc(String briefDesc) { 129 | this.briefDesc = briefDesc; 130 | } 131 | public String getBriefDesc() { 132 | return briefDesc; 133 | } 134 | 135 | public void setArtist(Artist artist) { 136 | this.artist = artist; 137 | } 138 | public Artist getArtist() { 139 | return artist; 140 | } 141 | 142 | public void setSongs(List songs) { 143 | this.songs = songs; 144 | } 145 | public List getSongs() { 146 | return songs; 147 | } 148 | 149 | public void setAlias(List alias) { 150 | this.alias = alias; 151 | } 152 | public List getAlias() { 153 | return alias; 154 | } 155 | 156 | public void setStatus(int status) { 157 | this.status = status; 158 | } 159 | public int getStatus() { 160 | return status; 161 | } 162 | 163 | public void setCopyrightId(int copyrightId) { 164 | this.copyrightId = copyrightId; 165 | } 166 | public int getCopyrightId() { 167 | return copyrightId; 168 | } 169 | 170 | public void setCommentThreadId(String commentThreadId) { 171 | this.commentThreadId = commentThreadId; 172 | } 173 | public String getCommentThreadId() { 174 | return commentThreadId; 175 | } 176 | 177 | public void setArtists(List artists) { 178 | this.artists = artists; 179 | } 180 | public List getArtists() { 181 | return artists; 182 | } 183 | 184 | public void setPicId_str(String picId_str) { 185 | this.picId_str = picId_str; 186 | } 187 | public String getPicId_str() { 188 | return picId_str; 189 | } 190 | 191 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/Demo/DocumentDemo.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.Demo; 2 | 3 | import org.jsoup.Jsoup; 4 | import org.jsoup.nodes.Document; 5 | import org.jsoup.nodes.Element; 6 | import org.jsoup.select.Elements; 7 | 8 | import java.io.IOException; 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | 12 | /** 13 | * Decument测试，利用Jsoup解析 14 | * 爬取知乎推荐Demo 15 | * 存在问题： 16 | * 1、拿到的页面不是完整的页面，只包含两个答案！！！ 17 | * （需要使用HttpUtil进行js响应）(或者通过抓包模拟请求拿到api) 18 | * 2、在遍历时未将回答用户和回答内容分开（已分） 19 | */ 20 | public class DocumentDemo { 21 | 22 | public static void main(String[] args){ 23 | String url = "https://www.zhihu.com/explore/recommendations"; 24 | Map cookies = null; 25 | try { 26 | cookies = new HashMap<>(); 27 | cookies.put("_xsrf","5f79c6ff-89f2-4a18-95bb-74b8d61045db"); 28 | cookies.put("_zap","ca28f69f-0954-4d63-b64b-4584f55cc70f"); 29 | cookies.put("capsion_ticket","2|1:0|10:1528194797|14:capsion_ticket|44:OTA4M2U1NmRkYjgzNGZiODllMmNjYjE0NTBmYWE3Njc=|523be751c42809bb43f6f6ff6da5bd5f478810abf7c7d61dafd4148f75d2286b"); 30 | cookies.put("d_c0","AFCl2Sn0sw2PTi5LJlMvkdzRG-9gvoCLuJ0=|1528194792"); 31 | cookies.put("q_c1","58d6c8c5b562461fa5ddb9aad2e4d2b5|1528194792000|1528194792000"); 32 | cookies.put("z_c0","2|1:0|10:1528194799|4:z_c0|92:Mi4xRXhQeEFBQUFBQUFBVUtYWktmU3pEU1lBQUFCZ0FsVk43N1FEWEFDNEZXWERMNnNnV0c1cVNZaGY2ai1TNko0dkJn|0989639006f63d51d2082e7a4b065c2b126b003c17c36cb72b5007077e88746c"); 33 | Document document = Jsoup.connect(url).timeout(10000) 34 | //模拟火狐浏览器 35 | .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36") 36 | .get(); 37 | 38 | Element main = document.getElementById("zh-recommend-list-full"); 39 | Elements qUrl = main.select("div:nth-child(2)").select("h2") 40 | .select("a[class=question_link]"); 41 | for(Element question:qUrl){ 42 | //abs:href表示拿到绝对路径 43 | String URL = question.attr("abs:href"); 44 | //下载URL对应的页面,忽略知乎专栏 45 | if(URL.contains("zhuanlan") || "".equals(URL)){ 46 | //专栏的数据 47 | System.out.println("知乎专栏。。"); 48 | continue; 49 | }else{ 50 | System.out.println("正在爬取数据。。。"); 51 | //爬知乎时注意选择浏览器的版本，过低的版本访问不了知乎 52 | Document qDoc = Jsoup.connect(URL).timeout(10000) 53 | .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36") 54 | .get(); 55 | //可直接用选择器 56 | Elements questionMain = qDoc.select("#root > div > main > div > div:nth-child(11) > div.QuestionHeader > div.QuestionHeader-content > div.QuestionHeader-main"); 57 | Elements title = questionMain.select("h1"); 58 | Elements detail = questionMain.select("div:nth-child(3)") 59 | .select("div") 60 | .select("div") 61 | .select("div") 62 | .select("span"); 63 | 64 | 65 | Elements qDocAllUrl = qDoc.select("#root > div > main > div > div.Question-main > div > div:nth-child(3) > a"); 66 | //拿到全部答案的url 67 | String qAllUrl = qDocAllUrl.attr("abs:href"); 68 | //解析url 69 | Document qDocAll = Jsoup.connect(qAllUrl) 70 | .cookies(cookies) 71 | .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36") 72 | .get(); 73 | //拿到main回答 74 | Elements ansMain = qDocAll.select("#QuestionAnswers-answers"); 75 | //拿到回答数 76 | Elements ansNum = ansMain.select("div > div > div.List-header > h4 > span"); 77 | 78 | System.out.println("\n"+"链接："+URL 79 | +"\n"+"标题："+title.text() 80 | +"\n"+"问题描述："+detail.text()); 81 | System.out.println("\n"+"问题回答数："+ansNum.text()); 82 | //拿到回答列表 83 | Elements ansList = qDocAll.select("#QuestionAnswers-answers > div > div > div:nth-child(2) > div > div.List-item"); 84 | //遍历列表 85 | for(Element ans:ansList){ 86 | //回答内容 87 | Elements content = ans.select("div.RichContent.RichContent--unescapable > div.RichContent-inner > span"); 88 | //回答者姓名 89 | Elements username = ans.select("#null-toggle > a"); 90 | System.out.println("\n回答用户；" +username.text()+ 91 | "\n回答内容："+content.text()); 92 | 93 | } 94 | 95 | 96 | 97 | } 98 | 99 | 100 | } 101 | 102 | 103 | } catch (IOException e) { 104 | e.printStackTrace(); 105 | } 106 | 107 | 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/Demo/HttpClientUtil.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.Demo; 2 | 3 | import com.google.gson.Gson; 4 | import org.apache.http.HttpEntity; 5 | import org.apache.http.HttpResponse; 6 | import org.apache.http.NameValuePair; 7 | import org.apache.http.client.HttpClient; 8 | import org.apache.http.client.entity.UrlEncodedFormEntity; 9 | import org.apache.http.client.methods.HttpPost; 10 | import org.apache.http.impl.client.HttpClients; 11 | import org.apache.http.message.BasicNameValuePair; 12 | import org.apache.http.util.EntityUtils; 13 | import org.jsoup.Jsoup; 14 | import org.jsoup.nodes.Document; 15 | import org.jsoup.nodes.Element; 16 | import org.jsoup.select.Elements; 17 | 18 | import java.io.File; 19 | import java.io.FileWriter; 20 | import java.util.*; 21 | 22 | /** 23 | * 爬取知乎推荐 24 | */ 25 | public class HttpClientUtil { 26 | 27 | public static String doPost(String url, Map map, String charset){ 28 | HttpClient httpClient = null; 29 | HttpPost httpPost = null; 30 | String result = null; 31 | try{ 32 | httpClient = HttpClients.createDefault(); 33 | httpPost = new HttpPost(url); 34 | //设置参数 35 | List list = new ArrayList(); 36 | Iterator iterator = map.entrySet().iterator(); 37 | while(iterator.hasNext()){ 38 | Map.Entry elem = (Map.Entry) iterator.next(); 39 | list.add(new BasicNameValuePair(elem.getKey(),(String) elem.getValue())); 40 | } 41 | if(list.size() > 0){ 42 | UrlEncodedFormEntity entity = new UrlEncodedFormEntity(list,charset); 43 | httpPost.setEntity(entity); 44 | } 45 | HttpResponse response = httpClient.execute(httpPost); 46 | if(response != null){ 47 | HttpEntity resEntity = response.getEntity(); 48 | if(resEntity != null){ 49 | result = EntityUtils.toString(resEntity,charset); 50 | } 51 | } 52 | }catch(Exception ex){ 53 | ex.printStackTrace(); 54 | } 55 | return result; 56 | } 57 | 58 | 59 | public static void spiderZH2(String url,int offset){ 60 | try { 61 | //String curl ="https://www.zhihu.com/node/ExploreRecommendListV2"; 62 | Map createMap = new HashMap(); 63 | String charset = "utf-8"; 64 | // method 提交的参数 65 | createMap.put("method", "next"); 66 | Map map = new HashMap(); 67 | // 分页显示的数据 68 | map.put("limit", 20); 69 | map.put("offset", offset); 70 | createMap.put("method", "next"); 71 | Gson gson = new Gson(); 72 | String mapStr = gson.toJson(map); 73 | // 请求的参数 74 | createMap.put("params", mapStr); 75 | // 根据httpclient模仿post请求 76 | String httpOrgCreateTestRtn = HttpClientUtil.doPost(url,createMap,charset); 77 | Map maps = gson.fromJson(httpOrgCreateTestRtn, Map.class); 78 | String html = maps.get("msg").toString(); 79 | Document doc = Jsoup.parse(html); 80 | Elements elements = 81 | doc.select("div[class=zm-item]").select("h2"). 82 | select("a[class=question_link]"); 83 | File file = new File("D:\\Code\\TemporaryCode\\zhihuwenda.txt"); 84 | // 遍历每个问题节点 85 | for (Element question : elements) { 86 | // 获取连接地址 87 | String qUrl = question.attr("href"); 88 | // 这里需要判断urlhttp格式 89 | if(!qUrl.contains("https://")){ 90 | qUrl = "https://www.zhihu.com"+qUrl; 91 | } 92 | Document document2=Jsoup.connect(qUrl) 93 | .userAgent("Mozilla/5.0 " 94 | + "(iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) " 95 | + "AppleWebKit/533.17.9" 96 | + " (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5") 97 | .get(); 98 | // 问题标题 99 | Elements title = document2.select("#root").select("div").select("main"). 100 | select("div").select("div:nth-child(11)").select("div.QuestionHeader"). 101 | select("div.QuestionHeader-content").select("div.QuestionHeader-main"). 102 | select("h1"); 103 | // 回答问题的内容 104 | Elements content = document2.select("#root").select("div").select("main"). 105 | select("div").select("div.Question-main").select("div.Question-mainColumn"). 106 | select("div.Card.AnswerCard").select("div").select("div"). 107 | select("div.RichContent.RichContent--unescapable"). 108 | select("div.RichContent-inner"); 109 | if (!file.getParentFile().exists()) {//判断路径是否存在，如果不存在，则创建上一级目录文件夹 110 | file.getParentFile().mkdirs(); 111 | } 112 | FileWriter fileWriter=new FileWriter(file, true); 113 | fileWriter.write("=============链接:"+qUrl+"\r\n"); 114 | fileWriter.write("=============标题:"+title.get(0).text()+"\r\n"); 115 | fileWriter.write("=============回答:"+content.get(0).text()+"\r\n"); 116 | fileWriter.close(); 117 | } 118 | 119 | } catch (Exception e) { 120 | // TODO Auto-generated catch block 121 | e.printStackTrace(); 122 | } 123 | } 124 | 125 | public static void main(String [] args){ 126 | // 这里采用循环的方式去除列表 127 | String url = "https://www.zhihu.com/node/ExploreRecommendListV2"; 128 | for(int i=1;i<1000;i++){ 129 | spiderZH2(url,59+i*20); 130 | } 131 | } 132 | 133 | 134 | 135 | 136 | } 137 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.reptilian.test 8 | ReptilianDemo 9 | 1.0-SNAPSHOT 10 | war 11 | 12 | 13 | UTF-8 14 | UTF-8 15 | 1.8 16 | 17 | 18 | 19 | org.springframework.boot 20 | spring-boot-starter-parent 21 | 2.0.2.RELEASE 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | org.mybatis.spring.boot 31 | mybatis-spring-boot-starter 32 | 1.3.2 33 | 34 | 35 | 36 | org.springframework.boot 37 | spring-boot-starter-web 38 | 39 | 40 | 41 | org.springframework.boot 42 | spring-boot-starter-test 43 | test 44 | 45 | 46 | 47 | org.springframework 48 | spring-test 49 | 50 | 51 | junit 52 | junit 53 | 4.12 54 | 55 | 56 | 57 | org.hamcrest 58 | hamcrest-all 59 | 1.3 60 | 61 | 62 | 63 | org.springframework.boot 64 | spring-boot-test-autoconfigure 65 | 66 | 67 | 68 | org.springframework.boot 69 | spring-boot-configuration-processor 70 | true 71 | 72 | 73 | 74 | mysql 75 | mysql-connector-java 76 | 5.1.26 77 | 78 | 79 | org.springframework.boot 80 | spring-boot-starter-data-jpa 81 | 82 | 83 | 84 | org.springframework.boot 85 | spring-boot-starter-aop 86 | 87 | 88 | 89 | org.apache.httpcomponents 90 | httpclient 91 | 4.5.2 92 | 93 | 94 | 95 | org.apache.httpcomponents 96 | httpclient-cache 97 | 4.5 98 | 99 | 100 | 101 | org.apache.httpcomponents 102 | httpmime 103 | 4.3.2 104 | 105 | 106 | 107 | 108 | org.jsoup 109 | jsoup 110 | 1.10.3 111 | 112 | 113 | 114 | 115 | 116 | net.sourceforge.htmlunit 117 | htmlunit 118 | 2.29 119 | 120 | 121 | 122 | 123 | 124 | com.google.code.gson 125 | gson 126 | 2.8.2 127 | 128 | 129 | 130 | 131 | us.codecraft 132 | webmagic-core 133 | 0.5.3 134 | 135 | 136 | us.codecraft 137 | webmagic-extension 138 | 0.5.3 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | org.springframework.boot 147 | spring-boot-maven-plugin 148 | 149 | 150 | 151 | org.mybatis.generator 152 | mybatis-generator-maven-plugin 153 | 1.3.2 154 | 155 | true 156 | 157 | 158 | 159 | 160 | org.springframework.boot 161 | spring-boot-maven-plugin 162 | 163 | 164 | 165 | org.mybatis.generator 166 | mybatis-generator-maven-plugin 167 | 1.3.2 168 | 169 | ${basedir}/src/main/resources/generator/generatorConfig.xml 170 | true 171 | true 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/entity/custom/Songs.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 bejson.com 3 | */ 4 | package com.netMusic.entity.custom; 5 | import java.util.List; 6 | 7 | /** 8 | * Auto-generated: 2018-06-06 20:59:46 9 | * 10 | * @author bejson.com (i@bejson.com) 11 | * @website http://www.bejson.com/java2pojo/ 12 | */ 13 | public class Songs { 14 | 15 | private String name; 16 | private long id; 17 | private int position; 18 | private List alias; 19 | private int status; 20 | private int fee; 21 | private int copyrightId; 22 | private String disc; 23 | private int no; 24 | private List artists; 25 | private Album album; 26 | private boolean starred; 27 | private int popularity; 28 | private int score; 29 | private int starredNum; 30 | private long duration; 31 | private int playedNum; 32 | private int dayPlays; 33 | private int hearTime; 34 | private String ringtone; 35 | private String crbt; 36 | private String audition; 37 | private String copyFrom; 38 | private String commentThreadId; 39 | private String rtUrl; 40 | private int ftype; 41 | private List rtUrls; 42 | private int copyright; 43 | private String mp3Url; 44 | private int rtype; 45 | private String rurl; 46 | private long mvid; 47 | public void setName(String name) { 48 | this.name = name; 49 | } 50 | public String getName() { 51 | return name; 52 | } 53 | 54 | public void setId(long id) { 55 | this.id = id; 56 | } 57 | public long getId() { 58 | return id; 59 | } 60 | 61 | public void setPosition(int position) { 62 | this.position = position; 63 | } 64 | public int getPosition() { 65 | return position; 66 | } 67 | 68 | public void setAlias(List alias) { 69 | this.alias = alias; 70 | } 71 | public List getAlias() { 72 | return alias; 73 | } 74 | 75 | public void setStatus(int status) { 76 | this.status = status; 77 | } 78 | public int getStatus() { 79 | return status; 80 | } 81 | 82 | public void setFee(int fee) { 83 | this.fee = fee; 84 | } 85 | public int getFee() { 86 | return fee; 87 | } 88 | 89 | public void setCopyrightId(int copyrightId) { 90 | this.copyrightId = copyrightId; 91 | } 92 | public int getCopyrightId() { 93 | return copyrightId; 94 | } 95 | 96 | public void setDisc(String disc) { 97 | this.disc = disc; 98 | } 99 | public String getDisc() { 100 | return disc; 101 | } 102 | 103 | public void setNo(int no) { 104 | this.no = no; 105 | } 106 | public int getNo() { 107 | return no; 108 | } 109 | 110 | public void setArtists(List artists) { 111 | this.artists = artists; 112 | } 113 | public List getArtists() { 114 | return artists; 115 | } 116 | 117 | public void setAlbum(Album album) { 118 | this.album = album; 119 | } 120 | public Album getAlbum() { 121 | return album; 122 | } 123 | 124 | public void setStarred(boolean starred) { 125 | this.starred = starred; 126 | } 127 | public boolean getStarred() { 128 | return starred; 129 | } 130 | 131 | public void setPopularity(int popularity) { 132 | this.popularity = popularity; 133 | } 134 | public int getPopularity() { 135 | return popularity; 136 | } 137 | 138 | public void setScore(int score) { 139 | this.score = score; 140 | } 141 | public int getScore() { 142 | return score; 143 | } 144 | 145 | public void setStarredNum(int starredNum) { 146 | this.starredNum = starredNum; 147 | } 148 | public int getStarredNum() { 149 | return starredNum; 150 | } 151 | 152 | public void setDuration(long duration) { 153 | this.duration = duration; 154 | } 155 | public long getDuration() { 156 | return duration; 157 | } 158 | 159 | public void setPlayedNum(int playedNum) { 160 | this.playedNum = playedNum; 161 | } 162 | public int getPlayedNum() { 163 | return playedNum; 164 | } 165 | 166 | public void setDayPlays(int dayPlays) { 167 | this.dayPlays = dayPlays; 168 | } 169 | public int getDayPlays() { 170 | return dayPlays; 171 | } 172 | 173 | public void setHearTime(int hearTime) { 174 | this.hearTime = hearTime; 175 | } 176 | public int getHearTime() { 177 | return hearTime; 178 | } 179 | 180 | public void setRingtone(String ringtone) { 181 | this.ringtone = ringtone; 182 | } 183 | public String getRingtone() { 184 | return ringtone; 185 | } 186 | 187 | public void setCrbt(String crbt) { 188 | this.crbt = crbt; 189 | } 190 | public String getCrbt() { 191 | return crbt; 192 | } 193 | 194 | public void setAudition(String audition) { 195 | this.audition = audition; 196 | } 197 | public String getAudition() { 198 | return audition; 199 | } 200 | 201 | public void setCopyFrom(String copyFrom) { 202 | this.copyFrom = copyFrom; 203 | } 204 | public String getCopyFrom() { 205 | return copyFrom; 206 | } 207 | 208 | public void setCommentThreadId(String commentThreadId) { 209 | this.commentThreadId = commentThreadId; 210 | } 211 | public String getCommentThreadId() { 212 | return commentThreadId; 213 | } 214 | 215 | public void setRtUrl(String rtUrl) { 216 | this.rtUrl = rtUrl; 217 | } 218 | public String getRtUrl() { 219 | return rtUrl; 220 | } 221 | 222 | public void setFtype(int ftype) { 223 | this.ftype = ftype; 224 | } 225 | public int getFtype() { 226 | return ftype; 227 | } 228 | 229 | public void setRtUrls(List rtUrls) { 230 | this.rtUrls = rtUrls; 231 | } 232 | public List getRtUrls() { 233 | return rtUrls; 234 | } 235 | 236 | public void setCopyright(int copyright) { 237 | this.copyright = copyright; 238 | } 239 | public int getCopyright() { 240 | return copyright; 241 | } 242 | 243 | 244 | public void setMp3Url(String mp3Url) { 245 | this.mp3Url = mp3Url; 246 | } 247 | public String getMp3Url() { 248 | return mp3Url; 249 | } 250 | 251 | public void setRtype(int rtype) { 252 | this.rtype = rtype; 253 | } 254 | public int getRtype() { 255 | return rtype; 256 | } 257 | 258 | public void setRurl(String rurl) { 259 | this.rurl = rurl; 260 | } 261 | public String getRurl() { 262 | return rurl; 263 | } 264 | 265 | public void setMvid(long mvid) { 266 | this.mvid = mvid; 267 | } 268 | public long getMvid() { 269 | return mvid; 270 | } 271 | 272 | 273 | } -------------------------------------------------------------------------------- /src/main/java/com/netMusic/Demo/HttpClientDemo2.java: -------------------------------------------------------------------------------- 1 | //package com.netMusic.Demo; 2 | // 3 | //import com.google.gson.Gson; 4 | //import com.netMusic.entity.Record; 5 | //import com.netMusic.entity.Singer; 6 | //import com.netMusic.entity.Song; 7 | //import com.netMusic.entity.custom.*; 8 | //import com.netMusic.repository.RecordRepository; 9 | //import com.netMusic.repository.SingerRepository; 10 | //import com.netMusic.repository.SongRepository; 11 | //import org.apache.http.HttpEntity; 12 | //import org.apache.http.HttpResponse; 13 | //import org.apache.http.NameValuePair; 14 | //import org.apache.http.client.HttpClient; 15 | //import org.apache.http.client.entity.UrlEncodedFormEntity; 16 | //import org.apache.http.client.methods.HttpPost; 17 | //import org.apache.http.impl.client.HttpClients; 18 | //import org.apache.http.message.BasicNameValuePair; 19 | //import org.apache.http.util.EntityUtils; 20 | //import org.springframework.beans.factory.annotation.Autowired; 21 | //import org.springframework.stereotype.Component; 22 | // 23 | //import java.util.*; 24 | // 25 | ///** 26 | // * 模拟post请求，获取返回数据 27 | // */ 28 | //@Component 29 | //public class HttpClientDemo2 { 30 | // 31 | // @Autowired 32 | // private RecordRepository recordRepository; 33 | // @Autowired 34 | // private SingerRepository singerRepository; 35 | // @Autowired 36 | // private SongRepository songRepository; 37 | // 38 | // 39 | // //拿到音乐的json数据 40 | // public static String doPost(String url, Map map, String charest) { 41 | // 42 | // HttpClient httpClient = null; 43 | // HttpPost httpPost = null; 44 | // String result = null; 45 | // try { 46 | // httpClient = HttpClients.createDefault(); 47 | // httpPost = new HttpPost(url); 48 | // //设置请求头，反反盗链 49 | // httpPost.setHeader("origin","https://www.xunmzy.com"); 50 | // httpPost.setHeader("referer","https://www.xunmzy.com/tool/fm/"); 51 | // httpPost.setHeader("user-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"); 52 | // //设置参数 53 | // List list = new ArrayList<>(); 54 | // Iterator iterator = map.entrySet().iterator(); 55 | // while (iterator.hasNext()) { 56 | // Map.Entry elem = (Map.Entry) iterator.next(); 57 | // list.add(new BasicNameValuePair(elem.getKey(), (String) elem.getValue())); 58 | // } 59 | // if (list.size() > 0) { 60 | // //封装实体 61 | // UrlEncodedFormEntity entity = new UrlEncodedFormEntity(list, charest); 62 | // httpPost.setEntity(entity); 63 | // } 64 | // //拿到返回结果 65 | // HttpResponse response = httpClient.execute(httpPost); 66 | // if (response != null) { 67 | // HttpEntity resEntity = response.getEntity(); 68 | // if (resEntity != null) { 69 | // result = EntityUtils.toString(resEntity, charest); 70 | // } 71 | // } 72 | // 73 | // } catch (Exception e) { 74 | // e.printStackTrace(); 75 | // } 76 | // 77 | // return result; 78 | // 79 | // } 80 | // 81 | // 82 | // public static Msg spiderMusic(String url,String offset,String songName){ 83 | // 84 | // Map map = new HashMap<>(); 85 | // String charest = "utf-8"; 86 | // //post请求的参数 87 | // map.put("types","CloudMusic"); 88 | // map.put("SongName",songName); 89 | // //偏移量数据 90 | // map.put("OffSet",offset); 91 | // //模拟post请求 92 | // String resJson = doPost(url,map,charest); 93 | // //利用Gson转化为javaBean 94 | // Gson gson = new Gson(); 95 | //// Type type = new TypeToken>() {}.getType(); 96 | //// Map creatMap = gson.fromJson(resJson,type); 97 | //// Iterator iterator = creatMap.entrySet().iterator(); 98 | //// if(iterator.hasNext()){ 99 | //// Map.Entry elem = (Map.Entry) iterator.next(); 100 | //// System.out.println(elem.getKey()+":"+elem.getValue()); 101 | //// } 102 | // //转为msg 103 | // 104 | // Msg msg = gson.fromJson(resJson.toString(),Msg.class); 105 | // 106 | // return msg; 107 | // 108 | // 109 | // 110 | // } 111 | // 112 | // 113 | // public static void main(String[] args){ 114 | // 115 | // HttpClientDemo2 httpClientDemo2 = new HttpClientDemo2(); 116 | // 117 | // String url = "https://www.xunmzy.com/tool/fm/music/index.php"; 118 | // for(int i=0;i<20;i++){ 119 | // Msg msg = spiderMusic(url,i*30+"","陈奕迅"); 120 | // System.out.println(msg); 121 | // 122 | // List list = msg.getResult().getSongs(); 123 | // 124 | // System.out.println(list.size()); 125 | // 126 | // for(Songs songs:list){ 127 | // //用自定义javaBean保存数据 128 | // httpClientDemo2.saveSong(songs); 129 | // //保存唱片 130 | // httpClientDemo2.saveRecord(songs); 131 | // //保存歌手 132 | // httpClientDemo2.saveSinger(songs); 133 | // } 134 | // 135 | // } 136 | // 137 | // 138 | // 139 | // } 140 | // 141 | // private void saveSinger(Songs songs) { 142 | // Singer singer = new Singer(); 143 | // List artists = songs.getArtists(); 144 | // for(Artists artist:artists){ 145 | // singer.setId(artist.getId()); 146 | // singer.setName(artist.getName()); 147 | // singer.setAlbumSize(artist.getAlbumSize()); 148 | // singer.setImg1v1Id(artist.getImg1v1Id()); 149 | // singer.setMusicSize(artist.getMusicSize()); 150 | // singer.setPicId(artist.getPicId()); 151 | // singer.setPicUrl(artist.getPicUrl()); 152 | // singer.setImg1v1Url(artist.getImg1v1Url()); 153 | // //save操作。。。 154 | // Singer singerTemp = singerRepository.getOne(singer.getId()); 155 | // if(singerTemp.getId()==null){ 156 | // singerRepository.save(singer); 157 | // } 158 | // 159 | // 160 | // } 161 | // } 162 | // 163 | // private void saveRecord(Songs songs) { 164 | // Record record = new Record(); 165 | // Album album = songs.getAlbum(); 166 | // record.setId(album.getId()); 167 | // record.setCommentThreadId(album.getCommentThreadId()); 168 | // record.setCompany(album.getCompany()); 169 | // record.setDescription(album.getDescription()); 170 | // record.setName(album.getName()); 171 | // record.setPicUrl(album.getPicUrl()); 172 | // record.setSingerId(album.getArtist().getId()); 173 | // record.setTags(album.getTags()); 174 | // record.setType(album.getType()); 175 | // 176 | // //save操作。。 177 | // Record recordTemp = recordRepository.getOne(record.getId()); 178 | // if(recordTemp.getId()==null){ 179 | // recordRepository.save(record); 180 | // } 181 | // 182 | // } 183 | // 184 | // ////用自定义javaBean保存数据 185 | // private void saveSong(Songs songs) { 186 | // Song song = new Song(); 187 | // song.setId(songs.getId()); 188 | // song.setName(songs.getName()); 189 | // song.setCommentThreadId(songs.getCommentThreadId()); 190 | // song.setMp3Url(songs.getMp3Url()); 191 | // song.setMvid(songs.getMvid()); 192 | // song.setRecordId(songs.getAlbum().getId()); 193 | // //保存song。。。 194 | // songRepository.save(song); 195 | // 196 | // } 197 | // 198 | // 199 | //} 200 | -------------------------------------------------------------------------------- /src/main/resources/mapper/SingerSongMapper.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | and ${criterion.condition} 18 | 19 | 20 | and ${criterion.condition} #{criterion.value} 21 | 22 | 23 | and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} 24 | 25 | 26 | and ${criterion.condition} 27 | 28 | #{listItem} 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | and ${criterion.condition} 47 | 48 | 49 | and ${criterion.condition} #{criterion.value} 50 | 51 | 52 | and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} 53 | 54 | 55 | and ${criterion.condition} 56 | 57 | #{listItem} 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | singer_id, song_id, intro 69 | 70 | 84 | 91 | 92 | delete from singer_song 93 | where singer_id = #{singerId,jdbcType=INTEGER} 94 | and song_id = #{songId,jdbcType=INTEGER} 95 | 96 | 97 | delete from singer_song 98 | 99 | 100 | 101 | 102 | 103 | insert into singer_song (singer_id, song_id, intro 104 | ) 105 | values (#{singerId,jdbcType=INTEGER}, #{songId,jdbcType=INTEGER}, #{intro,jdbcType=VARCHAR} 106 | ) 107 | 108 | 109 | insert into singer_song 110 | 111 | 112 | singer_id, 113 | 114 | 115 | song_id, 116 | 117 | 118 | intro, 119 | 120 | 121 | 122 | 123 | #{singerId,jdbcType=INTEGER}, 124 | 125 | 126 | #{songId,jdbcType=INTEGER}, 127 | 128 | 129 | #{intro,jdbcType=VARCHAR}, 130 | 131 | 132 | 133 | 139 | 140 | update singer_song 141 | 142 | 143 | singer_id = #{record.singerId,jdbcType=INTEGER}, 144 | 145 | 146 | song_id = #{record.songId,jdbcType=INTEGER}, 147 | 148 | 149 | intro = #{record.intro,jdbcType=VARCHAR}, 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | update singer_song 158 | set singer_id = #{record.singerId,jdbcType=INTEGER}, 159 | song_id = #{record.songId,jdbcType=INTEGER}, 160 | intro = #{record.intro,jdbcType=VARCHAR} 161 | 162 | 163 | 164 | 165 | 166 | update singer_song 167 | 168 | 169 | intro = #{intro,jdbcType=VARCHAR}, 170 | 171 | 172 | where singer_id = #{singerId,jdbcType=INTEGER} 173 | and song_id = #{songId,jdbcType=INTEGER} 174 | 175 | 176 | update singer_song 177 | set intro = #{intro,jdbcType=VARCHAR} 178 | where singer_id = #{singerId,jdbcType=INTEGER} 179 | and song_id = #{songId,jdbcType=INTEGER} 180 | 181 | 182 | 185 | -------------------------------------------------------------------------------- /src/main/resources/mapper/SingerMapper.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | and ${criterion.condition} 19 | 20 | 21 | and ${criterion.condition} #{criterion.value} 22 | 23 | 24 | and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} 25 | 26 | 27 | and ${criterion.condition} 28 | 29 | #{listItem} 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | and ${criterion.condition} 48 | 49 | 50 | and ${criterion.condition} #{criterion.value} 51 | 52 | 53 | and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} 54 | 55 | 56 | and ${criterion.condition} 57 | 58 | #{listItem} 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | id, name, intro, pic_url 70 | 71 | 85 | 91 | 92 | delete from singer 93 | where id = #{id,jdbcType=INTEGER} 94 | 95 | 96 | delete from singer 97 | 98 | 99 | 100 | 101 | 102 | insert into singer (id, name, intro, 103 | pic_url) 104 | values (#{id,jdbcType=INTEGER}, #{name,jdbcType=VARCHAR}, #{intro,jdbcType=VARCHAR}, 105 | #{picUrl,jdbcType=VARCHAR}) 106 | 107 | 108 | insert into singer 109 | 110 | 111 | id, 112 | 113 | 114 | name, 115 | 116 | 117 | intro, 118 | 119 | 120 | pic_url, 121 | 122 | 123 | 124 | 125 | #{id,jdbcType=INTEGER}, 126 | 127 | 128 | #{name,jdbcType=VARCHAR}, 129 | 130 | 131 | #{intro,jdbcType=VARCHAR}, 132 | 133 | 134 | #{picUrl,jdbcType=VARCHAR}, 135 | 136 | 137 | 138 | 144 | 145 | update singer 146 | 147 | 148 | id = #{record.id,jdbcType=INTEGER}, 149 | 150 | 151 | name = #{record.name,jdbcType=VARCHAR}, 152 | 153 | 154 | intro = #{record.intro,jdbcType=VARCHAR}, 155 | 156 | 157 | pic_url = #{record.picUrl,jdbcType=VARCHAR}, 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | update singer 166 | set id = #{record.id,jdbcType=INTEGER}, 167 | name = #{record.name,jdbcType=VARCHAR}, 168 | intro = #{record.intro,jdbcType=VARCHAR}, 169 | pic_url = #{record.picUrl,jdbcType=VARCHAR} 170 | 171 | 172 | 173 | 174 | 175 | update singer 176 | 177 | 178 | name = #{name,jdbcType=VARCHAR}, 179 | 180 | 181 | intro = #{intro,jdbcType=VARCHAR}, 182 | 183 | 184 | pic_url = #{picUrl,jdbcType=VARCHAR}, 185 | 186 | 187 | where id = #{id,jdbcType=INTEGER} 188 | 189 | 190 | update singer 191 | set name = #{name,jdbcType=VARCHAR}, 192 | intro = #{intro,jdbcType=VARCHAR}, 193 | pic_url = #{picUrl,jdbcType=VARCHAR} 194 | where id = #{id,jdbcType=INTEGER} 195 | 196 | -------------------------------------------------------------------------------- /src/main/resources/mapper/SongMapper.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | and ${criterion.condition} 20 | 21 | 22 | and ${criterion.condition} #{criterion.value} 23 | 24 | 25 | and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} 26 | 27 | 28 | and ${criterion.condition} 29 | 30 | #{listItem} 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | and ${criterion.condition} 49 | 50 | 51 | and ${criterion.condition} #{criterion.value} 52 | 53 | 54 | and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} 55 | 56 | 57 | and ${criterion.condition} 58 | 59 | #{listItem} 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | id, comment_thread_id, mp3url, name, record_id 71 | 72 | 86 | 92 | 93 | delete from song 94 | where id = #{id,jdbcType=BIGINT} 95 | 96 | 97 | delete from song 98 | 99 | 100 | 101 | 102 | 103 | insert into song (id, comment_thread_id, mp3url, 104 | name, record_id) 105 | values (#{id,jdbcType=BIGINT}, #{commentThreadId,jdbcType=VARCHAR}, #{mp3url,jdbcType=VARCHAR}, 106 | #{name,jdbcType=VARCHAR}, #{recordId,jdbcType=INTEGER}) 107 | 108 | 109 | insert into song 110 | 111 | 112 | id, 113 | 114 | 115 | comment_thread_id, 116 | 117 | 118 | mp3url, 119 | 120 | 121 | name, 122 | 123 | 124 | record_id, 125 | 126 | 127 | 128 | 129 | #{id,jdbcType=BIGINT}, 130 | 131 | 132 | #{commentThreadId,jdbcType=VARCHAR}, 133 | 134 | 135 | #{mp3url,jdbcType=VARCHAR}, 136 | 137 | 138 | #{name,jdbcType=VARCHAR}, 139 | 140 | 141 | #{recordId,jdbcType=INTEGER}, 142 | 143 | 144 | 145 | 151 | 152 | update song 153 | 154 | 155 | id = #{record.id,jdbcType=BIGINT}, 156 | 157 | 158 | comment_thread_id = #{record.commentThreadId,jdbcType=VARCHAR}, 159 | 160 | 161 | mp3url = #{record.mp3url,jdbcType=VARCHAR}, 162 | 163 | 164 | name = #{record.name,jdbcType=VARCHAR}, 165 | 166 | 167 | record_id = #{record.recordId,jdbcType=INTEGER}, 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | update song 176 | set id = #{record.id,jdbcType=BIGINT}, 177 | comment_thread_id = #{record.commentThreadId,jdbcType=VARCHAR}, 178 | mp3url = #{record.mp3url,jdbcType=VARCHAR}, 179 | name = #{record.name,jdbcType=VARCHAR}, 180 | record_id = #{record.recordId,jdbcType=INTEGER} 181 | 182 | 183 | 184 | 185 | 186 | update song 187 | 188 | 189 | comment_thread_id = #{commentThreadId,jdbcType=VARCHAR}, 190 | 191 | 192 | mp3url = #{mp3url,jdbcType=VARCHAR}, 193 | 194 | 195 | name = #{name,jdbcType=VARCHAR}, 196 | 197 | 198 | record_id = #{recordId,jdbcType=INTEGER}, 199 | 200 | 201 | where id = #{id,jdbcType=BIGINT} 202 | 203 | 204 | update song 205 | set comment_thread_id = #{commentThreadId,jdbcType=VARCHAR}, 206 | mp3url = #{mp3url,jdbcType=VARCHAR}, 207 | name = #{name,jdbcType=VARCHAR}, 208 | record_id = #{recordId,jdbcType=INTEGER} 209 | where id = #{id,jdbcType=BIGINT} 210 | 211 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/spider/NetMusicGrab.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.spider; 2 | 3 | import com.google.gson.Gson; 4 | import com.google.gson.reflect.TypeToken; 5 | import com.netMusic.entity.Record; 6 | import com.netMusic.entity.Singer; 7 | import com.netMusic.entity.custom2.SongMsg; 8 | import com.netMusic.utils.CharacterUtils; 9 | import com.netMusic.utils.HttpClientUtil; 10 | import org.apache.http.Header; 11 | import org.apache.http.message.BasicHeader; 12 | import org.jsoup.Jsoup; 13 | import org.jsoup.nodes.Document; 14 | import org.jsoup.nodes.Element; 15 | import org.jsoup.select.Elements; 16 | import org.slf4j.Logger; 17 | import org.slf4j.LoggerFactory; 18 | 19 | import java.io.IOException; 20 | import java.lang.reflect.Type; 21 | import java.text.ParseException; 22 | import java.text.SimpleDateFormat; 23 | import java.util.ArrayList; 24 | import java.util.Date; 25 | import java.util.List; 26 | 27 | /** 28 | * 歌曲爬取类 29 | */ 30 | public class NetMusicGrab { 31 | 32 | private final static Logger logger = LoggerFactory.getLogger(NetMusicGrab.class); 33 | 34 | public static List getSongList(String url,String charest) { 35 | Document document = getDocument(url, charest); 36 | if(document!=null){ 37 | Elements elements = document.select("#song-list-pre-data"); 38 | System.out.println("json数据如下"); 39 | String resJson = elements.text(); 40 | // logger.info(resJson); 41 | Gson gson = new Gson(); 42 | Type listType = new TypeToken>() {}.getType(); 43 | if(resJson!=null && !resJson.contains("html")); 44 | List msgList = gson.fromJson(resJson, listType); 45 | return msgList; 46 | } 47 | return null; 48 | } 49 | 50 | 51 | public static Singer getSinger(String url,String charest){ 52 | 53 | Document document = getDocument(url,charest); 54 | if(document!=null){ 55 | Elements singerName = document.select("#artist-name"); 56 | System.out.println("歌手名字："+singerName.text()); 57 | Elements singerAlias = document.select("#artist-alias"); 58 | String name=""; 59 | if(singerAlias!=null && !"".equals(singerAlias.text())){ 60 | System.out.println("歌手别名："+singerAlias.text()); 61 | name = singerName.text()+"/"+singerAlias.text(); 62 | }else{ 63 | name = singerName.text(); 64 | } 65 | 66 | Elements desc = document.select("body > div.g-bd4.f-cb > div.g-mn4 > div > div > div:nth-child(3) > div > p:nth-child(2)"); 67 | Elements image = document.select("body > div.g-bd4.f-cb > div.g-mn4 > div > div > div.n-artist.f-cb > img"); 68 | String intro = desc.text(); 69 | String picUrl = image.attr("src"); 70 | Singer singer = new Singer(); 71 | singer.setName(name); 72 | if(intro.length()>=255 || "".equals(intro)){ 73 | intro = "暂无介绍"; 74 | } 75 | singer.setIntro(intro); 76 | singer.setPicUrl(picUrl); 77 | return singer; 78 | } 79 | 80 | return null; 81 | } 82 | 83 | public static List getRecordList(String url,String charest){ 84 | 85 | List recordList = new ArrayList<>(); 86 | String publishTime = null; 87 | String company = null; 88 | 89 | Document document = getDocument(url,charest); 90 | if(document!=null){ 91 | Elements elements = document.select("#m-song-module"); 92 | for(Element element :elements){ 93 | //专辑链接 94 | Elements urlElement = element.select("#m-song-module > li:nth-child(1) > div > a.msk"); 95 | String albumUrl = urlElement.select(".msk").attr("href"); 96 | if(!albumUrl.contains("https://")){ 97 | albumUrl = "https://music.163.com"+albumUrl; 98 | } 99 | Elements elementsAlbumId = element.select("#m-song-module > li:nth-child(1) > div > a.icon-play.f-alpha"); 100 | String albumId = elementsAlbumId.attr("data-res-id"); 101 | try { 102 | Document albumDec = Jsoup.connect(albumUrl). 103 | userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36") 104 | .get(); 105 | 106 | Elements main = albumDec.select("body > div.g-bd4.f-cb.p-share > div.g-mn4 > div > div > div.m-info.f-cb > div.cnt > div > div.topblk"); 107 | 108 | 109 | Elements elementsName = main.select("div > div > h2"); 110 | String albumName = elementsName.text(); 111 | Elements elementsPublishTime = main.select("p:nth-child(3)"); 112 | if(elementsPublishTime!=null && !"".equals(elementsPublishTime.text()) ){ 113 | publishTime = elementsPublishTime.text().substring(5); 114 | } 115 | Elements elementsCompany = main.select("p:nth-child(4)"); 116 | 117 | if(elementsCompany!=null && !"".equals(elementsCompany.text()) ){ 118 | company = elementsCompany.text().substring(5); 119 | } 120 | //如果包含中文，则说明该字段为company 121 | if(CharacterUtils.isContainChinese(publishTime)){ 122 | company = publishTime; 123 | publishTime = null; 124 | } 125 | 126 | Elements elementsCommentId = albumDec.select("#cnt_comment_count"); 127 | String commentId = elementsCommentId.text(); 128 | if(CharacterUtils.isContainChinese(commentId)){ 129 | commentId = null; 130 | } 131 | Elements elementsImg = albumDec.select("body > div.g-bd4.f-cb.p-share > div.g-mn4 > div > div > div.m-info.f-cb > div.cover.u-cover.u-cover-alb > img"); 132 | String picUrl = elementsImg.attr("data-src"); 133 | Elements elementsIntro = albumDec.select("#album-desc-dot"); 134 | String intro = elementsIntro.text(); 135 | if(intro.length()>=255 || intro.length()<=1){ 136 | intro = "暂无介绍"; 137 | } 138 | Record record = new Record(); 139 | record.setId(Integer.valueOf(albumId)); 140 | record.setIntro(intro); 141 | record.setPicUrl(picUrl); 142 | record.setName(albumName); 143 | record.setCompany(company); 144 | record.setCommentThreadId(commentId); 145 | if(publishTime!=null){ 146 | SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); 147 | Date date = sdf.parse(publishTime); 148 | record.setPublishTime(date); 149 | }else{ 150 | record.setPublishTime(null); 151 | } 152 | recordList.add(record); 153 | 154 | } catch (IOException e) { 155 | e.printStackTrace(); 156 | }catch (Exception e){ 157 | e.printStackTrace(); 158 | } 159 | } 160 | 161 | } 162 | 163 | return recordList; 164 | } 165 | 166 | 167 | 168 | 169 | private static Document getDocument(String url, String charest) { 170 | List
headerList = new ArrayList<>(); 171 | headerList.add(new BasicHeader("Host", "music.163.com")); 172 | headerList.add(new BasicHeader("Referer", "https://music.163.com/")); 173 | headerList.add(new BasicHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36")); 174 | 175 | String result = HttpClientUtil.doGet(url, headerList, charest); 176 | if(result!=null && !result.contains("n-for404")){ 177 | return Jsoup.parse(result); 178 | } 179 | return null; 180 | } 181 | 182 | 183 | public static void main(String[] args) throws ParseException { 184 | // 测试获取歌手信息 185 | // List listId = BaseUtil.getRandomNumber(1000,10000, 186 | // } 187 | //测试专辑信息100); 188 | //// for(Integer id:listId){ 189 | //// Singer singer = getSinger("https://music.163.com/artist/desc?id="+id,"utf-8"); 190 | //// System.out.println(singer); 191 | // getRecordList("https://music.163.com/artist/album?id=3684&limit=100&offset=0","utf-8"); 192 | 193 | //测试字符串转时间 194 | // SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); 195 | // Date date = sdf.parse("2017-12-12"); 196 | // System.out.println(date); 197 | 198 | } 199 | } -------------------------------------------------------------------------------- /.idea/uiDesigner.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | -------------------------------------------------------------------------------- /src/main/resources/mapper/RecordMapper.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | and ${criterion.condition} 23 | 24 | 25 | and ${criterion.condition} #{criterion.value} 26 | 27 | 28 | and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} 29 | 30 | 31 | and ${criterion.condition} 32 | 33 | #{listItem} 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | and ${criterion.condition} 52 | 53 | 54 | and ${criterion.condition} #{criterion.value} 55 | 56 | 57 | and ${criterion.condition} #{criterion.value} and #{criterion.secondValue} 58 | 59 | 60 | and ${criterion.condition} 61 | 62 | #{listItem} 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | id, comment_thread_id, company, intro, name, pic_url, singer_id, publish_time 74 | 75 | 89 | 95 | 96 | delete from record 97 | where id = #{id,jdbcType=INTEGER} 98 | 99 | 100 | delete from record 101 | 102 | 103 | 104 | 105 | 106 | insert into record (id, comment_thread_id, company, 107 | intro, name, pic_url, 108 | singer_id, publish_time) 109 | values (#{id,jdbcType=INTEGER}, #{commentThreadId,jdbcType=VARCHAR}, #{company,jdbcType=VARCHAR}, 110 | #{intro,jdbcType=VARCHAR}, #{name,jdbcType=VARCHAR}, #{picUrl,jdbcType=VARCHAR}, 111 | #{singerId,jdbcType=INTEGER}, #{publishTime,jdbcType=TIMESTAMP}) 112 | 113 | 114 | insert into record 115 | 116 | 117 | id, 118 | 119 | 120 | comment_thread_id, 121 | 122 | 123 | company, 124 | 125 | 126 | intro, 127 | 128 | 129 | name, 130 | 131 | 132 | pic_url, 133 | 134 | 135 | singer_id, 136 | 137 | 138 | publish_time, 139 | 140 | 141 | 142 | 143 | #{id,jdbcType=INTEGER}, 144 | 145 | 146 | #{commentThreadId,jdbcType=VARCHAR}, 147 | 148 | 149 | #{company,jdbcType=VARCHAR}, 150 | 151 | 152 | #{intro,jdbcType=VARCHAR}, 153 | 154 | 155 | #{name,jdbcType=VARCHAR}, 156 | 157 | 158 | #{picUrl,jdbcType=VARCHAR}, 159 | 160 | 161 | #{singerId,jdbcType=INTEGER}, 162 | 163 | 164 | #{publishTime,jdbcType=TIMESTAMP}, 165 | 166 | 167 | 168 | 174 | 175 | update record 176 | 177 | 178 | id = #{record.id,jdbcType=INTEGER}, 179 | 180 | 181 | comment_thread_id = #{record.commentThreadId,jdbcType=VARCHAR}, 182 | 183 | 184 | company = #{record.company,jdbcType=VARCHAR}, 185 | 186 | 187 | intro = #{record.intro,jdbcType=VARCHAR}, 188 | 189 | 190 | name = #{record.name,jdbcType=VARCHAR}, 191 | 192 | 193 | pic_url = #{record.picUrl,jdbcType=VARCHAR}, 194 | 195 | 196 | singer_id = #{record.singerId,jdbcType=INTEGER}, 197 | 198 | 199 | publish_time = #{record.publishTime,jdbcType=TIMESTAMP}, 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | update record 208 | set id = #{record.id,jdbcType=INTEGER}, 209 | comment_thread_id = #{record.commentThreadId,jdbcType=VARCHAR}, 210 | company = #{record.company,jdbcType=VARCHAR}, 211 | intro = #{record.intro,jdbcType=VARCHAR}, 212 | name = #{record.name,jdbcType=VARCHAR}, 213 | pic_url = #{record.picUrl,jdbcType=VARCHAR}, 214 | singer_id = #{record.singerId,jdbcType=INTEGER}, 215 | publish_time = #{record.publishTime,jdbcType=TIMESTAMP} 216 | 217 | 218 | 219 | 220 | 221 | update record 222 | 223 | 224 | comment_thread_id = #{commentThreadId,jdbcType=VARCHAR}, 225 | 226 | 227 | company = #{company,jdbcType=VARCHAR}, 228 | 229 | 230 | intro = #{intro,jdbcType=VARCHAR}, 231 | 232 | 233 | name = #{name,jdbcType=VARCHAR}, 234 | 235 | 236 | pic_url = #{picUrl,jdbcType=VARCHAR}, 237 | 238 | 239 | singer_id = #{singerId,jdbcType=INTEGER}, 240 | 241 | 242 | publish_time = #{publishTime,jdbcType=TIMESTAMP}, 243 | 244 | 245 | where id = #{id,jdbcType=INTEGER} 246 | 247 | 248 | update record 249 | set comment_thread_id = #{commentThreadId,jdbcType=VARCHAR}, 250 | company = #{company,jdbcType=VARCHAR}, 251 | intro = #{intro,jdbcType=VARCHAR}, 252 | name = #{name,jdbcType=VARCHAR}, 253 | pic_url = #{picUrl,jdbcType=VARCHAR}, 254 | singer_id = #{singerId,jdbcType=INTEGER}, 255 | publish_time = #{publishTime,jdbcType=TIMESTAMP} 256 | where id = #{id,jdbcType=INTEGER} 257 | 258 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/entity/SingerSongExample.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.entity; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | public class SingerSongExample { 7 | protected String orderByClause; 8 | 9 | protected boolean distinct; 10 | 11 | protected List oredCriteria; 12 | 13 | public SingerSongExample() { 14 | oredCriteria = new ArrayList(); 15 | } 16 | 17 | public void setOrderByClause(String orderByClause) { 18 | this.orderByClause = orderByClause; 19 | } 20 | 21 | public String getOrderByClause() { 22 | return orderByClause; 23 | } 24 | 25 | public void setDistinct(boolean distinct) { 26 | this.distinct = distinct; 27 | } 28 | 29 | public boolean isDistinct() { 30 | return distinct; 31 | } 32 | 33 | public List getOredCriteria() { 34 | return oredCriteria; 35 | } 36 | 37 | public void or(Criteria criteria) { 38 | oredCriteria.add(criteria); 39 | } 40 | 41 | public Criteria or() { 42 | Criteria criteria = createCriteriaInternal(); 43 | oredCriteria.add(criteria); 44 | return criteria; 45 | } 46 | 47 | public Criteria createCriteria() { 48 | Criteria criteria = createCriteriaInternal(); 49 | if (oredCriteria.size() == 0) { 50 | oredCriteria.add(criteria); 51 | } 52 | return criteria; 53 | } 54 | 55 | protected Criteria createCriteriaInternal() { 56 | Criteria criteria = new Criteria(); 57 | return criteria; 58 | } 59 | 60 | public void clear() { 61 | oredCriteria.clear(); 62 | orderByClause = null; 63 | distinct = false; 64 | } 65 | 66 | protected abstract static class GeneratedCriteria { 67 | protected List criteria; 68 | 69 | protected GeneratedCriteria() { 70 | super(); 71 | criteria = new ArrayList(); 72 | } 73 | 74 | public boolean isValid() { 75 | return criteria.size() > 0; 76 | } 77 | 78 | public List getAllCriteria() { 79 | return criteria; 80 | } 81 | 82 | public List getCriteria() { 83 | return criteria; 84 | } 85 | 86 | protected void addCriterion(String condition) { 87 | if (condition == null) { 88 | throw new RuntimeException("Value for condition cannot be null"); 89 | } 90 | criteria.add(new Criterion(condition)); 91 | } 92 | 93 | protected void addCriterion(String condition, Object value, String property) { 94 | if (value == null) { 95 | throw new RuntimeException("Value for " + property + " cannot be null"); 96 | } 97 | criteria.add(new Criterion(condition, value)); 98 | } 99 | 100 | protected void addCriterion(String condition, Object value1, Object value2, String property) { 101 | if (value1 == null || value2 == null) { 102 | throw new RuntimeException("Between values for " + property + " cannot be null"); 103 | } 104 | criteria.add(new Criterion(condition, value1, value2)); 105 | } 106 | 107 | public Criteria andSingerIdIsNull() { 108 | addCriterion("singer_id is null"); 109 | return (Criteria) this; 110 | } 111 | 112 | public Criteria andSingerIdIsNotNull() { 113 | addCriterion("singer_id is not null"); 114 | return (Criteria) this; 115 | } 116 | 117 | public Criteria andSingerIdEqualTo(Integer value) { 118 | addCriterion("singer_id =", value, "singerId"); 119 | return (Criteria) this; 120 | } 121 | 122 | public Criteria andSingerIdNotEqualTo(Integer value) { 123 | addCriterion("singer_id <>", value, "singerId"); 124 | return (Criteria) this; 125 | } 126 | 127 | public Criteria andSingerIdGreaterThan(Integer value) { 128 | addCriterion("singer_id >", value, "singerId"); 129 | return (Criteria) this; 130 | } 131 | 132 | public Criteria andSingerIdGreaterThanOrEqualTo(Integer value) { 133 | addCriterion("singer_id >=", value, "singerId"); 134 | return (Criteria) this; 135 | } 136 | 137 | public Criteria andSingerIdLessThan(Integer value) { 138 | addCriterion("singer_id <", value, "singerId"); 139 | return (Criteria) this; 140 | } 141 | 142 | public Criteria andSingerIdLessThanOrEqualTo(Integer value) { 143 | addCriterion("singer_id <=", value, "singerId"); 144 | return (Criteria) this; 145 | } 146 | 147 | public Criteria andSingerIdIn(List values) { 148 | addCriterion("singer_id in", values, "singerId"); 149 | return (Criteria) this; 150 | } 151 | 152 | public Criteria andSingerIdNotIn(List values) { 153 | addCriterion("singer_id not in", values, "singerId"); 154 | return (Criteria) this; 155 | } 156 | 157 | public Criteria andSingerIdBetween(Integer value1, Integer value2) { 158 | addCriterion("singer_id between", value1, value2, "singerId"); 159 | return (Criteria) this; 160 | } 161 | 162 | public Criteria andSingerIdNotBetween(Integer value1, Integer value2) { 163 | addCriterion("singer_id not between", value1, value2, "singerId"); 164 | return (Criteria) this; 165 | } 166 | 167 | public Criteria andSongIdIsNull() { 168 | addCriterion("song_id is null"); 169 | return (Criteria) this; 170 | } 171 | 172 | public Criteria andSongIdIsNotNull() { 173 | addCriterion("song_id is not null"); 174 | return (Criteria) this; 175 | } 176 | 177 | public Criteria andSongIdEqualTo(Integer value) { 178 | addCriterion("song_id =", value, "songId"); 179 | return (Criteria) this; 180 | } 181 | 182 | public Criteria andSongIdNotEqualTo(Integer value) { 183 | addCriterion("song_id <>", value, "songId"); 184 | return (Criteria) this; 185 | } 186 | 187 | public Criteria andSongIdGreaterThan(Integer value) { 188 | addCriterion("song_id >", value, "songId"); 189 | return (Criteria) this; 190 | } 191 | 192 | public Criteria andSongIdGreaterThanOrEqualTo(Integer value) { 193 | addCriterion("song_id >=", value, "songId"); 194 | return (Criteria) this; 195 | } 196 | 197 | public Criteria andSongIdLessThan(Integer value) { 198 | addCriterion("song_id <", value, "songId"); 199 | return (Criteria) this; 200 | } 201 | 202 | public Criteria andSongIdLessThanOrEqualTo(Integer value) { 203 | addCriterion("song_id <=", value, "songId"); 204 | return (Criteria) this; 205 | } 206 | 207 | public Criteria andSongIdIn(List values) { 208 | addCriterion("song_id in", values, "songId"); 209 | return (Criteria) this; 210 | } 211 | 212 | public Criteria andSongIdNotIn(List values) { 213 | addCriterion("song_id not in", values, "songId"); 214 | return (Criteria) this; 215 | } 216 | 217 | public Criteria andSongIdBetween(Integer value1, Integer value2) { 218 | addCriterion("song_id between", value1, value2, "songId"); 219 | return (Criteria) this; 220 | } 221 | 222 | public Criteria andSongIdNotBetween(Integer value1, Integer value2) { 223 | addCriterion("song_id not between", value1, value2, "songId"); 224 | return (Criteria) this; 225 | } 226 | 227 | public Criteria andIntroIsNull() { 228 | addCriterion("intro is null"); 229 | return (Criteria) this; 230 | } 231 | 232 | public Criteria andIntroIsNotNull() { 233 | addCriterion("intro is not null"); 234 | return (Criteria) this; 235 | } 236 | 237 | public Criteria andIntroEqualTo(String value) { 238 | addCriterion("intro =", value, "intro"); 239 | return (Criteria) this; 240 | } 241 | 242 | public Criteria andIntroNotEqualTo(String value) { 243 | addCriterion("intro <>", value, "intro"); 244 | return (Criteria) this; 245 | } 246 | 247 | public Criteria andIntroGreaterThan(String value) { 248 | addCriterion("intro >", value, "intro"); 249 | return (Criteria) this; 250 | } 251 | 252 | public Criteria andIntroGreaterThanOrEqualTo(String value) { 253 | addCriterion("intro >=", value, "intro"); 254 | return (Criteria) this; 255 | } 256 | 257 | public Criteria andIntroLessThan(String value) { 258 | addCriterion("intro <", value, "intro"); 259 | return (Criteria) this; 260 | } 261 | 262 | public Criteria andIntroLessThanOrEqualTo(String value) { 263 | addCriterion("intro <=", value, "intro"); 264 | return (Criteria) this; 265 | } 266 | 267 | public Criteria andIntroLike(String value) { 268 | addCriterion("intro like", value, "intro"); 269 | return (Criteria) this; 270 | } 271 | 272 | public Criteria andIntroNotLike(String value) { 273 | addCriterion("intro not like", value, "intro"); 274 | return (Criteria) this; 275 | } 276 | 277 | public Criteria andIntroIn(List values) { 278 | addCriterion("intro in", values, "intro"); 279 | return (Criteria) this; 280 | } 281 | 282 | public Criteria andIntroNotIn(List values) { 283 | addCriterion("intro not in", values, "intro"); 284 | return (Criteria) this; 285 | } 286 | 287 | public Criteria andIntroBetween(String value1, String value2) { 288 | addCriterion("intro between", value1, value2, "intro"); 289 | return (Criteria) this; 290 | } 291 | 292 | public Criteria andIntroNotBetween(String value1, String value2) { 293 | addCriterion("intro not between", value1, value2, "intro"); 294 | return (Criteria) this; 295 | } 296 | } 297 | 298 | public static class Criteria extends GeneratedCriteria { 299 | 300 | protected Criteria() { 301 | super(); 302 | } 303 | } 304 | 305 | public static class Criterion { 306 | private String condition; 307 | 308 | private Object value; 309 | 310 | private Object secondValue; 311 | 312 | private boolean noValue; 313 | 314 | private boolean singleValue; 315 | 316 | private boolean betweenValue; 317 | 318 | private boolean listValue; 319 | 320 | private String typeHandler; 321 | 322 | public String getCondition() { 323 | return condition; 324 | } 325 | 326 | public Object getValue() { 327 | return value; 328 | } 329 | 330 | public Object getSecondValue() { 331 | return secondValue; 332 | } 333 | 334 | public boolean isNoValue() { 335 | return noValue; 336 | } 337 | 338 | public boolean isSingleValue() { 339 | return singleValue; 340 | } 341 | 342 | public boolean isBetweenValue() { 343 | return betweenValue; 344 | } 345 | 346 | public boolean isListValue() { 347 | return listValue; 348 | } 349 | 350 | public String getTypeHandler() { 351 | return typeHandler; 352 | } 353 | 354 | protected Criterion(String condition) { 355 | super(); 356 | this.condition = condition; 357 | this.typeHandler = null; 358 | this.noValue = true; 359 | } 360 | 361 | protected Criterion(String condition, Object value, String typeHandler) { 362 | super(); 363 | this.condition = condition; 364 | this.value = value; 365 | this.typeHandler = typeHandler; 366 | if (value instanceof List) { 367 | this.listValue = true; 368 | } else { 369 | this.singleValue = true; 370 | } 371 | } 372 | 373 | protected Criterion(String condition, Object value) { 374 | this(condition, value, null); 375 | } 376 | 377 | protected Criterion(String condition, Object value, Object secondValue, String typeHandler) { 378 | super(); 379 | this.condition = condition; 380 | this.value = value; 381 | this.secondValue = secondValue; 382 | this.typeHandler = typeHandler; 383 | this.betweenValue = true; 384 | } 385 | 386 | protected Criterion(String condition, Object value, Object secondValue) { 387 | this(condition, value, secondValue, null); 388 | } 389 | } 390 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 使用java爬取网易云音乐 2 | 3 | 4 | - **目的**：抓取网易云音乐热门歌手及其歌曲、专辑等信息保存到数据库 5 | - **技术点**： 6 | - 使用HttpClient和Jsoup进行模拟请求并对网页进行解析 7 | - 使用springBoot+maven构建管理项目 8 | - 使用mybatis作为数据访问 9 | - **数据库设计** 10 | - song : id,name,comment_thread_id,mv_id,record_id,mp3Url 11 | - singer: id,name,intro,picUrl 12 | - record: id,name,picUrl,singer_id,tags,type,intro,company,comment_thread_id 13 | - singer_song: singer_id,song_id,intro 14 | **这里考虑到一首歌曲可对应多个歌手，故建立一个中间表保存双方的id (多对多)** 15 | 16 | 爬取网站的分析 17 | 18 | 1.歌曲页面的分析 19 | ------- 20 | 21 | **目标网站**：[https://music.163.com/#/artist?id=6452](https://music.163.com/#/artist?id=6452) 22 | 我们先以该网页进行分析，在对网页抓包中我们发现有个名为artist?id=2116的请求，在该请求返回的数据中我们发现就是该网页歌手对应的页面（即html） 23 | ![这里写图片描述](https://img-blog.csdn.net/20180611222252865?watermark/2/text/aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlWXVLMA==/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70) 24 | 在该返回的html中，在标签``中就存在该歌手的50首歌曲的json数据。对于所要做的系统来说，50首歌曲数据已经够了，为此我们拿到该数据即可。 25 | ![这里写图片描述](https://img-blog.csdn.net/20180611222921494?watermark/2/text/aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlWXVLMA==/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70) 26 | 27 | 但是在其中我们并没有发现歌曲的url。当对歌曲进行点击后，抓包(url?csrf_token=)发现 28 | 浏览器向https://music.163.com/weapi/song/enhance/player/url?csrf_token= 发送了post请求，返回的就是带有当前歌曲url的json数据。 29 | 其中有两个参数 30 | **params**：IvwCQv++yCDpnlC+Tog13k9WVWYgou3LauQ60jh9YPGSL1AkcUxma7r1Hs8yTaXBGYluJV7tl0xrTmtgl3qxQ6TZYNBIxfFvkaqqDxRKJvgsBOYV2SpD8mRfxrgbesqH 31 | 32 | **encSecKey**：2b29c882a53743986c4f5aa279f1f2353a84a8d773071877252dbfc805c34b2065de3738945ea1bbdb29602b85d7ef4382d4d77c44c9eed6cb0c88ce7d3e37883b222b77381929a367b2fc062c3499ebfc7135c9d3a51b2fb8bb316f4c8006d2e30141b9be9de6bd017096fdeaf645e4450c88999febad081f9b6cc0e83fbaaa 33 | 34 | 这两个参数是加密过的。 35 | 找到https://s3.music.126.net/web/s/core.js?acad6e83fe6f991104f40576c001942a 36 | 在里面搜索encSecKey，找到加密函数，经过了两次加密。。 37 | 38 | 加密技术不太好搞，可以换种别的方法。 39 | 40 | 在对音乐生成外链的抓包中，意外地发现个url：http://music.163.com/song/media/outer/url?id=25730757 41 | 该url可直接拿到歌曲的url，这样我们只需要记住“http://music.163.com/song/media/outer/url?id=” 链接，最后加上歌曲id就可以拿到歌曲mp3的url了。 42 | 43 | **歌曲url:"https://music.163.com/artist?id="+singerId;** 44 | **歌曲mp3Url:“http://music.163.com/song/media/outer/url?id=” +songId;** 45 | 46 | 2.歌手页面的分析 47 | ------- 48 | 49 | **目标网站**：https://music.163.com/#/artist/desc?id=2116 50 | 以所需要的图片数据为例，F12打开开发者工具，找到图片所在标签，右键复制元素的selector，得到#auto-id-herlzRnQZM09uulQ > div.g-bd4.f-cb > div.g-mn4 > div > div > div.n-artist.f-cb > img。其它所需数据也是通过此方法拿到对应的selector，之后用Jsoup解析即可得到对应的元素。 51 | 52 | **歌手url:"https://music.163.com/artist/desc?id="+singerId;** 53 | 54 | ![这里写图片描述](https://img-blog.csdn.net/20180611232427391?watermark/2/text/aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlWXVLMA==/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70) 55 | 56 | 3.专辑页面的分析 57 | ------- 58 | **目标网站**：https://music.163.com/#/artist/album?id=2116 59 | 我们发现，专辑页面进行了分页，在进行下一页过程的抓包中，发现专辑分页的规律是以limit和offset两个参数决定的。在对几个热门歌手分析中可以发现歌手的专辑数是没有超过两位数的，所以我们可以简单地假设一个歌手的专辑数不会超过100。因为要尽可能不跳转页面发请求，所以得到一个 60 | **url: https://music.163.com/artist/album?id="+singerId+"&limit=100&offset=0** 61 | 我们以这个url去请求页面，以歌手为例的拿取元素位置即可得到专辑所需要的信息。 62 | 63 | 代码逻辑分析 64 | ------ 65 | 贴几个核心代码： 66 | **爬虫基类：** 67 | ``` 68 | package com.netMusic.spider; 69 | 70 | import com.google.gson.Gson; 71 | import com.google.gson.reflect.TypeToken; 72 | import com.netMusic.entity.Record; 73 | import com.netMusic.entity.Singer; 74 | import com.netMusic.entity.custom2.SongMsg; 75 | import com.netMusic.utils.CharacterUtils; 76 | import com.netMusic.utils.HttpClientUtil; 77 | import org.apache.http.Header; 78 | import org.apache.http.message.BasicHeader; 79 | import org.jsoup.Jsoup; 80 | import org.jsoup.nodes.Document; 81 | import org.jsoup.nodes.Element; 82 | import org.jsoup.select.Elements; 83 | import org.slf4j.Logger; 84 | import org.slf4j.LoggerFactory; 85 | 86 | import java.io.IOException; 87 | import java.lang.reflect.Type; 88 | import java.text.ParseException; 89 | import java.text.SimpleDateFormat; 90 | import java.util.ArrayList; 91 | import java.util.Date; 92 | import java.util.List; 93 | 94 | /** 95 | * 歌曲爬取类 96 | */ 97 | public class NetMusicGrab { 98 | 99 | private final static Logger logger = LoggerFactory.getLogger(NetMusicGrab.class); 100 | 101 | public static List getSongList(String url,String charest) { 102 | Document document = getDocument(url, charest); 103 | if(document!=null){ 104 | Elements elements = document.select("#song-list-pre-data"); 105 | System.out.println("json数据如下"); 106 | String resJson = elements.text(); 107 | // logger.info(resJson); 108 | Gson gson = new Gson(); 109 | Type listType = new TypeToken>() {}.getType(); 110 | if(resJson!=null && !resJson.contains("html")); 111 | List msgList = gson.fromJson(resJson, listType); 112 | return msgList; 113 | } 114 | return null; 115 | } 116 | 117 | 118 | public static Singer getSinger(String url,String charest){ 119 | 120 | Document document = getDocument(url,charest); 121 | if(document!=null){ 122 | Elements singerName = document.select("#artist-name"); 123 | System.out.println("歌手名字："+singerName.text()); 124 | Elements singerAlias = document.select("#artist-alias"); 125 | String name=""; 126 | if(singerAlias!=null && !"".equals(singerAlias.text())){ 127 | System.out.println("歌手别名："+singerAlias.text()); 128 | name = singerName.text()+"/"+singerAlias.text(); 129 | }else{ 130 | name = singerName.text(); 131 | } 132 | 133 | Elements desc = document.select("body > div.g-bd4.f-cb > div.g-mn4 > div > div > div:nth-child(3) > div > p:nth-child(2)"); 134 | Elements image = document.select("body > div.g-bd4.f-cb > div.g-mn4 > div > div > div.n-artist.f-cb > img"); 135 | String intro = desc.text(); 136 | String picUrl = image.attr("src"); 137 | Singer singer = new Singer(); 138 | singer.setName(name); 139 | if(intro.length()>=255 || "".equals(intro)){ 140 | intro = "暂无介绍"; 141 | } 142 | singer.setIntro(intro); 143 | singer.setPicUrl(picUrl); 144 | return singer; 145 | } 146 | 147 | return null; 148 | } 149 | 150 | public static List getRecordList(String url,String charest){ 151 | 152 | List recordList = new ArrayList<>(); 153 | String publishTime = null; 154 | String company = null; 155 | 156 | Document document = getDocument(url,charest); 157 | if(document!=null){ 158 | Elements elements = document.select("#m-song-module"); 159 | for(Element element :elements){ 160 | //专辑链接 161 | Elements urlElement = element.select("#m-song-module > li:nth-child(1) > div > a.msk"); 162 | String albumUrl = urlElement.select(".msk").attr("href"); 163 | if(!albumUrl.contains("https://")){ 164 | albumUrl = "https://music.163.com"+albumUrl; 165 | } 166 | Elements elementsAlbumId = element.select("#m-song-module > li:nth-child(1) > div > a.icon-play.f-alpha"); 167 | String albumId = elementsAlbumId.attr("data-res-id"); 168 | try { 169 | Document albumDec = Jsoup.connect(albumUrl). 170 | userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36") 171 | .get(); 172 | 173 | Elements main = albumDec.select("body > div.g-bd4.f-cb.p-share > div.g-mn4 > div > div > div.m-info.f-cb > div.cnt > div > div.topblk"); 174 | 175 | 176 | Elements elementsName = main.select("div > div > h2"); 177 | String albumName = elementsName.text(); 178 | Elements elementsPublishTime = main.select("p:nth-child(3)"); 179 | if(elementsPublishTime!=null && !"".equals(elementsPublishTime.text()) ){ 180 | publishTime = elementsPublishTime.text().substring(5); 181 | } 182 | Elements elementsCompany = main.select("p:nth-child(4)"); 183 | 184 | if(elementsCompany!=null && !"".equals(elementsCompany.text()) ){ 185 | company = elementsCompany.text().substring(5); 186 | } 187 | //如果包含中文，则说明该字段为company 188 | if(CharacterUtils.isContainChinese(publishTime)){ 189 | company = publishTime; 190 | publishTime = null; 191 | } 192 | 193 | Elements elementsCommentId = albumDec.select("#cnt_comment_count"); 194 | String commentId = elementsCommentId.text(); 195 | if(CharacterUtils.isContainChinese(commentId)){ 196 | commentId = null; 197 | } 198 | Elements elementsImg = albumDec.select("body > div.g-bd4.f-cb.p-share > div.g-mn4 > div > div > div.m-info.f-cb > div.cover.u-cover.u-cover-alb > img"); 199 | String picUrl = elementsImg.attr("data-src"); 200 | Elements elementsIntro = albumDec.select("#album-desc-dot"); 201 | String intro = elementsIntro.text(); 202 | if(intro.length()>=255 || intro.length()<=1){ 203 | intro = "暂无介绍"; 204 | } 205 | Record record = new Record(); 206 | record.setId(Integer.valueOf(albumId)); 207 | record.setIntro(intro); 208 | record.setPicUrl(picUrl); 209 | record.setName(albumName); 210 | record.setCompany(company); 211 | record.setCommentThreadId(commentId); 212 | if(publishTime!=null){ 213 | SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); 214 | Date date = sdf.parse(publishTime); 215 | record.setPublishTime(date); 216 | }else{ 217 | record.setPublishTime(null); 218 | } 219 | recordList.add(record); 220 | 221 | } catch (IOException e) { 222 | e.printStackTrace(); 223 | }catch (Exception e){ 224 | e.printStackTrace(); 225 | } 226 | } 227 | 228 | } 229 | 230 | return recordList; 231 | } 232 | 233 | 234 | 235 | 236 | private static Document getDocument(String url, String charest) { 237 | List
headerList = new ArrayList<>(); 238 | headerList.add(new BasicHeader("Host", "music.163.com")); 239 | headerList.add(new BasicHeader("Referer", "https://music.163.com/")); 240 | headerList.add(new BasicHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36")); 241 | 242 | String result = HttpClientUtil.doGet(url, headerList, charest); 243 | if(result!=null && !result.contains("n-for404")){ 244 | return Jsoup.parse(result); 245 | } 246 | return null; 247 | } 248 | 249 | 250 | public static void main(String[] args) throws ParseException { 251 | // 测试获取歌手信息 252 | // List listId = BaseUtil.getRandomNumber(1000,10000, 253 | // } 254 | //测试专辑信息100); 255 | //// for(Integer id:listId){ 256 | //// Singer singer = getSinger("https://music.163.com/artist/desc?id="+id,"utf-8"); 257 | //// System.out.println(singer); 258 | // getRecordList("https://music.163.com/artist/album?id=3684&limit=100&offset=0","utf-8"); 259 | 260 | //测试字符串转时间 261 | // SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); 262 | // Date date = sdf.parse("2017-12-12"); 263 | // System.out.println(date); 264 | 265 | } 266 | } 267 | ``` 268 | 269 | 因为考虑到速度问题，采用简单的多线程开启方式，这里设置了三个子线程，分别爬取专辑，歌曲，歌手。同时，为使项目启动时即开始爬，添加了监听类，用来启动线程。 270 | **SpiderSingerRunnable类：** 271 | ![这里写图片描述](https://img-blog.csdn.net/20180611234852626?watermark/2/text/aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlWXVLMA==/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70) 272 | **SpiderRecordRunnable类：** 273 | 274 | ![这里写图片描述](https://img-blog.csdn.net/20180611235017698?watermark/2/text/aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlWXVLMA==/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70) 275 | ![这里写图片描述](https://img-blog.csdn.net/20180611235027615?watermark/2/text/aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlWXVLMA==/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70) 276 | 277 | **SpiderSongRunnnable类：** 278 | ![这里写图片描述](https://img-blog.csdn.net/20180611235157980?watermark/2/text/aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlWXVLMA==/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70) 279 | ![这里写图片描述](https://img-blog.csdn.net/20180611235210688?watermark/2/text/aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlWXVLMA==/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70) 280 | ![这里写图片描述](https://img-blog.csdn.net/2018061123522083?watermark/2/text/aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlWXVLMA==/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70) 281 | 282 | **SaveDataListener类：** 283 | ![这里写图片描述](https://img-blog.csdn.net/20180611235421858?watermark/2/text/aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlWXVLMA==/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70) 284 | ![这里写图片描述](https://img-blog.csdn.net/20180611235430348?watermark/2/text/aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2tlWXVLMA==/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70) 285 | 286 | 具体的代码可以在我的[github](https://github.com/yijiajia/ReptilianDemo)找到，剩余的逻辑和一些自己封装的工具包就不贴了也相对简单。 287 | 贴一下博客：https://blog.csdn.net/keYuK0/article/details/80659044 288 | 289 | 学习总结 290 | ---- 291 | 292 | 经过测试，20分钟左右爬取了大约14万的歌曲数据，期间有几次被封ip了，直接404，为解决这个问题得用上ip连接池不断地更换ip地址发出请求，不过目前来讲已经够了，之后有时间再去学学。还有个问题是，虽然采用了多线程，但我发现并没有将多线程用得很好，希望有时间去学下Redis缓存队列再对代码进行更新吧。 293 | -------------------------------------------------------------------------------- /src/main/java/com/netMusic/entity/SingerExample.java: -------------------------------------------------------------------------------- 1 | package com.netMusic.entity; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | public class SingerExample { 7 | protected String orderByClause; 8 | 9 | protected boolean distinct; 10 | 11 | protected List oredCriteria; 12 | 13 | public SingerExample() { 14 | oredCriteria = new ArrayList(); 15 | } 16 | 17 | public void setOrderByClause(String orderByClause) { 18 | this.orderByClause = orderByClause; 19 | } 20 | 21 | public String getOrderByClause() { 22 | return orderByClause; 23 | } 24 | 25 | public void setDistinct(boolean distinct) { 26 | this.distinct = distinct; 27 | } 28 | 29 | public boolean isDistinct() { 30 | return distinct; 31 | } 32 | 33 | public List getOredCriteria() { 34 | return oredCriteria; 35 | } 36 | 37 | public void or(Criteria criteria) { 38 | oredCriteria.add(criteria); 39 | } 40 | 41 | public Criteria or() { 42 | Criteria criteria = createCriteriaInternal(); 43 | oredCriteria.add(criteria); 44 | return criteria; 45 | } 46 | 47 | public Criteria createCriteria() { 48 | Criteria criteria = createCriteriaInternal(); 49 | if (oredCriteria.size() == 0) { 50 | oredCriteria.add(criteria); 51 | } 52 | return criteria; 53 | } 54 | 55 | protected Criteria createCriteriaInternal() { 56 | Criteria criteria = new Criteria(); 57 | return criteria; 58 | } 59 | 60 | public void clear() { 61 | oredCriteria.clear(); 62 | orderByClause = null; 63 | distinct = false; 64 | } 65 | 66 | protected abstract static class GeneratedCriteria { 67 | protected List criteria; 68 | 69 | protected GeneratedCriteria() { 70 | super(); 71 | criteria = new ArrayList(); 72 | } 73 | 74 | public boolean isValid() { 75 | return criteria.size() > 0; 76 | } 77 | 78 | public List getAllCriteria() { 79 | return criteria; 80 | } 81 | 82 | public List getCriteria() { 83 | return criteria; 84 | } 85 | 86 | protected void addCriterion(String condition) { 87 | if (condition == null) { 88 | throw new RuntimeException("Value for condition cannot be null"); 89 | } 90 | criteria.add(new Criterion(condition)); 91 | } 92 | 93 | protected void addCriterion(String condition, Object value, String property) { 94 | if (value == null) { 95 | throw new RuntimeException("Value for " + property + " cannot be null"); 96 | } 97 | criteria.add(new Criterion(condition, value)); 98 | } 99 | 100 | protected void addCriterion(String condition, Object value1, Object value2, String property) { 101 | if (value1 == null || value2 == null) { 102 | throw new RuntimeException("Between values for " + property + " cannot be null"); 103 | } 104 | criteria.add(new Criterion(condition, value1, value2)); 105 | } 106 | 107 | public Criteria andIdIsNull() { 108 | addCriterion("id is null"); 109 | return (Criteria) this; 110 | } 111 | 112 | public Criteria andIdIsNotNull() { 113 | addCriterion("id is not null"); 114 | return (Criteria) this; 115 | } 116 | 117 | public Criteria andIdEqualTo(Integer value) { 118 | addCriterion("id =", value, "id"); 119 | return (Criteria) this; 120 | } 121 | 122 | public Criteria andIdNotEqualTo(Integer value) { 123 | addCriterion("id <>", value, "id"); 124 | return (Criteria) this; 125 | } 126 | 127 | public Criteria andIdGreaterThan(Integer value) { 128 | addCriterion("id >", value, "id"); 129 | return (Criteria) this; 130 | } 131 | 132 | public Criteria andIdGreaterThanOrEqualTo(Integer value) { 133 | addCriterion("id >=", value, "id"); 134 | return (Criteria) this; 135 | } 136 | 137 | public Criteria andIdLessThan(Integer value) { 138 | addCriterion("id <", value, "id"); 139 | return (Criteria) this; 140 | } 141 | 142 | public Criteria andIdLessThanOrEqualTo(Integer value) { 143 | addCriterion("id <=", value, "id"); 144 | return (Criteria) this; 145 | } 146 | 147 | public Criteria andIdIn(List values) { 148 | addCriterion("id in", values, "id"); 149 | return (Criteria) this; 150 | } 151 | 152 | public Criteria andIdNotIn(List values) { 153 | addCriterion("id not in", values, "id"); 154 | return (Criteria) this; 155 | } 156 | 157 | public Criteria andIdBetween(Integer value1, Integer value2) { 158 | addCriterion("id between", value1, value2, "id"); 159 | return (Criteria) this; 160 | } 161 | 162 | public Criteria andIdNotBetween(Integer value1, Integer value2) { 163 | addCriterion("id not between", value1, value2, "id"); 164 | return (Criteria) this; 165 | } 166 | 167 | public Criteria andNameIsNull() { 168 | addCriterion("name is null"); 169 | return (Criteria) this; 170 | } 171 | 172 | public Criteria andNameIsNotNull() { 173 | addCriterion("name is not null"); 174 | return (Criteria) this; 175 | } 176 | 177 | public Criteria andNameEqualTo(String value) { 178 | addCriterion("name =", value, "name"); 179 | return (Criteria) this; 180 | } 181 | 182 | public Criteria andNameNotEqualTo(String value) { 183 | addCriterion("name <>", value, "name"); 184 | return (Criteria) this; 185 | } 186 | 187 | public Criteria andNameGreaterThan(String value) { 188 | addCriterion("name >", value, "name"); 189 | return (Criteria) this; 190 | } 191 | 192 | public Criteria andNameGreaterThanOrEqualTo(String value) { 193 | addCriterion("name >=", value, "name"); 194 | return (Criteria) this; 195 | } 196 | 197 | public Criteria andNameLessThan(String value) { 198 | addCriterion("name <", value, "name"); 199 | return (Criteria) this; 200 | } 201 | 202 | public Criteria andNameLessThanOrEqualTo(String value) { 203 | addCriterion("name <=", value, "name"); 204 | return (Criteria) this; 205 | } 206 | 207 | public Criteria andNameLike(String value) { 208 | addCriterion("name like", value, "name"); 209 | return (Criteria) this; 210 | } 211 | 212 | public Criteria andNameNotLike(String value) { 213 | addCriterion("name not like", value, "name"); 214 | return (Criteria) this; 215 | } 216 | 217 | public Criteria andNameIn(List values) { 218 | addCriterion("name in", values, "name"); 219 | return (Criteria) this; 220 | } 221 | 222 | public Criteria andNameNotIn(List values) { 223 | addCriterion("name not in", values, "name"); 224 | return (Criteria) this; 225 | } 226 | 227 | public Criteria andNameBetween(String value1, String value2) { 228 | addCriterion("name between", value1, value2, "name"); 229 | return (Criteria) this; 230 | } 231 | 232 | public Criteria andNameNotBetween(String value1, String value2) { 233 | addCriterion("name not between", value1, value2, "name"); 234 | return (Criteria) this; 235 | } 236 | 237 | public Criteria andIntroIsNull() { 238 | addCriterion("intro is null"); 239 | return (Criteria) this; 240 | } 241 | 242 | public Criteria andIntroIsNotNull() { 243 | addCriterion("intro is not null"); 244 | return (Criteria) this; 245 | } 246 | 247 | public Criteria andIntroEqualTo(String value) { 248 | addCriterion("intro =", value, "intro"); 249 | return (Criteria) this; 250 | } 251 | 252 | public Criteria andIntroNotEqualTo(String value) { 253 | addCriterion("intro <>", value, "intro"); 254 | return (Criteria) this; 255 | } 256 | 257 | public Criteria andIntroGreaterThan(String value) { 258 | addCriterion("intro >", value, "intro"); 259 | return (Criteria) this; 260 | } 261 | 262 | public Criteria andIntroGreaterThanOrEqualTo(String value) { 263 | addCriterion("intro >=", value, "intro"); 264 | return (Criteria) this; 265 | } 266 | 267 | public Criteria andIntroLessThan(String value) { 268 | addCriterion("intro <", value, "intro"); 269 | return (Criteria) this; 270 | } 271 | 272 | public Criteria andIntroLessThanOrEqualTo(String value) { 273 | addCriterion("intro <=", value, "intro"); 274 | return (Criteria) this; 275 | } 276 | 277 | public Criteria andIntroLike(String value) { 278 | addCriterion("intro like", value, "intro"); 279 | return (Criteria) this; 280 | } 281 | 282 | public Criteria andIntroNotLike(String value) { 283 | addCriterion("intro not like", value, "intro"); 284 | return (Criteria) this; 285 | } 286 | 287 | public Criteria andIntroIn(List values) { 288 | addCriterion("intro in", values, "intro"); 289 | return (Criteria) this; 290 | } 291 | 292 | public Criteria andIntroNotIn(List values) { 293 | addCriterion("intro not in", values, "intro"); 294 | return (Criteria) this; 295 | } 296 | 297 | public Criteria andIntroBetween(String value1, String value2) { 298 | addCriterion("intro between", value1, value2, "intro"); 299 | return (Criteria) this; 300 | } 301 | 302 | public Criteria andIntroNotBetween(String value1, String value2) { 303 | addCriterion("intro not between", value1, value2, "intro"); 304 | return (Criteria) this; 305 | } 306 | 307 | public Criteria andPicUrlIsNull() { 308 | addCriterion("pic_url is null"); 309 | return (Criteria) this; 310 | } 311 | 312 | public Criteria andPicUrlIsNotNull() { 313 | addCriterion("pic_url is not null"); 314 | return (Criteria) this; 315 | } 316 | 317 | public Criteria andPicUrlEqualTo(String value) { 318 | addCriterion("pic_url =", value, "picUrl"); 319 | return (Criteria) this; 320 | } 321 | 322 | public Criteria andPicUrlNotEqualTo(String value) { 323 | addCriterion("pic_url <>", value, "picUrl"); 324 | return (Criteria) this; 325 | } 326 | 327 | public Criteria andPicUrlGreaterThan(String value) { 328 | addCriterion("pic_url >", value, "picUrl"); 329 | return (Criteria) this; 330 | } 331 | 332 | public Criteria andPicUrlGreaterThanOrEqualTo(String value) { 333 | addCriterion("pic_url >=", value, "picUrl"); 334 | return (Criteria) this; 335 | } 336 | 337 | public Criteria andPicUrlLessThan(String value) { 338 | addCriterion("pic_url <", value, "picUrl"); 339 | return (Criteria) this; 340 | } 341 | 342 | public Criteria andPicUrlLessThanOrEqualTo(String value) { 343 | addCriterion("pic_url <=", value, "picUrl"); 344 | return (Criteria) this; 345 | } 346 | 347 | public Criteria andPicUrlLike(String value) { 348 | addCriterion("pic_url like", value, "picUrl"); 349 | return (Criteria) this; 350 | } 351 | 352 | public Criteria andPicUrlNotLike(String value) { 353 | addCriterion("pic_url not like", value, "picUrl"); 354 | return (Criteria) this; 355 | } 356 | 357 | public Criteria andPicUrlIn(List values) { 358 | addCriterion("pic_url in", values, "picUrl"); 359 | return (Criteria) this; 360 | } 361 | 362 | public Criteria andPicUrlNotIn(List values) { 363 | addCriterion("pic_url not in", values, "picUrl"); 364 | return (Criteria) this; 365 | } 366 | 367 | public Criteria andPicUrlBetween(String value1, String value2) { 368 | addCriterion("pic_url between", value1, value2, "picUrl"); 369 | return (Criteria) this; 370 | } 371 | 372 | public Criteria andPicUrlNotBetween(String value1, String value2) { 373 | addCriterion("pic_url not between", value1, value2, "picUrl"); 374 | return (Criteria) this; 375 | } 376 | } 377 | 378 | public static class Criteria extends GeneratedCriteria { 379 | 380 | protected Criteria() { 381 | super(); 382 | } 383 | } 384 | 385 | public static class Criterion { 386 | private String condition; 387 | 388 | private Object value; 389 | 390 | private Object secondValue; 391 | 392 | private boolean noValue; 393 | 394 | private boolean singleValue; 395 | 396 | private boolean betweenValue; 397 | 398 | private boolean listValue; 399 | 400 | private String typeHandler; 401 | 402 | public String getCondition() { 403 | return condition; 404 | } 405 | 406 | public Object getValue() { 407 | return value; 408 | } 409 | 410 | public Object getSecondValue() { 411 | return secondValue; 412 | } 413 | 414 | public boolean isNoValue() { 415 | return noValue; 416 | } 417 | 418 | public boolean isSingleValue() { 419 | return singleValue; 420 | } 421 | 422 | public boolean isBetweenValue() { 423 | return betweenValue; 424 | } 425 | 426 | public boolean isListValue() { 427 | return listValue; 428 | } 429 | 430 | public String getTypeHandler() { 431 | return typeHandler; 432 | } 433 | 434 | protected Criterion(String condition) { 435 | super(); 436 | this.condition = condition; 437 | this.typeHandler = null; 438 | this.noValue = true; 439 | } 440 | 441 | protected Criterion(String condition, Object value, String typeHandler) { 442 | super(); 443 | this.condition = condition; 444 | this.value = value; 445 | this.typeHandler = typeHandler; 446 | if (value instanceof List) { 447 | this.listValue = true; 448 | } else { 449 | this.singleValue = true; 450 | } 451 | } 452 | 453 | protected Criterion(String condition, Object value) { 454 | this(condition, value, null); 455 | } 456 | 457 | protected Criterion(String condition, Object value, Object secondValue, String typeHandler) { 458 | super(); 459 | this.condition = condition; 460 | this.value = value; 461 | this.secondValue = secondValue; 462 | this.typeHandler = typeHandler; 463 | this.betweenValue = true; 464 | } 465 | 466 | protected Criterion(String condition, Object value, Object secondValue) { 467 | this(condition, value, secondValue, null); 468 | } 469 | } 470 | } --------------------------------------------------------------------------------