`| 图片资源不存在83 | * increase the RAM buffer. But if you do this, increase the max heap size to the JVM (eg add -Xmx512m or -Xmx1g): 84 | */ 85 | // writerConfig.setRAMBufferSizeMB(256.0); 86 | 87 | IndexWriter iWriter = new IndexWriter(directory, writerConfig); 88 | 89 | int count = 0; 90 | Document doc = null; 91 | Field field = null; 92 | 93 | /** declaring string type */ 94 | FieldType stringType = new FieldType(); 95 | stringType.setTokenized(true); 96 | 97 | /** Looping through resultset and adding data to index file */ 98 | while (rs.next()) { 99 | doc = new Document(); 100 | 101 | /** adding id in document */ 102 | field = new IntField("id", rs.getInt("id"), Field.Store.YES); 103 | doc.add(field); 104 | 105 | /** adding name in document */ 106 | field = new TextField("title", rs.getString("title"), Field.Store.YES); 107 | doc.add(field); 108 | 109 | /** adding details in document */ 110 | field = new TextField("body", rs.getString("body"), Field.Store.YES); 111 | doc.add(field); 112 | 113 | /** Adding doc to iWriter */ 114 | iWriter.addDocument(doc); 115 | count++; 116 | } 117 | 118 | System.out.println(count + " record indexed"); 119 | 120 | /** Closing iWriter */ 121 | iWriter.commit(); 122 | iWriter.close(); 123 | 124 | /** Closing JDBC connection */ 125 | rs.close(); 126 | stmt.close(); 127 | conn.close(); 128 | 129 | } catch (Exception e) { 130 | e.printStackTrace(); 131 | } 132 | 133 | } 134 | 135 | /** 136 | * to search the keywords 137 | * 138 | * @param keyword 139 | */ 140 | public void search(String keyword) { 141 | 142 | System.out.println("-- Seaching --"); 143 | 144 | try { 145 | /** Searching */ 146 | Directory index = FSDirectory.open(FileSystems.getDefault().getPath("./index2222")); 147 | 148 | IndexReader directoryReader = DirectoryReader.open(index); 149 | 150 | // IndexReader directoryReader = DirectoryReader 151 | // .open(FSDirectory.open(FileSystems.getDefault().getPath("./index2222"))); 152 | 153 | IndexSearcher searcher = new IndexSearcher(directoryReader); 154 | // StandardAnalyzer keywordAnalyzer = new StandardAnalyzer(); 155 | Analyzer luceneAnalyzer = new JcsegAnalyzer5X(JcsegTaskConfig.COMPLEX_MODE); 156 | 157 | /** MultiFieldQueryParser is used to search multiple fields */ 158 | String[] filesToSearch = { "id", "title", "body" }; 159 | MultiFieldQueryParser mqp = new MultiFieldQueryParser(filesToSearch, luceneAnalyzer); 160 | 161 | /** search the given keyword */ 162 | Query query = mqp.parse(keyword); 163 | System.out.println("query >> " + query); 164 | // 165 | // /** defining the sorting on filed "name" */ 166 | Sort nameSort = new Sort(new SortField("id", Type.STRING)); 167 | 168 | /** run the query */ 169 | TopDocs hits = searcher.search(query, 1000); 170 | System.out.println("Results found >> " + hits.totalHits); 171 | 172 | Document doc = null; 173 | for (int i = 0; i < hits.totalHits; i++) { 174 | /** get the next document */ 175 | doc = searcher.doc(hits.scoreDocs[i].doc); 176 | System.out.println("==========" + (i + 1) + " : Start Record=========\nId :: " + doc.get("id") 177 | + "\ntitle :: " + doc.get("title") + "\n==========End Record=========\n"); 178 | } 179 | } catch (Exception e) { 180 | e.printStackTrace(); 181 | } 182 | 183 | } 184 | 185 | /** 186 | * main method to check the output 187 | * 188 | * @param args 189 | */ 190 | public static void main(String[] args) { 191 | 192 | SearchDao obj = new SearchDao(); 193 | 194 | /** creating index */ 195 | // obj.createIndex(); 196 | 197 | /** searching simple keyword */ 198 | System.out.println("==================searching simple keyword==========================="); 199 | obj.search("电院课表"); 200 | 201 | // /** searching simple keyword */ 202 | // System.out.println("==================searching simple 203 | // keyword==========================="); 204 | // obj.search("褚"); 205 | // 206 | // /** searching using wild card */ 207 | // System.out.println("==================searching using wild 208 | // card==========================="); 209 | // obj.search("791"); 210 | // 211 | // /** searching using logical OR operator */ 212 | // System.out.println("==================searching using logical OR 213 | // operator==========================="); 214 | // obj.search("院"); 215 | } 216 | 217 | } -------------------------------------------------------------------------------- /src/com/chenxb/dao/SummaryDao.java: -------------------------------------------------------------------------------- 1 | package com.chenxb.dao; 2 | 3 | import java.sql.Connection; 4 | import java.sql.PreparedStatement; 5 | import java.sql.ResultSet; 6 | import java.sql.SQLException; 7 | import java.util.List; 8 | 9 | import org.jsoup.Jsoup; 10 | import org.jsoup.nodes.Document; 11 | 12 | import com.chenxb.model.ArticleItem; 13 | import com.chenxb.util.MysqlTool; 14 | import com.chenxb.util.TableName; 15 | import com.hankcs.hanlp.HanLP; 16 | 17 | /** 18 | * 增加文章的摘要 19 | * 可以先把数据存入 mysql，再读取mysql 的数据增加摘要 20 | * 也可以爬虫的时候就把摘要存入 mysql 21 | * @author tomchen 22 | * 23 | */ 24 | public class SummaryDao { 25 | private Connection connection; 26 | 27 | public SummaryDao() throws Exception { 28 | connection = new MysqlTool().getConnection(); 29 | } 30 | 31 | /** 32 | * 根据 type 找到数据库表名称 33 | * 再从该表里找出 id 对应的新闻 34 | * @param type 35 | * @param id 36 | * @return 37 | * @throws SQLException 38 | */ 39 | public int updateSummary(int type, int id) throws SQLException { 40 | // 根据 type 找出对应的 table 名称 41 | String tableName = TableName.getTableByType(type); 42 | 43 | String body = getArticleBody(type, id); 44 | // body是 html 表示的 45 | Document doc = Jsoup.parse(body); 46 | List
ImageTool.convertUrl() 无法解析图片
"); 67 | builder.append("图片 src = " + origin + "
"); 68 | MailTool.sendException(builder.toString(), currentPage, MailTool.IMAGE_UNUSUAL); 69 | return origin; 70 | } 71 | 72 | } 73 | 74 | /** imageKey 为输入参数 75 | * 76 | * @param currentPage 77 | * @param origin 78 | * @param imageKey 79 | * @return 80 | */ 81 | public static String convertUrl(int currentPage, String origin, String imageKey) { 82 | // 图片资源不一定都是在 uploads 文件夹下面 83 | // 也有可能外链到其他网站的图片 84 | if (origin.startsWith(Constant.HTTP_PREFIX)) { 85 | // 以绝对路径开头，最前面是网站域名 86 | // 比如 http://see.xidian.edu.cn/uploads/image/20141120/201411**.png 87 | // http://imgtec.eetrend.com/sites/*** 88 | uploadByUrl(currentPage, origin, imageKey); 89 | return imageKey; 90 | } else if (origin.startsWith(IMAGE_BASE)) { 91 | // 相对路径，比如/uploads/image/20141120/20141120**.jpg 92 | // /Public/kindeditor/php/../../../uploads/image/20151116/20151116114927_39484.jpg 93 | // 把图片上传给七牛 94 | // if 的先后顺序，先判断是否是全路径，再判断是不是相对路径 95 | String wholeURl = Constant.SEE_URL + origin; 96 | 97 | uploadByUrl(currentPage, wholeURl, imageKey); 98 | return imageKey; 99 | } else if (origin.startsWith(IMAGE_OLD__BASE)) { 100 | // 老图片路径 /uploads/old 101 | String wholeURl = Constant.SEE_URL + origin; 102 | uploadByUrl(currentPage, wholeURl, imageKey); 103 | return imageKey; 104 | } else { 105 | // 这部分 todo，识别其他格式的图片 106 | // 或者试图访问这个图片，但失败了，则不是完整的 url 107 | StringBuilder builder = new StringBuilder(); 108 | builder.append("ImageTool.convertUrl() 无法解析图片
"); 109 | builder.append("图片 url = " + origin + "
"); 110 | MailTool.sendException(builder.toString(), currentPage, MailTool.IMAGE_UNUSUAL); 111 | return origin; 112 | } 113 | 114 | } 115 | 116 | /** 117 | * 118 | * @param url 119 | * 给定图片的 url 120 | * @return 将图片上传至七牛，返回七牛上图片的 url 121 | * @throws QiniuException 122 | */ 123 | private static void uploadByUrl(int currentPage, String originalUrl, String key) { 124 | FetchRunnable f = new FetchRunnable(currentPage, originalUrl, key); 125 | new Thread(f).start(); 126 | } 127 | 128 | } 129 | 130 | /** 131 | * 图片上传使用多线程 132 | * 有问题？上传失败如何回滚？ 133 | * 失败的概率很小，暂时不考虑 134 | * 或者失败了发邮件通知 135 | * @author tomchen 136 | * 137 | */ 138 | class FetchRunnable implements Runnable { 139 | private static final String ACCESS_KEY = "**-*********"; // 你的access_key 140 | private static final String SECRET_KEY = "**-*********"; // 你的secret_key 141 | private static final String BUCKET_NAME = "*****"; // 你的secret_key 142 | 143 | private int currentPage; 144 | private String url; 145 | private String key; 146 | 147 | public FetchRunnable(int currentPage, String url, String key) { 148 | this.currentPage = currentPage; 149 | this.url = url; 150 | this.key = key; 151 | } 152 | 153 | @Override 154 | public void run() { 155 | // 获取到 Access Key 和 Secret Key 之后，您可以按照如下方式进行密钥配置 156 | Auth auth = Auth.create(ACCESS_KEY, SECRET_KEY); 157 | // 获取空间管理器 158 | BucketManager bucketManager = new BucketManager(auth); 159 | try { 160 | // 要求url可公网正常访问BucketManager.fetch(url, bucketName, key); 161 | // @param url 网络上一个资源文件的URL 162 | // @param bucketName 空间名称 163 | // @param key 空间内文件的key[唯一的] 164 | bucketManager.fetch(url, BUCKET_NAME, key); 165 | } catch (QiniuException e) { 166 | // 处理已知的部分资源不存在 167 | if (StringUtils.endsWithAny(url, Constant.DOC_JPG_SUFFIX, Constant.XLS_JPG_SUFFIX, Constant.RAR_JPG_SUFFIX, 168 | Constant.ZIP_JPG_SUFFIX)) { 169 | // 已经手工上传了这几种图标 170 | return; 171 | } 172 | 173 | StringWriter errors = new StringWriter(); 174 | e.printStackTrace(new PrintWriter(errors)); 175 | 176 | StringBuilder builder = new StringBuilder(errors.toString()); 177 | builder.append("ImageTool.uploadByUrl(url, key)发生异常！
"); 178 | builder.append("url = " + url + "
"); 179 | builder.append("key = " + key + "
"); 180 | MailTool.sendException(builder.toString(), currentPage, MailTool.ARTICLE_ITEM_BIZ); 181 | 182 | // to do 失败了邮件通知 183 | } 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /src/com/chenxb/util/JobScheduler.java: -------------------------------------------------------------------------------- 1 | package com.chenxb.util; 2 | 3 | import java.util.LinkedList; 4 | import java.util.List; 5 | import java.util.Random; 6 | 7 | import com.chenxb.biz.ArticleBiz; 8 | import com.chenxb.biz.ColumnBiz; 9 | 10 | public class JobScheduler { 11 | 12 | public static void main(String[] args) throws Exception { 13 | int[] ids = ColumnBiz.parseColumn(2, 3); 14 | List新闻页面 url = " + Constant.ARTICLE_BASE_URL + currentPage + ".html" + "
"); 79 | builder.append("异常发生时间 " + TimeTool.getCurrentTime() + "
"); 80 | // /* 设置Content 浏览器解析编码和格式等 */ 81 | msg.setContent(builder.toString(), "text/html;charset=utf-8"); 82 | // 设置发件人的邮箱 83 | msg.setFrom(new InternetAddress("905073281@qq.com")); 84 | 85 | // 给收件人的地址发送上面的 Message 86 | transport.sendMessage(msg, new Address[] { new InternetAddress("studychen@foxmail.com") }); 87 | transport.close(); 88 | } catch (MessagingException e) { 89 | e.printStackTrace(); 90 | return false; 91 | } 92 | 93 | // 无异常抛出，表示发送成功 94 | return true; 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/com/chenxb/util/MysqlTool.java: -------------------------------------------------------------------------------- 1 | package com.chenxb.util; 2 | 3 | import java.sql.Connection; 4 | import java.sql.DriverManager; 5 | import java.sql.SQLException; 6 | 7 | import com.sina.sae.util.SaeUserInfo; 8 | 9 | public class MysqlTool { 10 | 11 | // 是否使用新浪云 12 | private static boolean isCloud = false; 13 | 14 | public MysqlTool() { 15 | // JDBC驱动程序 16 | try { 17 | Class.forName("com.mysql.jdbc.Driver").newInstance(); 18 | } catch (Exception e) { 19 | e.printStackTrace(); 20 | } 21 | } 22 | 23 | // 为了方便分析错误，将异常全部抛出到最顶层 24 | public Connection getConnection() throws Exception { 25 | 26 | // 后面unicode和utf8设置防止中文乱码 27 | String url = "jdbc:mysql://127.0.0.1:3306/see_news?useSSL=false&useUnicode=true&characterEncoding=utf-8"; 28 | String name = "root"; 29 | String password = "chenxb123"; 30 | 31 | if (isCloud) { 32 | String appName = SaeUserInfo.getAppName(); 33 | String mysqlName = "app_" + appName; 34 | url = "jdbc:mysql://w.rdc.sae.sina.com.cn:3307/" + mysqlName + "?autoReconnect=true"; 35 | name = SaeUserInfo.getAccessKey(); 36 | password = SaeUserInfo.getSecretKey(); 37 | } 38 | Connection con = DriverManager.getConnection(url, name, password); 39 | 40 | return con; 41 | 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/com/chenxb/util/StreamTool.java: -------------------------------------------------------------------------------- 1 | package com.chenxb.util; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.ByteArrayOutputStream; 5 | import java.io.InputStream; 6 | import java.io.InputStreamReader; 7 | 8 | public class StreamTool { 9 | 10 | /** 11 | * 利用ByteArrayOutputStream将流转化为字符串 12 | * 13 | * @param in 14 | * 需要读取的InputStream 15 | * @return 读取的字符串 16 | * @throws Exception 17 | */ 18 | public static String inToStringByByte(InputStream in) throws Exception { 19 | ByteArrayOutputStream outStr = new ByteArrayOutputStream(); 20 | byte[] buffer = new byte[1024]; 21 | // 这部分有问题，一个中文3个byte，如何确定1024最末尾的正好是一个中文 22 | int len = 0; 23 | StringBuilder content = new StringBuilder(); 24 | while ((len = in.read(buffer)) != -1) { 25 | content.append(new String(buffer, 0, len, "UTF-8")); 26 | } 27 | outStr.close(); 28 | return content.toString(); 29 | } 30 | 31 | /** 32 | * 利用BufferedReader将流转化为字符串 33 | * 34 | * @param in 35 | * 需要读取的InputStream 36 | * @return 读取的字符串 37 | * @throws Exception 38 | */ 39 | public static String inToStringByReader(InputStream in) throws Exception { 40 | BufferedReader reader = null; 41 | StringBuilder content = new StringBuilder(); 42 | reader = new BufferedReader(new InputStreamReader(in)); 43 | String line = ""; 44 | while ((line = reader.readLine()) != null) { 45 | System.out.println(line); 46 | content.append(line); 47 | } 48 | return content.toString(); 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/com/chenxb/util/StringTool.java: -------------------------------------------------------------------------------- 1 | package com.chenxb.util; 2 | 3 | import java.math.BigInteger; 4 | import java.security.MessageDigest; 5 | import java.security.NoSuchAlgorithmException; 6 | import java.util.UUID; 7 | 8 | public class StringTool { 9 | 10 | public static String createUUID() { 11 | String s = UUID.randomUUID().toString(); 12 | return s.replaceAll("-", ""); 13 | } 14 | 15 | public static String createMD5(String plaintext) { 16 | MessageDigest m; 17 | try { 18 | m = MessageDigest.getInstance("MD5"); 19 | m.reset(); 20 | m.update(plaintext.getBytes()); 21 | byte[] digest = m.digest(); 22 | BigInteger bigInt = new BigInteger(1, digest); 23 | return bigInt.toString(16); 24 | } catch (NoSuchAlgorithmException e) { 25 | } 26 | return plaintext; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/com/chenxb/util/TableName.java: -------------------------------------------------------------------------------- 1 | package com.chenxb.util; 2 | 3 | public class TableName { 4 | public static final String LATEST = "latest";// 最新消息 5 | public static final String NOTIFIC = "notific";// 校园通知 6 | public static final String BACHELOR = "bachelor";// 本科教学 学士 7 | public static final String MASTER = "master";// 研究生 硕士 8 | public static final String ACADEMIC = "academic";// 学术交流 9 | public static final String JOB = "job";// 就业招聘 10 | 11 | /** 12 | * 这儿的设计能不能更优雅 13 | * 0是最新消息表，1是校园通知表 14 | * @param type 获取对应的表名称 15 | * @return 16 | */ 17 | public static String getTableByType(int type) { 18 | switch (type) { 19 | case ColumnType.LATEST: 20 | return LATEST; 21 | case ColumnType.NOTIFIC: 22 | return NOTIFIC; 23 | case ColumnType.BACHELOR: 24 | return BACHELOR; 25 | case ColumnType.MASTER: 26 | return MASTER; 27 | case ColumnType.ACADEMIC: 28 | return ACADEMIC; 29 | case ColumnType.JOB: 30 | return JOB; 31 | default: 32 | return LATEST; 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/com/chenxb/util/TimeTool.java: -------------------------------------------------------------------------------- 1 | package com.chenxb.util; 2 | 3 | import java.text.SimpleDateFormat; 4 | import java.util.Date; 5 | import java.util.Random; 6 | 7 | public class TimeTool { 8 | private static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss E"); 9 | 10 | private static final Random random = new Random(); 11 | private static final int WAIT_TIME = 60 * 1000;// 毫秒 12 | 13 | /** 14 | * 15 | * @return 格式化，得到当前的日期 16 | */ 17 | public static String getCurrentTime() { 18 | return dateFormat.format(new Date()); 19 | } 20 | 21 | /** 22 | * 等待一段时间，防止对被爬虫的网站负载太大 23 | */ 24 | public static void sleepSomeTime() { 25 | try { 26 | Thread.sleep(random.nextInt(WAIT_TIME) + WAIT_TIME); 27 | } catch (InterruptedException e) { 28 | e.printStackTrace(); 29 | } 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/com/chenxb/util/UrlTool.java: -------------------------------------------------------------------------------- 1 | package com.chenxb.util; 2 | 3 | import java.util.regex.Pattern; 4 | 5 | import org.apache.commons.lang3.StringUtils; 6 | 7 | public class UrlTool { 8 | // LATEST,//最新消息 9 | // NOTIFIC, //校园通知 10 | // BACHELOR, //本科教学 学士 11 | // MASTER, //研究生 硕士 12 | // RESEARCH, //科研 13 | // ACADEMIC //学术交流 14 | 15 | private static final String LATEST_URL = "http://see.xidian.edu.cn/index.php/index/more"; 16 | // 格式为http://see.xidian.edu.cn/html/category/5/2.html 17 | private static final String NOTIFIC_URL = "http://see.xidian.edu.cn/html/category/"; 18 | 19 | public static final Pattern VALID_EMAIL_REGEX = Pattern.compile("^[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,6}$", 20 | Pattern.CASE_INSENSITIVE); 21 | 22 | /** 23 | * 24 | * @param type 25 | * @param currentPage 26 | * 不是无限大，有一定范围 27 | * @return 28 | */ 29 | public static String generateUrl(int type, int currentPage) { 30 | currentPage = currentPage > 0 ? currentPage : 1; 31 | switch (type) { 32 | case ColumnType.LATEST: 33 | return LATEST_URL; 34 | case ColumnType.NOTIFIC: 35 | case ColumnType.BACHELOR: 36 | case ColumnType.MASTER: 37 | case ColumnType.ACADEMIC: 38 | case ColumnType.JOB: 39 | return NOTIFIC_URL + type + "/" + currentPage + ".html"; 40 | default: 41 | return LATEST_URL; 42 | } 43 | } 44 | 45 | /** 46 | * 这儿的tag 不和UrlTool.dealAttachmentUrl() 无法解析url
"); 90 | builder.append("异常 href = " + originTrim + "
"); 91 | MailTool.sendException(builder.toString(), currentPage, MailTool.HREF_UNUSUAL); 92 | return originTrim; 93 | } 94 | } 95 | 96 | } 97 | -------------------------------------------------------------------------------- /src/hanlp.properties: -------------------------------------------------------------------------------- 1 | #本配置文件中的路径的根目录，根目录+其他路径=绝对路径 2 | #Windows用户请注意，路径分隔符统一使用/ 3 | root=./ 4 | #核心词典路径 5 | CoreDictionaryPath=data/dictionary/CoreNatureDictionary.txt 6 | #2元语法词典路径 7 | BiGramDictionaryPath=data/dictionary/CoreNatureDictionary.ngram.txt 8 | #停用词词典路径 9 | CoreStopWordDictionaryPath=data/dictionary/stopwords.txt 10 | #同义词词典路径 11 | CoreSynonymDictionaryDictionaryPath=data/dictionary/synonym/CoreSynonym.txt 12 | #人名词典路径 13 | PersonDictionaryPath=data/dictionary/person/nr.txt 14 | #人名词典转移矩阵路径 15 | PersonDictionaryTrPath=data/dictionary/person/nr.tr.txt 16 | #繁简词典路径 17 | TraditionalChineseDictionaryPath=data/dictionary/tc/TraditionalChinese.txt 18 | #自定义词典路径，用;隔开多个自定义词典，空格开头表示在同一个目录，使用“文件名 词性”形式则表示这个词典的词性默认是该词性。优先级递减。 19 | #另外data/dictionary/custom/CustomDictionary.txt是个高质量的词库，请不要删除 20 | CustomDictionaryPath=data/dictionary/custom/CustomDictionary.txt; 现代汉语补充词库.txt; 全国地名大全.txt ns; 人名词典.txt; 机构名词典.txt; 上海地名.txt ns;data/dictionary/person/nrf.txt nrf 21 | #CRF分词模型路径 22 | CRFSegmentModelPath=data/model/segment/CRFSegmentModel.txt 23 | #HMM分词模型 24 | HMMSegmentModelPath=data/model/segment/HMMSegmentModel.bin 25 | #分词结果是否展示词性 26 | ShowTermNature=true --------------------------------------------------------------------------------