GET and POST
50 | * methods.
51 | *
52 | * @param request servlet request
53 | * @param response servlet response
54 | * @throws ServletException if a servlet-specific error occurs
55 | * @throws IOException if an I/O error occurs
56 | */
57 | protected void processRequest(HttpServletRequest request, HttpServletResponse response)
58 | throws ServletException, IOException {
59 | long start = System.currentTimeMillis();
60 | request.setCharacterEncoding("UTF-8");
61 | response.setContentType("application/json;charset=UTF-8");
62 | String url = request.getParameter("url");
63 | String nextPageCssQuery = request.getParameter("nextPageCssQuery");
64 | String nextPageText = request.getParameter("nextPageText");
65 | String titleCssQuery = request.getParameter("titleCssQuery");
66 | String lastTime = request.getParameter("lastTime");
67 | String proxyHost = request.getParameter("proxyHost");
68 | String proxyPort = request.getParameter("proxyPort");
69 | LOGGER.info("url:"+url);
70 | LOGGER.info("nextPageCssQuery:"+nextPageCssQuery);
71 | LOGGER.info("nextPageText:"+nextPageText);
72 | LOGGER.info("titleCssQuery:"+titleCssQuery);
73 | ListGET method.
86 | *
87 | * @param request servlet request
88 | * @param response servlet response
89 | * @throws ServletException if a servlet-specific error occurs
90 | * @throws IOException if an I/O error occurs
91 | */
92 | @Override
93 | protected void doGet(HttpServletRequest request, HttpServletResponse response)
94 | throws ServletException, IOException {
95 | processRequest(request, response);
96 | }
97 |
98 | /**
99 | * Handles the HTTP POST method.
100 | *
101 | * @param request servlet request
102 | * @param response servlet response
103 | * @throws ServletException if a servlet-specific error occurs
104 | * @throws IOException if an I/O error occurs
105 | */
106 | @Override
107 | protected void doPost(HttpServletRequest request, HttpServletResponse response)
108 | throws ServletException, IOException {
109 | processRequest(request, response);
110 | }
111 |
112 | /**
113 | * Returns a short description of the servlet.
114 | *
115 | * @return a String containing servlet description
116 | */
117 | @Override
118 | public String getServletInfo() {
119 | return "Short description";
120 | }// GET and POST
55 | * methods.
56 | *
57 | * @param request servlet request
58 | * @param response servlet response
59 | * @throws ServletException if a servlet-specific error occurs
60 | * @throws IOException if an I/O error occurs
61 | */
62 | protected void processRequest(HttpServletRequest request, HttpServletResponse response)
63 | throws ServletException, IOException {
64 | request.setCharacterEncoding("UTF-8");
65 | response.setContentType("application/json;charset=UTF-8");
66 | long start = System.currentTimeMillis();
67 | //获取栏目文章和链接
68 | String url = request.getParameter("url");
69 | String nextPageCssQuery = request.getParameter("nextPageCssQuery");
70 | String nextPageText = request.getParameter("nextPageText");
71 | String titleCssQuery = request.getParameter("titleCssQuery");
72 | String lastTime = request.getParameter("lastTime");
73 | String proxyHost = request.getParameter("proxyHost");
74 | String proxyPort = request.getParameter("proxyPort");
75 | LOGGER.info("url:"+url);
76 | LOGGER.info("nextPageCssQuery:"+nextPageCssQuery);
77 | LOGGER.info("nextPageText:"+nextPageText);
78 | LOGGER.info("titleCssQuery:"+titleCssQuery);
79 | ListGET method.
103 | *
104 | * @param request servlet request
105 | * @param response servlet response
106 | * @throws ServletException if a servlet-specific error occurs
107 | * @throws IOException if an I/O error occurs
108 | */
109 | @Override
110 | protected void doGet(HttpServletRequest request, HttpServletResponse response)
111 | throws ServletException, IOException {
112 | processRequest(request, response);
113 | }
114 |
115 | /**
116 | * Handles the HTTP POST method.
117 | *
118 | * @param request servlet request
119 | * @param response servlet response
120 | * @throws ServletException if a servlet-specific error occurs
121 | * @throws IOException if an I/O error occurs
122 | */
123 | @Override
124 | protected void doPost(HttpServletRequest request, HttpServletResponse response)
125 | throws ServletException, IOException {
126 | processRequest(request, response);
127 | }
128 |
129 | /**
130 | * Returns a short description of the servlet.
131 | *
132 | * @return a String containing servlet description
133 | */
134 | @Override
135 | public String getServletInfo() {
136 | return "Short description";
137 | }// GET and POST
49 | * methods.
50 | *
51 | * @param request servlet request
52 | * @param response servlet response
53 | * @throws ServletException if a servlet-specific error occurs
54 | * @throws IOException if an I/O error occurs
55 | */
56 | protected void processRequest(HttpServletRequest request, HttpServletResponse response)
57 | throws ServletException, IOException {
58 | long start = System.currentTimeMillis();
59 | request.setCharacterEncoding("UTF-8");
60 | response.setContentType("application/json;charset=UTF-8");
61 | String url = request.getParameter("url");
62 | String keyword = request.getParameter("keyword");
63 | LOGGER.info("url:"+url);
64 | LOGGER.info("keyword:"+keyword);
65 | Rank rank = new Rank();
66 | rank.setUrl(url);
67 | rank.setKeyword(keyword);
68 | RANKER.rank(rank);
69 |
70 | try (PrintWriter out = response.getWriter()) {
71 | String json = MAPPER.writeValueAsString(rank);
72 | out.println(json);
73 | }
74 | long cost = System.currentTimeMillis() - start;
75 | LOGGER.info("GetRank 耗时 "+cost+" 毫秒");
76 | }
77 |
78 | // GET method.
81 | *
82 | * @param request servlet request
83 | * @param response servlet response
84 | * @throws ServletException if a servlet-specific error occurs
85 | * @throws IOException if an I/O error occurs
86 | */
87 | @Override
88 | protected void doGet(HttpServletRequest request, HttpServletResponse response)
89 | throws ServletException, IOException {
90 | processRequest(request, response);
91 | }
92 |
93 | /**
94 | * Handles the HTTP POST method.
95 | *
96 | * @param request servlet request
97 | * @param response servlet response
98 | * @throws ServletException if a servlet-specific error occurs
99 | * @throws IOException if an I/O error occurs
100 | */
101 | @Override
102 | protected void doPost(HttpServletRequest request, HttpServletResponse response)
103 | throws ServletException, IOException {
104 | processRequest(request, response);
105 | }
106 |
107 | /**
108 | * Returns a short description of the servlet.
109 | *
110 | * @return a String containing servlet description
111 | */
112 | @Override
113 | public String getServletInfo() {
114 | return "Short description";
115 | }// 检查博文数目：" + articles.size()+"
"); 267 | AtomicInteger i = new AtomicInteger(); 268 | result.entrySet().stream().sorted((a,b)->b.getValue().size()-a.getValue().size()).forEach(e -> { 269 | String query = null; 270 | try { 271 | query = URLEncoder.encode(e.getKey().getTitle(), "UTF-8"); 272 | } catch (UnsupportedEncodingException ex) { 273 | LOGGER.error("url构造失败", ex); 274 | return; 275 | } 276 | String originURL = e.getKey().getUrl(); 277 | if(e.getValue().size()>0) { 278 | LOGGER.info(""+i.incrementAndGet()+"、" + e.getKey().getTitle() + " 抄袭链接有("+e.getValue().size()+")个
"); 279 | LOGGER.info("原文链接：" + originURL + ""); 280 | LOGGER.info("抄袭链接："); 281 | LOGGER.info("
- ");
282 | e.getValue().stream().sorted().forEach(url-> LOGGER.info("
- " + url + " ")); 283 | LOGGER.info("
"); 286 | LOGGER.info("原文链接：" + originURL + " 无抄袭链接
"); 287 | } 288 | }); 289 | } 290 | } 291 | -------------------------------------------------------------------------------- /src/main/java/org/seo/rank/impl/BaiduRanker.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * APDPlat - Application Product Development Platform 4 | * Copyright (c) 2013, 杨尚川, yang-shangchuan@qq.com 5 | * 6 | * This program is free software: you can redistribute it and/or modify 7 | * it under the terms of the GNU General Public License as published by 8 | * the Free Software Foundation, either version 3 of the License, or 9 | * (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program. If not, see
- ");
313 | map.entrySet().stream().sorted((a,b)->a.getValue()-b.getValue()).forEach(e -> {
314 | String query = null;
315 | try {
316 | query = URLEncoder.encode(e.getKey(), "UTF-8");
317 | } catch (UnsupportedEncodingException ex) {
318 | LOGGER.error("url构造失败", ex);
319 | return ;
320 | }
321 | LOGGER.info("
- " + e.getKey() + "(" + e.getValue() + ") "); 322 | }); 323 | LOGGER.info("
检查的博文数：" + commons.size() + "，相似度大于等于" + THRESHOLD_RATE + "的博文数：" + similarCount.get() + "，相似度小于" + THRESHOLD_RATE + "的博文数：" + (commons.size() - similarCount.get())+"
"); 327 | AtomicInteger i = new AtomicInteger(); 328 | result 329 | .entrySet() 330 | .stream() 331 | .sorted((a, b) -> b.getValue().compareTo(a.getValue())) 332 | .forEach(e -> { 333 | LOGGER.info(""); 334 | LOGGER.info(""+i.incrementAndGet() + "、检查博文" + "：" + e.getKey()+"，相似度分值："+e.getValue().doubleValue()+"
"); 335 | LOGGER.info("\t博文地址1："+om.get(e.getKey())+""); 336 | LOGGER.info("\t博文地址2："+im.get(e.getKey())+"
"); 337 | }); 338 | } 339 | public static void main(String[] args) throws Exception{ 340 | GenericWebPageSimilarChecker genericWebPageSimilarChecker = new GenericWebPageSimilarChecker(); 341 | genericWebPageSimilarChecker.verifyYscBlog(); 342 | } 343 | } 344 | -------------------------------------------------------------------------------- /src/main/java/org/seo/rank/impl/ITEYEBlogSimilarChecker.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * APDPlat - Application Product Development Platform 4 | * Copyright (c) 2013, 杨尚川, yang-shangchuan@qq.com 5 | * 6 | * This program is free software: you can redistribute it and/or modify 7 | * it under the terms of the GNU General Public License as published by 8 | * the Free Software Foundation, either version 3 of the License, or 9 | * (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program. If not, see
检查的博文数：" + commons.size() + "
"); 230 | AtomicInteger i = new AtomicInteger(); 231 | result 232 | .entrySet() 233 | .stream() 234 | .forEach(e -> { 235 | LOGGER.info(""); 236 | LOGGER.info(""+i.incrementAndGet() + "、检查博文" + "：" + e.getKey()+"，相似度分值："+e.getValue()+"
"); 237 | LOGGER.info("\t博文地址1："+om.get(e.getKey())+""); 238 | LOGGER.info("\t博文地址2："+im.get(e.getKey())+"
"); 239 | }); 240 | } 241 | public static void main(String[] args) throws Exception{ 242 | List
oschina和iteye都有("+commons.size()+")：
"); 201 | AtomicInteger j = new AtomicInteger(); 202 | commons.forEach(item -> LOGGER.info(j.incrementAndGet()+"、"+item+" oschina iteye")); 203 | 204 | List
oschina独有("+oschina.size()+")：
"); 206 | AtomicInteger l = new AtomicInteger(); 207 | oschina.forEach(item -> LOGGER.info(l.incrementAndGet()+"、"+item+"")); 208 | 209 | List
iteye独有("+iteye.size()+")：
"); 211 | AtomicInteger k = new AtomicInteger(); 212 | iteye.forEach(item -> LOGGER.info(k.incrementAndGet()+"、"+item+"")); 213 | } 214 | public static void main(String[] args){ 215 | //iteyeBlog(); 216 | //iteyeNews(); 217 | //iteyeMagazines(); 218 | //csdnBlog(); 219 | //oschinaNews(); 220 | //oschinaBlog(); 221 | //baidu("Java应用级产品开发平台APDPlat作者杨尚川专访"); 222 | blogCompare(); 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /src/main/java/org/seo/rank/model/Article.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * APDPlat - Application Product Development Platform 4 | * Copyright (c) 2013, 杨尚川, yang-shangchuan@qq.com 5 | * 6 | * This program is free software: you can redistribute it and/or modify 7 | * it under the terms of the GNU General Public License as published by 8 | * the Free Software Foundation, either version 3 of the License, or 9 | * (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program. If not, see
| 排名 | 票数 | 项目名称 | 项目作者 | 项目描述 |
| " + i.incrementAndGet() + " | " + e.getValue() + " | " + projectName + " | " + owner + " | " + des + " |