├── .gitignore
├── README.md
├── pom.xml
└── src
    ├── main
        ├── docker
        │   └── Dockerfile
        ├── java
        │   └── edu
        │   │   └── nwnu
        │   │       └── ququzone
        │   │           └── extractor
        │   │               ├── ExtractorConfiguration.java
        │   │               ├── result
        │   │                   ├── FailureResult.java
        │   │                   ├── Result.java
        │   │                   └── SuccessResult.java
        │   │               ├── service
        │   │                   ├── AbstractExtractor.java
        │   │                   ├── Extractor.java
        │   │                   ├── ParseException.java
        │   │                   ├── ReadabilityExtractor.java
        │   │                   └── RowBlockExtractor.java
        │   │               └── utlis
        │   │                   └── StringUtils.java
        └── resources
        │   ├── application.properties
        │   └── index.tpl
    └── test
        ├── java
            └── edu
            │   └── nwnu
            │       └── ququzone
            │           └── extractor
            │               └── service
            │                   └── ReadabilityExtractorTest.java
        └── resources
            └── test.html


/.gitignore:
--------------------------------------------------------------------------------
1 | .idea
2 | *.iml
3 | *.log
4 | target
5 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | html smart extractor
 2 | ====================
 3 | 
 4 | ### Introduction
 5 | 
 6 | A micro-service for extract main content from url.
 7 | 
 8 | ### Usage
 9 | 
10 | 1. Package
11 | 
12 | 	```
13 | 	$ mvn clean package
14 | 	```
15 | 
16 | 2. Run
17 | 
18 | 	```
19 | 	$ java -jar target/smart-extractor.jar
20 | 	$ open http://localhost:8080
21 | 	```
22 | 
23 | ### API
24 | 
25 | 1. Extract `http://localhost:8080/extract?url={url}`
26 | 
27 | 	```
28 | 	$ curl -i -X GET http://localhost:8080/extract\?url\=https://medium.com/@benjaminhardy/8-things-every-person-should-do-before-8-a-m-cc0233e15c8d
29 | 	```
30 | 
31 | ### Build Docker Image
32 | 
33 | 	$ mvn clean package
34 | 	$ mvn package docker:build
35 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 3 |          xmlns="http://maven.apache.org/POM/4.0.0"
 4 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 5 |     <modelVersion>4.0.0</modelVersion>
 6 | 
 7 |     <groupId>edu.nwnu.ququzone</groupId>
 8 |     <artifactId>smart-extractor</artifactId>
 9 |     <version>0.0.1-SNAPSHOT</version>
10 |     <packaging>jar</packaging>
11 | 
12 |     <parent>
13 |         <groupId>org.springframework.boot</groupId>
14 |         <artifactId>spring-boot-starter-parent</artifactId>
15 |         <version>1.2.7.RELEASE</version>
16 |     </parent>
17 | 
18 |     <properties>
19 |         <java.version>1.8</java.version>
20 |         <docker.image.prefix>ququzone</docker.image.prefix>
21 |     </properties>
22 | 
23 |     <dependencies>
24 |         <dependency>
25 |             <groupId>org.springframework.boot</groupId>
26 |             <artifactId>spring-boot-starter-web</artifactId>
27 |         </dependency>
28 | 
29 |         <dependency>
30 |             <groupId>com.squareup.okhttp</groupId>
31 |             <artifactId>okhttp</artifactId>
32 |             <version>2.5.0</version>
33 |         </dependency>
34 |         <dependency>
35 |             <groupId>org.jsoup</groupId>
36 |             <artifactId>jsoup</artifactId>
37 |             <version>1.8.3</version>
38 |         </dependency>
39 |         <dependency>
40 |             <groupId>com.google.guava</groupId>
41 |             <artifactId>guava</artifactId>
42 |             <version>18.0</version>
43 |         </dependency>
44 | 
45 |         <dependency>
46 |             <groupId>org.testng</groupId>
47 |             <artifactId>testng</artifactId>
48 |             <version>6.9.9</version>
49 |             <scope>test</scope>
50 |         </dependency>
51 | 
52 |         <dependency>
53 |             <groupId>com.googlecode.juniversalchardet</groupId>
54 |             <artifactId>juniversalchardet</artifactId>
55 |             <version>1.0.3</version>
56 |         </dependency>
57 |     </dependencies>
58 | 
59 |     <build>
60 |         <finalName>smart-extractor</finalName>
61 |         <plugins>
62 |             <plugin>
63 |                 <groupId>org.springframework.boot</groupId>
64 |                 <artifactId>spring-boot-maven-plugin</artifactId>
65 |             </plugin>
66 |             <plugin>
67 |                 <groupId>com.spotify</groupId>
68 |                 <artifactId>docker-maven-plugin</artifactId>
69 |                 <version>0.2.3</version>
70 |                 <configuration>
71 |                     <imageName>${docker.image.prefix}/${project.artifactId}</imageName>
72 |                     <dockerDirectory>src/main/docker</dockerDirectory>
73 |                     <resources>
74 |                         <resource>
75 |                             <targetPath>/</targetPath>
76 |                             <directory>${project.build.directory}</directory>
77 |                             <include>${project.build.finalName}.jar</include>
78 |                         </resource>
79 |                     </resources>
80 |                 </configuration>
81 |             </plugin>
82 |         </plugins>
83 |     </build>
84 | </project>
85 | 


--------------------------------------------------------------------------------
/src/main/docker/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM java:8
2 | VOLUME /tmp
3 | ADD smart-extractor.jar app.jar
4 | RUN bash -c 'touch /app.jar'
5 | ENTRYPOINT ["java","-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"]


--------------------------------------------------------------------------------
/src/main/java/edu/nwnu/ququzone/extractor/ExtractorConfiguration.java:
--------------------------------------------------------------------------------
 1 | package edu.nwnu.ququzone.extractor;
 2 | 
 3 | import com.google.common.io.ByteStreams;
 4 | import edu.nwnu.ququzone.extractor.result.Result;
 5 | import edu.nwnu.ququzone.extractor.service.Extractor;
 6 | import org.springframework.beans.factory.annotation.Autowired;
 7 | import org.springframework.beans.factory.annotation.Qualifier;
 8 | import org.springframework.boot.SpringApplication;
 9 | import org.springframework.boot.autoconfigure.SpringBootApplication;
10 | import org.springframework.web.bind.annotation.RequestMapping;
11 | import org.springframework.web.bind.annotation.RequestMethod;
12 | import org.springframework.web.bind.annotation.RequestParam;
13 | import org.springframework.web.bind.annotation.RestController;
14 | 
15 | import java.io.IOException;
16 | 
17 | /**
18 |  * extractor spring configuration.
19 |  *
20 |  * @author Yang XuePing
21 |  */
22 | @SpringBootApplication
23 | @RestController
24 | public class ExtractorConfiguration {
25 |     private static String tpl;
26 | 
27 |     static {
28 |         try {
29 |             tpl = new String(ByteStreams.toByteArray(ExtractorConfiguration.class.getClassLoader().getResourceAsStream("index.tpl")));
30 |         } catch (IOException e) {
31 |         }
32 |     }
33 | 
34 |     @Autowired
35 |     @Qualifier("rowBlockExtractor")
36 |     private Extractor extractor;
37 | 
38 |     public static void main(String[] args) {
39 |         SpringApplication.run(ExtractorConfiguration.class, args);
40 |     }
41 | 
42 |     @RequestMapping(value = "/", method = RequestMethod.GET)
43 |     public String index() {
44 |         return tpl;
45 |     }
46 | 
47 | 
48 |     @RequestMapping(value = "/extract", method = RequestMethod.GET)
49 |     public Result extract(@RequestParam String url) {
50 |         return extractor.extract(url);
51 |     }
52 | }
53 | 


--------------------------------------------------------------------------------
/src/main/java/edu/nwnu/ququzone/extractor/result/FailureResult.java:
--------------------------------------------------------------------------------
 1 | package edu.nwnu.ququzone.extractor.result;
 2 | 
 3 | /**
 4 |  * failure result
 5 |  *
 6 |  * @author Yang XuePing
 7 |  */
 8 | public class FailureResult extends Result {
 9 |     private final String msg;
10 | 
11 |     public FailureResult(String msg) {
12 |         super(false);
13 |         this.msg = msg;
14 |     }
15 | 
16 |     public String getMsg() {
17 |         return msg;
18 |     }
19 | }
20 | 


--------------------------------------------------------------------------------
/src/main/java/edu/nwnu/ququzone/extractor/result/Result.java:
--------------------------------------------------------------------------------
 1 | package edu.nwnu.ququzone.extractor.result;
 2 | 
 3 | /**
 4 |  * extractor result.
 5 |  *
 6 |  * @author Yang XuePing
 7 |  */
 8 | public class Result {
 9 |     private final boolean success;
10 | 
11 |     public Result(boolean success) {
12 |         this.success = success;
13 |     }
14 | 
15 |     public boolean isSuccess() {
16 |         return success;
17 |     }
18 | }
19 | 


--------------------------------------------------------------------------------
/src/main/java/edu/nwnu/ququzone/extractor/result/SuccessResult.java:
--------------------------------------------------------------------------------
 1 | package edu.nwnu.ququzone.extractor.result;
 2 | 
 3 | /**
 4 |  * successful result.
 5 |  *
 6 |  * @author Yang XuePing
 7 |  */
 8 | public class SuccessResult extends Result {
 9 |     private final String title;
10 |     private final String html;
11 | 
12 |     public SuccessResult(String title, String html) {
13 |         super(true);
14 |         this.title = title;
15 |         this.html = html;
16 |     }
17 | 
18 |     public String getTitle() {
19 |         return title;
20 |     }
21 | 
22 |     public String getHtml() {
23 |         return html;
24 |     }
25 | }
26 | 


--------------------------------------------------------------------------------
/src/main/java/edu/nwnu/ququzone/extractor/service/AbstractExtractor.java:
--------------------------------------------------------------------------------
 1 | package edu.nwnu.ququzone.extractor.service;
 2 | 
 3 | import com.squareup.okhttp.OkHttpClient;
 4 | import com.squareup.okhttp.Request;
 5 | import com.squareup.okhttp.Response;
 6 | import edu.nwnu.ququzone.extractor.result.FailureResult;
 7 | import edu.nwnu.ququzone.extractor.result.Result;
 8 | import org.jsoup.Jsoup;
 9 | import org.jsoup.nodes.Document;
10 | import org.mozilla.universalchardet.UniversalDetector;
11 | import org.slf4j.Logger;
12 | import org.slf4j.LoggerFactory;
13 | 
14 | import java.io.IOException;
15 | import java.util.concurrent.TimeUnit;
16 | 
17 | /**
18 |  * abstract extractor.
19 |  *
20 |  * @author Yang XuePing
21 |  */
22 | public abstract class AbstractExtractor implements Extractor {
23 |     private static final Logger LOG = LoggerFactory.getLogger(AbstractExtractor.class);
24 | 
25 |     protected OkHttpClient client;
26 | 
27 |     protected AbstractExtractor() {
28 |         this.client = new OkHttpClient();
29 |         this.client.setConnectTimeout(20, TimeUnit.SECONDS);
30 |         this.client.setReadTimeout(20, TimeUnit.SECONDS);
31 |     }
32 | 
33 |     @Override
34 |     public Result extract(String url) {
35 |         try {
36 |             Document doc = getDocument(url);
37 |             if (doc == null) {
38 |                 return new FailureResult(String.format("fetch %s document error.", url));
39 |             }
40 |             return parse(doc);
41 |         } catch (ParseException e) {
42 |             LOG.error(String.format("parse %s document exception.", url), e);
43 |             return new FailureResult(e.getMessage());
44 |         } catch (Exception e) {
45 |             LOG.error(String.format("fetch %s document exception.", url), e);
46 |             return new FailureResult(String.format("fetch %s document exception.", url));
47 |         }
48 |     }
49 | 
50 |     protected abstract Result parse(Document doc);
51 | 
52 |     protected Document getDocument(String url) {
53 |         Request request = new Request.Builder()
54 |                 .url(url)
55 |                 .build();
56 |         try {
57 |             Response response = client.newCall(request).execute();
58 |             if (response.isSuccessful()) {
59 |                 byte[] data = response.body().bytes();
60 |                 String encoding = detectEncoding(data);
61 |                 return Jsoup.parse(new String(data, encoding), url);
62 |             } else {
63 |                 throw new RuntimeException(String.format("get %s document error", url));
64 |             }
65 |         } catch (IOException e) {
66 |             LOG.error("fetch document error:" + url, e);
67 |         }
68 |         return null;
69 |     }
70 | 
71 |     protected String detectEncoding(byte[] data) {
72 |         UniversalDetector detector = new UniversalDetector(null);
73 |         detector.handleData(data, 0, data.length);
74 |         detector.dataEnd();
75 |         String encoding = detector.getDetectedCharset();
76 |         detector.reset();
77 |         if (encoding == null) {
78 |             encoding = "UTF-8";
79 |         }
80 |         return encoding;
81 |     }
82 | }
83 | 


--------------------------------------------------------------------------------
/src/main/java/edu/nwnu/ququzone/extractor/service/Extractor.java:
--------------------------------------------------------------------------------
 1 | package edu.nwnu.ququzone.extractor.service;
 2 | 
 3 | import edu.nwnu.ququzone.extractor.result.Result;
 4 | 
 5 | /**
 6 |  * extract page main content.
 7 |  *
 8 |  * @author Yang XuePing
 9 |  */
10 | public interface Extractor {
11 |     Result extract(String url);
12 | }
13 | 


--------------------------------------------------------------------------------
/src/main/java/edu/nwnu/ququzone/extractor/service/ParseException.java:
--------------------------------------------------------------------------------
 1 | package edu.nwnu.ququzone.extractor.service;
 2 | 
 3 | /**
 4 |  * parse exception.
 5 |  *
 6 |  * @author Yang XuePing
 7 |  */
 8 | public class ParseException extends RuntimeException {
 9 |     public ParseException(String message) {
10 |         super(message);
11 |     }
12 | }
13 | 


--------------------------------------------------------------------------------
/src/main/java/edu/nwnu/ququzone/extractor/service/ReadabilityExtractor.java:
--------------------------------------------------------------------------------
  1 | package edu.nwnu.ququzone.extractor.service;
  2 | 
  3 | import edu.nwnu.ququzone.extractor.result.FailureResult;
  4 | import edu.nwnu.ququzone.extractor.result.Result;
  5 | import edu.nwnu.ququzone.extractor.result.SuccessResult;
  6 | import org.jsoup.nodes.Document;
  7 | import org.jsoup.nodes.Element;
  8 | import org.jsoup.nodes.TextNode;
  9 | import org.springframework.stereotype.Component;
 10 | 
 11 | import java.util.*;
 12 | import java.util.regex.Pattern;
 13 | 
 14 | /**
 15 |  * readability extractor.
 16 |  *
 17 |  * @author Yang XuePing
 18 |  */
 19 | @Component("readabilityExtractor")
 20 | public class ReadabilityExtractor extends AbstractExtractor {
 21 |     private static final Map<String, Pattern> REGEXPS = new HashMap<>();
 22 | 
 23 |     private static final Set<String> DEFAULT_TAGS_TO_SCORE = new HashSet<>();
 24 | 
 25 |     private static final Set<String> DIV_TO_P_ELEMS = new HashSet<>();
 26 | 
 27 |     static {
 28 |         REGEXPS.put("unlikelyCandidates", Pattern.compile("banner|combx|comment|community|disqus|extra|foot|header|" +
 29 |                 "menu|related|remark|rss|share|shoutbox|sidebar|skyscraper|sponsor|ad-break|agegate|pagination|pager|popup|nav|navbar"));
 30 |         REGEXPS.put("okMaybeItsACandidate", Pattern.compile("and|article|body|column|main|shadow"));
 31 |         REGEXPS.put("positive", Pattern.compile("article|body|content|entry|hentry|main|page|pagination|post|text|blog|story"));
 32 |         REGEXPS.put("negative", Pattern.compile("hidden|banner|combx|comment|com-|contact|foot|footer|footnote|masthead|media|" +
 33 |                 "meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget"));
 34 |         REGEXPS.put("extraneous", Pattern.compile("print|archive|comment|discuss|e[\\-]?mail|share|reply|all|login|sign|single|utility"));
 35 |         REGEXPS.put("byline", Pattern.compile("byline|author|dateline|writtenby"));
 36 |         REGEXPS.put("replaceFonts", Pattern.compile("<(/?)font[^>]*>"));
 37 |         REGEXPS.put("normalize", Pattern.compile("\\s{2,}"));
 38 |         REGEXPS.put("videos", Pattern.compile("//(www\\.)?(dailymotion|youtube|youtube-nocookie|player\\.vimeo)\\.com"));
 39 |         REGEXPS.put("nextLink", Pattern.compile("(next|weiter|continue|>([^\\|]|$)|»([^\\|]|$))"));
 40 |         REGEXPS.put("prevLink", Pattern.compile("(prev|earl|old|new|<|«)"));
 41 |         REGEXPS.put("whitespace", Pattern.compile("^\\s*$"));
 42 |         REGEXPS.put("hasContent", Pattern.compile("\\S$"));
 43 | 
 44 |         DEFAULT_TAGS_TO_SCORE.add("section");
 45 |         DEFAULT_TAGS_TO_SCORE.add("h2");
 46 |         DEFAULT_TAGS_TO_SCORE.add("h3");
 47 |         DEFAULT_TAGS_TO_SCORE.add("h4");
 48 |         DEFAULT_TAGS_TO_SCORE.add("h5");
 49 |         DEFAULT_TAGS_TO_SCORE.add("h6");
 50 |         DEFAULT_TAGS_TO_SCORE.add("p");
 51 |         DEFAULT_TAGS_TO_SCORE.add("td");
 52 |         DEFAULT_TAGS_TO_SCORE.add("pre");
 53 | 
 54 |         DIV_TO_P_ELEMS.add("a");
 55 |         DIV_TO_P_ELEMS.add("blockquote");
 56 |         DIV_TO_P_ELEMS.add("dl");
 57 |         DIV_TO_P_ELEMS.add("div");
 58 |         DIV_TO_P_ELEMS.add("img");
 59 |         DIV_TO_P_ELEMS.add("ol");
 60 |         DIV_TO_P_ELEMS.add("p");
 61 |         DIV_TO_P_ELEMS.add("pre");
 62 |         DIV_TO_P_ELEMS.add("table");
 63 |         DIV_TO_P_ELEMS.add("ul");
 64 |         DIV_TO_P_ELEMS.add("select");
 65 |     }
 66 | 
 67 |     private static void removeElementByTag(Element element, String tag) {
 68 |         element.getElementsByTag(tag).forEach(node -> node.remove());
 69 |     }
 70 | 
 71 |     private static Element setNodeTag(Element element, String tag) {
 72 |         Element replacement = element.ownerDocument().createElement(tag);
 73 |         replacement.html(element.html());
 74 |         element.attributes().forEach(attr -> replacement.attr(attr.getKey(), attr.getValue()));
 75 |         element.replaceWith(replacement);
 76 |         return replacement;
 77 |     }
 78 | 
 79 |     private static boolean isEmpty(String str) {
 80 |         return str == null || "".equals(str.trim());
 81 |     }
 82 | 
 83 |     private static String getArticleTitle(Document doc) {
 84 |         String title = doc.title();
 85 |         if (isEmpty(title)) {
 86 |             title = doc.select("head title").text().trim();
 87 |             if (title.isEmpty()) {
 88 |                 title = doc.select("head meta[name=title]").attr("content").trim();
 89 |                 if (title.isEmpty()) {
 90 |                     title = doc.select("head meta[property=og:title]").attr("content").trim();
 91 |                     if (title.isEmpty()) {
 92 |                         title = doc.getElementsByTag("h1").text();
 93 |                     }
 94 |                 }
 95 |             }
 96 |         }
 97 |         return title;
 98 |     }
 99 | 
100 |     private static String grabArticle(Element body) {
101 |         List<Element> elementsToScore = new LinkedList<>();
102 |         Element node = body;
103 |         while (node != null) {
104 |             String matchString = node.className() + " " + node.id();
105 |             if (REGEXPS.get("unlikelyCandidates").matcher(matchString).find() &&
106 |                     !REGEXPS.get("okMaybeItsACandidate").matcher(matchString).find() &&
107 |                     !"body".equalsIgnoreCase(node.tagName()) &&
108 |                     !"a".equalsIgnoreCase(node.tagName())) {
109 |                 node = removeAndGetNext(node);
110 |                 continue;
111 |             }
112 | 
113 |             if (DEFAULT_TAGS_TO_SCORE.contains(node.tagName().toLowerCase())) {
114 |                 elementsToScore.add(node);
115 |             }
116 | 
117 |             if ("div".equalsIgnoreCase(node.tagName())) {
118 |                 if (node.children().size() == 1 && "p".equalsIgnoreCase(node.child(0).tagName())) {
119 |                     node.replaceWith(node.child(0));
120 |                 } else if (!hasChildBlockElement(node)) {
121 |                     node = setNodeTag(node, "p");
122 |                     elementsToScore.add(node);
123 |                 } else {
124 |                     node.childNodes().forEach(childNode -> {
125 |                         if (childNode instanceof TextNode) {
126 |                             if (!"".equals(childNode.outerHtml().trim())) {
127 |                                 Element p = childNode.ownerDocument().createElement("p");
128 |                                 p.html(childNode.outerHtml());
129 |                                 childNode.replaceWith(p);
130 |                             }
131 |                         }
132 |                     });
133 |                 }
134 |             }
135 |             node = getNextNode(node, false);
136 |         }
137 | 
138 |         final List<Element> candidates = new LinkedList<>();
139 |         elementsToScore.forEach(elementToScore -> {
140 |             if (elementToScore.parent() == null || isEmpty(elementToScore.parent().tagName())) {
141 |                 return;
142 |             }
143 |             String innerText = getInnerText(elementToScore, true);
144 |             if (innerText.length() < 10) {
145 |                 return;
146 |             }
147 |             List<Element> ancestors = getNodeAncestors(elementToScore, 3);
148 |             if (ancestors.size() == 0) {
149 |                 return;
150 |             }
151 |             int contentScore = 0;
152 |             contentScore += 1;
153 |             contentScore += innerText.split(",").length;
154 |             contentScore += Math.min(innerText.length() / 100, 3);
155 | 
156 |             // TODO
157 |         });
158 | 
159 |         return "";
160 |     }
161 | 
162 |     private static List<Element> getNodeAncestors(Element node, int maxDepth) {
163 |         int i = 0;
164 |         List<Element> ancestors = new LinkedList<>();
165 |         while (node.parent() != null) {
166 |             ancestors.add(node.parent());
167 |             if (++i == maxDepth)
168 |                 break;
169 |             node = node.parent();
170 |         }
171 |         return ancestors;
172 |     }
173 | 
174 |     private static String getInnerText(Element node, boolean normalizeSpaces) {
175 |         String textContent = node.text().trim();
176 |         return normalizeSpaces ? textContent.replaceAll(REGEXPS.get("normalize").pattern(), " ") : textContent;
177 |     }
178 | 
179 |     private static boolean hasChildBlockElement(Element node) {
180 |         return node.children().stream().anyMatch(child ->
181 |                 DIV_TO_P_ELEMS.contains(child.tagName().toLowerCase()) || hasChildBlockElement(child));
182 |     }
183 | 
184 |     private static Element removeAndGetNext(Element node) {
185 |         Element nextNode = getNextNode(node, true);
186 |         node.remove();
187 |         return nextNode;
188 |     }
189 | 
190 |     private static Element getNextNode(Element node, boolean ignoreSelfAndKids) {
191 |         if (!ignoreSelfAndKids && node.children().size() > 0) {
192 |             return node.child(0);
193 |         }
194 |         if (node.nextElementSibling() != null) {
195 |             return node.nextElementSibling();
196 |         }
197 |         do {
198 |             node = node.parent();
199 |         } while (node != null && node.nextElementSibling() == null);
200 |         return node == null ? null : node.nextElementSibling();
201 |     }
202 | 
203 |     @Override
204 |     protected Result parse(Document doc) {
205 |         removeElementByTag(doc, "script");
206 |         removeElementByTag(doc, "noscript");
207 | 
208 |         removeElementByTag(doc, "style");
209 |         doc.body().getElementsByTag("font").forEach(font -> setNodeTag(font, "span"));
210 | 
211 |         String articleTitle = getArticleTitle(doc);
212 |         String articleContent = grabArticle(doc.body());
213 | 
214 |         if (articleContent != null) {
215 |             return new SuccessResult(articleTitle, articleContent);
216 |         }
217 | 
218 |         return new FailureResult("Can't extract main content");
219 |     }
220 | }
221 | 


--------------------------------------------------------------------------------
/src/main/java/edu/nwnu/ququzone/extractor/service/RowBlockExtractor.java:
--------------------------------------------------------------------------------
  1 | package edu.nwnu.ququzone.extractor.service;
  2 | 
  3 | import edu.nwnu.ququzone.extractor.result.Result;
  4 | import edu.nwnu.ququzone.extractor.result.SuccessResult;
  5 | import edu.nwnu.ququzone.extractor.utlis.StringUtils;
  6 | import org.jsoup.nodes.Document;
  7 | import org.jsoup.nodes.Element;
  8 | import org.jsoup.nodes.Node;
  9 | import org.jsoup.nodes.TextNode;
 10 | import org.springframework.stereotype.Component;
 11 | 
 12 | import java.util.HashMap;
 13 | import java.util.LinkedList;
 14 | import java.util.List;
 15 | import java.util.Map;
 16 | 
 17 | /**
 18 |  * row block extractor implement.
 19 |  *
 20 |  * @author Yang XuePing
 21 |  */
 22 | @Component("rowBlockExtractor")
 23 | public class RowBlockExtractor extends AbstractExtractor {
 24 |     @Override
 25 |     protected Result parse(Document doc) {
 26 |         return new SuccessResult(getArticleTitle(doc), getContentElement(doc).html());
 27 |     }
 28 | 
 29 |     public Element getContentElement(Document doc) {
 30 |         Map<Element, CountInfo> infoMap = new HashMap<>();
 31 |         clean(doc);
 32 |         computeInfo(doc.body(), infoMap);
 33 |         double maxScore = 0;
 34 |         Element content = null;
 35 |         for (Map.Entry<Element, CountInfo> entry : infoMap.entrySet()) {
 36 |             Element tag = entry.getKey();
 37 |             if (tag.tagName().equals("a") || tag == doc.body()) {
 38 |                 continue;
 39 |             }
 40 |             double score = computeScore(tag, infoMap);
 41 |             if (score > maxScore) {
 42 |                 maxScore = score;
 43 |                 content = tag;
 44 |             }
 45 |         }
 46 |         if (content == null) {
 47 |             throw new RuntimeException("extraction failed");
 48 |         }
 49 |         return content;
 50 |     }
 51 | 
 52 |     private String getArticleTitle(Document doc) {
 53 |         String title = doc.title();
 54 |         if (!StringUtils.isEmpty(title)) {
 55 |             return title;
 56 |         }
 57 |         title = doc.select("head title").text().trim();
 58 |         if (!title.isEmpty()) {
 59 |             return title;
 60 |         }
 61 |         title = doc.select("head meta[name=title]").attr("content").trim();
 62 |         if (!title.isEmpty()) {
 63 |             return title;
 64 |         }
 65 |         title = doc.select("head meta[property=og:title]").attr("content").trim();
 66 |         if (!title.isEmpty()) {
 67 |             return title;
 68 |         }
 69 |         return doc.getElementsByTag("h1").text().trim();
 70 |     }
 71 | 
 72 |     protected void clean(Document doc) {
 73 |         doc.select("script,noscript,style,iframe,br").remove();
 74 |     }
 75 | 
 76 |     protected CountInfo computeInfo(Node node, Map<Element, CountInfo> infoMap) {
 77 |         if (node instanceof Element) {
 78 |             Element tag = (Element) node;
 79 | 
 80 |             CountInfo countInfo = new CountInfo();
 81 |             for (Node childNode : tag.childNodes()) {
 82 |                 CountInfo childCountInfo = computeInfo(childNode, infoMap);
 83 |                 countInfo.textCount += childCountInfo.textCount;
 84 |                 countInfo.linkTextCount += childCountInfo.linkTextCount;
 85 |                 countInfo.tagCount += childCountInfo.tagCount;
 86 |                 countInfo.linkTagCount += childCountInfo.linkTagCount;
 87 |                 countInfo.leafList.addAll(childCountInfo.leafList);
 88 |                 countInfo.densitySum += childCountInfo.density;
 89 |                 countInfo.pCount += childCountInfo.pCount;
 90 |             }
 91 |             countInfo.tagCount++;
 92 |             String tagName = tag.tagName();
 93 |             if ("a".equals(tagName)) {
 94 |                 countInfo.linkTextCount = countInfo.textCount;
 95 |                 countInfo.linkTagCount++;
 96 |             }
 97 |             if ("p".equals(tagName)) {
 98 |                 countInfo.pCount++;
 99 |             }
100 | 
101 |             int pureLen = countInfo.textCount - countInfo.linkTextCount;
102 |             int len = countInfo.tagCount - countInfo.linkTagCount;
103 |             if (pureLen == 0 || len == 0) {
104 |                 countInfo.density = 0;
105 |             } else {
106 |                 countInfo.density = (pureLen + 0.0) / len;
107 |             }
108 |             infoMap.put(tag, countInfo);
109 |             return countInfo;
110 |         }
111 |         if (node instanceof TextNode) {
112 |             TextNode tn = (TextNode) node;
113 |             CountInfo countInfo = new CountInfo();
114 |             int len = tn.text().length();
115 |             countInfo.textCount = len;
116 |             countInfo.leafList.add(len);
117 |             return countInfo;
118 |         }
119 |         return new CountInfo();
120 |     }
121 | 
122 |     protected double computeScore(Element tag, Map<Element, CountInfo> infoMap) {
123 |         CountInfo countInfo = infoMap.get(tag);
124 |         double var = Math.sqrt(computeVar(countInfo.leafList) + 1);
125 |         double score = Math.log(var) * countInfo.densitySum * Math.log(countInfo.textCount - countInfo.linkTextCount + 1) * Math.log10(countInfo.pCount + 2);
126 |         return score;
127 |     }
128 | 
129 |     private double computeVar(List<Integer> data) {
130 |         if (data.size() == 0) return 0;
131 | 
132 |         if (data.size() == 1) {
133 |             return data.get(0) / 2;
134 |         }
135 | 
136 |         double sum = 0;
137 |         for (Integer i : data) {
138 |             sum += i;
139 |         }
140 |         double ave = sum / data.size();
141 |         sum = 0;
142 |         for (Integer i : data) {
143 |             sum += (i - ave) * (i - ave);
144 |         }
145 |         sum = sum / data.size();
146 |         return sum;
147 |     }
148 | 
149 |     private static class CountInfo {
150 |         int textCount = 0;
151 |         int linkTextCount = 0;
152 |         int tagCount = 0;
153 |         int linkTagCount = 0;
154 |         double density = 0;
155 |         double densitySum = 0;
156 |         double score = 0;
157 |         int pCount = 0;
158 |         List<Integer> leafList = new LinkedList<>();
159 |     }
160 | }
161 | 


--------------------------------------------------------------------------------
/src/main/java/edu/nwnu/ququzone/extractor/utlis/StringUtils.java:
--------------------------------------------------------------------------------
 1 | package edu.nwnu.ququzone.extractor.utlis;
 2 | 
 3 | /**
 4 |  * String utls.
 5 |  *
 6 |  * @author Yang XuePing
 7 |  */
 8 | public final class StringUtils {
 9 |     public static boolean isEmpty(String str) {
10 |         return str == null || "".equals(str.trim());
11 |     }
12 | }
13 | 


--------------------------------------------------------------------------------
/src/main/resources/application.properties:
--------------------------------------------------------------------------------
1 | spring.application.name=smart-extractor
2 | spring.profiles.active=production
3 | 
4 | logging.file=smart-extractor.log
5 | 
6 | spring.mvc.view.prefix=/WEB-INF/jsp/
7 | spring.mvc.view.suffix=.jsp


--------------------------------------------------------------------------------
/src/main/resources/index.tpl:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html lang="en">
 3 | <head>
 4 |     <title>Smart Extractor</title>
 5 |     <script src="https://cdnjs.cloudflare.com/ajax/libs/fetch/0.10.1/fetch.min.js"></script>
 6 |     <script src="https://cdnjs.cloudflare.com/ajax/libs/react/0.14.2/react.min.js"></script>
 7 |     <script src="https://cdnjs.cloudflare.com/ajax/libs/react/0.14.2/react-dom.min.js"></script>
 8 |     <script src="https://cdnjs.cloudflare.com/ajax/libs/babel-core/5.8.23/browser.min.js"></script>
 9 | </head>
10 | <body>
11 | <div>
12 |     <h2>Extract main content form web</h2>
13 |     <div id="extract">
14 |     </div>
15 | </div>
16 | <script type="text/babel">
17 |     var ExtractForm = React.createClass({
18 |         handleSubmit: function(e) {
19 |             e.preventDefault();
20 |             var url = this.refs.url.value.trim();
21 |             if (!url) {
22 |                 return;
23 |             }
24 |             var that = this;
25 |             fetch('/extract?url=' + encodeURIComponent(url))
26 |                 .then(function(res) {
27 |                     return res.json();
28 |                 }).then(function(json) {
29 |                     that.refs.url.value = '';
30 |                     if (json.success) {
31 |                         that.props.onExtracted(json.title, json.html);
32 |                     } else {
33 |                         that.props.onExtracted('error', json.msg);
34 |                     }
35 |                 });
36 |         },
37 | 
38 |         render: function() {
39 |             return (
40 |                 <div>
41 |                     <form onSubmit={this.handleSubmit}>
42 |                         <div>
43 |                             <label>URL: </label>
44 |                             <input name="url" type="text" placeholder="url" ref="url" />
45 |                         </div>
46 |                         <div>
47 |                             <button id="extract-btn">extract</button>
48 |                         </div>
49 |                     </form>
50 |                 </div>
51 |             );
52 |         }
53 |     });
54 |     var ExtractContent = React.createClass({
55 |         render: function() {
56 |             return (
57 |                 <div>
58 |                     <hr />
59 |                     <div>
60 |                         <h2>{this.props.title}</h2>
61 |                     </div>
62 |                     <div dangerouslySetInnerHTML={{__html: this.props.content}} />
63 |                 </div>
64 |             );
65 |         }
66 |     });
67 |     var ExtractApp = React.createClass({
68 |         getInitialState: function() {
69 |             return {title: 'NONE', content: ''};
70 |         },
71 | 
72 |         onExtracted: function(title, content) {
73 |             this.setState({title: title || "", content: content || ""});
74 |         },
75 | 
76 |         render: function() {
77 |             return (
78 |                 <div>
79 |                     <ExtractForm onExtracted={this.onExtracted} />
80 |                     <ExtractContent title={this.state.title} content={this.state.content} />
81 |                 </div>
82 |             );
83 |         }
84 |     });
85 |     ReactDOM.render(
86 |         <ExtractApp />,
87 |         document.getElementById('extract')
88 |     );
89 | </script>
90 | </body>
91 | </html>


--------------------------------------------------------------------------------
/src/test/java/edu/nwnu/ququzone/extractor/service/ReadabilityExtractorTest.java:
--------------------------------------------------------------------------------
 1 | package edu.nwnu.ququzone.extractor.service;
 2 | 
 3 | import org.jsoup.Jsoup;
 4 | import org.jsoup.nodes.Document;
 5 | import org.testng.annotations.Test;
 6 | 
 7 | import java.net.URL;
 8 | 
 9 | /**
10 |  * test readability extractor.
11 |  *
12 |  * @author Yang XuePing
13 |  */
14 | public class ReadabilityExtractorTest {
15 |     @Test
16 |     public void testCrawl() throws Exception {
17 |         Document doc = Jsoup.parse(ReadabilityExtractorTest.class.getClassLoader().getResourceAsStream("test.html"), "UTF-8", "/");
18 |         // Document doc = Jsoup.parse(new URL("http://192.168.1.102:3000"), 2000);
19 | 
20 |         ReadabilityExtractor extractor = new ReadabilityExtractor();
21 |         extractor.parse(doc);
22 |     }
23 | }
24 | 


--------------------------------------------------------------------------------
/src/test/resources/test.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html lang="en">
 3 | <head>
 4 |     <meta charset="utf-8">
 5 |     <meta http-equiv="X-UA-Compatible" content="IE=edge">
 6 |     <meta name="viewport" content="width=device-width, initial-scale=1">
 7 |     <title>Bootstrap 101 Template</title>
 8 | 
 9 |     <!-- Bootstrap -->
10 |     <link href="css/bootstrap.min.css" rel="stylesheet">
11 | 
12 |     <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
13 |     <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
14 |     <!--[if lt IE 9]>
15 |     <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
16 |     <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
17 |     <![endif]-->
18 |     <style>
19 |         body {
20 |             font-family: "Courier New", Courier, monospace;
21 |         }
22 |     </style>
23 | </head>
24 | <body>
25 | <nav class="navbar navbar-inverse navbar-fixed-top">
26 |     <div class="container">
27 |         <div class="navbar-header">
28 |             <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
29 |                 <span class="sr-only">Toggle navigation</span>
30 |                 <span class="icon-bar"></span>
31 |                 <span class="icon-bar"></span>
32 |                 <span class="icon-bar"></span>
33 |             </button>
34 |             <a class="navbar-brand" href="#">Project name</a>
35 |         </div>
36 |         <div id="navbar" class="collapse navbar-collapse">
37 |             <ul class="nav navbar-nav">
38 |                 <li class="active"><a href="#">Home</a></li>
39 |                 <li><a href="#about">About</a></li>
40 |                 <li><a href="#contact">Contact</a></li>
41 |             </ul>
42 |         </div><!--/.nav-collapse -->
43 |     </div>
44 | </nav>
45 | 
46 | <div class="container">
47 |     <div class="starter-template">
48 |         <h1>Bootstrap starter template</h1>
49 |         <p class="lead">Use this document as a way to quickly start any new project.<br> All you get is this text and a mostly barebones HTML document.</p>
50 |     </div>
51 | </div>
52 | 
53 | <footer class="footer">
54 |     <p>&copy; Company 2014</p>
55 | </footer>
56 | 
57 | <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
58 | <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js"></script>
59 | <!-- Include all compiled plugins (below), or include individual files as needed -->
60 | <script src="js/bootstrap.min.js"></script>
61 | <script type="application/javascript">
62 |     function test() {
63 |         alert("edu.nwnu.ququzone.extractor.service.test");
64 |     }
65 | </script>
66 | </body>
67 | </html>


--------------------------------------------------------------------------------