20 | '
89 | " "
90 | " test section
102 |
103 |
107 |
108 |
113 |
114 |
115 |
118 |
119 |
120 | """
121 | doc = Document(sample)
122 | s = doc.summary()
123 | # print(s)
124 | assert "punctuation" in s
125 | assert not "comment" in s
126 | assert not "aside" in s
127 |
128 | # Many spaces make some regexes run forever
129 | @timeout(3)
130 | def test_many_repeated_spaces(self):
131 | long_space = " " * 1000000
132 | sample = "
189 |
190 | 这是标题
191 |
192 |
193 | Short Part | これは長いです
223 |
224 |
225 | ]*>[ \n\r\t]*){2,}',re.I),
46 | # 'replaceFontsRe': re.compile(r'<(\/?)font[^>]*>',re.I),
47 | # 'trimRe': re.compile(r'^\s+|\s+$/'),
48 | # 'normalizeRe': re.compile(r'\s{2,}/'),
49 | # 'killBreaksRe': re.compile(r'((\s| ?)*){1,}/'),
50 | "videoRe": re.compile(r"https?:\/\/(www\.)?(youtube|vimeo)\.com", re.I),
51 | # skipFootnoteLink: /^\s*(\[?[a-z0-9]{1,2}\]?|^|edit|citation needed)\s*$/i,
52 | }
53 |
54 |
55 | class Unparseable(ValueError):
56 | pass
57 |
58 |
59 | def to_int(x):
60 | if not x:
61 | return None
62 | x = x.strip()
63 | if x.endswith("px"):
64 | return int(x[:-2])
65 | if x.endswith("em"):
66 | return int(x[:-2]) * 12
67 | return int(x)
68 |
69 |
70 | def clean(text):
71 | # Many spaces make the following regexes run forever
72 | text = re.sub(r"\s{255,}", " " * 255, text)
73 | text = re.sub(r"\s*\n\s*", "\n", text)
74 | text = re.sub(r"\t|[ \t]{2,}", " ", text)
75 | return text.strip()
76 |
77 |
78 | def text_length(i):
79 | return len(clean(i.text_content() or ""))
80 |
81 |
82 | def compile_pattern(elements):
83 | if not elements:
84 | return None
85 | elif isinstance(elements, re.Pattern):
86 | return elements
87 | elif isinstance(elements, (str, bytes)):
88 | if isinstance(elements, bytes):
89 | elements = str(elements, "utf-8")
90 | elements = elements.split(",")
91 | if isinstance(elements, (list, tuple)):
92 | return re.compile("|".join([re.escape(x.strip()) for x in elements]), re.U)
93 | else:
94 | raise Exception(f"Unknown type for the pattern: {type(elements)}")
95 | # assume string or string like object
96 |
97 |
98 | class Document:
99 | """Class to build a etree document out of html."""
100 |
101 | def __init__(
102 | self,
103 | input,
104 | positive_keywords=None,
105 | negative_keywords=None,
106 | url=None,
107 | min_text_length=25,
108 | retry_length=250,
109 | xpath=False,
110 | handle_failures="discard",
111 | ):
112 | """Generate the document
113 |
114 | :param input: string of the html content.
115 | :param positive_keywords: regex, list or comma-separated string of patterns in classes and ids
116 | :param negative_keywords: regex, list or comma-separated string in classes and ids
117 | :param min_text_length: Tunable. Set to a higher value for more precise detection of longer texts.
118 | :param retry_length: Tunable. Set to a lower value for better detection of very small texts.
119 | :param xpath: If set to True, adds x="..." attribute to each HTML node,
120 | containing xpath path pointing to original document path (allows to
121 | reconstruct selected summary in original document).
122 | :param handle_failures: Parameter passed to `lxml` for handling failure during exception.
123 | Support options = ["discard", "ignore", None]
124 |
125 | Examples:
126 | positive_keywords=["news-item", "block"]
127 | positive_keywords=["news-item, block"]
128 | positive_keywords=re.compile("news|block")
129 | negative_keywords=["mysidebar", "related", "ads"]
130 |
131 | The Document class is not re-enterable.
132 | It is designed to create a new Document() for each HTML file to process it.
133 |
134 | API methods:
135 | .title() -- full title
136 | .short_title() -- cleaned up title
137 | .content() -- full content
138 | .summary() -- cleaned up content
139 | """
140 | self.input = input
141 | self.html = None
142 | self.encoding = None
143 | self.positive_keywords = compile_pattern(positive_keywords)
144 | self.negative_keywords = compile_pattern(negative_keywords)
145 | self.url = url
146 | self.min_text_length = min_text_length
147 | self.retry_length = retry_length
148 | self.xpath = xpath
149 | self.handle_failures = handle_failures
150 |
151 | def _html(self, force=False):
152 | if force or self.html is None:
153 | self.html = self._parse(self.input)
154 | if self.xpath:
155 | root = self.html.getroottree()
156 | for i in self.html.getiterator():
157 | # print root.getpath(i)
158 | i.attrib["x"] = root.getpath(i)
159 | return self.html
160 |
161 | def _parse(self, input):
162 | if isinstance(input, (_ElementTree, HtmlElement)):
163 | doc = input
164 | self.encoding = 'utf-8'
165 | else:
166 | doc, self.encoding = build_doc(input)
167 | doc = html_cleaner.clean_html(doc)
168 | base_href = self.url
169 | if base_href:
170 | # trying to guard against bad links like
171 | try:
172 | # such support is added in lxml 3.3.0
173 | doc.make_links_absolute(
174 | base_href,
175 | resolve_base_href=True,
176 | handle_failures=self.handle_failures,
177 | )
178 | except TypeError: # make_links_absolute() got an unexpected keyword argument 'handle_failures'
179 | # then we have lxml < 3.3.0
180 | # please upgrade to lxml >= 3.3.0 if you're failing here!
181 | doc.make_links_absolute(
182 | base_href,
183 | resolve_base_href=True,
184 | handle_failures=self.handle_failures,
185 | )
186 | else:
187 | doc.resolve_base_href(handle_failures=self.handle_failures)
188 | return doc
189 |
190 | def content(self):
191 | """Returns document body"""
192 | return get_body(self._html(True))
193 |
194 | def title(self):
195 | """Returns document title"""
196 | return get_title(self._html(True))
197 |
198 | def author(self):
199 | """Returns document author"""
200 | return get_author(self._html(True))
201 |
202 | def short_title(self):
203 | """Returns cleaned up document title"""
204 | return shorten_title(self._html(True))
205 |
206 | def get_clean_html(self):
207 | """
208 | An internal method, which can be overridden in subclasses, for example,
209 | to disable or to improve DOM-to-text conversion in .summary() method
210 | """
211 | return clean_attributes(tounicode(self.html, method="html"))
212 |
213 | def summary(self, html_partial=False, keep_all_images=False):
214 | """
215 | Given a HTML file, extracts the text of the article.
216 |
217 | :param html_partial: return only the div of the document, don't wrap
218 | in html and body tags.
219 | :param keep_all_images: Keep all images in summary.
220 |
221 | Warning: It mutates internal DOM representation of the HTML document,
222 | so it is better to call other API methods before this one.
223 | """
224 | try:
225 | ruthless = True
226 | while True:
227 | self._html(True)
228 | for i in self.tags(self.html, "script", "style"):
229 | i.drop_tree()
230 | for i in self.tags(self.html, "body"):
231 | i.set("id", "readabilityBody")
232 | if ruthless:
233 | self.remove_unlikely_candidates()
234 | self.transform_misused_divs_into_paragraphs()
235 | candidates = self.score_paragraphs()
236 |
237 | best_candidate = self.select_best_candidate(candidates)
238 |
239 | if best_candidate:
240 | article = self.get_article(
241 | candidates, best_candidate, html_partial=html_partial
242 | )
243 | else:
244 | if ruthless:
245 | log.info("ruthless removal did not work. ")
246 | ruthless = False
247 | log.debug(
248 | "ended up stripping too much - "
249 | "going for a safer _parse"
250 | )
251 | # try again
252 | continue
253 | else:
254 | log.debug(
255 | "Ruthless and lenient parsing did not work. "
256 | "Returning raw html"
257 | )
258 | article = self.html.find("body")
259 | if article is None:
260 | article = self.html
261 | cleaned_article = self.sanitize(article, candidates, keep_all_images)
262 |
263 | article_length = len(cleaned_article or "")
264 | retry_length = self.retry_length
265 | of_acceptable_length = article_length >= retry_length
266 | if ruthless and not of_acceptable_length:
267 | ruthless = False
268 | # Loop through and try again.
269 | continue
270 | else:
271 | return cleaned_article
272 | except Exception as e:
273 | log.exception("error getting summary: ")
274 | raise Unparseable(str(e)).with_traceback(sys.exc_info()[2])
275 |
276 | def get_article(self, candidates, best_candidate, html_partial=False):
277 | # Now that we have the top candidate, look through its siblings for
278 | # content that might also be related.
279 | # Things like preambles, content split by ads that we removed, etc.
280 | sibling_score_threshold = max([10, best_candidate["content_score"] * 0.2])
281 | # create a new html document with a html->body->div
282 | if html_partial:
283 | output = fragment_fromstring("")
284 | else:
285 | output = document_fromstring("")
286 | best_elem = best_candidate["elem"]
287 | parent = best_elem.getparent()
288 | siblings = parent.getchildren() if parent is not None else [best_elem]
289 | for sibling in siblings:
290 | # in lxml there no concept of simple text
291 | # if isinstance(sibling, NavigableString): continue
292 | append = False
293 | if sibling is best_elem:
294 | append = True
295 | sibling_key = sibling # HashableElement(sibling)
296 | if (
297 | sibling_key in candidates
298 | and candidates[sibling_key]["content_score"] >= sibling_score_threshold
299 | ):
300 | append = True
301 |
302 | if sibling.tag == "p":
303 | link_density = self.get_link_density(sibling)
304 | node_content = sibling.text or ""
305 | node_length = len(node_content)
306 |
307 | if node_length > 80 and link_density < 0.25:
308 | append = True
309 | elif (
310 | node_length <= 80
311 | and link_density == 0
312 | and re.search(r"\.( |$)", node_content)
313 | ):
314 | append = True
315 |
316 | if append:
317 | # We don't want to append directly to output, but the div
318 | # in html->body->div
319 | if html_partial:
320 | output.append(sibling)
321 | else:
322 | output.getchildren()[0].getchildren()[0].append(sibling)
323 | # if output is not None:
324 | # output.append(best_elem)
325 | return output
326 |
327 | def select_best_candidate(self, candidates):
328 | if not candidates:
329 | return None
330 |
331 | sorted_candidates = sorted(
332 | candidates.values(), key=lambda x: x["content_score"], reverse=True
333 | )
334 | for candidate in sorted_candidates[:5]:
335 | elem = candidate["elem"]
336 | log.debug("Top 5 : {:6.3f} {}".format(candidate["content_score"], describe(elem)))
337 |
338 | best_candidate = sorted_candidates[0]
339 | return best_candidate
340 |
341 | def get_link_density(self, elem):
342 | link_length = 0
343 | for i in elem.findall(".//a"):
344 | link_length += text_length(i)
345 | # if len(elem.findall(".//div") or elem.findall(".//p")):
346 | # link_length = link_length
347 | total_length = text_length(elem)
348 | return float(link_length) / max(total_length, 1)
349 |
350 | def score_paragraphs(self):
351 | MIN_LEN = self.min_text_length
352 | candidates = {}
353 | ordered = []
354 | for elem in self.tags(self._html(), "p", "pre", "td"):
355 | parent_node = elem.getparent()
356 | if parent_node is None:
357 | continue
358 | grand_parent_node = parent_node.getparent()
359 |
360 | inner_text = clean(elem.text_content() or "")
361 | inner_text_len = len(inner_text)
362 |
363 | # If this paragraph is less than 25 characters
364 | # don't even count it.
365 | if inner_text_len < MIN_LEN:
366 | continue
367 |
368 | if parent_node not in candidates:
369 | candidates[parent_node] = self.score_node(parent_node)
370 | ordered.append(parent_node)
371 |
372 | if grand_parent_node is not None and grand_parent_node not in candidates:
373 | candidates[grand_parent_node] = self.score_node(grand_parent_node)
374 | ordered.append(grand_parent_node)
375 |
376 | content_score = 1
377 | content_score += len(inner_text.split(","))
378 | content_score += min((inner_text_len / 100), 3)
379 | # if elem not in candidates:
380 | # candidates[elem] = self.score_node(elem)
381 |
382 | # WTF? candidates[elem]['content_score'] += content_score
383 | candidates[parent_node]["content_score"] += content_score
384 | if grand_parent_node is not None:
385 | candidates[grand_parent_node]["content_score"] += content_score / 2.0
386 |
387 | # Scale the final candidates score based on link density. Good content
388 | # should have a relatively small link density (5% or less) and be
389 | # mostly unaffected by this operation.
390 | for elem in ordered:
391 | candidate = candidates[elem]
392 | ld = self.get_link_density(elem)
393 | score = candidate["content_score"]
394 | log.debug(
395 | "Branch %6.3f %s link density %.3f -> %6.3f"
396 | % (score, describe(elem), ld, score * (1 - ld))
397 | )
398 | candidate["content_score"] *= 1 - ld
399 |
400 | return candidates
401 |
402 | def class_weight(self, e):
403 | weight = 0
404 | for feature in [e.get("class", None), e.get("id", None)]:
405 | if feature:
406 | if REGEXES["negativeRe"].search(feature):
407 | weight -= 25
408 |
409 | if REGEXES["positiveRe"].search(feature):
410 | weight += 25
411 |
412 | if self.positive_keywords and self.positive_keywords.search(feature):
413 | weight += 25
414 |
415 | if self.negative_keywords and self.negative_keywords.search(feature):
416 | weight -= 25
417 |
418 | if self.positive_keywords and self.positive_keywords.match("tag-" + e.tag):
419 | weight += 25
420 |
421 | if self.negative_keywords and self.negative_keywords.match("tag-" + e.tag):
422 | weight -= 25
423 |
424 | return weight
425 |
426 | def score_node(self, elem):
427 | content_score = self.class_weight(elem)
428 | name = elem.tag.lower()
429 | if name in ["div", "article"]:
430 | content_score += 5
431 | elif name in ["pre", "td", "blockquote"]:
432 | content_score += 3
433 | elif name in ["address", "ol", "ul", "dl", "dd", "dt", "li", "form", "aside"]:
434 | content_score -= 3
435 | elif name in [
436 | "h1",
437 | "h2",
438 | "h3",
439 | "h4",
440 | "h5",
441 | "h6",
442 | "th",
443 | "header",
444 | "footer",
445 | "nav",
446 | ]:
447 | content_score -= 5
448 | return {"content_score": content_score, "elem": elem}
449 |
450 | def remove_unlikely_candidates(self):
451 | for elem in self.html.findall(".//*"):
452 | s = "{} {}".format(elem.get("class", ""), elem.get("id", ""))
453 | if len(s) < 2:
454 | continue
455 | if (
456 | REGEXES["unlikelyCandidatesRe"].search(s)
457 | and (not REGEXES["okMaybeItsACandidateRe"].search(s))
458 | and elem.tag not in ["html", "body"]
459 | ):
460 | log.debug("Removing unlikely candidate - %s" % describe(elem))
461 | elem.drop_tree()
462 |
463 | def transform_misused_divs_into_paragraphs(self):
464 | for elem in self.tags(self.html, "div"):
465 | # transform
1234567890123456789012345
" 91 | " " 92 | "" 93 | ) 94 | doc = Document(sample) 95 | doc.summary() 96 | 97 | def test_correct_cleanup(self): 98 | sample = """ 99 | 100 | 101 |Lot of text here.
104 | 105 |More text is written here, and contains punctuation and dots.
106 |The comment is also helpful, but it's 116 | still not the correct item to be extracted.
117 |It's even longer than the article itself!"
foo" + long_space + "
" 133 | 134 | doc = Document(sample) 135 | s = doc.summary() 136 | 137 | assert "foo" in s 138 | 139 | def test_not_self_closing(self): 140 | sample = 'foobar
' 141 | doc = Document(sample) 142 | assert ( 143 | 'foobar
' 144 | == doc.summary() 145 | ) 146 | 147 | def test_utf8_kanji(self): 148 | """Using the UTF-8 kanji sample, load article which is written in kanji""" 149 | sample = load_sample("utf-8-kanji.sample.html") 150 | doc = Document(sample) 151 | res = doc.summary() 152 | assert 0 < len(res) < 10000 153 | 154 | def test_author_present(self): 155 | sample = load_sample("the-hurricane-rubin-carter-denzel-washington.html") 156 | doc = Document(sample) 157 | assert 'Alex von Tunzelmann' == doc.author() 158 | 159 | def test_author_absent(self): 160 | sample = load_sample("si-game.sample.html") 161 | doc = Document(sample) 162 | assert '[no-author]' == doc.author() 163 | 164 | def test_keep_images_present(self): 165 | sample = load_sample("summary-keep-all-images.sample.html") 166 | 167 | doc = Document(sample) 168 | 169 | assert "一些无关紧要的内容
194 |
195 |
201 | 主要文章标题
196 |这是主要内容的第一段。
197 |これはコンテンツの第2段落です。
198 |이것은 콘텐츠의 세 번째 단락입니다.
199 |This is the fourth paragraph.
200 |More irrelevant stuff
202 |
203 |
204 | """
205 | doc = Document(html)
206 | summary = doc.summary()
207 | # Check that the main CJK content is present in the summary
208 | self.assertTrue("这是主要内容的第一段" in summary)
209 | self.assertTrue("これはコンテンツの第2段落です" in summary)
210 | self.assertTrue("이것은 콘텐츠의 세 번째 단락입니다" in summary)
211 | # Check that irrelevant content is mostly gone
212 | self.assertFalse("一些无关紧要的内容" in summary)
213 |
214 | def test_shorten_title_delimiter_bug(self):
215 | """Test that shorten_title handles delimiters correctly when the last part is valid.
216 |
217 | This specifically targets a potential bug where 'p1' might be used instead of 'pl'.
218 | """
219 | html = """
220 |
221 |
222 | Content
226 |
227 |
228 | """
229 | doc = Document(html)
230 | # With the bug, this call might raise NameError: name 'p1' is not defined
231 | # With the fix, it should correctly return the last part.
232 | short_title = doc.short_title()
233 | self.assertEqual(short_title, "これは長いです")
234 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright [yyyy] [name of copyright owner]
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/readability/readability.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | import logging
3 | import re
4 | import sys
5 | import urllib.request
6 | import urllib.parse
7 | import urllib.error
8 |
9 | from lxml.etree import tostring
10 | from lxml.etree import tounicode
11 | from lxml.etree import _ElementTree
12 | from lxml.html import document_fromstring
13 | from lxml.html import fragment_fromstring
14 | from lxml.html import HtmlElement
15 |
16 | from .cleaners import clean_attributes
17 | from .cleaners import html_cleaner
18 | from .htmls import build_doc
19 | from .htmls import get_body
20 | from .htmls import get_title
21 | from .htmls import get_author
22 | from .htmls import shorten_title
23 | from .debug import describe, text_content
24 |
25 |
26 | log = logging.getLogger("readability.readability")
27 |
28 | REGEXES = {
29 | "unlikelyCandidatesRe": re.compile(
30 | r"combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter",
31 | re.I,
32 | ),
33 | "okMaybeItsACandidateRe": re.compile(r"and|article|body|column|main|shadow", re.I),
34 | "positiveRe": re.compile(
35 | r"article|body|content|entry|hentry|main|page|pagination|post|text|blog|story",
36 | re.I,
37 | ),
38 | "negativeRe": re.compile(
39 | r"combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|sponsor|shopping|tags|tool|widget",
40 | re.I,
41 | ),
42 | "divToPElementsRe": re.compile(
43 | r"<(a|blockquote|dl|div|img|ol|p|pre|table|ul)", re.I
44 | ),
45 | # 'replaceBrsRe': re.compile(r'(s that do not contain other block elements into
466 | # s,