'
239 | self.assertEqual(self.soup(markup).div.encode("utf8"), markup.encode("utf8"))
240 |
241 | class TestUnicodeDammit(unittest.TestCase):
242 | """Standalone tests of UnicodeDammit."""
243 |
244 | def test_unicode_input(self):
245 | markup = "I'm already Unicode! \N{SNOWMAN}"
246 | dammit = UnicodeDammit(markup)
247 | self.assertEqual(dammit.unicode_markup, markup)
248 |
249 | def test_smart_quotes_to_unicode(self):
250 | markup = b"\x91\x92\x93\x94"
251 | dammit = UnicodeDammit(markup)
252 | self.assertEqual(
253 | dammit.unicode_markup, "\u2018\u2019\u201c\u201d")
254 |
255 | def test_smart_quotes_to_xml_entities(self):
256 | markup = b"\x91\x92\x93\x94"
257 | dammit = UnicodeDammit(markup, smart_quotes_to="xml")
258 | self.assertEqual(
259 | dammit.unicode_markup, "‘’“”")
260 |
261 | def test_smart_quotes_to_html_entities(self):
262 | markup = b"\x91\x92\x93\x94"
263 | dammit = UnicodeDammit(markup, smart_quotes_to="html")
264 | self.assertEqual(
265 | dammit.unicode_markup, "‘’“”")
266 |
267 | def test_smart_quotes_to_ascii(self):
268 | markup = b"\x91\x92\x93\x94"
269 | dammit = UnicodeDammit(markup, smart_quotes_to="ascii")
270 | self.assertEqual(
271 | dammit.unicode_markup, """''""""")
272 |
273 | def test_detect_utf8(self):
274 | utf8 = b"\xc3\xa9"
275 | dammit = UnicodeDammit(utf8)
276 | self.assertEqual(dammit.unicode_markup, '\xe9')
277 | self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
278 |
279 | def test_convert_hebrew(self):
280 | hebrew = b"\xed\xe5\xec\xf9"
281 | dammit = UnicodeDammit(hebrew, ["iso-8859-8"])
282 | self.assertEqual(dammit.original_encoding.lower(), 'iso-8859-8')
283 | self.assertEqual(dammit.unicode_markup, '\u05dd\u05d5\u05dc\u05e9')
284 |
285 | def test_dont_see_smart_quotes_where_there_are_none(self):
286 | utf_8 = b"\343\202\261\343\203\274\343\202\277\343\202\244 Watch"
287 | dammit = UnicodeDammit(utf_8)
288 | self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
289 | self.assertEqual(dammit.unicode_markup.encode("utf-8"), utf_8)
290 |
291 | def test_ignore_inappropriate_codecs(self):
292 | utf8_data = "Räksmörgås".encode("utf-8")
293 | dammit = UnicodeDammit(utf8_data, ["iso-8859-8"])
294 | self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
295 |
296 | def test_ignore_invalid_codecs(self):
297 | utf8_data = "Räksmörgås".encode("utf-8")
298 | for bad_encoding in ['.utf8', '...', 'utF---16.!']:
299 | dammit = UnicodeDammit(utf8_data, [bad_encoding])
300 | self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
301 |
302 | def test_detect_html5_style_meta_tag(self):
303 |
304 | for data in (
305 | b'',
306 | b"",
307 | b"",
308 | b""):
309 | dammit = UnicodeDammit(data, is_html=True)
310 | self.assertEqual(
311 | "euc-jp", dammit.original_encoding)
312 |
313 | def test_last_ditch_entity_replacement(self):
314 | # This is a UTF-8 document that contains bytestrings
315 | # completely incompatible with UTF-8 (ie. encoded with some other
316 | # encoding).
317 | #
318 | # Since there is no consistent encoding for the document,
319 | # Unicode, Dammit will eventually encode the document as UTF-8
320 | # and encode the incompatible characters as REPLACEMENT
321 | # CHARACTER.
322 | #
323 | # If chardet is installed, it will detect that the document
324 | # can be converted into ISO-8859-1 without errors. This happens
325 | # to be the wrong encoding, but it is a consistent encoding, so the
326 | # code we're testing here won't run.
327 | #
328 | # So we temporarily disable chardet if it's present.
329 | doc = b"""\357\273\277
330 | \330\250\330\252\330\261
331 | \310\322\321\220\312\321\355\344"""
332 | chardet = bs4.dammit.chardet_dammit
333 | logging.disable(logging.WARNING)
334 | try:
335 | def noop(str):
336 | return None
337 | bs4.dammit.chardet_dammit = noop
338 | dammit = UnicodeDammit(doc)
339 | self.assertEqual(True, dammit.contains_replacement_characters)
340 | self.assertTrue("\ufffd" in dammit.unicode_markup)
341 |
342 | soup = BeautifulSoup(doc, "html.parser")
343 | self.assertTrue(soup.contains_replacement_characters)
344 | finally:
345 | logging.disable(logging.NOTSET)
346 | bs4.dammit.chardet_dammit = chardet
347 |
348 | def test_byte_order_mark_removed(self):
349 | # A document written in UTF-16LE will have its byte order marker stripped.
350 | data = b'\xff\xfe<\x00a\x00>\x00\xe1\x00\xe9\x00<\x00/\x00a\x00>\x00'
351 | dammit = UnicodeDammit(data)
352 | self.assertEqual("áé", dammit.unicode_markup)
353 | self.assertEqual("utf-16le", dammit.original_encoding)
354 |
355 | def test_detwingle(self):
356 | # Here's a UTF8 document.
357 | utf8 = ("\N{SNOWMAN}" * 3).encode("utf8")
358 |
359 | # Here's a Windows-1252 document.
360 | windows_1252 = (
361 | "\N{LEFT DOUBLE QUOTATION MARK}Hi, I like Windows!"
362 | "\N{RIGHT DOUBLE QUOTATION MARK}").encode("windows_1252")
363 |
364 | # Through some unholy alchemy, they've been stuck together.
365 | doc = utf8 + windows_1252 + utf8
366 |
367 | # The document can't be turned into UTF-8:
368 | self.assertRaises(UnicodeDecodeError, doc.decode, "utf8")
369 |
370 | # Unicode, Dammit thinks the whole document is Windows-1252,
371 | # and decodes it into "☃☃☃“Hi, I like Windows!”☃☃☃"
372 |
373 | # But if we run it through fix_embedded_windows_1252, it's fixed:
374 |
375 | fixed = UnicodeDammit.detwingle(doc)
376 | self.assertEqual(
377 | "☃☃☃“Hi, I like Windows!”☃☃☃", fixed.decode("utf8"))
378 |
379 | def test_detwingle_ignores_multibyte_characters(self):
380 | # Each of these characters has a UTF-8 representation ending
381 | # in \x93. \x93 is a smart quote if interpreted as
382 | # Windows-1252. But our code knows to skip over multibyte
383 | # UTF-8 characters, so they'll survive the process unscathed.
384 | for tricky_unicode_char in (
385 | "\N{LATIN SMALL LIGATURE OE}", # 2-byte char '\xc5\x93'
386 | "\N{LATIN SUBSCRIPT SMALL LETTER X}", # 3-byte char '\xe2\x82\x93'
387 | "\xf0\x90\x90\x93", # This is a CJK character, not sure which one.
388 | ):
389 | input = tricky_unicode_char.encode("utf8")
390 | self.assertTrue(input.endswith(b'\x93'))
391 | output = UnicodeDammit.detwingle(input)
392 | self.assertEqual(output, input)
393 |
394 | class TestNamedspacedAttribute(SoupTest):
395 |
396 | def test_name_may_be_none(self):
397 | a = NamespacedAttribute("xmlns", None)
398 | self.assertEqual(a, "xmlns")
399 |
400 | def test_attribute_is_equivalent_to_colon_separated_string(self):
401 | a = NamespacedAttribute("a", "b")
402 | self.assertEqual("a:b", a)
403 |
404 | def test_attributes_are_equivalent_if_prefix_and_name_identical(self):
405 | a = NamespacedAttribute("a", "b", "c")
406 | b = NamespacedAttribute("a", "b", "c")
407 | self.assertEqual(a, b)
408 |
409 | # The actual namespace is not considered.
410 | c = NamespacedAttribute("a", "b", None)
411 | self.assertEqual(a, c)
412 |
413 | # But name and prefix are important.
414 | d = NamespacedAttribute("a", "z", "c")
415 | self.assertNotEqual(a, d)
416 |
417 | e = NamespacedAttribute("z", "b", "c")
418 | self.assertNotEqual(a, e)
419 |
420 |
421 | class TestAttributeValueWithCharsetSubstitution(unittest.TestCase):
422 |
423 | def test_content_meta_attribute_value(self):
424 | value = CharsetMetaAttributeValue("euc-jp")
425 | self.assertEqual("euc-jp", value)
426 | self.assertEqual("euc-jp", value.original_value)
427 | self.assertEqual("utf8", value.encode("utf8"))
428 |
429 |
430 | def test_content_meta_attribute_value(self):
431 | value = ContentMetaAttributeValue("text/html; charset=euc-jp")
432 | self.assertEqual("text/html; charset=euc-jp", value)
433 | self.assertEqual("text/html; charset=euc-jp", value.original_value)
434 | self.assertEqual("text/html; charset=utf8", value.encode("utf8"))
435 |
--------------------------------------------------------------------------------
/bs4/testing.py:
--------------------------------------------------------------------------------
1 | """Helper classes for tests."""
2 |
3 | import copy
4 | import functools
5 | import unittest
6 | from unittest import TestCase
7 | from bs4 import BeautifulSoup
8 | from bs4.element import (
9 | CharsetMetaAttributeValue,
10 | Comment,
11 | ContentMetaAttributeValue,
12 | Doctype,
13 | SoupStrainer,
14 | )
15 |
16 | from bs4.builder import HTMLParserTreeBuilder
17 | default_builder = HTMLParserTreeBuilder
18 |
19 |
20 | class SoupTest(unittest.TestCase):
21 |
22 | @property
23 | def default_builder(self):
24 | return default_builder()
25 |
26 | def soup(self, markup, **kwargs):
27 | """Build a Beautiful Soup object from markup."""
28 | builder = kwargs.pop('builder', self.default_builder)
29 | return BeautifulSoup(markup, builder=builder, **kwargs)
30 |
31 | def document_for(self, markup):
32 | """Turn an HTML fragment into a document.
33 |
34 | The details depend on the builder.
35 | """
36 | return self.default_builder.test_fragment_to_document(markup)
37 |
38 | def assertSoupEquals(self, to_parse, compare_parsed_to=None):
39 | builder = self.default_builder
40 | obj = BeautifulSoup(to_parse, builder=builder)
41 | if compare_parsed_to is None:
42 | compare_parsed_to = to_parse
43 |
44 | self.assertEqual(obj.decode(), self.document_for(compare_parsed_to))
45 |
46 |
47 | class HTMLTreeBuilderSmokeTest(object):
48 |
49 | """A basic test of a treebuilder's competence.
50 |
51 | Any HTML treebuilder, present or future, should be able to pass
52 | these tests. With invalid markup, there's room for interpretation,
53 | and different parsers can handle it differently. But with the
54 | markup in these tests, there's not much room for interpretation.
55 | """
56 |
57 | def assertDoctypeHandled(self, doctype_fragment):
58 | """Assert that a given doctype string is handled correctly."""
59 | doctype_str, soup = self._document_with_doctype(doctype_fragment)
60 |
61 | # Make sure a Doctype object was created.
62 | doctype = soup.contents[0]
63 | self.assertEqual(doctype.__class__, Doctype)
64 | self.assertEqual(doctype, doctype_fragment)
65 | self.assertEqual(str(soup)[:len(doctype_str)], doctype_str)
66 |
67 | # Make sure that the doctype was correctly associated with the
68 | # parse tree and that the rest of the document parsed.
69 | self.assertEqual(soup.p.contents[0], 'foo')
70 |
71 | def _document_with_doctype(self, doctype_fragment):
72 | """Generate and parse a document with the given doctype."""
73 | doctype = '' % doctype_fragment
74 | markup = doctype + '\n
foo
'
75 | soup = self.soup(markup)
76 | return doctype, soup
77 |
78 | def test_normal_doctypes(self):
79 | """Make sure normal, everyday HTML doctypes are handled correctly."""
80 | self.assertDoctypeHandled("html")
81 | self.assertDoctypeHandled(
82 | 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"')
83 |
84 | def test_empty_doctype(self):
85 | soup = self.soup("")
86 | doctype = soup.contents[0]
87 | self.assertEqual("", doctype.strip())
88 |
89 | def test_public_doctype_with_url(self):
90 | doctype = 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"'
91 | self.assertDoctypeHandled(doctype)
92 |
93 | def test_system_doctype(self):
94 | self.assertDoctypeHandled('foo SYSTEM "http://www.example.com/"')
95 |
96 | def test_namespaced_system_doctype(self):
97 | # We can handle a namespaced doctype with a system ID.
98 | self.assertDoctypeHandled('xsl:stylesheet SYSTEM "htmlent.dtd"')
99 |
100 | def test_namespaced_public_doctype(self):
101 | # Test a namespaced doctype with a public id.
102 | self.assertDoctypeHandled('xsl:stylesheet PUBLIC "htmlent.dtd"')
103 |
104 | def test_real_xhtml_document(self):
105 | """A real XHTML document should come out more or less the same as it went in."""
106 | markup = b"""
107 |
108 |
109 | Hello.
110 | Goodbye.
111 | """
112 | soup = self.soup(markup)
113 | self.assertEqual(
114 | soup.encode("utf-8").replace(b"\n", b""),
115 | markup.replace(b"\n", b""))
116 |
117 | def test_deepcopy(self):
118 | """Make sure you can copy the tree builder.
119 |
120 | This is important because the builder is part of a
121 | BeautifulSoup object, and we want to be able to copy that.
122 | """
123 | copy.deepcopy(self.default_builder)
124 |
125 | def test_p_tag_is_never_empty_element(self):
126 | """A
tag is never designated as an empty-element tag.
127 |
128 | Even if the markup shows it as an empty-element tag, it
129 | shouldn't be presented that way.
130 | """
131 | soup = self.soup("
")
132 | self.assertFalse(soup.p.is_empty_element)
133 | self.assertEqual(str(soup.p), "")
134 |
135 | def test_unclosed_tags_get_closed(self):
136 | """A tag that's not closed by the end of the document should be closed.
137 |
138 | This applies to all tags except empty-element tags.
139 | """
140 | self.assertSoupEquals("
", "
")
141 | self.assertSoupEquals("", "")
142 |
143 | self.assertSoupEquals(" ", " ")
144 |
145 | def test_br_is_always_empty_element_tag(self):
146 | """A tag is designated as an empty-element tag.
147 |
148 | Some parsers treat as one tag, some parsers as
149 | two tags, but it should always be an empty-element tag.
150 | """
151 | soup = self.soup(" ")
152 | self.assertTrue(soup.br.is_empty_element)
153 | self.assertEqual(str(soup.br), " ")
154 |
155 | def test_nested_formatting_elements(self):
156 | self.assertSoupEquals("")
157 |
158 | def test_comment(self):
159 | # Comments are represented as Comment objects.
160 | markup = "
foobaz
"
161 | self.assertSoupEquals(markup)
162 |
163 | soup = self.soup(markup)
164 | comment = soup.find(text="foobar")
165 | self.assertEqual(comment.__class__, Comment)
166 |
167 | # The comment is properly integrated into the tree.
168 | foo = soup.find(text="foo")
169 | self.assertEqual(comment, foo.next_element)
170 | baz = soup.find(text="baz")
171 | self.assertEqual(comment, baz.previous_element)
172 |
173 | def test_preserved_whitespace_in_pre_and_textarea(self):
174 | """Whitespace must be preserved in
and ")
177 |
178 | def test_nested_inline_elements(self):
179 | """Inline elements can be nested indefinitely."""
180 | b_tag = "Inside a B tag"
181 | self.assertSoupEquals(b_tag)
182 |
183 | nested_b_tag = "
")
215 |
216 | def test_deeply_nested_multivalued_attribute(self):
217 | # html5lib can set the attributes of the same tag many times
218 | # as it rearranges the tree. This has caused problems with
219 | # multivalued attributes.
220 | markup = '
".encode("utf-8")
349 | soup = self.soup(text)
350 | self.assertEqual(soup.p.encode("utf-8"), expected)
351 |
352 | def test_real_iso_latin_document(self):
353 | # Smoke test of interrelated functionality, using an
354 | # easy-to-understand document.
355 |
356 | # Here it is in Unicode. Note that it claims to be in ISO-Latin-1.
357 | unicode_html = '
Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!
'
358 |
359 | # That's because we're going to encode it into ISO-Latin-1, and use
360 | # that to test.
361 | iso_latin_html = unicode_html.encode("iso-8859-1")
362 |
363 | # Parse the ISO-Latin-1 HTML.
364 | soup = self.soup(iso_latin_html)
365 | # Encode it to UTF-8.
366 | result = soup.encode("utf-8")
367 |
368 | # What do we expect the result to look like? Well, it would
369 | # look like unicode_html, except that the META tag would say
370 | # UTF-8 instead of ISO-Latin-1.
371 | expected = unicode_html.replace("ISO-Latin-1", "utf-8")
372 |
373 | # And, of course, it would be in UTF-8, not Unicode.
374 | expected = expected.encode("utf-8")
375 |
376 | # Ta-da!
377 | self.assertEqual(result, expected)
378 |
379 | def test_real_shift_jis_document(self):
380 | # Smoke test to make sure the parser can handle a document in
381 | # Shift-JIS encoding, without choking.
382 | shift_jis_html = (
383 | b'
')
388 | unicode_html = shift_jis_html.decode("shift-jis")
389 | soup = self.soup(unicode_html)
390 |
391 | # Make sure the parse tree is correctly encoded to various
392 | # encodings.
393 | self.assertEqual(soup.encode("utf-8"), unicode_html.encode("utf-8"))
394 | self.assertEqual(soup.encode("euc_jp"), unicode_html.encode("euc_jp"))
395 |
396 | def test_real_hebrew_document(self):
397 | # A real-world test to make sure we can convert ISO-8859-9 (a
398 | # Hebrew encoding) to UTF-8.
399 | hebrew_document = b'Hebrew (ISO 8859-8) in Visual Directionality
Hebrew (ISO 8859-8) in Visual Directionality
\xed\xe5\xec\xf9'
400 | soup = self.soup(
401 | hebrew_document, from_encoding="iso8859-8")
402 | self.assertEqual(soup.original_encoding, 'iso8859-8')
403 | self.assertEqual(
404 | soup.encode('utf-8'),
405 | hebrew_document.decode("iso8859-8").encode("utf-8"))
406 |
407 | def test_meta_tag_reflects_current_encoding(self):
408 | # Here's the tag saying that a document is
409 | # encoded in Shift-JIS.
410 | meta_tag = ('')
412 |
413 | # Here's a document incorporating that meta tag.
414 | shift_jis_html = (
415 | '\n%s\n'
416 | ''
417 | 'Shift-JIS markup goes here.') % meta_tag
418 | soup = self.soup(shift_jis_html)
419 |
420 | # Parse the document, and the charset is seemingly unaffected.
421 | parsed_meta = soup.find('meta', {'http-equiv': 'Content-type'})
422 | content = parsed_meta['content']
423 | self.assertEqual('text/html; charset=x-sjis', content)
424 |
425 | # But that value is actually a ContentMetaAttributeValue object.
426 | self.assertTrue(isinstance(content, ContentMetaAttributeValue))
427 |
428 | # And it will take on a value that reflects its current
429 | # encoding.
430 | self.assertEqual('text/html; charset=utf8', content.encode("utf8"))
431 |
432 | # For the rest of the story, see TestSubstitutions in
433 | # test_tree.py.
434 |
435 | def test_html5_style_meta_tag_reflects_current_encoding(self):
436 | # Here's the tag saying that a document is
437 | # encoded in Shift-JIS.
438 | meta_tag = ('')
439 |
440 | # Here's a document incorporating that meta tag.
441 | shift_jis_html = (
442 | '\n%s\n'
443 | ''
444 | 'Shift-JIS markup goes here.') % meta_tag
445 | soup = self.soup(shift_jis_html)
446 |
447 | # Parse the document, and the charset is seemingly unaffected.
448 | parsed_meta = soup.find('meta', id="encoding")
449 | charset = parsed_meta['charset']
450 | self.assertEqual('x-sjis', charset)
451 |
452 | # But that value is actually a CharsetMetaAttributeValue object.
453 | self.assertTrue(isinstance(charset, CharsetMetaAttributeValue))
454 |
455 | # And it will take on a value that reflects its current
456 | # encoding.
457 | self.assertEqual('utf8', charset.encode("utf8"))
458 |
459 | def test_tag_with_no_attributes_can_have_attributes_added(self):
460 | data = self.soup("text")
461 | data.a['foo'] = 'bar'
462 | self.assertEqual('text', data.a.decode())
463 |
464 | class XMLTreeBuilderSmokeTest(object):
465 |
466 | def test_docstring_generated(self):
467 | soup = self.soup("")
468 | self.assertEqual(
469 | soup.encode(), b'\n')
470 |
471 | def test_real_xhtml_document(self):
472 | """A real XHTML document should come out *exactly* the same as it went in."""
473 | markup = b"""
474 |
475 |
476 | Hello.
477 | Goodbye.
478 | """
479 | soup = self.soup(markup)
480 | self.assertEqual(
481 | soup.encode("utf-8"), markup)
482 |
483 | def test_formatter_processes_script_tag_for_xml_documents(self):
484 | doc = """
485 |
487 | """
488 | soup = BeautifulSoup(doc, "xml")
489 | # lxml would have stripped this while parsing, but we can add
490 | # it later.
491 | soup.script.string = 'console.log("< < hey > > ");'
492 | encoded = soup.encode()
493 | self.assertTrue(b"< < hey > >" in encoded)
494 |
495 | def test_can_parse_unicode_document(self):
496 | markup = 'Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!'
497 | soup = self.soup(markup)
498 | self.assertEqual('Sacr\xe9 bleu!', soup.root.string)
499 |
500 | def test_popping_namespaced_tag(self):
501 | markup = 'b2012-07-02T20:33:42Zcd'
502 | soup = self.soup(markup)
503 | self.assertEqual(
504 | str(soup.rss), markup)
505 |
506 | def test_docstring_includes_correct_encoding(self):
507 | soup = self.soup("")
508 | self.assertEqual(
509 | soup.encode("latin1"),
510 | b'\n')
511 |
512 | def test_large_xml_document(self):
513 | """A large XML document should come out the same as it went in."""
514 | markup = (b'\n'
515 | + b'0' * (2**12)
516 | + b'')
517 | soup = self.soup(markup)
518 | self.assertEqual(soup.encode("utf-8"), markup)
519 |
520 |
521 | def test_tags_are_empty_element_if_and_only_if_they_are_empty(self):
522 | self.assertSoupEquals("
", "
")
523 | self.assertSoupEquals("
foo
")
524 |
525 | def test_namespaces_are_preserved(self):
526 | markup = 'This tag is in the a namespaceThis tag is in the b namespace'
527 | soup = self.soup(markup)
528 | root = soup.root
529 | self.assertEqual("http://example.com/", root['xmlns:a'])
530 | self.assertEqual("http://example.net/", root['xmlns:b'])
531 |
532 | def test_closing_namespaced_tag(self):
533 | markup = '