17 | Dormouse
18 |
19 | """)
20 |
21 | assert html.root_element.name == "div"
22 | ```
23 |
24 | ## elements
25 |
26 | Recursively get all [elements](element.md) in the HTML.
27 |
28 | ```python
29 | from minestrone import HTML
30 | html = HTML("""
31 |
32 | Dormouse
33 |
34 | """)
35 |
36 | assert [e.name for e in html.elements] == ["div", "span"]
37 | ```
38 |
39 | ## query
40 |
41 | Takes a CSS selector and returns an iterator of [`Element`](element.md) items.
42 |
43 | ### Query by element name
44 |
45 | ```python
46 | from minestrone import HTML
47 | html = HTML("""
48 | The Dormouse's Story
49 |There was a table...
50 | """) 51 | 52 | for h1 in html.query("h1"): 53 | assert str(h1) == "The Dormouse's Story
" 54 | ``` 55 | 56 | ### Query by id 57 | 58 | ```python 59 | from minestrone import HTML 60 | html = HTML(""" 61 | 65 | """) 66 | 67 | for a in html.query("a#elsie"): 68 | assert str(a) == 'Elsie' 69 | ``` 70 | 71 | ### Query by class 72 | 73 | ```python 74 | from minestrone import HTML 75 | html = HTML(""" 76 | 80 | """) 81 | 82 | elsie_link = next(html.query("ul li a.sister")) 83 | assert str(elsie_link) == 'Elsie' 84 | 85 | lacie_link = next(html.query("ul li a.sister")) 86 | assert str(lacie_link) == 'Lacie' 87 | ``` 88 | 89 | ## query_to_list 90 | 91 | Exactly the same as [query](querying.md#query) except it returns a list of [`Element`](element.md) items instead of a generator. This is sometimes more useful than the `query` above, but it can take more time to parse and more memory to store the data if the HTML document is large. 92 | 93 | ```python 94 | from minestrone import HTML 95 | html = HTML(""" 96 | 100 | """) 101 | 102 | assert len(html.query_to_list("a")) == 2 103 | assert str(html.query_to_list("a")[0]) == 'Elsie' 104 | assert html.query_to_list("a") == list(html.query("a")) 105 | ``` 106 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "minestrone" 3 | authors = [{name = "Adam Hill", email = "adam@adamghill.com"}] 4 | dynamic = ["version", "description"] 5 | license = { file = "LICENSE" } 6 | 7 | [tool.poetry] 8 | name = "minestrone" 9 | version = "0.8.0" 10 | description = "Search, modify, and parse messy HTML with ease." 11 | authors = ["adamghill-
100 |
- 1 101 |
- 2 102 |
- 3 103 |
-
117 |
- 1 118 |
- item
- item
- item
- another item
- 1
-
161 |
- 1 162 |
-
173 |
- 1 174 |
- 2 175 |
The Dormouse's story
""" 29 | ) 30 | 31 | expected = """ 32 | 33 |The Dormouse's story
""" 37 | 38 | actual = str(html) 39 | 40 | assert actual == expected 41 | 42 | 43 | def test_html_fragments(): 44 | html = HTML( 45 | """The Dormouse's story
46 | """ 51 | ) 52 | 53 | expected = """The Dormouse's story
54 | """ 59 | 60 | actual = str(html) 61 | 62 | assert actual == expected 63 | 64 | 65 | def test_html_html_strings(html_fragment): 66 | html = HTML(html_fragment) 67 | 68 | assert str(html_fragment) == str(html) 69 | 70 | 71 | def test_html_html_soups(html_fragment): 72 | html = HTML(html_fragment) 73 | 74 | assert html_fragment._soup == html._soup 75 | 76 | 77 | def test_html_wrong_type(): 78 | with pytest.raises(Exception): 79 | HTML(1) 80 | 81 | 82 | def test_html_repr(html_fragment): 83 | assert repr(html_fragment) == str(html_fragment) 84 | 85 | 86 | def test_html_parser_fragment(html_fragment_str): 87 | html_parsed_with_html = HTML(html_fragment_str, parser=Parser.HTML) 88 | assert html_parsed_with_html 89 | 90 | html_parsed_with_lxml = HTML(html_fragment_str, parser=Parser.LXML) 91 | assert html_parsed_with_lxml 92 | 93 | html_parsed_with_html5 = HTML(html_fragment_str, parser=Parser.HTML5) 94 | assert html_parsed_with_html5 95 | 96 | 97 | def test_html_parser_fragment_html(): 98 | assert str(HTML("dormouse", parser=Parser.HTML)) == "dormouse" 99 | 100 | 101 | def test_html_parser_fragment_lxml(): 102 | assert ( 103 | str(HTML("dormouse", parser=Parser.LXML)) 104 | == "dormouse" 105 | ) 106 | 107 | 108 | def test_html_parser_fragment_html5(): 109 | assert ( 110 | str(HTML("dormouse", parser=Parser.HTML5)) 111 | == "dormouse" 112 | ) 113 | 114 | 115 | def test_html_parser_doc(html_doc_str): 116 | html_parsed_with_html = HTML(html_doc_str, parser=Parser.HTML) 117 | assert html_parsed_with_html 118 | 119 | html_parsed_with_lxml = HTML(html_doc_str, parser=Parser.LXML) 120 | assert html_parsed_with_lxml 121 | 122 | html_parsed_with_html5 = HTML(html_doc_str, parser=Parser.HTML5) 123 | assert html_parsed_with_html5 124 | 125 | 126 | def test_html_encoding(): 127 | html = HTML(b"\xed\xe5\xec\xf9
") 128 | assert str(html) == "翴檛
" 129 | assert html.encoding == "big5" 130 | 131 | html = HTML(b"\xed\xe5\xec\xf9
", encoding="iso-8859-8") 132 | assert str(html) == "םולש
" 133 | assert html.encoding == "iso-8859-8" 134 | -------------------------------------------------------------------------------- /minestrone/__init__.py: -------------------------------------------------------------------------------- 1 | """minestrone - Search, modify, and parse messy HTML with ease.""" 2 | 3 | from enum import Enum 4 | from typing import Iterator, List, Optional, Union 5 | 6 | import bs4 7 | 8 | from .element import Element, Text 9 | from .formatter import UnsortedAttributes 10 | 11 | __all__ = [ 12 | "HTML", 13 | "Element", 14 | "Text", 15 | "Parser", 16 | ] 17 | 18 | 19 | class Parser(Enum): 20 | HTML = "html.parser" 21 | LXML = "lxml" 22 | HTML5 = "html5lib" 23 | 24 | 25 | class HTML: 26 | encoding: Optional[str] = "utf-8" 27 | 28 | def __init__( 29 | self, 30 | html: Union[str, "HTML"], 31 | parser: Parser = Parser.HTML, 32 | encoding: str = None, 33 | ): 34 | self.html = html 35 | self.parser = parser 36 | 37 | if isinstance(html, str) or isinstance(html, bytes): 38 | self._soup = bs4.BeautifulSoup( 39 | html, features=parser.value, from_encoding=encoding 40 | ) 41 | elif isinstance(html, HTML): 42 | self._soup = html._soup 43 | self.html = html.html 44 | else: 45 | raise Exception("Unknown type to init HTML") 46 | 47 | if encoding: 48 | self.encoding = encoding 49 | else: 50 | self.encoding = self._soup.original_encoding 51 | 52 | def query(self, selector: str) -> Iterator[Element]: 53 | """Returns an iterator of `Element`s that match the CSS selector.""" 54 | 55 | for _tag in self._soup.select(selector): 56 | yield Element.convert_from_tag(self._soup, _tag) 57 | 58 | def query_to_list(self, selector: str) -> List[Element]: 59 | """Returns a list of `Element`s that match the CSS selector.""" 60 | 61 | return list(self.query(selector)) 62 | 63 | def prettify( 64 | self, indent: int = 2, max_line_length: int = 88, use_bs4: bool = False 65 | ) -> str: 66 | """Prettify HTML. 67 | 68 | Args: 69 | indent: How many spaces to indent for each level in the hierarchy. Defaults to 2. 70 | max_line_length: How long the line can reach before indenting another level. Defaults to 88. If `None` it will never used. 71 | use_bs4: Whether to use the `BeautifulSoup` `prettify` function or `minestrone`. Defaults to `False`. 72 | """ 73 | 74 | if use_bs4: 75 | return self._soup.prettify() 76 | 77 | strings = [] 78 | 79 | for top_level_child in self._soup.contents: 80 | if isinstance(top_level_child, bs4.Doctype) and top_level_child: 81 | strings.append("\n") 84 | elif isinstance(top_level_child, bs4.Tag): 85 | element = Element.convert_from_tag(self._soup, top_level_child) 86 | strings.append(element.prettify(indent, max_line_length)) 87 | elif isinstance(top_level_child, bs4.Comment): 88 | strings.append("") 91 | strings.append("\n") 92 | elif isinstance(top_level_child, str) and top_level_child != "\n": 93 | strings.append(top_level_child.strip()) 94 | strings.append("\n") 95 | 96 | return "".join(strings) 97 | 98 | @property 99 | def root_element(self) -> Optional[Element]: 100 | """Gets the root `Element` for the HTML.""" 101 | 102 | for _element in self._soup.contents: 103 | if isinstance(_element, bs4.element.Tag) and _element.name: 104 | return Element.convert_from_tag(self._soup, _element) 105 | 106 | return None 107 | 108 | @property 109 | def elements(self) -> Iterator[Element]: 110 | """Recursively yield all `Element`s in the HTML.""" 111 | 112 | for _element in self._soup.descendants: 113 | if isinstance(_element, bs4.element.Tag) and _element.name: 114 | yield Element.convert_from_tag(self._soup, _element) 115 | 116 | def __str__(self): 117 | # Cleans up `BeautifulSoup` modifications 118 | self._soup.smooth() 119 | 120 | # Prevent `BeautifulSoup` from re-ordering attributes in alphabetical order 121 | return self._soup.encode(formatter=UnsortedAttributes()).decode() 122 | 123 | def __repr__(self): 124 | return self.__str__() 125 | -------------------------------------------------------------------------------- /minestrone/element/prettifier.py: -------------------------------------------------------------------------------- 1 | import bs4 2 | 3 | from . import VOID_ELEMENTS, Element 4 | 5 | 6 | def prettify_element( 7 | element: Element, 8 | indent: int, 9 | max_line_length: int, 10 | spaces: str = "", 11 | ) -> str: 12 | def __increase_spaces(_spaces): 13 | return " " * (len(_spaces) + indent) 14 | 15 | def __decrease_spaces(_spaces): 16 | return " " * (len(_spaces) - indent) 17 | 18 | def __append_newline_if_needed(_strings): 19 | if not _strings[-1].endswith("\n"): 20 | _strings.append("\n") 21 | 22 | def __append_string(_strings, _string): 23 | if _string: 24 | _strings.append(_string) 25 | 26 | strings = [] 27 | __append_string(strings, spaces) 28 | __append_string(strings, element.tag_string) 29 | 30 | content_children = [ 31 | c 32 | for c in element._self.contents 33 | if (isinstance(c, str) and c != "\n") 34 | or isinstance(c, bs4.element.Tag) 35 | or isinstance(c, bs4.element.Comment) 36 | ] 37 | has_children = False 38 | 39 | for child in element.children: 40 | if has_children is False: 41 | if content_children: 42 | extra_child_spaces = __increase_spaces(spaces) 43 | 44 | for content_child in content_children.copy(): 45 | content_children.pop(0) 46 | 47 | if isinstance(content_child, str): 48 | child_text = content_child.strip() 49 | 50 | if child_text: 51 | # Make sure that any newlines are indented to the correct number of spaces 52 | child_text = child_text.replace( 53 | "\n", f"\n{extra_child_spaces}" 54 | ) 55 | 56 | __append_string(strings, "\n") 57 | __append_string(strings, extra_child_spaces) 58 | __append_string(strings, child_text) 59 | else: 60 | break 61 | 62 | if element.name not in VOID_ELEMENTS: 63 | # Only increase the number of spaces if the current element can have children 64 | # and it's the first child 65 | spaces = __increase_spaces(spaces) 66 | 67 | __append_newline_if_needed(strings) 68 | 69 | has_children = True 70 | __append_string( 71 | strings, prettify_element(child, indent, max_line_length, spaces=spaces) 72 | ) 73 | 74 | if content_children: 75 | extra_child_spaces = __increase_spaces(spaces) 76 | 77 | for child in content_children.copy(): 78 | content_children.pop(0) 79 | 80 | if isinstance(child, str): 81 | child_text = child.strip() 82 | 83 | if child_text: 84 | # Make sure that any newlines are indented to the correct number of spaces 85 | child_text = child_text.replace("\n", f"\n{spaces}") 86 | __append_string(strings, spaces) 87 | 88 | if isinstance(child, bs4.Comment): 89 | __append_string(strings, "") 95 | else: 96 | break 97 | 98 | if has_children: 99 | spaces = __decrease_spaces(spaces) 100 | 101 | __append_newline_if_needed(strings) 102 | __append_string(strings, spaces) 103 | __append_string(strings, element.closing_tag_string) 104 | else: 105 | is_long_line = False 106 | 107 | if max_line_length is not None and len(element._self.text) > max_line_length: 108 | is_long_line = True 109 | 110 | if is_long_line: 111 | spaces = __increase_spaces(spaces) 112 | __append_string(strings, "\n") 113 | __append_string(strings, spaces) 114 | 115 | __append_string(strings, element._self.text) 116 | 117 | if is_long_line: 118 | spaces = __decrease_spaces(spaces) 119 | __append_string(strings, "\n") 120 | __append_string(strings, spaces) 121 | 122 | __append_string(strings, element.closing_tag_string) 123 | 124 | __append_newline_if_needed(strings) 125 | 126 | return "".join(strings) 127 | -------------------------------------------------------------------------------- /docs/source/parsing.md: -------------------------------------------------------------------------------- 1 | # Parsing 2 | 3 | The `HTML` class parses a string of HTML and provides methods to [query](querying.md) the DOM for specific elements. 4 | 5 | ## \_\_init\_\_ 6 | 7 | Creates an `HTML` object from a `str` or `bytes`. 8 | 9 | ```python 10 | from minestrone import HTML 11 | html = HTML(""" 12 | 13 | 14 |The Dormouse's Story
18 | 19 | 23 | 24 | 25 | """) 26 | ``` 27 | 28 | If closing tags are missing, then they will be added as needed to make the HTML valid. 29 | 30 | ```python 31 | from minestrone import HTML 32 | assert str(HTML("dormouse")) == "dormouse" 33 | ``` 34 | 35 | ### parser 36 | Three parsers are available in `minestrone` and they all have different trade-offs. By default, the built-in, pure Python `html.parser` is used. `lxml` can be used for faster parsing speed. `html5lib` is another option to ensure a valid HTML5 document. 37 | 38 | ```{note} 39 | `lxml` and `html5lib` are not installed with `minestrone` by default and must be specifically installed. 40 | 41 | - `poetry add minestrone[lxml]` or `pip install minestrone[lxml]` 42 | - `poetry add minestrone[html5]` or `pip install minestrone[html5]` 43 | ``` 44 | 45 | ```{note} 46 | BeautifulSoup has a [summary table](https://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser) of the three parsers. There is also a more detailed [breakdown of the differences](https://www.crummy.com/software/BeautifulSoup/bs4/doc/#differences-between-parsers) between the parsers. 47 | ``` 48 | 49 | ### Parser.HTML 50 | 51 | ```python 52 | from minestrone import HTML, Parser 53 | assert str(HTML("dormouse"), parser=Parser.HTML) == "dormouse" 54 | ``` 55 | 56 | ### Parser.LXML 57 | 58 | ```python 59 | from minestrone import HTML, Parser 60 | assert str(HTML("dormouse"), parser=Parser.LXML) == "dormouse" 61 | ``` 62 | 63 | ### Parser.HTML5 64 | 65 | ```python 66 | from minestrone import HTML, Parser 67 | assert str(HTML("dormouse"), parser=Parser.HTML5) == "dormouse" 68 | ``` 69 | 70 | ## encoding 71 | 72 | `Beautiful Soup` [attempts to decipher the encoding](https://www.crummy.com/software/BeautifulSoup/bs4/doc/#encodings) of the HTML string, however it isn't always correct. An encoding can be passed along if necessary. 73 | 74 | ```python 75 | from minestrone import HTML, Parser 76 | html_bytes = b"\xed\xe5\xec\xf9
" 77 | 78 | assert str(HTML(html_bytes)) == "翴檛
" 79 | assert HTML(html_bytes).encoding == "big5" 80 | 81 | assert str(HTML(html_bytes), encoding="iso-8859-8") == "םולש
" 82 | assert HTML(html_bytes).encoding == "iso-8859-8" 83 | ``` 84 | 85 | ## prettify 86 | 87 | Returns a prettified version of the HTML. 88 | 89 | ```python 90 | html = HTML(""" 91 | 92 | 93 |The Dormouse's Story
97 | 98 | 102 | 103 | 104 | """) 105 | 106 | assert html.prettify() == """ 107 | 108 |The Dormouse's Story
112 | 120 | 121 | 122 | """ 123 | ``` 124 | 125 | 126 | ## \_\_str\_\_ 127 | 128 | Returns the `HTML` object as a string. 129 | 130 | ```python 131 | from minestrone import HTML 132 | html = HTML(""" 133 | 134 | 135 |The Dormouse's Story
139 | 140 | 144 | 145 | 146 | """) 147 | 148 | assert str(html) == """ 149 | 150 |The Dormouse's Story
154 | 158 | 159 | """ 160 | ``` 161 | 162 | ```{note} 163 | Rendering the `HTML` into a string _will_ remove preceding spaces. 164 | ``` 165 | -------------------------------------------------------------------------------- /tests/element/test_create_tags.py: -------------------------------------------------------------------------------- 1 | from minestrone import HTML, Text 2 | 3 | 4 | def test_create_tag_with_true_attribute_value(html_fragment: HTML): 5 | elsie = next(html_fragment.query("a#elsie")) 6 | actual = elsie._create_tag("button", "Save", disabled=True) 7 | 8 | expected = '' 9 | 10 | assert str(actual) == expected 11 | 12 | 13 | def test_create_tag_with_klass(html_fragment: HTML): 14 | elsie = next(html_fragment.query("a#elsie")) 15 | actual = elsie._create_tag("button", "Save", klass="test-class") 16 | 17 | expected = '' 18 | 19 | assert str(actual) == expected 20 | 21 | 22 | def test_prepend(html_fragment: HTML): 23 | elsie = next(html_fragment.query("a#elsie")) 24 | elsie.prepend("span", "test prepend content", klass="test-class") 25 | 26 | expected = """""" 31 | 32 | actual = str(html_fragment) 33 | 34 | assert actual == expected 35 | 36 | 37 | def test_text_str(html_fragment: HTML): 38 | elsie = next(html_fragment.query("a#elsie")) 39 | text = elsie.prepend(text="test prepend text content") 40 | 41 | expected = "test prepend text content" 42 | actual = str(text) 43 | 44 | assert actual == expected 45 | 46 | 47 | def test_text_repr(html_fragment: HTML): 48 | elsie = next(html_fragment.query("a#elsie")) 49 | text = elsie.prepend(text="test prepend text content") 50 | 51 | expected = "test prepend text content" 52 | actual = repr(text) 53 | 54 | assert actual == expected 55 | 56 | 57 | def test_prepend_text(html_fragment: HTML): 58 | elsie = next(html_fragment.query("a#elsie")) 59 | elsie.prepend(text="test prepend text content") 60 | 61 | expected = """""" 66 | 67 | actual = str(html_fragment) 68 | 69 | assert actual == expected 70 | 71 | 72 | def test_append(html_fragment: HTML): 73 | elsie = next(html_fragment.query("a#elsie")) 74 | elsie.append("span", "test append content", klass="test-class") 75 | 76 | expected = """""" 81 | 82 | actual = str(html_fragment) 83 | 84 | assert actual == expected 85 | 86 | 87 | def test_append_text(html_fragment: HTML): 88 | elsie = next(html_fragment.query("a#elsie")) 89 | 90 | text = elsie.append(text="test append text content") 91 | assert text 92 | assert isinstance(text, Text) 93 | 94 | expected = """""" 99 | 100 | actual = str(html_fragment) 101 | 102 | assert actual == expected 103 | 104 | 105 | def test_append_text_to_text(html_fragment: HTML): 106 | elsie = next(html_fragment.query("a#elsie")) 107 | first_text = elsie.append(text="test append 1") 108 | assert first_text 109 | assert isinstance(first_text, Text) 110 | 111 | second_text = first_text.append(text=" test append 2") 112 | assert second_text 113 | assert isinstance(second_text, Text) 114 | 115 | expected = """""" 120 | 121 | actual = str(html_fragment) 122 | 123 | assert actual == expected 124 | 125 | 126 | def test_append_multiple(html_fragment: HTML): 127 | elsie = next(html_fragment.query("a#elsie")) 128 | new_tag = elsie.append("span", "test append content 1", klass="test-class") 129 | new_tag.append("span", "test append content 2", klass="test-class") 130 | 131 | expected = """""" 136 | 137 | actual = str(html_fragment) 138 | 139 | assert actual == expected 140 | -------------------------------------------------------------------------------- /tests/test_benchmarks.py: -------------------------------------------------------------------------------- 1 | # Run this with `poe t tests/test_benchmarks.py --benchmark-only` 2 | from typing import Iterator 3 | 4 | import re 5 | from minestrone import HTML, Element 6 | from html.parser import HTMLParser 7 | 8 | from lxml import html as lxml_html 9 | from bs4 import BeautifulSoup, Tag 10 | 11 | UNICORN_MODEL_REGEX = re.compile( 12 | r"(unicorn:model|u:model)(\.[^=]+)?=[\"'](?P
16 |
"""
34 |
35 | EXPECTED = ["address", "city", "state", "zip_code"]
36 |
37 |
38 | def _parse_beautiful_soup(soup: BeautifulSoup):
39 | unicorn_model_names = []
40 |
41 | for tag in soup.descendants:
42 | if isinstance(tag, Tag) and tag.name:
43 | for attr in tag.attrs.keys():
44 | if attr.startswith("unicorn:model") or attr.startswith("u:model"):
45 | unicorn_model_names.append(tag.attrs[attr])
46 |
47 | return unicorn_model_names
48 |
49 |
50 | def test_beautiful_soup_html_parser_with_existing_soup(benchmark):
51 | html_parser_soup = BeautifulSoup(HTML_FRAGMENT, features="html.parser")
52 |
53 | def _():
54 | return _parse_beautiful_soup(html_parser_soup)
55 |
56 | actual = benchmark(_)
57 | assert EXPECTED == actual
58 |
59 |
60 | def test_beautiful_soup_html_parser(benchmark):
61 | def _():
62 | soup = BeautifulSoup(HTML_FRAGMENT, features="html.parser")
63 | return _parse_beautiful_soup(soup)
64 |
65 | actual = benchmark(_)
66 | assert EXPECTED == actual
67 |
68 |
69 | def test_beautiful_soup_lxml(benchmark):
70 | def _():
71 | soup = BeautifulSoup(HTML_FRAGMENT, features="lxml")
72 | return _parse_beautiful_soup(soup)
73 |
74 | actual = benchmark(_)
75 | assert EXPECTED == actual
76 |
77 |
78 | def test_beautiful_soup_lxml_with_existing_soup(benchmark):
79 | lxml_soup = BeautifulSoup(HTML_FRAGMENT, features="lxml")
80 |
81 | def _():
82 | return _parse_beautiful_soup(lxml_soup)
83 |
84 | actual = benchmark(_)
85 | assert EXPECTED == actual
86 |
87 |
88 | def test_regex(benchmark):
89 | def _():
90 | return [
91 | m.group("unicorn_model_name")
92 | for m in UNICORN_MODEL_REGEX.finditer(HTML_FRAGMENT)
93 | ]
94 |
95 | actual = benchmark(_)
96 |
97 | assert EXPECTED == actual
98 |
99 |
100 | def _minestrone_get_unicorn_models(element: Element) -> Iterator[str]:
101 | for attribute in element.attributes.keys():
102 | if attribute.startswith("unicorn:model") or attribute.startswith("u:model"):
103 | yield element.attributes[attribute]
104 |
105 |
106 | def test_minestrone(benchmark):
107 | def _():
108 | minestrone_html = HTML(HTML_FRAGMENT)
109 | unicorn_model_names = []
110 |
111 | for element in minestrone_html.elements:
112 | for attribute_value in _minestrone_get_unicorn_models(element):
113 | unicorn_model_names.append(attribute_value)
114 |
115 | return unicorn_model_names
116 |
117 | actual = benchmark(_)
118 |
119 | assert EXPECTED == actual
120 |
121 |
122 | def test_minestrone_with_existing_html(benchmark):
123 | minestrone_html = HTML(HTML_FRAGMENT)
124 |
125 | def _():
126 | unicorn_model_names = []
127 |
128 | for element in minestrone_html.elements:
129 | for attribute_value in _minestrone_get_unicorn_models(element):
130 | unicorn_model_names.append(attribute_value)
131 |
132 | return unicorn_model_names
133 |
134 | actual = benchmark(_)
135 |
136 | assert EXPECTED == actual
137 |
138 |
139 | # def test_parsel(benchmark):
140 | # from parsel import Selector
141 | # def _():
142 | # selector = Selector(html)
143 |
144 | # r = selector.css("input::attr(unicorn:model)").getall()
145 |
146 | # print(r)
147 |
148 | # return []
149 |
150 | # unicorn_model_names = benchmark(_)
151 | # assert len(unicorn_model_names) == 4
152 |
153 |
154 | def test_lxml_html(benchmark):
155 | def _():
156 | unicorn_model_names = []
157 |
158 | for element in lxml_html.fragment_fromstring(HTML_FRAGMENT).iter():
159 | for attrs in element.items():
160 | if attrs[0].startswith("unicorn:model") or attrs[0].startswith(
161 | "u:model"
162 | ):
163 | unicorn_model_names.append(attrs[1])
164 |
165 | return unicorn_model_names
166 |
167 | actual = benchmark(_)
168 |
169 | assert EXPECTED == actual
170 |
171 |
172 | class UnicornModelParser(HTMLParser):
173 | def feed(self, data):
174 | self.unicorn_model_names = []
175 |
176 | super().reset()
177 | super().feed(data)
178 |
179 | def handle_starttag(self, tag, attrs: list):
180 | for attr in attrs:
181 | if attr[0].startswith("unicorn:model") or attr[0].startswith("u:model"):
182 | self.unicorn_model_names.append(attr[1])
183 |
184 |
185 | def test_html_parser(benchmark):
186 | def _():
187 | parser = UnicornModelParser()
188 | parser.feed(HTML_FRAGMENT)
189 |
190 | return parser.unicorn_model_names
191 |
192 | actual = benchmark(_)
193 |
194 | assert EXPECTED == actual
195 |
--------------------------------------------------------------------------------
/tests/element/test_element.py:
--------------------------------------------------------------------------------
1 | import bs4
2 | import pytest
3 |
4 | from minestrone import HTML, Element
5 |
6 |
7 | def test_get_text(html_doc):
8 | tillie = next(html_doc.query("a#tillie"))
9 |
10 | expected = "Tillie"
11 | actual = tillie.text
12 |
13 | assert actual == expected
14 |
15 |
16 | def test_get_text_with_tags():
17 | html = HTML(
18 | """
19 |
17 |
33 | Step 2
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 | Address: 123 Main St
26 | City: Anytown
27 | State: CA
28 | Zip code: 12345
29 |
30 |
31 |
32 | 26 | City: Anytown
27 | State: CA
28 | Zip code: 12345
29 |
stuff in here header
20 | """
21 | )
22 | el = next(html.query("h1"))
23 |
24 | expected = "stuff in here header"
25 | actual = el.text
26 |
27 | assert actual == expected
28 |
29 |
30 | def test_set_text(html_doc):
31 | tillie = next(html_doc.query("a#tillie"))
32 |
33 | tillie.text = "Billie"
34 |
35 | expected = "Billie"
36 | actual = tillie.text
37 |
38 | assert actual == expected
39 |
40 |
41 | def test_get_name(html_doc):
42 | tillie = next(html_doc.query("a#tillie"))
43 |
44 | expected = "a"
45 | actual = tillie.name
46 |
47 | assert actual == expected
48 |
49 |
50 | def test_element_klass():
51 | actual = Element.create("button", "Save", klass="test-class")
52 |
53 | expected = ''
54 |
55 | assert str(actual) == expected
56 |
57 |
58 | def test_element_get_attributes():
59 | span = Element.create(
60 | "span",
61 | "test attrs content",
62 | klass="test-class1 test-class2",
63 | disabled=True,
64 | id="span1",
65 | )
66 |
67 | assert span.name == "span"
68 | assert span.text == "test attrs content"
69 | assert span.attributes == {
70 | "class": "test-class1 test-class2",
71 | "disabled": "disabled",
72 | "id": "span1",
73 | }
74 |
75 |
76 | def test_element_set_attributes_id():
77 | span = Element.create(
78 | "span",
79 | )
80 |
81 | assert span.id is None
82 |
83 | span.attributes = {"id": "test-id"}
84 |
85 | assert span.id == "test-id"
86 | assert span.attributes == {
87 | "id": "test-id",
88 | }
89 |
90 |
91 | def test_element_set_id():
92 | span = Element.create(
93 | "span",
94 | )
95 |
96 | assert span.id is None
97 |
98 | span.id = "test-id"
99 |
100 | assert span.id == "test-id"
101 | assert span.attributes == {
102 | "id": "test-id",
103 | }
104 |
105 |
106 | def test_element_set_attributes_klass():
107 | span = Element.create(
108 | "span",
109 | klass="test-class1",
110 | )
111 |
112 | span.attributes = {"klass": "test-class2 test-class3"}
113 |
114 | assert span.name == "span"
115 | assert span.attributes == {
116 | "class": "test-class2 test-class3",
117 | }
118 | assert span.classes == ["test-class2", "test-class3"]
119 |
120 |
121 | def test_element_set_attributes_css():
122 | span = Element.create(
123 | "span",
124 | klass="test-class1",
125 | )
126 |
127 | span.attributes = {"css": "test-class2 test-class3"}
128 |
129 | assert span.name == "span"
130 | assert span.attributes == {
131 | "class": "test-class2 test-class3",
132 | }
133 | assert span.classes == ["test-class2", "test-class3"]
134 |
135 |
136 | def test_element_set_attributes_class_list():
137 | span = Element.create(
138 | "span",
139 | klass="test-class1",
140 | )
141 |
142 | span.attributes = {"css": ["test-class2", "test-class3"]}
143 |
144 | assert span.name == "span"
145 | assert span.attributes == {
146 | "class": "test-class2 test-class3",
147 | }
148 | assert span.classes == ["test-class2", "test-class3"]
149 |
150 |
151 | def test_element_set_attributes_class_tuple():
152 | span = Element.create(
153 | "span",
154 | klass="test-class1",
155 | )
156 |
157 | span.attributes = {"css": ("test-class2", "test-class3")}
158 |
159 | assert span.name == "span"
160 | assert span.attributes == {
161 | "class": "test-class2 test-class3",
162 | }
163 | assert span.classes == ["test-class2", "test-class3"]
164 |
165 |
166 | def test_element_set_attributes_invalid_type():
167 | span = Element.create("span")
168 |
169 | with pytest.raises(Exception):
170 | span.attributes = {"css": 0}
171 |
172 |
173 | def test_element_classes_from_klass_kwarg():
174 | span = Element.create(
175 | "span",
176 | klass="test-class1 test-class2",
177 | )
178 |
179 | assert span.classes == ["test-class1", "test-class2"]
180 |
181 |
182 | def test_element_classes_from_css_kwarg():
183 | span = Element.create(
184 | "span",
185 | css="test-class1 test-class2",
186 | )
187 |
188 | assert span.classes == ["test-class1", "test-class2"]
189 |
190 |
191 | def test_element_classes_empty():
192 | span = Element.create("span")
193 |
194 | assert span.classes == []
195 |
196 |
197 | def test_element_children(html_doc):
198 | ul = next(html_doc.query("ul"))
199 |
200 | assert len(list(ul.children)) == 3
201 |
202 | # get generator so next() will work to get all children
203 | children = ul.children
204 |
205 | first_li = next(children)
206 | elsie = next(first_li.children)
207 | assert elsie.id == "elsie"
208 | assert len(list(elsie.children)) == 0
209 |
210 | second_li = next(children)
211 | lacie = next(second_li.children)
212 | assert lacie.id == "lacie"
213 |
214 | third_li = next(children)
215 | tillie = next(third_li.children)
216 | assert tillie.id == "tillie"
217 |
218 |
219 | def test_element_parent(html_doc):
220 | elsie = next(html_doc.query("#elsie"))
221 |
222 | assert elsie.parent
223 | assert elsie.parent.name == "li"
224 | assert elsie.parent.parent.name == "ul"
225 |
226 |
227 | def test_element_parent_none(html_doc: HTML):
228 | assert html_doc.root_element
229 | assert html_doc.root_element.parent
230 | assert html_doc.root_element.parent.parent is None
231 |
232 |
233 | def test_create_without_soup():
234 | span = Element.create(
235 | "span",
236 | )
237 |
238 | assert span._soup
239 |
240 |
241 | def test_create_with_soup():
242 | soup = bs4.BeautifulSoup()
243 |
244 | span = Element.create(
245 | "span",
246 | soup=soup,
247 | )
248 |
249 | assert span._soup
250 | assert id(span._soup) == id(soup)
251 |
252 |
253 | def test_repr():
254 | span = Element.create(
255 | "span",
256 | )
257 |
258 | assert repr(span) == ""
259 |
260 |
261 | def test_tag_string(html_doc):
262 | ul = next(html_doc.query("ul"))
263 |
264 | expected = "- "
265 | actual = ul.tag_string
266 |
267 | assert actual == expected
268 |
269 |
270 | def test_tag_string_with_attributes(html_doc):
271 | tillie = next(html_doc.query("a#tillie"))
272 |
273 | expected = ''
274 | actual = tillie.tag_string
275 |
276 | assert actual == expected
277 |
278 |
279 | def test_tag_string_with_list_attributes():
280 | tillie = HTML('').root_element
281 |
282 | expected = ''
283 | actual = tillie.tag_string
284 |
285 | assert actual == expected
286 |
287 |
288 | def test_closing_tag_string(html_doc):
289 | tillie = next(html_doc.query("a#tillie"))
290 |
291 | expected = ""
292 | actual = tillie.closing_tag_string
293 |
294 | assert actual == expected
295 |
296 |
297 | def test_remove_children(html_doc):
298 | ul = next(html_doc.query("ul"))
299 | assert len(list(ul.children)) == 3
300 |
301 | ul.remove_children()
302 | assert len(list(ul.children)) == 0
303 |
304 |
305 | def test_insert(html_doc):
306 | ul = next(html_doc.query("ul"))
307 | assert len(list(ul.children)) == 3
308 |
309 | element = Element.create("li", "insert")
310 |
311 | ul.insert(element)
312 | assert len(list(ul.children)) == 4
313 |
314 | expected = "
- insert " 315 | actual = str(list(ul.children)[0]) 316 | 317 | assert actual == expected 318 | 319 | 320 | def test_insert_index(html_doc): 321 | ul = next(html_doc.query("ul")) 322 | element = Element.create("li", "insert-index") 323 | 324 | ul.insert(element, -1) 325 | assert len(list(ul.children)) == 4 326 | 327 | expected = "
- insert-index " 328 | actual = str(list(ul.children)[-1]) 329 | 330 | assert actual == expected 331 | -------------------------------------------------------------------------------- /minestrone/element/__init__.py: -------------------------------------------------------------------------------- 1 | from abc import ABC 2 | from typing import Dict, Generator, List, Optional, Sequence, Union 3 | 4 | import bs4 5 | 6 | from ..formatter import UnsortedAttributes 7 | 8 | # List of void elements from: https://www.thoughtco.com/html-singleton-tags-3468620 9 | VOID_ELEMENTS = set( 10 | ( 11 | "area", 12 | "base", 13 | "br", 14 | "col", 15 | "command", 16 | "embed", 17 | "hr", 18 | "img", 19 | "input", 20 | "keygen", 21 | "link", 22 | "meta", 23 | "param", 24 | "source", 25 | "track", 26 | "wbr", 27 | ) 28 | ) 29 | HTML5_CLOSING_TAG = "" 30 | XHTML_CLOSING_TAG = "/>" # TODO: Be able to set XHTML mode 31 | 32 | 33 | class Content(ABC): 34 | """Base class for `Text` and `Element` classes.""" 35 | 36 | _soup: bs4.BeautifulSoup 37 | _self: Union[bs4.element.PageElement, bs4.element.Tag, bs4.element.Tag] 38 | 39 | def append( 40 | self, name: Optional[str] = None, text: Optional[str] = None, **kwargs 41 | ) -> Union["Element", "Text"]: 42 | """Adds `Text` or a new `Element` after the current `Element`. 43 | 44 | Args: 45 | name: The tag name of the new `Element` or `None` for `Text`. 46 | text: The text content either by itself or within the element, e.g. "test" 47 | would be the text for `test`. 48 | kwargs: Any attributes that should be added to the new element. Use "css" 49 | or "klass" for "class"; use the value of `True` for attributes that 50 | do not require a value like `disabled`. `None` for `Text`. 51 | 52 | Returns: 53 | The newly added `Element` or `Text`. 54 | """ 55 | 56 | if name is None: 57 | assert text is not None, "Text content is required" 58 | assert not kwargs, "No attributes can be set for text content" 59 | 60 | self._self.insert_after(text) 61 | 62 | # Get the just inserted text 63 | next_sibling = self._self.next_sibling 64 | 65 | if next_sibling is not None and isinstance( 66 | next_sibling, bs4.element.NavigableString 67 | ): 68 | return Text.convert_from_navigable_string(self._soup, next_sibling) 69 | 70 | raise Exception("Could not find text") 71 | else: 72 | tag = self._create_tag(name, text=text, **kwargs) 73 | self._self.insert_after(tag) 74 | 75 | return Element.convert_from_tag(self._soup, tag) 76 | 77 | def prepend(self, name=None, text=None, **kwargs) -> Union["Element", "Text"]: 78 | """Adds a new element before the current element. 79 | 80 | Args: 81 | name: The name of the element. 82 | text: The `text content` of the element, e.g. "test" is the text for 83 | `test` 84 | kwargs: Any attributes that should be added to the new element. Use "css" 85 | or "klass" for "class"; use the value of `True` for attributes that 86 | do not require a value like `disabled`. 87 | 88 | Returns: 89 | The newly created element or `None` if text content was prepended. 90 | """ 91 | 92 | if name is None: 93 | assert text is not None, "Text content is required" 94 | assert not kwargs, "No attributes can be set for text content" 95 | 96 | self._self.insert_before(text) 97 | 98 | # Get the just inserted text 99 | previous_sibling = self._self.previous_sibling 100 | 101 | if previous_sibling is not None and isinstance( 102 | previous_sibling, bs4.element.NavigableString 103 | ): 104 | return Text.convert_from_navigable_string(self._soup, previous_sibling) 105 | 106 | raise Exception("Could not find text") 107 | else: 108 | tag = self._create_tag(name, text=text, **kwargs) 109 | self._self.insert_before(tag) 110 | 111 | return Element.convert_from_tag(self._soup, tag) 112 | 113 | def _create_tag(self, name, text=None, **kwargs) -> bs4.element.Tag: 114 | """Creates a new bs4 tag to be inserted.""" 115 | 116 | attrs = self._convert_attributes(kwargs) 117 | 118 | tag = self._soup.new_tag(name, attrs=attrs) 119 | 120 | if text: 121 | tag.string = text 122 | 123 | return tag 124 | 125 | def _convert_attributes(self, attributes: Dict) -> Dict: 126 | """Converts the attributes dictionary. Handles "klass"/"css" to "class" conversion. 127 | Also handles a value of `True` for attributes like `disabled`. 128 | 129 | Returns: 130 | A new dictionary. 131 | """ 132 | 133 | attrs = {} 134 | 135 | for key, value in attributes.items(): 136 | if key == "klass" or key == "css": 137 | key = "class" 138 | 139 | if value is True: 140 | value = key 141 | 142 | attrs[key] = value 143 | 144 | return attrs 145 | 146 | 147 | class Text(Content): 148 | _self: bs4.element.NavigableString 149 | 150 | @staticmethod 151 | def convert_from_navigable_string( 152 | soup: bs4.BeautifulSoup, 153 | navigable_string: bs4.element.NavigableString, 154 | ) -> "Text": 155 | text = Text() 156 | text._soup = soup 157 | text._self = navigable_string 158 | 159 | return text 160 | 161 | def __str__(self): 162 | return str(self._self) 163 | 164 | def __repr__(self): 165 | return self.__str__() 166 | 167 | 168 | class Element(Content): 169 | _self: bs4.element.Tag 170 | 171 | @staticmethod 172 | def create( 173 | name: str, text: str = None, soup: bs4.BeautifulSoup = None, **kwargs 174 | ) -> "Element": 175 | """Create the `Element`.""" 176 | 177 | element = Element() 178 | 179 | if soup is None: 180 | soup = bs4.BeautifulSoup() 181 | 182 | element._soup = soup 183 | 184 | tag = element._create_tag(name, text=text, **kwargs) 185 | element._self = tag 186 | 187 | return element 188 | 189 | @staticmethod 190 | def convert_from_tag(soup: bs4.BeautifulSoup, tag: bs4.element.Tag) -> "Element": 191 | """Gets the name of the `Element`.""" 192 | 193 | element = Element() 194 | element._soup = soup 195 | element._self = tag 196 | 197 | return element 198 | 199 | @property 200 | def name(self) -> str: 201 | """Gets the name of the `Element`.""" 202 | 203 | return self._self.name 204 | 205 | @property 206 | def id(self) -> Optional[str]: 207 | """Gets the id of the `Element`.""" 208 | 209 | return self._self.attrs.get("id") 210 | 211 | @id.setter 212 | def id(self, string: str) -> None: 213 | """Set the `id` of the `Element`.""" 214 | 215 | self._self.attrs["id"] = string.__class__(string) 216 | 217 | @property 218 | def attributes(self) -> Dict: 219 | """Attributes of the `Element`.""" 220 | 221 | attrs = {} 222 | 223 | for k, v in self._self.attrs.items(): 224 | if k == "class": 225 | attrs["class"] = " ".join(v) 226 | else: 227 | attrs[k] = v 228 | 229 | return attrs 230 | 231 | @attributes.setter 232 | def attributes(self, value: Dict): 233 | attrs = self._convert_attributes(value) 234 | 235 | for k, v in attrs.items(): 236 | if k == "class": 237 | if isinstance(v, str): 238 | attrs[k] = v.split(" ") 239 | elif isinstance(v, Sequence): 240 | attrs[k] = list(v) 241 | else: 242 | raise Exception("Invalid type for CSS classes") 243 | 244 | self._self.attrs = attrs 245 | 246 | @property 247 | def classes(self) -> List[str]: 248 | """Return all classes of the element.""" 249 | 250 | return list(self._self.attrs.get("class", [])) 251 | 252 | @property 253 | def children(self) -> Generator["Element", None, None]: 254 | """Returns an iterator of `Element`s of the children of the element.""" 255 | 256 | for child in self._self.children: 257 | if isinstance(child, bs4.element.Tag): 258 | yield Element.convert_from_tag(self._soup, child) 259 | 260 | @property 261 | def parent(self) -> Optional["Element"]: 262 | """Returns the parent `Element` of the element.""" 263 | 264 | if self._self.parent: 265 | return Element.convert_from_tag(self._soup, self._self.parent) 266 | 267 | return None 268 | 269 | @property 270 | def text(self) -> Optional[str]: 271 | """Gets the `text content` of the element. 272 | 273 | For example: 274 | `Hello World` would return "Hello World" 275 | 276 | `
- 20 | Elsie 21 | 22 |
- 23 | Lacie 24 | 25 |
- 26 | Tillie 27 | 28 | """ 29 | 30 | html = HTML( 31 | """ 32 |
- 33 | Elsie 34 | 35 |
- 36 | Lacie 37 | 38 |
- 39 | Tillie 40 | 41 | """ 42 | ) 43 | actual = html.prettify() 44 | 45 | eq(actual, expected) 46 | 47 | 48 | def test_html_prettify_minimal(): 49 | expected = """ 60 | """ 61 | 62 | html = HTML( 63 | """ 64 | 75 | """ 76 | ) 77 | actual = html.prettify() 78 | 79 | eq(actual, expected) 80 | 81 | 82 | def test_html_prettify_4_space_indent(): 83 | expected = """ 94 | """ 95 | 96 | html = HTML( 97 | """ 98 | 109 | """ 110 | ) 111 | actual = html.prettify(indent=4) 112 | 113 | eq(actual, expected) 114 | 115 | 116 | def test_html_prettify_longlines(): 117 | expected = """ 124 | """ 125 | 126 | html = HTML( 127 | '' 128 | ) 129 | actual = html.prettify() 130 | 131 | eq(actual, expected) 132 | 133 | 134 | def test_html_prettify_longlines_none_max_line_length(): 135 | expected = """ 140 | """ 141 | 142 | html = HTML( 143 | '' 144 | ) 145 | actual = html.prettify(max_line_length=None) 146 | 147 | eq(actual, expected) 148 | 149 | 150 | def test_html_prettify_with_text_children(): 151 | expected = """
- 153 | extra text 154 | Voluptatum qui 155 | even more text 156 | 157 |
- extra textVoluptatum quieven more text
- 171 | extra text1 172 | Voluptatum qui1 173 | even more text1 174 | 175 |
- 176 | extra text2 177 | Voluptatum qui2 178 | even more text2 179 | 180 |
- extra text1Voluptatum qui1even more text1 187 |
- extra text2Voluptatum qui2even more text2 188 |
- 216 | 0 217 | 1 218 | Voluptatum qui1 219 | 2 220 | 3 221 | Voluptatum qui2 222 | 4 223 | Voluptatum qui3 224 | 5 225 | 6 226 | Voluptatum qui4 227 | 7 228 | 229 |
- 0 237 | 1Voluptatum qui12 238 | 3Voluptatum qui24 239 | Voluptatum qui35 240 | 6Voluptatum qui4 241 | 7 242 |
- 1 321 |
- 1 328 |
Hello World
` would return "Hello World"
277 | """
278 |
279 | return "".join([str(c) for c in self._self.contents])
280 |
281 | @text.setter
282 | def text(self, string: str) -> None:
283 | """Set the `text content` of the element."""
284 |
285 | self._self.clear()
286 | self._self.append(string.__class__(string))
287 |
288 | @property
289 | def tag_string(self) -> str:
290 | _attributes = self.attributes.items()
291 |
292 | if not _attributes:
293 | return f"<{self.name}>"
294 |
295 | _tag_string = f"<{self.name}"
296 |
297 | for key, value in _attributes:
298 | if isinstance(value, list):
299 | value = " ".join(value)
300 |
301 | _tag_string += f' {key}="{value}"'
302 |
303 | _tag_string = f"{_tag_string}>"
304 |
305 | return _tag_string
306 |
307 | @property
308 | def closing_tag_string(self) -> str:
309 | if self._self.is_empty_element or self.name in VOID_ELEMENTS:
310 | return HTML5_CLOSING_TAG
311 |
312 | return f""
313 |
314 | def insert(self, element: "Element", index: int = 0) -> None:
315 | """Insert a child element into this element."""
316 |
317 | self._self.insert(index, element._self)
318 |
319 | def remove_children(self) -> None:
320 | """Remove all children from the element."""
321 |
322 | self._self.clear()
323 |
324 | def prettify(self, indent: int = 2, max_line_length: int = 88):
325 | # Import here to avoid circular imports
326 | from .prettifier import prettify_element
327 |
328 | return prettify_element(self, indent, max_line_length)
329 |
330 | def __str__(self):
331 | return self._self.encode(formatter=UnsortedAttributes()).decode()
332 |
333 | def __repr__(self):
334 | return self.__str__()
335 |
--------------------------------------------------------------------------------
/tests/html/test_prettify.py:
--------------------------------------------------------------------------------
1 | import time
2 |
3 | import pytest
4 |
5 | from minestrone import HTML
6 |
7 | PRINT_TIMINGS = True
8 |
9 |
10 | def eq(actual, expected):
11 | print(expected)
12 | print("===")
13 | print(actual)
14 |
15 | assert actual == expected
16 |
17 |
18 | def test_html_prettify_no_root():
19 | expected = """-
152 |
-
170 |
-
186 |
-
215 |
-
236 |
The Dormouse's story
257 | 268 | 269 | 270 | """ 271 | 272 | actual = html_doc.prettify() 273 | 274 | eq(actual, expected) 275 | 276 | 277 | def test_html_prettify_doc_example(): 278 | expected = """ 279 | 280 |The Dormouse's Story
284 | 292 | 293 | 294 | """ 295 | 296 | html = HTML( 297 | """ 298 | 299 | 300 |The Dormouse's Story
304 | 305 | 309 | 310 | 311 | """ 312 | ) 313 | actual = html.prettify() 314 | 315 | eq(actual, expected) 316 | 317 | 318 | def test_html_prettify_doc_example_2(): 319 | expected = """-
320 |
-
327 |
351 |
|
381 |
356 |
|
400 |
433 |
434 |
435 |
436 |
437 | """
438 |
439 | html = HTML(
440 | """
441 |
442 |
443 |
444 | """
445 | )
446 | actual = html.prettify()
447 |
448 | eq(actual, expected)
449 |
450 | expected = """
451 |
452 |
453 | """
454 |
455 | html = HTML(
456 | """
457 |
458 |
459 | """
460 | )
461 | actual = html.prettify()
462 |
463 | eq(actual, expected)
464 |
465 |
466 | @pytest.mark.parametrize(
467 | "name",
468 | [
469 | "hacker-news",
470 | "amazon",
471 | "bbc",
472 | "bing",
473 | "bootstrap",
474 | "coding-horror",
475 | "github",
476 | "google",
477 | "ny-times",
478 | "reddit",
479 | "stack-overflow",
480 | "twitter",
481 | "wikipedia",
482 | # "ecma-262", # this takes a long time
483 | ],
484 | )
485 | def test_html_prettify_samples(name):
486 | try:
487 | with open(f"tests/html/samples/expected/{name}.html") as f:
488 | expected = f.read()
489 | except:
490 | expected = ""
491 |
492 | with open(f"tests/html/samples/{name}.html") as f:
493 | html = f.read()
494 |
495 | start_time = time.time()
496 | actual = HTML(html).prettify()
497 |
498 | if PRINT_TIMINGS:
499 | elapsed_seconds = time.time() - start_time
500 |
501 | print(f"minestrone parse {name} took: {elapsed_seconds} seconds")
502 |
503 | if elapsed_seconds > 1:
504 | assert 0, f"{name} took longer than a second to prettify"
505 |
506 | # if expected == "":
507 | # with open(f"tests/html/samples/expected/{name}.html", "w") as f:
508 | # f.write(actual)
509 |
510 | eq(actual, expected)
511 |
--------------------------------------------------------------------------------
/tests/html/samples/hacker-news.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
15 |
|
50 | |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
54 |
|
1227 | |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
1230 | 
1231 |
1237 | 1260 | 1261 | 1265 | |
1267 | |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||