` is 51 | /// not considered inline. 52 | /// 53 | /// Tags include: `<%=tags_with('inline')%>`. 54 | pub fn is_inline(&self) -> bool { 55 | self.is_inline 56 | } 57 | 58 | /// Return true if the tag represents metadata only, where any content is 59 | /// not displayed text. e.g. ``. 60 | /// 61 | /// Tags include: `<%=tags_with('meta')%>`. 62 | pub fn is_meta(&self) -> bool { 63 | self.is_meta 64 | } 65 | 66 | /// Return true if the tag is banned/blacklisted: where no content should 67 | /// be extracted, displayed, or otherwise used. 68 | /// 69 | /// Tags include: `<%=tags_with('banned')%>`. 70 | pub fn is_banned(&self) -> bool { 71 | self.is_banned 72 | } 73 | 74 | /// Return true if the given name is part of the _basic_ set of known 75 | /// attributes for this element. 76 | /// 77 | /// This _basic set_ of attributes excludes, among other things, attributes 78 | /// that are used exclusively for styling purposes. 79 | pub fn has_basic_attr(&self, name: &LocalName) -> bool { 80 | self.basic_attrs.binary_search(name).is_ok() 81 | } 82 | } 83 | 84 | impl Default for TagMeta { 85 | fn default() -> TagMeta { 86 | TagMeta { 87 | is_empty: false, 88 | is_deprecated: false, 89 | is_inline: false, 90 | is_meta: false, 91 | is_banned: false, 92 | basic_attrs: vec![], 93 | } 94 | } 95 | } 96 | 97 | /// `Namespace` constants 98 | pub mod ns { 99 | use html5ever::ns; 100 | use crate::dom::Namespace; 101 | 102 | pub const HTML: Namespace = ns!(html); 103 | } 104 | 105 | /// HTML tag constants 106 | pub mod t { 107 | use html5ever::local_name as lname; 108 | use crate::dom::LocalName; 109 | 110 | % tags.each do |tag| 111 | % if tag.flags.include?('undefined') 112 | lazy_static::lazy_static! { 113 | % if tag.desc 114 | /// Tag `<<%= tag.name %>>`: <%= tag.desc %>. 115 | % unless tag.flags.empty? 116 | /// (meta: <%= tag.flags.join(' ') %>) 117 | % end 118 | /// 119 | /// This is a lazy static (struct) as its not defined by html5ever. 120 | % end 121 | pub static ref <%=const(tag.name.upcase)%>: LocalName = "<%=tag.name%>".into(); 122 | } 123 | % else 124 | % if tag.desc 125 | /// Tag `<<%= tag.name %>>`: <%= tag.desc %>. 126 | % unless tag.flags.empty? 127 | /// (meta: <%= tag.flags.join(' ') %>) 128 | % end 129 | % end 130 | pub const <%=twidth(const(tag.name.upcase) + ':', 4)%> LocalName = lname!("<%=tag.name%>"); 131 | % end 132 | % end 133 | } 134 | 135 | /// HTML attribute constants 136 | pub mod a { 137 | use html5ever::local_name as lname; 138 | use crate::dom::LocalName; 139 | 140 | % attributes.each do |a| 141 | % if a.flags.include?('undefined') 142 | lazy_static::lazy_static! { 143 | % if a.desc 144 | /// Attribute <%= a.name %>: <%= a.desc %>. 145 | /// 146 | /// This is a lazy static (struct) as its not defined by html5ever. 147 | % end 148 | pub static ref <%=const(a.name.upcase)%>: LocalName = "<%=a.name%>".into(); 149 | } 150 | % else 151 | % if a.desc 152 | /// Attribute <%= a.name %>: <%= a.desc %>. 153 | % end 154 | pub const <%=awidth(const(a.name.upcase) + ':', 4)%> LocalName = lname!("<%=a.name%>"); 155 | % end 156 | % end 157 | } 158 | 159 | fn init_tag_metadata() -> HashMap
is not labeled inline.) 15 | # M :: Metadata elements (content not visible text), i.e. head 16 | # B :: Banned/blacklisted elements from which text should not be extracted. 17 | # U :: Currently undefined by our HTML parser provider 18 | # 19 | # Sources 20 | # https://developer.mozilla.org/en-US/docs/Web/HTML/Element 21 | # https://html.spec.whatwg.org/ 22 | # https://www.w3.org/TR/html4/ 23 | # https://www.w3.org/TR/xhtml11/ 24 | 25 | a , S T F 5 I , anchor 26 | abbr , S T F 5 I , abbreviation 27 | acronym , S T F D I , acronym 28 | address , S T F 5 , contact information for the author or owner 29 | applet , T F D , embedded applet 30 | area ,E S T F 5 , area inside an image-map 31 | article , 5 , Structure: an independent content element 32 | aside , 5 , Structure: tengentially related content 33 | audio , 5 I , Sound content 34 | b , S T F 5 I , bold text 35 | base ,E S T F 5 M , default address or target for all links on a page 36 | basefont ,E T F D I M , default font; color; or size for the text in a page 37 | bdi , 5 I , Text isolated from surrounding for BIDI formatting 38 | bdo , S T F 5 I , the text direction 39 | big , S T F D I , big text 40 | blink , D I , blinking text 41 | blockquote , S T F 5 , long quotation 42 | body , S T F 5 , the document's body 43 | br ,E S T F 5 , single line break 44 | button , S T F 5 I B, push button 45 | canvas , 5 I , canvas for drawing graphics and animations 46 | caption , S T F 5 , table caption 47 | center , T F D , centered text 48 | cite , S T F 5 I , citation 49 | code , S T F 5 I , computer code text 50 | col ,E S T F 5 , attribute values for one or more columns in a table 51 | colgroup , S T F 5 , group of columns in a table for formatting 52 | content , D B, Shadow DOM content placeholder element 53 | data , 5 I , adds machine-oriented data representation 54 | datalist , 5 I B, container for option elements 55 | dd , S T F 5 , description of a term in a definition list 56 | del , S T F 5 I , deleted text 57 | details , 5 , optional additional details (also: summary) 58 | dfn , S T F 5 I , definition term 59 | dialog , 5 , dialog box or other interactive component 60 | dir , T F D , directory list 61 | div , S T F 5 , section in a document 62 | dl , S T F 5 , definition list 63 | dt , S T F 5 , term (an item) in a definition list 64 | em , S T F 5 I , emphasized text 65 | embed ,E 5 I , embed content by external app or plug-in 66 | fieldset , S T F 5 B, border around elements in a form 67 | figcaption , 5 , Structure: a figure caption 68 | figure , 5 , Structure: self contained content that can be moved 69 | font , T F D I , font; color; or size for text 70 | footer , 5 , Structure: a footer of a section 71 | form , S T F 5 , form for user input 72 | frame ,E F D B, window (a frame) in a frameset 73 | frameset , F D B, set of frames 74 | h1 , S T F 5 , heading level 1 75 | h2 , S T F 5 , heading level 2 76 | h3 , S T F 5 , heading level 3 77 | h4 , S T F 5 , heading level 4 78 | h5 , S T F 5 , heading level 5 79 | h6 , S T F 5 , heading level 6 80 | head , S T F 5 M , information about the document 81 | header , 5 , Structure: a header of a section 82 | hgroup , 5 , Structure: a group of headings 83 | hr ,E S T F 5 , horizontal line 84 | html , S T F 5 , document 85 | i , S T F 5 I , italic text 86 | iframe , T F 5 I , inline frame 87 | img ,E S T F 5 I , image 88 | input ,E S T F 5 I B, input control 89 | ins , S T F 5 I , inserted text 90 | isindex , T F D , searchable index related to a document 91 | kbd , S T F 5 I , keyboard text 92 | label , S T F 5 I B, label for input or other element 93 | legend , S T F 5 B, caption for a fieldset element 94 | li , S T F 5 , list item 95 | link ,E S T F 5 M , relationship with an external resource 96 | listing , D , preformated text 97 | main , 5 , identify central topic/functional content 98 | map , S T F 5 I , image-map 99 | mark , 5 I , Text marked/highlighted for reference purposes 100 | menu , T F 5 D , menu list 101 | menuitem ,E 5 D , a command in a menu 102 | meta ,E S T F 5 M , metadata 103 | meter , 5 I , a linear guage for a scaler value 104 | nav , 5 , Structure: container for navigational links 105 | nobr , D I , contained text; white-space: nowrap 106 | noframes , T F D B, alternate content where frames not supported 107 | noscript , S T F 5 I B, alternate content script not supported 108 | object , S T F 5 I B, embedded object 109 | ol , S T F 5 , ordered list 110 | optgroup , S T F 5 B, group of related options in a select list 111 | option , S T F 5 B, option in a select list 112 | output , 5 I , content is (scripted) outcome of a user action. 113 | p , S T F 5 , paragraph 114 | param ,E S T F 5 , parameter for an object 115 | picture , 5 I , container for multiple img/source DPI 116 | plaintext , D , like xmp; no close tag 117 | pre , S T F 5 , preformatted text 118 | progress , 5 I , a progress bar 119 | q , S T F 5 I , short quotation 120 | rb , 5 , ruby base text 121 | rbc , 5 U, ruby base container (complex) 122 | rp , 5 , ruby simple text container 123 | rt , 5 , ruby annotation text 124 | rtc , 5 , ruby text container (complex) 125 | ruby , 5 I , ruby pronunciation aid 126 | s , T F 5 D I , strikethrough text 127 | samp , S T F 5 I , sample computer code 128 | script , S T F 5 I B, client-side script 129 | section , 5 , Structure: generic document/application section 130 | select , S T F 5 I B, select list (drop-down list) 131 | slot , 5 I B, (Shadow) DOM placeholder element 132 | small , S T F 5 I , small text 133 | source ,E 5 , source for picture/audio/video elements 134 | span , S T F 5 I , section in a document 135 | strike , T F D I , strikethrough text 136 | strong , S T F 5 I , strong text 137 | style , S T F 5 B, style information for a document 138 | sub , S T F 5 I , subscripted text 139 | summary , 5 , summary of details element 140 | sup , S T F 5 I , superscripted text 141 | svg , 5 , inline scalable vector graphics 142 | table , S T F 5 , table 143 | tbody , S T F 5 , Groups the body content in a table 144 | td , S T F 5 , cell in a table 145 | template , 5 B, html sub-tree notrenderered except by script 146 | textarea , S T F 5 I B, multi-line text input control 147 | tfoot , S T F 5 , Groups the footer content in a table 148 | th , S T F 5 , header cell in a table 149 | thead , S T F 5 , Groups the header content in a table 150 | time , 5 I , A date or time 151 | title , S T F 5 M , the title of a document 152 | tr , S T F 5 , row in a table 153 | tt , S T F D I , teletype text 154 | u , T F 5 D I , underlined text 155 | ul , S T F 5 , unordered list 156 | var , S T F 5 I , variable part of a text 157 | video , 5 I , video container 158 | wbr ,E 5 I , A line break opportunity 159 | xmp , D , preformatted text 160 | -------------------------------------------------------------------------------- /marked/clippy.toml: -------------------------------------------------------------------------------- 1 | msrv = "1.38.0" 2 | -------------------------------------------------------------------------------- /marked/samples/documento_utf16be_bom.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dekellum/marked/1010a62cfd57e8ecd0ce505f57a3a4e744b1cba1/marked/samples/documento_utf16be_bom.html -------------------------------------------------------------------------------- /marked/samples/documento_utf16be_meta_utf16le.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dekellum/marked/1010a62cfd57e8ecd0ce505f57a3a4e744b1cba1/marked/samples/documento_utf16be_meta_utf16le.html -------------------------------------------------------------------------------- /marked/samples/documento_utf16le.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dekellum/marked/1010a62cfd57e8ecd0ce505f57a3a4e744b1cba1/marked/samples/documento_utf16le.html -------------------------------------------------------------------------------- /marked/samples/documento_utf16le_bom.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dekellum/marked/1010a62cfd57e8ecd0ce505f57a3a4e744b1cba1/marked/samples/documento_utf16le_bom.html -------------------------------------------------------------------------------- /marked/samples/documento_utf16le_meta_utf8.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dekellum/marked/1010a62cfd57e8ecd0ce505f57a3a4e744b1cba1/marked/samples/documento_utf16le_meta_utf8.html -------------------------------------------------------------------------------- /marked/samples/documento_utf8.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |
¿De donde eres tú?
7 | 8 | 9 | -------------------------------------------------------------------------------- /marked/samples/documento_utf8_bom.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |¿De donde eres tú?
7 | 8 | 9 | -------------------------------------------------------------------------------- /marked/samples/documento_utf8_meta.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |¿De donde eres tú?
8 | 9 | 10 | -------------------------------------------------------------------------------- /marked/samples/documento_utf8_meta_utf16.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |¿De donde eres tú?
9 | 10 | 11 | -------------------------------------------------------------------------------- /marked/samples/documento_windows1252_meta.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dekellum/marked/1010a62cfd57e8ecd0ce505f57a3a4e744b1cba1/marked/samples/documento_windows1252_meta.html -------------------------------------------------------------------------------- /marked/samples/iro0094_shiftjis_meta.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dekellum/marked/1010a62cfd57e8ecd0ce505f57a3a4e744b1cba1/marked/samples/iro0094_shiftjis_meta.html -------------------------------------------------------------------------------- /marked/samples/matsunami_eucjp_meta.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dekellum/marked/1010a62cfd57e8ecd0ce505f57a3a4e744b1cba1/marked/samples/matsunami_eucjp_meta.html -------------------------------------------------------------------------------- /marked/samples/russez_windows1251_meta.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dekellum/marked/1010a62cfd57e8ecd0ce505f57a3a4e744b1cba1/marked/samples/russez_windows1251_meta.html -------------------------------------------------------------------------------- /marked/src/chars.rs: -------------------------------------------------------------------------------- 1 | use tendril::StrTendril; 2 | 3 | /// Replace or remove sequences of white-space and/or control characters, and 4 | /// optionally remove leading/trailing spaces. 5 | /// 6 | /// _What_ char classes to replace is given via `ws` and `ctrl` flags. If a 7 | /// sequence is _all_ control or zero-width spaces, then it is simple removed 8 | /// (without replacement). If there is at least one non-zero width white-space 9 | /// character then the sequence is replaces with U+0020 SPACE. The string (st) 10 | /// is only lazily re-allocated (replaced) if a change is required. 11 | pub(crate) fn replace_chars( 12 | st: &mut StrTendril, 13 | ws: bool, 14 | ctrl: bool, 15 | trim_start: bool, 16 | trim_end: bool) 17 | { 18 | let mut last = 0; 19 | let mut ost = None; // output lazy allocated 20 | let mut replacing = 0u8; 21 | 22 | let ins = st.as_ref(); 23 | for (i, ch) in ins.char_indices() { 24 | let rmask = replace_mask(ch, ws, ctrl); 25 | if rmask > 0 { 26 | if replacing == 0 { 27 | if ost.is_none() { 28 | ost = Some(StrTendril::new()); 29 | } 30 | ost.as_mut().unwrap().push_slice(&ins[last..i]); 31 | } 32 | replacing |= rmask; 33 | } else if replacing > 0 { 34 | if replacing >= 2 && 35 | (ost.as_ref().unwrap().len32() > 0 || !trim_start) 36 | { 37 | ost.as_mut().unwrap().push_char(' '); 38 | } 39 | last = i; 40 | replacing = 0; 41 | } 42 | } 43 | if replacing > 0 { 44 | if replacing >= 2 && !trim_end { 45 | ost.as_mut().unwrap().push_char(' '); 46 | } 47 | } else if ost.is_some() { 48 | ost.as_mut().unwrap().push_slice(&ins[last..]); 49 | } 50 | if ost.is_some() { 51 | *st = ost.take().unwrap(); 52 | } 53 | } 54 | 55 | // Compare CharClass to flags and return bit-1 (control or zero-width) or bit-2 56 | // (whitespace). 57 | fn replace_mask(c: char, ws: bool, ctrl: bool) -> u8 { 58 | use CharClass::*; 59 | match char_class(c) { 60 | ZeroSpace | Control if ctrl => 1, 61 | WhiteSpace if ws => 2, 62 | _ => 0, 63 | } 64 | } 65 | 66 | // Character classes of internal interest (not the same as Unicode classes). 67 | #[derive(Debug, Eq, PartialEq)] 68 | enum CharClass { 69 | Unclassified, 70 | WhiteSpace, 71 | ZeroSpace, 72 | Control, 73 | } 74 | 75 | /// True if all contained characters are classified as whitespace or controls. 76 | pub(crate) fn is_all_ctrl_ws(st: &StrTendril) -> bool { 77 | st.as_ref().chars().all(|c| char_class(c) != CharClass::Unclassified) 78 | } 79 | 80 | // Return CharClass for a char 81 | fn char_class(c: char) -> CharClass { 82 | use CharClass::*; 83 | match c { 84 | '\u{0000}'..='\u{0008}' => Control, // C0 (XML disallowed) 85 | '\u{0009}' | // HT 86 | '\u{000A}' | // LF 87 | '\u{000B}' => WhiteSpace, // VT 88 | '\u{000C}' => Control, // FF (C0) 89 | '\u{000D}' => WhiteSpace, // CR 90 | '\u{000E}'..='\u{001F}' => Control, // C0 91 | '\u{0020}' => WhiteSpace, // SPACE 92 | 93 | '\u{007F}' | // DEL (C0) 94 | '\u{0080}'..='\u{009F}' => Control, // C1 (XML disallowed) 95 | '\u{00A0}' => WhiteSpace, // NO-BREAK SPACE (NBSP) 96 | 97 | // Not always (zero) white; shows hypen when line is wrapped. 98 | // '\u{00AD}' => Un- // SOFT HYPHEN, 99 | 100 | // Not white, rendered with a line: 101 | // '\u{1680}' => Un- // OGHAM SPACE MARK 102 | 103 | // Effects subsequent characters in Mongolian: 104 | // '\u{180E}' => Un- // MONGOLIAN VOWEL SEPARATOR 105 | 106 | '\u{2000}'..='\u{200A}' => WhiteSpace, // EN QUAD..HAIR SPACE 107 | '\u{200B}' | // ZERO WIDTH SPACE 108 | '\u{200C}' => ZeroSpace, // ZERO WIDTH NON-JOINER 109 | 110 | '\u{2028}' | // LINE SEPARATOR 111 | '\u{2029}' | // PARAGRAPH SEPARATOR 112 | 113 | '\u{202F}' | // NARROW NO-BREAK SPACE 114 | 115 | '\u{205F}' => WhiteSpace, // MEDIUM MATHEMATICAL SPACE 116 | '\u{2060}' => ZeroSpace, // WORD JOINER 117 | 118 | '\u{3000}' => WhiteSpace, // IDEOGRAPHIC SPACE 119 | 120 | '\u{FEFF}' => ZeroSpace, // BOM or ZERO WIDTH NON-BREAKING 121 | '\u{FFFE}' | // Bad BOM (not assigned) 122 | '\u{FFFF}' => Control, // Not assigned (invalid) 123 | _ => Unclassified, 124 | } 125 | 126 | // FIXME: see markup5ever/data/mod.rs: C1_REPLACEMENTS replaced with 127 | // alt higher unicode characters. This should occur _before_ above 128 | // transform, at least for HTML? 129 | } 130 | 131 | #[cfg(test)] 132 | mod tests { 133 | use super::*; 134 | use tendril::SliceExt; 135 | 136 | #[test] 137 | fn test_char_class() { 138 | use CharClass::*; 139 | assert_eq!(Unclassified, char_class('x')); 140 | assert_eq!(Control, char_class('\u{0008}')); 141 | assert_eq!(ZeroSpace, char_class('\u{2060}')); 142 | assert_eq!(WhiteSpace, char_class('\n')); 143 | assert_eq!(WhiteSpace, char_class('\n')); 144 | } 145 | 146 | #[test] 147 | fn replace() { 148 | assert_clean("", "" ); 149 | assert_clean("", "\u{2060}" ); 150 | assert_clean(" ", " "); 151 | assert_clean(" ", "\t \r\n"); 152 | 153 | assert_clean("x", "x" ); 154 | assert_clean(" x ", " x "); 155 | assert_clean(" x", " x\u{2060}" ); 156 | assert_clean("x ", "x " ); 157 | 158 | assert_clean("aa b ", "\u{009F}a\u{009F}a b " ); 159 | 160 | assert_clean("aa b c ", "aa b c " ); 161 | assert_clean("aa b c", "aa \t b c" ); 162 | assert_clean(" aa b c", "\t aa \t b c"); 163 | } 164 | 165 | // Assert that super-ASCII character boundaries are properly observed 166 | #[test] 167 | fn replace_multibyte() { 168 | assert_clean("Ψ", "Ψ" ); 169 | assert_clean(" Ψ ", " Ψ "); 170 | assert_clean(" Ψ", " Ψ\u{2060}" ); 171 | assert_clean("Ψ ", "Ψ " ); 172 | 173 | assert_clean("αα β ", "\u{009F}α\u{009F}α β " ); 174 | 175 | assert_clean("αα β γ ", "αα β γ " ); 176 | assert_clean("αα β γ", "αα \t β γ" ); 177 | assert_clean(" αα β γ", "\t αα \t β γ"); 178 | } 179 | 180 | #[test] 181 | fn replace_ctrl_only() { 182 | assert_clean_ctrl("", "" ); 183 | assert_clean_ctrl("", "\u{2060}" ); 184 | assert_clean_ctrl(" ", " "); 185 | 186 | assert_clean_ctrl("x", "x" ); 187 | assert_clean_ctrl(" x ", " x "); 188 | assert_clean_ctrl(" x", " x\u{2060}" ); 189 | assert_clean_ctrl("x ", "x " ); 190 | 191 | assert_clean_ctrl("aaa β ", "\u{009F}a\u{009F}aa β " ); 192 | 193 | assert_clean_ctrl("aa β c ", "aa β c " ); 194 | assert_clean_ctrl("aa \t β c", "aa \t β c" ); 195 | assert_clean_ctrl("\t aa \t β c", "\t aa \t β c"); 196 | } 197 | 198 | #[test] 199 | fn replace_trim() { 200 | assert_clean_trim("", ""); 201 | assert_clean_trim("", "\t \r\n"); 202 | assert_clean_trim("", "\u{0000}"); //NUL 203 | assert_clean_trim("", "\u{FFFE}"); //BAD BOM 204 | assert_clean_trim("", "\u{00A0}\u{2007}\u{202F}"); 205 | 206 | assert_clean_trim("x", "x" ); 207 | assert_clean_trim("x", " x "); 208 | assert_clean_trim("x", " x" ); 209 | assert_clean_trim("x", "x " ); 210 | 211 | assert_clean_trim("aa b", " a\u{009F}a\u{009F} b " ); 212 | 213 | assert_clean_trim("aa b c", "aa b c " ); 214 | assert_clean_trim("aa b c", "aa \t b c" ); 215 | assert_clean_trim("aa b c", "\t aa \t b c"); 216 | } 217 | 218 | #[test] 219 | fn replace_trim_left() { 220 | assert_clean_trim_l("", ""); 221 | assert_clean_trim_l(" ", " "); 222 | assert_clean_trim_l(" ", "\t \r\n"); 223 | } 224 | 225 | #[test] 226 | fn replace_trim_right() { 227 | assert_clean_trim_r("", ""); 228 | assert_clean_trim_r("", " "); 229 | assert_clean_trim_r("", "\t \r\n"); 230 | } 231 | 232 | fn assert_clean_trim(exp: &str, src: &str) { 233 | let mut st = src.to_tendril(); 234 | replace_chars(&mut st, true, true, true, true); 235 | assert_eq!(exp, st.as_ref()); 236 | } 237 | 238 | fn assert_clean_trim_l(exp: &str, src: &str) { 239 | let mut st = src.to_tendril(); 240 | replace_chars(&mut st, true, true, true, false); 241 | assert_eq!(exp, st.as_ref()); 242 | } 243 | 244 | fn assert_clean_trim_r(exp: &str, src: &str) { 245 | let mut st = src.to_tendril(); 246 | replace_chars(&mut st, true, true, false, true); 247 | assert_eq!(exp, st.as_ref()); 248 | } 249 | 250 | fn assert_clean(exp: &str, src: &str) { 251 | let mut st = src.to_tendril(); 252 | replace_chars(&mut st, true, true, false, false); 253 | assert_eq!(exp, st.as_ref()); 254 | } 255 | 256 | fn assert_clean_ctrl(exp: &str, src: &str) { 257 | let mut st = src.to_tendril(); 258 | replace_chars(&mut st, false, true, false, false); 259 | assert_eq!(exp, st.as_ref()); 260 | } 261 | } 262 | -------------------------------------------------------------------------------- /marked/src/decode.rs: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 David Kellum 2 | // 3 | // The `Decoder` implemented here was originally derived from 4 | // `tendril::stream::LossyDecoder` and `tendril::stream::TendrilSink` 5 | // (version 0.4.1) source as found: 6 | // 7 | // https://github.com/servo/tendril 8 | // Copyright © 2015 Keegan McAllister 9 | // Licensed under the Apache license v2.0, or the MIT license 10 | 11 | //! Support for streaming charset decoding. 12 | 13 | use std::borrow::Cow; 14 | use std::io; 15 | 16 | use log::trace; 17 | use encoding_rs as enc; 18 | use enc::DecoderResult; 19 | 20 | use tendril::{Tendril, TendrilSink, Atomicity, NonAtomic}; 21 | use tendril::fmt as form; 22 | use tendril::stream::Utf8LossyDecoder; 23 | 24 | mod encoding_hint; 25 | 26 | pub use encoding_hint::{ 27 | EncodingHint, SharedEncodingHint, 28 | }; 29 | 30 | use crate::READ_BUFFER_SIZE; 31 | 32 | /// A `TendrilSink` adaptor that takes bytes, decodes them as the given 33 | /// character encoding, while replacing any ill-formed byte sequences with 34 | /// U+FFFD replacement characters, and emits Unicode (`StrTendril`). 35 | /// 36 | /// This allocates new tendrils for encodings other than UTF-8. 37 | pub struct Decoder` element. Non-text oriented inline elements like 259 | /// `