s around
456 | "paragraphs" that are wrapped in non-block-level tags, such as anchors,
457 | phrase emphasis, and spans. The list of tags we're looking for is
458 | hard-coded.
459 |
460 | @param raw {boolean} indicates if these are raw HTML blocks in
461 | the original source. It makes a difference in "safe" mode.
462 | """
463 | if '<' not in text:
464 | return text
465 |
466 | # Pass `raw` value into our calls to self._hash_html_block_sub.
467 | hash_html_block_sub = _curry(self._hash_html_block_sub, raw=raw)
468 |
469 | # First, look for nested blocks, e.g.:
470 | #
", start_idx) + 3
502 | except ValueError, ex:
503 | break
504 |
505 | # Start position for next comment block search.
506 | start = end_idx
507 |
508 | # Validate whitespace before comment.
509 | if start_idx:
510 | # - Up to `tab_width - 1` spaces before start_idx.
511 | for i in range(self.tab_width - 1):
512 | if text[start_idx - 1] != ' ':
513 | break
514 | start_idx -= 1
515 | if start_idx == 0:
516 | break
517 | # - Must be preceded by 2 newlines or hit the start of
518 | # the document.
519 | if start_idx == 0:
520 | pass
521 | elif start_idx == 1 and text[0] == '\n':
522 | start_idx = 0 # to match minute detail of Markdown.pl regex
523 | elif text[start_idx - 2:start_idx] == '\n\n':
524 | pass
525 | else:
526 | break
527 |
528 | # Validate whitespace after comment.
529 | # - Any number of spaces and tabs.
530 | while end_idx < len(text):
531 | if text[end_idx] not in ' \t':
532 | break
533 | end_idx += 1
534 | # - Must be following by 2 newlines or hit end of text.
535 | if text[end_idx:end_idx + 2] not in ('', '\n', '\n\n'):
536 | continue
537 |
538 | # Escape and hash (must match `_hash_html_block_sub`).
539 | html = text[start_idx:end_idx]
540 | if raw and self.safe_mode:
541 | html = self._sanitize_html(html)
542 | key = _hash_text(html)
543 | self.html_blocks[key] = html
544 | text = text[:start_idx] + "\n\n" + \
545 | key + "\n\n" + text[end_idx:]
546 |
547 | if "xml" in self.extras:
548 | # Treat XML processing instructions and namespaced one-liner
549 | # tags as if they were block HTML tags. E.g., if standalone
550 | # (i.e. are their own paragraph), the following do not get
551 | # wrapped in a tag:
552 | #
553 | #
554 | #
555 | _xml_oneliner_re = _xml_oneliner_re_from_tab_width(self.tab_width)
556 | text = _xml_oneliner_re.sub(hash_html_block_sub, text)
557 |
558 | return text
559 |
560 | def _strip_link_definitions(self, text):
561 | # Strips link definitions from text, stores the URLs and titles in
562 | # hash references.
563 | less_than_tab = self.tab_width - 1
564 |
565 | # Link defs are in the form:
566 | # [id]: url "optional title"
567 | _link_def_re = re.compile(r"""
568 | ^[ ]{0,%d}\[(.+)\]: # id = \1
569 | [ \t]*
570 | \n? # maybe *one* newline
571 | [ \t]*
572 | ? # url = \2
573 | [ \t]*
574 | (?:
575 | \n? # maybe one newline
576 | [ \t]*
577 | (?<=\s) # lookbehind for whitespace
578 | ['"(]
579 | ([^\n]*) # title = \3
580 | ['")]
581 | [ \t]*
582 | )? # title is optional
583 | (?:\n+|\Z)
584 | """ % less_than_tab, re.X | re.M | re.U)
585 | return _link_def_re.sub(self._extract_link_def_sub, text)
586 |
587 | def _extract_link_def_sub(self, match):
588 | id, url, title = match.groups()
589 | key = id.lower() # Link IDs are case-insensitive
590 | self.urls[key] = self._encode_amps_and_angles(url)
591 | if title:
592 | self.titles[key] = title.replace('"', '"')
593 | return ""
594 |
595 | def _extract_footnote_def_sub(self, match):
596 | id, text = match.groups()
597 | text = _dedent(text, skip_first_line=not text.startswith('\n')).strip()
598 | normed_id = re.sub(r'\W', '-', id)
599 | # Ensure footnote text ends with a couple newlines (for some
600 | # block gamut matches).
601 | self.footnotes[normed_id] = text + "\n\n"
602 | return ""
603 |
604 | def _strip_footnote_definitions(self, text):
605 | """A footnote definition looks like this:
606 |
607 | [^note-id]: Text of the note.
608 |
609 | May include one or more indented paragraphs.
610 |
611 | Where,
612 | - The 'note-id' can be pretty much anything, though typically it
613 | is the number of the footnote.
614 | - The first paragraph may start on the next line, like so:
615 |
616 | [^note-id]:
617 | Text of the note.
618 | """
619 | less_than_tab = self.tab_width - 1
620 | footnote_def_re = re.compile(r'''
621 | ^[ ]{0,%d}\[\^(.+)\]: # id = \1
622 | [ \t]*
623 | ( # footnote text = \2
624 | # First line need not start with the spaces.
625 | (?:\s*.*\n+)
626 | (?:
627 | (?:[ ]{%d} | \t) # Subsequent lines must be indented.
628 | .*\n+
629 | )*
630 | )
631 | # Lookahead for non-space at line-start, or end of doc.
632 | (?:(?=^[ ]{0,%d}\S)|\Z)
633 | ''' % (less_than_tab, self.tab_width, self.tab_width),
634 | re.X | re.M)
635 | return footnote_def_re.sub(self._extract_footnote_def_sub, text)
636 |
637 | _hr_res = [
638 | re.compile(r"^[ ]{0,2}([ ]?\*[ ]?){3,}[ \t]*$", re.M),
639 | re.compile(r"^[ ]{0,2}([ ]?\-[ ]?){3,}[ \t]*$", re.M),
640 | re.compile(r"^[ ]{0,2}([ ]?\_[ ]?){3,}[ \t]*$", re.M),
641 | ]
642 |
643 | def _run_block_gamut(self, text):
644 | # These are all the transformations that form block-level
645 | # tags like paragraphs, headers, and list items.
646 |
647 | text = self._do_headers(text)
648 |
649 | # Do Horizontal Rules:
650 | hr = "\n tags around block-level tags.
667 | text = self._hash_html_blocks(text)
668 |
669 | text = self._form_paragraphs(text)
670 |
671 | return text
672 |
673 | def _pyshell_block_sub(self, match):
674 | lines = match.group(0).splitlines(0)
675 | _dedentlines(lines)
676 | indent = ' ' * self.tab_width
677 | s = ('\n' # separate from possible cuddled paragraph
678 | + indent + ('\n' + indent).join(lines)
679 | + '\n\n')
680 | return s
681 |
682 | def _prepare_pyshell_blocks(self, text):
683 | """Ensure that Python interactive shell sessions are put in
684 | code blocks -- even if not properly indented.
685 | """
686 | if ">>>" not in text:
687 | return text
688 |
689 | less_than_tab = self.tab_width - 1
690 | _pyshell_block_re = re.compile(r"""
691 | ^([ ]{0,%d})>>>[ ].*\n # first line
692 | ^(\1.*\S+.*\n)* # any number of subsequent lines
693 | ^\n # ends with a blank line
694 | """ % less_than_tab, re.M | re.X)
695 |
696 | return _pyshell_block_re.sub(self._pyshell_block_sub, text)
697 |
698 | def _run_span_gamut(self, text):
699 | # These are all the transformations that occur *within* block-level
700 | # tags like paragraphs, headers, and list items.
701 |
702 | text = self._do_code_spans(text)
703 |
704 | text = self._escape_special_chars(text)
705 |
706 | # Process anchor and image tags.
707 | text = self._do_links(text)
708 |
709 | # Make links out of things like ``
710 | # Must come after _do_links(), because you can use < and >
711 | # delimiters in inline links like [this]().
712 | text = self._do_auto_links(text)
713 |
714 | if "link-patterns" in self.extras:
715 | text = self._do_link_patterns(text)
716 |
717 | text = self._encode_amps_and_angles(text)
718 |
719 | text = self._do_italics_and_bold(text)
720 |
721 | # Do hard breaks:
722 | text = re.sub(r" {2,}\n", "
734 | |
735 | # auto-link (e.g., )
736 | <\w+[^>]*>
737 | |
738 | # comment
739 | |
740 | <\?.*?\?> # processing instruction
741 | )
742 | """, re.X)
743 |
744 | def _escape_special_chars(self, text):
745 | # Python markdown note: the HTML tokenization here differs from
746 | # that in Markdown.pl, hence the behaviour for subtle cases can
747 | # differ (I believe the tokenizer here does a better job because
748 | # it isn't susceptible to unmatched '<' and '>' in HTML tags).
749 | # Note, however, that '>' is not allowed in an auto-link URL
750 | # here.
751 | escaped = []
752 | is_html_markup = False
753 | for token in self._sorta_html_tokenize_re.split(text):
754 | if is_html_markup:
755 | # Within tags/HTML-comments/auto-links, encode * and _
756 | # so they don't conflict with their use in Markdown for
757 | # italics and strong. We're replacing each such
758 | # character with its corresponding MD5 checksum value;
759 | # this is likely overkill, but it should prevent us from
760 | # colliding with the escape values by accident.
761 | escaped.append(token.replace('*', g_escape_table['*'])
762 | .replace('_', g_escape_table['_']))
763 | else:
764 | escaped.append(self._encode_backslash_escapes(token))
765 | is_html_markup = not is_html_markup
766 | return ''.join(escaped)
767 |
768 | def _hash_html_spans(self, text):
769 | # Used for safe_mode.
770 |
771 | def _is_auto_link(s):
772 | if ':' in s and self._auto_link_re.match(s):
773 | return True
774 | elif '@' in s and self._auto_email_link_re.match(s):
775 | return True
776 | return False
777 |
778 | tokens = []
779 | is_html_markup = False
780 | for token in self._sorta_html_tokenize_re.split(text):
781 | if is_html_markup and not _is_auto_link(token):
782 | sanitized = self._sanitize_html(token)
783 | key = _hash_text(sanitized)
784 | self.html_spans[key] = sanitized
785 | tokens.append(key)
786 | else:
787 | tokens.append(token)
788 | is_html_markup = not is_html_markup
789 | return ''.join(tokens)
790 |
791 | def _unhash_html_spans(self, text):
792 | for key, sanitized in self.html_spans.items():
793 | text = text.replace(key, sanitized)
794 | return text
795 |
796 | def _sanitize_html(self, s):
797 | if self.safe_mode == "replace":
798 | return self.html_removed_text
799 | elif self.safe_mode == "escape":
800 | replacements = [
801 | ('&', '&'),
802 | ('<', '<'),
803 | ('>', '>'),
804 | ]
805 | for before, after in replacements:
806 | s = s.replace(before, after)
807 | return s
808 | else:
809 | raise MarkdownError("invalid value for 'safe_mode': %r (must be "
810 | "'escape' or 'replace')" % self.safe_mode)
811 |
812 | _tail_of_inline_link_re = re.compile(r'''
813 | # Match tail of: [text](/url/) or [text](/url/ "title")
814 | \( # literal paren
815 | [ \t]*
816 | (?P # \1
817 | <.*?>
818 | |
819 | .*?
820 | )
821 | [ \t]*
822 | ( # \2
823 | (['"]) # quote char = \3
824 | (?P.*?)
825 | \3 # matching quote
826 | )? # title is optional
827 | \)
828 | ''', re.X | re.S)
829 | _tail_of_reference_link_re = re.compile(r'''
830 | # Match tail of: [text][id]
831 | [ ]? # one optional space
832 | (?:\n[ ]*)? # one optional newline followed by spaces
833 | \[
834 | (?P<id>.*?)
835 | \]
836 | ''', re.X | re.S)
837 |
838 | def _do_links(self, text):
839 | """Turn Markdown link shortcuts into XHTML <a> and <img> tags.
840 |
841 | This is a combination of Markdown.pl's _DoAnchors() and
842 | _DoImages(). They are done together because that simplified the
843 | approach. It was necessary to use a different approach than
844 | Markdown.pl because of the lack of atomic matching support in
845 | Python's regex engine used in $g_nested_brackets.
846 | """
847 | MAX_LINK_TEXT_SENTINEL = 3000 # markdown2 issue 24
848 |
849 | # `anchor_allowed_pos` is used to support img links inside
850 | # anchors, but not anchors inside anchors. An anchor's start
851 | # pos must be `>= anchor_allowed_pos`.
852 | anchor_allowed_pos = 0
853 |
854 | curr_pos = 0
855 | while True: # Handle the next link.
856 | # The next '[' is the start of:
857 | # - an inline anchor: [text](url "title")
858 | # - a reference anchor: [text][id]
859 | # - an inline img: 
860 | # - a reference img: ![text][id]
861 | # - a footnote ref: [^id]
862 | # (Only if 'footnotes' extra enabled)
863 | # - a footnote defn: [^id]: ...
864 | # (Only if 'footnotes' extra enabled) These have already
865 | # been stripped in _strip_footnote_definitions() so no
866 | # need to watch for them.
867 | # - a link definition: [id]: url "title"
868 | # These have already been stripped in
869 | # _strip_link_definitions() so no need to watch for them.
870 | # - not markup: [...anything else...
871 | try:
872 | start_idx = text.index('[', curr_pos)
873 | except ValueError:
874 | break
875 | text_length = len(text)
876 |
877 | # Find the matching closing ']'.
878 | # Markdown.pl allows *matching* brackets in link text so we
879 | # will here too. Markdown.pl *doesn't* currently allow
880 | # matching brackets in img alt text -- we'll differ in that
881 | # regard.
882 | bracket_depth = 0
883 | for p in range(start_idx + 1, min(start_idx + MAX_LINK_TEXT_SENTINEL,
884 | text_length)):
885 | ch = text[p]
886 | if ch == ']':
887 | bracket_depth -= 1
888 | if bracket_depth < 0:
889 | break
890 | elif ch == '[':
891 | bracket_depth += 1
892 | else:
893 | # Closing bracket not found within sentinel length.
894 | # This isn't markup.
895 | curr_pos = start_idx + 1
896 | continue
897 | link_text = text[start_idx + 1:p]
898 |
899 | # Possibly a footnote ref?
900 | if "footnotes" in self.extras and link_text.startswith("^"):
901 | normed_id = re.sub(r'\W', '-', link_text[1:])
902 | if normed_id in self.footnotes:
903 | self.footnote_ids.append(normed_id)
904 | result = '<sup class="footnote-ref" id="fnref-%s">' \
905 | '<a href="#fn-%s">%s</a></sup>' \
906 | % (normed_id, normed_id, len(self.footnote_ids))
907 | text = text[:start_idx] + result + text[p + 1:]
908 | else:
909 | # This id isn't defined, leave the markup alone.
910 | curr_pos = p + 1
911 | continue
912 |
913 | # Now determine what this is by the remainder.
914 | p += 1
915 | if p == text_length:
916 | return text
917 |
918 | # Inline anchor or img?
919 | if text[p] == '(': # attempt at perf improvement
920 | match = self._tail_of_inline_link_re.match(text, p)
921 | if match:
922 | # Handle an inline anchor or img.
923 | is_img = start_idx > 0 and text[start_idx - 1] == "!"
924 | if is_img:
925 | start_idx -= 1
926 |
927 | url, title = match.group("url"), match.group("title")
928 | if url and url[0] == '<':
929 | url = url[1:-1] # '<url>' -> 'url'
930 | # We've got to encode these to avoid conflicting
931 | # with italics/bold.
932 | url = url.replace('*', g_escape_table['*']) \
933 | .replace('_', g_escape_table['_'])
934 | if title:
935 | title_str = ' title="%s"' \
936 | % title.replace('*', g_escape_table['*']) \
937 | .replace('_', g_escape_table['_']) \
938 | .replace('"', '"')
939 | else:
940 | title_str = ''
941 | if is_img:
942 | result = '<img src="%s" alt="%s"%s%s' \
943 | % (url, link_text.replace('"', '"'),
944 | title_str, self.empty_element_suffix)
945 | curr_pos = start_idx + len(result)
946 | text = text[:start_idx] + result + text[match.end():]
947 | elif start_idx >= anchor_allowed_pos:
948 | result_head = '<a href="%s"%s>' % (url, title_str)
949 | result = '%s%s</a>' % (result_head, link_text)
950 | # <img> allowed from curr_pos on, <a> from
951 | # anchor_allowed_pos on.
952 | curr_pos = start_idx + len(result_head)
953 | anchor_allowed_pos = start_idx + len(result)
954 | text = text[:start_idx] + result + text[match.end():]
955 | else:
956 | # Anchor not allowed here.
957 | curr_pos = start_idx + 1
958 | continue
959 |
960 | # Reference anchor or img?
961 | else:
962 | match = self._tail_of_reference_link_re.match(text, p)
963 | if match:
964 | # Handle a reference-style anchor or img.
965 | is_img = start_idx > 0 and text[start_idx - 1] == "!"
966 | if is_img:
967 | start_idx -= 1
968 | link_id = match.group("id").lower()
969 | if not link_id:
970 | link_id = link_text.lower() # for links like [this][]
971 | if link_id in self.urls:
972 | url = self.urls[link_id]
973 | # We've got to encode these to avoid conflicting
974 | # with italics/bold.
975 | url = url.replace('*', g_escape_table['*']) \
976 | .replace('_', g_escape_table['_'])
977 | title = self.titles.get(link_id)
978 | if title:
979 | title = title.replace('*', g_escape_table['*']) \
980 | .replace('_', g_escape_table['_'])
981 | title_str = ' title="%s"' % title
982 | else:
983 | title_str = ''
984 | if is_img:
985 | result = '<img src="%s" alt="%s"%s%s' \
986 | % (url, link_text.replace('"', '"'),
987 | title_str, self.empty_element_suffix)
988 | curr_pos = start_idx + len(result)
989 | text = text[:start_idx] + \
990 | result + text[match.end():]
991 | elif start_idx >= anchor_allowed_pos:
992 | result = '<a href="%s"%s>%s</a>' \
993 | % (url, title_str, link_text)
994 | result_head = '<a href="%s"%s>' % (url, title_str)
995 | result = '%s%s</a>' % (result_head, link_text)
996 | # <img> allowed from curr_pos on, <a> from
997 | # anchor_allowed_pos on.
998 | curr_pos = start_idx + len(result_head)
999 | anchor_allowed_pos = start_idx + len(result)
1000 | text = text[:start_idx] + \
1001 | result + text[match.end():]
1002 | else:
1003 | # Anchor not allowed here.
1004 | curr_pos = start_idx + 1
1005 | else:
1006 | # This id isn't defined, leave the markup alone.
1007 | curr_pos = match.end()
1008 | continue
1009 |
1010 | # Otherwise, it isn't markup.
1011 | curr_pos = start_idx + 1
1012 |
1013 | return text
1014 |
1015 | _setext_h_re = re.compile(r'^(.+)[ \t]*\n(=+|-+)[ \t]*\n+', re.M)
1016 |
1017 | def _setext_h_sub(self, match):
1018 | n = {"=": 1, "-": 2}[match.group(2)[0]]
1019 | demote_headers = self.extras.get("demote-headers")
1020 | if demote_headers:
1021 | n = min(n + demote_headers, 6)
1022 | return "<h%d>%s</h%d>\n\n" \
1023 | % (n, self._run_span_gamut(match.group(1)), n)
1024 |
1025 | _atx_h_re = re.compile(r'''
1026 | ^(\#{1,6}) # \1 = string of #'s
1027 | [ \t]*
1028 | (.+?) # \2 = Header text
1029 | [ \t]*
1030 | (?<!\\) # ensure not an escaped trailing '#'
1031 | \#* # optional closing #'s (not counted)
1032 | \n+
1033 | ''', re.X | re.M)
1034 |
1035 | def _atx_h_sub(self, match):
1036 | n = len(match.group(1))
1037 | demote_headers = self.extras.get("demote-headers")
1038 | if demote_headers:
1039 | n = min(n + demote_headers, 6)
1040 | return "<h%d>%s</h%d>\n\n" \
1041 | % (n, self._run_span_gamut(match.group(2)), n)
1042 |
1043 | def _do_headers(self, text):
1044 | # Setext-style headers:
1045 | # Header 1
1046 | # ========
1047 | #
1048 | # Header 2
1049 | # --------
1050 | text = self._setext_h_re.sub(self._setext_h_sub, text)
1051 |
1052 | # atx-style headers:
1053 | # # Header 1
1054 | # ## Header 2
1055 | # ## Header 2 with closing hashes ##
1056 | # ...
1057 | # ###### Header 6
1058 | text = self._atx_h_re.sub(self._atx_h_sub, text)
1059 |
1060 | return text
1061 |
1062 | _marker_ul_chars = '*+-'
1063 | _marker_any = r'(?:[%s]|\d+\.)' % _marker_ul_chars
1064 | _marker_ul = '(?:[%s])' % _marker_ul_chars
1065 | _marker_ol = r'(?:\d+\.)'
1066 |
1067 | def _list_sub(self, match):
1068 | lst = match.group(1)
1069 | lst_type = match.group(3) in self._marker_ul_chars and "ul" or "ol"
1070 | result = self._process_list_items(lst)
1071 | if self.list_level:
1072 | return "<%s>\n%s</%s>\n" % (lst_type, result, lst_type)
1073 | else:
1074 | return "<%s>\n%s</%s>\n\n" % (lst_type, result, lst_type)
1075 |
1076 | def _do_lists(self, text):
1077 | # Form HTML ordered (numbered) and unordered (bulleted) lists.
1078 |
1079 | for marker_pat in (self._marker_ul, self._marker_ol):
1080 | # Re-usable pattern to match any entire ul or ol list:
1081 | less_than_tab = self.tab_width - 1
1082 | whole_list = r'''
1083 | ( # \1 = whole list
1084 | ( # \2
1085 | [ ]{0,%d}
1086 | (%s) # \3 = first list item marker
1087 | [ \t]+
1088 | )
1089 | (?:.+?)
1090 | ( # \4
1091 | \Z
1092 | |
1093 | \n{2,}
1094 | (?=\S)
1095 | (?! # Negative lookahead for another list item marker
1096 | [ \t]*
1097 | %s[ \t]+
1098 | )
1099 | )
1100 | )
1101 | ''' % (less_than_tab, marker_pat, marker_pat)
1102 |
1103 | # We use a different prefix before nested lists than top-level lists.
1104 | # See extended comment in _process_list_items().
1105 | #
1106 | # Note: There's a bit of duplication here. My original implementation
1107 | # created a scalar regex pattern as the conditional result of the test on
1108 | # $g_list_level, and then only ran the $text =~ s{...}{...}egmx
1109 | # substitution once, using the scalar as the pattern. This worked,
1110 | # everywhere except when running under MT on my hosting account at Pair
1111 | # Networks. There, this caused all rebuilds to be killed by the reaper (or
1112 | # perhaps they crashed, but that seems incredibly unlikely given that the
1113 | # same script on the same server ran fine *except* under MT. I've spent
1114 | # more time trying to figure out why this is happening than I'd like to
1115 | # admit. My only guess, backed up by the fact that this workaround works,
1116 | # is that Perl optimizes the substition when it can figure out that the
1117 | # pattern will never change, and when this optimization isn't on, we run
1118 | # afoul of the reaper. Thus, the slightly redundant code to that uses two
1119 | # static s/// patterns rather than one conditional pattern.
1120 |
1121 | if self.list_level:
1122 | sub_list_re = re.compile("^" + whole_list, re.X | re.M | re.S)
1123 | text = sub_list_re.sub(self._list_sub, text)
1124 | else:
1125 | list_re = re.compile(r"(?:(?<=\n\n)|\A\n?)" + whole_list,
1126 | re.X | re.M | re.S)
1127 | text = list_re.sub(self._list_sub, text)
1128 |
1129 | return text
1130 |
1131 | _list_item_re = re.compile(r'''
1132 | (\n)? # leading line = \1
1133 | (^[ \t]*) # leading whitespace = \2
1134 | (%s) [ \t]+ # list marker = \3
1135 | ((?:.+?) # list item text = \4
1136 | (\n{1,2})) # eols = \5
1137 | (?= \n* (\Z | \2 (%s) [ \t]+))
1138 | ''' % (_marker_any, _marker_any),
1139 | re.M | re.X | re.S)
1140 |
1141 | _last_li_endswith_two_eols = False
1142 |
1143 | def _list_item_sub(self, match):
1144 | item = match.group(4)
1145 | leading_line = match.group(1)
1146 | leading_space = match.group(2)
1147 | if leading_line or "\n\n" in item or self._last_li_endswith_two_eols:
1148 | item = self._run_block_gamut(self._outdent(item))
1149 | else:
1150 | # Recursion for sub-lists:
1151 | item = self._do_lists(self._outdent(item))
1152 | if item.endswith('\n'):
1153 | item = item[:-1]
1154 | item = self._run_span_gamut(item)
1155 | self._last_li_endswith_two_eols = (len(match.group(5)) == 2)
1156 | return "<li>%s</li>\n" % item
1157 |
1158 | def _process_list_items(self, list_str):
1159 | # Process the contents of a single ordered or unordered list,
1160 | # splitting it into individual list items.
1161 |
1162 | # The $g_list_level global keeps track of when we're inside a list.
1163 | # Each time we enter a list, we increment it; when we leave a list,
1164 | # we decrement. If it's zero, we're not in a list anymore.
1165 | #
1166 | # We do this because when we're not inside a list, we want to treat
1167 | # something like this:
1168 | #
1169 | # I recommend upgrading to version
1170 | # 8. Oops, now this line is treated
1171 | # as a sub-list.
1172 | #
1173 | # As a single paragraph, despite the fact that the second line starts
1174 | # with a digit-period-space sequence.
1175 | #
1176 | # Whereas when we're inside a list (or sub-list), that line will be
1177 | # treated as the start of a sub-list. What a kludge, huh? This is
1178 | # an aspect of Markdown's syntax that's hard to parse perfectly
1179 | # without resorting to mind-reading. Perhaps the solution is to
1180 | # change the syntax rules such that sub-lists must start with a
1181 | # starting cardinal number; e.g. "1." or "a.".
1182 | self.list_level += 1
1183 | self._last_li_endswith_two_eols = False
1184 | list_str = list_str.rstrip('\n') + '\n'
1185 | list_str = self._list_item_re.sub(self._list_item_sub, list_str)
1186 | self.list_level -= 1
1187 | return list_str
1188 |
1189 | def _get_pygments_lexer(self, lexer_name):
1190 | try:
1191 | from pygments import lexers, util
1192 | except ImportError:
1193 | return None
1194 | try:
1195 | return lexers.get_lexer_by_name(lexer_name)
1196 | except util.ClassNotFound:
1197 | return None
1198 |
1199 | def _color_with_pygments(self, codeblock, lexer, **formatter_opts):
1200 | import pygments
1201 | import pygments.formatters
1202 |
1203 | class HtmlCodeFormatter(pygments.formatters.HtmlFormatter):
1204 |
1205 | def _wrap_code(self, inner):
1206 | """A function for use in a Pygments Formatter which
1207 | wraps in <code> tags.
1208 | """
1209 | yield 0, "<code>"
1210 | for tup in inner:
1211 | yield tup
1212 | yield 0, "</code>"
1213 |
1214 | def wrap(self, source, outfile):
1215 | """Return the source with a code, pre, and div."""
1216 | return self._wrap_div(self._wrap_pre(self._wrap_code(source)))
1217 |
1218 | formatter = HtmlCodeFormatter(cssclass="codehilite", **formatter_opts)
1219 | return pygments.highlight(codeblock, lexer, formatter)
1220 |
1221 | def _code_block_sub(self, match):
1222 | codeblock = match.group(1)
1223 | codeblock = self._outdent(codeblock)
1224 | codeblock = self._detab(codeblock)
1225 | codeblock = codeblock.lstrip('\n') # trim leading newlines
1226 | codeblock = codeblock.rstrip() # trim trailing whitespace
1227 |
1228 | if "code-color" in self.extras and codeblock.startswith(":::"):
1229 | lexer_name, rest = codeblock.split('\n', 1)
1230 | lexer_name = lexer_name[3:].strip()
1231 | lexer = self._get_pygments_lexer(lexer_name)
1232 | codeblock = rest.lstrip("\n") # Remove lexer declaration line.
1233 | if lexer:
1234 | formatter_opts = self.extras['code-color'] or {}
1235 | colored = self._color_with_pygments(codeblock, lexer,
1236 | **formatter_opts)
1237 | return "\n\n%s\n\n" % colored
1238 |
1239 | codeblock = self._encode_code(codeblock)
1240 | return "\n\n<pre><code>%s\n</code></pre>\n\n" % codeblock
1241 |
1242 | def _do_code_blocks(self, text):
1243 | """Process Markdown `<pre><code>` blocks."""
1244 | code_block_re = re.compile(r'''
1245 | (?:\n\n|\A)
1246 | ( # $1 = the code block -- one or more lines, starting with a space/tab
1247 | (?:
1248 | (?:[ ]{%d} | \t) # Lines must start with a tab or a tab-width of spaces
1249 | .*\n+
1250 | )+
1251 | )
1252 | ((?=^[ ]{0,%d}\S)|\Z) # Lookahead for non-space at line-start, or end of doc
1253 | ''' % (self.tab_width, self.tab_width),
1254 | re.M | re.X)
1255 |
1256 | return code_block_re.sub(self._code_block_sub, text)
1257 |
1258 | # Rules for a code span:
1259 | # - backslash escapes are not interpreted in a code span
1260 | # - to include one or or a run of more backticks the delimiters must
1261 | # be a longer run of backticks
1262 | # - cannot start or end a code span with a backtick; pad with a
1263 | # space and that space will be removed in the emitted HTML
1264 | # See `test/tm-cases/escapes.text` for a number of edge-case
1265 | # examples.
1266 | _code_span_re = re.compile(r'''
1267 | (?<!\\)
1268 | (`+) # \1 = Opening run of `
1269 | (?!`) # See Note A test/tm-cases/escapes.text
1270 | (.+?) # \2 = The code block
1271 | (?<!`)
1272 | \1 # Matching closer
1273 | (?!`)
1274 | ''', re.X | re.S)
1275 |
1276 | def _code_span_sub(self, match):
1277 | c = match.group(2).strip(" \t")
1278 | c = self._encode_code(c)
1279 | return "<code>%s</code>" % c
1280 |
1281 | def _do_code_spans(self, text):
1282 | # * Backtick quotes are used for <code></code> spans.
1283 | #
1284 | # * You can use multiple backticks as the delimiters if you want to
1285 | # include literal backticks in the code span. So, this input:
1286 | #
1287 | # Just type ``foo `bar` baz`` at the prompt.
1288 | #
1289 | # Will translate to:
1290 | #
1291 | # <p>Just type <code>foo `bar` baz</code> at the prompt.</p>
1292 | #
1293 | # There's no arbitrary limit to the number of backticks you
1294 | # can use as delimters. If you need three consecutive backticks
1295 | # in your code, use four for delimiters, etc.
1296 | #
1297 | # * You can use spaces to get literal backticks at the edges:
1298 | #
1299 | # ... type `` `bar` `` ...
1300 | #
1301 | # Turns to:
1302 | #
1303 | # ... type <code>`bar`</code> ...
1304 | return self._code_span_re.sub(self._code_span_sub, text)
1305 |
1306 | def _encode_code(self, text):
1307 | """Encode/escape certain characters inside Markdown code runs.
1308 | The point is that in code, these characters are literals,
1309 | and lose their special Markdown meanings.
1310 | """
1311 | replacements = [
1312 | # Encode all ampersands; HTML entities are not
1313 | # entities within a Markdown code span.
1314 | ('&', '&'),
1315 | # Do the angle bracket song and dance:
1316 | ('<', '<'),
1317 | ('>', '>'),
1318 | # Now, escape characters that are magic in Markdown:
1319 | ('*', g_escape_table['*']),
1320 | ('_', g_escape_table['_']),
1321 | ('{', g_escape_table['{']),
1322 | ('}', g_escape_table['}']),
1323 | ('[', g_escape_table['[']),
1324 | (']', g_escape_table[']']),
1325 | ('\\', g_escape_table['\\']),
1326 | ]
1327 | for before, after in replacements:
1328 | text = text.replace(before, after)
1329 | return text
1330 |
1331 | _strong_re = re.compile(r"(\*\*|__)(?=\S)(.+?[*_]*)(?<=\S)\1", re.S)
1332 | _em_re = re.compile(r"(\*|_)(?=\S)(.+?)(?<=\S)\1", re.S)
1333 | _code_friendly_strong_re = re.compile(
1334 | r"\*\*(?=\S)(.+?[*_]*)(?<=\S)\*\*", re.S)
1335 | _code_friendly_em_re = re.compile(r"\*(?=\S)(.+?)(?<=\S)\*", re.S)
1336 |
1337 | def _do_italics_and_bold(self, text):
1338 | # <strong> must go first:
1339 | if "code-friendly" in self.extras:
1340 | text = self._code_friendly_strong_re.sub(
1341 | r"<strong>\1</strong>", text)
1342 | text = self._code_friendly_em_re.sub(r"<em>\1</em>", text)
1343 | else:
1344 | text = self._strong_re.sub(r"<strong>\2</strong>", text)
1345 | text = self._em_re.sub(r"<em>\2</em>", text)
1346 | return text
1347 |
1348 | _block_quote_re = re.compile(r'''
1349 | ( # Wrap whole match in \1
1350 | (
1351 | ^[ \t]*>[ \t]? # '>' at the start of a line
1352 | .+\n # rest of the first line
1353 | (.+\n)* # subsequent consecutive lines
1354 | \n* # blanks
1355 | )+
1356 | )
1357 | ''', re.M | re.X)
1358 | _bq_one_level_re = re.compile('^[ \t]*>[ \t]?', re.M)
1359 |
1360 | _html_pre_block_re = re.compile(r'(\s*<pre>.+?</pre>)', re.S)
1361 |
1362 | def _dedent_two_spaces_sub(self, match):
1363 | return re.sub(r'(?m)^ ', '', match.group(1))
1364 |
1365 | def _block_quote_sub(self, match):
1366 | bq = match.group(1)
1367 | bq = self._bq_one_level_re.sub('', bq) # trim one level of quoting
1368 | bq = self._ws_only_line_re.sub('', bq) # trim whitespace-only lines
1369 | bq = self._run_block_gamut(bq) # recurse
1370 |
1371 | bq = re.sub('(?m)^', ' ', bq)
1372 | # These leading spaces screw with <pre> content, so we need to fix
1373 | # that:
1374 | bq = self._html_pre_block_re.sub(self._dedent_two_spaces_sub, bq)
1375 |
1376 | return "<blockquote>\n%s\n</blockquote>\n\n" % bq
1377 |
1378 | def _do_block_quotes(self, text):
1379 | if '>' not in text:
1380 | return text
1381 | return self._block_quote_re.sub(self._block_quote_sub, text)
1382 |
1383 | def _form_paragraphs(self, text):
1384 | # Strip leading and trailing lines:
1385 | text = text.strip('\n')
1386 |
1387 | # Wrap <p> tags.
1388 | grafs = re.split(r"\n{2,}", text)
1389 | for i, graf in enumerate(grafs):
1390 | if graf in self.html_blocks:
1391 | # Unhashify HTML blocks
1392 | grafs[i] = self.html_blocks[graf]
1393 | else:
1394 | # Wrap <p> tags.
1395 | graf = self._run_span_gamut(graf)
1396 | grafs[i] = "<p>" + graf.lstrip(" \t") + "</p>"
1397 |
1398 | return "\n\n".join(grafs)
1399 |
1400 | def _add_footnotes(self, text):
1401 | if self.footnotes:
1402 | footer = [
1403 | '<div class="footnotes">',
1404 | '<hr' + self.empty_element_suffix,
1405 | '<ol>',
1406 | ]
1407 | for i, id in enumerate(self.footnote_ids):
1408 | if i != 0:
1409 | footer.append('')
1410 | footer.append('<li id="fn-%s">' % id)
1411 | footer.append(self._run_block_gamut(self.footnotes[id]))
1412 | backlink = ('<a href="#fnref-%s" '
1413 | 'class="footnoteBackLink" '
1414 | 'title="Jump back to footnote %d in the text.">'
1415 | '↩</a>' % (id, i + 1))
1416 | if footer[-1].endswith("</p>"):
1417 | footer[-1] = footer[-1][:-len("</p>")] \
1418 | + ' ' + backlink + "</p>"
1419 | else:
1420 | footer.append("\n<p>%s</p>" % backlink)
1421 | footer.append('</li>')
1422 | footer.append('</ol>')
1423 | footer.append('</div>')
1424 | return text + '\n\n' + '\n'.join(footer)
1425 | else:
1426 | return text
1427 |
1428 | # Ampersand-encoding based entirely on Nat Irons's Amputator MT plugin:
1429 | # http://bumppo.net/projects/amputator/
1430 | _ampersand_re = re.compile(r'&(?!#?[xX]?(?:[0-9a-fA-F]+|\w+);)')
1431 | _naked_lt_re = re.compile(r'<(?![a-z/?\$!])', re.I)
1432 | _naked_gt_re = re.compile(r'''(?<![a-z?!/'"-])>''', re.I)
1433 |
1434 | def _encode_amps_and_angles(self, text):
1435 | # Smart processing for ampersands and angle brackets that need
1436 | # to be encoded.
1437 | text = self._ampersand_re.sub('&', text)
1438 |
1439 | # Encode naked <'s
1440 | text = self._naked_lt_re.sub('<', text)
1441 |
1442 | # Encode naked >'s
1443 | # Note: Other markdown implementations (e.g. Markdown.pl, PHP
1444 | # Markdown) don't do this.
1445 | text = self._naked_gt_re.sub('>', text)
1446 | return text
1447 |
1448 | def _encode_backslash_escapes(self, text):
1449 | for ch, escape in g_escape_table.items():
1450 | text = text.replace("\\" + ch, escape)
1451 | return text
1452 |
1453 | _auto_link_re = re.compile(r'<((https?|ftp):[^\'">\s]+)>', re.I)
1454 |
1455 | def _auto_link_sub(self, match):
1456 | g1 = match.group(1)
1457 | return '<a href="%s">%s</a>' % (g1, g1)
1458 |
1459 | _auto_email_link_re = re.compile(r"""
1460 | <
1461 | (?:mailto:)?
1462 | (
1463 | [-.\w]+
1464 | \@
1465 | [-\w]+(\.[-\w]+)*\.[a-z]+
1466 | )
1467 | >
1468 | """, re.I | re.X | re.U)
1469 |
1470 | def _auto_email_link_sub(self, match):
1471 | return self._encode_email_address(
1472 | self._unescape_special_chars(match.group(1)))
1473 |
1474 | def _do_auto_links(self, text):
1475 | text = self._auto_link_re.sub(self._auto_link_sub, text)
1476 | text = self._auto_email_link_re.sub(self._auto_email_link_sub, text)
1477 | return text
1478 |
1479 | def _encode_email_address(self, addr):
1480 | # Input: an email address, e.g. "foo@example.com"
1481 | #
1482 | # Output: the email address as a mailto link, with each character
1483 | # of the address encoded as either a decimal or hex entity, in
1484 | # the hopes of foiling most address harvesting spam bots. E.g.:
1485 | #
1486 | # <a href="mailto:foo@e
1487 | # xample.com">foo
1488 | # @example.com</a>
1489 | #
1490 | # Based on a filter by Matthew Wickline, posted to the BBEdit-Talk
1491 | # mailing list: <http://tinyurl.com/yu7ue>
1492 | chars = [_xml_encode_email_char_at_random(ch)
1493 | for ch in "mailto:" + addr]
1494 | # Strip the mailto: from the visible part.
1495 | addr = '<a href="%s">%s</a>' \
1496 | % (''.join(chars), ''.join(chars[7:]))
1497 | return addr
1498 |
1499 | def _do_link_patterns(self, text):
1500 | """Caveat emptor: there isn't much guarding against link
1501 | patterns being formed inside other standard Markdown links, e.g.
1502 | inside a [link def][like this].
1503 |
1504 | Dev Notes: *Could* consider prefixing regexes with a negative
1505 | lookbehind assertion to attempt to guard against this.
1506 | """
1507 | link_from_hash = {}
1508 | for regex, repl in self.link_patterns:
1509 | replacements = []
1510 | for match in regex.finditer(text):
1511 | if hasattr(repl, "__call__"):
1512 | href = repl(match)
1513 | else:
1514 | href = match.expand(repl)
1515 | replacements.append((match.span(), href))
1516 | for (start, end), href in reversed(replacements):
1517 | escaped_href = (
1518 | href.replace('"', '"') # b/c of attr quote
1519 | # To avoid markdown <em> and <strong>:
1520 | .replace('*', g_escape_table['*'])
1521 | .replace('_', g_escape_table['_']))
1522 | link = '<a href="%s">%s</a>' % (escaped_href, text[start:end])
1523 | hash = md5(link).hexdigest()
1524 | link_from_hash[hash] = link
1525 | text = text[:start] + hash + text[end:]
1526 | for hash, link in link_from_hash.items():
1527 | text = text.replace(hash, link)
1528 | return text
1529 |
1530 | def _unescape_special_chars(self, text):
1531 | # Swap back in all the special characters we've hidden.
1532 | for ch, hash in g_escape_table.items():
1533 | text = text.replace(hash, ch)
1534 | return text
1535 |
1536 | def _outdent(self, text):
1537 | # Remove one level of line-leading tabs or spaces
1538 | return self._outdent_re.sub('', text)
1539 |
1540 |
1541 | class MarkdownWithExtras(Markdown):
1542 | """A markdowner class that enables most extras:
1543 |
1544 | - footnotes
1545 | - code-color (only has effect if 'pygments' Python module on path)
1546 |
1547 | These are not included:
1548 | - pyshell (specific to Python-related documenting)
1549 | - code-friendly (because it *disables* part of the syntax)
1550 | - link-patterns (because you need to specify some actual
1551 | link-patterns anyway)
1552 | """
1553 | extras = ["footnotes", "code-color"]
1554 |
1555 |
1556 | #---- internal support functions
1557 |
1558 | # From http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52549
1559 | def _curry(*args, **kwargs):
1560 | function, args = args[0], args[1:]
1561 |
1562 | def result(*rest, **kwrest):
1563 | combined = kwargs.copy()
1564 | combined.update(kwrest)
1565 | return function(*args + rest, **combined)
1566 | return result
1567 |
1568 | # Recipe: regex_from_encoded_pattern (1.0)
1569 |
1570 |
1571 | def _regex_from_encoded_pattern(s):
1572 | """'foo' -> re.compile(re.escape('foo'))
1573 | '/foo/' -> re.compile('foo')
1574 | '/foo/i' -> re.compile('foo', re.I)
1575 | """
1576 | if s.startswith('/') and s.rfind('/') != 0:
1577 | # Parse it: /PATTERN/FLAGS
1578 | idx = s.rfind('/')
1579 | pattern, flags_str = s[1:idx], s[idx + 1:]
1580 | flag_from_char = {
1581 | "i": re.IGNORECASE,
1582 | "l": re.LOCALE,
1583 | "s": re.DOTALL,
1584 | "m": re.MULTILINE,
1585 | "u": re.UNICODE,
1586 | }
1587 | flags = 0
1588 | for char in flags_str:
1589 | try:
1590 | flags |= flag_from_char[char]
1591 | except KeyError:
1592 | raise ValueError("unsupported regex flag: '%s' in '%s' "
1593 | "(must be one of '%s')"
1594 | % (char, s, ''.join(flag_from_char.keys())))
1595 | return re.compile(s[1:idx], flags)
1596 | else: # not an encoded regex
1597 | return re.compile(re.escape(s))
1598 |
1599 | # Recipe: dedent (0.1.2)
1600 |
1601 |
1602 | def _dedentlines(lines, tabsize=8, skip_first_line=False):
1603 | """_dedentlines(lines, tabsize=8, skip_first_line=False) -> dedented lines
1604 |
1605 | "lines" is a list of lines to dedent.
1606 | "tabsize" is the tab width to use for indent width calculations.
1607 | "skip_first_line" is a boolean indicating if the first line should
1608 | be skipped for calculating the indent width and for dedenting.
1609 | This is sometimes useful for docstrings and similar.
1610 |
1611 | Same as dedent() except operates on a sequence of lines. Note: the
1612 | lines list is modified **in-place**.
1613 | """
1614 | DEBUG = False
1615 | if DEBUG:
1616 | print "dedent: dedent(..., tabsize=%d, skip_first_line=%r)"\
1617 | % (tabsize, skip_first_line)
1618 | indents = []
1619 | margin = None
1620 | for i, line in enumerate(lines):
1621 | if i == 0 and skip_first_line:
1622 | continue
1623 | indent = 0
1624 | for ch in line:
1625 | if ch == ' ':
1626 | indent += 1
1627 | elif ch == '\t':
1628 | indent += tabsize - (indent % tabsize)
1629 | elif ch in '\r\n':
1630 | continue # skip all-whitespace lines
1631 | else:
1632 | break
1633 | else:
1634 | continue # skip all-whitespace lines
1635 | if DEBUG:
1636 | print "dedent: indent=%d: %r" % (indent, line)
1637 | if margin is None:
1638 | margin = indent
1639 | else:
1640 | margin = min(margin, indent)
1641 | if DEBUG:
1642 | print "dedent: margin=%r" % margin
1643 |
1644 | if margin is not None and margin > 0:
1645 | for i, line in enumerate(lines):
1646 | if i == 0 and skip_first_line:
1647 | continue
1648 | removed = 0
1649 | for j, ch in enumerate(line):
1650 | if ch == ' ':
1651 | removed += 1
1652 | elif ch == '\t':
1653 | removed += tabsize - (removed % tabsize)
1654 | elif ch in '\r\n':
1655 | if DEBUG:
1656 | print "dedent: %r: EOL -> strip up to EOL" % line
1657 | lines[i] = lines[i][j:]
1658 | break
1659 | else:
1660 | raise ValueError("unexpected non-whitespace char %r in "
1661 | "line %r while removing %d-space margin"
1662 | % (ch, line, margin))
1663 | if DEBUG:
1664 | print "dedent: %r: %r -> removed %d/%d"\
1665 | % (line, ch, removed, margin)
1666 | if removed == margin:
1667 | lines[i] = lines[i][j + 1:]
1668 | break
1669 | elif removed > margin:
1670 | lines[i] = ' ' * (removed - margin) + lines[i][j + 1:]
1671 | break
1672 | else:
1673 | if removed:
1674 | lines[i] = lines[i][removed:]
1675 | return lines
1676 |
1677 |
1678 | def _dedent(text, tabsize=8, skip_first_line=False):
1679 | """_dedent(text, tabsize=8, skip_first_line=False) -> dedented text
1680 |
1681 | "text" is the text to dedent.
1682 | "tabsize" is the tab width to use for indent width calculations.
1683 | "skip_first_line" is a boolean indicating if the first line should
1684 | be skipped for calculating the indent width and for dedenting.
1685 | This is sometimes useful for docstrings and similar.
1686 |
1687 | textwrap.dedent(s), but don't expand tabs to spaces
1688 | """
1689 | lines = text.splitlines(1)
1690 | _dedentlines(lines, tabsize=tabsize, skip_first_line=skip_first_line)
1691 | return ''.join(lines)
1692 |
1693 |
1694 | class _memoized(object):
1695 | """Decorator that caches a function's return value each time it is called.
1696 | If called later with the same arguments, the cached value is returned, and
1697 | not re-evaluated.
1698 |
1699 | http://wiki.python.org/moin/PythonDecoratorLibrary
1700 | """
1701 |
1702 | def __init__(self, func):
1703 | self.func = func
1704 | self.cache = {}
1705 |
1706 | def __call__(self, *args):
1707 | try:
1708 | return self.cache[args]
1709 | except KeyError:
1710 | self.cache[args] = value = self.func(*args)
1711 | return value
1712 | except TypeError:
1713 | # uncachable -- for instance, passing a list as an argument.
1714 | # Better to not cache than to blow up entirely.
1715 | return self.func(*args)
1716 |
1717 | def __repr__(self):
1718 | """Return the function's docstring."""
1719 | return self.func.__doc__
1720 |
1721 |
1722 | def _xml_oneliner_re_from_tab_width(tab_width):
1723 | """Standalone XML processing instruction regex."""
1724 | return re.compile(r"""
1725 | (?:
1726 | (?<=\n\n) # Starting after a blank line
1727 | | # or
1728 | \A\n? # the beginning of the doc
1729 | )
1730 | ( # save in $1
1731 | [ ]{0,%d}
1732 | (?:
1733 | <\?\w+\b\s+.*?\?> # XML processing instruction
1734 | |
1735 | <\w+:\w+\b\s+.*?/> # namespaced single tag
1736 | )
1737 | [ \t]*
1738 | (?=\n{2,}|\Z) # followed by a blank line or end of document
1739 | )
1740 | """ % (tab_width - 1), re.X)
1741 | _xml_oneliner_re_from_tab_width = _memoized(_xml_oneliner_re_from_tab_width)
1742 |
1743 |
1744 | def _hr_tag_re_from_tab_width(tab_width):
1745 | return re.compile(r"""
1746 | (?:
1747 | (?<=\n\n) # Starting after a blank line
1748 | | # or
1749 | \A\n? # the beginning of the doc
1750 | )
1751 | ( # save in \1
1752 | [ ]{0,%d}
1753 | <(hr) # start tag = \2
1754 | \b # word break
1755 | ([^<>])*? #
1756 | /?> # the matching end tag
1757 | [ \t]*
1758 | (?=\n{2,}|\Z) # followed by a blank line or end of document
1759 | )
1760 | """ % (tab_width - 1), re.X)
1761 | _hr_tag_re_from_tab_width = _memoized(_hr_tag_re_from_tab_width)
1762 |
1763 |
1764 | def _xml_encode_email_char_at_random(ch):
1765 | r = random()
1766 | # Roughly 10% raw, 45% hex, 45% dec.
1767 | # '@' *must* be encoded. I [John Gruber] insist.
1768 | # Issue 26: '_' must be encoded.
1769 | if r > 0.9 and ch not in "@_":
1770 | return ch
1771 | elif r < 0.45:
1772 | # The [1:] is to drop leading '0': 0x63 -> x63
1773 | return '&#%s;' % hex(ord(ch))[1:]
1774 | else:
1775 | return '&#%s;' % ord(ch)
1776 |
1777 |
1778 | def _hash_text(text):
1779 | return 'md5:' + md5(text.encode("utf-8")).hexdigest()
1780 |
1781 |
1782 | #---- mainline
1783 |
1784 | class _NoReflowFormatter(optparse.IndentedHelpFormatter):
1785 | """An optparse formatter that does NOT reflow the description."""
1786 |
1787 | def format_description(self, description):
1788 | return description or ""
1789 |
1790 |
1791 | def _test():
1792 | import doctest
1793 | doctest.testmod()
1794 |
1795 |
1796 | def main(argv=None):
1797 | if argv is None:
1798 | argv = sys.argv
1799 | if not logging.root.handlers:
1800 | logging.basicConfig()
1801 |
1802 | usage = "usage: %prog [PATHS...]"
1803 | version = "%prog " + __version__
1804 | parser = optparse.OptionParser(prog="markdown2", usage=usage,
1805 | version=version, description=cmdln_desc,
1806 | formatter=_NoReflowFormatter())
1807 | parser.add_option("-v", "--verbose", dest="log_level",
1808 | action="store_const", const=logging.DEBUG,
1809 | help="more verbose output")
1810 | parser.add_option("--encoding",
1811 | help="specify encoding of text content")
1812 | parser.add_option("--html4tags", action="store_true", default=False,
1813 | help="use HTML 4 style for empty element tags")
1814 | parser.add_option("-s", "--safe", metavar="MODE", dest="safe_mode",
1815 | help="sanitize literal HTML: 'escape' escapes "
1816 | "HTML meta chars, 'replace' replaces with an "
1817 | "[HTML_REMOVED] note")
1818 | parser.add_option("-x", "--extras", action="append",
1819 | help="Turn on specific extra features (not part of "
1820 | "the core Markdown spec). Supported values: "
1821 | "'code-friendly' disables _/__ for emphasis; "
1822 | "'code-color' adds code-block syntax coloring; "
1823 | "'link-patterns' adds auto-linking based on patterns; "
1824 | "'footnotes' adds the footnotes syntax;"
1825 | "'xml' passes one-liner processing instructions and namespaced XML tags;"
1826 | "'pyshell' to put unindented Python interactive shell sessions in a <code> block.")
1827 | parser.add_option("--use-file-vars",
1828 | help="Look for and use Emacs-style 'markdown-extras' "
1829 | "file var to turn on extras. See "
1830 | "<http://code.google.com/p/python-markdown2/wiki/Extras>.")
1831 | parser.add_option("--link-patterns-file",
1832 | help="path to a link pattern file")
1833 | parser.add_option("--self-test", action="store_true",
1834 | help="run internal self-tests (some doctests)")
1835 | parser.add_option("--compare", action="store_true",
1836 | help="run against Markdown.pl as well (for testing)")
1837 | parser.set_defaults(log_level=logging.INFO, compare=False,
1838 | encoding="utf-8", safe_mode=None, use_file_vars=False)
1839 | opts, paths = parser.parse_args()
1840 | log.setLevel(opts.log_level)
1841 |
1842 | if opts.self_test:
1843 | return _test()
1844 |
1845 | if opts.extras:
1846 | extras = {}
1847 | for s in opts.extras:
1848 | splitter = re.compile("[,;: ]+")
1849 | for e in splitter.split(s):
1850 | if '=' in e:
1851 | ename, earg = e.split('=', 1)
1852 | try:
1853 | earg = int(earg)
1854 | except ValueError:
1855 | pass
1856 | else:
1857 | ename, earg = e, None
1858 | extras[ename] = earg
1859 | else:
1860 | extras = None
1861 |
1862 | if opts.link_patterns_file:
1863 | link_patterns = []
1864 | f = open(opts.link_patterns_file)
1865 | try:
1866 | for i, line in enumerate(f.readlines()):
1867 | if not line.strip():
1868 | continue
1869 | if line.lstrip().startswith("#"):
1870 | continue
1871 | try:
1872 | pat, href = line.rstrip().rsplit(None, 1)
1873 | except ValueError:
1874 | raise MarkdownError("%s:%d: invalid link pattern line: %r"
1875 | % (opts.link_patterns_file, i + 1, line))
1876 | link_patterns.append(
1877 | (_regex_from_encoded_pattern(pat), href))
1878 | finally:
1879 | f.close()
1880 | else:
1881 | link_patterns = None
1882 |
1883 | from os.path import join, dirname, abspath, exists
1884 | markdown_pl = join(dirname(dirname(abspath(__file__))), "test",
1885 | "Markdown.pl")
1886 | for path in paths:
1887 | if opts.compare:
1888 | print "==== Markdown.pl ===="
1889 | perl_cmd = 'perl %s "%s"' % (markdown_pl, path)
1890 | o = os.popen(perl_cmd)
1891 | perl_html = o.read()
1892 | o.close()
1893 | sys.stdout.write(perl_html)
1894 | print "==== markdown2.py ===="
1895 | html = markdown_path(path, encoding=opts.encoding,
1896 | html4tags=opts.html4tags,
1897 | safe_mode=opts.safe_mode,
1898 | extras=extras, link_patterns=link_patterns,
1899 | use_file_vars=opts.use_file_vars)
1900 | sys.stdout.write(
1901 | html.encode(sys.stdout.encoding or "utf-8", 'xmlcharrefreplace'))
1902 | if opts.compare:
1903 | test_dir = join(dirname(dirname(abspath(__file__))), "test")
1904 | if exists(join(test_dir, "test_markdown2.py")):
1905 | sys.path.insert(0, test_dir)
1906 | from test_markdown2 import norm_html_from_html
1907 | norm_html = norm_html_from_html(html)
1908 | norm_perl_html = norm_html_from_html(perl_html)
1909 | else:
1910 | norm_html = html
1911 | norm_perl_html = perl_html
1912 | print "==== match? %r ====" % (norm_perl_html == norm_html)
1913 |
1914 |
1915 | if __name__ == "__main__":
1916 | sys.exit(main(sys.argv))
1917 |
--------------------------------------------------------------------------------
{{comment.content}} 17 |
18 | 19 |