├── .github └── workflows │ ├── backend-tests.yml │ └── codeql.yml ├── .golangci.yaml ├── CODEOWNERS ├── README.md ├── ast ├── ast.go ├── block.go ├── inline.go └── util.go ├── go.mod ├── go.sum ├── gomark.go ├── parser ├── auto_link.go ├── blockquote.go ├── bold.go ├── bold_italic.go ├── code.go ├── code_block.go ├── embedded_content.go ├── escaping_character.go ├── heading.go ├── highlight.go ├── horizontal_rule.go ├── html_element.go ├── image.go ├── italic.go ├── line_break.go ├── link.go ├── math.go ├── math_block.go ├── ordered_list_item.go ├── paragraph.go ├── parser.go ├── referenced_content.go ├── spoiler.go ├── strikethrough.go ├── subscript.go ├── superscript.go ├── table.go ├── tag.go ├── task_list_item.go ├── tests │ ├── auto_link_test.go │ ├── blockquote_test.go │ ├── bold_italic_test.go │ ├── bold_test.go │ ├── code_block_test.go │ ├── code_test.go │ ├── embedded_content_test.go │ ├── escaping_character_test.go │ ├── heading_test.go │ ├── highlight_test.go │ ├── horizontal_rule_test.go │ ├── html_element_test.go │ ├── image_test.go │ ├── italic_test.go │ ├── link_test.go │ ├── list_test.go │ ├── math_block_test.go │ ├── math_test.go │ ├── ordered_list_item_test.go │ ├── paragraph_test.go │ ├── parser_test.go │ ├── referenced_content_test.go │ ├── spoiler_test.go │ ├── strikethrough_test.go │ ├── subscript_test.go │ ├── superscript_test.go │ ├── table_test.go │ ├── tag_test.go │ ├── task_list_item_test.go │ └── unordered_list_item_test.go ├── text.go ├── tokenizer │ ├── tokenizer.go │ └── tokenizer_test.go └── unordered_list_item.go ├── renderer ├── html │ ├── html.go │ └── html_test.go ├── renderer.go └── string │ ├── string.go │ └── string_test.go └── restore ├── restore.go └── restore_test.go /.github/workflows/backend-tests.yml: -------------------------------------------------------------------------------- 1 | name: Backend Test 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: 8 | - main 9 | paths: 10 | - "go.mod" 11 | - "go.sum" 12 | - "**.go" 13 | 14 | jobs: 15 | go-static-checks: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v4 19 | - uses: actions/setup-go@v5 20 | with: 21 | go-version: 1.21 22 | check-latest: true 23 | cache: true 24 | - name: Verify go.mod is tidy 25 | run: | 26 | go mod tidy -go=1.21 27 | git diff --exit-code 28 | - name: golangci-lint 29 | uses: golangci/golangci-lint-action@v6 30 | with: 31 | version: v1.56.1 32 | args: --verbose --timeout=3m 33 | skip-cache: true 34 | 35 | go-tests: 36 | runs-on: ubuntu-latest 37 | steps: 38 | - uses: actions/checkout@v4 39 | - uses: actions/setup-go@v5 40 | with: 41 | go-version: 1.21 42 | check-latest: true 43 | cache: true 44 | - name: Run all tests 45 | run: go test -v ./... | tee test.log; exit ${PIPESTATUS[0]} 46 | - name: Pretty print tests running time 47 | run: grep --color=never -e '--- PASS:' -e '--- FAIL:' test.log | sed 's/[:()]//g' | awk '{print $2,$3,$4}' | sort -t' ' -nk3 -r | awk '{sum += $3; print $1,$2,$3,sum"s"}' 48 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [main] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [main] 20 | paths: 21 | - "go.mod" 22 | - "go.sum" 23 | - "**.go" 24 | - "proto/**" 25 | - "web/**" 26 | 27 | jobs: 28 | analyze: 29 | name: Analyze 30 | runs-on: ubuntu-latest 31 | permissions: 32 | actions: read 33 | contents: read 34 | security-events: write 35 | 36 | strategy: 37 | fail-fast: false 38 | matrix: 39 | language: ["go", "javascript"] 40 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 41 | # Learn more about CodeQL language support at https://git.io/codeql-language-support 42 | 43 | steps: 44 | - name: Checkout repository 45 | uses: actions/checkout@v4 46 | 47 | # Initializes the CodeQL tools for scanning. 48 | - name: Initialize CodeQL 49 | uses: github/codeql-action/init@v2 50 | with: 51 | languages: ${{ matrix.language }} 52 | # If you wish to specify custom queries, you can do so here or in a config file. 53 | # By default, queries listed here will override any specified in a config file. 54 | # Prefix the list here with "+" to use these queries and those in the config file. 55 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 56 | 57 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 58 | # If this step fails, then you should remove it and run the build manually (see below) 59 | - name: Autobuild 60 | uses: github/codeql-action/autobuild@v2 61 | 62 | # ℹ️ Command-line programs to run using the OS shell. 63 | # 📚 https://git.io/JvXDl 64 | 65 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 66 | # and modify them (or add more) to build your code if your project 67 | # uses a compiled language 68 | 69 | #- run: | 70 | # make bootstrap 71 | # make release 72 | 73 | - name: Perform CodeQL Analysis 74 | uses: github/codeql-action/analyze@v2 75 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | run: 2 | timeout: 10m 3 | linters: 4 | enable: 5 | - errcheck 6 | - goimports 7 | - revive 8 | - govet 9 | - staticcheck 10 | - misspell 11 | - gocritic 12 | - sqlclosecheck 13 | - rowserrcheck 14 | - nilerr 15 | - godot 16 | - forbidigo 17 | - mirror 18 | - bodyclose 19 | 20 | linters-settings: 21 | goimports: 22 | # Put imports beginning with prefix after 3rd-party packages. 23 | local-prefixes: github.com/usememos/memos 24 | revive: 25 | # Default to run all linters so that new rules in the future could automatically be added to the static check. 26 | enable-all-rules: true 27 | rules: 28 | # The following rules are too strict and make coding harder. We do not enable them for now. 29 | - name: file-header 30 | disabled: true 31 | - name: line-length-limit 32 | disabled: true 33 | - name: function-length 34 | disabled: true 35 | - name: max-public-structs 36 | disabled: true 37 | - name: function-result-limit 38 | disabled: true 39 | - name: banned-characters 40 | disabled: true 41 | - name: argument-limit 42 | disabled: true 43 | - name: cognitive-complexity 44 | disabled: true 45 | - name: cyclomatic 46 | disabled: true 47 | - name: confusing-results 48 | disabled: true 49 | - name: add-constant 50 | disabled: true 51 | - name: flag-parameter 52 | disabled: true 53 | - name: nested-structs 54 | disabled: true 55 | - name: import-shadowing 56 | disabled: true 57 | - name: early-return 58 | disabled: true 59 | - name: use-any 60 | disabled: true 61 | - name: exported 62 | disabled: true 63 | - name: unhandled-error 64 | disabled: true 65 | - name: if-return 66 | disabled: true 67 | - name: max-control-nesting 68 | disabled: true 69 | gocritic: 70 | disabled-checks: 71 | - ifElseChain 72 | govet: 73 | settings: 74 | printf: # The name of the analyzer, run `go tool vet help` to see the list of all analyzers 75 | funcs: # Run `go tool vet help printf` to see the full configuration of `printf`. 76 | - common.Errorf 77 | enable-all: true 78 | disable: 79 | - fieldalignment 80 | - shadow 81 | forbidigo: 82 | forbid: 83 | - 'fmt\.Errorf(# Please use errors\.Wrap\|Wrapf\|Errorf instead)?' 84 | - 'ioutil\.ReadDir(# Please use os\.ReadDir)?' 85 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # These owners will be the default owners for everything in the repo. 2 | * @boojack 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gomark 2 | -------------------------------------------------------------------------------- /ast/ast.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | type NodeType string 4 | 5 | // Block nodes. 6 | const ( 7 | LineBreakNode NodeType = "LINE_BREAK" 8 | ParagraphNode NodeType = "PARAGRAPH" 9 | CodeBlockNode NodeType = "CODE_BLOCK" 10 | HeadingNode NodeType = "HEADING" 11 | HorizontalRuleNode NodeType = "HORIZONTAL_RULE" 12 | BlockquoteNode NodeType = "BLOCKQUOTE" 13 | ListNode NodeType = "LIST" 14 | OrderedListItemNode NodeType = "ORDERED_LIST_ITEM" 15 | UnorderedListItemNode NodeType = "UNORDERED_LIST_ITEM" 16 | TaskListItemNode NodeType = "TASK_LIST_ITEM" 17 | MathBlockNode NodeType = "MATH_BLOCK" 18 | TableNode NodeType = "TABLE" 19 | EmbeddedContentNode NodeType = "EMBEDDED_CONTENT" 20 | ) 21 | 22 | // Inline nodes. 23 | const ( 24 | TextNode NodeType = "TEXT" 25 | BoldNode NodeType = "BOLD" 26 | ItalicNode NodeType = "ITALIC" 27 | BoldItalicNode NodeType = "BOLD_ITALIC" 28 | CodeNode NodeType = "CODE" 29 | ImageNode NodeType = "IMAGE" 30 | LinkNode NodeType = "LINK" 31 | AutoLinkNode NodeType = "AUTO_LINK" 32 | TagNode NodeType = "TAG" 33 | StrikethroughNode NodeType = "STRIKETHROUGH" 34 | EscapingCharacterNode NodeType = "ESCAPING_CHARACTER" 35 | MathNode NodeType = "MATH" 36 | HighlightNode NodeType = "HIGHLIGHT" 37 | SubscriptNode NodeType = "SUBSCRIPT" 38 | SuperscriptNode NodeType = "SUPERSCRIPT" 39 | ReferencedContentNode NodeType = "REFERENCED_CONTENT" 40 | SpoilerNode NodeType = "SPOILER" 41 | HTMLElementNode NodeType = "HTML_ELEMENT" 42 | ) 43 | 44 | type Node interface { 45 | // Type returns a node type. 46 | Type() NodeType 47 | 48 | // Restore returns a string representation of this node. 49 | Restore() string 50 | } 51 | 52 | type BaseNode struct { 53 | } 54 | 55 | func IsBlockNode(node Node) bool { 56 | switch node.Type() { 57 | case ParagraphNode, CodeBlockNode, HeadingNode, HorizontalRuleNode, BlockquoteNode, ListNode, OrderedListItemNode, UnorderedListItemNode, TaskListItemNode, TableNode, EmbeddedContentNode: 58 | return true 59 | default: 60 | return false 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /ast/block.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | type BaseBlock struct { 9 | BaseNode 10 | } 11 | 12 | type LineBreak struct { 13 | BaseBlock 14 | } 15 | 16 | func (*LineBreak) Type() NodeType { 17 | return LineBreakNode 18 | } 19 | 20 | func (*LineBreak) Restore() string { 21 | return "\n" 22 | } 23 | 24 | type Paragraph struct { 25 | BaseBlock 26 | 27 | Children []Node 28 | } 29 | 30 | func (*Paragraph) Type() NodeType { 31 | return ParagraphNode 32 | } 33 | 34 | func (n *Paragraph) Restore() string { 35 | var result string 36 | for _, child := range n.Children { 37 | result += child.Restore() 38 | } 39 | return result 40 | } 41 | 42 | type CodeBlock struct { 43 | BaseBlock 44 | 45 | Language string 46 | Content string 47 | } 48 | 49 | func (*CodeBlock) Type() NodeType { 50 | return CodeBlockNode 51 | } 52 | 53 | func (n *CodeBlock) Restore() string { 54 | return fmt.Sprintf("```%s\n%s\n```", n.Language, n.Content) 55 | } 56 | 57 | type Heading struct { 58 | BaseBlock 59 | 60 | Level int 61 | Children []Node 62 | } 63 | 64 | func (*Heading) Type() NodeType { 65 | return HeadingNode 66 | } 67 | 68 | func (n *Heading) Restore() string { 69 | var result string 70 | for _, child := range n.Children { 71 | result += child.Restore() 72 | } 73 | symbol := "" 74 | for i := 0; i < n.Level; i++ { 75 | symbol += "#" 76 | } 77 | return fmt.Sprintf("%s %s", symbol, result) 78 | } 79 | 80 | type HorizontalRule struct { 81 | BaseBlock 82 | 83 | // Symbol is "*" or "-" or "_". 84 | Symbol string 85 | } 86 | 87 | func (*HorizontalRule) Type() NodeType { 88 | return HorizontalRuleNode 89 | } 90 | 91 | func (n *HorizontalRule) Restore() string { 92 | return n.Symbol + n.Symbol + n.Symbol 93 | } 94 | 95 | type Blockquote struct { 96 | BaseBlock 97 | 98 | Children []Node 99 | } 100 | 101 | func (*Blockquote) Type() NodeType { 102 | return BlockquoteNode 103 | } 104 | 105 | func (n *Blockquote) Restore() string { 106 | var result string 107 | for i, child := range n.Children { 108 | result += fmt.Sprintf("> %s", child.Restore()) 109 | if i != len(n.Children)-1 { 110 | result += "\n" 111 | } 112 | } 113 | return result 114 | } 115 | 116 | type ListKind string 117 | 118 | const ( 119 | UnorderedList ListKind = "ul" 120 | OrderedList ListKind = "ol" 121 | DescrpitionList ListKind = "dl" 122 | ) 123 | 124 | type List struct { 125 | BaseBlock 126 | 127 | Kind ListKind 128 | Indent int 129 | Children []Node 130 | } 131 | 132 | func (*List) Type() NodeType { 133 | return ListNode 134 | } 135 | 136 | func (n *List) Restore() string { 137 | var result string 138 | for _, child := range n.Children { 139 | result += child.Restore() 140 | } 141 | return result 142 | } 143 | 144 | type OrderedListItem struct { 145 | BaseBlock 146 | 147 | // Number is the number of the list. 148 | Number string 149 | // Indent is the number of spaces. 150 | Indent int 151 | Children []Node 152 | } 153 | 154 | func (*OrderedListItem) Type() NodeType { 155 | return OrderedListItemNode 156 | } 157 | 158 | func (n *OrderedListItem) Restore() string { 159 | var result string 160 | for _, child := range n.Children { 161 | result += child.Restore() 162 | } 163 | return fmt.Sprintf("%s%s. %s", strings.Repeat(" ", n.Indent), n.Number, result) 164 | } 165 | 166 | type UnorderedListItem struct { 167 | BaseBlock 168 | 169 | // Symbol is "*" or "-" or "+". 170 | Symbol string 171 | // Indent is the number of spaces. 172 | Indent int 173 | Children []Node 174 | } 175 | 176 | func (*UnorderedListItem) Type() NodeType { 177 | return UnorderedListItemNode 178 | } 179 | 180 | func (n *UnorderedListItem) Restore() string { 181 | var result string 182 | for _, child := range n.Children { 183 | result += child.Restore() 184 | } 185 | return fmt.Sprintf("%s%s %s", strings.Repeat(" ", n.Indent), n.Symbol, result) 186 | } 187 | 188 | type TaskListItem struct { 189 | BaseBlock 190 | 191 | // Symbol is "*" or "-" or "+". 192 | Symbol string 193 | // Indent is the number of spaces. 194 | Indent int 195 | Complete bool 196 | Children []Node 197 | } 198 | 199 | func (*TaskListItem) Type() NodeType { 200 | return TaskListItemNode 201 | } 202 | 203 | func (n *TaskListItem) Restore() string { 204 | var result string 205 | for _, child := range n.Children { 206 | result += child.Restore() 207 | } 208 | complete := " " 209 | if n.Complete { 210 | complete = "x" 211 | } 212 | return fmt.Sprintf("%s%s [%s] %s", strings.Repeat(" ", n.Indent), n.Symbol, complete, result) 213 | } 214 | 215 | type MathBlock struct { 216 | BaseBlock 217 | 218 | Content string 219 | } 220 | 221 | func (*MathBlock) Type() NodeType { 222 | return MathBlockNode 223 | } 224 | 225 | func (n *MathBlock) Restore() string { 226 | return fmt.Sprintf("$$\n%s\n$$", n.Content) 227 | } 228 | 229 | type Table struct { 230 | BaseBlock 231 | 232 | Header []Node 233 | Delimiter []string 234 | Rows [][]Node 235 | } 236 | 237 | func (*Table) Type() NodeType { 238 | return TableNode 239 | } 240 | 241 | func (n *Table) Restore() string { 242 | result := "" 243 | for _, header := range n.Header { 244 | result += fmt.Sprintf("| %s ", header.Restore()) 245 | } 246 | result += "|\n" 247 | for _, d := range n.Delimiter { 248 | result += fmt.Sprintf("| %s ", d) 249 | } 250 | result += "|\n" 251 | for index, row := range n.Rows { 252 | for _, cell := range row { 253 | result += fmt.Sprintf("| %s ", cell.Restore()) 254 | } 255 | result += "|" 256 | if index != len(n.Rows)-1 { 257 | result += "\n" 258 | } 259 | } 260 | return result 261 | } 262 | 263 | type EmbeddedContent struct { 264 | BaseBlock 265 | 266 | ResourceName string 267 | Params string 268 | } 269 | 270 | func (*EmbeddedContent) Type() NodeType { 271 | return EmbeddedContentNode 272 | } 273 | 274 | func (n *EmbeddedContent) Restore() string { 275 | params := "" 276 | if n.Params != "" { 277 | params = fmt.Sprintf("?%s", n.Params) 278 | } 279 | result := fmt.Sprintf("![[%s%s]]", n.ResourceName, params) 280 | return result 281 | } 282 | -------------------------------------------------------------------------------- /ast/inline.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | type BaseInline struct { 9 | BaseNode 10 | } 11 | 12 | type Text struct { 13 | BaseInline 14 | 15 | Content string 16 | } 17 | 18 | func (*Text) Type() NodeType { 19 | return TextNode 20 | } 21 | 22 | func (n *Text) Restore() string { 23 | return n.Content 24 | } 25 | 26 | type Bold struct { 27 | BaseInline 28 | 29 | // Symbol is "*" or "_". 30 | Symbol string 31 | Children []Node 32 | } 33 | 34 | func (*Bold) Type() NodeType { 35 | return BoldNode 36 | } 37 | 38 | func (n *Bold) Restore() string { 39 | symbol := n.Symbol + n.Symbol 40 | children := "" 41 | for _, child := range n.Children { 42 | children += child.Restore() 43 | } 44 | return fmt.Sprintf("%s%s%s", symbol, children, symbol) 45 | } 46 | 47 | type Italic struct { 48 | BaseInline 49 | 50 | // Symbol is "*" or "_". 51 | Symbol string 52 | Children []Node 53 | } 54 | 55 | func (*Italic) Type() NodeType { 56 | return ItalicNode 57 | } 58 | 59 | func (n *Italic) Restore() string { 60 | content := "" 61 | for _, child := range n.Children { 62 | content += child.Restore() 63 | } 64 | return fmt.Sprintf("%s%s%s", n.Symbol, content, n.Symbol) 65 | } 66 | 67 | type BoldItalic struct { 68 | BaseInline 69 | 70 | // Symbol is "*" or "_". 71 | Symbol string 72 | Content string 73 | } 74 | 75 | func (*BoldItalic) Type() NodeType { 76 | return BoldItalicNode 77 | } 78 | 79 | func (n *BoldItalic) Restore() string { 80 | symbol := n.Symbol + n.Symbol + n.Symbol 81 | return fmt.Sprintf("%s%s%s", symbol, n.Content, symbol) 82 | } 83 | 84 | type Code struct { 85 | BaseInline 86 | 87 | Content string 88 | } 89 | 90 | func (*Code) Type() NodeType { 91 | return CodeNode 92 | } 93 | 94 | func (n *Code) Restore() string { 95 | return fmt.Sprintf("`%s`", n.Content) 96 | } 97 | 98 | type Image struct { 99 | BaseInline 100 | 101 | AltText string 102 | URL string 103 | } 104 | 105 | func (*Image) Type() NodeType { 106 | return ImageNode 107 | } 108 | 109 | func (n *Image) Restore() string { 110 | return fmt.Sprintf("![%s](%s)", n.AltText, n.URL) 111 | } 112 | 113 | type Link struct { 114 | BaseInline 115 | 116 | Content []Node 117 | URL string 118 | } 119 | 120 | func (*Link) Type() NodeType { 121 | return LinkNode 122 | } 123 | 124 | func (n *Link) Restore() string { 125 | content := "" 126 | for _, child := range n.Content { 127 | content += child.Restore() 128 | } 129 | return fmt.Sprintf("[%s](%s)", content, n.URL) 130 | } 131 | 132 | type AutoLink struct { 133 | BaseInline 134 | 135 | URL string 136 | IsRawText bool 137 | } 138 | 139 | func (*AutoLink) Type() NodeType { 140 | return AutoLinkNode 141 | } 142 | 143 | func (n *AutoLink) Restore() string { 144 | if n.IsRawText { 145 | return n.URL 146 | } 147 | return fmt.Sprintf("<%s>", n.URL) 148 | } 149 | 150 | type Tag struct { 151 | BaseInline 152 | 153 | Content string 154 | } 155 | 156 | func (*Tag) Type() NodeType { 157 | return TagNode 158 | } 159 | 160 | func (n *Tag) Restore() string { 161 | return fmt.Sprintf("#%s", n.Content) 162 | } 163 | 164 | type Strikethrough struct { 165 | BaseInline 166 | 167 | Content string 168 | } 169 | 170 | func (*Strikethrough) Type() NodeType { 171 | return StrikethroughNode 172 | } 173 | 174 | func (n *Strikethrough) Restore() string { 175 | return fmt.Sprintf("~~%s~~", n.Content) 176 | } 177 | 178 | type EscapingCharacter struct { 179 | BaseInline 180 | 181 | Symbol string 182 | } 183 | 184 | func (*EscapingCharacter) Type() NodeType { 185 | return EscapingCharacterNode 186 | } 187 | 188 | func (n *EscapingCharacter) Restore() string { 189 | return fmt.Sprintf("\\%s", n.Symbol) 190 | } 191 | 192 | type Math struct { 193 | BaseInline 194 | 195 | Content string 196 | } 197 | 198 | func (*Math) Type() NodeType { 199 | return MathNode 200 | } 201 | 202 | func (n *Math) Restore() string { 203 | return fmt.Sprintf("$%s$", n.Content) 204 | } 205 | 206 | type Highlight struct { 207 | BaseInline 208 | 209 | Content string 210 | } 211 | 212 | func (*Highlight) Type() NodeType { 213 | return HighlightNode 214 | } 215 | 216 | func (n *Highlight) Restore() string { 217 | return fmt.Sprintf("==%s==", n.Content) 218 | } 219 | 220 | type Subscript struct { 221 | BaseInline 222 | 223 | Content string 224 | } 225 | 226 | func (*Subscript) Type() NodeType { 227 | return SubscriptNode 228 | } 229 | 230 | func (n *Subscript) Restore() string { 231 | return fmt.Sprintf("~%s~", n.Content) 232 | } 233 | 234 | type Superscript struct { 235 | BaseInline 236 | 237 | Content string 238 | } 239 | 240 | func (*Superscript) Type() NodeType { 241 | return SuperscriptNode 242 | } 243 | 244 | func (n *Superscript) Restore() string { 245 | return fmt.Sprintf("^%s^", n.Content) 246 | } 247 | 248 | type ReferencedContent struct { 249 | BaseInline 250 | 251 | ResourceName string 252 | Params string 253 | } 254 | 255 | func (*ReferencedContent) Type() NodeType { 256 | return ReferencedContentNode 257 | } 258 | 259 | func (n *ReferencedContent) Restore() string { 260 | params := "" 261 | if n.Params != "" { 262 | params = fmt.Sprintf("?%s", n.Params) 263 | } 264 | result := fmt.Sprintf("[[%s%s]]", n.ResourceName, params) 265 | return result 266 | } 267 | 268 | type Spoiler struct { 269 | BaseInline 270 | 271 | Content string 272 | } 273 | 274 | func (*Spoiler) Type() NodeType { 275 | return SpoilerNode 276 | } 277 | 278 | func (n *Spoiler) Restore() string { 279 | return fmt.Sprintf("||%s||", n.Content) 280 | } 281 | 282 | type HTMLElement struct { 283 | BaseInline 284 | 285 | TagName string 286 | Attributes map[string]string 287 | } 288 | 289 | func (*HTMLElement) Type() NodeType { 290 | return HTMLElementNode 291 | } 292 | 293 | func (n *HTMLElement) Restore() string { 294 | attributes := []string{} 295 | for key, value := range n.Attributes { 296 | attributes = append(attributes, fmt.Sprintf(`%s="%s"`, key, value)) 297 | } 298 | attrStr := "" 299 | if len(attributes) > 0 { 300 | attrStr = " " + strings.Join(attributes, " ") 301 | } 302 | return fmt.Sprintf("<%s%s />", n.TagName, attrStr) 303 | } 304 | -------------------------------------------------------------------------------- /ast/util.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import "slices" 4 | 5 | func IsListItemNode(node Node) bool { 6 | nodeType := node.Type() 7 | return slices.Contains([]NodeType{ 8 | OrderedListItemNode, UnorderedListItemNode, TaskListItemNode, 9 | }, nodeType) 10 | } 11 | 12 | func GetListItemKindAndIndent(node Node) (ListKind, int) { 13 | switch n := node.(type) { 14 | case *OrderedListItem: 15 | return OrderedList, n.Indent 16 | case *UnorderedListItem: 17 | return UnorderedList, n.Indent 18 | case *TaskListItem: 19 | return DescrpitionList, n.Indent 20 | default: 21 | return "", 0 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/usememos/gomark 2 | 3 | go 1.21 4 | 5 | require github.com/stretchr/testify v1.8.4 6 | 7 | require ( 8 | github.com/davecgh/go-spew v1.1.1 // indirect 9 | github.com/pmezard/go-difflib v1.0.0 // indirect 10 | gopkg.in/yaml.v3 v3.0.1 // indirect 11 | ) 12 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 4 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 5 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= 6 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 7 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 8 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 9 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 10 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 11 | -------------------------------------------------------------------------------- /gomark.go: -------------------------------------------------------------------------------- 1 | package gomark 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser" 6 | "github.com/usememos/gomark/parser/tokenizer" 7 | "github.com/usememos/gomark/restore" 8 | ) 9 | 10 | // Parse parses the given markdown string and returns a list of nodes. 11 | func Parse(markdown string) (nodes []ast.Node, err error) { 12 | tokens := tokenizer.Tokenize(markdown) 13 | return parser.Parse(tokens) 14 | } 15 | 16 | // Restore restores the given nodes to a markdown string. 17 | func Restore(nodes []ast.Node) string { 18 | return restore.Restore(nodes) 19 | } 20 | -------------------------------------------------------------------------------- /parser/auto_link.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "net/url" 5 | 6 | "github.com/usememos/gomark/ast" 7 | "github.com/usememos/gomark/parser/tokenizer" 8 | ) 9 | 10 | type AutoLinkParser struct{} 11 | 12 | func NewAutoLinkParser() *AutoLinkParser { 13 | return &AutoLinkParser{} 14 | } 15 | 16 | func (*AutoLinkParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 17 | if len(tokens) < 3 { 18 | return nil, 0 19 | } 20 | 21 | matchedTokens := tokenizer.GetFirstLine(tokens) 22 | urlStr, isRawText := "", true 23 | if matchedTokens[0].Type == tokenizer.LessThan { 24 | greaterThanIndex := tokenizer.FindUnescaped(matchedTokens, tokenizer.GreaterThan) 25 | if greaterThanIndex < 0 { 26 | return nil, 0 27 | } 28 | matchedTokens = matchedTokens[:greaterThanIndex+1] 29 | urlStr = tokenizer.Stringify(matchedTokens[1 : len(matchedTokens)-1]) 30 | isRawText = false 31 | } else { 32 | contentTokens := []*tokenizer.Token{} 33 | for _, token := range matchedTokens { 34 | if token.Type == tokenizer.Space { 35 | break 36 | } 37 | contentTokens = append(contentTokens, token) 38 | } 39 | if len(contentTokens) == 0 { 40 | return nil, 0 41 | } 42 | 43 | matchedTokens = contentTokens 44 | u, err := url.Parse(tokenizer.Stringify(matchedTokens)) 45 | if err != nil || u.Scheme == "" || u.Host == "" { 46 | return nil, 0 47 | } 48 | urlStr = tokenizer.Stringify(matchedTokens) 49 | } 50 | 51 | return &ast.AutoLink{ 52 | URL: urlStr, 53 | IsRawText: isRawText, 54 | }, len(matchedTokens) 55 | } 56 | -------------------------------------------------------------------------------- /parser/blockquote.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type BlockquoteParser struct{} 9 | 10 | func NewBlockquoteParser() *BlockquoteParser { 11 | return &BlockquoteParser{} 12 | } 13 | 14 | func (*BlockquoteParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | rows := tokenizer.Split(tokens, tokenizer.NewLine) 16 | contentRows := [][]*tokenizer.Token{} 17 | for _, row := range rows { 18 | if len(row) < 2 || row[0].Type != tokenizer.GreaterThan || row[1].Type != tokenizer.Space { 19 | break 20 | } 21 | contentRows = append(contentRows, row) 22 | } 23 | if len(contentRows) == 0 { 24 | return nil, 0 25 | } 26 | 27 | children := []ast.Node{} 28 | size := 0 29 | 30 | for index, row := range contentRows { 31 | contentTokens := row[2:] 32 | var node ast.Node 33 | if len(contentTokens) == 0 { 34 | node = &ast.Paragraph{ 35 | Children: []ast.Node{&ast.Text{Content: " "}}, 36 | } 37 | } else { 38 | nodes, err := ParseBlockWithParsers(contentTokens, []BlockParser{NewBlockquoteParser(), NewParagraphParser()}) 39 | if err != nil { 40 | return nil, 0 41 | } 42 | if len(nodes) != 1 { 43 | return nil, 0 44 | } 45 | node = nodes[0] 46 | } 47 | children = append(children, node) 48 | size += len(row) 49 | if index != len(contentRows)-1 { 50 | size++ // NewLine. 51 | } 52 | } 53 | if len(children) == 0 { 54 | return nil, 0 55 | } 56 | return &ast.Blockquote{ 57 | Children: children, 58 | }, size 59 | } 60 | -------------------------------------------------------------------------------- /parser/bold.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type BoldParser struct{} 9 | 10 | func NewBoldParser() InlineParser { 11 | return &BoldParser{} 12 | } 13 | 14 | func (*BoldParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | matchedTokens := tokenizer.GetFirstLine(tokens) 16 | if len(matchedTokens) < 5 { 17 | return nil, 0 18 | } 19 | 20 | prefixTokens := matchedTokens[:2] 21 | if prefixTokens[0].Type != prefixTokens[1].Type { 22 | return nil, 0 23 | } 24 | prefixTokenType := prefixTokens[0].Type 25 | if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underscore { 26 | return nil, 0 27 | } 28 | 29 | cursor, matched := 2, false 30 | for ; cursor < len(matchedTokens)-1; cursor++ { 31 | token, nextToken := matchedTokens[cursor], matchedTokens[cursor+1] 32 | if token.Type == tokenizer.NewLine || nextToken.Type == tokenizer.NewLine { 33 | return nil, 0 34 | } 35 | if token.Type == prefixTokenType && nextToken.Type == prefixTokenType { 36 | matchedTokens = matchedTokens[:cursor+2] 37 | matched = true 38 | break 39 | } 40 | } 41 | if !matched { 42 | return nil, 0 43 | } 44 | 45 | size := len(matchedTokens) 46 | children, err := ParseInlineWithParsers(matchedTokens[2:size-2], []InlineParser{NewLinkParser(), NewTextParser()}) 47 | if err != nil || len(children) == 0 { 48 | return nil, 0 49 | } 50 | return &ast.Bold{ 51 | Symbol: prefixTokenType, 52 | Children: children, 53 | }, size 54 | } 55 | -------------------------------------------------------------------------------- /parser/bold_italic.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type BoldItalicParser struct{} 9 | 10 | func NewBoldItalicParser() InlineParser { 11 | return &BoldItalicParser{} 12 | } 13 | 14 | func (*BoldItalicParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | matchedTokens := tokenizer.GetFirstLine(tokens) 16 | if len(matchedTokens) < 7 { 17 | return nil, 0 18 | } 19 | prefixTokens := matchedTokens[:3] 20 | if prefixTokens[0].Type != prefixTokens[1].Type || prefixTokens[0].Type != prefixTokens[2].Type || prefixTokens[1].Type != prefixTokens[2].Type { 21 | return nil, 0 22 | } 23 | prefixTokenType := prefixTokens[0].Type 24 | if prefixTokenType != tokenizer.Asterisk { 25 | return nil, 0 26 | } 27 | 28 | cursor, matched := 3, false 29 | for ; cursor < len(matchedTokens)-2; cursor++ { 30 | token, nextToken, endToken := matchedTokens[cursor], matchedTokens[cursor+1], matchedTokens[cursor+2] 31 | if token.Type == tokenizer.NewLine || nextToken.Type == tokenizer.NewLine || endToken.Type == tokenizer.NewLine { 32 | return nil, 0 33 | } 34 | if token.Type == prefixTokenType && nextToken.Type == prefixTokenType && endToken.Type == prefixTokenType { 35 | matchedTokens = matchedTokens[:cursor+3] 36 | matched = true 37 | break 38 | } 39 | } 40 | if !matched { 41 | return nil, 0 42 | } 43 | 44 | size := len(matchedTokens) 45 | contentTokens := matchedTokens[3 : size-3] 46 | if len(contentTokens) == 0 { 47 | return nil, 0 48 | } 49 | 50 | return &ast.BoldItalic{ 51 | Symbol: prefixTokenType, 52 | Content: tokenizer.Stringify(contentTokens), 53 | }, size 54 | } 55 | -------------------------------------------------------------------------------- /parser/code.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type CodeParser struct{} 9 | 10 | func NewCodeParser() *CodeParser { 11 | return &CodeParser{} 12 | } 13 | 14 | func (*CodeParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | matchedTokens := tokenizer.GetFirstLine(tokens) 16 | if len(matchedTokens) < 3 { 17 | return nil, 0 18 | } 19 | if matchedTokens[0].Type != tokenizer.Backtick { 20 | return nil, 0 21 | } 22 | nextBacktickIndex := tokenizer.FindUnescaped(matchedTokens[1:], tokenizer.Backtick) 23 | if nextBacktickIndex < 0 { 24 | return nil, 0 25 | } 26 | matchedTokens = matchedTokens[:1+nextBacktickIndex+1] 27 | contentTokens := matchedTokens[1 : len(matchedTokens)-1] 28 | if len(contentTokens) == 0 { 29 | return nil, 0 30 | } 31 | return &ast.Code{ 32 | Content: tokenizer.Stringify(contentTokens), 33 | }, len(matchedTokens) 34 | } 35 | -------------------------------------------------------------------------------- /parser/code_block.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "slices" 5 | 6 | "github.com/usememos/gomark/ast" 7 | "github.com/usememos/gomark/parser/tokenizer" 8 | ) 9 | 10 | type CodeBlockParser struct { 11 | Language string 12 | Content string 13 | } 14 | 15 | func NewCodeBlockParser() *CodeBlockParser { 16 | return &CodeBlockParser{} 17 | } 18 | 19 | func (*CodeBlockParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 20 | rows := tokenizer.Split(tokens, tokenizer.NewLine) 21 | if len(rows) < 3 { 22 | return nil, 0 23 | } 24 | 25 | firstRow := rows[0] 26 | if len(firstRow) < 3 { 27 | return nil, 0 28 | } 29 | if firstRow[0].Type != tokenizer.Backtick || firstRow[1].Type != tokenizer.Backtick || firstRow[2].Type != tokenizer.Backtick { 30 | return nil, 0 31 | } 32 | languageTokens := []*tokenizer.Token{} 33 | if len(firstRow) > 3 { 34 | languageTokens = firstRow[3:] 35 | // Check if language is valid. 36 | availableLanguageTokenTypes := []tokenizer.TokenType{tokenizer.Text, tokenizer.Number, tokenizer.Underscore} 37 | for _, token := range languageTokens { 38 | if !slices.Contains(availableLanguageTokenTypes, token.Type) { 39 | return nil, 0 40 | } 41 | } 42 | } 43 | 44 | contentRows := [][]*tokenizer.Token{} 45 | matched := false 46 | for _, row := range rows[1:] { 47 | if len(row) == 3 && row[0].Type == tokenizer.Backtick && row[1].Type == tokenizer.Backtick && row[2].Type == tokenizer.Backtick { 48 | matched = true 49 | break 50 | } 51 | contentRows = append(contentRows, row) 52 | } 53 | if !matched { 54 | return nil, 0 55 | } 56 | 57 | contentTokens := []*tokenizer.Token{} 58 | for index, row := range contentRows { 59 | contentTokens = append(contentTokens, row...) 60 | if index != len(contentRows)-1 { 61 | contentTokens = append(contentTokens, &tokenizer.Token{ 62 | Type: tokenizer.NewLine, 63 | Value: "\n", 64 | }) 65 | } 66 | } 67 | 68 | return &ast.CodeBlock{ 69 | Content: tokenizer.Stringify(contentTokens), 70 | Language: tokenizer.Stringify(languageTokens), 71 | }, 4 + len(languageTokens) + len(contentTokens) + 4 72 | } 73 | -------------------------------------------------------------------------------- /parser/embedded_content.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type EmbeddedContentParser struct{} 9 | 10 | func NewEmbeddedContentParser() *EmbeddedContentParser { 11 | return &EmbeddedContentParser{} 12 | } 13 | 14 | func (*EmbeddedContentParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | matchedTokens := tokenizer.GetFirstLine(tokens) 16 | if len(matchedTokens) < 6 { 17 | return nil, 0 18 | } 19 | if matchedTokens[0].Type != tokenizer.ExclamationMark || matchedTokens[1].Type != tokenizer.LeftSquareBracket || matchedTokens[2].Type != tokenizer.LeftSquareBracket { 20 | return nil, 0 21 | } 22 | matched := false 23 | for index, token := range matchedTokens[:len(matchedTokens)-1] { 24 | if token.Type == tokenizer.RightSquareBracket && matchedTokens[index+1].Type == tokenizer.RightSquareBracket && index+1 == len(matchedTokens)-1 { 25 | matched = true 26 | break 27 | } 28 | } 29 | if !matched { 30 | return nil, 0 31 | } 32 | 33 | contentTokens := matchedTokens[3 : len(matchedTokens)-2] 34 | resourceName, params := tokenizer.Stringify(contentTokens), "" 35 | questionMarkIndex := tokenizer.FindUnescaped(contentTokens, tokenizer.QuestionMark) 36 | if questionMarkIndex > 0 { 37 | resourceName, params = tokenizer.Stringify(contentTokens[:questionMarkIndex]), tokenizer.Stringify(contentTokens[questionMarkIndex+1:]) 38 | } 39 | return &ast.EmbeddedContent{ 40 | ResourceName: resourceName, 41 | Params: params, 42 | }, len(matchedTokens) 43 | } 44 | -------------------------------------------------------------------------------- /parser/escaping_character.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type EscapingCharacterParser struct{} 9 | 10 | func NewEscapingCharacterParser() *EscapingCharacterParser { 11 | return &EscapingCharacterParser{} 12 | } 13 | 14 | func (*EscapingCharacterParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | if len(tokens) < 2 { 16 | return nil, 0 17 | } 18 | if tokens[0].Type != tokenizer.Backslash { 19 | return nil, 0 20 | } 21 | if tokens[1].Type == tokenizer.NewLine || tokens[1].Type == tokenizer.Space || tokens[1].Type == tokenizer.Text || tokens[1].Type == tokenizer.Number { 22 | return nil, 0 23 | } 24 | return &ast.EscapingCharacter{ 25 | Symbol: tokens[1].Value, 26 | }, 2 27 | } 28 | -------------------------------------------------------------------------------- /parser/heading.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type HeadingParser struct{} 9 | 10 | func NewHeadingParser() *HeadingParser { 11 | return &HeadingParser{} 12 | } 13 | 14 | func (*HeadingParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | matchedTokens := tokenizer.GetFirstLine(tokens) 16 | spaceIndex := tokenizer.FindUnescaped(matchedTokens, tokenizer.Space) 17 | if spaceIndex < 0 { 18 | return nil, 0 19 | } 20 | 21 | for _, token := range matchedTokens[:spaceIndex] { 22 | if token.Type != tokenizer.PoundSign { 23 | return nil, 0 24 | } 25 | } 26 | level := spaceIndex 27 | if level == 0 || level > 6 { 28 | return nil, 0 29 | } 30 | 31 | contentTokens := matchedTokens[level+1:] 32 | if len(contentTokens) == 0 { 33 | return nil, 0 34 | } 35 | children, err := ParseInline(contentTokens) 36 | if err != nil { 37 | return nil, 0 38 | } 39 | return &ast.Heading{ 40 | Level: level, 41 | Children: children, 42 | }, len(contentTokens) + level + 1 43 | } 44 | -------------------------------------------------------------------------------- /parser/highlight.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type HighlightParser struct{} 9 | 10 | func NewHighlightParser() InlineParser { 11 | return &HighlightParser{} 12 | } 13 | 14 | func (*HighlightParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | matchedToken := tokenizer.GetFirstLine(tokens) 16 | if len(matchedToken) < 5 { 17 | return nil, 0 18 | } 19 | 20 | prefixTokens := matchedToken[:2] 21 | if prefixTokens[0].Type != prefixTokens[1].Type { 22 | return nil, 0 23 | } 24 | prefixTokenType := prefixTokens[0].Type 25 | if prefixTokenType != tokenizer.EqualSign { 26 | return nil, 0 27 | } 28 | 29 | cursor, matched := 2, false 30 | for ; cursor < len(matchedToken)-1; cursor++ { 31 | token, nextToken := matchedToken[cursor], matchedToken[cursor+1] 32 | if token.Type == prefixTokenType && nextToken.Type == prefixTokenType { 33 | matched = true 34 | break 35 | } 36 | } 37 | if !matched { 38 | return nil, 0 39 | } 40 | 41 | return &ast.Highlight{ 42 | Content: tokenizer.Stringify(matchedToken[2:cursor]), 43 | }, cursor + 2 44 | } 45 | -------------------------------------------------------------------------------- /parser/horizontal_rule.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type HorizontalRuleParser struct{} 9 | 10 | func NewHorizontalRuleParser() *HorizontalRuleParser { 11 | return &HorizontalRuleParser{} 12 | } 13 | 14 | func (*HorizontalRuleParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | matchedTokens := tokenizer.GetFirstLine(tokens) 16 | if len(matchedTokens) < 3 { 17 | return nil, 0 18 | } 19 | if len(matchedTokens) > 3 && matchedTokens[3].Type != tokenizer.NewLine { 20 | return nil, 0 21 | } 22 | if matchedTokens[0].Type != matchedTokens[1].Type || matchedTokens[0].Type != matchedTokens[2].Type || matchedTokens[1].Type != matchedTokens[2].Type { 23 | return nil, 0 24 | } 25 | if matchedTokens[0].Type != tokenizer.Hyphen && matchedTokens[0].Type != tokenizer.Asterisk { 26 | return nil, 0 27 | } 28 | return &ast.HorizontalRule{ 29 | Symbol: matchedTokens[0].Type, 30 | }, 3 31 | } 32 | -------------------------------------------------------------------------------- /parser/html_element.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "slices" 5 | 6 | "github.com/usememos/gomark/ast" 7 | "github.com/usememos/gomark/parser/tokenizer" 8 | ) 9 | 10 | type HTMLElementParser struct{} 11 | 12 | func NewHTMLElementParser() *HTMLElementParser { 13 | return &HTMLElementParser{} 14 | } 15 | 16 | var ( 17 | availableHTMLElements = []string{ 18 | "br", 19 | } 20 | ) 21 | 22 | func (*HTMLElementParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 23 | if len(tokens) < 5 { 24 | return nil, 0 25 | } 26 | if tokens[0].Type != tokenizer.LessThan { 27 | return nil, 0 28 | } 29 | tagName := tokenizer.Stringify([]*tokenizer.Token{tokens[1]}) 30 | if !slices.Contains(availableHTMLElements, tagName) { 31 | return nil, 0 32 | } 33 | 34 | greaterThanIndex := tokenizer.FindUnescaped(tokens, tokenizer.GreaterThan) 35 | if greaterThanIndex+1 < 5 || tokens[greaterThanIndex-1].Type != tokenizer.Slash || tokens[greaterThanIndex-2].Type != tokenizer.Space { 36 | return nil, 0 37 | } 38 | 39 | matchedTokens := tokens[:greaterThanIndex+1] 40 | attributeTokens := matchedTokens[2 : greaterThanIndex-2] 41 | // TODO: Implement attribute parser. 42 | if len(attributeTokens) != 0 { 43 | return nil, 0 44 | } 45 | 46 | return &ast.HTMLElement{ 47 | TagName: tagName, 48 | Attributes: make(map[string]string), 49 | }, len(matchedTokens) 50 | } 51 | -------------------------------------------------------------------------------- /parser/image.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type ImageParser struct{} 9 | 10 | func NewImageParser() *ImageParser { 11 | return &ImageParser{} 12 | } 13 | 14 | func (*ImageParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | matchedTokens := tokenizer.GetFirstLine(tokens) 16 | if len(matchedTokens) < 5 { 17 | return nil, 0 18 | } 19 | if matchedTokens[0].Type != tokenizer.ExclamationMark { 20 | return nil, 0 21 | } 22 | if matchedTokens[1].Type != tokenizer.LeftSquareBracket { 23 | return nil, 0 24 | } 25 | cursor, altTokens := 2, []*tokenizer.Token{} 26 | for ; cursor < len(matchedTokens)-2; cursor++ { 27 | if matchedTokens[cursor].Type == tokenizer.RightSquareBracket { 28 | break 29 | } 30 | altTokens = append(altTokens, matchedTokens[cursor]) 31 | } 32 | if matchedTokens[cursor+1].Type != tokenizer.LeftParenthesis { 33 | return nil, 0 34 | } 35 | 36 | cursor += 2 37 | urlTokens, matched := []*tokenizer.Token{}, false 38 | for _, token := range matchedTokens[cursor:] { 39 | if token.Type == tokenizer.Space { 40 | return nil, 0 41 | } 42 | if token.Type == tokenizer.RightParenthesis { 43 | matched = true 44 | break 45 | } 46 | urlTokens = append(urlTokens, token) 47 | } 48 | if !matched || len(urlTokens) == 0 { 49 | return nil, 0 50 | } 51 | 52 | return &ast.Image{ 53 | AltText: tokenizer.Stringify(altTokens), 54 | URL: tokenizer.Stringify(urlTokens), 55 | }, 5 + len(altTokens) + len(urlTokens) 56 | } 57 | -------------------------------------------------------------------------------- /parser/italic.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type ItalicParser struct { 9 | ContentTokens []*tokenizer.Token 10 | } 11 | 12 | func NewItalicParser() *ItalicParser { 13 | return &ItalicParser{} 14 | } 15 | 16 | func (*ItalicParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 17 | matchedTokens := tokenizer.GetFirstLine(tokens) 18 | if len(matchedTokens) < 3 { 19 | return nil, 0 20 | } 21 | 22 | prefixTokens := matchedTokens[:1] 23 | if prefixTokens[0].Type != tokenizer.Asterisk && prefixTokens[0].Type != tokenizer.Underscore { 24 | return nil, 0 25 | } 26 | prefixTokenType := prefixTokens[0].Type 27 | contentTokens := []*tokenizer.Token{} 28 | matched := false 29 | for _, token := range matchedTokens[1:] { 30 | if token.Type == prefixTokenType { 31 | matched = true 32 | break 33 | } 34 | contentTokens = append(contentTokens, token) 35 | } 36 | if !matched || len(contentTokens) == 0 { 37 | return nil, 0 38 | } 39 | 40 | children, err := ParseInlineWithParsers(contentTokens, []InlineParser{NewLinkParser(), NewTextParser()}) 41 | if err != nil || len(children) == 0 { 42 | return nil, 0 43 | } 44 | 45 | return &ast.Italic{ 46 | Symbol: prefixTokenType, 47 | Children: children, 48 | }, len(contentTokens) + 2 49 | } 50 | -------------------------------------------------------------------------------- /parser/line_break.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type LineBreakParser struct{} 9 | 10 | func NewLineBreakParser() *LineBreakParser { 11 | return &LineBreakParser{} 12 | } 13 | 14 | func (*LineBreakParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | if len(tokens) == 0 { 16 | return nil, 0 17 | } 18 | if tokens[0].Type != tokenizer.NewLine { 19 | return nil, 0 20 | } 21 | return &ast.LineBreak{}, 1 22 | } 23 | -------------------------------------------------------------------------------- /parser/link.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type LinkParser struct{} 9 | 10 | func NewLinkParser() *LinkParser { 11 | return &LinkParser{} 12 | } 13 | 14 | func (*LinkParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | matchedTokens := tokenizer.GetFirstLine(tokens) 16 | if len(matchedTokens) < 5 { 17 | return nil, 0 18 | } 19 | if matchedTokens[0].Type != tokenizer.LeftSquareBracket { 20 | return nil, 0 21 | } 22 | 23 | rightSquareBracketIndex := tokenizer.FindUnescaped(matchedTokens[1:], tokenizer.RightSquareBracket) 24 | if rightSquareBracketIndex == -1 { 25 | return nil, 0 26 | } 27 | contentTokens := matchedTokens[1 : rightSquareBracketIndex+1] 28 | if tokenizer.FindUnescaped(contentTokens, tokenizer.LeftSquareBracket) != -1 { 29 | return nil, 0 30 | } 31 | if len(contentTokens)+4 >= len(matchedTokens) { 32 | return nil, 0 33 | } 34 | if matchedTokens[2+len(contentTokens)].Type != tokenizer.LeftParenthesis { 35 | return nil, 0 36 | } 37 | urlTokens, matched := []*tokenizer.Token{}, false 38 | for _, token := range matchedTokens[3+len(contentTokens):] { 39 | if token.Type == tokenizer.Space { 40 | return nil, 0 41 | } 42 | if token.Type == tokenizer.RightParenthesis { 43 | matched = true 44 | break 45 | } 46 | urlTokens = append(urlTokens, token) 47 | } 48 | if !matched || len(urlTokens) == 0 { 49 | return nil, 0 50 | } 51 | 52 | contentNodes, err := ParseInlineWithParsers(contentTokens, []InlineParser{NewEscapingCharacterParser(), NewTextParser()}) 53 | if err != nil { 54 | return nil, 0 55 | } 56 | return &ast.Link{ 57 | Content: contentNodes, 58 | URL: tokenizer.Stringify(urlTokens), 59 | }, 4 + len(contentTokens) + len(urlTokens) 60 | } 61 | -------------------------------------------------------------------------------- /parser/math.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type MathParser struct{} 9 | 10 | func NewMathParser() *MathParser { 11 | return &MathParser{} 12 | } 13 | 14 | func (*MathParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | matchedTokens := tokenizer.GetFirstLine(tokens) 16 | if len(matchedTokens) < 3 { 17 | return nil, 0 18 | } 19 | 20 | if matchedTokens[0].Type != tokenizer.DollarSign { 21 | return nil, 0 22 | } 23 | 24 | contentTokens := []*tokenizer.Token{} 25 | matched := false 26 | for _, token := range matchedTokens[1:] { 27 | if token.Type == tokenizer.DollarSign { 28 | matched = true 29 | break 30 | } 31 | contentTokens = append(contentTokens, token) 32 | } 33 | if !matched || len(contentTokens) == 0 { 34 | return nil, 0 35 | } 36 | return &ast.Math{ 37 | Content: tokenizer.Stringify(contentTokens), 38 | }, len(contentTokens) + 2 39 | } 40 | -------------------------------------------------------------------------------- /parser/math_block.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type MathBlockParser struct{} 9 | 10 | func NewMathBlockParser() *MathBlockParser { 11 | return &MathBlockParser{} 12 | } 13 | 14 | func (*MathBlockParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | rows := tokenizer.Split(tokens, tokenizer.NewLine) 16 | if len(rows) < 3 { 17 | return nil, 0 18 | } 19 | firstRow := rows[0] 20 | if len(firstRow) != 2 { 21 | return nil, 0 22 | } 23 | if firstRow[0].Type != tokenizer.DollarSign || firstRow[1].Type != tokenizer.DollarSign { 24 | return nil, 0 25 | } 26 | 27 | contentRows := [][]*tokenizer.Token{} 28 | matched := false 29 | for _, row := range rows[1:] { 30 | if len(row) == 2 && row[0].Type == tokenizer.DollarSign && row[1].Type == tokenizer.DollarSign { 31 | matched = true 32 | break 33 | } 34 | contentRows = append(contentRows, row) 35 | } 36 | if !matched { 37 | return nil, 0 38 | } 39 | 40 | contentTokens := []*tokenizer.Token{} 41 | for index, row := range contentRows { 42 | contentTokens = append(contentTokens, row...) 43 | if index != len(contentRows)-1 { 44 | contentTokens = append(contentTokens, &tokenizer.Token{ 45 | Type: tokenizer.NewLine, 46 | Value: "\n", 47 | }) 48 | } 49 | } 50 | return &ast.MathBlock{ 51 | Content: tokenizer.Stringify(contentTokens), 52 | }, 3 + len(contentTokens) + 3 53 | } 54 | -------------------------------------------------------------------------------- /parser/ordered_list_item.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type OrderedListItemParser struct{} 9 | 10 | func NewOrderedListItemParser() *OrderedListItemParser { 11 | return &OrderedListItemParser{} 12 | } 13 | 14 | func (*OrderedListItemParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | matchedTokens := tokenizer.GetFirstLine(tokens) 16 | indent := 0 17 | for _, token := range matchedTokens { 18 | if token.Type == tokenizer.Space { 19 | indent++ 20 | } else { 21 | break 22 | } 23 | } 24 | if len(matchedTokens) < indent+3 { 25 | return nil, 0 26 | } 27 | 28 | corsor := indent 29 | if matchedTokens[corsor].Type != tokenizer.Number || matchedTokens[corsor+1].Type != tokenizer.Dot || matchedTokens[corsor+2].Type != tokenizer.Space { 30 | return nil, 0 31 | } 32 | 33 | contentTokens := matchedTokens[corsor+3:] 34 | if len(contentTokens) == 0 { 35 | return nil, 0 36 | } 37 | children, err := ParseInline(contentTokens) 38 | if err != nil { 39 | return nil, 0 40 | } 41 | return &ast.OrderedListItem{ 42 | Number: matchedTokens[indent].Value, 43 | Indent: indent, 44 | Children: children, 45 | }, indent + 3 + len(contentTokens) 46 | } 47 | -------------------------------------------------------------------------------- /parser/paragraph.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type ParagraphParser struct { 9 | ContentTokens []*tokenizer.Token 10 | } 11 | 12 | func NewParagraphParser() *ParagraphParser { 13 | return &ParagraphParser{} 14 | } 15 | 16 | func (*ParagraphParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 17 | matchedTokens := tokenizer.GetFirstLine(tokens) 18 | if len(matchedTokens) == 0 { 19 | return nil, 0 20 | } 21 | 22 | children, err := ParseInline(matchedTokens) 23 | if err != nil { 24 | return nil, 0 25 | } 26 | return &ast.Paragraph{ 27 | Children: children, 28 | }, len(matchedTokens) 29 | } 30 | -------------------------------------------------------------------------------- /parser/parser.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type BaseParser interface { 9 | Match(tokens []*tokenizer.Token) (ast.Node, int) 10 | } 11 | 12 | type InlineParser interface { 13 | BaseParser 14 | } 15 | 16 | type BlockParser interface { 17 | BaseParser 18 | } 19 | 20 | func Parse(tokens []*tokenizer.Token) ([]ast.Node, error) { 21 | return ParseBlock(tokens) 22 | } 23 | 24 | var defaultBlockParsers = []BlockParser{ 25 | NewCodeBlockParser(), 26 | NewTableParser(), 27 | NewHorizontalRuleParser(), 28 | NewHeadingParser(), 29 | NewBlockquoteParser(), 30 | NewOrderedListItemParser(), 31 | NewTaskListItemParser(), 32 | NewUnorderedListItemParser(), 33 | NewMathBlockParser(), 34 | NewEmbeddedContentParser(), 35 | NewParagraphParser(), 36 | NewLineBreakParser(), 37 | } 38 | 39 | func ParseBlock(tokens []*tokenizer.Token) ([]ast.Node, error) { 40 | return ParseBlockWithParsers(tokens, defaultBlockParsers) 41 | } 42 | 43 | func ParseBlockWithParsers(tokens []*tokenizer.Token, blockParsers []BlockParser) ([]ast.Node, error) { 44 | nodes := []ast.Node{} 45 | for len(tokens) > 0 { 46 | for _, blockParser := range blockParsers { 47 | node, size := blockParser.Match(tokens) 48 | if node != nil && size != 0 { 49 | // Consume matched tokens. 50 | tokens = tokens[size:] 51 | nodes = append(nodes, node) 52 | break 53 | } 54 | } 55 | } 56 | 57 | nodes = mergeListItemNodes(nodes) 58 | return nodes, nil 59 | } 60 | 61 | var defaultInlineParsers = []InlineParser{ 62 | NewEscapingCharacterParser(), 63 | NewHTMLElementParser(), 64 | NewBoldItalicParser(), 65 | NewImageParser(), 66 | NewLinkParser(), 67 | NewAutoLinkParser(), 68 | NewBoldParser(), 69 | NewItalicParser(), 70 | NewSpoilerParser(), 71 | NewHighlightParser(), 72 | NewCodeParser(), 73 | NewSubscriptParser(), 74 | NewSuperscriptParser(), 75 | NewMathParser(), 76 | NewReferencedContentParser(), 77 | NewTagParser(), 78 | NewStrikethroughParser(), 79 | NewLineBreakParser(), 80 | NewTextParser(), 81 | } 82 | 83 | func ParseInline(tokens []*tokenizer.Token) ([]ast.Node, error) { 84 | return ParseInlineWithParsers(tokens, defaultInlineParsers) 85 | } 86 | 87 | func ParseInlineWithParsers(tokens []*tokenizer.Token, inlineParsers []InlineParser) ([]ast.Node, error) { 88 | nodes := []ast.Node{} 89 | for len(tokens) > 0 { 90 | for _, inlineParser := range inlineParsers { 91 | node, size := inlineParser.Match(tokens) 92 | if node != nil && size != 0 { 93 | // Consume matched tokens. 94 | tokens = tokens[size:] 95 | nodes = append(nodes, node) 96 | break 97 | } 98 | } 99 | } 100 | return mergeTextNodes(nodes), nil 101 | } 102 | 103 | func mergeListItemNodes(nodes []ast.Node) []ast.Node { 104 | var result []ast.Node 105 | var stack []*ast.List 106 | 107 | for _, node := range nodes { 108 | nodeType := node.Type() 109 | 110 | // Handle line breaks. 111 | if nodeType == ast.LineBreakNode { 112 | // If the stack is not empty and the last node is a list node, add the line break to the list. 113 | if len(stack) > 0 && len(result) > 0 && result[len(result)-1].Type() == ast.ListNode { 114 | stack[len(stack)-1].Children = append(stack[len(stack)-1].Children, node) 115 | } else { 116 | result = append(result, node) 117 | } 118 | continue 119 | } 120 | 121 | if ast.IsListItemNode(node) { 122 | itemKind, itemIndent := ast.GetListItemKindAndIndent(node) 123 | 124 | // Create a new List node if the stack is empty or the current item should be a child of the last item. 125 | if len(stack) == 0 || (itemKind != stack[len(stack)-1].Kind || itemIndent > stack[len(stack)-1].Indent) { 126 | newList := &ast.List{ 127 | Kind: itemKind, 128 | Indent: itemIndent, 129 | Children: []ast.Node{node}, 130 | } 131 | 132 | // Add the new List node to the stack or the result. 133 | if len(stack) > 0 && itemIndent > stack[len(stack)-1].Indent { 134 | stack[len(stack)-1].Children = append(stack[len(stack)-1].Children, newList) 135 | } else { 136 | result = append(result, newList) 137 | } 138 | stack = append(stack, newList) 139 | } else { 140 | // Pop the stack until the current item should be a sibling of the last item. 141 | for len(stack) > 0 && (itemKind != stack[len(stack)-1].Kind || itemIndent < stack[len(stack)-1].Indent) { 142 | stack = stack[:len(stack)-1] 143 | } 144 | 145 | // Add the current item to the last List node in the stack or the result. 146 | if len(stack) > 0 { 147 | stack[len(stack)-1].Children = append(stack[len(stack)-1].Children, node) 148 | } else { 149 | result = append(result, node) 150 | } 151 | } 152 | } else { 153 | result = append(result, node) 154 | // Reset the stack if the current node is not a list item node. 155 | stack = nil 156 | } 157 | } 158 | 159 | return result 160 | } 161 | 162 | func mergeTextNodes(nodes []ast.Node) []ast.Node { 163 | if len(nodes) == 0 { 164 | return nodes 165 | } 166 | result := []ast.Node{nodes[0]} 167 | for i := 1; i < len(nodes); i++ { 168 | if nodes[i].Type() == ast.TextNode && result[len(result)-1].Type() == ast.TextNode { 169 | result[len(result)-1].(*ast.Text).Content += nodes[i].(*ast.Text).Content 170 | } else { 171 | result = append(result, nodes[i]) 172 | } 173 | } 174 | return result 175 | } 176 | -------------------------------------------------------------------------------- /parser/referenced_content.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type ReferencedContentParser struct{} 9 | 10 | func NewReferencedContentParser() *ReferencedContentParser { 11 | return &ReferencedContentParser{} 12 | } 13 | 14 | func (*ReferencedContentParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | matchedTokens := tokenizer.GetFirstLine(tokens) 16 | if len(matchedTokens) < 5 { 17 | return nil, 0 18 | } 19 | if matchedTokens[0].Type != tokenizer.LeftSquareBracket || matchedTokens[1].Type != tokenizer.LeftSquareBracket { 20 | return nil, 0 21 | } 22 | 23 | contentTokens := []*tokenizer.Token{} 24 | matched := false 25 | for index, token := range matchedTokens[2 : len(matchedTokens)-1] { 26 | if token.Type == tokenizer.RightSquareBracket && matchedTokens[2+index+1].Type == tokenizer.RightSquareBracket { 27 | matched = true 28 | break 29 | } 30 | contentTokens = append(contentTokens, token) 31 | } 32 | if !matched { 33 | return nil, 0 34 | } 35 | 36 | resourceName, params := tokenizer.Stringify(contentTokens), "" 37 | questionMarkIndex := tokenizer.FindUnescaped(contentTokens, tokenizer.QuestionMark) 38 | if questionMarkIndex > 0 { 39 | resourceName, params = tokenizer.Stringify(contentTokens[:questionMarkIndex]), tokenizer.Stringify(contentTokens[questionMarkIndex+1:]) 40 | } 41 | return &ast.ReferencedContent{ 42 | ResourceName: resourceName, 43 | Params: params, 44 | }, len(contentTokens) + 4 45 | } 46 | -------------------------------------------------------------------------------- /parser/spoiler.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type SpoilerParser struct{} 9 | 10 | func NewSpoilerParser() InlineParser { 11 | return &SpoilerParser{} 12 | } 13 | 14 | func (*SpoilerParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | matchedTokens := tokenizer.GetFirstLine(tokens) 16 | if len(matchedTokens) < 5 { 17 | return nil, 0 18 | } 19 | 20 | prefixTokens := matchedTokens[:2] 21 | if prefixTokens[0].Type != prefixTokens[1].Type { 22 | return nil, 0 23 | } 24 | prefixTokenType := prefixTokens[0].Type 25 | if prefixTokenType != tokenizer.Pipe { 26 | return nil, 0 27 | } 28 | 29 | cursor, matched := 2, false 30 | for ; cursor < len(matchedTokens)-1; cursor++ { 31 | token, nextToken := matchedTokens[cursor], matchedTokens[cursor+1] 32 | if token.Type == tokenizer.NewLine || nextToken.Type == tokenizer.NewLine { 33 | return nil, 0 34 | } 35 | if token.Type == prefixTokenType && nextToken.Type == prefixTokenType { 36 | matchedTokens = matchedTokens[:cursor+2] 37 | matched = true 38 | break 39 | } 40 | } 41 | if !matched { 42 | return nil, 0 43 | } 44 | 45 | size := len(matchedTokens) 46 | content := tokenizer.Stringify(matchedTokens[2 : size-2]) 47 | return &ast.Spoiler{ 48 | Content: content, 49 | }, size 50 | } 51 | -------------------------------------------------------------------------------- /parser/strikethrough.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type StrikethroughParser struct{} 9 | 10 | func NewStrikethroughParser() *StrikethroughParser { 11 | return &StrikethroughParser{} 12 | } 13 | 14 | func (*StrikethroughParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | matchedTokens := tokenizer.GetFirstLine(tokens) 16 | if len(matchedTokens) < 5 { 17 | return nil, 0 18 | } 19 | if matchedTokens[0].Type != tokenizer.Tilde || matchedTokens[1].Type != tokenizer.Tilde { 20 | return nil, 0 21 | } 22 | 23 | contentTokens := []*tokenizer.Token{} 24 | matched := false 25 | for cursor := 2; cursor < len(matchedTokens)-1; cursor++ { 26 | token, nextToken := matchedTokens[cursor], matchedTokens[cursor+1] 27 | if token.Type == tokenizer.Tilde && nextToken.Type == tokenizer.Tilde { 28 | matched = true 29 | break 30 | } 31 | contentTokens = append(contentTokens, token) 32 | } 33 | if !matched || len(contentTokens) == 0 { 34 | return nil, 0 35 | } 36 | return &ast.Strikethrough{ 37 | Content: tokenizer.Stringify(contentTokens), 38 | }, len(contentTokens) + 4 39 | } 40 | -------------------------------------------------------------------------------- /parser/subscript.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type SubscriptParser struct{} 9 | 10 | func NewSubscriptParser() *SubscriptParser { 11 | return &SubscriptParser{} 12 | } 13 | 14 | func (*SubscriptParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | matchedTokens := tokenizer.GetFirstLine(tokens) 16 | if len(matchedTokens) < 3 { 17 | return nil, 0 18 | } 19 | if matchedTokens[0].Type != tokenizer.Tilde { 20 | return nil, 0 21 | } 22 | 23 | contentTokens := []*tokenizer.Token{} 24 | matched := false 25 | for _, token := range matchedTokens[1:] { 26 | if token.Type == tokenizer.Tilde { 27 | matched = true 28 | break 29 | } 30 | contentTokens = append(contentTokens, token) 31 | } 32 | if !matched || len(contentTokens) == 0 { 33 | return nil, 0 34 | } 35 | 36 | return &ast.Subscript{ 37 | Content: tokenizer.Stringify(contentTokens), 38 | }, len(contentTokens) + 2 39 | } 40 | -------------------------------------------------------------------------------- /parser/superscript.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type SuperscriptParser struct{} 9 | 10 | func NewSuperscriptParser() *SuperscriptParser { 11 | return &SuperscriptParser{} 12 | } 13 | 14 | func (*SuperscriptParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | matchedTokens := tokenizer.GetFirstLine(tokens) 16 | if len(matchedTokens) < 3 { 17 | return nil, 0 18 | } 19 | if matchedTokens[0].Type != tokenizer.Caret { 20 | return nil, 0 21 | } 22 | 23 | contentTokens := []*tokenizer.Token{} 24 | matched := false 25 | for _, token := range matchedTokens[1:] { 26 | if token.Type == tokenizer.Caret { 27 | matched = true 28 | break 29 | } 30 | contentTokens = append(contentTokens, token) 31 | } 32 | if !matched || len(contentTokens) == 0 { 33 | return nil, 0 34 | } 35 | 36 | return &ast.Superscript{ 37 | Content: tokenizer.Stringify(contentTokens), 38 | }, len(contentTokens) + 2 39 | } 40 | -------------------------------------------------------------------------------- /parser/table.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type TableParser struct{} 9 | 10 | func NewTableParser() *TableParser { 11 | return &TableParser{} 12 | } 13 | 14 | func (*TableParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | rawRows := tokenizer.Split(tokens, tokenizer.NewLine) 16 | if len(rawRows) < 3 { 17 | return nil, 0 18 | } 19 | 20 | headerTokens := rawRows[0] 21 | if len(headerTokens) < 3 { 22 | return nil, 0 23 | } 24 | 25 | delimiterTokens := rawRows[1] 26 | if len(delimiterTokens) < 3 { 27 | return nil, 0 28 | } 29 | 30 | // Check header. 31 | if len(headerTokens) < 5 { 32 | return nil, 0 33 | } 34 | headerCells, ok := matchTableCellTokens(headerTokens) 35 | if headerCells == 0 || !ok { 36 | return nil, 0 37 | } 38 | 39 | // Check delimiter. 40 | if len(delimiterTokens) < 5 { 41 | return nil, 0 42 | } 43 | delimiterCells, ok := matchTableCellTokens(delimiterTokens) 44 | if delimiterCells != headerCells || !ok { 45 | return nil, 0 46 | } 47 | for index, t := range tokenizer.Split(delimiterTokens, tokenizer.Pipe) { 48 | if index == 0 || index == headerCells { 49 | if len(t) != 0 { 50 | return nil, 0 51 | } 52 | continue 53 | } 54 | // Each delimiter cell should be like ` --- `, ` :-- `, ` --: `, ` :-: `. 55 | if len(t) < 5 { 56 | return nil, 0 57 | } 58 | 59 | delimiterTokens := t[1 : len(t)-1] 60 | if len(delimiterTokens) < 3 { 61 | return nil, 0 62 | } 63 | if (delimiterTokens[0].Type != tokenizer.Colon && 64 | delimiterTokens[0].Type != tokenizer.Hyphen) || 65 | (delimiterTokens[len(delimiterTokens)-1].Type != tokenizer.Colon && 66 | delimiterTokens[len(delimiterTokens)-1].Type != tokenizer.Hyphen) { 67 | return nil, 0 68 | } 69 | for _, token := range delimiterTokens[1 : len(delimiterTokens)-1] { 70 | if token.Type != tokenizer.Hyphen { 71 | return nil, 0 72 | } 73 | } 74 | } 75 | 76 | // Check rows. 77 | rows := rawRows[2:] 78 | matchedRows := 0 79 | for _, rowTokens := range rows { 80 | cells, ok := matchTableCellTokens(rowTokens) 81 | if cells != headerCells || !ok { 82 | break 83 | } 84 | matchedRows++ 85 | } 86 | if matchedRows == 0 { 87 | return nil, 0 88 | } 89 | rows = rows[:matchedRows] 90 | 91 | headerNodes := make([]ast.Node, 0) 92 | delimiter := make([]string, 0) 93 | rowsNodes := make([][]ast.Node, 0) 94 | 95 | cols := len(tokenizer.Split(headerTokens, tokenizer.Pipe)) - 2 96 | for _, t := range tokenizer.Split(headerTokens, tokenizer.Pipe)[1 : cols+1] { 97 | if len(t) < 3 { 98 | headerNodes = append(headerNodes, &ast.Text{}) 99 | } else { 100 | cellTokens := t[1 : len(t)-1] 101 | nodes, err := ParseBlockWithParsers(cellTokens, []BlockParser{NewHeadingParser(), NewParagraphParser()}) 102 | if err != nil { 103 | return nil, 0 104 | } 105 | if len(nodes) != 1 { 106 | return nil, 0 107 | } 108 | headerNodes = append(headerNodes, nodes[0]) 109 | } 110 | } 111 | for _, t := range tokenizer.Split(delimiterTokens, tokenizer.Pipe)[1 : cols+1] { 112 | if len(t) < 3 { 113 | delimiter = append(delimiter, "") 114 | } else { 115 | delimiter = append(delimiter, tokenizer.Stringify(t[1:len(t)-1])) 116 | } 117 | } 118 | for _, row := range rows { 119 | rowNodes := make([]ast.Node, 0) 120 | for _, t := range tokenizer.Split(row, tokenizer.Pipe)[1 : cols+1] { 121 | if len(t) < 3 { 122 | rowNodes = append(rowNodes, &ast.Text{}) 123 | } else { 124 | nodes, err := ParseBlockWithParsers(t[1:len(t)-1], []BlockParser{NewHeadingParser(), NewParagraphParser()}) 125 | if err != nil { 126 | return nil, 0 127 | } 128 | if len(nodes) != 1 { 129 | return nil, 0 130 | } 131 | rowNodes = append(rowNodes, nodes[0]) 132 | } 133 | } 134 | rowsNodes = append(rowsNodes, rowNodes) 135 | } 136 | 137 | size := len(headerTokens) + len(delimiterTokens) + 2 138 | for _, row := range rows { 139 | size += len(row) 140 | } 141 | size = size + len(rows) - 1 142 | 143 | return &ast.Table{ 144 | Header: headerNodes, 145 | Delimiter: delimiter, 146 | Rows: rowsNodes, 147 | }, size 148 | } 149 | 150 | func matchTableCellTokens(tokens []*tokenizer.Token) (int, bool) { 151 | if len(tokens) == 0 { 152 | return 0, false 153 | } 154 | 155 | pipes := 0 156 | for _, token := range tokens { 157 | if token.Type == tokenizer.Pipe { 158 | pipes++ 159 | } 160 | } 161 | cells := tokenizer.Split(tokens, tokenizer.Pipe) 162 | if len(cells) != pipes+1 { 163 | return 0, false 164 | } 165 | if len(cells[0]) != 0 || len(cells[len(cells)-1]) != 0 { 166 | return 0, false 167 | } 168 | for _, cellTokens := range cells[1 : len(cells)-1] { 169 | if len(cellTokens) == 0 { 170 | return 0, false 171 | } 172 | if cellTokens[0].Type != tokenizer.Space { 173 | return 0, false 174 | } 175 | if cellTokens[len(cellTokens)-1].Type != tokenizer.Space { 176 | return 0, false 177 | } 178 | } 179 | 180 | return len(cells) - 1, true 181 | } 182 | -------------------------------------------------------------------------------- /parser/tag.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type TagParser struct{} 9 | 10 | func NewTagParser() *TagParser { 11 | return &TagParser{} 12 | } 13 | 14 | func (*TagParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | matchedTokens := tokenizer.GetFirstLine(tokens) 16 | if len(matchedTokens) < 2 { 17 | return nil, 0 18 | } 19 | if matchedTokens[0].Type != tokenizer.PoundSign { 20 | return nil, 0 21 | } 22 | 23 | contentTokens := []*tokenizer.Token{} 24 | for _, token := range matchedTokens[1:] { 25 | if token.Type == tokenizer.Space || token.Type == tokenizer.PoundSign || token.Type == tokenizer.Backslash { 26 | break 27 | } 28 | contentTokens = append(contentTokens, token) 29 | } 30 | if len(contentTokens) == 0 { 31 | return nil, 0 32 | } 33 | 34 | return &ast.Tag{ 35 | Content: tokenizer.Stringify(contentTokens), 36 | }, len(contentTokens) + 1 37 | } 38 | -------------------------------------------------------------------------------- /parser/task_list_item.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type TaskListItemParser struct{} 9 | 10 | func NewTaskListItemParser() *TaskListItemParser { 11 | return &TaskListItemParser{} 12 | } 13 | 14 | func (*TaskListItemParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | matchedTokens := tokenizer.GetFirstLine(tokens) 16 | indent := 0 17 | for _, token := range matchedTokens { 18 | if token.Type == tokenizer.Space { 19 | indent++ 20 | } else { 21 | break 22 | } 23 | } 24 | if len(matchedTokens) < indent+6 { 25 | return nil, 0 26 | } 27 | 28 | symbolToken := matchedTokens[indent] 29 | if symbolToken.Type != tokenizer.Hyphen && symbolToken.Type != tokenizer.Asterisk && symbolToken.Type != tokenizer.PlusSign { 30 | return nil, 0 31 | } 32 | if matchedTokens[indent+1].Type != tokenizer.Space { 33 | return nil, 0 34 | } 35 | if matchedTokens[indent+2].Type != tokenizer.LeftSquareBracket || (matchedTokens[indent+3].Type != tokenizer.Space && matchedTokens[indent+3].Value != "x") || matchedTokens[indent+4].Type != tokenizer.RightSquareBracket { 36 | return nil, 0 37 | } 38 | if matchedTokens[indent+5].Type != tokenizer.Space { 39 | return nil, 0 40 | } 41 | 42 | contentTokens := matchedTokens[indent+6:] 43 | if len(contentTokens) == 0 { 44 | return nil, 0 45 | } 46 | children, err := ParseInline(contentTokens) 47 | if err != nil { 48 | return nil, 0 49 | } 50 | return &ast.TaskListItem{ 51 | Symbol: symbolToken.Type, 52 | Indent: indent, 53 | Complete: matchedTokens[indent+3].Value == "x", 54 | Children: children, 55 | }, indent + len(contentTokens) + 6 56 | } 57 | -------------------------------------------------------------------------------- /parser/tests/auto_link_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/usememos/gomark/ast" 9 | "github.com/usememos/gomark/parser" 10 | "github.com/usememos/gomark/parser/tokenizer" 11 | ) 12 | 13 | func TestAutoLinkParser(t *testing.T) { 14 | tests := []struct { 15 | text string 16 | node ast.Node 17 | }{ 18 | { 19 | text: "", 24 | node: &ast.AutoLink{ 25 | URL: "https://example.com", 26 | }, 27 | }, 28 | { 29 | text: "https://example.com", 30 | node: &ast.AutoLink{ 31 | URL: "https://example.com", 32 | IsRawText: true, 33 | }, 34 | }, 35 | } 36 | 37 | for _, test := range tests { 38 | tokens := tokenizer.Tokenize(test.text) 39 | node, _ := parser.NewAutoLinkParser().Match(tokens) 40 | require.Equal(t, node, test.node) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /parser/tests/blockquote_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/usememos/gomark/ast" 9 | "github.com/usememos/gomark/parser" 10 | "github.com/usememos/gomark/parser/tokenizer" 11 | ) 12 | 13 | func TestBlockquoteParser(t *testing.T) { 14 | tests := []struct { 15 | text string 16 | node ast.Node 17 | }{ 18 | { 19 | text: ">Hello world", 20 | node: nil, 21 | }, 22 | { 23 | text: "> Hello world", 24 | node: &ast.Blockquote{ 25 | Children: []ast.Node{ 26 | &ast.Paragraph{ 27 | Children: []ast.Node{ 28 | &ast.Text{ 29 | Content: "Hello world", 30 | }, 31 | }, 32 | }, 33 | }, 34 | }, 35 | }, 36 | { 37 | text: "> 你好", 38 | node: &ast.Blockquote{ 39 | Children: []ast.Node{ 40 | &ast.Paragraph{ 41 | Children: []ast.Node{ 42 | &ast.Text{ 43 | Content: "你好", 44 | }, 45 | }, 46 | }, 47 | }, 48 | }, 49 | }, 50 | { 51 | text: "> Hello\n> world", 52 | node: &ast.Blockquote{ 53 | Children: []ast.Node{ 54 | &ast.Paragraph{ 55 | Children: []ast.Node{ 56 | &ast.Text{ 57 | Content: "Hello", 58 | }, 59 | }, 60 | }, 61 | &ast.Paragraph{ 62 | Children: []ast.Node{ 63 | &ast.Text{ 64 | Content: "world", 65 | }, 66 | }, 67 | }, 68 | }, 69 | }, 70 | }, 71 | { 72 | text: "> Hello\n> > world", 73 | node: &ast.Blockquote{ 74 | Children: []ast.Node{ 75 | &ast.Paragraph{ 76 | Children: []ast.Node{ 77 | &ast.Text{ 78 | Content: "Hello", 79 | }, 80 | }, 81 | }, 82 | &ast.Blockquote{ 83 | Children: []ast.Node{ 84 | &ast.Paragraph{ 85 | Children: []ast.Node{ 86 | &ast.Text{ 87 | Content: "world", 88 | }, 89 | }, 90 | }, 91 | }, 92 | }, 93 | }, 94 | }, 95 | }, 96 | } 97 | 98 | for _, test := range tests { 99 | tokens := tokenizer.Tokenize(test.text) 100 | node, _ := parser.NewBlockquoteParser().Match(tokens) 101 | require.Equal(t, test.node, node) 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /parser/tests/bold_italic_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/usememos/gomark/ast" 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestBoldItalicParser(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | node ast.Node 16 | }{ 17 | { 18 | text: "*Hello world!", 19 | node: nil, 20 | }, 21 | { 22 | text: "*** Hello * *", 23 | node: nil, 24 | }, 25 | { 26 | text: "*** Hello **", 27 | node: nil, 28 | }, 29 | { 30 | text: "***Hello***", 31 | node: &ast.BoldItalic{ 32 | Symbol: "*", 33 | Content: "Hello", 34 | }, 35 | }, 36 | { 37 | text: "*** Hello ***", 38 | node: &ast.BoldItalic{ 39 | Symbol: "*", 40 | Content: " Hello ", 41 | }, 42 | }, 43 | } 44 | 45 | for _, test := range tests { 46 | tokens := tokenizer.Tokenize(test.text) 47 | node, _ := parser.NewBoldItalicParser().Match(tokens) 48 | require.Equal(t, test.node, node) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /parser/tests/bold_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/usememos/gomark/ast" 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestBoldParser(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | node ast.Node 16 | }{ 17 | { 18 | text: "*Hello world!", 19 | node: nil, 20 | }, 21 | { 22 | text: "****", 23 | node: nil, 24 | }, 25 | { 26 | text: "**Hello**", 27 | node: &ast.Bold{ 28 | Symbol: "*", 29 | Children: []ast.Node{ 30 | &ast.Text{ 31 | Content: "Hello", 32 | }, 33 | }, 34 | }, 35 | }, 36 | { 37 | text: "** Hello **", 38 | node: &ast.Bold{ 39 | Symbol: "*", 40 | Children: []ast.Node{ 41 | &ast.Text{ 42 | Content: " Hello ", 43 | }, 44 | }, 45 | }, 46 | }, 47 | { 48 | text: "** Hello * *", 49 | node: nil, 50 | }, 51 | { 52 | text: "* * Hello **", 53 | node: nil, 54 | }, 55 | { 56 | text: "__Hello__", 57 | node: &ast.Bold{ 58 | Symbol: "_", 59 | Children: []ast.Node{ 60 | &ast.Text{ 61 | Content: "Hello", 62 | }, 63 | }, 64 | }, 65 | }, 66 | { 67 | text: "__ Hello __", 68 | node: &ast.Bold{ 69 | Symbol: "_", 70 | Children: []ast.Node{ 71 | &ast.Text{ 72 | Content: " Hello ", 73 | }, 74 | }, 75 | }, 76 | }, 77 | { 78 | text: "__ Hello _ _", 79 | node: nil, 80 | }, 81 | { 82 | text: "_ _ Hello __", 83 | node: nil, 84 | }, 85 | } 86 | 87 | for _, test := range tests { 88 | tokens := tokenizer.Tokenize(test.text) 89 | node, _ := parser.NewBoldParser().Match(tokens) 90 | require.Equal(t, test.node, node) 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /parser/tests/code_block_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/usememos/gomark/ast" 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestCodeBlockParser(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | node ast.Node 16 | }{ 17 | { 18 | text: "```Hello world!```", 19 | node: nil, 20 | }, 21 | { 22 | text: "```\nHello\n```", 23 | node: &ast.CodeBlock{ 24 | Language: "", 25 | Content: "Hello", 26 | }, 27 | }, 28 | { 29 | text: "```\nHello world!\n```", 30 | node: &ast.CodeBlock{ 31 | Language: "", 32 | Content: "Hello world!", 33 | }, 34 | }, 35 | { 36 | text: "```java\nHello \n world!\n```", 37 | node: &ast.CodeBlock{ 38 | Language: "java", 39 | Content: "Hello \n world!", 40 | }, 41 | }, 42 | { 43 | text: "```java\nHello \n world!\n```111", 44 | node: nil, 45 | }, 46 | { 47 | text: "```java\nHello \n world!\n``` 111", 48 | node: nil, 49 | }, 50 | { 51 | text: "```java\nHello \n world!\n```\n123123", 52 | node: &ast.CodeBlock{ 53 | Language: "java", 54 | Content: "Hello \n world!", 55 | }, 56 | }, 57 | } 58 | 59 | for _, test := range tests { 60 | tokens := tokenizer.Tokenize(test.text) 61 | node, _ := parser.NewCodeBlockParser().Match(tokens) 62 | require.Equal(t, test.node, node) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /parser/tests/code_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/usememos/gomark/ast" 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestCodeParser(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | node ast.Node 16 | }{ 17 | { 18 | text: "`Hello world!", 19 | node: nil, 20 | }, 21 | { 22 | text: "`Hello world!`", 23 | node: &ast.Code{ 24 | Content: "Hello world!", 25 | }, 26 | }, 27 | { 28 | text: "`Hello \nworld!`", 29 | node: nil, 30 | }, 31 | } 32 | 33 | for _, test := range tests { 34 | tokens := tokenizer.Tokenize(test.text) 35 | node, _ := parser.NewCodeParser().Match(tokens) 36 | require.Equal(t, test.node, node) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /parser/tests/embedded_content_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/usememos/gomark/ast" 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestEmbeddedContentParser(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | node ast.Node 16 | }{ 17 | { 18 | text: "![[Hello world]", 19 | node: nil, 20 | }, 21 | { 22 | text: "![[Hello world]]", 23 | node: &ast.EmbeddedContent{ 24 | ResourceName: "Hello world", 25 | }, 26 | }, 27 | { 28 | text: "![[memos/1]]", 29 | node: &ast.EmbeddedContent{ 30 | ResourceName: "memos/1", 31 | }, 32 | }, 33 | { 34 | text: "![[resources/101]] \n123", 35 | node: nil, 36 | }, 37 | { 38 | text: "![[resources/101]]\n123", 39 | node: &ast.EmbeddedContent{ 40 | ResourceName: "resources/101", 41 | }, 42 | }, 43 | { 44 | text: "![[resources/101?align=center]]\n123", 45 | node: &ast.EmbeddedContent{ 46 | ResourceName: "resources/101", 47 | Params: "align=center", 48 | }, 49 | }, 50 | { 51 | text: "![[resources/6uxnhT98q8vN8anBbUbRGu?align=center]]", 52 | node: &ast.EmbeddedContent{ 53 | ResourceName: "resources/6uxnhT98q8vN8anBbUbRGu", 54 | Params: "align=center", 55 | }, 56 | }, 57 | } 58 | 59 | for _, test := range tests { 60 | tokens := tokenizer.Tokenize(test.text) 61 | node, _ := parser.NewEmbeddedContentParser().Match(tokens) 62 | require.Equal(t, test.node, node) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /parser/tests/escaping_character_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/usememos/gomark/ast" 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestEscapingCharacterParser(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | node ast.Node 16 | }{ 17 | { 18 | text: `\#`, 19 | node: &ast.EscapingCharacter{ 20 | Symbol: "#", 21 | }, 22 | }, 23 | { 24 | text: `\' test`, 25 | node: &ast.EscapingCharacter{ 26 | Symbol: "'", 27 | }, 28 | }, 29 | } 30 | 31 | for _, test := range tests { 32 | tokens := tokenizer.Tokenize(test.text) 33 | node, _ := parser.NewEscapingCharacterParser().Match(tokens) 34 | require.Equal(t, test.node, node) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /parser/tests/heading_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/usememos/gomark/ast" 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestHeadingParser(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | node ast.Node 16 | }{ 17 | { 18 | text: "*Hello world", 19 | node: nil, 20 | }, 21 | { 22 | text: "## Hello World\n123", 23 | node: &ast.Heading{ 24 | Level: 2, 25 | Children: []ast.Node{ 26 | &ast.Text{ 27 | Content: "Hello World", 28 | }, 29 | }, 30 | }, 31 | }, 32 | { 33 | text: "# # Hello World", 34 | node: &ast.Heading{ 35 | Level: 1, 36 | Children: []ast.Node{ 37 | &ast.Text{ 38 | Content: "# Hello World", 39 | }, 40 | }, 41 | }, 42 | }, 43 | { 44 | text: " # 123123 Hello World", 45 | node: nil, 46 | }, 47 | { 48 | text: `# 123 49 | Hello World`, 50 | node: &ast.Heading{ 51 | Level: 1, 52 | Children: []ast.Node{ 53 | &ast.Text{ 54 | Content: "123 ", 55 | }, 56 | }, 57 | }, 58 | }, 59 | { 60 | text: "### **Hello** World", 61 | node: &ast.Heading{ 62 | Level: 3, 63 | Children: []ast.Node{ 64 | &ast.Bold{ 65 | Symbol: "*", 66 | Children: []ast.Node{ 67 | &ast.Text{ 68 | Content: "Hello", 69 | }, 70 | }, 71 | }, 72 | &ast.Text{ 73 | Content: " World", 74 | }, 75 | }, 76 | }, 77 | }, 78 | } 79 | 80 | for _, test := range tests { 81 | tokens := tokenizer.Tokenize(test.text) 82 | node, _ := parser.NewHeadingParser().Match(tokens) 83 | require.Equal(t, test.node, node) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /parser/tests/highlight_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/usememos/gomark/ast" 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestHighlightParser(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | node ast.Node 16 | }{ 17 | { 18 | text: "==Hello world!", 19 | node: nil, 20 | }, 21 | { 22 | text: "==Hello==", 23 | node: &ast.Highlight{ 24 | Content: "Hello", 25 | }, 26 | }, 27 | { 28 | text: "==Hello world==", 29 | node: &ast.Highlight{ 30 | Content: "Hello world", 31 | }, 32 | }, 33 | } 34 | 35 | for _, test := range tests { 36 | tokens := tokenizer.Tokenize(test.text) 37 | node, _ := parser.NewHighlightParser().Match(tokens) 38 | require.Equal(t, test.node, node) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /parser/tests/horizontal_rule_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/usememos/gomark/ast" 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestHorizontalRuleParser(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | node ast.Node 16 | }{ 17 | { 18 | text: "---", 19 | node: &ast.HorizontalRule{ 20 | Symbol: "-", 21 | }, 22 | }, 23 | { 24 | text: "---\naaa", 25 | node: &ast.HorizontalRule{ 26 | Symbol: "-", 27 | }, 28 | }, 29 | { 30 | text: "****", 31 | node: nil, 32 | }, 33 | { 34 | text: "***", 35 | node: &ast.HorizontalRule{ 36 | Symbol: "*", 37 | }, 38 | }, 39 | { 40 | text: "-*-", 41 | node: nil, 42 | }, 43 | } 44 | 45 | for _, test := range tests { 46 | tokens := tokenizer.Tokenize(test.text) 47 | node, _ := parser.NewHorizontalRuleParser().Match(tokens) 48 | require.Equal(t, test.node, node) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /parser/tests/html_element_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/usememos/gomark/ast" 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestHTMLElementParser(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | node ast.Node 16 | }{ 17 | { 18 | text: "
", 19 | node: &ast.HTMLElement{ 20 | TagName: "br", 21 | Attributes: map[string]string{}, 22 | }, 23 | }, 24 | } 25 | 26 | for _, test := range tests { 27 | tokens := tokenizer.Tokenize(test.text) 28 | node, _ := parser.NewHTMLElementParser().Match(tokens) 29 | require.Equal(t, test.node, node) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /parser/tests/image_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/usememos/gomark/ast" 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestImageParser(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | node ast.Node 16 | }{ 17 | { 18 | text: "![](https://example.com)", 19 | node: &ast.Image{ 20 | AltText: "", 21 | URL: "https://example.com", 22 | }, 23 | }, 24 | { 25 | text: "! [](https://example.com)", 26 | node: nil, 27 | }, 28 | { 29 | text: "![alte]( htt ps :/ /example.com)", 30 | node: nil, 31 | }, 32 | { 33 | text: "![al te](https://example.com)", 34 | node: &ast.Image{ 35 | AltText: "al te", 36 | URL: "https://example.com", 37 | }, 38 | }, 39 | } 40 | for _, test := range tests { 41 | tokens := tokenizer.Tokenize(test.text) 42 | node, _ := parser.NewImageParser().Match(tokens) 43 | require.Equal(t, test.node, node) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /parser/tests/italic_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/usememos/gomark/ast" 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestItalicParser(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | node ast.Node 16 | }{ 17 | { 18 | text: "*Hello world!", 19 | node: nil, 20 | }, 21 | { 22 | text: "*Hello*", 23 | node: &ast.Italic{ 24 | Symbol: "*", 25 | Children: []ast.Node{ 26 | &ast.Text{ 27 | Content: "Hello", 28 | }, 29 | }, 30 | }, 31 | }, 32 | { 33 | text: "* Hello *", 34 | node: &ast.Italic{ 35 | Symbol: "*", 36 | Children: []ast.Node{ 37 | &ast.Text{ 38 | Content: " Hello ", 39 | }, 40 | }, 41 | }, 42 | }, 43 | { 44 | text: "*1* Hello * *", 45 | node: &ast.Italic{ 46 | Symbol: "*", 47 | Children: []ast.Node{ 48 | &ast.Text{ 49 | Content: "1", 50 | }, 51 | }, 52 | }, 53 | }, 54 | { 55 | text: "_Hello_", 56 | node: &ast.Italic{ 57 | Symbol: "_", 58 | Children: []ast.Node{ 59 | &ast.Text{ 60 | Content: "Hello", 61 | }, 62 | }, 63 | }, 64 | }, 65 | { 66 | text: "_ Hello _", 67 | node: &ast.Italic{ 68 | Symbol: "_", 69 | Children: []ast.Node{ 70 | &ast.Text{ 71 | Content: " Hello ", 72 | }, 73 | }, 74 | }, 75 | }, 76 | { 77 | text: "_1_ Hello _ _", 78 | node: &ast.Italic{ 79 | Symbol: "_", 80 | Children: []ast.Node{ 81 | &ast.Text{ 82 | Content: "1", 83 | }, 84 | }, 85 | }, 86 | }, 87 | { 88 | text: "*[Hello](https://example.com)*", 89 | node: &ast.Italic{ 90 | Symbol: "*", 91 | Children: []ast.Node{ 92 | &ast.Link{ 93 | Content: []ast.Node{ 94 | &ast.Text{ 95 | Content: "Hello", 96 | }, 97 | }, 98 | URL: "https://example.com", 99 | }, 100 | }, 101 | }, 102 | }, 103 | } 104 | 105 | for _, test := range tests { 106 | tokens := tokenizer.Tokenize(test.text) 107 | node, _ := parser.NewItalicParser().Match(tokens) 108 | require.Equal(t, test.node, node) 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /parser/tests/link_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/usememos/gomark/ast" 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestLinkParser(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | node ast.Node 16 | }{ 17 | { 18 | text: "[](https://example.com)", 19 | node: &ast.Link{ 20 | Content: []ast.Node{}, 21 | URL: "https://example.com", 22 | }, 23 | }, 24 | { 25 | text: "! [](https://example.com)", 26 | node: nil, 27 | }, 28 | { 29 | text: "[alte]( htt ps :/ /example.com)", 30 | node: nil, 31 | }, 32 | { 33 | text: "[your/slash](https://example.com)", 34 | node: &ast.Link{ 35 | Content: []ast.Node{&ast.Text{Content: "your/slash"}}, 36 | URL: "https://example.com", 37 | }, 38 | }, 39 | { 40 | text: "[hello world](https://example.com)", 41 | node: &ast.Link{ 42 | Content: []ast.Node{&ast.Text{Content: "hello world"}}, 43 | URL: "https://example.com", 44 | }, 45 | }, 46 | { 47 | text: "[hello world](https://example.com)", 48 | node: &ast.Link{ 49 | Content: []ast.Node{&ast.Text{Content: "hello world"}}, 50 | URL: "https://example.com", 51 | }, 52 | }, 53 | { 54 | text: `[\[link\]](https://example.com)`, 55 | node: &ast.Link{ 56 | Content: []ast.Node{&ast.EscapingCharacter{Symbol: "["}, &ast.Text{Content: `link`}, &ast.EscapingCharacter{Symbol: "]"}}, 57 | URL: "https://example.com", 58 | }, 59 | }, 60 | } 61 | for _, test := range tests { 62 | tokens := tokenizer.Tokenize(test.text) 63 | node, _ := parser.NewLinkParser().Match(tokens) 64 | require.Equal(t, test.node, node) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /parser/tests/list_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | 9 | "github.com/usememos/gomark/ast" 10 | "github.com/usememos/gomark/parser" 11 | "github.com/usememos/gomark/parser/tokenizer" 12 | ) 13 | 14 | func TestListParser(t *testing.T) { 15 | tests := []struct { 16 | text string 17 | nodes []ast.Node 18 | }{ 19 | { 20 | text: "1. hello\n\n", 21 | nodes: []ast.Node{ 22 | &ast.List{ 23 | Kind: ast.OrderedList, 24 | Children: []ast.Node{ 25 | &ast.OrderedListItem{ 26 | Number: "1", 27 | Children: []ast.Node{ 28 | &ast.Text{ 29 | Content: "hello", 30 | }, 31 | }, 32 | }, 33 | &ast.LineBreak{}, 34 | &ast.LineBreak{}, 35 | }, 36 | }, 37 | }, 38 | }, 39 | { 40 | text: "1. hello\n2. world", 41 | nodes: []ast.Node{ 42 | &ast.List{ 43 | Kind: ast.OrderedList, 44 | Children: []ast.Node{ 45 | &ast.OrderedListItem{ 46 | Number: "1", 47 | Children: []ast.Node{ 48 | &ast.Text{ 49 | Content: "hello", 50 | }, 51 | }, 52 | }, 53 | &ast.LineBreak{}, 54 | &ast.OrderedListItem{ 55 | Number: "2", 56 | Children: []ast.Node{ 57 | &ast.Text{ 58 | Content: "world", 59 | }, 60 | }, 61 | }, 62 | }, 63 | }, 64 | }, 65 | }, 66 | { 67 | text: "1. hello\n 2. world", 68 | nodes: []ast.Node{ 69 | &ast.List{ 70 | Kind: ast.OrderedList, 71 | Children: []ast.Node{ 72 | &ast.OrderedListItem{ 73 | Number: "1", 74 | Children: []ast.Node{ 75 | &ast.Text{ 76 | Content: "hello", 77 | }, 78 | }, 79 | }, 80 | &ast.LineBreak{}, 81 | &ast.List{ 82 | Kind: ast.OrderedList, 83 | Indent: 2, 84 | Children: []ast.Node{ 85 | &ast.OrderedListItem{ 86 | Number: "2", 87 | Indent: 2, 88 | Children: []ast.Node{ 89 | &ast.Text{ 90 | Content: "world", 91 | }, 92 | }, 93 | }, 94 | }, 95 | }, 96 | }, 97 | }, 98 | }, 99 | }, 100 | { 101 | text: "* hello\n * world\n * gomark", 102 | nodes: []ast.Node{ 103 | &ast.List{ 104 | Kind: ast.UnorderedList, 105 | Children: []ast.Node{ 106 | &ast.UnorderedListItem{ 107 | Symbol: "*", 108 | Children: []ast.Node{ 109 | &ast.Text{ 110 | Content: "hello", 111 | }, 112 | }, 113 | }, 114 | &ast.LineBreak{}, 115 | &ast.List{ 116 | Kind: ast.UnorderedList, 117 | Indent: 2, 118 | Children: []ast.Node{ 119 | &ast.UnorderedListItem{ 120 | Symbol: "*", 121 | Indent: 2, 122 | Children: []ast.Node{ 123 | &ast.Text{ 124 | Content: "world", 125 | }, 126 | }, 127 | }, 128 | &ast.LineBreak{}, 129 | &ast.UnorderedListItem{ 130 | Symbol: "*", 131 | Indent: 2, 132 | Children: []ast.Node{ 133 | &ast.Text{ 134 | Content: "gomark", 135 | }, 136 | }, 137 | }, 138 | }, 139 | }, 140 | }, 141 | }, 142 | }, 143 | }, 144 | { 145 | text: "* hello\n * world\n* gomark", 146 | nodes: []ast.Node{ 147 | &ast.List{ 148 | Kind: ast.UnorderedList, 149 | Children: []ast.Node{ 150 | &ast.UnorderedListItem{ 151 | Symbol: "*", 152 | Children: []ast.Node{ 153 | &ast.Text{ 154 | Content: "hello", 155 | }, 156 | }, 157 | }, 158 | &ast.LineBreak{}, 159 | &ast.List{ 160 | Kind: ast.UnorderedList, 161 | Indent: 2, 162 | Children: []ast.Node{ 163 | &ast.UnorderedListItem{ 164 | Symbol: "*", 165 | Indent: 2, 166 | Children: []ast.Node{ 167 | &ast.Text{ 168 | Content: "world", 169 | }, 170 | }, 171 | }, 172 | &ast.LineBreak{}, 173 | }, 174 | }, 175 | &ast.UnorderedListItem{ 176 | Symbol: "*", 177 | Children: []ast.Node{ 178 | &ast.Text{ 179 | Content: "gomark", 180 | }, 181 | }, 182 | }, 183 | }, 184 | }, 185 | }, 186 | }, 187 | { 188 | text: "* hello\nparagraph\n* world", 189 | nodes: []ast.Node{ 190 | &ast.List{ 191 | Kind: ast.UnorderedList, 192 | Children: []ast.Node{ 193 | &ast.UnorderedListItem{ 194 | Symbol: "*", 195 | Children: []ast.Node{ 196 | &ast.Text{ 197 | Content: "hello", 198 | }, 199 | }, 200 | }, 201 | &ast.LineBreak{}, 202 | }, 203 | }, 204 | &ast.Paragraph{ 205 | Children: []ast.Node{ 206 | &ast.Text{ 207 | Content: "paragraph", 208 | }, 209 | }, 210 | }, 211 | &ast.LineBreak{}, 212 | &ast.List{ 213 | Kind: ast.UnorderedList, 214 | Children: []ast.Node{ 215 | &ast.UnorderedListItem{ 216 | Symbol: "*", 217 | Children: []ast.Node{ 218 | &ast.Text{ 219 | Content: "world", 220 | }, 221 | }, 222 | }, 223 | }, 224 | }, 225 | }, 226 | }, 227 | } 228 | 229 | for _, test := range tests { 230 | tokens := tokenizer.Tokenize(test.text) 231 | nodes, _ := parser.Parse(tokens) 232 | require.ElementsMatch(t, test.nodes, nodes, fmt.Sprintf("Test case: %s", test.text)) 233 | } 234 | } 235 | -------------------------------------------------------------------------------- /parser/tests/math_block_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/usememos/gomark/ast" 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestMathBlockParser(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | node ast.Node 16 | }{ 17 | { 18 | text: "$$\n(1+x)^2\n$$", 19 | node: &ast.MathBlock{ 20 | Content: "(1+x)^2", 21 | }, 22 | }, 23 | { 24 | text: "$$\na=3\n$$", 25 | node: &ast.MathBlock{ 26 | Content: "a=3", 27 | }, 28 | }, 29 | } 30 | for _, test := range tests { 31 | tokens := tokenizer.Tokenize(test.text) 32 | node, _ := parser.NewMathBlockParser().Match(tokens) 33 | require.Equal(t, test.node, node) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /parser/tests/math_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/usememos/gomark/ast" 9 | "github.com/usememos/gomark/parser" 10 | "github.com/usememos/gomark/parser/tokenizer" 11 | ) 12 | 13 | func TestMathParser(t *testing.T) { 14 | tests := []struct { 15 | text string 16 | node ast.Node 17 | }{ 18 | { 19 | text: "$\\sqrt{3x-1}+(1+x)^2$", 20 | node: &ast.Math{ 21 | Content: "\\sqrt{3x-1}+(1+x)^2", 22 | }, 23 | }, 24 | } 25 | for _, test := range tests { 26 | tokens := tokenizer.Tokenize(test.text) 27 | node, _ := parser.NewMathParser().Match(tokens) 28 | require.Equal(t, test.node, node) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /parser/tests/ordered_list_item_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/usememos/gomark/ast" 9 | "github.com/usememos/gomark/parser" 10 | "github.com/usememos/gomark/parser/tokenizer" 11 | ) 12 | 13 | func TestOrderedListItemParser(t *testing.T) { 14 | tests := []struct { 15 | text string 16 | node ast.Node 17 | }{ 18 | { 19 | text: "1.asd", 20 | node: nil, 21 | }, 22 | { 23 | text: "1. Hello World", 24 | node: &ast.OrderedListItem{ 25 | Number: "1", 26 | Children: []ast.Node{ 27 | &ast.Text{ 28 | Content: "Hello World", 29 | }, 30 | }, 31 | }, 32 | }, 33 | { 34 | text: " 1. Hello World", 35 | node: &ast.OrderedListItem{ 36 | Number: "1", 37 | Indent: 2, 38 | Children: []ast.Node{ 39 | &ast.Text{ 40 | Content: "Hello World", 41 | }, 42 | }, 43 | }, 44 | }, 45 | { 46 | text: "1aa. Hello World", 47 | node: nil, 48 | }, 49 | { 50 | text: "22. Hello *World*", 51 | node: &ast.OrderedListItem{ 52 | Number: "22", 53 | Children: []ast.Node{ 54 | &ast.Text{ 55 | Content: "Hello ", 56 | }, 57 | &ast.Italic{ 58 | Symbol: "*", 59 | Children: []ast.Node{ 60 | &ast.Text{ 61 | Content: "World", 62 | }, 63 | }, 64 | }, 65 | }, 66 | }, 67 | }, 68 | } 69 | 70 | for _, test := range tests { 71 | tokens := tokenizer.Tokenize(test.text) 72 | node, _ := parser.NewOrderedListItemParser().Match(tokens) 73 | require.Equal(t, test.node, node) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /parser/tests/paragraph_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/usememos/gomark/ast" 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestParagraphParser(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | node ast.Node 16 | }{ 17 | { 18 | text: "", 19 | node: nil, 20 | }, 21 | { 22 | text: "\n", 23 | node: nil, 24 | }, 25 | { 26 | text: "Hello world!", 27 | node: &ast.Paragraph{ 28 | Children: []ast.Node{ 29 | &ast.Text{ 30 | Content: "Hello world!", 31 | }, 32 | }, 33 | }, 34 | }, 35 | { 36 | text: "Hello world!\n", 37 | node: &ast.Paragraph{ 38 | Children: []ast.Node{ 39 | &ast.Text{ 40 | Content: "Hello world!", 41 | }, 42 | }, 43 | }, 44 | }, 45 | { 46 | text: "Hello world!\n\nNew paragraph.", 47 | node: &ast.Paragraph{ 48 | Children: []ast.Node{ 49 | &ast.Text{ 50 | Content: "Hello world!", 51 | }, 52 | }, 53 | }, 54 | }, 55 | } 56 | 57 | for _, test := range tests { 58 | tokens := tokenizer.Tokenize(test.text) 59 | node, _ := parser.NewParagraphParser().Match(tokens) 60 | require.Equal(t, test.node, node) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /parser/tests/parser_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | 9 | "github.com/usememos/gomark/ast" 10 | "github.com/usememos/gomark/parser" 11 | "github.com/usememos/gomark/parser/tokenizer" 12 | ) 13 | 14 | func TestParser(t *testing.T) { 15 | tests := []struct { 16 | text string 17 | nodes []ast.Node 18 | }{ 19 | { 20 | text: "Hello world!", 21 | nodes: []ast.Node{ 22 | &ast.Paragraph{ 23 | Children: []ast.Node{ 24 | &ast.Text{ 25 | Content: "Hello world!", 26 | }, 27 | }, 28 | }, 29 | }, 30 | }, 31 | { 32 | text: "# Hello world!", 33 | nodes: []ast.Node{ 34 | &ast.Heading{ 35 | Level: 1, 36 | Children: []ast.Node{ 37 | &ast.Text{ 38 | Content: "Hello world!", 39 | }, 40 | }, 41 | }, 42 | }, 43 | }, 44 | { 45 | text: "\\# Hello world!", 46 | nodes: []ast.Node{ 47 | &ast.Paragraph{ 48 | Children: []ast.Node{ 49 | &ast.EscapingCharacter{ 50 | Symbol: "#", 51 | }, 52 | &ast.Text{ 53 | Content: " Hello world!", 54 | }, 55 | }, 56 | }, 57 | }, 58 | }, 59 | { 60 | text: "**Hello** world!", 61 | nodes: []ast.Node{ 62 | &ast.Paragraph{ 63 | Children: []ast.Node{ 64 | &ast.Bold{ 65 | Symbol: "*", 66 | Children: []ast.Node{ 67 | &ast.Text{ 68 | Content: "Hello", 69 | }, 70 | }, 71 | }, 72 | &ast.Text{ 73 | Content: " world!", 74 | }, 75 | }, 76 | }, 77 | }, 78 | }, 79 | { 80 | text: "Hello **world**!\nHere is a new line.", 81 | nodes: []ast.Node{ 82 | &ast.Paragraph{ 83 | Children: []ast.Node{ 84 | &ast.Text{ 85 | Content: "Hello ", 86 | }, 87 | &ast.Bold{ 88 | Symbol: "*", 89 | Children: []ast.Node{ 90 | &ast.Text{ 91 | Content: "world", 92 | }, 93 | }, 94 | }, 95 | &ast.Text{ 96 | Content: "!", 97 | }, 98 | }, 99 | }, 100 | &ast.LineBreak{}, 101 | &ast.Paragraph{ 102 | Children: []ast.Node{ 103 | &ast.Text{ 104 | Content: "Here is a new line.", 105 | }, 106 | }, 107 | }, 108 | }, 109 | }, 110 | { 111 | text: "Hello **world**!\n```javascript\nconsole.log(\"Hello world!\");\n```", 112 | nodes: []ast.Node{ 113 | &ast.Paragraph{ 114 | Children: []ast.Node{ 115 | &ast.Text{ 116 | Content: "Hello ", 117 | }, 118 | &ast.Bold{ 119 | Symbol: "*", 120 | Children: []ast.Node{ 121 | &ast.Text{ 122 | Content: "world", 123 | }, 124 | }, 125 | }, 126 | &ast.Text{ 127 | Content: "!", 128 | }, 129 | }, 130 | }, 131 | &ast.LineBreak{}, 132 | &ast.CodeBlock{ 133 | Language: "javascript", 134 | Content: "console.log(\"Hello world!\");", 135 | }, 136 | }, 137 | }, 138 | { 139 | text: "Hello world!\n\nNew paragraph.", 140 | nodes: []ast.Node{ 141 | &ast.Paragraph{ 142 | Children: []ast.Node{ 143 | &ast.Text{ 144 | Content: "Hello world!", 145 | }, 146 | }, 147 | }, 148 | &ast.LineBreak{}, 149 | &ast.LineBreak{}, 150 | &ast.Paragraph{ 151 | Children: []ast.Node{ 152 | &ast.Text{ 153 | Content: "New paragraph.", 154 | }, 155 | }, 156 | }, 157 | }, 158 | }, 159 | { 160 | text: "1. hello\n- [ ] world", 161 | nodes: []ast.Node{ 162 | &ast.List{ 163 | Kind: ast.OrderedList, 164 | Children: []ast.Node{ 165 | &ast.OrderedListItem{ 166 | Number: "1", 167 | Children: []ast.Node{ 168 | &ast.Text{ 169 | Content: "hello", 170 | }, 171 | }, 172 | }, 173 | &ast.LineBreak{}, 174 | }, 175 | }, 176 | &ast.List{ 177 | Kind: ast.DescrpitionList, 178 | Children: []ast.Node{ 179 | &ast.TaskListItem{ 180 | Symbol: tokenizer.Hyphen, 181 | Complete: false, 182 | Children: []ast.Node{ 183 | &ast.Text{ 184 | Content: "world", 185 | }, 186 | }, 187 | }, 188 | }, 189 | }, 190 | }, 191 | }, 192 | { 193 | text: "- [ ] hello\n- [x] world", 194 | nodes: []ast.Node{ 195 | &ast.List{ 196 | Kind: ast.DescrpitionList, 197 | Children: []ast.Node{ 198 | &ast.TaskListItem{ 199 | Symbol: tokenizer.Hyphen, 200 | Complete: false, 201 | Children: []ast.Node{ 202 | &ast.Text{ 203 | Content: "hello", 204 | }, 205 | }, 206 | }, 207 | &ast.LineBreak{}, 208 | &ast.TaskListItem{ 209 | Symbol: tokenizer.Hyphen, 210 | Complete: true, 211 | Children: []ast.Node{ 212 | &ast.Text{ 213 | Content: "world", 214 | }, 215 | }, 216 | }, 217 | }, 218 | }, 219 | }, 220 | }, 221 | { 222 | text: "\n\n", 223 | nodes: []ast.Node{ 224 | &ast.LineBreak{}, 225 | &ast.LineBreak{}, 226 | }, 227 | }, 228 | { 229 | text: "\n$$\na=3\n$$", 230 | nodes: []ast.Node{ 231 | &ast.LineBreak{}, 232 | &ast.MathBlock{ 233 | Content: "a=3", 234 | }, 235 | }, 236 | }, 237 | { 238 | text: "Hello\n![[memos/101]]", 239 | nodes: []ast.Node{ 240 | &ast.Paragraph{ 241 | Children: []ast.Node{ 242 | &ast.Text{ 243 | Content: "Hello", 244 | }, 245 | }, 246 | }, 247 | &ast.LineBreak{}, 248 | &ast.EmbeddedContent{ 249 | ResourceName: "memos/101", 250 | }, 251 | }, 252 | }, 253 | { 254 | text: "Hello\nworld
", 255 | nodes: []ast.Node{ 256 | &ast.Paragraph{ 257 | Children: []ast.Node{ 258 | &ast.Text{ 259 | Content: "Hello", 260 | }, 261 | }, 262 | }, 263 | &ast.LineBreak{}, 264 | &ast.Paragraph{ 265 | Children: []ast.Node{ 266 | &ast.Text{ 267 | Content: "world", 268 | }, 269 | &ast.HTMLElement{ 270 | TagName: "br", 271 | Attributes: map[string]string{}, 272 | }, 273 | }, 274 | }, 275 | }, 276 | }, 277 | { 278 | text: "Hello
world", 279 | nodes: []ast.Node{ 280 | &ast.Paragraph{ 281 | Children: []ast.Node{ 282 | &ast.Text{ 283 | Content: "Hello ", 284 | }, 285 | &ast.HTMLElement{ 286 | TagName: "br", 287 | Attributes: map[string]string{}, 288 | }, 289 | &ast.Text{ 290 | Content: " world", 291 | }, 292 | }, 293 | }, 294 | }, 295 | }, 296 | { 297 | text: "* unordered list item 1\n* unordered list item 2", 298 | nodes: []ast.Node{ 299 | &ast.List{ 300 | Kind: ast.UnorderedList, 301 | Children: []ast.Node{ 302 | &ast.UnorderedListItem{ 303 | Symbol: tokenizer.Asterisk, 304 | Children: []ast.Node{ 305 | &ast.Text{ 306 | Content: "unordered list item 1", 307 | }, 308 | }, 309 | }, 310 | &ast.LineBreak{}, 311 | &ast.UnorderedListItem{ 312 | Symbol: tokenizer.Asterisk, 313 | Children: []ast.Node{ 314 | &ast.Text{ 315 | Content: "unordered list item 2", 316 | }, 317 | }, 318 | }, 319 | }, 320 | }, 321 | }, 322 | }, 323 | { 324 | text: "* unordered list item\n\n1. ordered list item", 325 | nodes: []ast.Node{ 326 | &ast.List{ 327 | Kind: ast.UnorderedList, 328 | Children: []ast.Node{ 329 | &ast.UnorderedListItem{ 330 | Symbol: tokenizer.Asterisk, 331 | Children: []ast.Node{ 332 | &ast.Text{ 333 | Content: "unordered list item", 334 | }, 335 | }, 336 | }, 337 | &ast.LineBreak{}, 338 | &ast.LineBreak{}, 339 | }, 340 | }, 341 | &ast.List{ 342 | Kind: ast.OrderedList, 343 | Children: []ast.Node{ 344 | &ast.OrderedListItem{ 345 | Number: "1", 346 | Children: []ast.Node{ 347 | &ast.Text{ 348 | Content: "ordered list item", 349 | }, 350 | }, 351 | }, 352 | }, 353 | }, 354 | }, 355 | }, 356 | { 357 | text: "* unordered list item\nparagraph\n\n1. ordered list item", 358 | nodes: []ast.Node{ 359 | &ast.List{ 360 | Kind: ast.UnorderedList, 361 | Children: []ast.Node{ 362 | &ast.UnorderedListItem{ 363 | Symbol: tokenizer.Asterisk, 364 | Children: []ast.Node{ 365 | &ast.Text{ 366 | Content: "unordered list item", 367 | }, 368 | }, 369 | }, 370 | &ast.LineBreak{}, 371 | }, 372 | }, 373 | &ast.Paragraph{ 374 | Children: []ast.Node{ 375 | &ast.Text{ 376 | Content: "paragraph", 377 | }, 378 | }, 379 | }, 380 | &ast.LineBreak{}, 381 | &ast.LineBreak{}, 382 | &ast.List{ 383 | Kind: ast.OrderedList, 384 | Children: []ast.Node{ 385 | &ast.OrderedListItem{ 386 | Number: "1", 387 | Children: []ast.Node{ 388 | &ast.Text{ 389 | Content: "ordered list item", 390 | }, 391 | }, 392 | }, 393 | }, 394 | }, 395 | }, 396 | }, 397 | } 398 | 399 | for _, test := range tests { 400 | tokens := tokenizer.Tokenize(test.text) 401 | nodes, _ := parser.Parse(tokens) 402 | require.Equal(t, test.nodes, nodes, fmt.Sprintf("Test case: %s", test.text)) 403 | } 404 | } 405 | -------------------------------------------------------------------------------- /parser/tests/referenced_content_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/usememos/gomark/ast" 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestReferencedContentParser(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | node ast.Node 16 | }{ 17 | { 18 | text: "[[Hello world]", 19 | node: nil, 20 | }, 21 | { 22 | text: "[[Hello world]]", 23 | node: &ast.ReferencedContent{ 24 | ResourceName: "Hello world", 25 | }, 26 | }, 27 | { 28 | text: "[[memos/1]]", 29 | node: &ast.ReferencedContent{ 30 | ResourceName: "memos/1", 31 | }, 32 | }, 33 | { 34 | text: "[[resources/101]]111\n123", 35 | node: &ast.ReferencedContent{ 36 | ResourceName: "resources/101", 37 | }, 38 | }, 39 | { 40 | text: "[[resources/101?align=center]]", 41 | node: &ast.ReferencedContent{ 42 | ResourceName: "resources/101", 43 | Params: "align=center", 44 | }, 45 | }, 46 | { 47 | text: "[[resources/6uxnhT98q8vN8anBbUbRGu?align=center]]", 48 | node: &ast.ReferencedContent{ 49 | ResourceName: "resources/6uxnhT98q8vN8anBbUbRGu", 50 | Params: "align=center", 51 | }, 52 | }, 53 | } 54 | 55 | for _, test := range tests { 56 | tokens := tokenizer.Tokenize(test.text) 57 | node, _ := parser.NewReferencedContentParser().Match(tokens) 58 | require.Equal(t, test.node, node) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /parser/tests/spoiler_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/usememos/gomark/ast" 9 | "github.com/usememos/gomark/parser" 10 | "github.com/usememos/gomark/parser/tokenizer" 11 | ) 12 | 13 | func TestSpoilerParser(t *testing.T) { 14 | tests := []struct { 15 | text string 16 | node ast.Node 17 | }{ 18 | { 19 | text: "*Hello world!", 20 | node: nil, 21 | }, 22 | { 23 | text: "||Hello||", 24 | node: &ast.Spoiler{ 25 | Content: "Hello", 26 | }, 27 | }, 28 | } 29 | 30 | for _, test := range tests { 31 | tokens := tokenizer.Tokenize(test.text) 32 | node, _ := parser.NewSpoilerParser().Match(tokens) 33 | require.Equal(t, test.node, node) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /parser/tests/strikethrough_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/usememos/gomark/ast" 9 | "github.com/usememos/gomark/parser" 10 | "github.com/usememos/gomark/parser/tokenizer" 11 | ) 12 | 13 | func TestStrikethroughParser(t *testing.T) { 14 | tests := []struct { 15 | text string 16 | node ast.Node 17 | }{ 18 | { 19 | text: "~~Hello world", 20 | node: nil, 21 | }, 22 | { 23 | text: "~~Hello~~", 24 | node: &ast.Strikethrough{ 25 | Content: "Hello", 26 | }, 27 | }, 28 | { 29 | text: "~~ Hello ~~", 30 | node: &ast.Strikethrough{ 31 | Content: " Hello ", 32 | }, 33 | }, 34 | { 35 | text: "~~1~~ Hello ~~~", 36 | node: &ast.Strikethrough{ 37 | Content: "1", 38 | }, 39 | }, 40 | } 41 | 42 | for _, test := range tests { 43 | tokens := tokenizer.Tokenize(test.text) 44 | node, _ := parser.NewStrikethroughParser().Match(tokens) 45 | require.Equal(t, test.node, node) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /parser/tests/subscript_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/usememos/gomark/ast" 9 | "github.com/usememos/gomark/parser" 10 | "github.com/usememos/gomark/parser/tokenizer" 11 | ) 12 | 13 | func TestSubscriptParser(t *testing.T) { 14 | tests := []struct { 15 | text string 16 | node ast.Node 17 | }{ 18 | { 19 | text: "~Hello world!", 20 | node: nil, 21 | }, 22 | { 23 | text: "~Hello~", 24 | node: &ast.Subscript{ 25 | Content: "Hello", 26 | }, 27 | }, 28 | { 29 | text: "~ Hello ~", 30 | node: &ast.Subscript{ 31 | Content: " Hello ", 32 | }, 33 | }, 34 | { 35 | text: "~1~ Hello ~ ~", 36 | node: &ast.Subscript{ 37 | Content: "1", 38 | }, 39 | }, 40 | } 41 | 42 | for _, test := range tests { 43 | tokens := tokenizer.Tokenize(test.text) 44 | node, _ := parser.NewSubscriptParser().Match(tokens) 45 | require.Equal(t, test.node, node) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /parser/tests/superscript_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/usememos/gomark/ast" 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestSuperscriptParser(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | node ast.Node 16 | }{ 17 | { 18 | text: "^Hello world!", 19 | node: nil, 20 | }, 21 | { 22 | text: "^Hello^", 23 | node: &ast.Superscript{ 24 | Content: "Hello", 25 | }, 26 | }, 27 | { 28 | text: "^ Hello ^", 29 | node: &ast.Superscript{ 30 | Content: " Hello ", 31 | }, 32 | }, 33 | { 34 | text: "^1^ Hello ^ ^", 35 | node: &ast.Superscript{ 36 | Content: "1", 37 | }, 38 | }, 39 | } 40 | 41 | for _, test := range tests { 42 | tokens := tokenizer.Tokenize(test.text) 43 | node, _ := parser.NewSuperscriptParser().Match(tokens) 44 | require.Equal(t, test.node, node) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /parser/tests/table_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/usememos/gomark/ast" 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestTableParser(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | node ast.Node 16 | }{ 17 | { 18 | text: "| header |\n| --- |\n| cell |\n", 19 | node: &ast.Table{ 20 | Header: []ast.Node{ 21 | &ast.Paragraph{ 22 | Children: []ast.Node{ 23 | &ast.Text{Content: "header"}, 24 | }, 25 | }, 26 | }, 27 | Delimiter: []string{"---"}, 28 | Rows: [][]ast.Node{ 29 | { 30 | &ast.Paragraph{ 31 | Children: []ast.Node{ 32 | &ast.Text{Content: "cell"}, 33 | }, 34 | }, 35 | }, 36 | }, 37 | }, 38 | }, 39 | { 40 | text: "| **header1** | header2 |\n| --- | ---- |\n| cell1 | cell2 |\n| cell3 | cell4 |", 41 | node: &ast.Table{ 42 | Header: []ast.Node{ 43 | &ast.Paragraph{ 44 | Children: []ast.Node{ 45 | &ast.Bold{ 46 | Symbol: "*", 47 | Children: []ast.Node{ 48 | &ast.Text{Content: "header1"}, 49 | }, 50 | }, 51 | }, 52 | }, 53 | &ast.Paragraph{ 54 | Children: []ast.Node{ 55 | &ast.Text{Content: "header2"}, 56 | }, 57 | }, 58 | }, 59 | Delimiter: []string{"---", "----"}, 60 | Rows: [][]ast.Node{ 61 | { 62 | &ast.Paragraph{ 63 | Children: []ast.Node{ 64 | &ast.Text{Content: "cell1"}, 65 | }, 66 | }, 67 | &ast.Paragraph{ 68 | Children: []ast.Node{ 69 | &ast.Text{Content: "cell2"}, 70 | }, 71 | }, 72 | }, 73 | { 74 | &ast.Paragraph{ 75 | Children: []ast.Node{ 76 | &ast.Text{Content: "cell3"}, 77 | }, 78 | }, 79 | &ast.Paragraph{ 80 | Children: []ast.Node{ 81 | &ast.Text{Content: "cell4"}, 82 | }, 83 | }, 84 | }, 85 | }, 86 | }, 87 | }, 88 | } 89 | 90 | for _, test := range tests { 91 | tokens := tokenizer.Tokenize(test.text) 92 | node, _ := parser.NewTableParser().Match(tokens) 93 | require.Equal(t, test.node, node) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /parser/tests/tag_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/usememos/gomark/ast" 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestTagParser(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | node ast.Node 16 | }{ 17 | { 18 | text: "*Hello world", 19 | node: nil, 20 | }, 21 | { 22 | text: "# Hello World", 23 | node: nil, 24 | }, 25 | { 26 | text: "#tag", 27 | node: &ast.Tag{ 28 | Content: "tag", 29 | }, 30 | }, 31 | { 32 | text: "#tag/subtag 123", 33 | node: &ast.Tag{ 34 | Content: "tag/subtag", 35 | }, 36 | }, 37 | { 38 | text: `#tag\'s 123`, 39 | node: &ast.Tag{ 40 | Content: "tag", 41 | }, 42 | }, 43 | } 44 | 45 | for _, test := range tests { 46 | tokens := tokenizer.Tokenize(test.text) 47 | node, _ := parser.NewTagParser().Match(tokens) 48 | require.Equal(t, test.node, node) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /parser/tests/task_list_item_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | "github.com/usememos/gomark/ast" 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestTaskListItemParser(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | node ast.Node 16 | }{ 17 | { 18 | text: "*asd", 19 | node: nil, 20 | }, 21 | { 22 | text: "+ [ ] Hello World", 23 | node: &ast.TaskListItem{ 24 | Symbol: tokenizer.PlusSign, 25 | Children: []ast.Node{ 26 | &ast.Text{ 27 | Content: "Hello World", 28 | }, 29 | }, 30 | }, 31 | }, 32 | { 33 | text: " + [ ] Hello World", 34 | node: &ast.TaskListItem{ 35 | Symbol: tokenizer.PlusSign, 36 | Indent: 2, 37 | Complete: false, 38 | Children: []ast.Node{ 39 | &ast.Text{ 40 | Content: "Hello World", 41 | }, 42 | }, 43 | }, 44 | }, 45 | { 46 | text: "* [x] **Hello**", 47 | node: &ast.TaskListItem{ 48 | Symbol: tokenizer.Asterisk, 49 | Complete: true, 50 | Children: []ast.Node{ 51 | &ast.Bold{ 52 | Symbol: "*", 53 | Children: []ast.Node{ 54 | &ast.Text{ 55 | Content: "Hello", 56 | }, 57 | }, 58 | }, 59 | }, 60 | }, 61 | }, 62 | } 63 | 64 | for _, test := range tests { 65 | tokens := tokenizer.Tokenize(test.text) 66 | node, _ := parser.NewTaskListItemParser().Match(tokens) 67 | require.Equal(t, test.node, node) 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /parser/tests/unordered_list_item_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/usememos/gomark/ast" 9 | "github.com/usememos/gomark/parser" 10 | "github.com/usememos/gomark/parser/tokenizer" 11 | ) 12 | 13 | func TestUnorderedListItemParser(t *testing.T) { 14 | tests := []struct { 15 | text string 16 | node ast.Node 17 | }{ 18 | { 19 | text: "*asd", 20 | node: nil, 21 | }, 22 | { 23 | text: "+ Hello World", 24 | node: &ast.UnorderedListItem{ 25 | Symbol: tokenizer.PlusSign, 26 | Children: []ast.Node{ 27 | &ast.Text{ 28 | Content: "Hello World", 29 | }, 30 | }, 31 | }, 32 | }, 33 | { 34 | text: "* **Hello**", 35 | node: &ast.UnorderedListItem{ 36 | Symbol: tokenizer.Asterisk, 37 | Children: []ast.Node{ 38 | &ast.Bold{ 39 | Symbol: "*", 40 | Children: []ast.Node{ 41 | &ast.Text{ 42 | Content: "Hello", 43 | }, 44 | }, 45 | }, 46 | }, 47 | }, 48 | }, 49 | } 50 | 51 | for _, test := range tests { 52 | tokens := tokenizer.Tokenize(test.text) 53 | node, _ := parser.NewUnorderedListItemParser().Match(tokens) 54 | require.Equal(t, test.node, node) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /parser/text.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type TextParser struct { 9 | Content string 10 | } 11 | 12 | func NewTextParser() *TextParser { 13 | return &TextParser{} 14 | } 15 | 16 | func (*TextParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 17 | if len(tokens) == 0 { 18 | return nil, 0 19 | } 20 | return &ast.Text{ 21 | Content: tokens[0].String(), 22 | }, 1 23 | } 24 | -------------------------------------------------------------------------------- /parser/tokenizer/tokenizer.go: -------------------------------------------------------------------------------- 1 | package tokenizer 2 | 3 | type TokenType = string 4 | 5 | // Special character tokens. 6 | const ( 7 | Underscore TokenType = "_" 8 | Asterisk TokenType = "*" 9 | PoundSign TokenType = "#" 10 | Backtick TokenType = "`" 11 | LeftSquareBracket TokenType = "[" 12 | RightSquareBracket TokenType = "]" 13 | LeftParenthesis TokenType = "(" 14 | RightParenthesis TokenType = ")" 15 | ExclamationMark TokenType = "!" 16 | QuestionMark TokenType = "?" 17 | Tilde TokenType = "~" 18 | Hyphen TokenType = "-" 19 | PlusSign TokenType = "+" 20 | Dot TokenType = "." 21 | LessThan TokenType = "<" 22 | GreaterThan TokenType = ">" 23 | DollarSign TokenType = "$" 24 | EqualSign TokenType = "=" 25 | Pipe TokenType = "|" 26 | Colon TokenType = ":" 27 | Caret TokenType = "^" 28 | Apostrophe TokenType = "'" 29 | Backslash TokenType = "\\" 30 | Slash TokenType = "/" 31 | NewLine TokenType = "\n" 32 | Space TokenType = " " 33 | ) 34 | 35 | // Text based tokens. 36 | const ( 37 | Number TokenType = "number" 38 | Text TokenType = "" 39 | ) 40 | 41 | type Token struct { 42 | Type TokenType 43 | Value string 44 | } 45 | 46 | func NewToken(tp, text string) *Token { 47 | return &Token{ 48 | Type: tp, 49 | Value: text, 50 | } 51 | } 52 | 53 | func Tokenize(text string) []*Token { 54 | tokens := []*Token{} 55 | for _, c := range text { 56 | switch c { 57 | case '_': 58 | tokens = append(tokens, NewToken(Underscore, "_")) 59 | case '*': 60 | tokens = append(tokens, NewToken(Asterisk, "*")) 61 | case '#': 62 | tokens = append(tokens, NewToken(PoundSign, "#")) 63 | case '`': 64 | tokens = append(tokens, NewToken(Backtick, "`")) 65 | case '[': 66 | tokens = append(tokens, NewToken(LeftSquareBracket, "[")) 67 | case ']': 68 | tokens = append(tokens, NewToken(RightSquareBracket, "]")) 69 | case '(': 70 | tokens = append(tokens, NewToken(LeftParenthesis, "(")) 71 | case ')': 72 | tokens = append(tokens, NewToken(RightParenthesis, ")")) 73 | case '!': 74 | tokens = append(tokens, NewToken(ExclamationMark, "!")) 75 | case '?': 76 | tokens = append(tokens, NewToken(QuestionMark, "?")) 77 | case '~': 78 | tokens = append(tokens, NewToken(Tilde, "~")) 79 | case '-': 80 | tokens = append(tokens, NewToken(Hyphen, "-")) 81 | case '<': 82 | tokens = append(tokens, NewToken(LessThan, "<")) 83 | case '>': 84 | tokens = append(tokens, NewToken(GreaterThan, ">")) 85 | case '+': 86 | tokens = append(tokens, NewToken(PlusSign, "+")) 87 | case '.': 88 | tokens = append(tokens, NewToken(Dot, ".")) 89 | case '$': 90 | tokens = append(tokens, NewToken(DollarSign, "$")) 91 | case '=': 92 | tokens = append(tokens, NewToken(EqualSign, "=")) 93 | case '|': 94 | tokens = append(tokens, NewToken(Pipe, "|")) 95 | case ':': 96 | tokens = append(tokens, NewToken(Colon, ":")) 97 | case '^': 98 | tokens = append(tokens, NewToken(Caret, "^")) 99 | case '\'': 100 | tokens = append(tokens, NewToken(Apostrophe, "'")) 101 | case '\\': 102 | tokens = append(tokens, NewToken(Backslash, `\`)) 103 | case '/': 104 | tokens = append(tokens, NewToken(Slash, "/")) 105 | case '\n': 106 | tokens = append(tokens, NewToken(NewLine, "\n")) 107 | case ' ': 108 | tokens = append(tokens, NewToken(Space, " ")) 109 | default: 110 | var prevToken *Token 111 | if len(tokens) > 0 { 112 | prevToken = tokens[len(tokens)-1] 113 | } 114 | 115 | isNumber := c >= '0' && c <= '9' 116 | if prevToken != nil { 117 | if (prevToken.Type == Text && !isNumber) || (prevToken.Type == Number && isNumber) { 118 | prevToken.Value += string(c) 119 | continue 120 | } 121 | } 122 | 123 | if isNumber { 124 | tokens = append(tokens, NewToken(Number, string(c))) 125 | } else { 126 | tokens = append(tokens, NewToken(Text, string(c))) 127 | } 128 | } 129 | } 130 | return tokens 131 | } 132 | 133 | func (t *Token) String() string { 134 | return t.Value 135 | } 136 | 137 | func Stringify(tokens []*Token) string { 138 | text := "" 139 | for _, token := range tokens { 140 | text += token.String() 141 | } 142 | return text 143 | } 144 | 145 | func Split(tokens []*Token, delimiter TokenType) [][]*Token { 146 | if len(tokens) == 0 { 147 | return [][]*Token{} 148 | } 149 | 150 | result := make([][]*Token, 0) 151 | current := make([]*Token, 0) 152 | for _, token := range tokens { 153 | if token.Type == delimiter { 154 | result = append(result, current) 155 | current = make([]*Token, 0) 156 | } else { 157 | current = append(current, token) 158 | } 159 | } 160 | result = append(result, current) 161 | return result 162 | } 163 | 164 | func Find(tokens []*Token, target TokenType) int { 165 | for i, token := range tokens { 166 | if token.Type == target { 167 | return i 168 | } 169 | } 170 | return -1 171 | } 172 | 173 | func FindUnescaped(tokens []*Token, target TokenType) int { 174 | for i, token := range tokens { 175 | if token.Type == target && (i == 0 || (i > 0 && tokens[i-1].Type != Backslash)) { 176 | return i 177 | } 178 | } 179 | return -1 180 | } 181 | 182 | func GetFirstLine(tokens []*Token) []*Token { 183 | for i, token := range tokens { 184 | if token.Type == NewLine { 185 | return tokens[:i] 186 | } 187 | } 188 | return tokens 189 | } 190 | -------------------------------------------------------------------------------- /parser/tokenizer/tokenizer_test.go: -------------------------------------------------------------------------------- 1 | package tokenizer 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestTokenize(t *testing.T) { 10 | tests := []struct { 11 | text string 12 | tokens []*Token 13 | }{ 14 | { 15 | text: "*Hello world!", 16 | tokens: []*Token{ 17 | { 18 | Type: Asterisk, 19 | Value: "*", 20 | }, 21 | { 22 | Type: Text, 23 | Value: "Hello", 24 | }, 25 | { 26 | Type: Space, 27 | Value: " ", 28 | }, 29 | { 30 | Type: Text, 31 | Value: "world", 32 | }, 33 | { 34 | Type: ExclamationMark, 35 | Value: "!", 36 | }, 37 | }, 38 | }, 39 | { 40 | text: `# hello 41 | world`, 42 | tokens: []*Token{ 43 | { 44 | Type: PoundSign, 45 | Value: "#", 46 | }, 47 | { 48 | Type: Space, 49 | Value: " ", 50 | }, 51 | { 52 | Type: Text, 53 | Value: "hello", 54 | }, 55 | { 56 | Type: Space, 57 | Value: " ", 58 | }, 59 | { 60 | Type: NewLine, 61 | Value: "\n", 62 | }, 63 | { 64 | Type: Space, 65 | Value: " ", 66 | }, 67 | { 68 | Type: Text, 69 | Value: "world", 70 | }, 71 | }, 72 | }, 73 | } 74 | 75 | for _, test := range tests { 76 | result := Tokenize(test.text) 77 | require.Equal(t, test.tokens, result) 78 | } 79 | } 80 | 81 | func TestSplit(t *testing.T) { 82 | tests := []struct { 83 | tokens []*Token 84 | sep TokenType 85 | result [][]*Token 86 | }{ 87 | { 88 | tokens: []*Token{ 89 | { 90 | Type: Asterisk, 91 | Value: "*", 92 | }, 93 | { 94 | Type: Text, 95 | Value: "Hello", 96 | }, 97 | { 98 | Type: Space, 99 | Value: " ", 100 | }, 101 | { 102 | Type: Text, 103 | Value: "world", 104 | }, 105 | { 106 | Type: ExclamationMark, 107 | Value: "!", 108 | }, 109 | }, 110 | sep: Asterisk, 111 | result: [][]*Token{ 112 | {}, 113 | { 114 | { 115 | Type: Text, 116 | Value: "Hello", 117 | }, 118 | { 119 | Type: Space, 120 | Value: " ", 121 | }, 122 | { 123 | Type: Text, 124 | Value: "world", 125 | }, 126 | { 127 | Type: ExclamationMark, 128 | Value: "!", 129 | }, 130 | }, 131 | }, 132 | }, 133 | } 134 | 135 | for _, test := range tests { 136 | result := Split(test.tokens, test.sep) 137 | for index, tokens := range result { 138 | require.Equal(t, Stringify(test.result[index]), Stringify(tokens)) 139 | } 140 | } 141 | } 142 | 143 | func TestGetFirstLine(t *testing.T) { 144 | tests := []struct { 145 | tokens []*Token 146 | want []*Token 147 | }{ 148 | { 149 | tokens: []*Token{ 150 | { 151 | Type: Asterisk, 152 | Value: "hello world", 153 | }, 154 | { 155 | Type: NewLine, 156 | Value: "\n", 157 | }, 158 | }, 159 | want: []*Token{ 160 | { 161 | Type: Asterisk, 162 | Value: "hello world", 163 | }, 164 | }, 165 | }, 166 | } 167 | 168 | for _, test := range tests { 169 | result := GetFirstLine(test.tokens) 170 | require.Equal(t, test.want, result) 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /parser/unordered_list_item.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/usememos/gomark/ast" 5 | "github.com/usememos/gomark/parser/tokenizer" 6 | ) 7 | 8 | type UnorderedListItemParser struct{} 9 | 10 | func NewUnorderedListItemParser() *UnorderedListItemParser { 11 | return &UnorderedListItemParser{} 12 | } 13 | 14 | func (*UnorderedListItemParser) Match(tokens []*tokenizer.Token) (ast.Node, int) { 15 | matchedTokens := tokenizer.GetFirstLine(tokens) 16 | indent := 0 17 | for _, token := range matchedTokens { 18 | if token.Type == tokenizer.Space { 19 | indent++ 20 | } else { 21 | break 22 | } 23 | } 24 | if len(matchedTokens) < indent+2 { 25 | return nil, 0 26 | } 27 | 28 | symbolToken := matchedTokens[indent] 29 | if (symbolToken.Type != tokenizer.Hyphen && symbolToken.Type != tokenizer.Asterisk && symbolToken.Type != tokenizer.PlusSign) || matchedTokens[indent+1].Type != tokenizer.Space { 30 | return nil, 0 31 | } 32 | 33 | contentTokens := matchedTokens[indent+2:] 34 | if len(contentTokens) == 0 { 35 | return nil, 0 36 | } 37 | children, err := ParseInline(contentTokens) 38 | if err != nil { 39 | return nil, 0 40 | } 41 | return &ast.UnorderedListItem{ 42 | Symbol: symbolToken.Type, 43 | Indent: indent, 44 | Children: children, 45 | }, indent + len(contentTokens) + 2 46 | } 47 | -------------------------------------------------------------------------------- /renderer/html/html.go: -------------------------------------------------------------------------------- 1 | package html 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | 7 | "github.com/usememos/gomark/ast" 8 | ) 9 | 10 | type RendererContext struct { 11 | } 12 | 13 | // HTMLRenderer is a simple renderer that converts AST to HTML. 14 | type HTMLRenderer struct { 15 | output *bytes.Buffer 16 | context *RendererContext 17 | } 18 | 19 | // NewHTMLRenderer creates a new HTMLRender. 20 | func NewHTMLRenderer() *HTMLRenderer { 21 | return &HTMLRenderer{ 22 | output: new(bytes.Buffer), 23 | context: &RendererContext{}, 24 | } 25 | } 26 | 27 | // RenderNode renders a single AST node to HTML. 28 | func (r *HTMLRenderer) RenderNode(node ast.Node) { 29 | switch n := node.(type) { 30 | case *ast.LineBreak: 31 | r.renderLineBreak(n) 32 | case *ast.Paragraph: 33 | r.renderParagraph(n) 34 | case *ast.CodeBlock: 35 | r.renderCodeBlock(n) 36 | case *ast.Heading: 37 | r.renderHeading(n) 38 | case *ast.HorizontalRule: 39 | r.renderHorizontalRule(n) 40 | case *ast.Blockquote: 41 | r.renderBlockquote(n) 42 | case *ast.List: 43 | r.renderList(n) 44 | case *ast.UnorderedListItem: 45 | r.renderUnorderedListItem(n) 46 | case *ast.OrderedListItem: 47 | r.renderOrderedListItem(n) 48 | case *ast.TaskListItem: 49 | r.renderTaskListItem(n) 50 | case *ast.MathBlock: 51 | r.renderMathBlock(n) 52 | case *ast.Table: 53 | r.renderTable(n) 54 | case *ast.EmbeddedContent: 55 | r.renderEmbeddedContent(n) 56 | case *ast.Text: 57 | r.renderText(n) 58 | case *ast.Bold: 59 | r.renderBold(n) 60 | case *ast.Italic: 61 | r.renderItalic(n) 62 | case *ast.BoldItalic: 63 | r.renderBoldItalic(n) 64 | case *ast.Code: 65 | r.renderCode(n) 66 | case *ast.Image: 67 | r.renderImage(n) 68 | case *ast.Link: 69 | r.renderLink(n) 70 | case *ast.AutoLink: 71 | r.renderAutoLink(n) 72 | case *ast.Tag: 73 | r.renderTag(n) 74 | case *ast.Strikethrough: 75 | r.renderStrikethrough(n) 76 | case *ast.EscapingCharacter: 77 | r.renderEscapingCharacter(n) 78 | case *ast.Math: 79 | r.renderMath(n) 80 | case *ast.Highlight: 81 | r.renderHighlight(n) 82 | case *ast.Subscript: 83 | r.renderSubscript(n) 84 | case *ast.Superscript: 85 | r.renderSuperscript(n) 86 | case *ast.ReferencedContent: 87 | r.renderReferencedContent(n) 88 | case *ast.Spoiler: 89 | r.renderSpoiler(n) 90 | case *ast.HTMLElement: 91 | r.renderHTMLElement(n) 92 | default: 93 | // Handle other block types if needed. 94 | } 95 | } 96 | 97 | // RenderNodes renders a slice of AST nodes to HTML. 98 | func (r *HTMLRenderer) RenderNodes(nodes []ast.Node) { 99 | var prevNode ast.Node 100 | var skipNextLineBreakFlag bool 101 | for _, node := range nodes { 102 | if node.Type() == ast.LineBreakNode && skipNextLineBreakFlag { 103 | if prevNode != nil && ast.IsBlockNode(prevNode) { 104 | skipNextLineBreakFlag = false 105 | continue 106 | } 107 | } 108 | 109 | r.RenderNode(node) 110 | prevNode = node 111 | skipNextLineBreakFlag = true 112 | } 113 | } 114 | 115 | // Render renders the AST to HTML. 116 | func (r *HTMLRenderer) Render(astRoot []ast.Node) string { 117 | r.RenderNodes(astRoot) 118 | return r.output.String() 119 | } 120 | 121 | func (r *HTMLRenderer) renderLineBreak(*ast.LineBreak) { 122 | r.output.WriteString("
") 123 | } 124 | 125 | func (r *HTMLRenderer) renderParagraph(node *ast.Paragraph) { 126 | r.output.WriteString("

") 127 | r.RenderNodes(node.Children) 128 | r.output.WriteString("

") 129 | } 130 | 131 | func (r *HTMLRenderer) renderCodeBlock(node *ast.CodeBlock) { 132 | r.output.WriteString("
")
133 | 	r.output.WriteString(node.Content)
134 | 	r.output.WriteString("
") 135 | } 136 | 137 | func (r *HTMLRenderer) renderHeading(node *ast.Heading) { 138 | element := fmt.Sprintf("h%d", node.Level) 139 | r.output.WriteString(fmt.Sprintf("<%s>", element)) 140 | r.RenderNodes(node.Children) 141 | r.output.WriteString(fmt.Sprintf("", element)) 142 | } 143 | 144 | func (r *HTMLRenderer) renderHorizontalRule(_ *ast.HorizontalRule) { 145 | r.output.WriteString("
") 146 | } 147 | 148 | func (r *HTMLRenderer) renderBlockquote(node *ast.Blockquote) { 149 | r.output.WriteString("
") 150 | r.RenderNodes(node.Children) 151 | r.output.WriteString("
") 152 | } 153 | 154 | func (r *HTMLRenderer) renderList(node *ast.List) { 155 | switch node.Kind { 156 | case ast.OrderedList: 157 | r.output.WriteString("
    ") 158 | case ast.UnorderedList: 159 | r.output.WriteString("
") 169 | case ast.UnorderedList: 170 | r.output.WriteString("") 171 | case ast.DescrpitionList: 172 | r.output.WriteString("") 173 | } 174 | } 175 | 176 | func (r *HTMLRenderer) renderUnorderedListItem(node *ast.UnorderedListItem) { 177 | r.output.WriteString("
  • ") 178 | r.RenderNodes(node.Children) 179 | r.output.WriteString("
    • ") 180 | } 181 | 182 | func (r *HTMLRenderer) renderOrderedListItem(node *ast.OrderedListItem) { 183 | r.output.WriteString("
  • ") 184 | r.RenderNodes(node.Children) 185 | r.output.WriteString("
    • ") 186 | } 187 | 188 | func (r *HTMLRenderer) renderTaskListItem(node *ast.TaskListItem) { 189 | r.output.WriteString("
  • ") 190 | r.output.WriteString("") 195 | r.RenderNodes(node.Children) 196 | r.output.WriteString("
    • ") 197 | } 198 | 199 | func (r *HTMLRenderer) renderMathBlock(node *ast.MathBlock) { 200 | r.output.WriteString("
    ")
201 | 	r.output.WriteString(node.Content)
202 | 	r.output.WriteString("
    ") 203 | } 204 | 205 | func (r *HTMLRenderer) renderTable(node *ast.Table) { 206 | r.output.WriteString("") 207 | r.output.WriteString("") 208 | r.output.WriteString("") 209 | for _, cell := range node.Header { 210 | r.output.WriteString("") 213 | } 214 | r.output.WriteString("") 215 | r.output.WriteString("") 216 | r.output.WriteString("") 217 | for _, row := range node.Rows { 218 | r.output.WriteString("") 219 | for _, cell := range row { 220 | r.output.WriteString("") 223 | } 224 | r.output.WriteString("") 225 | } 226 | r.output.WriteString("") 227 | r.output.WriteString("
    ") 211 | r.RenderNodes([]ast.Node{cell}) 212 | r.output.WriteString("
    ") 221 | r.RenderNodes([]ast.Node{cell}) 222 | r.output.WriteString("
    ") 228 | } 229 | 230 | func (r *HTMLRenderer) renderEmbeddedContent(node *ast.EmbeddedContent) { 231 | r.output.WriteString("
    ") 232 | r.output.WriteString(node.ResourceName) 233 | if node.Params != "" { 234 | r.output.WriteString("?") 235 | r.output.WriteString(node.Params) 236 | } 237 | r.output.WriteString("
    ") 238 | } 239 | 240 | func (r *HTMLRenderer) renderText(node *ast.Text) { 241 | r.output.WriteString(node.Content) 242 | } 243 | 244 | func (r *HTMLRenderer) renderBold(node *ast.Bold) { 245 | r.output.WriteString("") 246 | r.RenderNodes(node.Children) 247 | r.output.WriteString("") 248 | } 249 | 250 | func (r *HTMLRenderer) renderItalic(node *ast.Italic) { 251 | r.output.WriteString("") 252 | r.RenderNodes(node.Children) 253 | r.output.WriteString("") 254 | } 255 | 256 | func (r *HTMLRenderer) renderBoldItalic(node *ast.BoldItalic) { 257 | r.output.WriteString("") 258 | r.output.WriteString(node.Content) 259 | r.output.WriteString("") 260 | } 261 | 262 | func (r *HTMLRenderer) renderCode(node *ast.Code) { 263 | r.output.WriteString("") 264 | r.output.WriteString(node.Content) 265 | r.output.WriteString("") 266 | } 267 | 268 | func (r *HTMLRenderer) renderImage(node *ast.Image) { 269 | r.output.WriteString(``) 272 | r.output.WriteString(node.AltText) 273 | r.output.WriteString(``) 274 | } 275 | 276 | func (r *HTMLRenderer) renderLink(node *ast.Link) { 277 | r.output.WriteString(``) 280 | r.RenderNodes(node.Content) 281 | r.output.WriteString("") 282 | } 283 | 284 | func (r *HTMLRenderer) renderAutoLink(node *ast.AutoLink) { 285 | r.output.WriteString(``) 288 | r.output.WriteString(node.URL) 289 | r.output.WriteString("") 290 | } 291 | 292 | func (r *HTMLRenderer) renderTag(node *ast.Tag) { 293 | r.output.WriteString(``) 294 | r.output.WriteString(`#`) 295 | r.output.WriteString(node.Content) 296 | r.output.WriteString(``) 297 | } 298 | 299 | func (r *HTMLRenderer) renderStrikethrough(node *ast.Strikethrough) { 300 | r.output.WriteString(``) 301 | r.output.WriteString(node.Content) 302 | r.output.WriteString(``) 303 | } 304 | 305 | func (r *HTMLRenderer) renderEscapingCharacter(node *ast.EscapingCharacter) { 306 | r.output.WriteString("\\") 307 | r.output.WriteString(node.Symbol) 308 | } 309 | 310 | func (r *HTMLRenderer) renderMath(node *ast.Math) { 311 | r.output.WriteString("") 312 | r.output.WriteString(node.Content) 313 | r.output.WriteString("") 314 | } 315 | 316 | func (r *HTMLRenderer) renderHighlight(node *ast.Highlight) { 317 | r.output.WriteString("") 318 | r.output.WriteString(node.Content) 319 | r.output.WriteString("") 320 | } 321 | 322 | func (r *HTMLRenderer) renderSubscript(node *ast.Subscript) { 323 | r.output.WriteString("") 324 | r.output.WriteString(node.Content) 325 | r.output.WriteString("") 326 | } 327 | 328 | func (r *HTMLRenderer) renderSuperscript(node *ast.Superscript) { 329 | r.output.WriteString("") 330 | r.output.WriteString(node.Content) 331 | r.output.WriteString("") 332 | } 333 | 334 | func (r *HTMLRenderer) renderReferencedContent(node *ast.ReferencedContent) { 335 | r.output.WriteString("
    ") 336 | r.output.WriteString(node.ResourceName) 337 | if node.Params != "" { 338 | r.output.WriteString("?") 339 | r.output.WriteString(node.Params) 340 | } 341 | r.output.WriteString("
    ") 342 | } 343 | 344 | func (r *HTMLRenderer) renderSpoiler(node *ast.Spoiler) { 345 | r.output.WriteString("
    ") 346 | r.output.WriteString(node.Content) 347 | r.output.WriteString("
    ") 348 | } 349 | 350 | func (r *HTMLRenderer) renderHTMLElement(node *ast.HTMLElement) { 351 | r.output.WriteString(fmt.Sprintf("<%s >", node.TagName)) 352 | } 353 | -------------------------------------------------------------------------------- /renderer/html/html_test.go: -------------------------------------------------------------------------------- 1 | package html 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | 9 | "github.com/usememos/gomark/parser" 10 | "github.com/usememos/gomark/parser/tokenizer" 11 | ) 12 | 13 | func TestHTMLRenderer(t *testing.T) { 14 | tests := []struct { 15 | text string 16 | expected string 17 | }{ 18 | { 19 | text: "Hello world!", 20 | expected: `

    Hello world!

    `, 21 | }, 22 | { 23 | text: "# Hello world!", 24 | expected: `

    Hello world!

    `, 25 | }, 26 | { 27 | text: "> Hello\n> world!", 28 | expected: `

    Hello

    world!

    `, 29 | }, 30 | { 31 | text: "*Hello* world!", 32 | expected: `

    Hello world!

    `, 33 | }, 34 | { 35 | text: "Hello world!\n\nNew paragraph.", 36 | expected: "

    Hello world!


    New paragraph.

    ", 37 | }, 38 | { 39 | text: "**Hello** world!", 40 | expected: `

    Hello world!

    `, 41 | }, 42 | { 43 | text: "#article #memo", 44 | expected: `

    #article #memo

    `, 45 | }, 46 | { 47 | text: "#article \\#memo", 48 | expected: `

    #article \#memo

    `, 49 | }, 50 | { 51 | text: "* Hello\n* world!", 52 | expected: ``, 53 | }, 54 | { 55 | text: "- [ ] hello\n- [x] world", 56 | expected: `
  • hello

  • world
    • `, 57 | }, 58 | } 59 | 60 | for _, test := range tests { 61 | tokens := tokenizer.Tokenize(test.text) 62 | nodes, err := parser.Parse(tokens) 63 | require.NoError(t, err) 64 | actual := NewHTMLRenderer().Render(nodes) 65 | require.Equal(t, test.expected, actual, fmt.Sprintf("Test case: %s", test.text)) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /renderer/renderer.go: -------------------------------------------------------------------------------- 1 | package renderer 2 | 3 | import ( 4 | htmlrenderer "github.com/usememos/gomark/renderer/html" 5 | stringrenderer "github.com/usememos/gomark/renderer/string" 6 | ) 7 | 8 | func NewHTMLRenderer() *htmlrenderer.HTMLRenderer { 9 | return htmlrenderer.NewHTMLRenderer() 10 | } 11 | 12 | func NewStringRenderer() *stringrenderer.StringRenderer { 13 | return stringrenderer.NewStringRenderer() 14 | } 15 | -------------------------------------------------------------------------------- /renderer/string/string.go: -------------------------------------------------------------------------------- 1 | package string 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | 7 | "github.com/usememos/gomark/ast" 8 | ) 9 | 10 | type RendererContext struct { 11 | } 12 | 13 | // StringRenderer renders AST to raw string. 14 | type StringRenderer struct { 15 | output *bytes.Buffer 16 | context *RendererContext 17 | } 18 | 19 | // NewStringRenderer creates a new StringRender. 20 | func NewStringRenderer() *StringRenderer { 21 | return &StringRenderer{ 22 | output: new(bytes.Buffer), 23 | context: &RendererContext{}, 24 | } 25 | } 26 | 27 | // RenderNode renders a single AST node to raw string. 28 | func (r *StringRenderer) RenderNode(node ast.Node) { 29 | switch n := node.(type) { 30 | case *ast.LineBreak: 31 | r.renderLineBreak(n) 32 | case *ast.Paragraph: 33 | r.renderParagraph(n) 34 | case *ast.CodeBlock: 35 | r.renderCodeBlock(n) 36 | case *ast.Heading: 37 | r.renderHeading(n) 38 | case *ast.HorizontalRule: 39 | r.renderHorizontalRule(n) 40 | case *ast.Blockquote: 41 | r.renderBlockquote(n) 42 | case *ast.List: 43 | r.renderList(n) 44 | case *ast.UnorderedListItem: 45 | r.renderUnorderedListItem(n) 46 | case *ast.OrderedListItem: 47 | r.renderOrderedListItem(n) 48 | case *ast.TaskListItem: 49 | r.renderTaskListItem(n) 50 | case *ast.MathBlock: 51 | r.renderMathBlock(n) 52 | case *ast.Table: 53 | r.renderTable(n) 54 | case *ast.EmbeddedContent: 55 | r.renderEmbeddedContent(n) 56 | case *ast.Text: 57 | r.renderText(n) 58 | case *ast.Bold: 59 | r.renderBold(n) 60 | case *ast.Italic: 61 | r.renderItalic(n) 62 | case *ast.BoldItalic: 63 | r.renderBoldItalic(n) 64 | case *ast.Code: 65 | r.renderCode(n) 66 | case *ast.Image: 67 | r.renderImage(n) 68 | case *ast.Link: 69 | r.renderLink(n) 70 | case *ast.AutoLink: 71 | r.renderAutoLink(n) 72 | case *ast.Tag: 73 | r.renderTag(n) 74 | case *ast.Strikethrough: 75 | r.renderStrikethrough(n) 76 | case *ast.EscapingCharacter: 77 | r.renderEscapingCharacter(n) 78 | case *ast.Math: 79 | r.renderMath(n) 80 | case *ast.Highlight: 81 | r.renderHighlight(n) 82 | case *ast.Subscript: 83 | r.renderSubscript(n) 84 | case *ast.Superscript: 85 | r.renderSuperscript(n) 86 | case *ast.ReferencedContent: 87 | r.renderReferencedContent(n) 88 | case *ast.Spoiler: 89 | r.renderSpoiler(n) 90 | case *ast.HTMLElement: 91 | r.renderHTMLElement(n) 92 | default: 93 | // Handle other block types if needed. 94 | } 95 | } 96 | 97 | // RenderNodes renders a slice of AST nodes to raw string. 98 | func (r *StringRenderer) RenderNodes(nodes []ast.Node) { 99 | var prevNode ast.Node 100 | var skipNextLineBreakFlag bool 101 | for _, node := range nodes { 102 | if node.Type() == ast.LineBreakNode && skipNextLineBreakFlag { 103 | if prevNode != nil && ast.IsBlockNode(prevNode) { 104 | skipNextLineBreakFlag = false 105 | continue 106 | } 107 | } 108 | 109 | r.RenderNode(node) 110 | prevNode = node 111 | skipNextLineBreakFlag = true 112 | } 113 | } 114 | 115 | // Render renders the AST to raw string. 116 | func (r *StringRenderer) Render(astRoot []ast.Node) string { 117 | r.RenderNodes(astRoot) 118 | return r.output.String() 119 | } 120 | 121 | func (r *StringRenderer) renderLineBreak(_ *ast.LineBreak) { 122 | r.output.WriteString("\n") 123 | } 124 | 125 | func (r *StringRenderer) renderParagraph(node *ast.Paragraph) { 126 | r.RenderNodes(node.Children) 127 | r.output.WriteString("\n") 128 | } 129 | 130 | func (r *StringRenderer) renderCodeBlock(node *ast.CodeBlock) { 131 | r.output.WriteString(node.Content) 132 | } 133 | 134 | func (r *StringRenderer) renderHeading(node *ast.Heading) { 135 | r.RenderNodes(node.Children) 136 | r.output.WriteString("\n") 137 | } 138 | 139 | func (r *StringRenderer) renderHorizontalRule(_ *ast.HorizontalRule) { 140 | r.output.WriteString("\n") 141 | } 142 | 143 | func (r *StringRenderer) renderBlockquote(node *ast.Blockquote) { 144 | r.RenderNodes(node.Children) 145 | r.output.WriteString("\n") 146 | } 147 | 148 | func (r *StringRenderer) renderList(node *ast.List) { 149 | for _, item := range node.Children { 150 | r.RenderNodes([]ast.Node{item}) 151 | } 152 | } 153 | 154 | func (r *StringRenderer) renderUnorderedListItem(node *ast.UnorderedListItem) { 155 | r.output.WriteString(node.Symbol) 156 | r.RenderNodes(node.Children) 157 | } 158 | 159 | func (r *StringRenderer) renderOrderedListItem(node *ast.OrderedListItem) { 160 | r.output.WriteString(fmt.Sprintf("%s. ", node.Number)) 161 | r.RenderNodes(node.Children) 162 | } 163 | 164 | func (r *StringRenderer) renderTaskListItem(node *ast.TaskListItem) { 165 | r.output.WriteString(node.Symbol) 166 | r.RenderNodes(node.Children) 167 | } 168 | 169 | func (r *StringRenderer) renderMathBlock(node *ast.MathBlock) { 170 | r.output.WriteString(node.Content) 171 | r.output.WriteString("\n") 172 | } 173 | 174 | func (r *StringRenderer) renderTable(node *ast.Table) { 175 | for _, cell := range node.Header { 176 | r.RenderNodes([]ast.Node{cell}) 177 | r.output.WriteString("\t") 178 | } 179 | r.output.WriteString("\n") 180 | for _, row := range node.Rows { 181 | for _, cell := range row { 182 | r.RenderNodes([]ast.Node{cell}) 183 | r.output.WriteString("\t") 184 | } 185 | r.output.WriteString("\n") 186 | } 187 | } 188 | 189 | func (*StringRenderer) renderEmbeddedContent(_ *ast.EmbeddedContent) {} 190 | 191 | func (r *StringRenderer) renderText(node *ast.Text) { 192 | r.output.WriteString(node.Content) 193 | } 194 | 195 | func (r *StringRenderer) renderBold(node *ast.Bold) { 196 | r.RenderNodes(node.Children) 197 | } 198 | 199 | func (r *StringRenderer) renderItalic(node *ast.Italic) { 200 | r.RenderNodes(node.Children) 201 | } 202 | 203 | func (r *StringRenderer) renderBoldItalic(node *ast.BoldItalic) { 204 | r.output.WriteString(node.Content) 205 | } 206 | 207 | func (r *StringRenderer) renderCode(node *ast.Code) { 208 | r.output.WriteString(node.Content) 209 | } 210 | 211 | func (*StringRenderer) renderImage(_ *ast.Image) {} 212 | 213 | func (r *StringRenderer) renderLink(node *ast.Link) { 214 | r.output.WriteString(node.URL) 215 | } 216 | 217 | func (r *StringRenderer) renderAutoLink(node *ast.AutoLink) { 218 | r.output.WriteString(node.URL) 219 | } 220 | 221 | func (r *StringRenderer) renderTag(node *ast.Tag) { 222 | r.output.WriteString(`#`) 223 | r.output.WriteString(node.Content) 224 | } 225 | 226 | func (r *StringRenderer) renderStrikethrough(node *ast.Strikethrough) { 227 | r.output.WriteString(node.Content) 228 | } 229 | 230 | func (r *StringRenderer) renderEscapingCharacter(node *ast.EscapingCharacter) { 231 | r.output.WriteString("\\") 232 | r.output.WriteString(node.Symbol) 233 | } 234 | 235 | func (r *StringRenderer) renderMath(node *ast.Math) { 236 | r.output.WriteString(node.Content) 237 | } 238 | 239 | func (r *StringRenderer) renderHighlight(node *ast.Highlight) { 240 | r.output.WriteString(node.Content) 241 | } 242 | 243 | func (r *StringRenderer) renderSubscript(node *ast.Subscript) { 244 | r.output.WriteString(node.Content) 245 | } 246 | 247 | func (r *StringRenderer) renderSuperscript(node *ast.Superscript) { 248 | r.output.WriteString(node.Content) 249 | } 250 | 251 | func (*StringRenderer) renderReferencedContent(_ *ast.ReferencedContent) {} 252 | 253 | func (r *StringRenderer) renderSpoiler(node *ast.Spoiler) { 254 | r.output.WriteString(node.Content) 255 | } 256 | 257 | func (r *StringRenderer) renderHTMLElement(*ast.HTMLElement) { 258 | r.output.WriteString("\n") 259 | } 260 | -------------------------------------------------------------------------------- /renderer/string/string_test.go: -------------------------------------------------------------------------------- 1 | package string 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/usememos/gomark/parser" 9 | "github.com/usememos/gomark/parser/tokenizer" 10 | ) 11 | 12 | func TestStringRender(t *testing.T) { 13 | tests := []struct { 14 | text string 15 | expected string 16 | }{ 17 | { 18 | text: "", 19 | expected: "", 20 | }, 21 | { 22 | text: "Hello world!", 23 | expected: "Hello world!\n", 24 | }, 25 | { 26 | text: "**Hello** world!", 27 | expected: "Hello world!\n", 28 | }, 29 | { 30 | text: "Test\n1. Hello\n2. World", 31 | expected: "Test\n1. Hello\n2. World", 32 | }, 33 | } 34 | 35 | for _, test := range tests { 36 | tokens := tokenizer.Tokenize(test.text) 37 | nodes, err := parser.Parse(tokens) 38 | require.NoError(t, err) 39 | actual := NewStringRenderer().Render(nodes) 40 | require.Equal(t, test.expected, actual) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /restore/restore.go: -------------------------------------------------------------------------------- 1 | package restore 2 | 3 | import "github.com/usememos/gomark/ast" 4 | 5 | func Restore(nodes []ast.Node) string { 6 | var result string 7 | for _, node := range nodes { 8 | result += node.Restore() 9 | } 10 | return result 11 | } 12 | -------------------------------------------------------------------------------- /restore/restore_test.go: -------------------------------------------------------------------------------- 1 | package restore 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | 8 | "github.com/usememos/gomark/ast" 9 | ) 10 | 11 | func TestRestore(t *testing.T) { 12 | tests := []struct { 13 | nodes []ast.Node 14 | rawText string 15 | }{ 16 | { 17 | nodes: nil, 18 | rawText: "", 19 | }, 20 | { 21 | nodes: []ast.Node{ 22 | &ast.Text{ 23 | Content: "Hello world!", 24 | }, 25 | }, 26 | rawText: "Hello world!", 27 | }, 28 | { 29 | nodes: []ast.Node{ 30 | &ast.Paragraph{ 31 | Children: []ast.Node{ 32 | &ast.Text{ 33 | Content: "Code: ", 34 | }, 35 | &ast.Code{ 36 | Content: "Hello world!", 37 | }, 38 | }, 39 | }, 40 | }, 41 | rawText: "Code: `Hello world!`", 42 | }, 43 | } 44 | 45 | for _, test := range tests { 46 | require.Equal(t, Restore(test.nodes), test.rawText) 47 | } 48 | } 49 | --------------------------------------------------------------------------------