├── .editorconfig ├── .gitattributes ├── .github ├── screenshot.png └── workflows │ ├── ci.yml │ ├── publish.yml │ └── release.yml ├── .gitignore ├── CMakeLists.txt ├── CONTRIBUTING.md ├── Cargo.toml ├── LICENSE ├── Makefile ├── Package.resolved ├── Package.swift ├── README.md ├── binding.gyp ├── bindings ├── go │ ├── binding_test.go │ ├── markdown.go │ └── markdown_inline.go ├── node │ ├── binding.cc │ ├── binding_test.js │ ├── index.d.ts │ ├── index.js │ └── inline.js ├── python │ ├── tests │ │ └── test_binding.py │ └── tree_sitter_markdown │ │ ├── __init__.py │ │ ├── __init__.pyi │ │ ├── binding.c │ │ └── py.typed ├── rust │ ├── benchmark.rs │ ├── build.rs │ ├── lib.rs │ └── parser.rs └── swift │ ├── .gitignore │ └── TreeSitterMarkdownTests │ └── TreeSitterMarkdownTests.swift ├── common ├── common.js ├── common.mak └── html_entities.json ├── go.mod ├── package-lock.json ├── package.json ├── pyproject.toml ├── scripts ├── build.js └── test.js ├── setup.py ├── tree-sitter-markdown-inline ├── CMakeLists.txt ├── Makefile ├── bindings │ ├── c │ │ ├── tree-sitter-markdown-inline.h │ │ └── tree-sitter-markdown-inline.pc.in │ └── swift │ │ └── TreeSitterMarkdownInline │ │ └── markdown_inline.h ├── grammar.js ├── package.json ├── queries │ ├── highlights.scm │ └── injections.scm ├── src │ ├── grammar.json │ ├── node-types.json │ ├── parser.c │ ├── scanner.c │ └── tree_sitter │ │ ├── alloc.h │ │ ├── array.h │ │ └── parser.h └── test │ └── corpus │ ├── extension_latex.txt │ ├── extension_strikethrough.txt │ ├── extension_wikilink.txt │ ├── failing.txt │ ├── issues.txt │ ├── spec.txt │ └── tags.txt ├── tree-sitter-markdown ├── CMakeLists.txt ├── Makefile ├── bindings │ ├── c │ │ ├── tree-sitter-markdown.h │ │ └── tree-sitter-markdown.pc.in │ └── swift │ │ └── TreeSitterMarkdown │ │ └── markdown.h ├── grammar.js ├── package.json ├── queries │ ├── highlights.scm │ └── injections.scm ├── src │ ├── grammar.json │ ├── node-types.json │ ├── parser.c │ ├── scanner.c │ └── tree_sitter │ │ ├── alloc.h │ │ ├── array.h │ │ └── parser.h └── test │ └── corpus │ ├── extension_minus_metadata.txt │ ├── extension_pipe_table.txt │ ├── extension_plus_metadata.txt │ ├── extension_task_list.txt │ ├── failing.txt │ ├── issues.txt │ └── spec.txt └── tree-sitter.json /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | 6 | [*.{json,toml,yml,gyp}] 7 | indent_style = space 8 | indent_size = 2 9 | 10 | [*.js] 11 | indent_style = space 12 | indent_size = 2 13 | quote_type = single 14 | 15 | [*.{c,cc,h}] 16 | indent_style = space 17 | indent_size = 4 18 | 19 | [*.rs] 20 | indent_style = space 21 | indent_size = 4 22 | 23 | [*.{py,pyi}] 24 | indent_style = space 25 | indent_size = 4 26 | 27 | [*.swift] 28 | indent_style = space 29 | indent_size = 4 30 | 31 | [*.go] 32 | indent_style = tab 33 | indent_size = 8 34 | 35 | [Makefile] 36 | indent_style = tab 37 | indent_size = 8 38 | 39 | [parser.c] 40 | indent_size = 2 41 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto eol=lf 2 | 3 | *.png -text 4 | 5 | # Generated source files 6 | */src/*.json linguist-generated 7 | */src/parser.c linguist-generated 8 | */src/tree_sitter/* linguist-generated 9 | 10 | # C bindings 11 | */bindings/c/* linguist-generated 12 | 13 | # Rust bindings 14 | bindings/rust/build.rs linguist-generated 15 | Cargo.lock linguist-generated 16 | 17 | # Node.js bindings 18 | bindings/node/* linguist-generated 19 | binding.gyp linguist-generated 20 | package.json linguist-generated 21 | package-lock.json linguist-generated 22 | 23 | # Python bindings 24 | bindings/python/** linguist-generated 25 | setup.py linguist-generated 26 | pyproject.toml linguist-generated 27 | 28 | # Go bindings 29 | bindings/go/* linguist-generated 30 | go.mod linguist-generated 31 | go.sum linguist-generated 32 | 33 | # Swift bindings 34 | /bindings/swift/** linguist-generated 35 | */bindings/swift/** linguist-generated 36 | Package.swift linguist-generated 37 | Package.resolved linguist-generated 38 | -------------------------------------------------------------------------------- /.github/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tree-sitter-grammars/tree-sitter-markdown/efb075cbd57ce33f694c2bb264b99cdba0f31789/.github/screenshot.png -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [split_parser] 6 | pull_request: 7 | workflow_dispatch: 8 | 9 | jobs: 10 | test: 11 | name: Test parsers 12 | runs-on: ${{matrix.os}} 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | os: [ubuntu-latest, windows-latest, macos-latest] 17 | steps: 18 | - name: Checkout repository 19 | uses: actions/checkout@v4 20 | 21 | - name: Set up tree-sitter 22 | uses: tree-sitter/setup-action@v2 23 | with: 24 | install-lib: false 25 | 26 | - name: Set up Node.js 27 | uses: actions/setup-node@v4 28 | with: 29 | node-version: ${{vars.NODE_VERSION}} 30 | 31 | - name: Build with all extensions 32 | run: node scripts/build.js 33 | env: 34 | ALL_EXTENSIONS: 1 35 | 36 | - name: Run tests 37 | uses: tree-sitter/parser-test-action@v2 38 | with: 39 | test-parser-cmd: node scripts/test.js 40 | 41 | - name: Rebuild with default extensions 42 | run: node scripts/build.js 43 | - name: Verify grammar consistency 44 | run: git diff --exit-code -- */src/grammar.json 45 | 46 | fuzz: 47 | name: Fuzz parsers 48 | runs-on: ubuntu-latest 49 | strategy: 50 | fail-fast: false 51 | matrix: 52 | parser: [tree-sitter-markdown, tree-sitter-markdown-inline] 53 | steps: 54 | - name: Checkout repository 55 | uses: actions/checkout@v4 56 | with: 57 | fetch-depth: 2 58 | - name: Check for scanner changes 59 | id: scanner-changes 60 | run: |- 61 | if git diff --quiet HEAD^ -- '${{matrix.parser}}/src/scanner.c'; then 62 | printf 'changed=false\n' >> "$GITHUB_OUTPUT" 63 | else 64 | printf 'changed=true\n' >> "$GITHUB_OUTPUT" 65 | fi 66 | - name: Fuzz ${{matrix.parser}} parser 67 | uses: tree-sitter/fuzz-action@v4 68 | if: steps.scanner-changes.outputs.changed == 'true' 69 | with: 70 | directory: ${{matrix.parser}} 71 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish package 2 | 3 | on: 4 | push: 5 | tags: ["*"] 6 | 7 | jobs: 8 | github: 9 | uses: tree-sitter/workflows/.github/workflows/release.yml@main 10 | permissions: 11 | contents: write 12 | id-token: write 13 | npm: 14 | uses: tree-sitter/workflows/.github/workflows/package-npm.yml@main 15 | with: 16 | package-name: "@tree-sitter-grammars/tree-sitter-markdown" 17 | secrets: 18 | NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} 19 | crates: 20 | uses: tree-sitter/workflows/.github/workflows/package-crates.yml@main 21 | with: 22 | package-name: tree-sitter-md 23 | secrets: 24 | CARGO_REGISTRY_TOKEN: ${{secrets.CARGO_TOKEN}} 25 | pypi: 26 | uses: tree-sitter/workflows/.github/workflows/package-pypi.yml@main 27 | with: 28 | package-name: tree-sitter-markdown 29 | secrets: 30 | PYPI_API_TOKEN: ${{secrets.PYPI_TOKEN}} 31 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Create release 2 | 3 | on: 4 | push: 5 | tags: ["*"] 6 | workflow_dispatch: 7 | 8 | concurrency: 9 | group: ${{github.workflow}}-${{github.ref}} 10 | cancel-in-progress: true 11 | 12 | permissions: 13 | contents: write 14 | id-token: write 15 | attestations: write 16 | 17 | jobs: 18 | release: 19 | uses: tree-sitter/workflows/.github/workflows/release.yml@main 20 | with: 21 | attestations: true 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Rust artifacts 2 | Cargo.lock 3 | target/ 4 | 5 | # Node artifacts 6 | build/ 7 | prebuilds/ 8 | node_modules/ 9 | *.tgz 10 | 11 | # Swift artifacts 12 | .build/ 13 | 14 | # Go artifacts 15 | go.sum 16 | _obj/ 17 | 18 | # Python artifacts 19 | .venv/ 20 | dist/ 21 | *.egg-info 22 | *.whl 23 | 24 | # C artifacts 25 | *.a 26 | *.so 27 | *.so.* 28 | *.dylib 29 | *.dll 30 | *.pc 31 | 32 | # Example dirs 33 | /examples/*/ 34 | 35 | # Grammar volatiles 36 | *.wasm 37 | *.obj 38 | *.o 39 | 40 | # Moved bindings 41 | bindings/c/ 42 | bindings/swift/TreeSitterMarkdown/ 43 | bindings/swift/TreeSitterMarkdownInline/ 44 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.13) 2 | 3 | project(tree-sitter-markdown 4 | VERSION "0.5.0" 5 | DESCRIPTION "Markdown grammar for tree-sitter" 6 | HOMEPAGE_URL "https://github.com/tree-sitter-grammars/tree-sitter-markdown" 7 | LANGUAGES C) 8 | 9 | option(BUILD_SHARED_LIBS "Build using shared libraries" ON) 10 | option(TREE_SITTER_REUSE_ALLOCATOR "Reuse the library allocator" OFF) 11 | option(ALL_EXTENSIONS "Enable all Markdown extensions" OFF) 12 | 13 | set(TREE_SITTER_ABI_VERSION 14 CACHE STRING "Tree-sitter ABI version") 14 | if(NOT ${TREE_SITTER_ABI_VERSION} MATCHES "^[0-9]+$") 15 | unset(TREE_SITTER_ABI_VERSION CACHE) 16 | message(FATAL_ERROR "TREE_SITTER_ABI_VERSION must be an integer") 17 | endif() 18 | 19 | find_program(TREE_SITTER_CLI tree-sitter DOC "Tree-sitter CLI") 20 | 21 | include(GNUInstallDirs) 22 | 23 | macro(add_parser name) 24 | add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c" 25 | DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/grammar.js" 26 | COMMAND "${CMAKE_COMMAND}" -E env ALL_EXTENSIONS=$ 27 | -- "${TREE_SITTER_CLI}" generate --abi=${TREE_SITTER_ABI_VERSION} 28 | BYPRODUCTS "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json" 29 | "${CMAKE_CURRENT_SOURCE_DIR}/src/node-types.json" 30 | "${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/alloc.h" 31 | "${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/array.h" 32 | "${CMAKE_CURRENT_SOURCE_DIR}/src/tree_sitter/parser.h" 33 | WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" 34 | COMMENT "Generating parser.c") 35 | 36 | add_library(tree-sitter-${name} 37 | "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c" 38 | "${CMAKE_CURRENT_SOURCE_DIR}/src/scanner.c") 39 | target_include_directories(tree-sitter-${name} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") 40 | 41 | target_compile_definitions(tree-sitter-${name} PRIVATE 42 | $<$:TREE_SITTER_REUSE_ALLOCATOR> 43 | $<$:TREE_SITTER_DEBUG>) 44 | 45 | set_target_properties(tree-sitter-${name} 46 | PROPERTIES 47 | C_STANDARD 11 48 | POSITION_INDEPENDENT_CODE ON 49 | SOVERSION "${TREE_SITTER_ABI_VERSION}.${PROJECT_VERSION_MAJOR}" 50 | DEFINE_SYMBOL "") 51 | 52 | configure_file("${CMAKE_CURRENT_SOURCE_DIR}/bindings/c/tree-sitter-${name}.pc.in" 53 | "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter-${name}.pc" @ONLY) 54 | 55 | install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/bindings/c/tree-sitter-${name}.h" 56 | DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/tree_sitter") 57 | install(FILES "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter-${name}.pc" 58 | DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig") 59 | install(TARGETS tree-sitter-${name} 60 | LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}") 61 | endmacro() 62 | 63 | add_subdirectory(tree-sitter-markdown tree-sitter-markdown) 64 | 65 | add_subdirectory(tree-sitter-markdown-inline tree-sitter-markdown-inline) 66 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | All contributions are welcome. Specifically, if you found a bug or have a 4 | suggestion for a new feature or markdown extension, you can always open an 5 | [issue] or [pull request]. 6 | 7 | ## Issues 8 | 9 | If you open an issue please give a short description of your bug or feature 10 | along with a code example. If there is a relevant spec like the [CommonMark 11 | Spec][commonmark] or the [GitHub Flavored Markdown Spec][gfm] please link it in 12 | the issue. 13 | 14 | Any feature suggestions are welcome. The grammar should by default only support 15 | very common syntax, but any extension can be implemented behind a compile time 16 | flag. (See below) 17 | 18 | Some bug reports belong in other repositories if they only concern the 19 | implementation of the grammar in a specific context like `nvim-treesitter`, but 20 | you can always open an issue here and I will point you in the right direction. 21 | 22 | ## Code Overview 23 | 24 | Please refer to the [tree-sitter spec] for more details on how to write a tree- 25 | sitter grammar. 26 | 27 | This parse is split into two grammars. One for block structure, which can be 28 | found in the `tree-sitter-markdown` folder, and one for inline structure, which 29 | can be found in the `tree-sitter-markdown-inline` folder. Components that are 30 | parts of either grammar can be found in the `common` folder. 31 | 32 | For either of the grammar the most important files are the `grammar.js` which 33 | defines most nodes and the `src/scanner.c` which defines nodes that cannot 34 | be parsed with normal tree-sitter rules. All other files in the `src` subfolder 35 | are auto-generated by running `tree-sitter generate`. (You need to install the 36 | [tree-sitter cli tool][tree-sitter-cli] and [Node.js][nodejs] first.) 37 | 38 | Some syntactical components can be enabled or disabled by environment variables 39 | at compile time. The logic for this can be found in the `common/grammar.js` 40 | file. 41 | 42 | Tests are located in the `test/corpus` subfolder: 43 | * `spec.txt` is taken from the examples in the [GFM spec][gfm]. 44 | * `failing.txt` are those examples from the spec that do not pass yet. 45 | * `issues.txt` are test cases covering solved issues. 46 | * `extension_<>.txt` are covering specific extensions. Some of these are also 47 | taken from the GFM spec. 48 | 49 | ## Pull Requests 50 | 51 | I will happily accept any pull requests. 52 | 53 | Before submitting any code please check the following: 54 | 55 | * You ran `tree-sitter generate` in the `tree-sitter-markdown` or 56 | `tree-sitter-markdown-inline` directories respectively after modifying any 57 | `grammar.js` file. 58 | * When running `tree-sitter test` only the cases defined in `failing.txt` or 59 | `extension_<>.txt` for not activated extensions fail for **both** grammars. 60 | * If you implemented new behavior please add tests. (In most cases these belong 61 | in a `extension_<>.txt`.) 62 | * You deleted any auto-generated bindings and files for debugging purposes 63 | like `log.html` 64 | 65 | ## Tests 66 | 67 | To run the tests, first install the development dependencies (see above), then 68 | execute: 69 | 70 | ```sh 71 | ALL_EXTENSIONS=1 node scripts/build.js 72 | node scripts/test.js 73 | ``` 74 | 75 | [issue]: https://github.com/tree-sitter-grammars/tree-sitter-markdown/issues/new 76 | [pull request]: https://github.com/tree-sitter-grammars/tree-sitter-markdown/compare 77 | [gfm]: https://github.github.com/gfm/ 78 | [commonmark]: https://spec.commonmark.org/ 79 | [tree-sitter spec]: https://tree-sitter.github.io/tree-sitter/ 80 | [tree-sitter-cli]: https://github.com/tree-sitter/tree-sitter/blob/master/cli/README.md 81 | [nodejs]: https://nodejs.org/ 82 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tree-sitter-md" 3 | description = "Markdown grammar for tree-sitter" 4 | version = "0.5.0" 5 | authors = ["MDeiml"] 6 | license = "MIT" 7 | readme = "README.md" 8 | keywords = ["incremental", "parsing", "tree-sitter", "markdown"] 9 | categories = ["parsing", "text-editors"] 10 | repository = "https://github.com/tree-sitter-grammars/tree-sitter-markdown" 11 | edition = "2021" 12 | autoexamples = false 13 | 14 | build = "bindings/rust/build.rs" 15 | include = [ 16 | "bindings/rust/*", 17 | "tree-sitter-markdown/src/*", 18 | "tree-sitter-markdown-inline/src/*", 19 | "tree-sitter-markdown/grammar.js", 20 | "tree-sitter-markdown-inline/grammar.js", 21 | "tree-sitter-markdown/queries/*", 22 | "tree-sitter-markdown-inline/queries/*", 23 | "common/grammar.js", 24 | "common/html_entities.json", 25 | ] 26 | 27 | [features] 28 | parser = ["tree-sitter"] 29 | 30 | [lib] 31 | path = "bindings/rust/lib.rs" 32 | 33 | [dependencies] 34 | tree-sitter-language = "0.1" 35 | tree-sitter = { version = "0.24", optional = true } 36 | 37 | [dev-dependencies] 38 | tree-sitter = "0.24.3" 39 | 40 | [build-dependencies] 41 | cc = "1.1.22" 42 | 43 | [[bin]] 44 | name = "benchmark" 45 | path = "bindings/rust/benchmark.rs" 46 | required-features = ["parser"] 47 | 48 | [profile.release] 49 | debug = true 50 | 51 | [package.metadata.docs.rs] 52 | features = ["parser"] 53 | rustdoc-args = ["--cfg", "docsrs"] 54 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Matthias Deiml 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all install uninstall clean: 2 | $(MAKE) -C tree-sitter-markdown $@ 3 | $(MAKE) -C tree-sitter-markdown-inline $@ 4 | 5 | .PHONY: all install uninstall clean 6 | -------------------------------------------------------------------------------- /Package.resolved: -------------------------------------------------------------------------------- 1 | { 2 | "object": { 3 | "pins": [ 4 | { 5 | "package": "SwiftTreeSitter", 6 | "repositoryURL": "https://github.com/ChimeHQ/SwiftTreeSitter", 7 | "state": { 8 | "branch": null, 9 | "revision": "2599e95310b3159641469d8a21baf2d3d200e61f", 10 | "version": "0.8.0" 11 | } 12 | } 13 | ] 14 | }, 15 | "version": 1 16 | } 17 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.3 2 | import PackageDescription 3 | 4 | let package = Package( 5 | name: "TreeSitterMarkdown", 6 | platforms: [.macOS(.v10_13), .iOS(.v11)], 7 | products: [ 8 | .library(name: "TreeSitterMarkdown", targets: ["TreeSitterMarkdown", "TreeSitterMarkdownInline"]), 9 | ], 10 | dependencies: [ 11 | .package(url: "https://github.com/ChimeHQ/SwiftTreeSitter", from: "0.8.0"), 12 | ], 13 | targets: [ 14 | .target( 15 | name: "TreeSitterMarkdown", 16 | path: "tree-sitter-markdown", 17 | sources: [ 18 | "src/parser.c", 19 | "src/scanner.c", 20 | ], 21 | resources: [ 22 | .copy("queries") 23 | ], 24 | publicHeadersPath: "bindings/swift", 25 | cSettings: [.headerSearchPath("src")] 26 | ), 27 | .target( 28 | name: "TreeSitterMarkdownInline", 29 | path: "tree-sitter-markdown-inline", 30 | exclude: [ 31 | "test", 32 | "grammar.js", 33 | ], 34 | sources: [ 35 | "src/parser.c", 36 | "src/scanner.c", 37 | ], 38 | resources: [ 39 | .copy("queries") 40 | ], 41 | publicHeadersPath: "bindings/swift", 42 | cSettings: [.headerSearchPath("src")] 43 | ), 44 | .testTarget( 45 | name: "TreeSitterMarkdownTests", 46 | dependencies: [ 47 | "SwiftTreeSitter", 48 | "TreeSitterMarkdown", 49 | "TreeSitterMarkdownInline", 50 | ], 51 | path: "bindings/swift/TreeSitterMarkdownTests" 52 | ) 53 | ], 54 | cLanguageStandard: .c11 55 | ) 56 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tree-sitter-markdown 2 | 3 | [![CI][ci]](https://github.com/tree-sitter-grammars/tree-sitter-markdown/actions) 4 | [![discord][discord]](https://discord.gg/w7nTvsVJhm) 5 | [![matrix][matrix]](https://matrix.to/#/#tree-sitter-chat:matrix.org) 6 | [![npm][npm]](https://www.npmjs.com/package/@tree-sitter-grammars/tree-sitter-markdown) 7 | [![crates][crates]](https://crates.io/crates/tree-sitter-md) 8 | [![pypi][pypi]](https://pypi.org/project/tree-sitter-markdown/) 9 | 10 | A Markdown parser for [tree-sitter]. 11 | 12 | ![screenshot](https://github.com/MDeiml/tree-sitter-markdown/blob/split_parser/.github/screenshot.png) 13 | 14 | The parser is designed to read markdown according to the [CommonMark Spec], 15 | but some extensions to the spec from different sources such as [Github flavored 16 | markdown] are also included. These can be toggled on or off at compile time. 17 | For specifics see [Extensions](#extensions) 18 | 19 | ## Goals 20 | 21 | Even though this parser has existed for some while and obvious issues are 22 | mostly solved, there are still lots of inaccuracies in the output. These stem 23 | from restricting a complex format such as markdown to the quite restricting 24 | tree-sitter parsing rules. 25 | 26 | As such it is not recommended to use this parser where correctness is 27 | important. The main goal for this parser is to provide syntactical information 28 | for syntax highlighting in parsers such as [neovim] and [helix]. 29 | 30 | ## Contributing 31 | 32 | All contributions are welcome. For details refer to [CONTRIBUTING.md]. 33 | 34 | ## Extensions 35 | 36 | Extensions can be enabled at compile time through environment variables. Some 37 | of them are on by default, these can be disabled with the environment variable 38 | `NO_DEFAULT_EXTENSIONS`. 39 | 40 | | Name | Environment variable | Specification | Default | Also enables | 41 | |:----:|:--------------------:|:-------------:|:-------:|:------------:| 42 | | Github flavored markdown | `EXTENSION_GFM` | [link](https://github.github.com/gfm/) | ✓ | Task lists, strikethrough, pipe tables | 43 | | Task lists | `EXTENSION_TASK_LIST` | [link](https://github.github.com/gfm/#task-list-items-extension-) | ✓ | | 44 | | Strikethrough | `EXTENSION_STRIKETHROUGH` | [link](https://github.github.com/gfm/#strikethrough-extension-) | ✓ | | 45 | | Pipe tables | `EXTENSION_PIPE_TABLE` | [link](https://github.github.com/gfm/#tables-extension-) | ✓ | | 46 | | YAML metadata | `EXTENSION_MINUS_METADATA` | [link](https://gohugo.io/content-management/front-matter/) | ✓ | | 47 | | TOML metadata | `EXTENSION_PLUS_METADATA` | [link](https://gohugo.io/content-management/front-matter/) | ✓ | | 48 | | Tags | `EXTENSION_TAGS` | [link](https://help.obsidian.md/Editing+and+formatting/Tags#Tag+format) | | | 49 | | Wiki Link | `EXTENSION_WIKI_LINK` | [link](https://help.obsidian.md/Linking+notes+and+files/Internal+links) | | | 50 | 51 | ## Usage in Editors 52 | 53 | For guides on how to use this parser in a specific editor, refer to that 54 | editor's specific documentation, e.g. 55 | * [neovim](https://github.com/nvim-treesitter/nvim-treesitter) 56 | * [helix](https://docs.helix-editor.com/guides/adding_languages.html) 57 | 58 | ## Standalone usage 59 | 60 | To use the two grammars, first parse the document with the block 61 | grammar. Then perform a second parse with the inline grammar using 62 | `ts_parser_set_included_ranges` to specify which parts are inline content. 63 | These parts are marked as `inline` nodes. Children of those inline nodes should 64 | be excluded from these ranges. For an example implementation see `lib.rs` in 65 | the `bindings` folder. 66 | 67 | ### Usage with WASM 68 | 69 | Unfortunately using this parser with WASM/web-tree-sitter does not work out of the box at the moment. This is because the parser uses some C functions that are not exported by tree-sitter by default. To fix this you can statically link the parser to tree-sitter. See also https://github.com/tree-sitter/tree-sitter/issues/949, https://github.com/MDeiml/tree-sitter-markdown/issues/126, and https://github.com/MDeiml/tree-sitter-markdown/issues/93 70 | 71 | [CommonMark Spec]: https://spec.commonmark.org/ 72 | [Github flavored markdown]: https://github.github.com/gfm/ 73 | [tree-sitter]: https://tree-sitter.github.io/tree-sitter/ 74 | [neovim]: https://neovim.io/ 75 | [helix]: https://helix-editor.com/ 76 | [CONTRIBUTING.md]: https://github.com/MDeiml/tree-sitter-markdown/blob/split_parser/CONTRIBUTING.md 77 | [ci]: https://img.shields.io/github/actions/workflow/status/tree-sitter-grammars/tree-sitter-markdown/ci.yml?logo=github&label=CI 78 | [discord]: https://img.shields.io/discord/1063097320771698699?logo=discord&label=discord 79 | [matrix]: https://img.shields.io/matrix/tree-sitter-chat%3Amatrix.org?logo=matrix&label=matrix 80 | [npm]: https://img.shields.io/npm/v/%40tree-sitter-grammars%2Ftree-sitter-markdown?logo=npm 81 | [crates]: https://img.shields.io/crates/v/tree-sitter-md?logo=rust 82 | [pypi]: https://img.shields.io/pypi/v/tree-sitter-markdown?logo=pypi&logoColor=ffd242 83 | -------------------------------------------------------------------------------- /binding.gyp: -------------------------------------------------------------------------------- 1 | { 2 | "targets": [ 3 | { 4 | "target_name": "tree_sitter_markdown_binding", 5 | "dependencies": [ 6 | " 2 | 3 | typedef struct TSLanguage TSLanguage; 4 | 5 | extern "C" TSLanguage * tree_sitter_markdown(); 6 | extern "C" TSLanguage * tree_sitter_markdown_inline(); 7 | 8 | // "tree-sitter", "language" hashed with BLAKE2 9 | const napi_type_tag LANGUAGE_TYPE_TAG = { 10 | 0x8AF2E5212AD58ABF, 0xD5006CAD83ABBA16 11 | }; 12 | 13 | Napi::Object Init(Napi::Env env, Napi::Object exports) { 14 | exports["name"] = Napi::String::New(env, "markdown"); 15 | auto markdown_language = Napi::External::New(env, tree_sitter_markdown()); 16 | markdown_language.TypeTag(&LANGUAGE_TYPE_TAG); 17 | exports["language"] = markdown_language; 18 | 19 | auto md_inline = Napi::Object::New(env); 20 | md_inline["name"] = Napi::String::New(env, "markdown_inline"); 21 | auto md_inline_language = Napi::External::New(env, tree_sitter_markdown_inline()); 22 | md_inline_language.TypeTag(&LANGUAGE_TYPE_TAG); 23 | md_inline["language"] = md_inline_language; 24 | exports["inline"] = md_inline; 25 | 26 | return exports; 27 | } 28 | 29 | NODE_API_MODULE(tree_sitter_markdown_binding, Init); 30 | -------------------------------------------------------------------------------- /bindings/node/binding_test.js: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | const assert = require("node:assert"); 4 | const { test } = require("node:test"); 5 | 6 | const Parser = require("tree-sitter"); 7 | 8 | test("can load block grammar", () => { 9 | const parser = new Parser(); 10 | assert.doesNotThrow(() => parser.setLanguage(require("."))); 11 | }); 12 | 13 | test("can load inline grammar", () => { 14 | const parser = new Parser(); 15 | assert.doesNotThrow(() => parser.setLanguage(require(".").inline)); 16 | }); 17 | -------------------------------------------------------------------------------- /bindings/node/index.d.ts: -------------------------------------------------------------------------------- 1 | type BaseNode = { 2 | type: string; 3 | named: boolean; 4 | }; 5 | 6 | type ChildNode = { 7 | multiple: boolean; 8 | required: boolean; 9 | types: BaseNode[]; 10 | }; 11 | 12 | type NodeInfo = 13 | | (BaseNode & { 14 | subtypes: BaseNode[]; 15 | }) 16 | | (BaseNode & { 17 | fields: { [name: string]: ChildNode }; 18 | children: ChildNode[]; 19 | }); 20 | 21 | type Language = { 22 | name: string; 23 | language: unknown; 24 | nodeTypeInfo: NodeInfo[]; 25 | }; 26 | 27 | declare const _exports: Language & { 28 | inline: Language 29 | }; 30 | export = _exports; 31 | -------------------------------------------------------------------------------- /bindings/node/index.js: -------------------------------------------------------------------------------- 1 | const root = require("path").join(__dirname, "..", ".."); 2 | 3 | module.exports = require("node-gyp-build")(root); 4 | 5 | try { 6 | module.exports.nodeTypeInfo = require("../../tree-sitter-markdown/src/node-types.json"); 7 | module.exports.inline.nodeTypeInfo = require("../../tree-sitter-markdown-inline/src/node-types.json"); 8 | } catch (_) { } 9 | -------------------------------------------------------------------------------- /bindings/node/inline.js: -------------------------------------------------------------------------------- 1 | module.exports = require(".").inline; 2 | -------------------------------------------------------------------------------- /bindings/python/tests/test_binding.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import tree_sitter, tree_sitter_markdown 4 | 5 | 6 | class TestLanguage(TestCase): 7 | def test_can_load_block_grammar(self): 8 | try: 9 | tree_sitter.Language(tree_sitter_markdown.language()) 10 | except Exception: 11 | self.fail("Error loading Markdown block grammar") 12 | 13 | def test_can_load_block_grammar(self): 14 | try: 15 | tree_sitter.Language(tree_sitter_markdown.inline_language()) 16 | except Exception: 17 | self.fail("Error loading Markdown inline grammar") 18 | -------------------------------------------------------------------------------- /bindings/python/tree_sitter_markdown/__init__.py: -------------------------------------------------------------------------------- 1 | "Markdown grammar for tree-sitter" 2 | 3 | from ._binding import language, inline_language 4 | 5 | __all__ = ["language", "inline_language"] 6 | -------------------------------------------------------------------------------- /bindings/python/tree_sitter_markdown/__init__.pyi: -------------------------------------------------------------------------------- 1 | def language() -> object: ... 2 | 3 | def inline_language() -> object: ... 4 | -------------------------------------------------------------------------------- /bindings/python/tree_sitter_markdown/binding.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | typedef struct TSLanguage TSLanguage; 4 | 5 | TSLanguage *tree_sitter_markdown(void); 6 | 7 | TSLanguage *tree_sitter_markdown_inline(void); 8 | 9 | static PyObject* _binding_language(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) { 10 | return PyCapsule_New(tree_sitter_markdown(), "tree_sitter.Language", NULL); 11 | } 12 | 13 | static PyObject* _binding_inline_language(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) { 14 | return PyCapsule_New(tree_sitter_markdown_inline(), "tree_sitter.Language", NULL); 15 | } 16 | 17 | static PyMethodDef methods[] = { 18 | {"language", _binding_language, METH_NOARGS, 19 | "Get the tree-sitter language for the block grammar."}, 20 | {"inline_language", _binding_inline_language, METH_NOARGS, 21 | "Get the tree-sitter language for the inline grammar."}, 22 | {NULL, NULL, 0, NULL} 23 | }; 24 | 25 | static struct PyModuleDef module = { 26 | .m_base = PyModuleDef_HEAD_INIT, 27 | .m_name = "_binding", 28 | .m_doc = NULL, 29 | .m_size = -1, 30 | .m_methods = methods 31 | }; 32 | 33 | PyMODINIT_FUNC PyInit__binding(void) { 34 | return PyModule_Create(&module); 35 | } 36 | -------------------------------------------------------------------------------- /bindings/python/tree_sitter_markdown/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tree-sitter-grammars/tree-sitter-markdown/efb075cbd57ce33f694c2bb264b99cdba0f31789/bindings/python/tree_sitter_markdown/py.typed -------------------------------------------------------------------------------- /bindings/rust/benchmark.rs: -------------------------------------------------------------------------------- 1 | use std::env::args; 2 | 3 | use tree_sitter::{InputEdit, Point}; 4 | use tree_sitter_md::{MarkdownParser, MarkdownTree}; 5 | 6 | fn main() { 7 | let filename = args() 8 | .nth_back(1) 9 | .take_if(|f| f != "benchmark") 10 | .unwrap_or("README.md".to_string()); 11 | let source = std::fs::read(filename).unwrap(); 12 | 13 | let mut parser = MarkdownParser::default(); 14 | let mut tree = parser.parse(&source, None).unwrap(); 15 | tree.edit(&InputEdit { 16 | start_byte: 0, 17 | old_end_byte: 1, 18 | new_end_byte: 0, 19 | start_position: Point::new(0, 0), 20 | old_end_position: Point::new(0, 1), 21 | new_end_position: Point::new(0, 0), 22 | }); 23 | reparse(&mut parser, &source[1..], tree); 24 | } 25 | 26 | fn reparse(parser: &mut MarkdownParser, source: &[u8], old_tree: MarkdownTree) { 27 | parser.parse(source, Some(&old_tree)).unwrap(); 28 | } 29 | -------------------------------------------------------------------------------- /bindings/rust/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let block_dir = std::path::Path::new("tree-sitter-markdown").join("src"); 3 | let inline_dir = std::path::Path::new("tree-sitter-markdown-inline").join("src"); 4 | 5 | let mut c_config = cc::Build::new(); 6 | c_config.std("c11").include(&block_dir); 7 | 8 | #[cfg(target_env = "msvc")] 9 | c_config.flag("-utf-8"); 10 | 11 | for path in &[ 12 | block_dir.join("parser.c"), 13 | block_dir.join("scanner.c"), 14 | inline_dir.join("parser.c"), 15 | inline_dir.join("scanner.c"), 16 | ] { 17 | c_config.file(path); 18 | println!("cargo:rerun-if-changed={}", path.to_str().unwrap()); 19 | } 20 | 21 | c_config.compile("tree-sitter-markdown"); 22 | } 23 | -------------------------------------------------------------------------------- /bindings/rust/lib.rs: -------------------------------------------------------------------------------- 1 | //! This crate provides Markdown language support for the [tree-sitter][] parsing library. 2 | //! 3 | //! It contains two grammars: [`LANGUAGE`] to parse the block structure of markdown documents and 4 | //! [`INLINE_LANGUAGE`] to parse inline content. 5 | //! 6 | //! It also supplies [`MarkdownParser`] as a convenience wrapper around the two grammars. 7 | //! [`MarkdownParser::parse`] returns a [`MarkdownTree`] instread of a [`Tree`][Tree]. This struct 8 | //! contains a block tree and an inline tree for each node in the block tree that has inline 9 | //! content. 10 | //! 11 | //! [LanguageFn]: https://docs.rs/tree-sitter-language/*/tree_sitter_language/struct.LanguageFn.html 12 | //! [Tree]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Tree.html 13 | //! [tree-sitter]: https://tree-sitter.github.io/ 14 | 15 | #![cfg_attr(docsrs, feature(doc_cfg))] 16 | 17 | use tree_sitter_language::LanguageFn; 18 | 19 | extern "C" { 20 | fn tree_sitter_markdown() -> *const (); 21 | fn tree_sitter_markdown_inline() -> *const (); 22 | } 23 | 24 | /// The tree-sitter [`LanguageFn`][LanguageFn] for the block grammar. 25 | pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_markdown) }; 26 | 27 | /// The tree-sitter [`LanguageFn`][LanguageFn] for the inline grammar. 28 | pub const INLINE_LANGUAGE: LanguageFn = 29 | unsafe { LanguageFn::from_raw(tree_sitter_markdown_inline) }; 30 | 31 | /// The syntax highlighting queries for the block grammar. 32 | pub const HIGHLIGHT_QUERY_BLOCK: &str = 33 | include_str!("../../tree-sitter-markdown/queries/highlights.scm"); 34 | 35 | /// The language injection queries for the block grammar. 36 | pub const INJECTION_QUERY_BLOCK: &str = 37 | include_str!("../../tree-sitter-markdown/queries/injections.scm"); 38 | 39 | /// The syntax highlighting queries for the inline grammar. 40 | pub const HIGHLIGHT_QUERY_INLINE: &str = 41 | include_str!("../../tree-sitter-markdown-inline/queries/highlights.scm"); 42 | 43 | /// The language injection queries for the inline grammar. 44 | pub const INJECTION_QUERY_INLINE: &str = 45 | include_str!("../../tree-sitter-markdown-inline/queries/injections.scm"); 46 | 47 | /// The content of the [`node-types.json`][] file for the block grammar. 48 | /// 49 | /// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types 50 | pub const NODE_TYPES_BLOCK: &str = include_str!("../../tree-sitter-markdown/src/node-types.json"); 51 | 52 | /// The content of the [`node-types.json`][] file for the inline grammar. 53 | /// 54 | /// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types 55 | pub const NODE_TYPES_INLINE: &str = 56 | include_str!("../../tree-sitter-markdown-inline/src/node-types.json"); 57 | 58 | #[cfg(feature = "parser")] 59 | #[cfg_attr(docsrs, doc(cfg(feature = "parser")))] 60 | mod parser; 61 | 62 | #[cfg(feature = "parser")] 63 | #[cfg_attr(docsrs, doc(cfg(feature = "parser")))] 64 | pub use parser::*; 65 | 66 | #[cfg(test)] 67 | mod tests { 68 | use super::*; 69 | 70 | #[test] 71 | fn can_load_block_grammar() { 72 | let mut parser = tree_sitter::Parser::new(); 73 | parser 74 | .set_language(&LANGUAGE.into()) 75 | .expect("Error loading Markdown block grammar"); 76 | } 77 | 78 | #[test] 79 | fn can_load_inline_grammar() { 80 | let mut parser = tree_sitter::Parser::new(); 81 | parser 82 | .set_language(&INLINE_LANGUAGE.into()) 83 | .expect("Error loading Markdown inline grammar"); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /bindings/rust/parser.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::num::NonZeroU16; 3 | 4 | use tree_sitter::{InputEdit, Language, Node, Parser, Point, Range, Tree, TreeCursor}; 5 | 6 | use crate::{INLINE_LANGUAGE, LANGUAGE}; 7 | 8 | /// A parser that produces [`MarkdownTree`]s. 9 | /// 10 | /// This is a convenience wrapper around [`LANGUAGE`] and [`INLINE_LANGUAGE`]. 11 | pub struct MarkdownParser { 12 | parser: Parser, 13 | block_language: Language, 14 | inline_language: Language, 15 | } 16 | 17 | /// A stateful object for walking a [`MarkdownTree`] efficiently. 18 | /// 19 | /// This exposes the same methdos as [`TreeCursor`], but abstracts away the 20 | /// double block / inline structure of [`MarkdownTree`]. 21 | pub struct MarkdownCursor<'a> { 22 | markdown_tree: &'a MarkdownTree, 23 | block_cursor: TreeCursor<'a>, 24 | inline_cursor: Option>, 25 | } 26 | 27 | impl<'a> MarkdownCursor<'a> { 28 | /// Get the cursor's current [`Node`]. 29 | pub fn node(&self) -> Node<'a> { 30 | match &self.inline_cursor { 31 | Some(cursor) => cursor.node(), 32 | None => self.block_cursor.node(), 33 | } 34 | } 35 | 36 | /// Returns `true` if the current node is from the (inline language)[INLINE_LANGUAGE] 37 | /// 38 | /// This information is needed to handle "tree-sitter internal" data like 39 | /// [`field_id`](Self::field_id) correctly. 40 | pub fn is_inline(&self) -> bool { 41 | self.inline_cursor.is_some() 42 | } 43 | 44 | /// Get the numerical field id of this tree cursor’s current node. 45 | /// 46 | /// You will need to call [`is_inline`](Self::is_inline) to find out if the 47 | /// current node is an inline or block node. 48 | /// 49 | /// See also [`field_name`](Self::field_name). 50 | pub fn field_id(&self) -> Option { 51 | match &self.inline_cursor { 52 | Some(cursor) => cursor.field_id(), 53 | None => self.block_cursor.field_id(), 54 | } 55 | } 56 | 57 | /// Get the field name of this tree cursor’s current node. 58 | /// 59 | /// You will need to call [`is_inline`](Self::is_inline) to find out if the 60 | /// current node is an inline or block node. 61 | pub fn field_name(&self) -> Option<&'static str> { 62 | match &self.inline_cursor { 63 | Some(cursor) => cursor.field_name(), 64 | None => self.block_cursor.field_name(), 65 | } 66 | } 67 | 68 | fn move_to_inline_tree(&mut self) -> bool { 69 | let node = self.block_cursor.node(); 70 | match node.kind() { 71 | "inline" | "pipe_table_cell" => { 72 | if let Some(inline_tree) = self.markdown_tree.inline_tree(&node) { 73 | self.inline_cursor = Some(inline_tree.walk()); 74 | return true; 75 | } 76 | } 77 | _ => (), 78 | } 79 | false 80 | } 81 | 82 | fn move_to_block_tree(&mut self) { 83 | self.inline_cursor = None; 84 | } 85 | 86 | /// Move this cursor to the first child of its current node. 87 | /// 88 | /// This returns `true` if the cursor successfully moved, and returns `false` if there were no 89 | /// children. 90 | /// If the cursor is currently at a node in the block tree and it has an associated inline tree, it 91 | /// will descend into the inline tree. 92 | pub fn goto_first_child(&mut self) -> bool { 93 | match &mut self.inline_cursor { 94 | Some(cursor) => cursor.goto_first_child(), 95 | None => { 96 | if self.move_to_inline_tree() { 97 | if !self.inline_cursor.as_mut().unwrap().goto_first_child() { 98 | self.move_to_block_tree(); 99 | false 100 | } else { 101 | true 102 | } 103 | } else { 104 | self.block_cursor.goto_first_child() 105 | } 106 | } 107 | } 108 | } 109 | 110 | /// Move this cursor to the parent of its current node. 111 | /// 112 | /// This returns true if the cursor successfully moved, and returns false if there was no 113 | /// parent node (the cursor was already on the root node). 114 | /// If the cursor moves to the root node of an inline tree, the it ascents to the associated 115 | /// node in the block tree. 116 | pub fn goto_parent(&mut self) -> bool { 117 | match &mut self.inline_cursor { 118 | Some(inline_cursor) => { 119 | inline_cursor.goto_parent(); 120 | if inline_cursor.node().parent().is_none() { 121 | self.move_to_block_tree(); 122 | } 123 | true 124 | } 125 | None => self.block_cursor.goto_parent(), 126 | } 127 | } 128 | 129 | /// Move this cursor to the next sibling of its current node. 130 | /// 131 | /// This returns true if the cursor successfully moved, and returns false if there was no next 132 | /// sibling node. 133 | pub fn goto_next_sibling(&mut self) -> bool { 134 | match &mut self.inline_cursor { 135 | Some(inline_cursor) => inline_cursor.goto_next_sibling(), 136 | None => self.block_cursor.goto_next_sibling(), 137 | } 138 | } 139 | 140 | /// Move this cursor to the first child of its current node that extends beyond the given byte offset. 141 | /// 142 | /// This returns the index of the child node if one was found, and returns None if no such child was found. 143 | /// If the cursor is currently at a node in the block tree and it has an associated inline tree, it 144 | /// will descend into the inline tree. 145 | pub fn goto_first_child_for_byte(&mut self, index: usize) -> Option { 146 | match &mut self.inline_cursor { 147 | Some(cursor) => cursor.goto_first_child_for_byte(index), 148 | None => { 149 | if self.move_to_inline_tree() { 150 | self.inline_cursor 151 | .as_mut() 152 | .unwrap() 153 | .goto_first_child_for_byte(index) 154 | } else { 155 | self.block_cursor.goto_first_child_for_byte(index) 156 | } 157 | } 158 | } 159 | } 160 | 161 | /// Move this cursor to the first child of its current node that extends beyond the given point. 162 | /// 163 | /// This returns the index of the child node if one was found, and returns None if no such child was found. 164 | /// If the cursor is currently at a node in the block tree and it has an associated inline tree, it 165 | /// will descend into the inline tree. 166 | pub fn goto_first_child_for_point(&mut self, index: Point) -> Option { 167 | match &mut self.inline_cursor { 168 | Some(cursor) => cursor.goto_first_child_for_point(index), 169 | None => { 170 | if self.move_to_inline_tree() { 171 | self.inline_cursor 172 | .as_mut() 173 | .unwrap() 174 | .goto_first_child_for_point(index) 175 | } else { 176 | self.block_cursor.goto_first_child_for_point(index) 177 | } 178 | } 179 | } 180 | } 181 | } 182 | 183 | /// An object that holds a combined markdown tree. 184 | #[derive(Debug, Clone)] 185 | pub struct MarkdownTree { 186 | block_tree: Tree, 187 | inline_trees: Vec, 188 | inline_indices: HashMap, 189 | } 190 | 191 | impl MarkdownTree { 192 | /// Edit the block tree and inline trees to keep them in sync with source code that has been 193 | /// edited. 194 | /// 195 | /// You must describe the edit both in terms of byte offsets and in terms of 196 | /// row/column coordinates. 197 | pub fn edit(&mut self, edit: &InputEdit) { 198 | self.block_tree.edit(edit); 199 | for inline_tree in self.inline_trees.iter_mut() { 200 | inline_tree.edit(edit); 201 | } 202 | } 203 | 204 | /// Returns the block tree for the parsed document 205 | pub fn block_tree(&self) -> &Tree { 206 | &self.block_tree 207 | } 208 | 209 | /// Returns the inline tree for the given inline node. 210 | /// 211 | /// Returns `None` if the given node does not have an associated inline tree. Either because 212 | /// the nodes type is not `inline` or because the inline content is empty. 213 | pub fn inline_tree(&self, parent: &Node) -> Option<&Tree> { 214 | let index = *self.inline_indices.get(&parent.id())?; 215 | Some(&self.inline_trees[index]) 216 | } 217 | 218 | /// Returns the list of all inline trees 219 | pub fn inline_trees(&self) -> &[Tree] { 220 | &self.inline_trees 221 | } 222 | 223 | /// Create a new [`MarkdownCursor`] starting from the root of the tree. 224 | pub fn walk(&self) -> MarkdownCursor { 225 | MarkdownCursor { 226 | markdown_tree: self, 227 | block_cursor: self.block_tree.walk(), 228 | inline_cursor: None, 229 | } 230 | } 231 | } 232 | 233 | impl Default for MarkdownParser { 234 | fn default() -> Self { 235 | let block_language = LANGUAGE.into(); 236 | let inline_language = INLINE_LANGUAGE.into(); 237 | let parser = Parser::new(); 238 | MarkdownParser { 239 | parser, 240 | block_language, 241 | inline_language, 242 | } 243 | } 244 | } 245 | 246 | impl MarkdownParser { 247 | /// Parse a slice of UTF8 text. 248 | /// 249 | /// # Arguments: 250 | /// * `text` The UTF8-encoded text to parse. 251 | /// * `old_tree` A previous syntax tree parsed from the same document. 252 | /// If the text of the document has changed since `old_tree` was 253 | /// created, then you must edit `old_tree` to match the new text using 254 | /// [MarkdownTree::edit]. 255 | /// 256 | /// Returns a [MarkdownTree] if parsing succeeded, or `None` if: 257 | /// * The timeout set with [tree_sitter::Parser::set_timeout_micros] expired 258 | /// * The cancellation flag set with [tree_sitter::Parser::set_cancellation_flag] was flipped 259 | pub fn parse_with, F: FnMut(usize, Point) -> T>( 260 | &mut self, 261 | callback: &mut F, 262 | old_tree: Option<&MarkdownTree>, 263 | ) -> Option { 264 | let MarkdownParser { 265 | parser, 266 | block_language, 267 | inline_language, 268 | } = self; 269 | parser 270 | .set_included_ranges(&[]) 271 | .expect("Can not set included ranges to whole document"); 272 | parser 273 | .set_language(block_language) 274 | .expect("Could not load block grammar"); 275 | let block_tree = parser.parse_with(callback, old_tree.map(|tree| &tree.block_tree))?; 276 | let (mut inline_trees, mut inline_indices) = if let Some(old_tree) = old_tree { 277 | let len = old_tree.inline_trees.len(); 278 | (Vec::with_capacity(len), HashMap::with_capacity(len)) 279 | } else { 280 | (Vec::new(), HashMap::new()) 281 | }; 282 | parser 283 | .set_language(inline_language) 284 | .expect("Could not load inline grammar"); 285 | let mut tree_cursor = block_tree.walk(); 286 | 287 | let mut i = 0; 288 | 'outer: loop { 289 | let node = loop { 290 | let kind = tree_cursor.node().kind(); 291 | if kind == "inline" || kind == "pipe_table_cell" || !tree_cursor.goto_first_child() 292 | { 293 | while !tree_cursor.goto_next_sibling() { 294 | if !tree_cursor.goto_parent() { 295 | break 'outer; 296 | } 297 | } 298 | } 299 | let kind = tree_cursor.node().kind(); 300 | if kind == "inline" || kind == "pipe_table_cell" { 301 | break tree_cursor.node(); 302 | } 303 | }; 304 | let mut range = node.range(); 305 | let mut ranges = Vec::new(); 306 | if tree_cursor.goto_first_child() { 307 | while tree_cursor.goto_next_sibling() { 308 | if !tree_cursor.node().is_named() { 309 | continue; 310 | } 311 | let child_range = tree_cursor.node().range(); 312 | ranges.push(Range { 313 | start_byte: range.start_byte, 314 | start_point: range.start_point, 315 | end_byte: child_range.start_byte, 316 | end_point: child_range.start_point, 317 | }); 318 | range.start_byte = child_range.end_byte; 319 | range.start_point = child_range.end_point; 320 | } 321 | tree_cursor.goto_parent(); 322 | } 323 | ranges.push(range); 324 | parser.set_included_ranges(&ranges).ok()?; 325 | let inline_tree = parser.parse_with( 326 | callback, 327 | old_tree.and_then(|old_tree| old_tree.inline_trees.get(i)), 328 | )?; 329 | inline_trees.push(inline_tree); 330 | inline_indices.insert(node.id(), i); 331 | i += 1; 332 | } 333 | drop(tree_cursor); 334 | inline_trees.shrink_to_fit(); 335 | inline_indices.shrink_to_fit(); 336 | Some(MarkdownTree { 337 | block_tree, 338 | inline_trees, 339 | inline_indices, 340 | }) 341 | } 342 | 343 | /// Parse a slice of UTF8 text. 344 | /// 345 | /// # Arguments: 346 | /// * `text` The UTF8-encoded text to parse. 347 | /// * `old_tree` A previous syntax tree parsed from the same document. 348 | /// If the text of the document has changed since `old_tree` was 349 | /// created, then you must edit `old_tree` to match the new text using 350 | /// [MarkdownTree::edit]. 351 | /// 352 | /// Returns a [MarkdownTree] if parsing succeeded, or `None` if: 353 | /// * The timeout set with [tree_sitter::Parser::set_timeout_micros] expired 354 | /// * The cancellation flag set with [tree_sitter::Parser::set_cancellation_flag] was flipped 355 | pub fn parse(&mut self, text: &[u8], old_tree: Option<&MarkdownTree>) -> Option { 356 | self.parse_with(&mut |byte, _| &text[byte..], old_tree) 357 | } 358 | } 359 | 360 | #[cfg(test)] 361 | mod tests { 362 | use tree_sitter::{InputEdit, Point}; 363 | 364 | use super::*; 365 | 366 | #[test] 367 | fn inline_ranges() { 368 | let code = "# title\n\nInline [content].\n"; 369 | let mut parser = MarkdownParser::default(); 370 | let mut tree = parser.parse(code.as_bytes(), None).unwrap(); 371 | 372 | let section = tree.block_tree().root_node().child(0).unwrap(); 373 | assert_eq!(section.kind(), "section"); 374 | let heading = section.child(0).unwrap(); 375 | assert_eq!(heading.kind(), "atx_heading"); 376 | let paragraph = section.child(1).unwrap(); 377 | assert_eq!(paragraph.kind(), "paragraph"); 378 | let inline = paragraph.child(0).unwrap(); 379 | assert_eq!(inline.kind(), "inline"); 380 | assert_eq!( 381 | tree.inline_tree(&inline) 382 | .unwrap() 383 | .root_node() 384 | .child(0) 385 | .unwrap() 386 | .kind(), 387 | "shortcut_link" 388 | ); 389 | 390 | let code = "# Title\n\nInline [content].\n"; 391 | tree.edit(&InputEdit { 392 | start_byte: 2, 393 | old_end_byte: 3, 394 | new_end_byte: 3, 395 | start_position: Point { row: 0, column: 2 }, 396 | old_end_position: Point { row: 0, column: 3 }, 397 | new_end_position: Point { row: 0, column: 3 }, 398 | }); 399 | let tree = parser.parse(code.as_bytes(), Some(&tree)).unwrap(); 400 | 401 | let section = tree.block_tree().root_node().child(0).unwrap(); 402 | assert_eq!(section.kind(), "section"); 403 | let heading = section.child(0).unwrap(); 404 | assert_eq!(heading.kind(), "atx_heading"); 405 | let paragraph = section.child(1).unwrap(); 406 | assert_eq!(paragraph.kind(), "paragraph"); 407 | let inline = paragraph.child(0).unwrap(); 408 | assert_eq!(inline.kind(), "inline"); 409 | assert_eq!( 410 | tree.inline_tree(&inline) 411 | .unwrap() 412 | .root_node() 413 | .named_child(0) 414 | .unwrap() 415 | .kind(), 416 | "shortcut_link" 417 | ); 418 | } 419 | 420 | #[test] 421 | fn markdown_cursor() { 422 | let code = "# title\n\nInline [content].\n"; 423 | let mut parser = MarkdownParser::default(); 424 | let tree = parser.parse(code.as_bytes(), None).unwrap(); 425 | let mut cursor = tree.walk(); 426 | assert_eq!(cursor.node().kind(), "document"); 427 | assert!(cursor.goto_first_child()); 428 | assert_eq!(cursor.node().kind(), "section"); 429 | assert!(cursor.goto_first_child()); 430 | assert_eq!(cursor.node().kind(), "atx_heading"); 431 | assert!(cursor.goto_next_sibling()); 432 | assert_eq!(cursor.node().kind(), "paragraph"); 433 | assert!(cursor.goto_first_child()); 434 | assert_eq!(cursor.node().kind(), "inline"); 435 | assert!(cursor.goto_first_child()); 436 | assert_eq!(cursor.node().kind(), "shortcut_link"); 437 | assert!(cursor.goto_parent()); 438 | assert!(cursor.goto_parent()); 439 | assert!(cursor.goto_parent()); 440 | assert!(cursor.goto_parent()); 441 | assert_eq!(cursor.node().kind(), "document"); 442 | } 443 | 444 | #[test] 445 | fn table() { 446 | let code = "| foo |\n| --- |\n| *bar*|\n"; 447 | let mut parser = MarkdownParser::default(); 448 | let tree = parser.parse(code.as_bytes(), None).unwrap(); 449 | dbg!(&tree.inline_trees()); 450 | let mut cursor = tree.walk(); 451 | 452 | assert_eq!(cursor.node().kind(), "document"); 453 | assert!(cursor.goto_first_child()); 454 | assert_eq!(cursor.node().kind(), "section"); 455 | assert!(cursor.goto_first_child()); 456 | assert_eq!(cursor.node().kind(), "pipe_table"); 457 | assert!(cursor.goto_first_child()); 458 | assert!(cursor.goto_next_sibling()); 459 | assert!(cursor.goto_next_sibling()); 460 | assert_eq!(cursor.node().kind(), "pipe_table_row"); 461 | assert!(cursor.goto_first_child()); 462 | assert!(cursor.goto_next_sibling()); 463 | assert_eq!(cursor.node().kind(), "pipe_table_cell"); 464 | assert!(cursor.goto_first_child()); 465 | assert_eq!(cursor.node().kind(), "emphasis"); 466 | } 467 | } 468 | -------------------------------------------------------------------------------- /bindings/swift/.gitignore: -------------------------------------------------------------------------------- 1 | TreeSitterMarkdown 2 | -------------------------------------------------------------------------------- /bindings/swift/TreeSitterMarkdownTests/TreeSitterMarkdownTests.swift: -------------------------------------------------------------------------------- 1 | import XCTest 2 | import SwiftTreeSitter 3 | import TreeSitterMarkdown 4 | import TreeSitterMarkdownInline 5 | 6 | final class TreeSitterXMLTests: XCTestCase { 7 | func testCanLoadBlockGrammar() throws { 8 | let parser = Parser() 9 | let language = Language(language: tree_sitter_markdown()) 10 | XCTAssertNoThrow(try parser.setLanguage(language), 11 | "Error loading Markdown block grammar") 12 | } 13 | 14 | func testCanLoadInlineGrammar() throws { 15 | let parser = Parser() 16 | let language = Language(language: tree_sitter_markdown_inline()) 17 | XCTAssertNoThrow(try parser.setLanguage(language), 18 | "Error loading Markdown inline grammar") 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /common/common.js: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | module.exports.EXTENSION_DEFAULT = !process.env.NO_DEFAULT_EXTENSIONS; 4 | module.exports.EXTENSION_GFM = process.env.EXTENSION_GFM || module.exports.EXTENSION_DEFAULT || process.env.ALL_EXTENSIONS; 5 | module.exports.EXTENSION_TASK_LIST = process.env.EXTENSION_TASK_LIST || module.exports.EXTENSION_GFM || process.env.ALL_EXTENSIONS; 6 | module.exports.EXTENSION_STRIKETHROUGH = process.env.EXTENSION_STRIKETHROUGH || module.exports.EXTENSION_GFM || process.env.ALL_EXTENSIONS; 7 | module.exports.EXTENSION_PIPE_TABLE = process.env.EXTENSION_PIPE_TABLE || module.exports.EXTENSION_GFM || process.env.ALL_EXTENSIONS; 8 | module.exports.EXTENSION_MINUS_METADATA = process.env.EXTENSION_MINUS_METADATA || module.exports.EXTENSION_DEFAULT || process.env.ALL_EXTENSIONS; 9 | module.exports.EXTENSION_PLUS_METADATA = process.env.EXTENSION_PLUS_METADATA || module.exports.EXTENSION_DEFAULT || process.env.ALL_EXTENSIONS; 10 | module.exports.EXTENSION_TAGS = process.env.EXTENSION_TAGS || process.env.ALL_EXTENSIONS; 11 | module.exports.EXTENSION_LATEX = process.env.EXTENSION_LATEX || module.exports.EXTENSION_DEFAULT || process.env.ALL_EXTENSIONS; 12 | module.exports.EXTENSION_WIKI_LINK = process.env.EXTENSION_WIKI_LINK || process.env.ALL_EXTENSIONS; 13 | 14 | const PUNCTUATION_CHARACTERS_REGEX = '!-/:-@\\[-`\\{-~'; 15 | const PUNCTUATION_CHARACTERS_ARRAY = [ 16 | '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', ':', ';', '<', 17 | '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', '{', '|', '}', '~' 18 | ]; 19 | 20 | const PRECEDENCE_LEVEL_LINK = 10; 21 | 22 | module.exports.PRECEDENCE_LEVEL_LINK = PRECEDENCE_LEVEL_LINK; 23 | 24 | module.exports.PUNCTUATION_CHARACTERS_REGEX = PUNCTUATION_CHARACTERS_REGEX; 25 | 26 | /** @type {Record) => RuleOrLiteral>} */ 27 | module.exports.rules = { 28 | 29 | // A backslash escape. This can often be part of different nodes like link labels 30 | // 31 | // https://github.github.com/gfm/#backslash-escapes 32 | backslash_escape: $ => $._backslash_escape, 33 | _backslash_escape: $ => new RegExp('\\\\[' + PUNCTUATION_CHARACTERS_REGEX + ']'), 34 | 35 | // HTML entity and numeric character references. 36 | // 37 | // The regex for entity references are build from the html_entities.json file. 38 | // 39 | // https://github.github.com/gfm/#entity-and-numeric-character-references 40 | entity_reference: $ => html_entity_regex(), 41 | numeric_character_reference: $ => /&#([0-9]{1,7}|[xX][0-9a-fA-F]{1,6});/, 42 | 43 | link_label: $ => seq('[', repeat1(choice( 44 | $._text_inline_no_link, 45 | $.backslash_escape, 46 | $.entity_reference, 47 | $.numeric_character_reference, 48 | $._soft_line_break 49 | )), ']'), 50 | 51 | link_destination: $ => prec.dynamic(PRECEDENCE_LEVEL_LINK, choice( 52 | seq('<', repeat(choice($._text_no_angle, $.backslash_escape, $.entity_reference, $.numeric_character_reference)), '>'), 53 | seq( 54 | choice( // first character is not a '<' 55 | $._word, 56 | punctuation_without($, ['<', '(', ')']), 57 | $.backslash_escape, 58 | $.entity_reference, 59 | $.numeric_character_reference, 60 | $._link_destination_parenthesis 61 | ), 62 | repeat(choice( 63 | $._word, 64 | punctuation_without($, ['(', ')']), 65 | $.backslash_escape, 66 | $.entity_reference, 67 | $.numeric_character_reference, 68 | $._link_destination_parenthesis 69 | )), 70 | ) 71 | )), 72 | _link_destination_parenthesis: $ => seq('(', repeat(choice( 73 | $._word, 74 | punctuation_without($, ['(', ')']), 75 | $.backslash_escape, 76 | $.entity_reference, 77 | $.numeric_character_reference, 78 | $._link_destination_parenthesis 79 | )), ')'), 80 | _text_no_angle: $ => choice($._word, punctuation_without($, ['<', '>']), $._whitespace), 81 | link_title: $ => choice( 82 | seq('"', repeat(choice( 83 | $._word, 84 | punctuation_without($, ['"']), 85 | $._whitespace, 86 | $.backslash_escape, 87 | $.entity_reference, 88 | $.numeric_character_reference, 89 | seq($._soft_line_break, optional(seq($._soft_line_break, $._trigger_error))) 90 | )), '"'), 91 | seq("'", repeat(choice( 92 | $._word, 93 | punctuation_without($, ["'"]), 94 | $._whitespace, 95 | $.backslash_escape, 96 | $.entity_reference, 97 | $.numeric_character_reference, 98 | seq($._soft_line_break, optional(seq($._soft_line_break, $._trigger_error))) 99 | )), "'"), 100 | seq('(', repeat(choice( 101 | $._word, 102 | punctuation_without($, ['(', ')']), 103 | $._whitespace, 104 | $.backslash_escape, 105 | $.entity_reference, 106 | $.numeric_character_reference, 107 | seq($._soft_line_break, optional(seq($._soft_line_break, $._trigger_error))) 108 | )), ')'), 109 | ), 110 | 111 | _newline_token: $ => /\n|\r\n?/, 112 | }; 113 | 114 | // Returns a rule that matches all characters that count as punctuation inside markdown, besides 115 | // a list of excluded punctuation characters. Calling this function with a empty list as the second 116 | // argument returns a rule that matches all punctuation. 117 | function punctuation_without($, chars) { 118 | return seq(choice(...PUNCTUATION_CHARACTERS_ARRAY.filter(c => !chars.includes(c))), optional($._last_token_punctuation)); 119 | } 120 | 121 | module.exports.punctuation_without = punctuation_without; 122 | 123 | // Constructs a regex that matches all html entity references. 124 | function html_entity_regex() { 125 | // A file with all html entities, should be kept up to date with 126 | // https://html.spec.whatwg.org/multipage/entities.json 127 | let html_entities = require("./html_entities.json"); 128 | let s = '&('; 129 | s += Object.keys(html_entities).map(name => name.substring(1, name.length - 1)).join('|'); 130 | s += ');'; 131 | return new RegExp(s); 132 | } 133 | -------------------------------------------------------------------------------- /common/common.mak: -------------------------------------------------------------------------------- 1 | ifeq ($(OS),Windows_NT) 2 | $(error Windows is not supported) 3 | endif 4 | 5 | HOMEPAGE_URL := https://github.com/tree-sitter-grammars/tree-sitter-markdown 6 | VERSION := 0.5.0 7 | 8 | # repository 9 | SRC_DIR := src 10 | 11 | TS ?= tree-sitter 12 | 13 | # install directory layout 14 | PREFIX ?= /usr/local 15 | INCLUDEDIR ?= $(PREFIX)/include 16 | LIBDIR ?= $(PREFIX)/lib 17 | PCLIBDIR ?= $(LIBDIR)/pkgconfig 18 | 19 | # source/object files 20 | PARSER := $(SRC_DIR)/parser.c 21 | EXTRAS := $(filter-out $(PARSER),$(wildcard $(SRC_DIR)/*.c)) 22 | OBJS := $(patsubst %.c,%.o,$(PARSER) $(EXTRAS)) 23 | 24 | # flags 25 | ARFLAGS ?= rcs 26 | override CFLAGS += -I$(SRC_DIR) -std=c11 -fPIC 27 | 28 | # ABI versioning 29 | SONAME_MAJOR = $(shell sed -n 's/\#define LANGUAGE_VERSION //p' $(PARSER)) 30 | SONAME_MINOR = $(word 1,$(subst ., ,$(VERSION))) 31 | 32 | # OS-specific bits 33 | ifeq ($(shell uname),Darwin) 34 | SOEXT = dylib 35 | SOEXTVER_MAJOR = $(SONAME_MAJOR).$(SOEXT) 36 | SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).$(SOEXT) 37 | LINKSHARED = -dynamiclib -Wl,-install_name,$(LIBDIR)/lib$(LANGUAGE_NAME).$(SOEXTVER),-rpath,@executable_path/../Frameworks 38 | else 39 | SOEXT = so 40 | SOEXTVER_MAJOR = $(SOEXT).$(SONAME_MAJOR) 41 | SOEXTVER = $(SOEXT).$(SONAME_MAJOR).$(SONAME_MINOR) 42 | LINKSHARED = -shared -Wl,-soname,lib$(LANGUAGE_NAME).$(SOEXTVER) 43 | endif 44 | ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),) 45 | PCLIBDIR := $(PREFIX)/libdata/pkgconfig 46 | endif 47 | 48 | 49 | all: lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) $(LANGUAGE_NAME).pc 50 | 51 | lib$(LANGUAGE_NAME).a: $(OBJS) 52 | $(AR) $(ARFLAGS) $@ $^ 53 | 54 | lib$(LANGUAGE_NAME).$(SOEXT): $(OBJS) 55 | $(CC) $(LDFLAGS) $(LINKSHARED) $^ $(LDLIBS) -o $@ 56 | ifneq ($(STRIP),) 57 | $(STRIP) $@ 58 | endif 59 | 60 | $(LANGUAGE_NAME).pc: bindings/c/$(LANGUAGE_NAME).pc.in 61 | sed -e 's|@CMAKE_PROJECT_VERSION@|$(VERSION)|' \ 62 | -e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \ 63 | -e 's|@CMAKE_INSTALL_INCLUDEDIR@|$(INCLUDEDIR:$(PREFIX)/%=%)|' \ 64 | -e 's|@PROJECT_DESCRIPTION@|$(DESCRIPTION)|' \ 65 | -e 's|@CMAKE_PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \ 66 | -e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' \ 67 | -e 's|@TS_REQUIRES@|$(REQUIRES)|' $< > $@ 68 | 69 | $(PARSER): $(SRC_DIR)/grammar.js 70 | $(TS) generate $^ 71 | 72 | install: all 73 | install -Dm644 bindings/c/$(LANGUAGE_NAME).h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h 74 | install -Dm644 $(LANGUAGE_NAME).pc '$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc 75 | install -Dm755 lib$(LANGUAGE_NAME).a '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a 76 | install -m755 lib$(LANGUAGE_NAME).$(SOEXT) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER) 77 | ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) 78 | ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT) 79 | 80 | uninstall: 81 | $(RM) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a \ 82 | '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER) \ 83 | '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) \ 84 | '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT) \ 85 | '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h \ 86 | '$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc 87 | 88 | clean: 89 | $(RM) $(OBJS) $(LANGUAGE_NAME).pc lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) 90 | 91 | test: 92 | $(TS) test 93 | 94 | .PHONY: all install uninstall clean test 95 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tree-sitter-grammars/tree-sitter-markdown 2 | 3 | go 1.23 4 | 5 | toolchain go1.23.0 6 | 7 | require github.com/tree-sitter/go-tree-sitter v0.23.1 8 | 9 | require github.com/mattn/go-pointer v0.0.1 // indirect 10 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@tree-sitter-grammars/tree-sitter-markdown", 3 | "version": "0.5.0", 4 | "lockfileVersion": 3, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "@tree-sitter-grammars/tree-sitter-markdown", 9 | "version": "0.5.0", 10 | "hasInstallScript": true, 11 | "license": "MIT", 12 | "dependencies": { 13 | "node-addon-api": "^8.2.1", 14 | "node-gyp-build": "^4.8.2" 15 | }, 16 | "devDependencies": { 17 | "prebuildify": "^6.0.1", 18 | "tree-sitter-cli": "^0.24.3" 19 | }, 20 | "peerDependencies": { 21 | "tree-sitter": "^0.21.1" 22 | }, 23 | "peerDependenciesMeta": { 24 | "tree_sitter": { 25 | "optional": true 26 | } 27 | } 28 | }, 29 | "node_modules/base64-js": { 30 | "version": "1.5.1", 31 | "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", 32 | "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", 33 | "dev": true, 34 | "funding": [ 35 | { 36 | "type": "github", 37 | "url": "https://github.com/sponsors/feross" 38 | }, 39 | { 40 | "type": "patreon", 41 | "url": "https://www.patreon.com/feross" 42 | }, 43 | { 44 | "type": "consulting", 45 | "url": "https://feross.org/support" 46 | } 47 | ] 48 | }, 49 | "node_modules/bl": { 50 | "version": "4.1.0", 51 | "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", 52 | "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", 53 | "dev": true, 54 | "dependencies": { 55 | "buffer": "^5.5.0", 56 | "inherits": "^2.0.4", 57 | "readable-stream": "^3.4.0" 58 | } 59 | }, 60 | "node_modules/buffer": { 61 | "version": "5.7.1", 62 | "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", 63 | "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", 64 | "dev": true, 65 | "funding": [ 66 | { 67 | "type": "github", 68 | "url": "https://github.com/sponsors/feross" 69 | }, 70 | { 71 | "type": "patreon", 72 | "url": "https://www.patreon.com/feross" 73 | }, 74 | { 75 | "type": "consulting", 76 | "url": "https://feross.org/support" 77 | } 78 | ], 79 | "dependencies": { 80 | "base64-js": "^1.3.1", 81 | "ieee754": "^1.1.13" 82 | } 83 | }, 84 | "node_modules/chownr": { 85 | "version": "1.1.4", 86 | "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", 87 | "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", 88 | "dev": true 89 | }, 90 | "node_modules/end-of-stream": { 91 | "version": "1.4.4", 92 | "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", 93 | "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", 94 | "dev": true, 95 | "dependencies": { 96 | "once": "^1.4.0" 97 | } 98 | }, 99 | "node_modules/fs-constants": { 100 | "version": "1.0.0", 101 | "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", 102 | "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", 103 | "dev": true 104 | }, 105 | "node_modules/ieee754": { 106 | "version": "1.2.1", 107 | "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", 108 | "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", 109 | "dev": true, 110 | "funding": [ 111 | { 112 | "type": "github", 113 | "url": "https://github.com/sponsors/feross" 114 | }, 115 | { 116 | "type": "patreon", 117 | "url": "https://www.patreon.com/feross" 118 | }, 119 | { 120 | "type": "consulting", 121 | "url": "https://feross.org/support" 122 | } 123 | ] 124 | }, 125 | "node_modules/inherits": { 126 | "version": "2.0.4", 127 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", 128 | "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", 129 | "dev": true 130 | }, 131 | "node_modules/lru-cache": { 132 | "version": "6.0.0", 133 | "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", 134 | "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", 135 | "dev": true, 136 | "dependencies": { 137 | "yallist": "^4.0.0" 138 | }, 139 | "engines": { 140 | "node": ">=10" 141 | } 142 | }, 143 | "node_modules/minimist": { 144 | "version": "1.2.8", 145 | "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", 146 | "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", 147 | "dev": true, 148 | "funding": { 149 | "url": "https://github.com/sponsors/ljharb" 150 | } 151 | }, 152 | "node_modules/mkdirp-classic": { 153 | "version": "0.5.3", 154 | "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", 155 | "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", 156 | "dev": true 157 | }, 158 | "node_modules/node-abi": { 159 | "version": "3.56.0", 160 | "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.56.0.tgz", 161 | "integrity": "sha512-fZjdhDOeRcaS+rcpve7XuwHBmktS1nS1gzgghwKUQQ8nTy2FdSDr6ZT8k6YhvlJeHmmQMYiT/IH9hfco5zeW2Q==", 162 | "dev": true, 163 | "dependencies": { 164 | "semver": "^7.3.5" 165 | }, 166 | "engines": { 167 | "node": ">=10" 168 | } 169 | }, 170 | "node_modules/node-addon-api": { 171 | "version": "8.2.1", 172 | "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.2.1.tgz", 173 | "integrity": "sha512-vmEOvxwiH8tlOcv4SyE8RH34rI5/nWVaigUeAUPawC6f0+HoDthwI0vkMu4tbtsZrXq6QXFfrkhjofzKEs5tpA==", 174 | "license": "MIT", 175 | "engines": { 176 | "node": "^18 || ^20 || >= 21" 177 | } 178 | }, 179 | "node_modules/node-gyp-build": { 180 | "version": "4.8.2", 181 | "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.2.tgz", 182 | "integrity": "sha512-IRUxE4BVsHWXkV/SFOut4qTlagw2aM8T5/vnTsmrHJvVoKueJHRc/JaFND7QDDc61kLYUJ6qlZM3sqTSyx2dTw==", 183 | "bin": { 184 | "node-gyp-build": "bin.js", 185 | "node-gyp-build-optional": "optional.js", 186 | "node-gyp-build-test": "build-test.js" 187 | } 188 | }, 189 | "node_modules/npm-run-path": { 190 | "version": "3.1.0", 191 | "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-3.1.0.tgz", 192 | "integrity": "sha512-Dbl4A/VfiVGLgQv29URL9xshU8XDY1GeLy+fsaZ1AA8JDSfjvr5P5+pzRbWqRSBxk6/DW7MIh8lTM/PaGnP2kg==", 193 | "dev": true, 194 | "dependencies": { 195 | "path-key": "^3.0.0" 196 | }, 197 | "engines": { 198 | "node": ">=8" 199 | } 200 | }, 201 | "node_modules/once": { 202 | "version": "1.4.0", 203 | "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", 204 | "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", 205 | "dev": true, 206 | "dependencies": { 207 | "wrappy": "1" 208 | } 209 | }, 210 | "node_modules/path-key": { 211 | "version": "3.1.1", 212 | "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", 213 | "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", 214 | "dev": true, 215 | "engines": { 216 | "node": ">=8" 217 | } 218 | }, 219 | "node_modules/prebuildify": { 220 | "version": "6.0.1", 221 | "resolved": "https://registry.npmjs.org/prebuildify/-/prebuildify-6.0.1.tgz", 222 | "integrity": "sha512-8Y2oOOateom/s8dNBsGIcnm6AxPmLH4/nanQzL5lQMU+sC0CMhzARZHizwr36pUPLdvBnOkCNQzxg4djuFSgIw==", 223 | "dev": true, 224 | "dependencies": { 225 | "minimist": "^1.2.5", 226 | "mkdirp-classic": "^0.5.3", 227 | "node-abi": "^3.3.0", 228 | "npm-run-path": "^3.1.0", 229 | "pump": "^3.0.0", 230 | "tar-fs": "^2.1.0" 231 | }, 232 | "bin": { 233 | "prebuildify": "bin.js" 234 | } 235 | }, 236 | "node_modules/pump": { 237 | "version": "3.0.0", 238 | "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", 239 | "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", 240 | "dev": true, 241 | "dependencies": { 242 | "end-of-stream": "^1.1.0", 243 | "once": "^1.3.1" 244 | } 245 | }, 246 | "node_modules/readable-stream": { 247 | "version": "3.6.2", 248 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", 249 | "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", 250 | "dev": true, 251 | "dependencies": { 252 | "inherits": "^2.0.3", 253 | "string_decoder": "^1.1.1", 254 | "util-deprecate": "^1.0.1" 255 | }, 256 | "engines": { 257 | "node": ">= 6" 258 | } 259 | }, 260 | "node_modules/safe-buffer": { 261 | "version": "5.2.1", 262 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", 263 | "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", 264 | "dev": true, 265 | "funding": [ 266 | { 267 | "type": "github", 268 | "url": "https://github.com/sponsors/feross" 269 | }, 270 | { 271 | "type": "patreon", 272 | "url": "https://www.patreon.com/feross" 273 | }, 274 | { 275 | "type": "consulting", 276 | "url": "https://feross.org/support" 277 | } 278 | ] 279 | }, 280 | "node_modules/semver": { 281 | "version": "7.6.0", 282 | "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz", 283 | "integrity": "sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==", 284 | "dev": true, 285 | "dependencies": { 286 | "lru-cache": "^6.0.0" 287 | }, 288 | "bin": { 289 | "semver": "bin/semver.js" 290 | }, 291 | "engines": { 292 | "node": ">=10" 293 | } 294 | }, 295 | "node_modules/string_decoder": { 296 | "version": "1.3.0", 297 | "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", 298 | "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", 299 | "dev": true, 300 | "dependencies": { 301 | "safe-buffer": "~5.2.0" 302 | } 303 | }, 304 | "node_modules/tar-fs": { 305 | "version": "2.1.1", 306 | "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", 307 | "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", 308 | "dev": true, 309 | "dependencies": { 310 | "chownr": "^1.1.1", 311 | "mkdirp-classic": "^0.5.2", 312 | "pump": "^3.0.0", 313 | "tar-stream": "^2.1.4" 314 | } 315 | }, 316 | "node_modules/tar-stream": { 317 | "version": "2.2.0", 318 | "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", 319 | "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", 320 | "dev": true, 321 | "dependencies": { 322 | "bl": "^4.0.3", 323 | "end-of-stream": "^1.4.1", 324 | "fs-constants": "^1.0.0", 325 | "inherits": "^2.0.3", 326 | "readable-stream": "^3.1.1" 327 | }, 328 | "engines": { 329 | "node": ">=6" 330 | } 331 | }, 332 | "node_modules/tree-sitter": { 333 | "version": "0.21.1", 334 | "resolved": "https://registry.npmjs.org/tree-sitter/-/tree-sitter-0.21.1.tgz", 335 | "integrity": "sha512-7dxoA6kYvtgWw80265MyqJlkRl4yawIjO7S5MigytjELkX43fV2WsAXzsNfO7sBpPPCF5Gp0+XzHk0DwLCq3xQ==", 336 | "hasInstallScript": true, 337 | "peer": true, 338 | "dependencies": { 339 | "node-addon-api": "^8.0.0", 340 | "node-gyp-build": "^4.8.0" 341 | } 342 | }, 343 | "node_modules/tree-sitter-cli": { 344 | "version": "0.24.3", 345 | "resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.24.3.tgz", 346 | "integrity": "sha512-5vS0SiJf31tMTn9CYLsu5l18qXaw5MLFka3cuGxOB5f4TtgoUSK1Sog6rKmqBc7PvFJq37YcQBjj9giNy2cJPw==", 347 | "dev": true, 348 | "hasInstallScript": true, 349 | "license": "MIT", 350 | "bin": { 351 | "tree-sitter": "cli.js" 352 | }, 353 | "engines": { 354 | "node": ">=12.0.0" 355 | } 356 | }, 357 | "node_modules/util-deprecate": { 358 | "version": "1.0.2", 359 | "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", 360 | "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", 361 | "dev": true 362 | }, 363 | "node_modules/wrappy": { 364 | "version": "1.0.2", 365 | "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", 366 | "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", 367 | "dev": true 368 | }, 369 | "node_modules/yallist": { 370 | "version": "4.0.0", 371 | "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", 372 | "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", 373 | "dev": true 374 | } 375 | } 376 | } 377 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@tree-sitter-grammars/tree-sitter-markdown", 3 | "version": "0.5.0", 4 | "description": "Markdown grammar for tree-sitter", 5 | "repository": "github:tree-sitter-grammars/tree-sitter-markdown", 6 | "license": "MIT", 7 | "main": "bindings/node", 8 | "types": "bindings/node", 9 | "author": { 10 | "name": "MDeiml" 11 | }, 12 | "keywords": [ 13 | "incremental", 14 | "parsing", 15 | "tree-sitter", 16 | "markdown" 17 | ], 18 | "files": [ 19 | "binding.gyp", 20 | "prebuilds/**", 21 | "bindings/node/*", 22 | "tree-sitter.json", 23 | "tree-sitter-*/grammar.js", 24 | "tree-sitter-*/queries/*", 25 | "tree-sitter-*/src/**", 26 | "common/common.js", 27 | "common/html_entities.json" 28 | ], 29 | "dependencies": { 30 | "node-addon-api": "^8.2.1", 31 | "node-gyp-build": "^4.8.2" 32 | }, 33 | "devDependencies": { 34 | "prebuildify": "^6.0.1", 35 | "tree-sitter-cli": "^0.24.3" 36 | }, 37 | "peerDependencies": { 38 | "tree-sitter": "^0.21.1" 39 | }, 40 | "peerDependenciesMeta": { 41 | "tree_sitter": { 42 | "optional": true 43 | } 44 | }, 45 | "scripts": { 46 | "build": "node scripts/build.js", 47 | "install": "node-gyp-build", 48 | "prestart": "tree-sitter build --wasm", 49 | "start": "tree-sitter playground", 50 | "test": "node --test bindings/node/*_test.js" 51 | }, 52 | "publishConfig": { 53 | "access": "public" 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=42", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "tree-sitter-markdown" 7 | description = "Markdown grammar for tree-sitter" 8 | version = "0.5.0" 9 | keywords = ["incremental", "parsing", "tree-sitter", "markdown"] 10 | classifiers = [ 11 | "Intended Audience :: Developers", 12 | "License :: OSI Approved :: MIT License", 13 | "Topic :: Software Development :: Compilers", 14 | "Topic :: Text Processing :: Linguistic", 15 | "Typing :: Typed" 16 | ] 17 | authors = [ 18 | {name = "MDeiml"} 19 | ] 20 | requires-python = ">=3.9" 21 | license.text = "MIT" 22 | readme = "README.md" 23 | 24 | [project.urls] 25 | Homepage = "https://github.com/tree-sitter-grammars/tree-sitter-markdown" 26 | 27 | [project.optional-dependencies] 28 | core = ["tree-sitter~=0.23"] 29 | 30 | [tool.cibuildwheel] 31 | build = "cp39-*" 32 | build-frontend = "build" 33 | -------------------------------------------------------------------------------- /scripts/build.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const { execSync } = require("child_process"); 4 | const { join } = require("path"); 5 | 6 | for (const dir of ["tree-sitter-markdown", "tree-sitter-markdown-inline"]) { 7 | console.log(`building ${dir}`); 8 | execSync("tree-sitter generate", { 9 | stdio: "inherit", 10 | cwd: join(__dirname, "..", dir) 11 | }); 12 | } 13 | -------------------------------------------------------------------------------- /scripts/test.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const { execSync } = require("child_process"); 4 | const { join } = require("path"); 5 | 6 | const parsers = ["tree-sitter-markdown", "tree-sitter-markdown-inline"]; 7 | 8 | for (const dir of parsers) { 9 | console.log(`testing ${dir}`); 10 | try { 11 | execSync("tree-sitter test", { 12 | stdio: "inherit", 13 | cwd: join(__dirname, "..", dir) 14 | }); 15 | } catch(error) { 16 | process.exitCode |= parsers.indexOf(dir) + 1; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from platform import system 3 | 4 | from setuptools import Extension, find_packages, setup 5 | from setuptools.command.build import build 6 | from wheel.bdist_wheel import bdist_wheel 7 | 8 | 9 | class Build(build): 10 | def run(self): 11 | if (block_queries := Path("tree-sitter-markdown", "queries")).is_dir(): 12 | dest = Path(self.build_lib, "tree_sitter_markdown", "queries", "markdown") 13 | self.copy_tree(str(block_queries), str(dest)) 14 | if (inline_queries := Path("tree-sitter-markdown-inline", "queries")).is_dir(): 15 | dest = Path(self.build_lib, "tree_sitter_markdown", "queries", "markdown_inline") 16 | self.copy_tree(str(inline_queries), str(dest)) 17 | super().run() 18 | 19 | 20 | class BdistWheel(bdist_wheel): 21 | def get_tag(self): 22 | python, abi, platform = super().get_tag() 23 | if python.startswith("cp"): 24 | python, abi = "cp39", "abi3" 25 | return python, abi, platform 26 | 27 | 28 | setup( 29 | packages=find_packages("bindings/python"), 30 | package_dir={"": "bindings/python"}, 31 | package_data={ 32 | "tree_sitter_markdown": ["*.pyi", "py.typed"], 33 | "tree_sitter_markdown.queries": ["*.scm"], 34 | }, 35 | ext_package="tree_sitter_markdown", 36 | ext_modules=[ 37 | Extension( 38 | name="_binding", 39 | sources=[ 40 | "bindings/python/tree_sitter_markdown/binding.c", 41 | "tree-sitter-markdown/src/parser.c", 42 | "tree-sitter-markdown/src/scanner.c", 43 | "tree-sitter-markdown-inline/src/parser.c", 44 | "tree-sitter-markdown-inline/src/scanner.c", 45 | ], 46 | extra_compile_args=( 47 | ["-std=c11"] if system() != "Windows" else [] 48 | ), 49 | define_macros=[ 50 | ("Py_LIMITED_API", "0x03090000"), 51 | ("PY_SSIZE_T_CLEAN", None) 52 | ], 53 | include_dirs=["tree-sitter-markdown/src"], 54 | py_limited_api=True, 55 | ) 56 | ], 57 | cmdclass={ 58 | "build": Build, 59 | "bdist_wheel": BdistWheel 60 | }, 61 | zip_safe=False 62 | ) 63 | -------------------------------------------------------------------------------- /tree-sitter-markdown-inline/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(PROJECT_DESCRIPTION "Markdown inline grammar for tree-sitter") 2 | 3 | add_parser(markdown-inline) 4 | 5 | add_custom_target(test-inline "${TREE_SITTER_CLI}" test 6 | WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" 7 | DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c" 8 | COMMENT "tree-sitter test") 9 | -------------------------------------------------------------------------------- /tree-sitter-markdown-inline/Makefile: -------------------------------------------------------------------------------- 1 | LANGUAGE_NAME := tree-sitter-markdown-inline 2 | 3 | include ../common/common.mak 4 | -------------------------------------------------------------------------------- /tree-sitter-markdown-inline/bindings/c/tree-sitter-markdown-inline.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_MARKDOWN_INLINE_H_ 2 | #define TREE_SITTER_MARKDOWN_INLINE_H_ 3 | 4 | typedef struct TSLanguage TSLanguage; 5 | 6 | #ifdef __cplusplus 7 | extern "C" { 8 | #endif 9 | 10 | const TSLanguage *tree_sitter_markdown_inline(void); 11 | 12 | #ifdef __cplusplus 13 | } 14 | #endif 15 | 16 | #endif // TREE_SITTER_MARKDOWN_INLINE_H_ 17 | -------------------------------------------------------------------------------- /tree-sitter-markdown-inline/bindings/c/tree-sitter-markdown-inline.pc.in: -------------------------------------------------------------------------------- 1 | prefix=@CMAKE_INSTALL_PREFIX@ 2 | libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ 3 | includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ 4 | 5 | Name: tree-sitter-markdown-inline 6 | Description: @PROJECT_DESCRIPTION@ 7 | URL: @PROJECT_HOMEPAGE_URL@ 8 | Version: @PROJECT_VERSION@ 9 | Requires: @TS_REQUIRES@ 10 | Libs: -L${libdir} -ltree-sitter-markdown-inline 11 | Cflags: -I${includedir} 12 | -------------------------------------------------------------------------------- /tree-sitter-markdown-inline/bindings/swift/TreeSitterMarkdownInline/markdown_inline.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_MARKDOWN_INLINE_H_ 2 | #define TREE_SITTER_MARKDOWN_INLINE_H_ 3 | 4 | typedef struct TSLanguage TSLanguage; 5 | 6 | #ifdef __cplusplus 7 | extern "C" { 8 | #endif 9 | 10 | const TSLanguage *tree_sitter_markdown_inline(void); 11 | 12 | #ifdef __cplusplus 13 | } 14 | #endif 15 | 16 | #endif // TREE_SITTER_MARKDOWN_INLINE_H_ 17 | -------------------------------------------------------------------------------- /tree-sitter-markdown-inline/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "main": "../bindings/node/inline.js", 3 | "private": true, 4 | "scripts": { 5 | "build": "tree-sitter generate", 6 | "test": "tree-sitter test" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /tree-sitter-markdown-inline/queries/highlights.scm: -------------------------------------------------------------------------------- 1 | ;; From nvim-treesitter/nvim-treesitter 2 | [ 3 | (code_span) 4 | (link_title) 5 | ] @text.literal 6 | 7 | [ 8 | (emphasis_delimiter) 9 | (code_span_delimiter) 10 | ] @punctuation.delimiter 11 | 12 | (emphasis) @text.emphasis 13 | 14 | (strong_emphasis) @text.strong 15 | 16 | [ 17 | (link_destination) 18 | (uri_autolink) 19 | ] @text.uri 20 | 21 | [ 22 | (link_label) 23 | (link_text) 24 | (image_description) 25 | ] @text.reference 26 | 27 | [ 28 | (backslash_escape) 29 | (hard_line_break) 30 | ] @string.escape 31 | 32 | (image ["!" "[" "]" "(" ")"] @punctuation.delimiter) 33 | (inline_link ["[" "]" "(" ")"] @punctuation.delimiter) 34 | (shortcut_link ["[" "]"] @punctuation.delimiter) 35 | 36 | ; NOTE: extension not enabled by default 37 | ; (wiki_link ["[" "|" "]"] @punctuation.delimiter) 38 | -------------------------------------------------------------------------------- /tree-sitter-markdown-inline/queries/injections.scm: -------------------------------------------------------------------------------- 1 | ((html_tag) @injection.content (#set! injection.language "html")) 2 | ((latex_block) @injection.content (#set! injection.language "latex")) 3 | -------------------------------------------------------------------------------- /tree-sitter-markdown-inline/src/node-types.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "type": "backslash_escape", 4 | "named": true, 5 | "fields": {} 6 | }, 7 | { 8 | "type": "code_span", 9 | "named": true, 10 | "fields": {}, 11 | "children": { 12 | "multiple": true, 13 | "required": true, 14 | "types": [ 15 | { 16 | "type": "code_span_delimiter", 17 | "named": true 18 | } 19 | ] 20 | } 21 | }, 22 | { 23 | "type": "collapsed_reference_link", 24 | "named": true, 25 | "fields": {}, 26 | "children": { 27 | "multiple": false, 28 | "required": false, 29 | "types": [ 30 | { 31 | "type": "link_text", 32 | "named": true 33 | } 34 | ] 35 | } 36 | }, 37 | { 38 | "type": "emphasis", 39 | "named": true, 40 | "fields": {}, 41 | "children": { 42 | "multiple": true, 43 | "required": true, 44 | "types": [ 45 | { 46 | "type": "backslash_escape", 47 | "named": true 48 | }, 49 | { 50 | "type": "code_span", 51 | "named": true 52 | }, 53 | { 54 | "type": "collapsed_reference_link", 55 | "named": true 56 | }, 57 | { 58 | "type": "email_autolink", 59 | "named": true 60 | }, 61 | { 62 | "type": "emphasis", 63 | "named": true 64 | }, 65 | { 66 | "type": "emphasis_delimiter", 67 | "named": true 68 | }, 69 | { 70 | "type": "entity_reference", 71 | "named": true 72 | }, 73 | { 74 | "type": "full_reference_link", 75 | "named": true 76 | }, 77 | { 78 | "type": "hard_line_break", 79 | "named": true 80 | }, 81 | { 82 | "type": "html_tag", 83 | "named": true 84 | }, 85 | { 86 | "type": "image", 87 | "named": true 88 | }, 89 | { 90 | "type": "inline_link", 91 | "named": true 92 | }, 93 | { 94 | "type": "latex_block", 95 | "named": true 96 | }, 97 | { 98 | "type": "numeric_character_reference", 99 | "named": true 100 | }, 101 | { 102 | "type": "shortcut_link", 103 | "named": true 104 | }, 105 | { 106 | "type": "strikethrough", 107 | "named": true 108 | }, 109 | { 110 | "type": "strong_emphasis", 111 | "named": true 112 | }, 113 | { 114 | "type": "uri_autolink", 115 | "named": true 116 | } 117 | ] 118 | } 119 | }, 120 | { 121 | "type": "full_reference_link", 122 | "named": true, 123 | "fields": {}, 124 | "children": { 125 | "multiple": true, 126 | "required": true, 127 | "types": [ 128 | { 129 | "type": "link_label", 130 | "named": true 131 | }, 132 | { 133 | "type": "link_text", 134 | "named": true 135 | } 136 | ] 137 | } 138 | }, 139 | { 140 | "type": "hard_line_break", 141 | "named": true, 142 | "fields": {} 143 | }, 144 | { 145 | "type": "html_tag", 146 | "named": true, 147 | "fields": {} 148 | }, 149 | { 150 | "type": "image", 151 | "named": true, 152 | "fields": {}, 153 | "children": { 154 | "multiple": true, 155 | "required": false, 156 | "types": [ 157 | { 158 | "type": "image_description", 159 | "named": true 160 | }, 161 | { 162 | "type": "link_destination", 163 | "named": true 164 | }, 165 | { 166 | "type": "link_label", 167 | "named": true 168 | }, 169 | { 170 | "type": "link_title", 171 | "named": true 172 | } 173 | ] 174 | } 175 | }, 176 | { 177 | "type": "image_description", 178 | "named": true, 179 | "fields": {}, 180 | "children": { 181 | "multiple": true, 182 | "required": false, 183 | "types": [ 184 | { 185 | "type": "backslash_escape", 186 | "named": true 187 | }, 188 | { 189 | "type": "code_span", 190 | "named": true 191 | }, 192 | { 193 | "type": "collapsed_reference_link", 194 | "named": true 195 | }, 196 | { 197 | "type": "email_autolink", 198 | "named": true 199 | }, 200 | { 201 | "type": "emphasis", 202 | "named": true 203 | }, 204 | { 205 | "type": "entity_reference", 206 | "named": true 207 | }, 208 | { 209 | "type": "full_reference_link", 210 | "named": true 211 | }, 212 | { 213 | "type": "hard_line_break", 214 | "named": true 215 | }, 216 | { 217 | "type": "html_tag", 218 | "named": true 219 | }, 220 | { 221 | "type": "image", 222 | "named": true 223 | }, 224 | { 225 | "type": "inline_link", 226 | "named": true 227 | }, 228 | { 229 | "type": "latex_block", 230 | "named": true 231 | }, 232 | { 233 | "type": "numeric_character_reference", 234 | "named": true 235 | }, 236 | { 237 | "type": "shortcut_link", 238 | "named": true 239 | }, 240 | { 241 | "type": "strikethrough", 242 | "named": true 243 | }, 244 | { 245 | "type": "strong_emphasis", 246 | "named": true 247 | }, 248 | { 249 | "type": "uri_autolink", 250 | "named": true 251 | } 252 | ] 253 | } 254 | }, 255 | { 256 | "type": "inline", 257 | "named": true, 258 | "root": true, 259 | "fields": {}, 260 | "children": { 261 | "multiple": true, 262 | "required": false, 263 | "types": [ 264 | { 265 | "type": "backslash_escape", 266 | "named": true 267 | }, 268 | { 269 | "type": "code_span", 270 | "named": true 271 | }, 272 | { 273 | "type": "collapsed_reference_link", 274 | "named": true 275 | }, 276 | { 277 | "type": "email_autolink", 278 | "named": true 279 | }, 280 | { 281 | "type": "emphasis", 282 | "named": true 283 | }, 284 | { 285 | "type": "entity_reference", 286 | "named": true 287 | }, 288 | { 289 | "type": "full_reference_link", 290 | "named": true 291 | }, 292 | { 293 | "type": "hard_line_break", 294 | "named": true 295 | }, 296 | { 297 | "type": "html_tag", 298 | "named": true 299 | }, 300 | { 301 | "type": "image", 302 | "named": true 303 | }, 304 | { 305 | "type": "inline_link", 306 | "named": true 307 | }, 308 | { 309 | "type": "latex_block", 310 | "named": true 311 | }, 312 | { 313 | "type": "numeric_character_reference", 314 | "named": true 315 | }, 316 | { 317 | "type": "shortcut_link", 318 | "named": true 319 | }, 320 | { 321 | "type": "strikethrough", 322 | "named": true 323 | }, 324 | { 325 | "type": "strong_emphasis", 326 | "named": true 327 | }, 328 | { 329 | "type": "uri_autolink", 330 | "named": true 331 | } 332 | ] 333 | } 334 | }, 335 | { 336 | "type": "inline_link", 337 | "named": true, 338 | "fields": {}, 339 | "children": { 340 | "multiple": true, 341 | "required": false, 342 | "types": [ 343 | { 344 | "type": "link_destination", 345 | "named": true 346 | }, 347 | { 348 | "type": "link_text", 349 | "named": true 350 | }, 351 | { 352 | "type": "link_title", 353 | "named": true 354 | } 355 | ] 356 | } 357 | }, 358 | { 359 | "type": "latex_block", 360 | "named": true, 361 | "fields": {}, 362 | "children": { 363 | "multiple": true, 364 | "required": true, 365 | "types": [ 366 | { 367 | "type": "backslash_escape", 368 | "named": true 369 | }, 370 | { 371 | "type": "latex_span_delimiter", 372 | "named": true 373 | } 374 | ] 375 | } 376 | }, 377 | { 378 | "type": "link_destination", 379 | "named": true, 380 | "fields": {}, 381 | "children": { 382 | "multiple": true, 383 | "required": false, 384 | "types": [ 385 | { 386 | "type": "backslash_escape", 387 | "named": true 388 | }, 389 | { 390 | "type": "entity_reference", 391 | "named": true 392 | }, 393 | { 394 | "type": "numeric_character_reference", 395 | "named": true 396 | } 397 | ] 398 | } 399 | }, 400 | { 401 | "type": "link_label", 402 | "named": true, 403 | "fields": {}, 404 | "children": { 405 | "multiple": true, 406 | "required": false, 407 | "types": [ 408 | { 409 | "type": "backslash_escape", 410 | "named": true 411 | }, 412 | { 413 | "type": "entity_reference", 414 | "named": true 415 | }, 416 | { 417 | "type": "numeric_character_reference", 418 | "named": true 419 | } 420 | ] 421 | } 422 | }, 423 | { 424 | "type": "link_text", 425 | "named": true, 426 | "fields": {}, 427 | "children": { 428 | "multiple": true, 429 | "required": false, 430 | "types": [ 431 | { 432 | "type": "backslash_escape", 433 | "named": true 434 | }, 435 | { 436 | "type": "code_span", 437 | "named": true 438 | }, 439 | { 440 | "type": "email_autolink", 441 | "named": true 442 | }, 443 | { 444 | "type": "emphasis", 445 | "named": true 446 | }, 447 | { 448 | "type": "entity_reference", 449 | "named": true 450 | }, 451 | { 452 | "type": "hard_line_break", 453 | "named": true 454 | }, 455 | { 456 | "type": "html_tag", 457 | "named": true 458 | }, 459 | { 460 | "type": "image", 461 | "named": true 462 | }, 463 | { 464 | "type": "latex_block", 465 | "named": true 466 | }, 467 | { 468 | "type": "numeric_character_reference", 469 | "named": true 470 | }, 471 | { 472 | "type": "strikethrough", 473 | "named": true 474 | }, 475 | { 476 | "type": "strong_emphasis", 477 | "named": true 478 | }, 479 | { 480 | "type": "uri_autolink", 481 | "named": true 482 | } 483 | ] 484 | } 485 | }, 486 | { 487 | "type": "link_title", 488 | "named": true, 489 | "fields": {}, 490 | "children": { 491 | "multiple": true, 492 | "required": false, 493 | "types": [ 494 | { 495 | "type": "backslash_escape", 496 | "named": true 497 | }, 498 | { 499 | "type": "entity_reference", 500 | "named": true 501 | }, 502 | { 503 | "type": "numeric_character_reference", 504 | "named": true 505 | } 506 | ] 507 | } 508 | }, 509 | { 510 | "type": "shortcut_link", 511 | "named": true, 512 | "fields": {}, 513 | "children": { 514 | "multiple": false, 515 | "required": true, 516 | "types": [ 517 | { 518 | "type": "link_text", 519 | "named": true 520 | } 521 | ] 522 | } 523 | }, 524 | { 525 | "type": "strikethrough", 526 | "named": true, 527 | "fields": {}, 528 | "children": { 529 | "multiple": true, 530 | "required": true, 531 | "types": [ 532 | { 533 | "type": "backslash_escape", 534 | "named": true 535 | }, 536 | { 537 | "type": "code_span", 538 | "named": true 539 | }, 540 | { 541 | "type": "collapsed_reference_link", 542 | "named": true 543 | }, 544 | { 545 | "type": "email_autolink", 546 | "named": true 547 | }, 548 | { 549 | "type": "emphasis", 550 | "named": true 551 | }, 552 | { 553 | "type": "emphasis_delimiter", 554 | "named": true 555 | }, 556 | { 557 | "type": "entity_reference", 558 | "named": true 559 | }, 560 | { 561 | "type": "full_reference_link", 562 | "named": true 563 | }, 564 | { 565 | "type": "hard_line_break", 566 | "named": true 567 | }, 568 | { 569 | "type": "html_tag", 570 | "named": true 571 | }, 572 | { 573 | "type": "image", 574 | "named": true 575 | }, 576 | { 577 | "type": "inline_link", 578 | "named": true 579 | }, 580 | { 581 | "type": "latex_block", 582 | "named": true 583 | }, 584 | { 585 | "type": "numeric_character_reference", 586 | "named": true 587 | }, 588 | { 589 | "type": "shortcut_link", 590 | "named": true 591 | }, 592 | { 593 | "type": "strikethrough", 594 | "named": true 595 | }, 596 | { 597 | "type": "strong_emphasis", 598 | "named": true 599 | }, 600 | { 601 | "type": "uri_autolink", 602 | "named": true 603 | } 604 | ] 605 | } 606 | }, 607 | { 608 | "type": "strong_emphasis", 609 | "named": true, 610 | "fields": {}, 611 | "children": { 612 | "multiple": true, 613 | "required": true, 614 | "types": [ 615 | { 616 | "type": "backslash_escape", 617 | "named": true 618 | }, 619 | { 620 | "type": "code_span", 621 | "named": true 622 | }, 623 | { 624 | "type": "collapsed_reference_link", 625 | "named": true 626 | }, 627 | { 628 | "type": "email_autolink", 629 | "named": true 630 | }, 631 | { 632 | "type": "emphasis", 633 | "named": true 634 | }, 635 | { 636 | "type": "emphasis_delimiter", 637 | "named": true 638 | }, 639 | { 640 | "type": "entity_reference", 641 | "named": true 642 | }, 643 | { 644 | "type": "full_reference_link", 645 | "named": true 646 | }, 647 | { 648 | "type": "hard_line_break", 649 | "named": true 650 | }, 651 | { 652 | "type": "html_tag", 653 | "named": true 654 | }, 655 | { 656 | "type": "image", 657 | "named": true 658 | }, 659 | { 660 | "type": "inline_link", 661 | "named": true 662 | }, 663 | { 664 | "type": "latex_block", 665 | "named": true 666 | }, 667 | { 668 | "type": "numeric_character_reference", 669 | "named": true 670 | }, 671 | { 672 | "type": "shortcut_link", 673 | "named": true 674 | }, 675 | { 676 | "type": "strikethrough", 677 | "named": true 678 | }, 679 | { 680 | "type": "strong_emphasis", 681 | "named": true 682 | }, 683 | { 684 | "type": "uri_autolink", 685 | "named": true 686 | } 687 | ] 688 | } 689 | }, 690 | { 691 | "type": "!", 692 | "named": false 693 | }, 694 | { 695 | "type": "\"", 696 | "named": false 697 | }, 698 | { 699 | "type": "#", 700 | "named": false 701 | }, 702 | { 703 | "type": "$", 704 | "named": false 705 | }, 706 | { 707 | "type": "%", 708 | "named": false 709 | }, 710 | { 711 | "type": "&", 712 | "named": false 713 | }, 714 | { 715 | "type": "'", 716 | "named": false 717 | }, 718 | { 719 | "type": "(", 720 | "named": false 721 | }, 722 | { 723 | "type": ")", 724 | "named": false 725 | }, 726 | { 727 | "type": "*", 728 | "named": false 729 | }, 730 | { 731 | "type": "+", 732 | "named": false 733 | }, 734 | { 735 | "type": ",", 736 | "named": false 737 | }, 738 | { 739 | "type": "-", 740 | "named": false 741 | }, 742 | { 743 | "type": "-->", 744 | "named": false 745 | }, 746 | { 747 | "type": ".", 748 | "named": false 749 | }, 750 | { 751 | "type": "/", 752 | "named": false 753 | }, 754 | { 755 | "type": ":", 756 | "named": false 757 | }, 758 | { 759 | "type": ";", 760 | "named": false 761 | }, 762 | { 763 | "type": "<", 764 | "named": false 765 | }, 766 | { 767 | "type": "` 57 | 58 | -------------------------------------------------------------------------------- 59 | 60 | (inline 61 | (code_span 62 | (code_span_delimiter) 63 | (code_span_delimiter))) 64 | 65 | ================================================================================ 66 | #36 - Multiple code spans with HTML comments does not working properly (2) 67 | ================================================================================ 68 | `` foo 69 | 70 | -------------------------------------------------------------------------------- 71 | 72 | (inline 73 | (code_span 74 | (code_span_delimiter) 75 | (code_span_delimiter))) 76 | 77 | ================================================================================ 78 | #36 - Multiple code spans with HTML comments does not working properly (3) 79 | ================================================================================ 80 | `` foo `` 81 | 82 | -------------------------------------------------------------------------------- 83 | 84 | (inline 85 | (code_span 86 | (code_span_delimiter) 87 | (code_span_delimiter)) 88 | (code_span 89 | (code_span_delimiter) 90 | (code_span_delimiter))) 91 | 92 | ================================================================================ 93 | #75 - code spans with `[x][]` 94 | ================================================================================ 95 | `some code` 96 | normal text (or even nothing) `[index][]` 97 | 98 | -------------------------------------------------------------------------------- 99 | 100 | (inline 101 | (code_span 102 | (code_span_delimiter) 103 | (code_span_delimiter)) 104 | (code_span 105 | (code_span_delimiter) 106 | (code_span_delimiter))) 107 | -------------------------------------------------------------------------------- /tree-sitter-markdown-inline/test/corpus/tags.txt: -------------------------------------------------------------------------------- 1 | ================================================================================ 2 | Tags are working 3 | ================================================================================ 4 | #tag #_tag_with-symbols0123and/numbers #0123tag_starting_with_numbers 5 | 6 | -------------------------------------------------------------------------------- 7 | 8 | (inline 9 | (tag) 10 | (tag) 11 | (tag)) 12 | 13 | ================================================================================ 14 | Purely numeric tags are forbidden 15 | ================================================================================ 16 | #0123 17 | 18 | -------------------------------------------------------------------------------- 19 | 20 | (inline) 21 | 22 | -------------------------------------------------------------------------------- /tree-sitter-markdown/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(PROJECT_DESCRIPTION "Markdown block grammar for tree-sitter") 2 | set(TS_REQUIRES tree-sitter-markdown-inline) 3 | 4 | add_parser(markdown) 5 | 6 | add_custom_target(test "${TREE_SITTER_CLI}" test 7 | WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" 8 | DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c" 9 | COMMENT "tree-sitter test") 10 | -------------------------------------------------------------------------------- /tree-sitter-markdown/Makefile: -------------------------------------------------------------------------------- 1 | LANGUAGE_NAME := tree-sitter-markdown 2 | 3 | REQUIRES := tree-sitter-markdown-inline 4 | 5 | include ../common/common.mak 6 | -------------------------------------------------------------------------------- /tree-sitter-markdown/bindings/c/tree-sitter-markdown.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_MARKDOWN_H_ 2 | #define TREE_SITTER_MARKDOWN_H_ 3 | 4 | typedef struct TSLanguage TSLanguage; 5 | 6 | #ifdef __cplusplus 7 | extern "C" { 8 | #endif 9 | 10 | const TSLanguage *tree_sitter_markdown(void); 11 | 12 | #ifdef __cplusplus 13 | } 14 | #endif 15 | 16 | #endif // TREE_SITTER_MARKDOWN_H_ 17 | -------------------------------------------------------------------------------- /tree-sitter-markdown/bindings/c/tree-sitter-markdown.pc.in: -------------------------------------------------------------------------------- 1 | prefix=@CMAKE_INSTALL_PREFIX@ 2 | libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ 3 | includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ 4 | 5 | Name: tree-sitter-markdown 6 | Description: @PROJECT_DESCRIPTION@ 7 | URL: @PROJECT_HOMEPAGE_URL@ 8 | Version: @PROJECT_VERSION@ 9 | Requires: @TS_REQUIRES@ 10 | Libs: -L${libdir} -ltree-sitter-markdown 11 | Cflags: -I${includedir} 12 | -------------------------------------------------------------------------------- /tree-sitter-markdown/bindings/swift/TreeSitterMarkdown/markdown.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_MARKDOWN_H_ 2 | #define TREE_SITTER_MARKDOWN_H_ 3 | 4 | typedef struct TSLanguage TSLanguage; 5 | 6 | #ifdef __cplusplus 7 | extern "C" { 8 | #endif 9 | 10 | const TSLanguage *tree_sitter_markdown(void); 11 | 12 | #ifdef __cplusplus 13 | } 14 | #endif 15 | 16 | #endif // TREE_SITTER_MARKDOWN_H_ 17 | -------------------------------------------------------------------------------- /tree-sitter-markdown/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "main": "../bindings/node/index.js", 3 | "private": true, 4 | "scripts": { 5 | "build": "tree-sitter generate", 6 | "test": "tree-sitter test" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /tree-sitter-markdown/queries/highlights.scm: -------------------------------------------------------------------------------- 1 | ;From nvim-treesitter/nvim-treesitter 2 | (atx_heading (inline) @text.title) 3 | (setext_heading (paragraph) @text.title) 4 | 5 | [ 6 | (atx_h1_marker) 7 | (atx_h2_marker) 8 | (atx_h3_marker) 9 | (atx_h4_marker) 10 | (atx_h5_marker) 11 | (atx_h6_marker) 12 | (setext_h1_underline) 13 | (setext_h2_underline) 14 | ] @punctuation.special 15 | 16 | [ 17 | (link_title) 18 | (indented_code_block) 19 | (fenced_code_block) 20 | ] @text.literal 21 | 22 | [ 23 | (fenced_code_block_delimiter) 24 | ] @punctuation.delimiter 25 | 26 | (code_fence_content) @none 27 | 28 | [ 29 | (link_destination) 30 | ] @text.uri 31 | 32 | [ 33 | (link_label) 34 | ] @text.reference 35 | 36 | [ 37 | (list_marker_plus) 38 | (list_marker_minus) 39 | (list_marker_star) 40 | (list_marker_dot) 41 | (list_marker_parenthesis) 42 | (thematic_break) 43 | ] @punctuation.special 44 | 45 | [ 46 | (block_continuation) 47 | (block_quote_marker) 48 | ] @punctuation.special 49 | 50 | [ 51 | (backslash_escape) 52 | ] @string.escape 53 | -------------------------------------------------------------------------------- /tree-sitter-markdown/queries/injections.scm: -------------------------------------------------------------------------------- 1 | (fenced_code_block 2 | (info_string 3 | (language) @injection.language) 4 | (code_fence_content) @injection.content) 5 | 6 | ((html_block) @injection.content (#set! injection.language "html")) 7 | 8 | (document . (section . (thematic_break) (_) @injection.content (thematic_break)) (#set! injection.language "yaml")) 9 | 10 | ((minus_metadata) @injection.content (#set! injection.language "yaml")) 11 | 12 | ((plus_metadata) @injection.content (#set! injection.language "toml")) 13 | 14 | ((inline) @injection.content (#set! injection.language "markdown_inline")) 15 | -------------------------------------------------------------------------------- /tree-sitter-markdown/src/tree_sitter/alloc.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_ALLOC_H_ 2 | #define TREE_SITTER_ALLOC_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | // Allow clients to override allocation functions 13 | #ifdef TREE_SITTER_REUSE_ALLOCATOR 14 | 15 | extern void *(*ts_current_malloc)(size_t size); 16 | extern void *(*ts_current_calloc)(size_t count, size_t size); 17 | extern void *(*ts_current_realloc)(void *ptr, size_t size); 18 | extern void (*ts_current_free)(void *ptr); 19 | 20 | #ifndef ts_malloc 21 | #define ts_malloc ts_current_malloc 22 | #endif 23 | #ifndef ts_calloc 24 | #define ts_calloc ts_current_calloc 25 | #endif 26 | #ifndef ts_realloc 27 | #define ts_realloc ts_current_realloc 28 | #endif 29 | #ifndef ts_free 30 | #define ts_free ts_current_free 31 | #endif 32 | 33 | #else 34 | 35 | #ifndef ts_malloc 36 | #define ts_malloc malloc 37 | #endif 38 | #ifndef ts_calloc 39 | #define ts_calloc calloc 40 | #endif 41 | #ifndef ts_realloc 42 | #define ts_realloc realloc 43 | #endif 44 | #ifndef ts_free 45 | #define ts_free free 46 | #endif 47 | 48 | #endif 49 | 50 | #ifdef __cplusplus 51 | } 52 | #endif 53 | 54 | #endif // TREE_SITTER_ALLOC_H_ 55 | -------------------------------------------------------------------------------- /tree-sitter-markdown/src/tree_sitter/array.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_ARRAY_H_ 2 | #define TREE_SITTER_ARRAY_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include "./alloc.h" 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #ifdef _MSC_VER 17 | #pragma warning(push) 18 | #pragma warning(disable : 4101) 19 | #elif defined(__GNUC__) || defined(__clang__) 20 | #pragma GCC diagnostic push 21 | #pragma GCC diagnostic ignored "-Wunused-variable" 22 | #endif 23 | 24 | #define Array(T) \ 25 | struct { \ 26 | T *contents; \ 27 | uint32_t size; \ 28 | uint32_t capacity; \ 29 | } 30 | 31 | /// Initialize an array. 32 | #define array_init(self) \ 33 | ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL) 34 | 35 | /// Create an empty array. 36 | #define array_new() \ 37 | { NULL, 0, 0 } 38 | 39 | /// Get a pointer to the element at a given `index` in the array. 40 | #define array_get(self, _index) \ 41 | (assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index]) 42 | 43 | /// Get a pointer to the first element in the array. 44 | #define array_front(self) array_get(self, 0) 45 | 46 | /// Get a pointer to the last element in the array. 47 | #define array_back(self) array_get(self, (self)->size - 1) 48 | 49 | /// Clear the array, setting its size to zero. Note that this does not free any 50 | /// memory allocated for the array's contents. 51 | #define array_clear(self) ((self)->size = 0) 52 | 53 | /// Reserve `new_capacity` elements of space in the array. If `new_capacity` is 54 | /// less than the array's current capacity, this function has no effect. 55 | #define array_reserve(self, new_capacity) \ 56 | _array__reserve((Array *)(self), array_elem_size(self), new_capacity) 57 | 58 | /// Free any memory allocated for this array. Note that this does not free any 59 | /// memory allocated for the array's contents. 60 | #define array_delete(self) _array__delete((Array *)(self)) 61 | 62 | /// Push a new `element` onto the end of the array. 63 | #define array_push(self, element) \ 64 | (_array__grow((Array *)(self), 1, array_elem_size(self)), \ 65 | (self)->contents[(self)->size++] = (element)) 66 | 67 | /// Increase the array's size by `count` elements. 68 | /// New elements are zero-initialized. 69 | #define array_grow_by(self, count) \ 70 | do { \ 71 | if ((count) == 0) break; \ 72 | _array__grow((Array *)(self), count, array_elem_size(self)); \ 73 | memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \ 74 | (self)->size += (count); \ 75 | } while (0) 76 | 77 | /// Append all elements from one array to the end of another. 78 | #define array_push_all(self, other) \ 79 | array_extend((self), (other)->size, (other)->contents) 80 | 81 | /// Append `count` elements to the end of the array, reading their values from the 82 | /// `contents` pointer. 83 | #define array_extend(self, count, contents) \ 84 | _array__splice( \ 85 | (Array *)(self), array_elem_size(self), (self)->size, \ 86 | 0, count, contents \ 87 | ) 88 | 89 | /// Remove `old_count` elements from the array starting at the given `index`. At 90 | /// the same index, insert `new_count` new elements, reading their values from the 91 | /// `new_contents` pointer. 92 | #define array_splice(self, _index, old_count, new_count, new_contents) \ 93 | _array__splice( \ 94 | (Array *)(self), array_elem_size(self), _index, \ 95 | old_count, new_count, new_contents \ 96 | ) 97 | 98 | /// Insert one `element` into the array at the given `index`. 99 | #define array_insert(self, _index, element) \ 100 | _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element)) 101 | 102 | /// Remove one element from the array at the given `index`. 103 | #define array_erase(self, _index) \ 104 | _array__erase((Array *)(self), array_elem_size(self), _index) 105 | 106 | /// Pop the last element off the array, returning the element by value. 107 | #define array_pop(self) ((self)->contents[--(self)->size]) 108 | 109 | /// Assign the contents of one array to another, reallocating if necessary. 110 | #define array_assign(self, other) \ 111 | _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self)) 112 | 113 | /// Swap one array with another 114 | #define array_swap(self, other) \ 115 | _array__swap((Array *)(self), (Array *)(other)) 116 | 117 | /// Get the size of the array contents 118 | #define array_elem_size(self) (sizeof *(self)->contents) 119 | 120 | /// Search a sorted array for a given `needle` value, using the given `compare` 121 | /// callback to determine the order. 122 | /// 123 | /// If an existing element is found to be equal to `needle`, then the `index` 124 | /// out-parameter is set to the existing value's index, and the `exists` 125 | /// out-parameter is set to true. Otherwise, `index` is set to an index where 126 | /// `needle` should be inserted in order to preserve the sorting, and `exists` 127 | /// is set to false. 128 | #define array_search_sorted_with(self, compare, needle, _index, _exists) \ 129 | _array__search_sorted(self, 0, compare, , needle, _index, _exists) 130 | 131 | /// Search a sorted array for a given `needle` value, using integer comparisons 132 | /// of a given struct field (specified with a leading dot) to determine the order. 133 | /// 134 | /// See also `array_search_sorted_with`. 135 | #define array_search_sorted_by(self, field, needle, _index, _exists) \ 136 | _array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists) 137 | 138 | /// Insert a given `value` into a sorted array, using the given `compare` 139 | /// callback to determine the order. 140 | #define array_insert_sorted_with(self, compare, value) \ 141 | do { \ 142 | unsigned _index, _exists; \ 143 | array_search_sorted_with(self, compare, &(value), &_index, &_exists); \ 144 | if (!_exists) array_insert(self, _index, value); \ 145 | } while (0) 146 | 147 | /// Insert a given `value` into a sorted array, using integer comparisons of 148 | /// a given struct field (specified with a leading dot) to determine the order. 149 | /// 150 | /// See also `array_search_sorted_by`. 151 | #define array_insert_sorted_by(self, field, value) \ 152 | do { \ 153 | unsigned _index, _exists; \ 154 | array_search_sorted_by(self, field, (value) field, &_index, &_exists); \ 155 | if (!_exists) array_insert(self, _index, value); \ 156 | } while (0) 157 | 158 | // Private 159 | 160 | typedef Array(void) Array; 161 | 162 | /// This is not what you're looking for, see `array_delete`. 163 | static inline void _array__delete(Array *self) { 164 | if (self->contents) { 165 | ts_free(self->contents); 166 | self->contents = NULL; 167 | self->size = 0; 168 | self->capacity = 0; 169 | } 170 | } 171 | 172 | /// This is not what you're looking for, see `array_erase`. 173 | static inline void _array__erase(Array *self, size_t element_size, 174 | uint32_t index) { 175 | assert(index < self->size); 176 | char *contents = (char *)self->contents; 177 | memmove(contents + index * element_size, contents + (index + 1) * element_size, 178 | (self->size - index - 1) * element_size); 179 | self->size--; 180 | } 181 | 182 | /// This is not what you're looking for, see `array_reserve`. 183 | static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) { 184 | if (new_capacity > self->capacity) { 185 | if (self->contents) { 186 | self->contents = ts_realloc(self->contents, new_capacity * element_size); 187 | } else { 188 | self->contents = ts_malloc(new_capacity * element_size); 189 | } 190 | self->capacity = new_capacity; 191 | } 192 | } 193 | 194 | /// This is not what you're looking for, see `array_assign`. 195 | static inline void _array__assign(Array *self, const Array *other, size_t element_size) { 196 | _array__reserve(self, element_size, other->size); 197 | self->size = other->size; 198 | memcpy(self->contents, other->contents, self->size * element_size); 199 | } 200 | 201 | /// This is not what you're looking for, see `array_swap`. 202 | static inline void _array__swap(Array *self, Array *other) { 203 | Array swap = *other; 204 | *other = *self; 205 | *self = swap; 206 | } 207 | 208 | /// This is not what you're looking for, see `array_push` or `array_grow_by`. 209 | static inline void _array__grow(Array *self, uint32_t count, size_t element_size) { 210 | uint32_t new_size = self->size + count; 211 | if (new_size > self->capacity) { 212 | uint32_t new_capacity = self->capacity * 2; 213 | if (new_capacity < 8) new_capacity = 8; 214 | if (new_capacity < new_size) new_capacity = new_size; 215 | _array__reserve(self, element_size, new_capacity); 216 | } 217 | } 218 | 219 | /// This is not what you're looking for, see `array_splice`. 220 | static inline void _array__splice(Array *self, size_t element_size, 221 | uint32_t index, uint32_t old_count, 222 | uint32_t new_count, const void *elements) { 223 | uint32_t new_size = self->size + new_count - old_count; 224 | uint32_t old_end = index + old_count; 225 | uint32_t new_end = index + new_count; 226 | assert(old_end <= self->size); 227 | 228 | _array__reserve(self, element_size, new_size); 229 | 230 | char *contents = (char *)self->contents; 231 | if (self->size > old_end) { 232 | memmove( 233 | contents + new_end * element_size, 234 | contents + old_end * element_size, 235 | (self->size - old_end) * element_size 236 | ); 237 | } 238 | if (new_count > 0) { 239 | if (elements) { 240 | memcpy( 241 | (contents + index * element_size), 242 | elements, 243 | new_count * element_size 244 | ); 245 | } else { 246 | memset( 247 | (contents + index * element_size), 248 | 0, 249 | new_count * element_size 250 | ); 251 | } 252 | } 253 | self->size += new_count - old_count; 254 | } 255 | 256 | /// A binary search routine, based on Rust's `std::slice::binary_search_by`. 257 | /// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`. 258 | #define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \ 259 | do { \ 260 | *(_index) = start; \ 261 | *(_exists) = false; \ 262 | uint32_t size = (self)->size - *(_index); \ 263 | if (size == 0) break; \ 264 | int comparison; \ 265 | while (size > 1) { \ 266 | uint32_t half_size = size / 2; \ 267 | uint32_t mid_index = *(_index) + half_size; \ 268 | comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \ 269 | if (comparison <= 0) *(_index) = mid_index; \ 270 | size -= half_size; \ 271 | } \ 272 | comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \ 273 | if (comparison == 0) *(_exists) = true; \ 274 | else if (comparison < 0) *(_index) += 1; \ 275 | } while (0) 276 | 277 | /// Helper macro for the `_sorted_by` routines below. This takes the left (existing) 278 | /// parameter by reference in order to work with the generic sorting function above. 279 | #define _compare_int(a, b) ((int)*(a) - (int)(b)) 280 | 281 | #ifdef _MSC_VER 282 | #pragma warning(pop) 283 | #elif defined(__GNUC__) || defined(__clang__) 284 | #pragma GCC diagnostic pop 285 | #endif 286 | 287 | #ifdef __cplusplus 288 | } 289 | #endif 290 | 291 | #endif // TREE_SITTER_ARRAY_H_ 292 | -------------------------------------------------------------------------------- /tree-sitter-markdown/src/tree_sitter/parser.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_PARSER_H_ 2 | #define TREE_SITTER_PARSER_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #define ts_builtin_sym_error ((TSSymbol)-1) 13 | #define ts_builtin_sym_end 0 14 | #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 15 | 16 | #ifndef TREE_SITTER_API_H_ 17 | typedef uint16_t TSStateId; 18 | typedef uint16_t TSSymbol; 19 | typedef uint16_t TSFieldId; 20 | typedef struct TSLanguage TSLanguage; 21 | typedef struct TSLanguageMetadata { 22 | uint8_t major_version; 23 | uint8_t minor_version; 24 | uint8_t patch_version; 25 | } TSLanguageMetadata; 26 | #endif 27 | 28 | typedef struct { 29 | TSFieldId field_id; 30 | uint8_t child_index; 31 | bool inherited; 32 | } TSFieldMapEntry; 33 | 34 | // Used to index the field and supertype maps. 35 | typedef struct { 36 | uint16_t index; 37 | uint16_t length; 38 | } TSMapSlice; 39 | 40 | typedef struct { 41 | bool visible; 42 | bool named; 43 | bool supertype; 44 | } TSSymbolMetadata; 45 | 46 | typedef struct TSLexer TSLexer; 47 | 48 | struct TSLexer { 49 | int32_t lookahead; 50 | TSSymbol result_symbol; 51 | void (*advance)(TSLexer *, bool); 52 | void (*mark_end)(TSLexer *); 53 | uint32_t (*get_column)(TSLexer *); 54 | bool (*is_at_included_range_start)(const TSLexer *); 55 | bool (*eof)(const TSLexer *); 56 | void (*log)(const TSLexer *, const char *, ...); 57 | }; 58 | 59 | typedef enum { 60 | TSParseActionTypeShift, 61 | TSParseActionTypeReduce, 62 | TSParseActionTypeAccept, 63 | TSParseActionTypeRecover, 64 | } TSParseActionType; 65 | 66 | typedef union { 67 | struct { 68 | uint8_t type; 69 | TSStateId state; 70 | bool extra; 71 | bool repetition; 72 | } shift; 73 | struct { 74 | uint8_t type; 75 | uint8_t child_count; 76 | TSSymbol symbol; 77 | int16_t dynamic_precedence; 78 | uint16_t production_id; 79 | } reduce; 80 | uint8_t type; 81 | } TSParseAction; 82 | 83 | typedef struct { 84 | uint16_t lex_state; 85 | uint16_t external_lex_state; 86 | } TSLexMode; 87 | 88 | typedef struct { 89 | uint16_t lex_state; 90 | uint16_t external_lex_state; 91 | uint16_t reserved_word_set_id; 92 | } TSLexerMode; 93 | 94 | typedef union { 95 | TSParseAction action; 96 | struct { 97 | uint8_t count; 98 | bool reusable; 99 | } entry; 100 | } TSParseActionEntry; 101 | 102 | typedef struct { 103 | int32_t start; 104 | int32_t end; 105 | } TSCharacterRange; 106 | 107 | struct TSLanguage { 108 | uint32_t abi_version; 109 | uint32_t symbol_count; 110 | uint32_t alias_count; 111 | uint32_t token_count; 112 | uint32_t external_token_count; 113 | uint32_t state_count; 114 | uint32_t large_state_count; 115 | uint32_t production_id_count; 116 | uint32_t field_count; 117 | uint16_t max_alias_sequence_length; 118 | const uint16_t *parse_table; 119 | const uint16_t *small_parse_table; 120 | const uint32_t *small_parse_table_map; 121 | const TSParseActionEntry *parse_actions; 122 | const char * const *symbol_names; 123 | const char * const *field_names; 124 | const TSMapSlice *field_map_slices; 125 | const TSFieldMapEntry *field_map_entries; 126 | const TSSymbolMetadata *symbol_metadata; 127 | const TSSymbol *public_symbol_map; 128 | const uint16_t *alias_map; 129 | const TSSymbol *alias_sequences; 130 | const TSLexerMode *lex_modes; 131 | bool (*lex_fn)(TSLexer *, TSStateId); 132 | bool (*keyword_lex_fn)(TSLexer *, TSStateId); 133 | TSSymbol keyword_capture_token; 134 | struct { 135 | const bool *states; 136 | const TSSymbol *symbol_map; 137 | void *(*create)(void); 138 | void (*destroy)(void *); 139 | bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist); 140 | unsigned (*serialize)(void *, char *); 141 | void (*deserialize)(void *, const char *, unsigned); 142 | } external_scanner; 143 | const TSStateId *primary_state_ids; 144 | const char *name; 145 | const TSSymbol *reserved_words; 146 | uint16_t max_reserved_word_set_size; 147 | uint32_t supertype_count; 148 | const TSSymbol *supertype_symbols; 149 | const TSMapSlice *supertype_map_slices; 150 | const TSSymbol *supertype_map_entries; 151 | TSLanguageMetadata metadata; 152 | }; 153 | 154 | static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) { 155 | uint32_t index = 0; 156 | uint32_t size = len - index; 157 | while (size > 1) { 158 | uint32_t half_size = size / 2; 159 | uint32_t mid_index = index + half_size; 160 | const TSCharacterRange *range = &ranges[mid_index]; 161 | if (lookahead >= range->start && lookahead <= range->end) { 162 | return true; 163 | } else if (lookahead > range->end) { 164 | index = mid_index; 165 | } 166 | size -= half_size; 167 | } 168 | const TSCharacterRange *range = &ranges[index]; 169 | return (lookahead >= range->start && lookahead <= range->end); 170 | } 171 | 172 | /* 173 | * Lexer Macros 174 | */ 175 | 176 | #ifdef _MSC_VER 177 | #define UNUSED __pragma(warning(suppress : 4101)) 178 | #else 179 | #define UNUSED __attribute__((unused)) 180 | #endif 181 | 182 | #define START_LEXER() \ 183 | bool result = false; \ 184 | bool skip = false; \ 185 | UNUSED \ 186 | bool eof = false; \ 187 | int32_t lookahead; \ 188 | goto start; \ 189 | next_state: \ 190 | lexer->advance(lexer, skip); \ 191 | start: \ 192 | skip = false; \ 193 | lookahead = lexer->lookahead; 194 | 195 | #define ADVANCE(state_value) \ 196 | { \ 197 | state = state_value; \ 198 | goto next_state; \ 199 | } 200 | 201 | #define ADVANCE_MAP(...) \ 202 | { \ 203 | static const uint16_t map[] = { __VA_ARGS__ }; \ 204 | for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \ 205 | if (map[i] == lookahead) { \ 206 | state = map[i + 1]; \ 207 | goto next_state; \ 208 | } \ 209 | } \ 210 | } 211 | 212 | #define SKIP(state_value) \ 213 | { \ 214 | skip = true; \ 215 | state = state_value; \ 216 | goto next_state; \ 217 | } 218 | 219 | #define ACCEPT_TOKEN(symbol_value) \ 220 | result = true; \ 221 | lexer->result_symbol = symbol_value; \ 222 | lexer->mark_end(lexer); 223 | 224 | #define END_STATE() return result; 225 | 226 | /* 227 | * Parse Table Macros 228 | */ 229 | 230 | #define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT) 231 | 232 | #define STATE(id) id 233 | 234 | #define ACTIONS(id) id 235 | 236 | #define SHIFT(state_value) \ 237 | {{ \ 238 | .shift = { \ 239 | .type = TSParseActionTypeShift, \ 240 | .state = (state_value) \ 241 | } \ 242 | }} 243 | 244 | #define SHIFT_REPEAT(state_value) \ 245 | {{ \ 246 | .shift = { \ 247 | .type = TSParseActionTypeShift, \ 248 | .state = (state_value), \ 249 | .repetition = true \ 250 | } \ 251 | }} 252 | 253 | #define SHIFT_EXTRA() \ 254 | {{ \ 255 | .shift = { \ 256 | .type = TSParseActionTypeShift, \ 257 | .extra = true \ 258 | } \ 259 | }} 260 | 261 | #define REDUCE(symbol_name, children, precedence, prod_id) \ 262 | {{ \ 263 | .reduce = { \ 264 | .type = TSParseActionTypeReduce, \ 265 | .symbol = symbol_name, \ 266 | .child_count = children, \ 267 | .dynamic_precedence = precedence, \ 268 | .production_id = prod_id \ 269 | }, \ 270 | }} 271 | 272 | #define RECOVER() \ 273 | {{ \ 274 | .type = TSParseActionTypeRecover \ 275 | }} 276 | 277 | #define ACCEPT_INPUT() \ 278 | {{ \ 279 | .type = TSParseActionTypeAccept \ 280 | }} 281 | 282 | #ifdef __cplusplus 283 | } 284 | #endif 285 | 286 | #endif // TREE_SITTER_PARSER_H_ 287 | -------------------------------------------------------------------------------- /tree-sitter-markdown/test/corpus/extension_minus_metadata.txt: -------------------------------------------------------------------------------- 1 | ================================================================================ 2 | EXTENSION_MINUS_METADATA - https://pandoc.org/MANUAL.html#extension-yaml_metadata_block 3 | ================================================================================ 4 | --- 5 | title: 'This is the title: it contains a colon' 6 | author: 7 | - Author One 8 | - Author Two 9 | keywords: [nothing, nothingness] 10 | abstract: | 11 | This is the abstract. 12 | 13 | It consists of two paragraphs. 14 | --- 15 | 16 | -------------------------------------------------------------------------------- 17 | 18 | (document 19 | (minus_metadata)) 20 | -------------------------------------------------------------------------------- /tree-sitter-markdown/test/corpus/extension_pipe_table.txt: -------------------------------------------------------------------------------- 1 | ================================================================================ 2 | Example 198 - https://github.github.com/gfm/#example-198 3 | ================================================================================ 4 | | foo | bar | 5 | | --- | --- | 6 | | baz | bim | 7 | 8 | -------------------------------------------------------------------------------- 9 | 10 | (document 11 | (section 12 | (pipe_table 13 | (pipe_table_header 14 | (pipe_table_cell) 15 | (pipe_table_cell)) 16 | (pipe_table_delimiter_row 17 | (pipe_table_delimiter_cell) 18 | (pipe_table_delimiter_cell)) 19 | (pipe_table_row 20 | (pipe_table_cell) 21 | (pipe_table_cell))))) 22 | 23 | ================================================================================ 24 | Example 199 - https://github.github.com/gfm/#example-199 25 | ================================================================================ 26 | | abc | defghi | 27 | :-: | -----------: 28 | bar | baz 29 | 30 | -------------------------------------------------------------------------------- 31 | 32 | (document 33 | (section 34 | (pipe_table 35 | (pipe_table_header 36 | (pipe_table_cell) 37 | (pipe_table_cell)) 38 | (pipe_table_delimiter_row 39 | (pipe_table_delimiter_cell 40 | (pipe_table_align_left) 41 | (pipe_table_align_right)) 42 | (pipe_table_delimiter_cell 43 | (pipe_table_align_right))) 44 | (pipe_table_row 45 | (pipe_table_cell) 46 | (pipe_table_cell))))) 47 | 48 | ================================================================================ 49 | Example 200 - https://github.github.com/gfm/#example-200 50 | ================================================================================ 51 | | f\|oo | 52 | | ------ | 53 | | b `\|` az | 54 | | b **\|** im | 55 | 56 | -------------------------------------------------------------------------------- 57 | 58 | (document 59 | (section 60 | (pipe_table 61 | (pipe_table_header 62 | (pipe_table_cell)) 63 | (pipe_table_delimiter_row 64 | (pipe_table_delimiter_cell)) 65 | (pipe_table_row 66 | (pipe_table_cell)) 67 | (pipe_table_row 68 | (pipe_table_cell))))) 69 | 70 | ================================================================================ 71 | Example 201 - https://github.github.com/gfm/#example-201 72 | ================================================================================ 73 | | abc | def | 74 | | --- | --- | 75 | | bar | baz | 76 | > bar 77 | 78 | -------------------------------------------------------------------------------- 79 | 80 | (document 81 | (section 82 | (pipe_table 83 | (pipe_table_header 84 | (pipe_table_cell) 85 | (pipe_table_cell)) 86 | (pipe_table_delimiter_row 87 | (pipe_table_delimiter_cell) 88 | (pipe_table_delimiter_cell)) 89 | (pipe_table_row 90 | (pipe_table_cell) 91 | (pipe_table_cell))) 92 | (block_quote 93 | (block_quote_marker) 94 | (paragraph 95 | (inline))))) 96 | 97 | ================================================================================ 98 | Example 202 - https://github.github.com/gfm/#example-202 99 | ================================================================================ 100 | | abc | def | 101 | | --- | --- | 102 | | bar | baz | 103 | bar 104 | 105 | bar 106 | 107 | -------------------------------------------------------------------------------- 108 | 109 | (document 110 | (section 111 | (pipe_table 112 | (pipe_table_header 113 | (pipe_table_cell) 114 | (pipe_table_cell)) 115 | (pipe_table_delimiter_row 116 | (pipe_table_delimiter_cell) 117 | (pipe_table_delimiter_cell)) 118 | (pipe_table_row 119 | (pipe_table_cell) 120 | (pipe_table_cell)) 121 | (pipe_table_row 122 | (pipe_table_cell))) 123 | (paragraph 124 | (inline)))) 125 | 126 | ================================================================================ 127 | Example 203 - https://github.github.com/gfm/#example-203 128 | ================================================================================ 129 | | abc | def | 130 | | --- | 131 | | bar | 132 | 133 | -------------------------------------------------------------------------------- 134 | 135 | (document 136 | (section 137 | (paragraph 138 | (inline)))) 139 | 140 | ================================================================================ 141 | Example 204 - https://github.github.com/gfm/#example-204 142 | ================================================================================ 143 | | abc | def | 144 | | --- | --- | 145 | | bar | 146 | | bar | baz | boo | 147 | 148 | -------------------------------------------------------------------------------- 149 | 150 | (document 151 | (section 152 | (pipe_table 153 | (pipe_table_header 154 | (pipe_table_cell) 155 | (pipe_table_cell)) 156 | (pipe_table_delimiter_row 157 | (pipe_table_delimiter_cell) 158 | (pipe_table_delimiter_cell)) 159 | (pipe_table_row 160 | (pipe_table_cell)) 161 | (pipe_table_row 162 | (pipe_table_cell) 163 | (pipe_table_cell) 164 | (pipe_table_cell))))) 165 | 166 | ================================================================================ 167 | Example 205 - https://github.github.com/gfm/#example-205 168 | ================================================================================ 169 | | abc | def | 170 | | --- | --- | 171 | 172 | -------------------------------------------------------------------------------- 173 | 174 | (document 175 | (section 176 | (pipe_table 177 | (pipe_table_header 178 | (pipe_table_cell) 179 | (pipe_table_cell)) 180 | (pipe_table_delimiter_row 181 | (pipe_table_delimiter_cell) 182 | (pipe_table_delimiter_cell))))) 183 | 184 | ================================================================================ 185 | #112 - Works with table cells that only contain whitespce 186 | ================================================================================ 187 | | foo | bar | 188 | | --- | --- | 189 | | | bim | 190 | 191 | -------------------------------------------------------------------------------- 192 | 193 | (document 194 | (section 195 | (pipe_table 196 | (pipe_table_header 197 | (pipe_table_cell) 198 | (pipe_table_cell)) 199 | (pipe_table_delimiter_row 200 | (pipe_table_delimiter_cell) 201 | (pipe_table_delimiter_cell)) 202 | (pipe_table_row 203 | (pipe_table_cell) 204 | (pipe_table_cell))))) 205 | -------------------------------------------------------------------------------- /tree-sitter-markdown/test/corpus/extension_plus_metadata.txt: -------------------------------------------------------------------------------- 1 | ================================================================================ 2 | EXTENSION_PLUS_METADATA 3 | ================================================================================ 4 | +++ 5 | title: 'This is the title: it contains a colon' 6 | author: 7 | - Author One 8 | - Author Two 9 | keywords: [nothing, nothingness] 10 | abstract: | 11 | This is the abstract. 12 | 13 | It consists of two paragraphs. 14 | +++ 15 | 16 | -------------------------------------------------------------------------------- 17 | 18 | (document 19 | (plus_metadata)) 20 | -------------------------------------------------------------------------------- /tree-sitter-markdown/test/corpus/extension_task_list.txt: -------------------------------------------------------------------------------- 1 | ================================================================================ 2 | Example 279 - https://github.github.com/gfm/#example-279 3 | ================================================================================ 4 | - [ ] foo 5 | - [x] bar 6 | 7 | -------------------------------------------------------------------------------- 8 | 9 | (document 10 | (section 11 | (list 12 | (list_item 13 | (list_marker_minus) 14 | (task_list_marker_unchecked) 15 | (paragraph 16 | (inline))) 17 | (list_item 18 | (list_marker_minus) 19 | (task_list_marker_checked) 20 | (paragraph 21 | (inline)))))) 22 | 23 | ================================================================================ 24 | Example 280 - https://github.github.com/gfm/#example-280 25 | ================================================================================ 26 | - [x] foo 27 | - [ ] bar 28 | - [x] baz 29 | - [ ] bim 30 | 31 | -------------------------------------------------------------------------------- 32 | 33 | (document 34 | (section 35 | (list 36 | (list_item 37 | (list_marker_minus) 38 | (task_list_marker_checked) 39 | (paragraph 40 | (inline) 41 | (block_continuation)) 42 | (list 43 | (list_item 44 | (list_marker_minus) 45 | (task_list_marker_unchecked) 46 | (paragraph 47 | (inline) 48 | (block_continuation))) 49 | (list_item 50 | (list_marker_minus) 51 | (task_list_marker_checked) 52 | (paragraph 53 | (inline))))) 54 | (list_item 55 | (list_marker_minus) 56 | (task_list_marker_unchecked) 57 | (paragraph 58 | (inline)))))) 59 | 60 | ================================================================================ 61 | task list item marker … the letter x in either lowercase or UPPERCASE 62 | ================================================================================ 63 | - [ ] foo 64 | - [X] bar 65 | 66 | -------------------------------------------------------------------------------- 67 | 68 | (document 69 | (section 70 | (list 71 | (list_item 72 | (list_marker_minus) 73 | (task_list_marker_unchecked) 74 | (paragraph 75 | (inline))) 76 | (list_item 77 | (list_marker_minus) 78 | (task_list_marker_checked) 79 | (paragraph 80 | (inline)))))) 81 | -------------------------------------------------------------------------------- /tree-sitter-markdown/test/corpus/failing.txt: -------------------------------------------------------------------------------- 1 | ================================================================================ 2 | Example 107 - https://github.github.com/gfm/#example-107 3 | :skip 4 | ================================================================================ 5 | ``` 6 | aaa 7 | ``` 8 | 9 | -------------------------------------------------------------------------------- 10 | 11 | (document 12 | (section 13 | (fenced_code_block 14 | (fenced_code_block_delimiter) 15 | (block_continuation) 16 | (code_fence_content 17 | (block_continuation))))) 18 | -------------------------------------------------------------------------------- /tree-sitter-markdown/test/corpus/issues.txt: -------------------------------------------------------------------------------- 1 | ================================================================================ 2 | #17 - Titles not detected after an empty inner list item (bullet point) 3 | ================================================================================ 4 | * a 5 | * b 6 | * 7 | 8 | 9 | # C 10 | 11 | -------------------------------------------------------------------------------- 12 | 13 | (document 14 | (section 15 | (list 16 | (list_item 17 | (list_marker_star) 18 | (paragraph 19 | (inline) 20 | (block_continuation)) 21 | (list 22 | (list_item 23 | (list_marker_star) 24 | (paragraph 25 | (inline) 26 | (block_continuation))) 27 | (list_item 28 | (list_marker_star) 29 | (block_continuation) 30 | (block_continuation)))))) 31 | (section 32 | (atx_heading 33 | (atx_h1_marker) 34 | (inline)))) 35 | 36 | ================================================================================ 37 | #33 - Fenced code block attributes 38 | ================================================================================ 39 | ```{R} 40 | 1 + 1 41 | ``` 42 | 43 | ```{} 44 | 1 + 1 45 | ``` 46 | 47 | -------------------------------------------------------------------------------- 48 | 49 | (document 50 | (section 51 | (fenced_code_block 52 | (fenced_code_block_delimiter) 53 | (info_string 54 | (language)) 55 | (block_continuation) 56 | (code_fence_content 57 | (block_continuation)) 58 | (fenced_code_block_delimiter)) 59 | (fenced_code_block 60 | (fenced_code_block_delimiter) 61 | (info_string) 62 | (block_continuation) 63 | (code_fence_content 64 | (block_continuation)) 65 | (fenced_code_block_delimiter)))) 66 | 67 | ================================================================================ 68 | #72 - Can't create list item after a list item with a newline and indent 69 | ================================================================================ 70 | 1. a 71 | 1. b 72 | c 73 | 2. d 74 | 75 | -------------------------------------------------------------------------------- 76 | 77 | (document 78 | (section 79 | (list 80 | (list_item 81 | (list_marker_dot) 82 | (paragraph 83 | (inline))) 84 | (list_item 85 | (list_marker_dot) 86 | (paragraph 87 | (inline 88 | (block_continuation)))) 89 | (list_item 90 | (list_marker_dot) 91 | (paragraph 92 | (inline)))))) 93 | 94 | ================================================================================ 95 | #135 - Closing code block fence not recognized when it has trailing space 96 | ================================================================================ 97 | ``` 98 | // returns 2 99 | globalNS.method1(5, 10); 100 | ``` 101 | 102 | @example 103 | 104 | -------------------------------------------------------------------------------- 105 | 106 | (document 107 | (section 108 | (fenced_code_block 109 | (fenced_code_block_delimiter) 110 | (block_continuation) 111 | (code_fence_content 112 | (block_continuation) 113 | (block_continuation)) 114 | (fenced_code_block_delimiter)) 115 | (paragraph 116 | (inline)))) 117 | -------------------------------------------------------------------------------- /tree-sitter.json: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "version": "0.5.0", 4 | "license": "MIT", 5 | "authors": [ 6 | { 7 | "name": "MDeiml" 8 | } 9 | ], 10 | "description": "Markdown grammar for tree-sitter", 11 | "links": { 12 | "repository": "https://github.com/tree-sitter-grammars/tree-sitter-markdown" 13 | } 14 | }, 15 | "grammars": [ 16 | { 17 | "name": "markdown", 18 | "scope": "text.markdown", 19 | "path": "tree-sitter-markdown", 20 | "injection-regex": "^(markdown|md)$", 21 | "file-types": [ 22 | "md" 23 | ], 24 | "highlights": "tree-sitter-markdown/queries/highlights.scm", 25 | "injections": "tree-sitter-markdown/queries/injections.scm", 26 | "external-files": [ 27 | "common/common.js" 28 | ] 29 | }, 30 | { 31 | "name": "markdown_inline", 32 | "camelcase": "MarkdownInline", 33 | "scope": "text.markdown_inline", 34 | "path": "tree-sitter-markdown-inline", 35 | "highlights": "tree-sitter-markdown-inline/queries/highlights.scm", 36 | "injections": "tree-sitter-markdown-inline/queries/injections.scm", 37 | "external-files": [ 38 | "common/common.js" 39 | ] 40 | } 41 | ] 42 | } 43 | --------------------------------------------------------------------------------