├── .envrc ├── .gitattributes ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── CHANGELOG.md ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── bindings ├── CHANGELOG.md ├── Cargo.toml ├── LICENSE ├── build.rs ├── src │ ├── grammar.rs │ ├── lib.rs │ ├── node.rs │ ├── parser.rs │ ├── query.rs │ ├── query │ │ ├── predicate.rs │ │ └── property.rs │ ├── query_cursor.rs │ ├── ropey.rs │ ├── tree.rs │ └── tree_cursor.rs ├── vendor.sh └── vendor │ ├── LICENSE │ ├── include │ └── tree_sitter │ │ └── api.h │ └── src │ ├── alloc.c │ ├── alloc.h │ ├── array.h │ ├── atomic.h │ ├── clock.h │ ├── error_costs.h │ ├── get_changed_ranges.c │ ├── get_changed_ranges.h │ ├── host.h │ ├── language.c │ ├── language.h │ ├── length.h │ ├── lexer.c │ ├── lexer.h │ ├── lib.c │ ├── node.c │ ├── parser.c │ ├── parser.h │ ├── point.h │ ├── portable │ └── endian.h │ ├── query.c │ ├── reduce_action.h │ ├── reusable_node.h │ ├── stack.c │ ├── stack.h │ ├── subtree.c │ ├── subtree.h │ ├── tree.c │ ├── tree.h │ ├── tree_cursor.c │ ├── tree_cursor.h │ ├── ts_assert.h │ ├── unicode.h │ ├── unicode │ ├── ICU_SHA │ ├── LICENSE │ ├── README.md │ ├── ptypes.h │ ├── umachine.h │ ├── urename.h │ ├── utf.h │ ├── utf16.h │ └── utf8.h │ ├── wasm │ ├── stdlib-symbols.txt │ ├── stdlib.c │ └── wasm-stdlib.h │ ├── wasm_store.c │ └── wasm_store.h ├── cli ├── Cargo.toml ├── LICENSE ├── import.sh └── src │ ├── build.rs │ ├── flags.rs │ ├── generate_parser.rs │ ├── import.rs │ ├── init.rs │ ├── load.rs │ └── main.rs ├── default.nix ├── fixtures ├── highlighter │ ├── codefence_rust_doc_comments.md │ ├── comment.html │ ├── hello_world.rs │ ├── html_in_edoc_in_erlang.erl │ ├── injectionception.rs │ ├── non_local.rs │ ├── reference_highlight_starts_after_definition_ends.rs │ ├── rust_doc_comment.rs │ ├── rust_no_doc_comment.rs │ └── rust_parameter_locals.rs └── injections │ ├── html_in_edoc_in_erlang.erl │ ├── injectionception.rs │ ├── overlapping_injection.rs │ ├── rust_doc_comment.rs │ └── rust_no_doc_comment.rs ├── flake.lock ├── flake.nix ├── highlighter ├── CHANGELOG.md ├── Cargo.toml ├── LICENSE └── src │ ├── config.rs │ ├── fixtures.rs │ ├── highlighter.rs │ ├── injections_query.rs │ ├── lib.rs │ ├── locals.rs │ ├── parse.rs │ ├── pretty_print.rs │ ├── query_iter.rs │ ├── tests.rs │ ├── text_object.rs │ └── tree_cursor.rs ├── skidder ├── Cargo.toml ├── LICENSE ├── build.rs └── src │ ├── build.rs │ └── lib.rs └── test-grammars ├── comment ├── LICENSE ├── highlights.scm ├── metadata.json └── src │ ├── grammar.json │ ├── parser.c │ ├── scanner.c │ ├── tree_sitter │ └── parser.h │ └── tree_sitter_comment │ ├── chars.c │ ├── chars.h │ ├── parser.c │ ├── parser.h │ └── tokens.h ├── edoc ├── LICENSE ├── highlights.scm ├── injections.scm ├── metadata.json └── src │ ├── grammar.json │ ├── parser.c │ └── tree_sitter │ └── parser.h ├── erlang ├── LICENSE ├── highlights.scm ├── injections.scm ├── locals.scm ├── metadata.json ├── src │ ├── grammar.json │ ├── parser.c │ └── tree_sitter │ │ └── parser.h └── textobjects.scm ├── html ├── LICENSE ├── highlights.scm ├── injections.scm ├── metadata.json └── src │ ├── grammar.json │ ├── parser.c │ ├── scanner.cc │ ├── tag.h │ └── tree_sitter │ └── parser.h ├── markdown-inline ├── LICENSE ├── highlights.scm ├── injections.scm ├── metadata.json └── src │ ├── grammar.json │ ├── parser.c │ ├── scanner.c │ └── tree_sitter │ ├── alloc.h │ ├── array.h │ └── parser.h ├── markdown ├── LICENSE ├── highlights.scm ├── injections.scm ├── metadata.json └── src │ ├── grammar.json │ ├── parser.c │ ├── scanner.c │ └── tree_sitter │ ├── alloc.h │ ├── array.h │ └── parser.h └── rust ├── LICENSE ├── highlights.scm ├── injections.scm ├── locals.scm ├── metadata.json └── src ├── grammar.json ├── parser.c ├── scanner.c └── tree_sitter ├── alloc.h ├── array.h └── parser.h /.envrc: -------------------------------------------------------------------------------- 1 | watch_file flake.lock 2 | 3 | # try to use flakes, if it fails use normal nix (ie. shell.nix) 4 | use flake || use nix 5 | eval "$shellHook" 6 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # See 2 | test-grammars/*/*.scm linguist-vendored 3 | test-grammars/*/src/** linguist-vendored 4 | test-grammars/*/src/{parser.c,grammar.json,scanner.*} binary 5 | fixtures/** -linguist-detectable 6 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | pull_request: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | check-msrv: 10 | name: Check 11 | strategy: 12 | matrix: 13 | toolchain: 14 | - "1.76" 15 | - stable 16 | runs-on: ubuntu-latest 17 | steps: 18 | - name: Checkout sources 19 | uses: actions/checkout@v4 20 | 21 | - name: Install toolchain 22 | uses: dtolnay/rust-toolchain@master 23 | with: 24 | toolchain: ${{ matrix.toolchain}} 25 | 26 | - uses: Swatinem/rust-cache@v2 27 | 28 | - name: Run cargo check 29 | run: cargo check 30 | - name: Run cargo check without default features 31 | run: cargo check --no-default-features 32 | 33 | test: 34 | name: Test 35 | runs-on: ubuntu-latest 36 | steps: 37 | - name: Checkout sources 38 | uses: actions/checkout@v4 39 | 40 | - name: Install stable toolchain 41 | uses: dtolnay/rust-toolchain@stable 42 | 43 | - uses: Swatinem/rust-cache@v2 44 | 45 | - name: Run cargo test 46 | run: cargo test --workspace 47 | 48 | lints: 49 | name: Lints 50 | runs-on: ubuntu-latest 51 | steps: 52 | - name: Checkout sources 53 | uses: actions/checkout@v4 54 | 55 | - name: Install stable toolchain 56 | uses: dtolnay/rust-toolchain@stable 57 | with: 58 | components: rustfmt, clippy 59 | 60 | - uses: Swatinem/rust-cache@v2 61 | 62 | - name: Run cargo fmt 63 | run: cargo fmt --all --check 64 | 65 | - name: Run cargo clippy 66 | run: cargo clippy --workspace --all-targets -- -D warnings 67 | - name: Run cargo clippy without default features 68 | run: cargo clippy --workspace --all-targets --no-default-features -- -D warnings 69 | 70 | - name: Run cargo doc 71 | run: cargo doc --no-deps --workspace --document-private-items 72 | env: 73 | RUSTDOCFLAGS: -D warnings 74 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | result 3 | .direnv 4 | /test-grammars/*/*.so 5 | /test-grammars/*/.BUILD_COOKIE 6 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "2" 3 | members = ["bindings", "cli", "highlighter", "skidder"] 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `tree-house` 2 | 3 | This repository contains a number of crates used by the [Helix editor](https://github.com/helix-editor/helix) for integration with the [tree-sitter](https://github.com/tree-sitter/tree-sitter) C library. 4 | 5 | * `bindings/` contains the `tree-house-bindings` crate which provides Rust bindings over the C library. 6 | * `highlighter/` contains the `tree-house` crate which exposes a robust highlighter and query iterator for working across [injections]. 7 | * `skidder/` contains the `skidder` crate which exposes utilities for building a package repository for tree-sitter grammars. 8 | * `cli/` contains the `skidder-cli` crate which wraps `skidder` in a command line interface. 9 | -------------------------------------------------------------------------------- /bindings/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | 9 | 10 | ## [v0.2.0] - 2025-06-06 11 | 12 | ### Added 13 | 14 | * Added `TreeCursor::reset` 15 | * Added an iterator for recursively walking over the nodes in a `TreeCursor`: `TreeRecursiveWalker` 16 | 17 | ### Updated 18 | 19 | * Updated the tree-sitter C library to v0.25.6 20 | 21 | ## [v0.1.1] - 2025-05-14 22 | 23 | ### Fixed 24 | 25 | * Patched `endian.h` to include IllumOS 26 | 27 | ## [v0.1.0] - 2025-05-13 28 | 29 | ### Added 30 | 31 | * Initial publish 32 | -------------------------------------------------------------------------------- /bindings/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tree-house-bindings" 3 | description = "Homey Rust bindings for the tree-sitter C library" 4 | authors = ["Pascal Kuthe "] 5 | version = "0.2.0" 6 | edition = "2021" 7 | license = "MPL-2.0" 8 | repository = "https://github.com/helix-editor/tree-house" 9 | readme = "../README.md" 10 | rust-version = "1.76.0" 11 | 12 | [features] 13 | ropey = ["dep:ropey"] 14 | 15 | [dependencies] 16 | ropey = { version = "1.6", default-features = false, optional=true } 17 | regex-cursor = "0.1.5" 18 | libloading = "0.8" 19 | thiserror = "2.0" 20 | 21 | [build-dependencies] 22 | cc = "1.0" 23 | -------------------------------------------------------------------------------- /bindings/LICENSE: -------------------------------------------------------------------------------- 1 | ../LICENSE -------------------------------------------------------------------------------- /bindings/build.rs: -------------------------------------------------------------------------------- 1 | use std::path::Path; 2 | use std::{env, fs}; 3 | 4 | fn main() { 5 | if env::var_os("DISABLED_TS_BUILD").is_some() { 6 | return; 7 | } 8 | let mut config = cc::Build::new(); 9 | 10 | let manifest_path = Path::new(env!("CARGO_MANIFEST_DIR")); 11 | let include_path = manifest_path.join("vendor/include"); 12 | let src_path = manifest_path.join("vendor/src"); 13 | for entry in fs::read_dir(&src_path).unwrap() { 14 | let entry = entry.unwrap(); 15 | let path = src_path.join(entry.file_name()); 16 | println!("cargo:rerun-if-changed={}", path.to_str().unwrap()); 17 | } 18 | 19 | config 20 | .flag_if_supported("-std=c11") 21 | .flag_if_supported("-fvisibility=hidden") 22 | .flag_if_supported("-Wshadow") 23 | .flag_if_supported("-Wno-unused-parameter") 24 | .flag_if_supported("-Wno-incompatible-pointer-types") 25 | .include(&src_path) 26 | .include(&include_path) 27 | .define("_POSIX_C_SOURCE", "200112L") 28 | .define("_DEFAULT_SOURCE", None) 29 | .warnings(false) 30 | .file(src_path.join("lib.c")) 31 | .compile("tree-sitter"); 32 | } 33 | -------------------------------------------------------------------------------- /bindings/src/grammar.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | use std::path::{Path, PathBuf}; 3 | use std::ptr::NonNull; 4 | 5 | use libloading::{Library, Symbol}; 6 | 7 | /// Lowest supported ABI version of a grammar. 8 | // WARNING: update when updating vendored c sources 9 | // `TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION` 10 | pub const MIN_COMPATIBLE_ABI_VERSION: u32 = 13; 11 | // `TREE_SITTER_LANGUAGE_VERSION` 12 | pub const ABI_VERSION: u32 = 15; 13 | 14 | // opaque pointer 15 | enum GrammarData {} 16 | 17 | #[repr(transparent)] 18 | #[derive(Clone, Copy, PartialEq, Eq, Hash)] 19 | pub struct Grammar { 20 | ptr: NonNull, 21 | } 22 | 23 | unsafe impl Send for Grammar {} 24 | unsafe impl Sync for Grammar {} 25 | 26 | impl std::fmt::Debug for Grammar { 27 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 28 | f.debug_struct("Grammar").finish_non_exhaustive() 29 | } 30 | } 31 | 32 | impl Grammar { 33 | /// Loads a shared library containing a tree sitter grammar with name `name` 34 | // from `library_path`. 35 | /// 36 | /// # Safety 37 | /// 38 | /// `library_path` must be a valid tree sitter grammar 39 | pub unsafe fn new(name: &str, library_path: &Path) -> Result { 40 | let library = unsafe { 41 | Library::new(library_path).map_err(|err| Error::DlOpen { 42 | err, 43 | path: library_path.to_owned(), 44 | })? 45 | }; 46 | let language_fn_name = format!("tree_sitter_{}", name.replace('-', "_")); 47 | let grammar = unsafe { 48 | let language_fn: Symbol NonNull> = library 49 | .get(language_fn_name.as_bytes()) 50 | .map_err(|err| Error::DlSym { 51 | err, 52 | symbol: name.to_owned(), 53 | })?; 54 | Grammar { ptr: language_fn() } 55 | }; 56 | let version = grammar.abi_version(); 57 | if (MIN_COMPATIBLE_ABI_VERSION..=ABI_VERSION).contains(&version) { 58 | std::mem::forget(library); 59 | Ok(grammar) 60 | } else { 61 | Err(Error::IncompatibleVersion { version }) 62 | } 63 | } 64 | 65 | pub fn abi_version(self) -> u32 { 66 | unsafe { ts_language_abi_version(self) } 67 | } 68 | 69 | pub fn node_kind_is_visible(self, kind_id: u16) -> bool { 70 | let symbol_type = unsafe { ts_language_symbol_type(self, kind_id) }; 71 | symbol_type <= (SymbolType::Anonymous as u32) 72 | } 73 | } 74 | 75 | #[derive(thiserror::Error, Debug)] 76 | pub enum Error { 77 | #[error("Error opening dynamic library {path:?}")] 78 | DlOpen { 79 | #[source] 80 | err: libloading::Error, 81 | path: PathBuf, 82 | }, 83 | #[error("Failed to load symbol {symbol}")] 84 | DlSym { 85 | #[source] 86 | err: libloading::Error, 87 | symbol: String, 88 | }, 89 | #[error("Tried to load grammar with incompatible ABI {version}.")] 90 | IncompatibleVersion { version: u32 }, 91 | } 92 | 93 | /// An error that occurred when trying to assign an incompatible [`Grammar`] to 94 | /// a [`crate::parser::Parser`]. 95 | #[derive(Debug, PartialEq, Eq)] 96 | pub struct IncompatibleGrammarError { 97 | pub abi_version: u32, 98 | } 99 | 100 | impl fmt::Display for IncompatibleGrammarError { 101 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 102 | write!( 103 | f, 104 | "Tried to load grammar with incompatible ABI version {}.", 105 | self.abi_version, 106 | ) 107 | } 108 | } 109 | impl std::error::Error for IncompatibleGrammarError {} 110 | 111 | #[repr(u32)] 112 | #[allow(dead_code)] 113 | pub enum SymbolType { 114 | Regular, 115 | Anonymous, 116 | Supertype, 117 | Auxiliary, 118 | } 119 | 120 | extern "C" { 121 | /// Get the ABI version number for this language. This version number 122 | /// is used to ensure that languages were generated by a compatible version of 123 | /// Tree-sitter. See also `ts_parser_set_language`. 124 | pub fn ts_language_abi_version(grammar: Grammar) -> u32; 125 | 126 | /// Checks whether the given node type belongs to named nodes, anonymous nodes, or hidden 127 | /// nodes. 128 | /// 129 | /// See also `ts_node_is_named`. Hidden nodes are never returned from the API. 130 | pub fn ts_language_symbol_type(grammar: Grammar, symbol: u16) -> u32; 131 | } 132 | -------------------------------------------------------------------------------- /bindings/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod grammar; 2 | mod node; 3 | mod parser; 4 | pub mod query; 5 | mod query_cursor; 6 | mod tree; 7 | mod tree_cursor; 8 | 9 | #[cfg(feature = "ropey")] 10 | mod ropey; 11 | #[cfg(feature = "ropey")] 12 | pub use ropey::RopeInput; 13 | 14 | use std::ops; 15 | 16 | pub use grammar::{Grammar, IncompatibleGrammarError}; 17 | pub use node::Node; 18 | pub use parser::{Parser, ParserInputRaw}; 19 | pub use query::{Capture, Pattern, Query, QueryStr}; 20 | pub use query_cursor::{InactiveQueryCursor, MatchedNode, MatchedNodeIdx, QueryCursor, QueryMatch}; 21 | pub use tree::{InputEdit, Tree}; 22 | pub use tree_cursor::TreeCursor; 23 | 24 | #[repr(C)] 25 | #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] 26 | pub struct Point { 27 | pub row: u32, 28 | pub col: u32, 29 | } 30 | 31 | impl Point { 32 | pub const ZERO: Self = Self { row: 0, col: 0 }; 33 | pub const MAX: Self = Self { 34 | row: u32::MAX, 35 | col: u32::MAX, 36 | }; 37 | } 38 | 39 | #[repr(C)] 40 | #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] 41 | pub struct Range { 42 | pub start_point: Point, 43 | pub end_point: Point, 44 | pub start_byte: u32, 45 | pub end_byte: u32, 46 | } 47 | 48 | pub trait Input { 49 | type Cursor: regex_cursor::Cursor; 50 | fn cursor_at(&mut self, offset: u32) -> &mut Self::Cursor; 51 | fn eq(&mut self, range1: ops::Range, range2: ops::Range) -> bool; 52 | } 53 | 54 | pub trait IntoInput { 55 | type Input: Input; 56 | fn into_input(self) -> Self::Input; 57 | } 58 | 59 | impl IntoInput for T { 60 | type Input = T; 61 | 62 | fn into_input(self) -> T { 63 | self 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /bindings/src/query/property.rs: -------------------------------------------------------------------------------- 1 | use crate::query::predicate::{InvalidPredicateError, Predicate}; 2 | use crate::query::QueryStr; 3 | 4 | #[derive(Debug)] 5 | pub struct QueryProperty { 6 | pub key: QueryStr, 7 | pub val: Option, 8 | } 9 | 10 | impl QueryProperty { 11 | pub fn parse(predicate: &Predicate) -> Result { 12 | predicate.check_min_arg_count(1)?; 13 | predicate.check_max_arg_count(2)?; 14 | let key = predicate.query_str_arg(0)?; 15 | let val = (predicate.num_args() == 2) 16 | .then(|| predicate.query_str_arg(1)) 17 | .transpose()?; 18 | Ok(QueryProperty { key, val }) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /bindings/src/ropey.rs: -------------------------------------------------------------------------------- 1 | use std::ops; 2 | 3 | use regex_cursor::{Cursor, RopeyCursor}; 4 | use ropey::RopeSlice; 5 | 6 | use crate::{Input, IntoInput}; 7 | 8 | pub struct RopeInput<'a> { 9 | src: RopeSlice<'a>, 10 | cursor: regex_cursor::RopeyCursor<'a>, 11 | } 12 | 13 | impl<'a> RopeInput<'a> { 14 | pub fn new(src: RopeSlice<'a>) -> Self { 15 | RopeInput { 16 | src, 17 | cursor: regex_cursor::RopeyCursor::new(src), 18 | } 19 | } 20 | } 21 | 22 | impl<'a> IntoInput for RopeSlice<'a> { 23 | type Input = RopeInput<'a>; 24 | 25 | fn into_input(self) -> Self::Input { 26 | RopeInput { 27 | src: self, 28 | cursor: RopeyCursor::new(self), 29 | } 30 | } 31 | } 32 | 33 | impl<'a> Input for RopeInput<'a> { 34 | type Cursor = RopeyCursor<'a>; 35 | fn cursor_at(&mut self, offset: u32) -> &mut RopeyCursor<'a> { 36 | let offset = offset as usize; 37 | debug_assert!( 38 | offset <= self.src.len_bytes(), 39 | "parser offset out of bounds: {offset} > {}", 40 | self.src.len_bytes() 41 | ); 42 | // this cursor is optimized for contiguous reads which are by far the most common during parsing 43 | // very far jumps (like injections at the other end of the document) are handled 44 | // by starting a new cursor (new chunks iterator) 45 | if offset < self.cursor.offset() || offset - self.cursor.offset() > 4906 { 46 | self.cursor = regex_cursor::RopeyCursor::at(self.src, offset); 47 | } else { 48 | while self.cursor.offset() + self.cursor.chunk().len() <= offset { 49 | if !self.cursor.advance() { 50 | break; 51 | } 52 | } 53 | } 54 | &mut self.cursor 55 | } 56 | 57 | fn eq(&mut self, range1: ops::Range, range2: ops::Range) -> bool { 58 | let range1 = self 59 | .src 60 | .byte_slice(range1.start as usize..range1.end as usize); 61 | let range2 = self 62 | .src 63 | .byte_slice(range2.start as usize..range2.end as usize); 64 | range1 == range2 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /bindings/src/tree.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | use std::ptr::NonNull; 3 | 4 | use crate::node::{Node, NodeRaw}; 5 | use crate::{Point, TreeCursor}; 6 | 7 | // opaque pointers 8 | pub(super) enum SyntaxTreeData {} 9 | 10 | pub struct Tree { 11 | ptr: NonNull, 12 | } 13 | 14 | impl Tree { 15 | pub(super) unsafe fn from_raw(raw: NonNull) -> Tree { 16 | Tree { ptr: raw } 17 | } 18 | 19 | pub(super) fn as_raw(&self) -> NonNull { 20 | self.ptr 21 | } 22 | 23 | pub fn root_node(&self) -> Node<'_> { 24 | unsafe { Node::from_raw(ts_tree_root_node(self.ptr)).unwrap() } 25 | } 26 | 27 | pub fn edit(&mut self, edit: &InputEdit) { 28 | unsafe { ts_tree_edit(self.ptr, edit) } 29 | } 30 | 31 | pub fn walk(&self) -> TreeCursor<'_> { 32 | self.root_node().walk() 33 | } 34 | } 35 | 36 | impl fmt::Debug for Tree { 37 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 38 | write!(f, "{{Tree {:?}}}", self.root_node()) 39 | } 40 | } 41 | 42 | impl Drop for Tree { 43 | fn drop(&mut self) { 44 | unsafe { ts_tree_delete(self.ptr) } 45 | } 46 | } 47 | 48 | impl Clone for Tree { 49 | fn clone(&self) -> Self { 50 | unsafe { 51 | Tree { 52 | ptr: ts_tree_copy(self.ptr), 53 | } 54 | } 55 | } 56 | } 57 | 58 | unsafe impl Send for Tree {} 59 | unsafe impl Sync for Tree {} 60 | 61 | #[repr(C)] 62 | #[derive(Debug, Copy, Clone, PartialEq, Eq)] 63 | pub struct InputEdit { 64 | pub start_byte: u32, 65 | pub old_end_byte: u32, 66 | pub new_end_byte: u32, 67 | pub start_point: Point, 68 | pub old_end_point: Point, 69 | pub new_end_point: Point, 70 | } 71 | 72 | impl InputEdit { 73 | /// returns the offset between the old end of the edit and the new end of 74 | /// the edit. This offset needs to be added to every position that occurs 75 | /// after `self.old_end_byte` to may it to its old position 76 | /// 77 | /// This function assumes that the the source-file is smaller than 2GiB 78 | pub fn offset(&self) -> i32 { 79 | self.new_end_byte as i32 - self.old_end_byte as i32 80 | } 81 | } 82 | 83 | extern "C" { 84 | /// Create a shallow copy of the syntax tree. This is very fast. You need to 85 | /// copy a syntax tree in order to use it on more than one thread at a time, 86 | /// as syntax trees are not thread safe. 87 | fn ts_tree_copy(self_: NonNull) -> NonNull; 88 | /// Delete the syntax tree, freeing all of the memory that it used. 89 | fn ts_tree_delete(self_: NonNull); 90 | /// Get the root node of the syntax tree. 91 | fn ts_tree_root_node<'tree>(self_: NonNull) -> NodeRaw; 92 | /// Edit the syntax tree to keep it in sync with source code that has been 93 | /// edited. 94 | /// 95 | /// You must describe the edit both in terms of byte offsets and in terms of 96 | /// row/column coordinates. 97 | fn ts_tree_edit(self_: NonNull, edit: &InputEdit); 98 | } 99 | -------------------------------------------------------------------------------- /bindings/vendor.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | REMOTE=https://github.com/tree-sitter/tree-sitter.git 4 | BRANCH=v0.25.6 5 | 6 | rm -rf vendor 7 | rm -rf tmp 8 | git clone --depth 1 --branch $BRANCH $REMOTE tmp 9 | mkdir vendor 10 | mv tmp/lib/src vendor 11 | mv tmp/lib/include vendor 12 | mv tmp/LICENSE vendor 13 | rm -rf tmp 14 | -------------------------------------------------------------------------------- /bindings/vendor/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018-2024 Max Brunsfeld 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /bindings/vendor/src/alloc.c: -------------------------------------------------------------------------------- 1 | #include "alloc.h" 2 | #include "tree_sitter/api.h" 3 | #include 4 | 5 | static void *ts_malloc_default(size_t size) { 6 | void *result = malloc(size); 7 | if (size > 0 && !result) { 8 | fprintf(stderr, "tree-sitter failed to allocate %zu bytes", size); 9 | abort(); 10 | } 11 | return result; 12 | } 13 | 14 | static void *ts_calloc_default(size_t count, size_t size) { 15 | void *result = calloc(count, size); 16 | if (count > 0 && !result) { 17 | fprintf(stderr, "tree-sitter failed to allocate %zu bytes", count * size); 18 | abort(); 19 | } 20 | return result; 21 | } 22 | 23 | static void *ts_realloc_default(void *buffer, size_t size) { 24 | void *result = realloc(buffer, size); 25 | if (size > 0 && !result) { 26 | fprintf(stderr, "tree-sitter failed to reallocate %zu bytes", size); 27 | abort(); 28 | } 29 | return result; 30 | } 31 | 32 | // Allow clients to override allocation functions dynamically 33 | TS_PUBLIC void *(*ts_current_malloc)(size_t) = ts_malloc_default; 34 | TS_PUBLIC void *(*ts_current_calloc)(size_t, size_t) = ts_calloc_default; 35 | TS_PUBLIC void *(*ts_current_realloc)(void *, size_t) = ts_realloc_default; 36 | TS_PUBLIC void (*ts_current_free)(void *) = free; 37 | 38 | void ts_set_allocator( 39 | void *(*new_malloc)(size_t size), 40 | void *(*new_calloc)(size_t count, size_t size), 41 | void *(*new_realloc)(void *ptr, size_t size), 42 | void (*new_free)(void *ptr) 43 | ) { 44 | ts_current_malloc = new_malloc ? new_malloc : ts_malloc_default; 45 | ts_current_calloc = new_calloc ? new_calloc : ts_calloc_default; 46 | ts_current_realloc = new_realloc ? new_realloc : ts_realloc_default; 47 | ts_current_free = new_free ? new_free : free; 48 | } 49 | -------------------------------------------------------------------------------- /bindings/vendor/src/alloc.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_ALLOC_H_ 2 | #define TREE_SITTER_ALLOC_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #if defined(TREE_SITTER_HIDDEN_SYMBOLS) || defined(_WIN32) 13 | #define TS_PUBLIC 14 | #else 15 | #define TS_PUBLIC __attribute__((visibility("default"))) 16 | #endif 17 | 18 | TS_PUBLIC extern void *(*ts_current_malloc)(size_t size); 19 | TS_PUBLIC extern void *(*ts_current_calloc)(size_t count, size_t size); 20 | TS_PUBLIC extern void *(*ts_current_realloc)(void *ptr, size_t size); 21 | TS_PUBLIC extern void (*ts_current_free)(void *ptr); 22 | 23 | // Allow clients to override allocation functions 24 | #ifndef ts_malloc 25 | #define ts_malloc ts_current_malloc 26 | #endif 27 | #ifndef ts_calloc 28 | #define ts_calloc ts_current_calloc 29 | #endif 30 | #ifndef ts_realloc 31 | #define ts_realloc ts_current_realloc 32 | #endif 33 | #ifndef ts_free 34 | #define ts_free ts_current_free 35 | #endif 36 | 37 | #ifdef __cplusplus 38 | } 39 | #endif 40 | 41 | #endif // TREE_SITTER_ALLOC_H_ 42 | -------------------------------------------------------------------------------- /bindings/vendor/src/atomic.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_ATOMIC_H_ 2 | #define TREE_SITTER_ATOMIC_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #ifdef __TINYC__ 9 | 10 | static inline size_t atomic_load(const volatile size_t *p) { 11 | return *p; 12 | } 13 | 14 | static inline uint32_t atomic_inc(volatile uint32_t *p) { 15 | *p += 1; 16 | return *p; 17 | } 18 | 19 | static inline uint32_t atomic_dec(volatile uint32_t *p) { 20 | *p-= 1; 21 | return *p; 22 | } 23 | 24 | #elif defined(_WIN32) 25 | 26 | #include 27 | 28 | static inline size_t atomic_load(const volatile size_t *p) { 29 | return *p; 30 | } 31 | 32 | static inline uint32_t atomic_inc(volatile uint32_t *p) { 33 | return InterlockedIncrement((long volatile *)p); 34 | } 35 | 36 | static inline uint32_t atomic_dec(volatile uint32_t *p) { 37 | return InterlockedDecrement((long volatile *)p); 38 | } 39 | 40 | #else 41 | 42 | static inline size_t atomic_load(const volatile size_t *p) { 43 | #ifdef __ATOMIC_RELAXED 44 | return __atomic_load_n(p, __ATOMIC_RELAXED); 45 | #else 46 | return __sync_fetch_and_add((volatile size_t *)p, 0); 47 | #endif 48 | } 49 | 50 | static inline uint32_t atomic_inc(volatile uint32_t *p) { 51 | #ifdef __ATOMIC_RELAXED 52 | return __atomic_add_fetch(p, 1U, __ATOMIC_SEQ_CST); 53 | #else 54 | return __sync_add_and_fetch(p, 1U); 55 | #endif 56 | } 57 | 58 | static inline uint32_t atomic_dec(volatile uint32_t *p) { 59 | #ifdef __ATOMIC_RELAXED 60 | return __atomic_sub_fetch(p, 1U, __ATOMIC_SEQ_CST); 61 | #else 62 | return __sync_sub_and_fetch(p, 1U); 63 | #endif 64 | } 65 | 66 | #endif 67 | 68 | #endif // TREE_SITTER_ATOMIC_H_ 69 | -------------------------------------------------------------------------------- /bindings/vendor/src/clock.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_CLOCK_H_ 2 | #define TREE_SITTER_CLOCK_H_ 3 | 4 | #include 5 | #include 6 | 7 | typedef uint64_t TSDuration; 8 | 9 | #ifdef _WIN32 10 | 11 | // Windows: 12 | // * Represent a time as a performance counter value. 13 | // * Represent a duration as a number of performance counter ticks. 14 | 15 | #include 16 | typedef uint64_t TSClock; 17 | 18 | static inline TSDuration duration_from_micros(uint64_t micros) { 19 | LARGE_INTEGER frequency; 20 | QueryPerformanceFrequency(&frequency); 21 | return micros * (uint64_t)frequency.QuadPart / 1000000; 22 | } 23 | 24 | static inline uint64_t duration_to_micros(TSDuration self) { 25 | LARGE_INTEGER frequency; 26 | QueryPerformanceFrequency(&frequency); 27 | return self * 1000000 / (uint64_t)frequency.QuadPart; 28 | } 29 | 30 | static inline TSClock clock_null(void) { 31 | return 0; 32 | } 33 | 34 | static inline TSClock clock_now(void) { 35 | LARGE_INTEGER result; 36 | QueryPerformanceCounter(&result); 37 | return (uint64_t)result.QuadPart; 38 | } 39 | 40 | static inline TSClock clock_after(TSClock base, TSDuration duration) { 41 | return base + duration; 42 | } 43 | 44 | static inline bool clock_is_null(TSClock self) { 45 | return !self; 46 | } 47 | 48 | static inline bool clock_is_gt(TSClock self, TSClock other) { 49 | return self > other; 50 | } 51 | 52 | #elif defined(CLOCK_MONOTONIC) 53 | 54 | // POSIX with monotonic clock support (Linux, macOS) 55 | // * Represent a time as a monotonic (seconds, nanoseconds) pair. 56 | // * Represent a duration as a number of microseconds. 57 | // 58 | // On these platforms, parse timeouts will correspond accurately to 59 | // real time, regardless of what other processes are running. 60 | 61 | #include 62 | typedef struct timespec TSClock; 63 | 64 | static inline TSDuration duration_from_micros(uint64_t micros) { 65 | return micros; 66 | } 67 | 68 | static inline uint64_t duration_to_micros(TSDuration self) { 69 | return self; 70 | } 71 | 72 | static inline TSClock clock_now(void) { 73 | TSClock result; 74 | clock_gettime(CLOCK_MONOTONIC, &result); 75 | return result; 76 | } 77 | 78 | static inline TSClock clock_null(void) { 79 | return (TSClock) {0, 0}; 80 | } 81 | 82 | static inline TSClock clock_after(TSClock base, TSDuration duration) { 83 | TSClock result = base; 84 | result.tv_sec += duration / 1000000; 85 | result.tv_nsec += (duration % 1000000) * 1000; 86 | if (result.tv_nsec >= 1000000000) { 87 | result.tv_nsec -= 1000000000; 88 | ++(result.tv_sec); 89 | } 90 | return result; 91 | } 92 | 93 | static inline bool clock_is_null(TSClock self) { 94 | return !self.tv_sec && !self.tv_nsec; 95 | } 96 | 97 | static inline bool clock_is_gt(TSClock self, TSClock other) { 98 | if (self.tv_sec > other.tv_sec) return true; 99 | if (self.tv_sec < other.tv_sec) return false; 100 | return self.tv_nsec > other.tv_nsec; 101 | } 102 | 103 | #else 104 | 105 | // POSIX without monotonic clock support 106 | // * Represent a time as a process clock value. 107 | // * Represent a duration as a number of process clock ticks. 108 | // 109 | // On these platforms, parse timeouts may be affected by other processes, 110 | // which is not ideal, but is better than using a non-monotonic time API 111 | // like `gettimeofday`. 112 | 113 | #include 114 | typedef uint64_t TSClock; 115 | 116 | static inline TSDuration duration_from_micros(uint64_t micros) { 117 | return micros * (uint64_t)CLOCKS_PER_SEC / 1000000; 118 | } 119 | 120 | static inline uint64_t duration_to_micros(TSDuration self) { 121 | return self * 1000000 / (uint64_t)CLOCKS_PER_SEC; 122 | } 123 | 124 | static inline TSClock clock_null(void) { 125 | return 0; 126 | } 127 | 128 | static inline TSClock clock_now(void) { 129 | return (uint64_t)clock(); 130 | } 131 | 132 | static inline TSClock clock_after(TSClock base, TSDuration duration) { 133 | return base + duration; 134 | } 135 | 136 | static inline bool clock_is_null(TSClock self) { 137 | return !self; 138 | } 139 | 140 | static inline bool clock_is_gt(TSClock self, TSClock other) { 141 | return self > other; 142 | } 143 | 144 | #endif 145 | 146 | #endif // TREE_SITTER_CLOCK_H_ 147 | -------------------------------------------------------------------------------- /bindings/vendor/src/error_costs.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_ERROR_COSTS_H_ 2 | #define TREE_SITTER_ERROR_COSTS_H_ 3 | 4 | #define ERROR_STATE 0 5 | #define ERROR_COST_PER_RECOVERY 500 6 | #define ERROR_COST_PER_MISSING_TREE 110 7 | #define ERROR_COST_PER_SKIPPED_TREE 100 8 | #define ERROR_COST_PER_SKIPPED_LINE 30 9 | #define ERROR_COST_PER_SKIPPED_CHAR 1 10 | 11 | #endif 12 | -------------------------------------------------------------------------------- /bindings/vendor/src/get_changed_ranges.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_GET_CHANGED_RANGES_H_ 2 | #define TREE_SITTER_GET_CHANGED_RANGES_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include "./tree_cursor.h" 9 | #include "./subtree.h" 10 | 11 | typedef Array(TSRange) TSRangeArray; 12 | 13 | void ts_range_array_get_changed_ranges( 14 | const TSRange *old_ranges, unsigned old_range_count, 15 | const TSRange *new_ranges, unsigned new_range_count, 16 | TSRangeArray *differences 17 | ); 18 | 19 | bool ts_range_array_intersects( 20 | const TSRangeArray *self, unsigned start_index, 21 | uint32_t start_byte, uint32_t end_byte 22 | ); 23 | 24 | unsigned ts_subtree_get_changed_ranges( 25 | const Subtree *old_tree, const Subtree *new_tree, 26 | TreeCursor *cursor1, TreeCursor *cursor2, 27 | const TSLanguage *language, 28 | const TSRangeArray *included_range_differences, 29 | TSRange **ranges 30 | ); 31 | 32 | #ifdef __cplusplus 33 | } 34 | #endif 35 | 36 | #endif // TREE_SITTER_GET_CHANGED_RANGES_H_ 37 | -------------------------------------------------------------------------------- /bindings/vendor/src/host.h: -------------------------------------------------------------------------------- 1 | 2 | // Determine endian and pointer size based on known defines. 3 | // TS_BIG_ENDIAN and TS_PTR_SIZE can be set as -D compiler arguments 4 | // to override this. 5 | 6 | #if !defined(TS_BIG_ENDIAN) 7 | #if (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) \ 8 | || (defined( __APPLE_CC__) && (defined(__ppc__) || defined(__ppc64__))) 9 | #define TS_BIG_ENDIAN 1 10 | #else 11 | #define TS_BIG_ENDIAN 0 12 | #endif 13 | #endif 14 | 15 | #if !defined(TS_PTR_SIZE) 16 | #if UINTPTR_MAX == 0xFFFFFFFF 17 | #define TS_PTR_SIZE 32 18 | #else 19 | #define TS_PTR_SIZE 64 20 | #endif 21 | #endif 22 | -------------------------------------------------------------------------------- /bindings/vendor/src/length.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_LENGTH_H_ 2 | #define TREE_SITTER_LENGTH_H_ 3 | 4 | #include 5 | #include 6 | #include "./point.h" 7 | #include "tree_sitter/api.h" 8 | 9 | typedef struct { 10 | uint32_t bytes; 11 | TSPoint extent; 12 | } Length; 13 | 14 | static const Length LENGTH_UNDEFINED = {0, {0, 1}}; 15 | static const Length LENGTH_MAX = {UINT32_MAX, {UINT32_MAX, UINT32_MAX}}; 16 | 17 | static inline bool length_is_undefined(Length length) { 18 | return length.bytes == 0 && length.extent.column != 0; 19 | } 20 | 21 | static inline Length length_min(Length len1, Length len2) { 22 | return (len1.bytes < len2.bytes) ? len1 : len2; 23 | } 24 | 25 | static inline Length length_add(Length len1, Length len2) { 26 | Length result; 27 | result.bytes = len1.bytes + len2.bytes; 28 | result.extent = point_add(len1.extent, len2.extent); 29 | return result; 30 | } 31 | 32 | static inline Length length_sub(Length len1, Length len2) { 33 | Length result; 34 | result.bytes = (len1.bytes >= len2.bytes) ? len1.bytes - len2.bytes : 0; 35 | result.extent = point_sub(len1.extent, len2.extent); 36 | return result; 37 | } 38 | 39 | static inline Length length_zero(void) { 40 | Length result = {0, {0, 0}}; 41 | return result; 42 | } 43 | 44 | static inline Length length_saturating_sub(Length len1, Length len2) { 45 | if (len1.bytes > len2.bytes) { 46 | return length_sub(len1, len2); 47 | } else { 48 | return length_zero(); 49 | } 50 | } 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /bindings/vendor/src/lexer.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_LEXER_H_ 2 | #define TREE_SITTER_LEXER_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include "./length.h" 9 | #include "./subtree.h" 10 | #include "tree_sitter/api.h" 11 | #include "./parser.h" 12 | 13 | typedef struct { 14 | uint32_t value; 15 | bool valid; 16 | } ColumnData; 17 | 18 | typedef struct { 19 | TSLexer data; 20 | Length current_position; 21 | Length token_start_position; 22 | Length token_end_position; 23 | 24 | TSRange *included_ranges; 25 | const char *chunk; 26 | TSInput input; 27 | TSLogger logger; 28 | 29 | uint32_t included_range_count; 30 | uint32_t current_included_range_index; 31 | uint32_t chunk_start; 32 | uint32_t chunk_size; 33 | uint32_t lookahead_size; 34 | bool did_get_column; 35 | ColumnData column_data; 36 | 37 | char debug_buffer[TREE_SITTER_SERIALIZATION_BUFFER_SIZE]; 38 | } Lexer; 39 | 40 | void ts_lexer_init(Lexer *self); 41 | void ts_lexer_delete(Lexer *self); 42 | void ts_lexer_set_input(Lexer *self, TSInput input); 43 | void ts_lexer_reset(Lexer *self, Length position); 44 | void ts_lexer_start(Lexer *self); 45 | void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte); 46 | void ts_lexer_mark_end(Lexer *self); 47 | bool ts_lexer_set_included_ranges(Lexer *self, const TSRange *ranges, uint32_t count); 48 | TSRange *ts_lexer_included_ranges(const Lexer *self, uint32_t *count); 49 | 50 | #ifdef __cplusplus 51 | } 52 | #endif 53 | 54 | #endif // TREE_SITTER_LEXER_H_ 55 | -------------------------------------------------------------------------------- /bindings/vendor/src/lib.c: -------------------------------------------------------------------------------- 1 | #include "./alloc.c" 2 | #include "./get_changed_ranges.c" 3 | #include "./language.c" 4 | #include "./lexer.c" 5 | #include "./node.c" 6 | #include "./parser.c" 7 | #include "./query.c" 8 | #include "./stack.c" 9 | #include "./subtree.c" 10 | #include "./tree_cursor.c" 11 | #include "./tree.c" 12 | #include "./wasm_store.c" 13 | -------------------------------------------------------------------------------- /bindings/vendor/src/point.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_POINT_H_ 2 | #define TREE_SITTER_POINT_H_ 3 | 4 | #include "tree_sitter/api.h" 5 | 6 | #define POINT_ZERO ((TSPoint) {0, 0}) 7 | #define POINT_MAX ((TSPoint) {UINT32_MAX, UINT32_MAX}) 8 | 9 | static inline TSPoint point__new(unsigned row, unsigned column) { 10 | TSPoint result = {row, column}; 11 | return result; 12 | } 13 | 14 | static inline TSPoint point_add(TSPoint a, TSPoint b) { 15 | if (b.row > 0) 16 | return point__new(a.row + b.row, b.column); 17 | else 18 | return point__new(a.row, a.column + b.column); 19 | } 20 | 21 | static inline TSPoint point_sub(TSPoint a, TSPoint b) { 22 | if (a.row > b.row) 23 | return point__new(a.row - b.row, a.column); 24 | else 25 | return point__new(0, (a.column >= b.column) ? a.column - b.column : 0); 26 | } 27 | 28 | static inline bool point_lte(TSPoint a, TSPoint b) { 29 | return (a.row < b.row) || (a.row == b.row && a.column <= b.column); 30 | } 31 | 32 | static inline bool point_lt(TSPoint a, TSPoint b) { 33 | return (a.row < b.row) || (a.row == b.row && a.column < b.column); 34 | } 35 | 36 | static inline bool point_gt(TSPoint a, TSPoint b) { 37 | return (a.row > b.row) || (a.row == b.row && a.column > b.column); 38 | } 39 | 40 | static inline bool point_gte(TSPoint a, TSPoint b) { 41 | return (a.row > b.row) || (a.row == b.row && a.column >= b.column); 42 | } 43 | 44 | static inline bool point_eq(TSPoint a, TSPoint b) { 45 | return a.row == b.row && a.column == b.column; 46 | } 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /bindings/vendor/src/portable/endian.h: -------------------------------------------------------------------------------- 1 | // "License": Public Domain 2 | // I, Mathias Panzenböck, place this file hereby into the public domain. Use it at your own risk for whatever you like. 3 | // In case there are jurisdictions that don't support putting things in the public domain you can also consider it to 4 | // be "dual licensed" under the BSD, MIT and Apache licenses, if you want to. This code is trivial anyway. Consider it 5 | // an example on how to get the endian conversion functions on different platforms. 6 | 7 | // updates from https://github.com/mikepb/endian.h/issues/4 8 | 9 | #ifndef ENDIAN_H 10 | #define ENDIAN_H 11 | 12 | #if (defined(_WIN16) || defined(_WIN32) || defined(_WIN64)) && !defined(__WINDOWS__) 13 | 14 | # define __WINDOWS__ 15 | 16 | #endif 17 | 18 | #if defined(HAVE_ENDIAN_H) || \ 19 | defined(__linux__) || \ 20 | defined(__GNU__) || \ 21 | defined(__illumos__) || \ 22 | defined(__NetBSD__) || \ 23 | defined(__OpenBSD__) || \ 24 | defined(__CYGWIN__) || \ 25 | defined(__MSYS__) || \ 26 | defined(__EMSCRIPTEN__) || \ 27 | defined(__wasi__) 28 | 29 | #if defined(__NetBSD__) 30 | #define _NETBSD_SOURCE 1 31 | #endif 32 | 33 | # include 34 | 35 | #elif defined(HAVE_SYS_ENDIAN_H) || \ 36 | defined(__FreeBSD__) || \ 37 | defined(__DragonFly__) 38 | 39 | # include 40 | 41 | #elif defined(__APPLE__) 42 | # define __BYTE_ORDER BYTE_ORDER 43 | # define __BIG_ENDIAN BIG_ENDIAN 44 | # define __LITTLE_ENDIAN LITTLE_ENDIAN 45 | # define __PDP_ENDIAN PDP_ENDIAN 46 | 47 | # if !defined(_POSIX_C_SOURCE) 48 | # include 49 | 50 | # define htobe16(x) OSSwapHostToBigInt16(x) 51 | # define htole16(x) OSSwapHostToLittleInt16(x) 52 | # define be16toh(x) OSSwapBigToHostInt16(x) 53 | # define le16toh(x) OSSwapLittleToHostInt16(x) 54 | 55 | # define htobe32(x) OSSwapHostToBigInt32(x) 56 | # define htole32(x) OSSwapHostToLittleInt32(x) 57 | # define be32toh(x) OSSwapBigToHostInt32(x) 58 | # define le32toh(x) OSSwapLittleToHostInt32(x) 59 | 60 | # define htobe64(x) OSSwapHostToBigInt64(x) 61 | # define htole64(x) OSSwapHostToLittleInt64(x) 62 | # define be64toh(x) OSSwapBigToHostInt64(x) 63 | # define le64toh(x) OSSwapLittleToHostInt64(x) 64 | # else 65 | # if BYTE_ORDER == LITTLE_ENDIAN 66 | # define htobe16(x) __builtin_bswap16(x) 67 | # define htole16(x) (x) 68 | # define be16toh(x) __builtin_bswap16(x) 69 | # define le16toh(x) (x) 70 | 71 | # define htobe32(x) __builtin_bswap32(x) 72 | # define htole32(x) (x) 73 | # define be32toh(x) __builtin_bswap32(x) 74 | # define le32toh(x) (x) 75 | 76 | # define htobe64(x) __builtin_bswap64(x) 77 | # define htole64(x) (x) 78 | # define be64toh(x) __builtin_bswap64(x) 79 | # define le64toh(x) (x) 80 | # elif BYTE_ORDER == BIG_ENDIAN 81 | # define htobe16(x) (x) 82 | # define htole16(x) __builtin_bswap16(x) 83 | # define be16toh(x) (x) 84 | # define le16toh(x) __builtin_bswap16(x) 85 | 86 | # define htobe32(x) (x) 87 | # define htole32(x) __builtin_bswap32(x) 88 | # define be32toh(x) (x) 89 | # define le32toh(x) __builtin_bswap32(x) 90 | 91 | # define htobe64(x) (x) 92 | # define htole64(x) __builtin_bswap64(x) 93 | # define be64toh(x) (x) 94 | # define le64toh(x) __builtin_bswap64(x) 95 | # else 96 | # error byte order not supported 97 | # endif 98 | # endif 99 | 100 | #elif defined(__WINDOWS__) 101 | 102 | # if defined(_MSC_VER) && !defined(__clang__) 103 | # include 104 | # define B_SWAP_16(x) _byteswap_ushort(x) 105 | # define B_SWAP_32(x) _byteswap_ulong(x) 106 | # define B_SWAP_64(x) _byteswap_uint64(x) 107 | # else 108 | # define B_SWAP_16(x) __builtin_bswap16(x) 109 | # define B_SWAP_32(x) __builtin_bswap32(x) 110 | # define B_SWAP_64(x) __builtin_bswap64(x) 111 | # endif 112 | 113 | # if defined(__MINGW32__) || defined(HAVE_SYS_PARAM_H) 114 | # include 115 | # endif 116 | 117 | # ifndef BIG_ENDIAN 118 | # ifdef __BIG_ENDIAN 119 | # define BIG_ENDIAN __BIG_ENDIAN 120 | # elif defined(__ORDER_BIG_ENDIAN__) 121 | # define BIG_ENDIAN __ORDER_BIG_ENDIAN__ 122 | # else 123 | # define BIG_ENDIAN 4321 124 | # endif 125 | # endif 126 | 127 | # ifndef LITTLE_ENDIAN 128 | # ifdef __LITTLE_ENDIAN 129 | # define LITTLE_ENDIAN __LITTLE_ENDIAN 130 | # elif defined(__ORDER_LITTLE_ENDIAN__) 131 | # define LITTLE_ENDIAN __ORDER_LITTLE_ENDIAN__ 132 | # else 133 | # define LITTLE_ENDIAN 1234 134 | # endif 135 | # endif 136 | 137 | # ifndef BYTE_ORDER 138 | # ifdef __BYTE_ORDER 139 | # define BYTE_ORDER __BYTE_ORDER 140 | # elif defined(__BYTE_ORDER__) 141 | # define BYTE_ORDER __BYTE_ORDER__ 142 | # else 143 | /* assume LE on Windows if nothing was defined */ 144 | # define BYTE_ORDER LITTLE_ENDIAN 145 | # endif 146 | # endif 147 | 148 | # if BYTE_ORDER == LITTLE_ENDIAN 149 | 150 | # define htobe16(x) B_SWAP_16(x) 151 | # define htole16(x) (x) 152 | # define be16toh(x) B_SWAP_16(x) 153 | # define le16toh(x) (x) 154 | 155 | # define htobe32(x) B_SWAP_32(x) 156 | # define htole32(x) (x) 157 | # define be32toh(x) B_SWAP_32(x) 158 | # define le32toh(x) (x) 159 | 160 | # define htobe64(x) B_SWAP_64(x) 161 | # define htole64(x) (x) 162 | # define be64toh(x) B_SWAP_64(x) 163 | # define le64toh(x) (x) 164 | 165 | # elif BYTE_ORDER == BIG_ENDIAN 166 | 167 | # define htobe16(x) (x) 168 | # define htole16(x) B_SWAP_16(x) 169 | # define be16toh(x) (x) 170 | # define le16toh(x) B_SWAP_16(x) 171 | 172 | # define htobe32(x) (x) 173 | # define htole32(x) B_SWAP_32(x) 174 | # define be32toh(x) (x) 175 | # define le32toh(x) B_SWAP_32(x) 176 | 177 | # define htobe64(x) (x) 178 | # define htole64(x) B_SWAP_64(x) 179 | # define be64toh(x) (x) 180 | # define le64toh(x) B_SWAP_64(x) 181 | 182 | # else 183 | 184 | # error byte order not supported 185 | 186 | # endif 187 | 188 | #elif defined(__QNXNTO__) 189 | 190 | # include 191 | 192 | # define __LITTLE_ENDIAN 1234 193 | # define __BIG_ENDIAN 4321 194 | # define __PDP_ENDIAN 3412 195 | 196 | # if defined(__BIGENDIAN__) 197 | 198 | # define __BYTE_ORDER __BIG_ENDIAN 199 | 200 | # define htobe16(x) (x) 201 | # define htobe32(x) (x) 202 | # define htobe64(x) (x) 203 | 204 | # define htole16(x) ENDIAN_SWAP16(x) 205 | # define htole32(x) ENDIAN_SWAP32(x) 206 | # define htole64(x) ENDIAN_SWAP64(x) 207 | 208 | # elif defined(__LITTLEENDIAN__) 209 | 210 | # define __BYTE_ORDER __LITTLE_ENDIAN 211 | 212 | # define htole16(x) (x) 213 | # define htole32(x) (x) 214 | # define htole64(x) (x) 215 | 216 | # define htobe16(x) ENDIAN_SWAP16(x) 217 | # define htobe32(x) ENDIAN_SWAP32(x) 218 | # define htobe64(x) ENDIAN_SWAP64(x) 219 | 220 | # else 221 | 222 | # error byte order not supported 223 | 224 | # endif 225 | 226 | # define be16toh(x) ENDIAN_BE16(x) 227 | # define be32toh(x) ENDIAN_BE32(x) 228 | # define be64toh(x) ENDIAN_BE64(x) 229 | # define le16toh(x) ENDIAN_LE16(x) 230 | # define le32toh(x) ENDIAN_LE32(x) 231 | # define le64toh(x) ENDIAN_LE64(x) 232 | 233 | #else 234 | 235 | # error platform not supported 236 | 237 | #endif 238 | 239 | #endif 240 | -------------------------------------------------------------------------------- /bindings/vendor/src/reduce_action.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_REDUCE_ACTION_H_ 2 | #define TREE_SITTER_REDUCE_ACTION_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include "./array.h" 9 | #include "tree_sitter/api.h" 10 | 11 | typedef struct { 12 | uint32_t count; 13 | TSSymbol symbol; 14 | int dynamic_precedence; 15 | unsigned short production_id; 16 | } ReduceAction; 17 | 18 | typedef Array(ReduceAction) ReduceActionSet; 19 | 20 | static inline void ts_reduce_action_set_add(ReduceActionSet *self, 21 | ReduceAction new_action) { 22 | for (uint32_t i = 0; i < self->size; i++) { 23 | ReduceAction action = self->contents[i]; 24 | if (action.symbol == new_action.symbol && action.count == new_action.count) 25 | return; 26 | } 27 | array_push(self, new_action); 28 | } 29 | 30 | #ifdef __cplusplus 31 | } 32 | #endif 33 | 34 | #endif // TREE_SITTER_REDUCE_ACTION_H_ 35 | -------------------------------------------------------------------------------- /bindings/vendor/src/reusable_node.h: -------------------------------------------------------------------------------- 1 | #include "./subtree.h" 2 | 3 | typedef struct { 4 | Subtree tree; 5 | uint32_t child_index; 6 | uint32_t byte_offset; 7 | } StackEntry; 8 | 9 | typedef struct { 10 | Array(StackEntry) stack; 11 | Subtree last_external_token; 12 | } ReusableNode; 13 | 14 | static inline ReusableNode reusable_node_new(void) { 15 | return (ReusableNode) {array_new(), NULL_SUBTREE}; 16 | } 17 | 18 | static inline void reusable_node_clear(ReusableNode *self) { 19 | array_clear(&self->stack); 20 | self->last_external_token = NULL_SUBTREE; 21 | } 22 | 23 | static inline Subtree reusable_node_tree(ReusableNode *self) { 24 | return self->stack.size > 0 25 | ? self->stack.contents[self->stack.size - 1].tree 26 | : NULL_SUBTREE; 27 | } 28 | 29 | static inline uint32_t reusable_node_byte_offset(ReusableNode *self) { 30 | return self->stack.size > 0 31 | ? self->stack.contents[self->stack.size - 1].byte_offset 32 | : UINT32_MAX; 33 | } 34 | 35 | static inline void reusable_node_delete(ReusableNode *self) { 36 | array_delete(&self->stack); 37 | } 38 | 39 | static inline void reusable_node_advance(ReusableNode *self) { 40 | StackEntry last_entry = *array_back(&self->stack); 41 | uint32_t byte_offset = last_entry.byte_offset + ts_subtree_total_bytes(last_entry.tree); 42 | if (ts_subtree_has_external_tokens(last_entry.tree)) { 43 | self->last_external_token = ts_subtree_last_external_token(last_entry.tree); 44 | } 45 | 46 | Subtree tree; 47 | uint32_t next_index; 48 | do { 49 | StackEntry popped_entry = array_pop(&self->stack); 50 | next_index = popped_entry.child_index + 1; 51 | if (self->stack.size == 0) return; 52 | tree = array_back(&self->stack)->tree; 53 | } while (ts_subtree_child_count(tree) <= next_index); 54 | 55 | array_push(&self->stack, ((StackEntry) { 56 | .tree = ts_subtree_children(tree)[next_index], 57 | .child_index = next_index, 58 | .byte_offset = byte_offset, 59 | })); 60 | } 61 | 62 | static inline bool reusable_node_descend(ReusableNode *self) { 63 | StackEntry last_entry = *array_back(&self->stack); 64 | if (ts_subtree_child_count(last_entry.tree) > 0) { 65 | array_push(&self->stack, ((StackEntry) { 66 | .tree = ts_subtree_children(last_entry.tree)[0], 67 | .child_index = 0, 68 | .byte_offset = last_entry.byte_offset, 69 | })); 70 | return true; 71 | } else { 72 | return false; 73 | } 74 | } 75 | 76 | static inline void reusable_node_advance_past_leaf(ReusableNode *self) { 77 | while (reusable_node_descend(self)) {} 78 | reusable_node_advance(self); 79 | } 80 | 81 | static inline void reusable_node_reset(ReusableNode *self, Subtree tree) { 82 | reusable_node_clear(self); 83 | array_push(&self->stack, ((StackEntry) { 84 | .tree = tree, 85 | .child_index = 0, 86 | .byte_offset = 0, 87 | })); 88 | 89 | // Never reuse the root node, because it has a non-standard internal structure 90 | // due to transformations that are applied when it is accepted: adding the EOF 91 | // child and any extra children. 92 | if (!reusable_node_descend(self)) { 93 | reusable_node_clear(self); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /bindings/vendor/src/stack.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_PARSE_STACK_H_ 2 | #define TREE_SITTER_PARSE_STACK_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include "./array.h" 9 | #include "./subtree.h" 10 | #include 11 | 12 | typedef struct Stack Stack; 13 | 14 | typedef unsigned StackVersion; 15 | #define STACK_VERSION_NONE ((StackVersion)-1) 16 | 17 | typedef struct { 18 | SubtreeArray subtrees; 19 | StackVersion version; 20 | } StackSlice; 21 | typedef Array(StackSlice) StackSliceArray; 22 | 23 | typedef struct { 24 | Length position; 25 | unsigned depth; 26 | TSStateId state; 27 | } StackSummaryEntry; 28 | typedef Array(StackSummaryEntry) StackSummary; 29 | 30 | // Create a stack. 31 | Stack *ts_stack_new(SubtreePool *subtree_pool); 32 | 33 | // Release the memory reserved for a given stack. 34 | void ts_stack_delete(Stack *self); 35 | 36 | // Get the stack's current number of versions. 37 | uint32_t ts_stack_version_count(const Stack *self); 38 | 39 | // Get the stack's current number of halted versions. 40 | uint32_t ts_stack_halted_version_count(Stack *self); 41 | 42 | // Get the state at the top of the given version of the stack. If the stack is 43 | // empty, this returns the initial state, 0. 44 | TSStateId ts_stack_state(const Stack *self, StackVersion version); 45 | 46 | // Get the last external token associated with a given version of the stack. 47 | Subtree ts_stack_last_external_token(const Stack *self, StackVersion version); 48 | 49 | // Set the last external token associated with a given version of the stack. 50 | void ts_stack_set_last_external_token(Stack *self, StackVersion version, Subtree token); 51 | 52 | // Get the position of the given version of the stack within the document. 53 | Length ts_stack_position(const Stack *, StackVersion); 54 | 55 | // Push a tree and state onto the given version of the stack. 56 | // 57 | // This transfers ownership of the tree to the Stack. Callers that 58 | // need to retain ownership of the tree for their own purposes should 59 | // first retain the tree. 60 | void ts_stack_push(Stack *self, StackVersion version, Subtree subtree, bool pending, TSStateId state); 61 | 62 | // Pop the given number of entries from the given version of the stack. This 63 | // operation can increase the number of stack versions by revealing multiple 64 | // versions which had previously been merged. It returns an array that 65 | // specifies the index of each revealed version and the trees that were 66 | // removed from that version. 67 | StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, uint32_t count); 68 | 69 | // Remove an error at the top of the given version of the stack. 70 | SubtreeArray ts_stack_pop_error(Stack *self, StackVersion version); 71 | 72 | // Remove any pending trees from the top of the given version of the stack. 73 | StackSliceArray ts_stack_pop_pending(Stack *self, StackVersion version); 74 | 75 | // Remove all trees from the given version of the stack. 76 | StackSliceArray ts_stack_pop_all(Stack *self, StackVersion version); 77 | 78 | // Get the maximum number of tree nodes reachable from this version of the stack 79 | // since the last error was detected. 80 | unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version); 81 | 82 | int ts_stack_dynamic_precedence(Stack *self, StackVersion version); 83 | 84 | bool ts_stack_has_advanced_since_error(const Stack *self, StackVersion version); 85 | 86 | // Compute a summary of all the parse states near the top of the given 87 | // version of the stack and store the summary for later retrieval. 88 | void ts_stack_record_summary(Stack *self, StackVersion version, unsigned max_depth); 89 | 90 | // Retrieve a summary of all the parse states near the top of the 91 | // given version of the stack. 92 | StackSummary *ts_stack_get_summary(Stack *self, StackVersion version); 93 | 94 | // Get the total cost of all errors on the given version of the stack. 95 | unsigned ts_stack_error_cost(const Stack *self, StackVersion version); 96 | 97 | // Merge the given two stack versions if possible, returning true 98 | // if they were successfully merged and false otherwise. 99 | bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2); 100 | 101 | // Determine whether the given two stack versions can be merged. 102 | bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version2); 103 | 104 | Subtree ts_stack_resume(Stack *self, StackVersion version); 105 | 106 | void ts_stack_pause(Stack *self, StackVersion version, Subtree lookahead); 107 | 108 | void ts_stack_halt(Stack *self, StackVersion version); 109 | 110 | bool ts_stack_is_active(const Stack *self, StackVersion version); 111 | 112 | bool ts_stack_is_paused(const Stack *self, StackVersion version); 113 | 114 | bool ts_stack_is_halted(const Stack *self, StackVersion version); 115 | 116 | void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2); 117 | 118 | void ts_stack_swap_versions(Stack *, StackVersion v1, StackVersion v2); 119 | 120 | StackVersion ts_stack_copy_version(Stack *self, StackVersion version); 121 | 122 | // Remove the given version from the stack. 123 | void ts_stack_remove_version(Stack *self, StackVersion version); 124 | 125 | void ts_stack_clear(Stack *self); 126 | 127 | bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f); 128 | 129 | #ifdef __cplusplus 130 | } 131 | #endif 132 | 133 | #endif // TREE_SITTER_PARSE_STACK_H_ 134 | -------------------------------------------------------------------------------- /bindings/vendor/src/tree.c: -------------------------------------------------------------------------------- 1 | #include "tree_sitter/api.h" 2 | #include "./array.h" 3 | #include "./get_changed_ranges.h" 4 | #include "./length.h" 5 | #include "./subtree.h" 6 | #include "./tree_cursor.h" 7 | #include "./tree.h" 8 | 9 | TSTree *ts_tree_new( 10 | Subtree root, const TSLanguage *language, 11 | const TSRange *included_ranges, unsigned included_range_count 12 | ) { 13 | TSTree *result = ts_malloc(sizeof(TSTree)); 14 | result->root = root; 15 | result->language = ts_language_copy(language); 16 | result->included_ranges = ts_calloc(included_range_count, sizeof(TSRange)); 17 | memcpy(result->included_ranges, included_ranges, included_range_count * sizeof(TSRange)); 18 | result->included_range_count = included_range_count; 19 | return result; 20 | } 21 | 22 | TSTree *ts_tree_copy(const TSTree *self) { 23 | ts_subtree_retain(self->root); 24 | return ts_tree_new(self->root, self->language, self->included_ranges, self->included_range_count); 25 | } 26 | 27 | void ts_tree_delete(TSTree *self) { 28 | if (!self) return; 29 | 30 | SubtreePool pool = ts_subtree_pool_new(0); 31 | ts_subtree_release(&pool, self->root); 32 | ts_subtree_pool_delete(&pool); 33 | ts_language_delete(self->language); 34 | ts_free(self->included_ranges); 35 | ts_free(self); 36 | } 37 | 38 | TSNode ts_tree_root_node(const TSTree *self) { 39 | return ts_node_new(self, &self->root, ts_subtree_padding(self->root), 0); 40 | } 41 | 42 | TSNode ts_tree_root_node_with_offset( 43 | const TSTree *self, 44 | uint32_t offset_bytes, 45 | TSPoint offset_extent 46 | ) { 47 | Length offset = {offset_bytes, offset_extent}; 48 | return ts_node_new(self, &self->root, length_add(offset, ts_subtree_padding(self->root)), 0); 49 | } 50 | 51 | const TSLanguage *ts_tree_language(const TSTree *self) { 52 | return self->language; 53 | } 54 | 55 | void ts_tree_edit(TSTree *self, const TSInputEdit *edit) { 56 | for (unsigned i = 0; i < self->included_range_count; i++) { 57 | TSRange *range = &self->included_ranges[i]; 58 | if (range->end_byte >= edit->old_end_byte) { 59 | if (range->end_byte != UINT32_MAX) { 60 | range->end_byte = edit->new_end_byte + (range->end_byte - edit->old_end_byte); 61 | range->end_point = point_add( 62 | edit->new_end_point, 63 | point_sub(range->end_point, edit->old_end_point) 64 | ); 65 | if (range->end_byte < edit->new_end_byte) { 66 | range->end_byte = UINT32_MAX; 67 | range->end_point = POINT_MAX; 68 | } 69 | } 70 | } else if (range->end_byte > edit->start_byte) { 71 | range->end_byte = edit->start_byte; 72 | range->end_point = edit->start_point; 73 | } 74 | if (range->start_byte >= edit->old_end_byte) { 75 | range->start_byte = edit->new_end_byte + (range->start_byte - edit->old_end_byte); 76 | range->start_point = point_add( 77 | edit->new_end_point, 78 | point_sub(range->start_point, edit->old_end_point) 79 | ); 80 | if (range->start_byte < edit->new_end_byte) { 81 | range->start_byte = UINT32_MAX; 82 | range->start_point = POINT_MAX; 83 | } 84 | } else if (range->start_byte > edit->start_byte) { 85 | range->start_byte = edit->start_byte; 86 | range->start_point = edit->start_point; 87 | } 88 | } 89 | 90 | SubtreePool pool = ts_subtree_pool_new(0); 91 | self->root = ts_subtree_edit(self->root, edit, &pool); 92 | ts_subtree_pool_delete(&pool); 93 | } 94 | 95 | TSRange *ts_tree_included_ranges(const TSTree *self, uint32_t *length) { 96 | *length = self->included_range_count; 97 | TSRange *ranges = ts_calloc(self->included_range_count, sizeof(TSRange)); 98 | memcpy(ranges, self->included_ranges, self->included_range_count * sizeof(TSRange)); 99 | return ranges; 100 | } 101 | 102 | TSRange *ts_tree_get_changed_ranges(const TSTree *old_tree, const TSTree *new_tree, uint32_t *length) { 103 | TreeCursor cursor1 = {NULL, array_new(), 0}; 104 | TreeCursor cursor2 = {NULL, array_new(), 0}; 105 | ts_tree_cursor_init(&cursor1, ts_tree_root_node(old_tree)); 106 | ts_tree_cursor_init(&cursor2, ts_tree_root_node(new_tree)); 107 | 108 | TSRangeArray included_range_differences = array_new(); 109 | ts_range_array_get_changed_ranges( 110 | old_tree->included_ranges, old_tree->included_range_count, 111 | new_tree->included_ranges, new_tree->included_range_count, 112 | &included_range_differences 113 | ); 114 | 115 | TSRange *result; 116 | *length = ts_subtree_get_changed_ranges( 117 | &old_tree->root, &new_tree->root, &cursor1, &cursor2, 118 | old_tree->language, &included_range_differences, &result 119 | ); 120 | 121 | array_delete(&included_range_differences); 122 | array_delete(&cursor1.stack); 123 | array_delete(&cursor2.stack); 124 | return result; 125 | } 126 | 127 | #ifdef _WIN32 128 | 129 | #include 130 | #include 131 | 132 | int _ts_dup(HANDLE handle) { 133 | HANDLE dup_handle; 134 | if (!DuplicateHandle( 135 | GetCurrentProcess(), handle, 136 | GetCurrentProcess(), &dup_handle, 137 | 0, FALSE, DUPLICATE_SAME_ACCESS 138 | )) return -1; 139 | 140 | return _open_osfhandle((intptr_t)dup_handle, 0); 141 | } 142 | 143 | void ts_tree_print_dot_graph(const TSTree *self, int fd) { 144 | FILE *file = _fdopen(_ts_dup((HANDLE)_get_osfhandle(fd)), "a"); 145 | ts_subtree_print_dot_graph(self->root, self->language, file); 146 | fclose(file); 147 | } 148 | 149 | #elif !defined(__wasi__) // WASI doesn't support dup 150 | 151 | #include 152 | 153 | int _ts_dup(int file_descriptor) { 154 | return dup(file_descriptor); 155 | } 156 | 157 | void ts_tree_print_dot_graph(const TSTree *self, int file_descriptor) { 158 | FILE *file = fdopen(_ts_dup(file_descriptor), "a"); 159 | ts_subtree_print_dot_graph(self->root, self->language, file); 160 | fclose(file); 161 | } 162 | 163 | #else 164 | 165 | void ts_tree_print_dot_graph(const TSTree *self, int file_descriptor) { 166 | (void)self; 167 | (void)file_descriptor; 168 | } 169 | 170 | #endif 171 | -------------------------------------------------------------------------------- /bindings/vendor/src/tree.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_TREE_H_ 2 | #define TREE_SITTER_TREE_H_ 3 | 4 | #include "./subtree.h" 5 | 6 | #ifdef __cplusplus 7 | extern "C" { 8 | #endif 9 | 10 | typedef struct { 11 | const Subtree *child; 12 | const Subtree *parent; 13 | Length position; 14 | TSSymbol alias_symbol; 15 | } ParentCacheEntry; 16 | 17 | struct TSTree { 18 | Subtree root; 19 | const TSLanguage *language; 20 | TSRange *included_ranges; 21 | unsigned included_range_count; 22 | }; 23 | 24 | TSTree *ts_tree_new(Subtree root, const TSLanguage *language, const TSRange *included_ranges, unsigned included_range_count); 25 | TSNode ts_node_new(const TSTree *tree, const Subtree *subtree, Length position, TSSymbol alias); 26 | 27 | #ifdef __cplusplus 28 | } 29 | #endif 30 | 31 | #endif // TREE_SITTER_TREE_H_ 32 | -------------------------------------------------------------------------------- /bindings/vendor/src/tree_cursor.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_TREE_CURSOR_H_ 2 | #define TREE_SITTER_TREE_CURSOR_H_ 3 | 4 | #include "./subtree.h" 5 | 6 | typedef struct { 7 | const Subtree *subtree; 8 | Length position; 9 | uint32_t child_index; 10 | uint32_t structural_child_index; 11 | uint32_t descendant_index; 12 | } TreeCursorEntry; 13 | 14 | typedef struct { 15 | const TSTree *tree; 16 | Array(TreeCursorEntry) stack; 17 | TSSymbol root_alias_symbol; 18 | } TreeCursor; 19 | 20 | typedef enum { 21 | TreeCursorStepNone, 22 | TreeCursorStepHidden, 23 | TreeCursorStepVisible, 24 | } TreeCursorStep; 25 | 26 | void ts_tree_cursor_init(TreeCursor *self, TSNode node); 27 | void ts_tree_cursor_current_status( 28 | const TSTreeCursor *_self, 29 | TSFieldId *field_id, 30 | bool *has_later_siblings, 31 | bool *has_later_named_siblings, 32 | bool *can_have_later_siblings_with_this_field, 33 | TSSymbol *supertypes, 34 | unsigned *supertype_count 35 | ); 36 | 37 | TreeCursorStep ts_tree_cursor_goto_first_child_internal(TSTreeCursor *_self); 38 | TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(TSTreeCursor *_self); 39 | 40 | static inline Subtree ts_tree_cursor_current_subtree(const TSTreeCursor *_self) { 41 | const TreeCursor *self = (const TreeCursor *)_self; 42 | TreeCursorEntry *last_entry = array_back(&self->stack); 43 | return *last_entry->subtree; 44 | } 45 | 46 | TSNode ts_tree_cursor_parent_node(const TSTreeCursor *_self); 47 | 48 | #endif // TREE_SITTER_TREE_CURSOR_H_ 49 | -------------------------------------------------------------------------------- /bindings/vendor/src/ts_assert.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_ASSERT_H_ 2 | #define TREE_SITTER_ASSERT_H_ 3 | 4 | #ifdef NDEBUG 5 | #define ts_assert(e) ((void)(e)) 6 | #else 7 | #include 8 | #define ts_assert(e) assert(e) 9 | #endif 10 | 11 | #endif // TREE_SITTER_ASSERT_H_ 12 | -------------------------------------------------------------------------------- /bindings/vendor/src/unicode.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_UNICODE_H_ 2 | #define TREE_SITTER_UNICODE_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include 9 | #include 10 | 11 | #define U_EXPORT 12 | #define U_EXPORT2 13 | #include "unicode/utf8.h" 14 | #include "unicode/utf16.h" 15 | #include "portable/endian.h" 16 | 17 | #define U16_NEXT_LE(s, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \ 18 | (c)=le16toh((s)[(i)++]); \ 19 | if(U16_IS_LEAD(c)) { \ 20 | uint16_t __c2; \ 21 | if((i)!=(length) && U16_IS_TRAIL(__c2=(s)[(i)])) { \ 22 | ++(i); \ 23 | (c)=U16_GET_SUPPLEMENTARY((c), __c2); \ 24 | } \ 25 | } \ 26 | } UPRV_BLOCK_MACRO_END 27 | 28 | #define U16_NEXT_BE(s, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \ 29 | (c)=be16toh((s)[(i)++]); \ 30 | if(U16_IS_LEAD(c)) { \ 31 | uint16_t __c2; \ 32 | if((i)!=(length) && U16_IS_TRAIL(__c2=(s)[(i)])) { \ 33 | ++(i); \ 34 | (c)=U16_GET_SUPPLEMENTARY((c), __c2); \ 35 | } \ 36 | } \ 37 | } UPRV_BLOCK_MACRO_END 38 | 39 | static const int32_t TS_DECODE_ERROR = U_SENTINEL; 40 | 41 | static inline uint32_t ts_decode_utf8( 42 | const uint8_t *string, 43 | uint32_t length, 44 | int32_t *code_point 45 | ) { 46 | uint32_t i = 0; 47 | U8_NEXT(string, i, length, *code_point); 48 | return i; 49 | } 50 | 51 | static inline uint32_t ts_decode_utf16_le( 52 | const uint8_t *string, 53 | uint32_t length, 54 | int32_t *code_point 55 | ) { 56 | uint32_t i = 0; 57 | U16_NEXT_LE(((uint16_t *)string), i, length, *code_point); 58 | return i * 2; 59 | } 60 | 61 | static inline uint32_t ts_decode_utf16_be( 62 | const uint8_t *string, 63 | uint32_t length, 64 | int32_t *code_point 65 | ) { 66 | uint32_t i = 0; 67 | U16_NEXT_BE(((uint16_t *)string), i, length, *code_point); 68 | return i * 2; 69 | } 70 | 71 | #ifdef __cplusplus 72 | } 73 | #endif 74 | 75 | #endif // TREE_SITTER_UNICODE_H_ 76 | -------------------------------------------------------------------------------- /bindings/vendor/src/unicode/ICU_SHA: -------------------------------------------------------------------------------- 1 | 552b01f61127d30d6589aa4bf99468224979b661 2 | -------------------------------------------------------------------------------- /bindings/vendor/src/unicode/README.md: -------------------------------------------------------------------------------- 1 | # ICU Parts 2 | 3 | This directory contains a small subset of files from the Unicode organization's [ICU repository](https://github.com/unicode-org/icu). 4 | 5 | ### License 6 | 7 | The license for these files is contained in the `LICENSE` file within this directory. 8 | 9 | ### Contents 10 | 11 | * Source files taken from the [`icu4c/source/common/unicode`](https://github.com/unicode-org/icu/tree/552b01f61127d30d6589aa4bf99468224979b661/icu4c/source/common/unicode) directory: 12 | * `utf8.h` 13 | * `utf16.h` 14 | * `umachine.h` 15 | * Empty source files that are referenced by the above source files, but whose original contents in `libicu` are not needed: 16 | * `ptypes.h` 17 | * `urename.h` 18 | * `utf.h` 19 | * `ICU_SHA` - File containing the Git SHA of the commit in the `icu` repository from which the files were obtained. 20 | * `LICENSE` - The license file from the [`icu4c`](https://github.com/unicode-org/icu/tree/552b01f61127d30d6589aa4bf99468224979b661/icu4c) directory of the `icu` repository. 21 | * `README.md` - This text file. 22 | 23 | ### Updating ICU 24 | 25 | To incorporate changes from the upstream `icu` repository: 26 | 27 | * Update `ICU_SHA` with the new Git SHA. 28 | * Update `LICENSE` with the license text from the directory mentioned above. 29 | * Update `utf8.h`, `utf16.h`, and `umachine.h` with their new contents in the `icu` repository. 30 | -------------------------------------------------------------------------------- /bindings/vendor/src/unicode/ptypes.h: -------------------------------------------------------------------------------- 1 | // This file must exist in order for `utf8.h` and `utf16.h` to be used. 2 | -------------------------------------------------------------------------------- /bindings/vendor/src/unicode/urename.h: -------------------------------------------------------------------------------- 1 | // This file must exist in order for `utf8.h` and `utf16.h` to be used. 2 | -------------------------------------------------------------------------------- /bindings/vendor/src/unicode/utf.h: -------------------------------------------------------------------------------- 1 | // This file must exist in order for `utf8.h` and `utf16.h` to be used. 2 | -------------------------------------------------------------------------------- /bindings/vendor/src/wasm/stdlib-symbols.txt: -------------------------------------------------------------------------------- 1 | "calloc", 2 | "free", 3 | "iswalnum", 4 | "iswalpha", 5 | "iswblank", 6 | "iswdigit", 7 | "iswlower", 8 | "iswspace", 9 | "iswupper", 10 | "iswxdigit", 11 | "malloc", 12 | "memchr", 13 | "memcmp", 14 | "memcpy", 15 | "memmove", 16 | "memset", 17 | "realloc", 18 | "strcmp", 19 | "strlen", 20 | "strncat", 21 | "strncmp", 22 | "strncpy", 23 | "towlower", 24 | "towupper", 25 | -------------------------------------------------------------------------------- /bindings/vendor/src/wasm/stdlib.c: -------------------------------------------------------------------------------- 1 | // This file implements a very simple allocator for external scanners running 2 | // in WASM. Allocation is just bumping a static pointer and growing the heap 3 | // as needed, and freeing is mostly a noop. But in the special case of freeing 4 | // the last-allocated pointer, we'll reuse that pointer again. 5 | 6 | #ifdef TREE_SITTER_FEATURE_WASM 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | extern void tree_sitter_debug_message(const char *, size_t); 14 | 15 | #define PAGESIZE 0x10000 16 | #define MAX_HEAP_SIZE (4 * 1024 * 1024) 17 | 18 | typedef struct { 19 | size_t size; 20 | char data[0]; 21 | } Region; 22 | 23 | static Region *heap_end = NULL; 24 | static Region *heap_start = NULL; 25 | static Region *next = NULL; 26 | 27 | // Get the region metadata for the given heap pointer. 28 | static inline Region *region_for_ptr(void *ptr) { 29 | return ((Region *)ptr) - 1; 30 | } 31 | 32 | // Get the location of the next region after the given region, 33 | // if the given region had the given size. 34 | static inline Region *region_after(Region *self, size_t len) { 35 | char *address = self->data + len; 36 | char *aligned = (char *)((uintptr_t)(address + 3) & ~0x3); 37 | return (Region *)aligned; 38 | } 39 | 40 | static void *get_heap_end() { 41 | return (void *)(__builtin_wasm_memory_size(0) * PAGESIZE); 42 | } 43 | 44 | static int grow_heap(size_t size) { 45 | size_t new_page_count = ((size - 1) / PAGESIZE) + 1; 46 | return __builtin_wasm_memory_grow(0, new_page_count) != SIZE_MAX; 47 | } 48 | 49 | // Clear out the heap, and move it to the given address. 50 | void reset_heap(void *new_heap_start) { 51 | heap_start = new_heap_start; 52 | next = new_heap_start; 53 | heap_end = get_heap_end(); 54 | } 55 | 56 | void *malloc(size_t size) { 57 | Region *region_end = region_after(next, size); 58 | 59 | if (region_end > heap_end) { 60 | if ((char *)region_end - (char *)heap_start > MAX_HEAP_SIZE) { 61 | return NULL; 62 | } 63 | if (!grow_heap(size)) return NULL; 64 | heap_end = get_heap_end(); 65 | } 66 | 67 | void *result = &next->data; 68 | next->size = size; 69 | next = region_end; 70 | 71 | return result; 72 | } 73 | 74 | void free(void *ptr) { 75 | if (ptr == NULL) return; 76 | 77 | Region *region = region_for_ptr(ptr); 78 | Region *region_end = region_after(region, region->size); 79 | 80 | // When freeing the last allocated pointer, re-use that 81 | // pointer for the next allocation. 82 | if (region_end == next) { 83 | next = region; 84 | } 85 | } 86 | 87 | void *calloc(size_t count, size_t size) { 88 | void *result = malloc(count * size); 89 | memset(result, 0, count * size); 90 | return result; 91 | } 92 | 93 | void *realloc(void *ptr, size_t new_size) { 94 | if (ptr == NULL) { 95 | return malloc(new_size); 96 | } 97 | 98 | Region *region = region_for_ptr(ptr); 99 | Region *region_end = region_after(region, region->size); 100 | 101 | // When reallocating the last allocated region, return 102 | // the same pointer, and skip copying the data. 103 | if (region_end == next) { 104 | next = region; 105 | return malloc(new_size); 106 | } 107 | 108 | void *result = malloc(new_size); 109 | memcpy(result, ®ion->data, region->size); 110 | return result; 111 | } 112 | 113 | #endif 114 | -------------------------------------------------------------------------------- /bindings/vendor/src/wasm_store.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_WASM_H_ 2 | #define TREE_SITTER_WASM_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include "tree_sitter/api.h" 9 | #include "./parser.h" 10 | 11 | bool ts_wasm_store_start(TSWasmStore *self, TSLexer *lexer, const TSLanguage *language); 12 | void ts_wasm_store_reset(TSWasmStore *self); 13 | bool ts_wasm_store_has_error(const TSWasmStore *self); 14 | 15 | bool ts_wasm_store_call_lex_main(TSWasmStore *self, TSStateId state); 16 | bool ts_wasm_store_call_lex_keyword(TSWasmStore *self, TSStateId state); 17 | 18 | uint32_t ts_wasm_store_call_scanner_create(TSWasmStore *self); 19 | void ts_wasm_store_call_scanner_destroy(TSWasmStore *self, uint32_t scanner_address); 20 | bool ts_wasm_store_call_scanner_scan(TSWasmStore *self, uint32_t scanner_address, uint32_t valid_tokens_ix); 21 | uint32_t ts_wasm_store_call_scanner_serialize(TSWasmStore *self, uint32_t scanner_address, char *buffer); 22 | void ts_wasm_store_call_scanner_deserialize(TSWasmStore *self, uint32_t scanner, const char *buffer, unsigned length); 23 | 24 | void ts_wasm_language_retain(const TSLanguage *self); 25 | void ts_wasm_language_release(const TSLanguage *self); 26 | 27 | #ifdef __cplusplus 28 | } 29 | #endif 30 | 31 | #endif // TREE_SITTER_WASM_H_ 32 | -------------------------------------------------------------------------------- /cli/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "skidder-cli" 3 | version = "0.1.0" 4 | edition = "2021" 5 | description = "A package manager for tree-sitter" 6 | authors = ["Pascal Kuthe "] 7 | license = "MPL-2.0" 8 | repository = "https://github.com/helix-editor/tree-house" 9 | readme = "../README.md" 10 | rust-version = "1.76.0" 11 | 12 | [dependencies] 13 | anyhow = "1.0" 14 | serde = "1.0" 15 | serde_json = "1.0" 16 | walkdir = "2.5" 17 | xflags = "0.3" 18 | 19 | skidder = { path = "../skidder" } 20 | libloading = "0.8" 21 | tempfile = "3.12" 22 | indicatif = "0.17" 23 | 24 | -------------------------------------------------------------------------------- /cli/LICENSE: -------------------------------------------------------------------------------- 1 | ../LICENSE -------------------------------------------------------------------------------- /cli/import.sh: -------------------------------------------------------------------------------- 1 | #!/bin/env bash 2 | 3 | set -e 4 | cargo build --release 5 | ../target/release/skidder-cli import --metadata -r ../../../tree-sitter-grammars/ ../../../master/runtime/grammars/sources/* 6 | ../target/release/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/markdown/tree-sitter-markdown:markdown 7 | ../target/release/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/markdown/tree-sitter-markdown-inline:markdown-inline 8 | ../target/release/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/v/tree_sitter_v:v 9 | ../target/release/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/wat/wat 10 | ../target/release/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/wat/wast 11 | ../target/release/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/typescript/typescript 12 | ../target/release/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/typescript/tsx 13 | ../target/release/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/php-only/php_only:php-only 14 | ../target/release/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/ocaml/ocaml 15 | ../target/release/skidder-cli import --metadata -r ../../../tree-sitter-grammars ../../../master/runtime/grammars/sources/ocaml/interface:ocaml-interface 16 | -------------------------------------------------------------------------------- /cli/src/build.rs: -------------------------------------------------------------------------------- 1 | use std::num::NonZeroUsize; 2 | use std::path::PathBuf; 3 | 4 | use anyhow::Context; 5 | 6 | use crate::flags; 7 | 8 | impl flags::Build { 9 | pub fn run(self) -> anyhow::Result<()> { 10 | let repo = self 11 | .repo 12 | .canonicalize() 13 | .with_context(|| format!("failed to access {}", self.repo.display()))?; 14 | let config = skidder::Config { 15 | repos: vec![skidder::Repo::Local { path: repo }], 16 | index: PathBuf::new(), 17 | verbose: self.verbose, 18 | }; 19 | if let Some(grammar) = self.grammar { 20 | skidder::build_grammar(&config, &grammar, self.force)?; 21 | } else { 22 | skidder::build_all_grammars( 23 | &config, 24 | self.force, 25 | self.threads.and_then(NonZeroUsize::new), 26 | )?; 27 | } 28 | Ok(()) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /cli/src/flags.rs: -------------------------------------------------------------------------------- 1 | use std::path::PathBuf; 2 | 3 | xflags::xflags! { 4 | src "./src/flags.rs" 5 | 6 | cmd skidder { 7 | cmd import { 8 | /// Whether to import queries 9 | optional --import-queries 10 | /// Whether to (re)generate metadata 11 | optional --metadata 12 | /// The repository/directory where repos are copied into. 13 | /// Defaults to the current working directory 14 | optional -r,--repo repo: PathBuf 15 | /// The path of the grammars to import. The name of the directory 16 | /// will be used as the grammar name. To overwrite you can append 17 | /// the grammar name with a colon 18 | repeated path: PathBuf 19 | } 20 | cmd build { 21 | optional --verbose 22 | optional -j, --threads threads: usize 23 | optional -f, --force 24 | required repo: PathBuf 25 | optional grammar: String 26 | } 27 | cmd init-repo { 28 | required repo: PathBuf 29 | } 30 | cmd load-grammar { 31 | optional -r, --recursive 32 | required path: PathBuf 33 | } 34 | cmd regenerate-parser { 35 | optional -r, --recursive 36 | required path: PathBuf 37 | } 38 | default cmd version { 39 | optional --version 40 | } 41 | } 42 | } 43 | // generated start 44 | // The following code is generated by `xflags` macro. 45 | // Run `env UPDATE_XFLAGS=1 cargo build` to regenerate. 46 | #[derive(Debug)] 47 | pub struct Skidder { 48 | pub subcommand: SkidderCmd, 49 | } 50 | 51 | #[derive(Debug)] 52 | pub enum SkidderCmd { 53 | Version(Version), 54 | Import(Import), 55 | Build(Build), 56 | InitRepo(InitRepo), 57 | LoadGrammar(LoadGrammar), 58 | RegenerateParser(RegenerateParser), 59 | } 60 | 61 | #[derive(Debug)] 62 | pub struct Version { 63 | pub version: bool, 64 | } 65 | 66 | #[derive(Debug)] 67 | pub struct Import { 68 | pub path: Vec, 69 | 70 | pub import_queries: bool, 71 | pub metadata: bool, 72 | pub repo: Option, 73 | } 74 | 75 | #[derive(Debug)] 76 | pub struct Build { 77 | pub repo: PathBuf, 78 | pub grammar: Option, 79 | 80 | pub verbose: bool, 81 | pub threads: Option, 82 | pub force: bool, 83 | } 84 | 85 | #[derive(Debug)] 86 | pub struct InitRepo { 87 | pub repo: PathBuf, 88 | } 89 | 90 | #[derive(Debug)] 91 | pub struct LoadGrammar { 92 | pub path: PathBuf, 93 | 94 | pub recursive: bool, 95 | } 96 | 97 | #[derive(Debug)] 98 | pub struct RegenerateParser { 99 | pub path: PathBuf, 100 | 101 | pub recursive: bool, 102 | } 103 | 104 | impl Skidder { 105 | #[allow(dead_code)] 106 | pub fn from_env_or_exit() -> Self { 107 | Self::from_env_or_exit_() 108 | } 109 | 110 | #[allow(dead_code)] 111 | pub fn from_env() -> xflags::Result { 112 | Self::from_env_() 113 | } 114 | 115 | #[allow(dead_code)] 116 | pub fn from_vec(args: Vec) -> xflags::Result { 117 | Self::from_vec_(args) 118 | } 119 | } 120 | // generated end 121 | 122 | impl Skidder { 123 | pub const HELP: &str = Self::HELP_; 124 | } 125 | -------------------------------------------------------------------------------- /cli/src/generate_parser.rs: -------------------------------------------------------------------------------- 1 | use std::fs::{self, File}; 2 | use std::io::Write; 3 | use std::path::Path; 4 | use std::process::Command; 5 | 6 | use anyhow::{ensure, Context, Result}; 7 | use skidder::{decompress, Metadata}; 8 | use tempfile::TempDir; 9 | 10 | use crate::collect_grammars; 11 | use crate::flags::RegenerateParser; 12 | use crate::import::import_compressed; 13 | 14 | impl RegenerateParser { 15 | pub fn run(self) -> Result<()> { 16 | let paths = if self.recursive { 17 | collect_grammars(&self.path)? 18 | } else { 19 | vec![self.path.clone()] 20 | }; 21 | let temp_dir = 22 | TempDir::new().context("failed to create temporary directory for decompression")?; 23 | // create dummy file to prevent TS CLI from creating a full skeleton 24 | File::create(temp_dir.path().join("grammar.js")) 25 | .context("failed to create temporary directory for decompression")?; 26 | let mut failed = Vec::new(); 27 | for grammar_dir in paths { 28 | let grammar_name = grammar_dir.file_name().unwrap().to_str().unwrap(); 29 | if grammar_name <= "dart" { 30 | continue; 31 | } 32 | println!("checking {grammar_name}"); 33 | 34 | let compressed = Metadata::read(&grammar_dir.join("metadata.json")) 35 | .with_context(|| format!("failed to read metadata for {grammar_name}"))? 36 | .parser_definition() 37 | .unwrap() 38 | .compressed; 39 | 40 | let src_path = grammar_dir.join("src"); 41 | let src_grammar_path = src_path.join("grammar.json"); 42 | let grammar_path = temp_dir.path().join("grammar.json"); 43 | if !src_grammar_path.exists() { 44 | eprintln!("grammar.json not found for {grammar_name}"); 45 | failed.push(grammar_name.to_owned()); 46 | continue; 47 | } 48 | if compressed { 49 | let dst = File::create(&grammar_path).with_context(|| { 50 | format!( 51 | "failed to create grammar.json file in temporary build directory {}", 52 | temp_dir.path().display() 53 | ) 54 | })?; 55 | decompress_file(&src_grammar_path, dst).with_context(|| { 56 | format!("failed to decompress grammar.json for {grammar_name}") 57 | })?; 58 | } else { 59 | fs::copy(src_grammar_path, &grammar_path) 60 | .with_context(|| format!("failed to copy grammar.json for {grammar_name}"))?; 61 | } 62 | println!("running tree-sitter generate {}", grammar_path.display()); 63 | let res = Command::new("tree-sitter") 64 | .arg("generate") 65 | .arg("--no-bindings") 66 | .arg(&grammar_path) 67 | .current_dir(temp_dir.path()) 68 | .status() 69 | .with_context(|| { 70 | format!( 71 | "failed to execute tree-sitter generate {}", 72 | grammar_path.display() 73 | ) 74 | })? 75 | .success(); 76 | if !res { 77 | eprintln!( 78 | "failed to execute tree-sitter generate {}", 79 | grammar_path.display() 80 | ); 81 | failed.push(grammar_name.to_owned()); 82 | continue; 83 | } 84 | 85 | let new_parser_path = temp_dir.path().join("src").join("parser.c"); 86 | let old_parser_path = src_path.join("parser.c"); 87 | let mut old_parser = Vec::new(); 88 | decompress_file(&old_parser_path, &mut old_parser) 89 | .with_context(|| format!("failed to decompress parser for {grammar_name}"))?; 90 | let old_parser = String::from_utf8_lossy(&old_parser); 91 | let new_parser = fs::read_to_string(&new_parser_path) 92 | .context("tree-sitter cli did not generate parser.c")?; 93 | if old_parser.trim() == new_parser.trim() { 94 | continue; 95 | } 96 | failed.push(grammar_name.to_owned()); 97 | eprintln!("existing parser.c was outdated updating..."); 98 | if compressed { 99 | import_compressed(&new_parser_path, &old_parser_path).with_context(|| { 100 | format!("failed to compress new parser.c for {grammar_name}") 101 | })?; 102 | } else { 103 | fs::copy(&new_parser_path, &old_parser_path) 104 | .with_context(|| format!("failed to opy new parser.c for {grammar_name}"))?; 105 | } 106 | } 107 | ensure!( 108 | failed.is_empty(), 109 | "parser.c files is not up to date for {failed:?}!" 110 | ); 111 | Ok(()) 112 | } 113 | } 114 | 115 | fn decompress_file(src: &Path, dst: impl Write) -> Result<()> { 116 | File::open(src) 117 | .map_err(anyhow::Error::from) 118 | .and_then(|mut reader| decompress(&mut reader, dst)) 119 | .with_context(|| format!("failed to decompress {}", src.display()))?; 120 | Ok(()) 121 | } 122 | -------------------------------------------------------------------------------- /cli/src/init.rs: -------------------------------------------------------------------------------- 1 | use std::io::Write; 2 | use std::path::Path; 3 | 4 | use anyhow::Context; 5 | 6 | use crate::flags::InitRepo; 7 | 8 | impl InitRepo { 9 | pub fn run(self) -> anyhow::Result<()> { 10 | append(&self.repo.join(".gitignore"), "*/*.so\n*/.BUILD_COOKIE\n")?; 11 | append( 12 | &self.repo.join(".gitattributes"), 13 | "*/src/parser.c binary\n*/src/grammar.json binary\n", 14 | )?; 15 | Ok(()) 16 | } 17 | } 18 | 19 | fn append(path: &Path, contents: &str) -> anyhow::Result<()> { 20 | std::fs::File::options() 21 | .create(true) 22 | .append(true) 23 | .open(path)? 24 | .write_all(contents.as_bytes()) 25 | .with_context(|| format!("failed to write {}", path.display())) 26 | } 27 | -------------------------------------------------------------------------------- /cli/src/load.rs: -------------------------------------------------------------------------------- 1 | use std::ffi::c_void; 2 | 3 | use anyhow::{Context, Result}; 4 | use libloading::Symbol; 5 | 6 | use crate::collect_grammars; 7 | use crate::flags::LoadGrammar; 8 | 9 | impl LoadGrammar { 10 | pub fn run(self) -> Result<()> { 11 | let paths = if self.recursive { 12 | collect_grammars(&self.path)? 13 | } else { 14 | vec![self.path.clone()] 15 | }; 16 | for path in paths { 17 | let Some(name) = path.file_stem().unwrap().to_str() else { 18 | continue; 19 | }; 20 | println!("loading {}", path.display()); 21 | unsafe { 22 | let lib = libloading::Library::new(&path) 23 | .with_context(|| format!("failed to load {}", path.display()))?; 24 | let language_fn_name = format!("tree_sitter_{}", name.replace('-', "_")); 25 | let _language_fn: Symbol *mut c_void> = lib 26 | .get(language_fn_name.as_bytes()) 27 | .with_context(|| format!("failed to load {}", path.display()))?; 28 | } 29 | } 30 | Ok(()) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /cli/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::path::{Path, PathBuf}; 2 | use std::process::exit; 3 | 4 | use ::skidder::list_grammars; 5 | use anyhow::Result; 6 | 7 | mod build; 8 | mod flags; 9 | mod generate_parser; 10 | mod import; 11 | mod init; 12 | mod load; 13 | 14 | fn get_version() -> String { 15 | const GIT_HASH: Option<&str> = option_env!("GIT_HASH"); 16 | const CARGO_VERSION: &str = env!("CARGO_PKG_VERSION"); 17 | 18 | let owned = CARGO_VERSION.to_string(); 19 | if let Some(hash) = GIT_HASH { 20 | owned + "-" + hash 21 | } else { 22 | owned 23 | } 24 | } 25 | 26 | fn wrapped_main() -> Result<()> { 27 | let flags = flags::Skidder::from_env_or_exit(); 28 | match flags.subcommand { 29 | flags::SkidderCmd::Import(import_cmd) => import_cmd.run(), 30 | flags::SkidderCmd::Build(build_cmd) => build_cmd.run(), 31 | flags::SkidderCmd::InitRepo(init_cmd) => init_cmd.run(), 32 | flags::SkidderCmd::LoadGrammar(load_cmd) => load_cmd.run(), 33 | flags::SkidderCmd::RegenerateParser(generate_cmd) => generate_cmd.run(), 34 | flags::SkidderCmd::Version(flags::Version { version }) => { 35 | if version { 36 | println!("skidder-cli {}", get_version()); 37 | } else { 38 | println!("{}", flags::Skidder::HELP); 39 | } 40 | Ok(()) 41 | } 42 | } 43 | } 44 | 45 | pub fn main() { 46 | if let Err(err) = wrapped_main() { 47 | for error in err.chain() { 48 | eprintln!("error: {error}") 49 | } 50 | exit(1) 51 | } 52 | } 53 | 54 | fn collect_grammars(repo: &Path) -> Result> { 55 | let config = skidder::Config { 56 | repos: vec![skidder::Repo::Local { 57 | path: repo.to_owned(), 58 | }], 59 | index: PathBuf::new(), 60 | verbose: false, 61 | }; 62 | list_grammars(&config) 63 | } 64 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | { 2 | lib, 3 | rustPlatform, 4 | 5 | gitRev ? null, 6 | }: 7 | let 8 | fs = lib.fileset; 9 | 10 | files = fs.difference (fs.gitTracked ./.) ( 11 | fs.unions [ 12 | ./.github 13 | ./.envrc 14 | ./flake.lock 15 | (fs.fileFilter (file: lib.strings.hasInfix ".git" file.name) ./.) 16 | (fs.fileFilter (file: file.hasExt "md") ./.) 17 | (fs.fileFilter (file: file.hasExt "nix") ./.) 18 | ] 19 | ); 20 | in 21 | rustPlatform.buildRustPackage { 22 | strictDeps = true; 23 | pname = with builtins; (fromTOML (readFile ./cli/Cargo.toml)).package.name; 24 | version = with builtins; (fromTOML (readFile ./cli/Cargo.toml)).package.version; 25 | 26 | src = fs.toSource { 27 | root = ./.; 28 | fileset = files; 29 | }; 30 | 31 | cargoLock = { 32 | lockFile = ./Cargo.lock; 33 | allowBuiltinFetchGit = true; 34 | }; 35 | 36 | cargoBuildFlags = [ "-p skidder-cli" ]; 37 | 38 | doCheck = false; 39 | env.GIT_HASH = gitRev; 40 | 41 | meta.mainProgram = "skidder-cli"; 42 | } 43 | -------------------------------------------------------------------------------- /fixtures/highlighter/codefence_rust_doc_comments.md: -------------------------------------------------------------------------------- 1 | ```rust 2 | // ┡━┛┡━━┛╰─ markup.raw.block 3 | // │ ╰─ markup.raw.block label 4 | // ╰─ markup.raw.block punctuation.bracket 5 | /// `Something` 6 | // ┡┛╿╿╿┡━━━━━━━┛╿╰─ markup.raw.block comment 7 | // │ ││││ ╰─ markup.raw.block comment markup.raw.inline punctuation.bracket 8 | // │ │││╰─ markup.raw.block comment markup.raw.inline 9 | // │ ││╰─ markup.raw.block comment markup.raw.inline punctuation.bracket 10 | // │ │╰─ markup.raw.block comment 11 | // │ ╰─ markup.raw.block comment comment 12 | // ╰─ markup.raw.block comment 13 | /// Anything 14 | // ┡┛╿┗━━━━━━━┹─ markup.raw.block comment 15 | // │ ╰─ markup.raw.block comment comment 16 | // ╰─ markup.raw.block comment 17 | ``` 18 | // ┡━┛╰─ markup.raw.block 19 | // ╰─ markup.raw.block punctuation.bracket 20 | -------------------------------------------------------------------------------- /fixtures/highlighter/comment.html: -------------------------------------------------------------------------------- 1 |
2 | // ╿┡━┛╿┡━━━━━━━━━━┛┡━━┛┡━━━┛┡┛┡━┛╰─ punctuation.bracket 3 | // ││ ││ │ │ │ ╰─ tag 4 | // ││ ││ │ │ ╰─ punctuation.bracket 5 | // ││ ││ │ ╰─ comment 6 | // ││ ││ ╰─ comment warning 7 | // ││ │╰─ comment 8 | // ││ ╰─ punctuation.bracket 9 | // │╰─ tag 10 | // ╰─ punctuation.bracket 11 | -------------------------------------------------------------------------------- /fixtures/highlighter/hello_world.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | // ┡┛ ┡━━┛┡┛ ╰─ punctuation.bracket 3 | // │ │ ╰─ punctuation.bracket 4 | // │ ╰─ function 5 | // ╰─ keyword.function 6 | println!("hello world") 7 | // ┡━━━━━━┛╿┡━━━━━━━━━━━┛╰─ punctuation.bracket 8 | // │ │╰─ string 9 | // │ ╰─ punctuation.bracket 10 | // ╰─ function.macro 11 | } 12 | // ╰─ punctuation.bracket 13 | -------------------------------------------------------------------------------- /fixtures/highlighter/html_in_edoc_in_erlang.erl: -------------------------------------------------------------------------------- 1 | %%
  • foo 2 | // ┡━┛╿┡┛╿┗━┹─ comment 3 | // │ ││ ╰─ comment punctuation.bracket 4 | // │ │╰─ comment tag 5 | // │ ╰─ comment punctuation.bracket 6 | // ╰─ comment 7 | %% bar
    • 8 | // ┡━━━━┛┡┛┡┛╿╰─ comment 9 | // │ │ │ ╰─ comment punctuation.bracket 10 | // │ │ ╰─ comment tag 11 | // │ ╰─ comment punctuation.bracket 12 | // ╰─ comment 13 | -------------------------------------------------------------------------------- /fixtures/highlighter/injectionception.rs: -------------------------------------------------------------------------------- 1 | /// Says hello. 2 | // ┡┛╿┗━━━━━━━━━━┹─ comment 3 | // │ ╰─ comment comment 4 | // ╰─ comment 5 | /// 6 | // ┡┛╿╰─ comment 7 | // │ ╰─ comment comment 8 | // ╰─ comment 9 | /// # Example 10 | // ┡┛╿┡┛┗━━━━━━┹─ comment markup.heading.1 11 | // │ │╰─ comment markup.heading.1 markup.heading.marker 12 | // │ ╰─ comment comment 13 | // ╰─ comment 14 | /// 15 | // ┡┛╿╰─ comment 16 | // │ ╰─ comment comment 17 | // ╰─ comment 18 | /// ```rust 19 | // ┡┛╿┡━━┛┡━━┛╰─ comment markup.raw.block 20 | // │ ││ ╰─ comment markup.raw.block label 21 | // │ │╰─ comment markup.raw.block punctuation.bracket 22 | // │ ╰─ comment comment 23 | // ╰─ comment 24 | /// fn add(left: usize, right: usize) -> usize { 25 | // ┡┛╿╿┡┛╿┡━┛╿┡━━┛╿╿┡━━━┛╿╿┡━━━┛╿╿┡━━━┛╿╿┡┛╿┡━━━┛╿╿╰─ comment markup.raw.block 26 | // │ │││ ││ ││ │││ │││ │││ │││ ││ │╰─ comment markup.raw.block punctuation.bracket 27 | // │ │││ ││ ││ │││ │││ │││ │││ ││ ╰─ comment markup.raw.block 28 | // │ │││ ││ ││ │││ │││ │││ │││ │╰─ comment markup.raw.block type.builtin 29 | // │ │││ ││ ││ │││ │││ │││ │││ ╰─ comment markup.raw.block 30 | // │ │││ ││ ││ │││ │││ │││ ││╰─ comment markup.raw.block operator 31 | // │ │││ ││ ││ │││ │││ │││ │╰─ comment markup.raw.block 32 | // │ │││ ││ ││ │││ │││ │││ ╰─ comment markup.raw.block punctuation.bracket 33 | // │ │││ ││ ││ │││ │││ ││╰─ comment markup.raw.block type.builtin 34 | // │ │││ ││ ││ │││ │││ │╰─ comment markup.raw.block 35 | // │ │││ ││ ││ │││ │││ ╰─ comment markup.raw.block punctuation.delimiter 36 | // │ │││ ││ ││ │││ ││╰─ comment markup.raw.block variable.parameter 37 | // │ │││ ││ ││ │││ │╰─ comment markup.raw.block 38 | // │ │││ ││ ││ │││ ╰─ comment markup.raw.block punctuation.delimiter 39 | // │ │││ ││ ││ ││╰─ comment markup.raw.block type.builtin 40 | // │ │││ ││ ││ │╰─ comment markup.raw.block 41 | // │ │││ ││ ││ ╰─ comment markup.raw.block punctuation.delimiter 42 | // │ │││ ││ │╰─ comment markup.raw.block variable.parameter 43 | // │ │││ ││ ╰─ comment markup.raw.block punctuation.bracket 44 | // │ │││ │╰─ comment markup.raw.block function 45 | // │ │││ ╰─ comment markup.raw.block 46 | // │ ││╰─ comment markup.raw.block keyword.function 47 | // │ │╰─ comment markup.raw.block 48 | // │ ╰─ comment comment 49 | // ╰─ comment 50 | /// left + right 51 | // ┡┛╿┡━━━┛┡━━┛╿╿╿┡━━━┛╰─ comment markup.raw.block 52 | // │ ││ │ │││╰─ comment markup.raw.block variable.parameter 53 | // │ ││ │ ││╰─ comment markup.raw.block 54 | // │ ││ │ │╰─ comment markup.raw.block operator 55 | // │ ││ │ ╰─ comment markup.raw.block 56 | // │ ││ ╰─ comment markup.raw.block variable.parameter 57 | // │ │╰─ comment markup.raw.block 58 | // │ ╰─ comment comment 59 | // ╰─ comment 60 | /// } 61 | // ┡┛╿╿╿╰─ comment markup.raw.block 62 | // │ ││╰─ comment markup.raw.block punctuation.bracket 63 | // │ │╰─ comment markup.raw.block 64 | // │ ╰─ comment comment 65 | // ╰─ comment 66 | /// ``` 67 | // ┡┛╿┡━━┛╰─ comment markup.raw.block 68 | // │ │╰─ comment markup.raw.block punctuation.bracket 69 | // │ ╰─ comment comment 70 | // ╰─ comment 71 | pub fn hello() {} 72 | // ┡━┛ ┡┛ ┡━━━┛┡┛ ┗┹─ punctuation.bracket 73 | // │ │ │ ╰─ punctuation.bracket 74 | // │ │ ╰─ function 75 | // │ ╰─ keyword.function 76 | // ╰─ keyword 77 | -------------------------------------------------------------------------------- /fixtures/highlighter/non_local.rs: -------------------------------------------------------------------------------- 1 | fn foo(this: &Thing) { 2 | // ┡┛ ┡━┛╿┡━━┛╿ ╿┡━━━┛╿ ╰─ punctuation.bracket 3 | // │ │ ││ │ ││ ╰─ punctuation.bracket 4 | // │ │ ││ │ │╰─ type 5 | // │ │ ││ │ ╰─ keyword.storage.modifier.ref 6 | // │ │ ││ ╰─ punctuation.delimiter 7 | // │ │ │╰─ variable.parameter 8 | // │ │ ╰─ punctuation.bracket 9 | // │ ╰─ function 10 | // ╰─ keyword.function 11 | this 12 | // ┗━━┹─ variable.parameter 13 | } 14 | // ╰─ punctuation.bracket 15 | 16 | fn bar() { 17 | // ┡┛ ┡━┛┡┛ ╰─ punctuation.bracket 18 | // │ │ ╰─ punctuation.bracket 19 | // │ ╰─ function 20 | // ╰─ keyword.function 21 | this.foo(); 22 | // ┡━━┛╿┡━┛┡┛╰─ punctuation.delimiter 23 | // │ ││ ╰─ punctuation.bracket 24 | // │ │╰─ function 25 | // │ ╰─ punctuation.delimiter 26 | // ╰─ variable.builtin 27 | } 28 | // ╰─ punctuation.bracket 29 | -------------------------------------------------------------------------------- /fixtures/highlighter/reference_highlight_starts_after_definition_ends.rs: -------------------------------------------------------------------------------- 1 | fn event(tx: &Sender, event: MyEvent) { 2 | // ┡┛ ┡━━━┛╿┡┛╿ ╿┡━━━━┛╿ ┡━━━┛╿ ┡━━━━━┛╿ ╰─ punctuation.bracket 3 | // │ │ ││ │ ││ │ │ │ │ ╰─ punctuation.bracket 4 | // │ │ ││ │ ││ │ │ │ ╰─ type 5 | // │ │ ││ │ ││ │ │ ╰─ punctuation.delimiter 6 | // │ │ ││ │ ││ │ ╰─ variable.parameter 7 | // │ │ ││ │ ││ ╰─ punctuation.delimiter 8 | // │ │ ││ │ │╰─ type 9 | // │ │ ││ │ ╰─ keyword.storage.modifier.ref 10 | // │ │ ││ ╰─ punctuation.delimiter 11 | // │ │ │╰─ variable.parameter 12 | // │ │ ╰─ punctuation.bracket 13 | // │ ╰─ function 14 | // ╰─ keyword.function 15 | send_blocking(tx, event); 16 | // ┡━━━━━━━━━━━┛╿┡┛╿ ┡━━━┛╿╰─ punctuation.delimiter 17 | // │ ││ │ │ ╰─ punctuation.bracket 18 | // │ ││ │ ╰─ variable.parameter 19 | // │ ││ ╰─ punctuation.delimiter 20 | // │ │╰─ variable.parameter 21 | // │ ╰─ punctuation.bracket 22 | // ╰─ function 23 | } 24 | // ╰─ punctuation.bracket 25 | -------------------------------------------------------------------------------- /fixtures/highlighter/rust_doc_comment.rs: -------------------------------------------------------------------------------- 1 | /// **hello-world** 2 | // ┡┛╿╿┡┛┡━━━━━━━━━┛┡┛╰─ comment 3 | // │ │││ │ ╰─ comment markup.bold punctuation.bracket 4 | // │ │││ ╰─ comment markup.bold 5 | // │ ││╰─ comment markup.bold punctuation.bracket 6 | // │ │╰─ comment 7 | // │ ╰─ comment comment 8 | // ╰─ comment 9 | /// **foo 10 | // ┡┛╿╿┡┛┗━┹─ comment markup.bold 11 | // │ ││╰─ comment markup.bold punctuation.bracket 12 | // │ │╰─ comment 13 | // │ ╰─ comment comment 14 | // ╰─ comment 15 | fn foo() { 16 | // ┡┛ ┡━┛┡┛ ╰─ punctuation.bracket 17 | // │ │ ╰─ punctuation.bracket 18 | // │ ╰─ function 19 | // ╰─ keyword.function 20 | println!("hello world") 21 | // ┡━━━━━━┛╿┡━━━━━━━━━━━┛╰─ punctuation.bracket 22 | // │ │╰─ string 23 | // │ ╰─ punctuation.bracket 24 | // ╰─ function.macro 25 | } 26 | // ╰─ punctuation.bracket 27 | /// bar** 28 | // ┡┛╿┡━━┛┡┛╰─ comment 29 | // │ ││ ╰─ comment markup.bold punctuation.bracket 30 | // │ │╰─ comment markup.bold 31 | // │ ╰─ comment comment 32 | // ╰─ comment 33 | fn bar() { 34 | // ┡┛ ┡━┛┡┛ ╰─ punctuation.bracket 35 | // │ │ ╰─ punctuation.bracket 36 | // │ ╰─ function 37 | // ╰─ keyword.function 38 | println!("hello world") 39 | // ┡━━━━━━┛╿┡━━━━━━━━━━━┛╰─ punctuation.bracket 40 | // │ │╰─ string 41 | // │ ╰─ punctuation.bracket 42 | // ╰─ function.macro 43 | } 44 | // ╰─ punctuation.bracket 45 | -------------------------------------------------------------------------------- /fixtures/highlighter/rust_no_doc_comment.rs: -------------------------------------------------------------------------------- 1 | // TODO **hello-world** 2 | // ┡━┛┡━━┛┗━━━━━━━━━━━━━━━┹─ comment 3 | // │ ╰─ comment info 4 | // ╰─ comment 5 | fn foo() { 6 | // ┡┛ ┡━┛┡┛ ╰─ punctuation.bracket 7 | // │ │ ╰─ punctuation.bracket 8 | // │ ╰─ function 9 | // ╰─ keyword.function 10 | println!("hello world") 11 | // ┡━━━━━━┛╿┡━━━━━━━━━━━┛╰─ punctuation.bracket 12 | // │ │╰─ string 13 | // │ ╰─ punctuation.bracket 14 | // ╰─ function.macro 15 | } 16 | // ╰─ punctuation.bracket 17 | /// bar** 18 | // ┗━━━━━━━┹─ comment 19 | fn bar() { 20 | // ┡┛ ┡━┛┡┛ ╰─ punctuation.bracket 21 | // │ │ ╰─ punctuation.bracket 22 | // │ ╰─ function 23 | // ╰─ keyword.function 24 | println!("hello world") 25 | // ┡━━━━━━┛╿┡━━━━━━━━━━━┛╰─ punctuation.bracket 26 | // │ │╰─ string 27 | // │ ╰─ punctuation.bracket 28 | // ╰─ function.macro 29 | } 30 | // ╰─ punctuation.bracket 31 | -------------------------------------------------------------------------------- /fixtures/highlighter/rust_parameter_locals.rs: -------------------------------------------------------------------------------- 1 | fn add(left: usize, right: usize) -> usize { 2 | // ┡┛ ┡━┛╿┡━━┛╿ ┡━━━┛╿ ┡━━━┛╿ ┡━━━┛╿ ┡┛ ┡━━━┛ ╰─ punctuation.bracket 3 | // │ │ ││ │ │ │ │ │ │ │ │ ╰─ type.builtin 4 | // │ │ ││ │ │ │ │ │ │ │ ╰─ operator 5 | // │ │ ││ │ │ │ │ │ │ ╰─ punctuation.bracket 6 | // │ │ ││ │ │ │ │ │ ╰─ type.builtin 7 | // │ │ ││ │ │ │ │ ╰─ punctuation.delimiter 8 | // │ │ ││ │ │ │ ╰─ variable.parameter 9 | // │ │ ││ │ │ ╰─ punctuation.delimiter 10 | // │ │ ││ │ ╰─ type.builtin 11 | // │ │ ││ ╰─ punctuation.delimiter 12 | // │ │ │╰─ variable.parameter 13 | // │ │ ╰─ punctuation.bracket 14 | // │ ╰─ function 15 | // ╰─ keyword.function 16 | left + right 17 | // ┡━━┛ ╿ ┗━━━┹─ variable.parameter 18 | // │ ╰─ operator 19 | // ╰─ variable.parameter 20 | } 21 | // ╰─ punctuation.bracket 22 | 23 | fn add(left: usize, right: usize) -> usize { 24 | // ┡┛ ┡━┛╿┡━━┛╿ ┡━━━┛╿ ┡━━━┛╿ ┡━━━┛╿ ┡┛ ┡━━━┛ ╰─ punctuation.bracket 25 | // │ │ ││ │ │ │ │ │ │ │ │ ╰─ type.builtin 26 | // │ │ ││ │ │ │ │ │ │ │ ╰─ operator 27 | // │ │ ││ │ │ │ │ │ │ ╰─ punctuation.bracket 28 | // │ │ ││ │ │ │ │ │ ╰─ type.builtin 29 | // │ │ ││ │ │ │ │ ╰─ punctuation.delimiter 30 | // │ │ ││ │ │ │ ╰─ variable.parameter 31 | // │ │ ││ │ │ ╰─ punctuation.delimiter 32 | // │ │ ││ │ ╰─ type.builtin 33 | // │ │ ││ ╰─ punctuation.delimiter 34 | // │ │ │╰─ variable.parameter 35 | // │ │ ╰─ punctuation.bracket 36 | // │ ╰─ function 37 | // ╰─ keyword.function 38 | left + right 39 | // ┡━━┛ ╿ ┗━━━┹─ variable.parameter 40 | // │ ╰─ operator 41 | // ╰─ variable.parameter 42 | } 43 | // ╰─ punctuation.bracket 44 | -------------------------------------------------------------------------------- /fixtures/injections/html_in_edoc_in_erlang.erl: -------------------------------------------------------------------------------- 1 | %%
  • foo 2 | // ╿┡━━┛┗━┹─ edoc 3 | // │╰─ edoc html 4 | // ╰─ edoc 5 | %% bar
    • 6 | // ┡━━┛┡━━━┛╰─ edoc 7 | // │ ╰─ edoc html 8 | // ╰─ edoc 9 | -------------------------------------------------------------------------------- /fixtures/injections/injectionception.rs: -------------------------------------------------------------------------------- 1 | /// Says hello. 2 | // ┡━━━━━━━━━━┛╰─ markdown 3 | // ╰─ markdown markdown-inline 4 | /// 5 | // ╰─ markdown 6 | /// # Example 7 | // ┡━┛┡━━━━━┛╰─ markdown 8 | // │ ╰─ markdown markdown-inline 9 | // ╰─ markdown 10 | /// 11 | // ╰─ markdown 12 | /// ```rust 13 | // ┗━━━━━━┹─ markdown 14 | /// fn add(left: usize, right: usize) -> usize { 15 | // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┹─ markdown rust 16 | /// left + right 17 | // ┗━━━━━━━━━━━━━━━┹─ markdown rust 18 | /// } 19 | // ┗┹─ markdown rust 20 | /// ``` 21 | // ┗━━┹─ markdown 22 | pub fn hello() {} 23 | -------------------------------------------------------------------------------- /fixtures/injections/overlapping_injection.rs: -------------------------------------------------------------------------------- 1 | some_macro!((), (), "`rust` injection happens here"); 2 | // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┹─ rust 3 | -------------------------------------------------------------------------------- /fixtures/injections/rust_doc_comment.rs: -------------------------------------------------------------------------------- 1 | /// **hello-world** 2 | // ┗━━━━━━━━━━━━━━━┹─ markdown markdown-inline 3 | /// **foo 4 | // ┗━━━━┹─ markdown markdown-inline 5 | fn foo() { 6 | println!("hello world") 7 | // ┗━━━━━━━━━━━━━┹─ rust 8 | } 9 | /// bar** 10 | // ┡━━━━┛╰─ markdown 11 | // ╰─ markdown markdown-inline 12 | fn bar() { 13 | println!("hello world") 14 | // ┗━━━━━━━━━━━━━┹─ rust 15 | } 16 | 17 | -------------------------------------------------------------------------------- /fixtures/injections/rust_no_doc_comment.rs: -------------------------------------------------------------------------------- 1 | /// TODO **hello-world** 2 | // ┗━━━━━━━━━━━━━━━━━━━━━━━┹─ comment 3 | /// **foo 4 | // ┗━━━━━━━┹─ comment 5 | fn foo() { 6 | println!("hello world") 7 | // ┗━━━━━━━━━━━━━┹─ rust 8 | } 9 | /// bar** 10 | // ┗━━━━━━━┹─ comment 11 | fn bar() { 12 | println!("hello world") 13 | // ┗━━━━━━━━━━━━━┹─ rust 14 | } 15 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "nixpkgs": { 4 | "locked": { 5 | "lastModified": 1746904237, 6 | "narHash": "sha256-3e+AVBczosP5dCLQmMoMEogM57gmZ2qrVSrmq9aResQ=", 7 | "owner": "nixos", 8 | "repo": "nixpkgs", 9 | "rev": "d89fc19e405cb2d55ce7cc114356846a0ee5e956", 10 | "type": "github" 11 | }, 12 | "original": { 13 | "owner": "nixos", 14 | "ref": "nixos-unstable", 15 | "repo": "nixpkgs", 16 | "type": "github" 17 | } 18 | }, 19 | "root": { 20 | "inputs": { 21 | "nixpkgs": "nixpkgs", 22 | "rust-overlay": "rust-overlay" 23 | } 24 | }, 25 | "rust-overlay": { 26 | "inputs": { 27 | "nixpkgs": [ 28 | "nixpkgs" 29 | ] 30 | }, 31 | "locked": { 32 | "lastModified": 1747190175, 33 | "narHash": "sha256-s33mQ2s5L/2nyllhRTywgECNZyCqyF4MJeM3vG/GaRo=", 34 | "owner": "oxalica", 35 | "repo": "rust-overlay", 36 | "rev": "58160be7abad81f6f8cb53120d5b88c16e01c06d", 37 | "type": "github" 38 | }, 39 | "original": { 40 | "owner": "oxalica", 41 | "repo": "rust-overlay", 42 | "type": "github" 43 | } 44 | } 45 | }, 46 | "root": "root", 47 | "version": 7 48 | } 49 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | description = "A package manager for tree-sitter grammars"; 3 | 4 | inputs = { 5 | nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; 6 | rust-overlay = { 7 | url = "github:oxalica/rust-overlay"; 8 | inputs.nixpkgs.follows = "nixpkgs"; 9 | }; 10 | }; 11 | 12 | outputs = 13 | { 14 | self, 15 | nixpkgs, 16 | rust-overlay, 17 | }: 18 | let 19 | inherit (nixpkgs) lib; 20 | forEachSystem = lib.genAttrs lib.systems.flakeExposed; 21 | in 22 | { 23 | packages = forEachSystem ( 24 | system: 25 | let 26 | pkgs = import nixpkgs { 27 | inherit system; 28 | overlays = [ (import rust-overlay) ]; 29 | }; 30 | toolchain = pkgs.rust-bin.stable.latest.default; 31 | in 32 | { 33 | skidder-cli = pkgs.callPackage ./. { gitRev = self.rev or self.dirtyRev; }; 34 | default = self.packages.${system}.skidder-cli; 35 | } 36 | ); 37 | 38 | devShell = forEachSystem ( 39 | system: 40 | let 41 | pkgs = import nixpkgs { 42 | inherit system; 43 | overlays = [ (import rust-overlay) ]; 44 | }; 45 | toolchain = pkgs.rust-bin.stable.latest.default; 46 | in 47 | pkgs.mkShell { 48 | nativeBuildInputs = with pkgs; [ 49 | (toolchain.override { 50 | extensions = [ 51 | "rust-src" 52 | "clippy" 53 | "llvm-tools-preview" 54 | ]; 55 | }) 56 | rust-analyzer 57 | cargo-flamegraph 58 | cargo-llvm-cov 59 | valgrind 60 | ]; 61 | RUST_BACKTRACE = "1"; 62 | } 63 | ); 64 | }; 65 | } 66 | -------------------------------------------------------------------------------- /highlighter/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | 9 | 10 | ## [v0.2.0] - 2025-06-06 11 | 12 | ### Added 13 | 14 | * Added `Syntax::layers_for_byte_range` 15 | * Added `TreeCursor::reset` 16 | * Added an iterator for recursively walking over the nodes in a `TreeCursor`: `TreeRecursiveWalker` 17 | 18 | ### Changed 19 | 20 | * `InactiveQueryCursor::new` now takes the byte range and match limit as parameters 21 | 22 | ### Fixed 23 | 24 | * Included `LICENSE` in the crate package 25 | * Fixed an issue where a combined injection layer could be queried multiple times by `QueryIter` 26 | * Fixed an issue where a combined injection layer would not be re-parsed when an injection for the layer was removed by an edit 27 | 28 | ## [v0.1.0] - 2025-05-13 29 | 30 | ### Added 31 | 32 | * Initial publish 33 | 34 | -------------------------------------------------------------------------------- /highlighter/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tree-house" 3 | description = "A robust and cozy highlighter library for tree-sitter" 4 | authors = [ 5 | "Pascal Kuthe ", 6 | "Michael Davis " 7 | ] 8 | version = "0.2.0" 9 | edition = "2021" 10 | license = "MPL-2.0" 11 | repository = "https://github.com/helix-editor/tree-house" 12 | readme = "../README.md" 13 | rust-version = "1.76.0" 14 | 15 | [features] 16 | default = ["fixtures"] 17 | fixtures = ["dep:unicode-width", "dep:pretty_assertions"] 18 | 19 | [dependencies] 20 | ropey = { version = "1.6", default-features = false } 21 | arc-swap = "1" 22 | hashbrown = { version = "0.15" } 23 | regex = "1" 24 | regex-cursor = "0.1" 25 | slab = "0.4" 26 | unicode-width = { version = "=0.1.12", optional = true } 27 | pretty_assertions = { version = "1.4.0", optional = true } 28 | kstring = "2.0" 29 | 30 | [dev-dependencies] 31 | cov-mark = "2.0.0" 32 | indexmap = "2.5.0" 33 | skidder = { path = "../skidder" } 34 | 35 | [dependencies.tree-sitter] 36 | version = "0.2" 37 | package = "tree-house-bindings" 38 | path = "../bindings" 39 | features = ["ropey"] 40 | 41 | [dependencies.once_cell] 42 | version = "1.19" 43 | default-features = false 44 | features = ["std", "alloc"] 45 | -------------------------------------------------------------------------------- /highlighter/LICENSE: -------------------------------------------------------------------------------- 1 | ../LICENSE -------------------------------------------------------------------------------- /highlighter/src/config.rs: -------------------------------------------------------------------------------- 1 | use once_cell::sync::Lazy; 2 | use regex::Regex; 3 | use tree_sitter::{query, Grammar}; 4 | 5 | use crate::highlighter::{Highlight, HighlightQuery}; 6 | use crate::injections_query::{InjectionLanguageMarker, InjectionsQuery}; 7 | use crate::Language; 8 | 9 | use std::fmt::Write; 10 | 11 | #[derive(Debug)] 12 | pub struct LanguageConfig { 13 | pub grammar: Grammar, 14 | pub highlight_query: HighlightQuery, 15 | pub injection_query: InjectionsQuery, 16 | } 17 | 18 | impl LanguageConfig { 19 | pub fn new( 20 | grammar: Grammar, 21 | highlight_query_text: &str, 22 | injection_query_text: &str, 23 | local_query_text: &str, 24 | ) -> Result { 25 | // NOTE: the injection queries are parsed first since the local query is passed as-is 26 | // to `Query::new` in `InjectionsQuery::new`. This ensures that the more readable error 27 | // bubbles up first if the locals queries have an issue. 28 | let injection_query = 29 | InjectionsQuery::new(grammar, injection_query_text, local_query_text)?; 30 | let highlight_query = HighlightQuery::new(grammar, highlight_query_text, local_query_text)?; 31 | 32 | Ok(Self { 33 | grammar, 34 | highlight_query, 35 | injection_query, 36 | }) 37 | } 38 | 39 | pub fn configure(&self, mut f: impl FnMut(&str) -> Option) { 40 | self.highlight_query.configure(&mut f); 41 | self.injection_query.configure(&mut f); 42 | } 43 | } 44 | 45 | static INHERITS_REGEX: Lazy = 46 | Lazy::new(|| Regex::new(r";+\s*inherits\s*:?\s*([a-z_,()-]+)\s*").unwrap()); 47 | 48 | /// reads a query by invoking `read_query_text`, handles any `inherits` directives 49 | pub fn read_query(language: &str, mut read_query_text: impl FnMut(&str) -> String) -> String { 50 | fn read_query_impl(language: &str, read_query_text: &mut impl FnMut(&str) -> String) -> String { 51 | let query = read_query_text(language); 52 | 53 | // replaces all "; inherits (,)*" with the queries of the given language(s) 54 | INHERITS_REGEX 55 | .replace_all(&query, |captures: ®ex::Captures| { 56 | captures[1] 57 | .split(',') 58 | .fold(String::new(), |mut output, language| { 59 | // `write!` to a String cannot fail. 60 | write!( 61 | output, 62 | "\n{}\n", 63 | read_query_impl(language, &mut *read_query_text) 64 | ) 65 | .unwrap(); 66 | output 67 | }) 68 | }) 69 | .into_owned() 70 | } 71 | read_query_impl(language, &mut read_query_text) 72 | } 73 | 74 | pub trait LanguageLoader { 75 | fn language_for_marker(&self, marker: InjectionLanguageMarker) -> Option; 76 | fn get_config(&self, lang: Language) -> Option<&LanguageConfig>; 77 | } 78 | 79 | impl LanguageLoader for &'_ T 80 | where 81 | T: LanguageLoader, 82 | { 83 | fn language_for_marker(&self, marker: InjectionLanguageMarker) -> Option { 84 | T::language_for_marker(self, marker) 85 | } 86 | 87 | fn get_config(&self, lang: Language) -> Option<&LanguageConfig> { 88 | T::get_config(self, lang) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /highlighter/src/parse.rs: -------------------------------------------------------------------------------- 1 | use std::mem::take; 2 | use std::time::Duration; 3 | 4 | use ropey::RopeSlice; 5 | use tree_sitter::Parser; 6 | 7 | use crate::config::LanguageLoader; 8 | use crate::{Error, LayerData, Syntax}; 9 | 10 | impl Syntax { 11 | pub fn update( 12 | &mut self, 13 | source: RopeSlice, 14 | timeout: Duration, 15 | edits: &[tree_sitter::InputEdit], 16 | loader: &impl LanguageLoader, 17 | ) -> Result<(), Error> { 18 | // size limit of 512MiB, TS just cannot handle files this big (too 19 | // slow). Furthermore, TS uses 32 (signed) bit indices so this limit 20 | // must never be raised above 2GiB 21 | if source.len_bytes() >= 512 * 1024 * 1024 { 22 | return Err(Error::ExceededMaximumSize); 23 | } 24 | 25 | let mut queue = Vec::with_capacity(32); 26 | let root_flags = &mut self.layer_mut(self.root).flags; 27 | // The root layer is always considered. 28 | root_flags.touched = true; 29 | // If there was an edit then the root layer must've been modified. 30 | root_flags.modified = true; 31 | queue.push(self.root); 32 | 33 | let mut parser = Parser::new(); 34 | parser.set_timeout(timeout); 35 | 36 | while let Some(layer) = queue.pop() { 37 | let layer_data = self.layer_mut(layer); 38 | if layer_data.ranges.is_empty() { 39 | // Skip re-parsing and querying layers without any ranges. 40 | continue; 41 | } 42 | if let Some(tree) = &mut layer_data.parse_tree { 43 | if layer_data.flags.moved || layer_data.flags.modified { 44 | for edit in edits.iter().rev() { 45 | // Apply the edits in reverse. 46 | // If we applied them in order then edit 1 would disrupt the positioning 47 | // of edit 2. 48 | tree.edit(edit); 49 | } 50 | } 51 | if layer_data.flags.modified { 52 | // Re-parse the tree. 53 | layer_data.parse(&mut parser, source, loader)?; 54 | } 55 | } else { 56 | // always parse if this layer has never been parsed before 57 | layer_data.parse(&mut parser, source, loader)?; 58 | } 59 | self.run_injection_query(layer, edits, source, loader, |layer| queue.push(layer)); 60 | self.run_local_query(layer, source, loader); 61 | } 62 | 63 | if self.layer(self.root).parse_tree.is_none() { 64 | return Err(Error::NoRootConfig); 65 | } 66 | 67 | self.prune_dead_layers(); 68 | Ok(()) 69 | } 70 | 71 | /// Reset all `LayerUpdateFlags` and remove all untouched layers 72 | fn prune_dead_layers(&mut self) { 73 | self.layers 74 | .retain(|_, layer| take(&mut layer.flags).touched); 75 | } 76 | } 77 | 78 | impl LayerData { 79 | fn parse( 80 | &mut self, 81 | parser: &mut Parser, 82 | source: RopeSlice, 83 | loader: &impl LanguageLoader, 84 | ) -> Result<(), Error> { 85 | let Some(config) = loader.get_config(self.language) else { 86 | return Ok(()); 87 | }; 88 | if let Err(err) = parser.set_grammar(config.grammar) { 89 | return Err(Error::IncompatibleGrammar(self.language, err)); 90 | } 91 | parser 92 | .set_included_ranges(&self.ranges) 93 | .map_err(|_| Error::InvalidRanges)?; 94 | 95 | // HACK: 96 | // This is a workaround for a bug within the lexer (in the C library) or maybe within 97 | // tree-sitter-markdown which needs more debugging. When adding a new range to a combined 98 | // injection and passing the old tree, if the old tree doesn't already cover a wider range 99 | // than the newly added range, some assumptions are violated in the lexer and it tries to 100 | // access some invalid memory, resulting in a segfault. This workaround avoids that 101 | // situation by avoiding passing the old tree when the old tree's range doesn't cover the 102 | // total range of `self.ranges`. 103 | // 104 | // See . 105 | let tree = self.parse_tree.as_ref().filter(|tree| { 106 | let included_ranges_range = self.ranges.first().map(|r| r.start_byte).unwrap_or(0) 107 | ..self.ranges.last().map(|r| r.end_byte).unwrap_or(u32::MAX); 108 | // Allow re-parsing the root layer even though the range is larger. The root always 109 | // covers `0..u32::MAX`: 110 | if included_ranges_range == (0..u32::MAX) { 111 | return true; 112 | } 113 | let tree_range = tree.root_node().byte_range(); 114 | tree_range.start <= included_ranges_range.start 115 | && tree_range.end >= included_ranges_range.end 116 | }); 117 | 118 | let tree = parser.parse(source, tree).ok_or(Error::Timeout)?; 119 | self.parse_tree = Some(tree); 120 | Ok(()) 121 | } 122 | } 123 | 124 | #[derive(Debug, PartialEq, Eq, Default, Clone)] 125 | pub(crate) struct LayerUpdateFlags { 126 | pub reused: bool, 127 | pub modified: bool, 128 | pub moved: bool, 129 | pub touched: bool, 130 | } 131 | -------------------------------------------------------------------------------- /highlighter/src/pretty_print.rs: -------------------------------------------------------------------------------- 1 | use std::fmt; 2 | 3 | use tree_sitter::{SyntaxTreeNode, TreeCursor}; 4 | 5 | pub fn pretty_print_tree(fmt: &mut W, node: SyntaxTreeNode) -> fmt::Result { 6 | if node.child_count() == 0 { 7 | if node_is_visible(&node) { 8 | write!(fmt, "({})", node.kind()) 9 | } else { 10 | write!(fmt, "\"{}\"", node.kind()) 11 | } 12 | } else { 13 | pretty_print_tree_impl(fmt, &mut node.walk(), 0) 14 | } 15 | } 16 | 17 | fn pretty_print_tree_impl( 18 | fmt: &mut W, 19 | cursor: &mut TreeCursor, 20 | depth: usize, 21 | ) -> fmt::Result { 22 | let node = cursor.node(); 23 | let visible = node_is_visible(&node); 24 | 25 | if visible { 26 | let indentation_columns = depth * 2; 27 | write!(fmt, "{:indentation_columns$}", "")?; 28 | 29 | if let Some(field_name) = cursor.field_name() { 30 | write!(fmt, "{}: ", field_name)?; 31 | } 32 | 33 | write!(fmt, "({}", node.kind())?; 34 | } 35 | 36 | // Handle children. 37 | if cursor.goto_first_child() { 38 | loop { 39 | if node_is_visible(&cursor.node()) { 40 | fmt.write_char('\n')?; 41 | } 42 | 43 | pretty_print_tree_impl(fmt, cursor, depth + 1)?; 44 | 45 | if !cursor.goto_next_sibling() { 46 | break; 47 | } 48 | } 49 | 50 | let moved = cursor.goto_parent(); 51 | // The parent of the first child must exist, and must be `node`. 52 | debug_assert!(moved); 53 | debug_assert!(cursor.node() == node); 54 | } 55 | 56 | if visible { 57 | fmt.write_char(')')?; 58 | } 59 | 60 | Ok(()) 61 | } 62 | 63 | fn node_is_visible(node: &SyntaxTreeNode) -> bool { 64 | node.is_missing() || (node.is_named() && node.language().node_kind_is_visible(node.kind_id())) 65 | } 66 | -------------------------------------------------------------------------------- /highlighter/src/text_object.rs: -------------------------------------------------------------------------------- 1 | // TODO: rework using query iter 2 | 3 | use std::iter; 4 | 5 | use ropey::RopeSlice; 6 | 7 | use crate::TREE_SITTER_MATCH_LIMIT; 8 | use tree_sitter::{InactiveQueryCursor, Node, Query, RopeInput}; 9 | 10 | #[derive(Debug)] 11 | pub enum CapturedNode<'a> { 12 | Single(Node<'a>), 13 | /// Guaranteed to be not empty 14 | Grouped(Vec>), 15 | } 16 | 17 | impl CapturedNode<'_> { 18 | pub fn start_byte(&self) -> usize { 19 | match self { 20 | Self::Single(n) => n.start_byte() as usize, 21 | Self::Grouped(ns) => ns[0].start_byte() as usize, 22 | } 23 | } 24 | 25 | pub fn end_byte(&self) -> usize { 26 | match self { 27 | Self::Single(n) => n.end_byte() as usize, 28 | Self::Grouped(ns) => ns.last().unwrap().end_byte() as usize, 29 | } 30 | } 31 | } 32 | 33 | #[derive(Debug)] 34 | pub struct TextObjectQuery { 35 | pub query: Query, 36 | } 37 | 38 | impl TextObjectQuery { 39 | /// Run the query on the given node and return sub nodes which match given 40 | /// capture ("function.inside", "class.around", etc). 41 | /// 42 | /// Captures may contain multiple nodes by using quantifiers (+, *, etc), 43 | /// and support for this is partial and could use improvement. 44 | /// 45 | /// ```query 46 | /// (comment)+ @capture 47 | /// 48 | /// ; OR 49 | /// ( 50 | /// (comment)* 51 | /// . 52 | /// (function) 53 | /// ) @capture 54 | /// ``` 55 | pub fn capture_nodes<'a>( 56 | &'a self, 57 | capture_name: &str, 58 | node: Node<'a>, 59 | slice: RopeSlice<'a>, 60 | cursor: InactiveQueryCursor, 61 | ) -> Option>> { 62 | self.capture_nodes_any(&[capture_name], node, slice, cursor) 63 | } 64 | 65 | /// Find the first capture that exists out of all given `capture_names` 66 | /// and return sub nodes that match this capture. 67 | pub fn capture_nodes_any<'a>( 68 | &'a self, 69 | capture_names: &[&str], 70 | node: Node<'a>, 71 | slice: RopeSlice<'a>, 72 | mut cursor: InactiveQueryCursor, 73 | ) -> Option>> { 74 | let capture = capture_names 75 | .iter() 76 | .find_map(|cap| self.query.get_capture(cap))?; 77 | 78 | cursor.set_match_limit(TREE_SITTER_MATCH_LIMIT); 79 | let mut cursor = cursor.execute_query(&self.query, &node, RopeInput::new(slice)); 80 | let capture_node = iter::from_fn(move || { 81 | let (mat, _) = cursor.next_matched_node()?; 82 | Some(mat.nodes_for_capture(capture).cloned().collect()) 83 | }) 84 | .filter_map(move |nodes: Vec<_>| { 85 | if nodes.len() > 1 { 86 | Some(CapturedNode::Grouped(nodes)) 87 | } else { 88 | nodes.into_iter().map(CapturedNode::Single).next() 89 | } 90 | }); 91 | Some(capture_node) 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /highlighter/src/tree_cursor.rs: -------------------------------------------------------------------------------- 1 | use std::collections::VecDeque; 2 | 3 | use crate::tree_sitter::Node; 4 | use crate::{Layer, Syntax}; 5 | 6 | pub struct TreeCursor<'tree> { 7 | syntax: &'tree Syntax, 8 | current: Layer, 9 | cursor: tree_sitter::TreeCursor<'tree>, 10 | } 11 | 12 | impl<'tree> TreeCursor<'tree> { 13 | pub(crate) fn new(syntax: &'tree Syntax) -> Self { 14 | let cursor = syntax.tree().walk(); 15 | 16 | Self { 17 | syntax, 18 | current: syntax.root, 19 | cursor, 20 | } 21 | } 22 | 23 | pub fn node(&self) -> Node<'tree> { 24 | self.cursor.node() 25 | } 26 | 27 | pub fn goto_parent(&mut self) -> bool { 28 | if self.cursor.goto_parent() { 29 | return true; 30 | }; 31 | 32 | loop { 33 | // Ascend to the parent layer if one exists. 34 | let Some(parent) = self.syntax.layer(self.current).parent else { 35 | return false; 36 | }; 37 | 38 | self.current = parent; 39 | if let Some(tree) = self.syntax.layer(self.current).tree() { 40 | self.cursor = tree.walk(); 41 | break; 42 | } 43 | } 44 | 45 | true 46 | } 47 | 48 | pub fn goto_parent_with

    (&mut self, predicate: P) -> bool 49 | where 50 | P: Fn(&Node) -> bool, 51 | { 52 | while self.goto_parent() { 53 | if predicate(&self.node()) { 54 | return true; 55 | } 56 | } 57 | 58 | false 59 | } 60 | 61 | pub fn goto_first_child(&mut self) -> bool { 62 | let range = self.cursor.node().byte_range(); 63 | let layer = self.syntax.layer(self.current); 64 | if let Some((layer, tree)) = layer 65 | .injection_at_byte_idx(range.start) 66 | .filter(|injection| injection.range.end >= range.end) 67 | .and_then(|injection| { 68 | Some((injection.layer, self.syntax.layer(injection.layer).tree()?)) 69 | }) 70 | { 71 | // Switch to the child layer. 72 | self.current = layer; 73 | self.cursor = tree.walk(); 74 | return true; 75 | } 76 | 77 | self.cursor.goto_first_child() 78 | } 79 | 80 | pub fn goto_next_sibling(&mut self) -> bool { 81 | self.cursor.goto_next_sibling() 82 | } 83 | 84 | pub fn goto_previous_sibling(&mut self) -> bool { 85 | self.cursor.goto_previous_sibling() 86 | } 87 | 88 | pub fn reset_to_byte_range(&mut self, start: u32, end: u32) { 89 | let (layer, tree) = self.syntax.layer_and_tree_for_byte_range(start, end); 90 | self.current = layer; 91 | self.cursor = tree.walk(); 92 | 93 | loop { 94 | let node = self.cursor.node(); 95 | if start < node.start_byte() || end > node.end_byte() { 96 | self.cursor.goto_parent(); 97 | break; 98 | } 99 | if self.cursor.goto_first_child_for_byte(start).is_none() { 100 | break; 101 | } 102 | } 103 | } 104 | 105 | /// Returns an iterator over the children of the node the TreeCursor is on 106 | /// at the time this is called. 107 | pub fn children<'a>(&'a mut self) -> ChildIter<'a, 'tree> { 108 | let parent = self.node(); 109 | 110 | ChildIter { 111 | cursor: self, 112 | parent, 113 | } 114 | } 115 | } 116 | 117 | pub struct ChildIter<'a, 'tree> { 118 | cursor: &'a mut TreeCursor<'tree>, 119 | parent: Node<'tree>, 120 | } 121 | 122 | impl<'tree> Iterator for ChildIter<'_, 'tree> { 123 | type Item = Node<'tree>; 124 | 125 | fn next(&mut self) -> Option { 126 | // first iteration, just visit the first child 127 | if self.cursor.node() == self.parent { 128 | self.cursor.goto_first_child().then(|| self.cursor.node()) 129 | } else { 130 | self.cursor.goto_next_sibling().then(|| self.cursor.node()) 131 | } 132 | } 133 | } 134 | 135 | impl<'cursor, 'tree> IntoIterator for &'cursor mut TreeCursor<'tree> { 136 | type Item = Node<'tree>; 137 | type IntoIter = TreeRecursiveWalker<'cursor, 'tree>; 138 | 139 | fn into_iter(self) -> Self::IntoIter { 140 | let mut queue = VecDeque::new(); 141 | let root = self.node(); 142 | queue.push_back(root.clone()); 143 | 144 | TreeRecursiveWalker { 145 | cursor: self, 146 | queue, 147 | root, 148 | } 149 | } 150 | } 151 | 152 | pub struct TreeRecursiveWalker<'cursor, 'tree> { 153 | cursor: &'cursor mut TreeCursor<'tree>, 154 | queue: VecDeque>, 155 | root: Node<'tree>, 156 | } 157 | 158 | impl<'tree> Iterator for TreeRecursiveWalker<'_, 'tree> { 159 | type Item = Node<'tree>; 160 | 161 | fn next(&mut self) -> Option { 162 | let current = self.cursor.node(); 163 | 164 | if current != self.root && self.cursor.goto_next_sibling() { 165 | self.queue.push_back(current); 166 | return Some(self.cursor.node()); 167 | } 168 | 169 | while let Some(queued) = self.queue.pop_front() { 170 | self.cursor.cursor.reset(&queued); 171 | 172 | if !self.cursor.goto_first_child() { 173 | continue; 174 | } 175 | 176 | return Some(self.cursor.node()); 177 | } 178 | 179 | None 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /skidder/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "skidder" 3 | version = "0.1.0" 4 | edition = "2021" 5 | description = "A package manager for tree-sitter" 6 | authors = ["Pascal Kuthe "] 7 | license = "MPL-2.0" 8 | repository = "https://github.com/helix-editor/tree-house" 9 | readme = "../README.md" 10 | rust-version = "1.74.0" 11 | 12 | [dependencies] 13 | anyhow = "1.0" 14 | cc = "1.1" 15 | indicatif = "0.17" 16 | ruzstd = "0.7" 17 | serde = { version = "1.0", features = ["derive"] } 18 | serde_json = "1.0" 19 | sha1 = "0.10" 20 | tempfile = "3.10" 21 | walkdir = "2.5" 22 | 23 | -------------------------------------------------------------------------------- /skidder/LICENSE: -------------------------------------------------------------------------------- 1 | ../LICENSE -------------------------------------------------------------------------------- /skidder/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | println!( 3 | "cargo:rustc-env=BUILD_TARGET={}", 4 | std::env::var("TARGET").unwrap() 5 | ); 6 | } 7 | -------------------------------------------------------------------------------- /test-grammars/comment/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Santos Gallegos 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /test-grammars/comment/highlights.scm: -------------------------------------------------------------------------------- 1 | (tag 2 | (name) @ui.text 3 | (user)? @constant) 4 | 5 | ; Hint level tags 6 | ((tag (name) @hint) 7 | (#any-of? @hint "HINT" "MARK" "PASSED" "STUB" "MOCK")) 8 | 9 | ("text" @hint 10 | (#any-of? @hint "HINT" "MARK" "PASSED" "STUB" "MOCK")) 11 | 12 | ; Info level tags 13 | ((tag (name) @info) 14 | (#any-of? @info "INFO" "NOTE" "TODO" "PERF" "OPTIMIZE" "PERFORMANCE" "QUESTION" "ASK")) 15 | 16 | ("text" @info 17 | (#any-of? @info "INFO" "NOTE" "TODO" "PERF" "OPTIMIZE" "PERFORMANCE" "QUESTION" "ASK")) 18 | 19 | ; Warning level tags 20 | ((tag (name) @warning) 21 | (#any-of? @warning "HACK" "WARN" "WARNING" "TEST" "TEMP")) 22 | 23 | ("text" @warning 24 | (#any-of? @warning "HACK" "WARN" "WARNING" "TEST" "TEMP")) 25 | 26 | ; Error level tags 27 | ((tag (name) @error) 28 | (#any-of? @error "BUG" "FIXME" "ISSUE" "XXX" "FIX" "SAFETY" "FIXIT" "FAILED" "DEBUG" "INVARIANT" "COMPLIANCE")) 29 | 30 | ("text" @error 31 | (#any-of? @error "BUG" "FIXME" "ISSUE" "XXX" "FIX" "SAFETY" "FIXIT" "FAILED" "DEBUG" "INVARIANT" "COMPLIANCE")) 32 | 33 | ; Issue number (#123) 34 | ("text" @constant.numeric 35 | (#match? @constant.numeric "^#[0-9]+$")) 36 | 37 | ; User mention (@user) 38 | ("text" @tag 39 | (#match? @tag "^[@][a-zA-Z0-9_-]+$")) 40 | 41 | (uri) @markup.link.url 42 | -------------------------------------------------------------------------------- /test-grammars/comment/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "repo": "https://github.com/stsewd/tree-sitter-comment", 3 | "rev": "aefcc2813392eb6ffe509aa0fc8b4e9b57413ee1", 4 | "license": "MIT", 5 | "compressed": true 6 | } -------------------------------------------------------------------------------- /test-grammars/comment/src/grammar.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/helix-editor/tree-house/959309a5e86e71cd1140af9c35d0148eed6a4b12/test-grammars/comment/src/grammar.json -------------------------------------------------------------------------------- /test-grammars/comment/src/parser.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/helix-editor/tree-house/959309a5e86e71cd1140af9c35d0148eed6a4b12/test-grammars/comment/src/parser.c -------------------------------------------------------------------------------- /test-grammars/comment/src/scanner.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "tree_sitter_comment/parser.c" 4 | #include "tree_sitter_comment/tokens.h" 5 | 6 | void* tree_sitter_comment_external_scanner_create() 7 | { 8 | return NULL; 9 | } 10 | 11 | void tree_sitter_comment_external_scanner_destroy(void* payload) 12 | { 13 | } 14 | 15 | unsigned tree_sitter_comment_external_scanner_serialize( 16 | void* payload, 17 | char* buffer) 18 | { 19 | return 0; 20 | } 21 | 22 | void tree_sitter_comment_external_scanner_deserialize( 23 | void* payload, 24 | const char* buffer, 25 | unsigned length) 26 | { 27 | } 28 | 29 | bool tree_sitter_comment_external_scanner_scan( 30 | void* payload, 31 | TSLexer* lexer, 32 | const bool* valid_symbols) 33 | { 34 | return parse(lexer, valid_symbols); 35 | } 36 | -------------------------------------------------------------------------------- /test-grammars/comment/src/tree_sitter/parser.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_PARSER_H_ 2 | #define TREE_SITTER_PARSER_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #define ts_builtin_sym_error ((TSSymbol)-1) 13 | #define ts_builtin_sym_end 0 14 | #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 15 | 16 | typedef uint16_t TSStateId; 17 | 18 | #ifndef TREE_SITTER_API_H_ 19 | typedef uint16_t TSSymbol; 20 | typedef uint16_t TSFieldId; 21 | typedef struct TSLanguage TSLanguage; 22 | #endif 23 | 24 | typedef struct { 25 | TSFieldId field_id; 26 | uint8_t child_index; 27 | bool inherited; 28 | } TSFieldMapEntry; 29 | 30 | typedef struct { 31 | uint16_t index; 32 | uint16_t length; 33 | } TSFieldMapSlice; 34 | 35 | typedef struct { 36 | bool visible; 37 | bool named; 38 | bool supertype; 39 | } TSSymbolMetadata; 40 | 41 | typedef struct TSLexer TSLexer; 42 | 43 | struct TSLexer { 44 | int32_t lookahead; 45 | TSSymbol result_symbol; 46 | void (*advance)(TSLexer *, bool); 47 | void (*mark_end)(TSLexer *); 48 | uint32_t (*get_column)(TSLexer *); 49 | bool (*is_at_included_range_start)(const TSLexer *); 50 | bool (*eof)(const TSLexer *); 51 | }; 52 | 53 | typedef enum { 54 | TSParseActionTypeShift, 55 | TSParseActionTypeReduce, 56 | TSParseActionTypeAccept, 57 | TSParseActionTypeRecover, 58 | } TSParseActionType; 59 | 60 | typedef union { 61 | struct { 62 | uint8_t type; 63 | TSStateId state; 64 | bool extra; 65 | bool repetition; 66 | } shift; 67 | struct { 68 | uint8_t type; 69 | uint8_t child_count; 70 | TSSymbol symbol; 71 | int16_t dynamic_precedence; 72 | uint16_t production_id; 73 | } reduce; 74 | uint8_t type; 75 | } TSParseAction; 76 | 77 | typedef struct { 78 | uint16_t lex_state; 79 | uint16_t external_lex_state; 80 | } TSLexMode; 81 | 82 | typedef union { 83 | TSParseAction action; 84 | struct { 85 | uint8_t count; 86 | bool reusable; 87 | } entry; 88 | } TSParseActionEntry; 89 | 90 | struct TSLanguage { 91 | uint32_t version; 92 | uint32_t symbol_count; 93 | uint32_t alias_count; 94 | uint32_t token_count; 95 | uint32_t external_token_count; 96 | uint32_t state_count; 97 | uint32_t large_state_count; 98 | uint32_t production_id_count; 99 | uint32_t field_count; 100 | uint16_t max_alias_sequence_length; 101 | const uint16_t *parse_table; 102 | const uint16_t *small_parse_table; 103 | const uint32_t *small_parse_table_map; 104 | const TSParseActionEntry *parse_actions; 105 | const char * const *symbol_names; 106 | const char * const *field_names; 107 | const TSFieldMapSlice *field_map_slices; 108 | const TSFieldMapEntry *field_map_entries; 109 | const TSSymbolMetadata *symbol_metadata; 110 | const TSSymbol *public_symbol_map; 111 | const uint16_t *alias_map; 112 | const TSSymbol *alias_sequences; 113 | const TSLexMode *lex_modes; 114 | bool (*lex_fn)(TSLexer *, TSStateId); 115 | bool (*keyword_lex_fn)(TSLexer *, TSStateId); 116 | TSSymbol keyword_capture_token; 117 | struct { 118 | const bool *states; 119 | const TSSymbol *symbol_map; 120 | void *(*create)(void); 121 | void (*destroy)(void *); 122 | bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist); 123 | unsigned (*serialize)(void *, char *); 124 | void (*deserialize)(void *, const char *, unsigned); 125 | } external_scanner; 126 | const TSStateId *primary_state_ids; 127 | }; 128 | 129 | /* 130 | * Lexer Macros 131 | */ 132 | 133 | #define START_LEXER() \ 134 | bool result = false; \ 135 | bool skip = false; \ 136 | bool eof = false; \ 137 | int32_t lookahead; \ 138 | goto start; \ 139 | next_state: \ 140 | lexer->advance(lexer, skip); \ 141 | start: \ 142 | skip = false; \ 143 | lookahead = lexer->lookahead; 144 | 145 | #define ADVANCE(state_value) \ 146 | { \ 147 | state = state_value; \ 148 | goto next_state; \ 149 | } 150 | 151 | #define SKIP(state_value) \ 152 | { \ 153 | skip = true; \ 154 | state = state_value; \ 155 | goto next_state; \ 156 | } 157 | 158 | #define ACCEPT_TOKEN(symbol_value) \ 159 | result = true; \ 160 | lexer->result_symbol = symbol_value; \ 161 | lexer->mark_end(lexer); 162 | 163 | #define END_STATE() return result; 164 | 165 | /* 166 | * Parse Table Macros 167 | */ 168 | 169 | #define SMALL_STATE(id) id - LARGE_STATE_COUNT 170 | 171 | #define STATE(id) id 172 | 173 | #define ACTIONS(id) id 174 | 175 | #define SHIFT(state_value) \ 176 | {{ \ 177 | .shift = { \ 178 | .type = TSParseActionTypeShift, \ 179 | .state = state_value \ 180 | } \ 181 | }} 182 | 183 | #define SHIFT_REPEAT(state_value) \ 184 | {{ \ 185 | .shift = { \ 186 | .type = TSParseActionTypeShift, \ 187 | .state = state_value, \ 188 | .repetition = true \ 189 | } \ 190 | }} 191 | 192 | #define SHIFT_EXTRA() \ 193 | {{ \ 194 | .shift = { \ 195 | .type = TSParseActionTypeShift, \ 196 | .extra = true \ 197 | } \ 198 | }} 199 | 200 | #define REDUCE(symbol_val, child_count_val, ...) \ 201 | {{ \ 202 | .reduce = { \ 203 | .type = TSParseActionTypeReduce, \ 204 | .symbol = symbol_val, \ 205 | .child_count = child_count_val, \ 206 | __VA_ARGS__ \ 207 | }, \ 208 | }} 209 | 210 | #define RECOVER() \ 211 | {{ \ 212 | .type = TSParseActionTypeRecover \ 213 | }} 214 | 215 | #define ACCEPT_INPUT() \ 216 | {{ \ 217 | .type = TSParseActionTypeAccept \ 218 | }} 219 | 220 | #ifdef __cplusplus 221 | } 222 | #endif 223 | 224 | #endif // TREE_SITTER_PARSER_H_ 225 | -------------------------------------------------------------------------------- /test-grammars/comment/src/tree_sitter_comment/chars.c: -------------------------------------------------------------------------------- 1 | #include "chars.h" 2 | 3 | bool is_upper(int32_t c) 4 | { 5 | const int32_t upper = 65; 6 | const int32_t lower = 90; 7 | return c >= upper && c <= lower; 8 | } 9 | 10 | bool is_digit(int32_t c) 11 | { 12 | const int32_t upper = 48; 13 | const int32_t lower = 57; 14 | return c >= upper && c <= lower; 15 | } 16 | 17 | bool is_newline(int32_t c) 18 | { 19 | const int32_t newline_chars[] = { 20 | CHAR_EOF, 21 | CHAR_NEWLINE, 22 | CHAR_CARRIAGE_RETURN, 23 | }; 24 | const int length = sizeof(newline_chars) / sizeof(int32_t); 25 | for (int i = 0; i < length; i++) { 26 | if (c == newline_chars[i]) { 27 | return true; 28 | } 29 | } 30 | return false; 31 | } 32 | 33 | bool is_space(int32_t c) 34 | { 35 | const int32_t space_chars[] = { 36 | CHAR_SPACE, 37 | CHAR_FORM_FEED, 38 | CHAR_TAB, 39 | CHAR_VERTICAL_TAB, 40 | }; 41 | const int length = sizeof(space_chars) / sizeof(int32_t); 42 | bool is_space_char = false; 43 | for (int i = 0; i < length; i++) { 44 | if (c == space_chars[i]) { 45 | is_space_char = true; 46 | break; 47 | } 48 | } 49 | return is_space_char || is_newline(c); 50 | } 51 | 52 | /// Check if the character is allowed inside the name. 53 | bool is_internal_char(int32_t c) 54 | { 55 | const int32_t valid_chars[] = { 56 | '-', 57 | '_', 58 | }; 59 | const int length = sizeof(valid_chars) / sizeof(int32_t); 60 | for (int i = 0; i < length; i++) { 61 | if (c == valid_chars[i]) { 62 | return true; 63 | } 64 | } 65 | return false; 66 | } 67 | -------------------------------------------------------------------------------- /test-grammars/comment/src/tree_sitter_comment/chars.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_COMMENT_CHARS_H 2 | #define TREE_SITTER_COMMENT_CHARS_H 3 | 4 | #include 5 | #include 6 | 7 | #define CHAR_EOF 0 8 | #define CHAR_NEWLINE 10 9 | #define CHAR_CARRIAGE_RETURN 13 10 | 11 | #define CHAR_SPACE ' ' 12 | #define CHAR_FORM_FEED '\f' 13 | #define CHAR_TAB '\t' 14 | #define CHAR_VERTICAL_TAB '\v' 15 | 16 | bool is_internal_char(int32_t c); 17 | bool is_newline(int32_t c); 18 | bool is_space(int32_t c); 19 | bool is_upper(int32_t c); 20 | bool is_digit(int32_t c); 21 | 22 | #endif /* ifndef TREE_SITTER_COMMENT_CHARS_H */ 23 | -------------------------------------------------------------------------------- /test-grammars/comment/src/tree_sitter_comment/parser.c: -------------------------------------------------------------------------------- 1 | #include "parser.h" 2 | 3 | #include "chars.c" 4 | #include "tokens.h" 5 | #include 6 | #include 7 | 8 | /// Parse the name of the tag. 9 | /// 10 | /// They can be of the form: 11 | /// - TODO: 12 | /// - TODO: text 13 | /// - TODO(stsewd): 14 | /// - TODO(stsewd): text 15 | /// - TODO (stsewd): text 16 | bool parse_tagname(TSLexer* lexer, const bool* valid_symbols) 17 | { 18 | if (!is_upper(lexer->lookahead) || !valid_symbols[T_TAGNAME]) { 19 | return false; 20 | } 21 | 22 | int32_t previous = lexer->lookahead; 23 | lexer->advance(lexer, false); 24 | 25 | while (is_upper(lexer->lookahead) 26 | || is_digit(lexer->lookahead) 27 | || is_internal_char(lexer->lookahead)) { 28 | previous = lexer->lookahead; 29 | lexer->advance(lexer, false); 30 | } 31 | // The tag name ends here. 32 | // But we keep parsing to see if it's a valid tag name. 33 | lexer->mark_end(lexer); 34 | 35 | // It can't end with an internal char. 36 | if (is_internal_char(previous)) { 37 | return false; 38 | } 39 | 40 | // For the user component this is `\s*(`. 41 | // We don't parse that part, we just need to be sure it ends with `:\s`. 42 | if ((is_space(lexer->lookahead) && !is_newline(lexer->lookahead)) 43 | || lexer->lookahead == '(') { 44 | // Skip white spaces. 45 | while (is_space(lexer->lookahead) && !is_newline(lexer->lookahead)) { 46 | lexer->advance(lexer, false); 47 | } 48 | // Checking aperture. 49 | if (lexer->lookahead != '(') { 50 | return false; 51 | } 52 | lexer->advance(lexer, false); 53 | 54 | // Checking closure. 55 | int user_length = 0; 56 | while (lexer->lookahead != ')') { 57 | if (is_newline(lexer->lookahead)) { 58 | return false; 59 | } 60 | lexer->advance(lexer, false); 61 | user_length++; 62 | } 63 | if (user_length <= 0) { 64 | return false; 65 | } 66 | lexer->advance(lexer, false); 67 | } 68 | 69 | // It should end with `:`... 70 | if (lexer->lookahead != ':') { 71 | return false; 72 | } 73 | 74 | // ... and be followed by one space. 75 | lexer->advance(lexer, false); 76 | if (!is_space(lexer->lookahead)) { 77 | return false; 78 | } 79 | 80 | lexer->result_symbol = T_TAGNAME; 81 | return true; 82 | } 83 | 84 | bool parse(TSLexer* lexer, const bool* valid_symbols) 85 | { 86 | // If all valid symbols are true, tree-sitter is in correction mode. 87 | // We don't want to parse anything in that case. 88 | if (valid_symbols[T_INVALID_TOKEN]) { 89 | return false; 90 | } 91 | 92 | if (is_upper(lexer->lookahead) && valid_symbols[T_TAGNAME]) { 93 | return parse_tagname(lexer, valid_symbols); 94 | } 95 | 96 | return false; 97 | } 98 | -------------------------------------------------------------------------------- /test-grammars/comment/src/tree_sitter_comment/parser.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_COMMENT_PARSER_H 2 | #define TREE_SITTER_COMMENT_PARSER_H 3 | 4 | #include 5 | 6 | bool parse_tagname(TSLexer* lexer, const bool* valid_symbols); 7 | bool parse(TSLexer* lexer, const bool* valid_symbols); 8 | 9 | #endif /* ifndef TREE_SITTER_COMMENT_PARSER_H */ 10 | -------------------------------------------------------------------------------- /test-grammars/comment/src/tree_sitter_comment/tokens.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_COMMENT_TOKENS_H 2 | #define TREE_SITTER_COMMENT_TOKENS_H 3 | 4 | enum TokenType { 5 | T_TAGNAME, 6 | T_INVALID_TOKEN, 7 | }; 8 | 9 | #endif /* ifndef TREE_SITTER_COMMENT_TOKENS_H */ 10 | -------------------------------------------------------------------------------- /test-grammars/edoc/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Michael Davis 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /test-grammars/edoc/highlights.scm: -------------------------------------------------------------------------------- 1 | ((section 2 | (section_marker) @markup.heading.marker 3 | (section_content) @markup.heading.1 4 | (section_marker) @markup.heading.marker) 5 | (#eq? @markup.heading.marker "==")) 6 | 7 | ((section 8 | (section_marker) @markup.heading.marker 9 | (section_content) @markup.heading.2 10 | (section_marker) @markup.heading.marker) 11 | (#eq? @markup.heading.marker "===")) 12 | 13 | ((section 14 | (section_marker) @markup.heading.marker 15 | (section_content) @markup.heading.3 16 | (section_marker) @markup.heading.marker) 17 | (#eq? @markup.heading.marker "====")) 18 | 19 | (tag) @keyword 20 | (macro (tag) @function.macro) 21 | (macro_escape) @constant.character.escape 22 | (inline_quote) @markup.raw.inline 23 | (email_address) @markup.link.url 24 | 25 | (em_xhtml_tag 26 | (open_xhtml_tag) @tag 27 | (xhtml_tag_content) @markup.italic 28 | (close_xhtml_tag) @tag) 29 | 30 | (strong_xhtml_tag 31 | (open_xhtml_tag) @tag 32 | (xhtml_tag_content) @markup.bold 33 | (close_xhtml_tag) @tag) 34 | 35 | (module) @namespace 36 | (function) @function 37 | (type) @type 38 | 39 | ; could be @constant.numeric.integer but this looks similar to a capture 40 | (arity) @operator 41 | 42 | (expression [":" "/"] @operator) 43 | (expression ["(" ")"] @punctuation.delimiter) 44 | (macro ["{" "}"] @function.macro) 45 | 46 | [ 47 | (quote_marker) 48 | (language_identifier) 49 | (quote_content) 50 | ] @markup.raw.block 51 | 52 | (parameter) @variable.parameter 53 | -------------------------------------------------------------------------------- /test-grammars/edoc/injections.scm: -------------------------------------------------------------------------------- 1 | ((xhtml_tag) @injection.content 2 | (#set! injection.combined) 3 | (#set! injection.include-children) 4 | (#set! injection.language "html")) 5 | 6 | ((block_quote 7 | !language 8 | (quote_content) @injection.content) 9 | (#set! injection.language "erlang")) 10 | 11 | (block_quote 12 | language: (language_identifier) @injection.language 13 | (quote_content) @injection.content) 14 | 15 | ((macro 16 | (tag) @_tag 17 | (argument) @injection.content) 18 | (#eq? @_tag "@type") 19 | (#set! injection.language "erlang") 20 | (#set! injection.include-children)) 21 | -------------------------------------------------------------------------------- /test-grammars/edoc/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "repo": "https://github.com/the-mikedavis/tree-sitter-edoc", 3 | "rev": "74774af7b45dd9cefbf9510328fc6ff2374afc50", 4 | "license": "MIT", 5 | "compressed": true 6 | } -------------------------------------------------------------------------------- /test-grammars/edoc/src/grammar.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/helix-editor/tree-house/959309a5e86e71cd1140af9c35d0148eed6a4b12/test-grammars/edoc/src/grammar.json -------------------------------------------------------------------------------- /test-grammars/edoc/src/parser.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/helix-editor/tree-house/959309a5e86e71cd1140af9c35d0148eed6a4b12/test-grammars/edoc/src/parser.c -------------------------------------------------------------------------------- /test-grammars/edoc/src/tree_sitter/parser.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_PARSER_H_ 2 | #define TREE_SITTER_PARSER_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #define ts_builtin_sym_error ((TSSymbol)-1) 13 | #define ts_builtin_sym_end 0 14 | #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 15 | 16 | typedef uint16_t TSStateId; 17 | 18 | #ifndef TREE_SITTER_API_H_ 19 | typedef uint16_t TSSymbol; 20 | typedef uint16_t TSFieldId; 21 | typedef struct TSLanguage TSLanguage; 22 | #endif 23 | 24 | typedef struct { 25 | TSFieldId field_id; 26 | uint8_t child_index; 27 | bool inherited; 28 | } TSFieldMapEntry; 29 | 30 | typedef struct { 31 | uint16_t index; 32 | uint16_t length; 33 | } TSFieldMapSlice; 34 | 35 | typedef struct { 36 | bool visible; 37 | bool named; 38 | bool supertype; 39 | } TSSymbolMetadata; 40 | 41 | typedef struct TSLexer TSLexer; 42 | 43 | struct TSLexer { 44 | int32_t lookahead; 45 | TSSymbol result_symbol; 46 | void (*advance)(TSLexer *, bool); 47 | void (*mark_end)(TSLexer *); 48 | uint32_t (*get_column)(TSLexer *); 49 | bool (*is_at_included_range_start)(const TSLexer *); 50 | bool (*eof)(const TSLexer *); 51 | }; 52 | 53 | typedef enum { 54 | TSParseActionTypeShift, 55 | TSParseActionTypeReduce, 56 | TSParseActionTypeAccept, 57 | TSParseActionTypeRecover, 58 | } TSParseActionType; 59 | 60 | typedef union { 61 | struct { 62 | uint8_t type; 63 | TSStateId state; 64 | bool extra; 65 | bool repetition; 66 | } shift; 67 | struct { 68 | uint8_t type; 69 | uint8_t child_count; 70 | TSSymbol symbol; 71 | int16_t dynamic_precedence; 72 | uint16_t production_id; 73 | } reduce; 74 | uint8_t type; 75 | } TSParseAction; 76 | 77 | typedef struct { 78 | uint16_t lex_state; 79 | uint16_t external_lex_state; 80 | } TSLexMode; 81 | 82 | typedef union { 83 | TSParseAction action; 84 | struct { 85 | uint8_t count; 86 | bool reusable; 87 | } entry; 88 | } TSParseActionEntry; 89 | 90 | struct TSLanguage { 91 | uint32_t version; 92 | uint32_t symbol_count; 93 | uint32_t alias_count; 94 | uint32_t token_count; 95 | uint32_t external_token_count; 96 | uint32_t state_count; 97 | uint32_t large_state_count; 98 | uint32_t production_id_count; 99 | uint32_t field_count; 100 | uint16_t max_alias_sequence_length; 101 | const uint16_t *parse_table; 102 | const uint16_t *small_parse_table; 103 | const uint32_t *small_parse_table_map; 104 | const TSParseActionEntry *parse_actions; 105 | const char * const *symbol_names; 106 | const char * const *field_names; 107 | const TSFieldMapSlice *field_map_slices; 108 | const TSFieldMapEntry *field_map_entries; 109 | const TSSymbolMetadata *symbol_metadata; 110 | const TSSymbol *public_symbol_map; 111 | const uint16_t *alias_map; 112 | const TSSymbol *alias_sequences; 113 | const TSLexMode *lex_modes; 114 | bool (*lex_fn)(TSLexer *, TSStateId); 115 | bool (*keyword_lex_fn)(TSLexer *, TSStateId); 116 | TSSymbol keyword_capture_token; 117 | struct { 118 | const bool *states; 119 | const TSSymbol *symbol_map; 120 | void *(*create)(void); 121 | void (*destroy)(void *); 122 | bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist); 123 | unsigned (*serialize)(void *, char *); 124 | void (*deserialize)(void *, const char *, unsigned); 125 | } external_scanner; 126 | const TSStateId *primary_state_ids; 127 | }; 128 | 129 | /* 130 | * Lexer Macros 131 | */ 132 | 133 | #define START_LEXER() \ 134 | bool result = false; \ 135 | bool skip = false; \ 136 | bool eof = false; \ 137 | int32_t lookahead; \ 138 | goto start; \ 139 | next_state: \ 140 | lexer->advance(lexer, skip); \ 141 | start: \ 142 | skip = false; \ 143 | lookahead = lexer->lookahead; 144 | 145 | #define ADVANCE(state_value) \ 146 | { \ 147 | state = state_value; \ 148 | goto next_state; \ 149 | } 150 | 151 | #define SKIP(state_value) \ 152 | { \ 153 | skip = true; \ 154 | state = state_value; \ 155 | goto next_state; \ 156 | } 157 | 158 | #define ACCEPT_TOKEN(symbol_value) \ 159 | result = true; \ 160 | lexer->result_symbol = symbol_value; \ 161 | lexer->mark_end(lexer); 162 | 163 | #define END_STATE() return result; 164 | 165 | /* 166 | * Parse Table Macros 167 | */ 168 | 169 | #define SMALL_STATE(id) id - LARGE_STATE_COUNT 170 | 171 | #define STATE(id) id 172 | 173 | #define ACTIONS(id) id 174 | 175 | #define SHIFT(state_value) \ 176 | {{ \ 177 | .shift = { \ 178 | .type = TSParseActionTypeShift, \ 179 | .state = state_value \ 180 | } \ 181 | }} 182 | 183 | #define SHIFT_REPEAT(state_value) \ 184 | {{ \ 185 | .shift = { \ 186 | .type = TSParseActionTypeShift, \ 187 | .state = state_value, \ 188 | .repetition = true \ 189 | } \ 190 | }} 191 | 192 | #define SHIFT_EXTRA() \ 193 | {{ \ 194 | .shift = { \ 195 | .type = TSParseActionTypeShift, \ 196 | .extra = true \ 197 | } \ 198 | }} 199 | 200 | #define REDUCE(symbol_val, child_count_val, ...) \ 201 | {{ \ 202 | .reduce = { \ 203 | .type = TSParseActionTypeReduce, \ 204 | .symbol = symbol_val, \ 205 | .child_count = child_count_val, \ 206 | __VA_ARGS__ \ 207 | }, \ 208 | }} 209 | 210 | #define RECOVER() \ 211 | {{ \ 212 | .type = TSParseActionTypeRecover \ 213 | }} 214 | 215 | #define ACCEPT_INPUT() \ 216 | {{ \ 217 | .type = TSParseActionTypeAccept \ 218 | }} 219 | 220 | #ifdef __cplusplus 221 | } 222 | #endif 223 | 224 | #endif // TREE_SITTER_PARSER_H_ 225 | -------------------------------------------------------------------------------- /test-grammars/erlang/highlights.scm: -------------------------------------------------------------------------------- 1 | ; Comments 2 | (tripledot) @comment.discard 3 | 4 | [(comment) (line_comment) (shebang)] @comment 5 | 6 | ; Basic types 7 | (variable) @variable 8 | (atom) @string.special.symbol 9 | ((atom) @constant.builtin.boolean 10 | (#match? @constant.builtin.boolean "^(true|false)$")) 11 | [(string) (sigil)] @string 12 | (character) @constant.character 13 | (escape_sequence) @constant.character.escape 14 | 15 | (integer) @constant.numeric.integer 16 | (float) @constant.numeric.float 17 | 18 | ; Punctuation 19 | ["," "." "-" ";"] @punctuation.delimiter 20 | ["(" ")" "#" "{" "}" "[" "]" "<<" ">>"] @punctuation.bracket 21 | 22 | ; Operators 23 | (binary_operator operator: _ @operator) 24 | (unary_operator operator: _ @operator) 25 | ["/" ":" "->"] @operator 26 | 27 | (binary_operator 28 | left: (atom) @function 29 | operator: "/" 30 | right: (integer) @constant.numeric.integer) 31 | 32 | ((binary_operator operator: _ @keyword.operator) 33 | (#match? @keyword.operator "^\\w+$")) 34 | ((unary_operator operator: _ @keyword.operator) 35 | (#match? @keyword.operator "^\\w+$")) 36 | 37 | 38 | ; Keywords 39 | (attribute name: (atom) @keyword) 40 | 41 | ["case" "fun" "if" "of" "when" "end" "receive" "try" "catch" "after" "begin" "maybe"] @keyword 42 | 43 | ; Attributes 44 | ; module declaration 45 | (attribute 46 | name: (atom) @keyword 47 | (arguments (atom) @namespace) 48 | (#any-of? @keyword "module" "behaviour" "behavior")) 49 | 50 | (attribute 51 | name: (atom) @keyword 52 | (arguments 53 | . 54 | (atom) @namespace) 55 | (#eq? @keyword "import")) 56 | 57 | (attribute 58 | name: (atom) @keyword 59 | (arguments 60 | . 61 | [(atom) @type (macro)] 62 | [ 63 | (tuple (atom)? @variable.other.member) 64 | (tuple 65 | (binary_operator 66 | left: (atom) @variable.other.member 67 | operator: ["=" "::"])) 68 | (tuple 69 | (binary_operator 70 | left: 71 | (binary_operator 72 | left: (atom) @variable.other.member 73 | operator: "=") 74 | operator: "::")) 75 | ]) 76 | (#eq? @keyword "record")) 77 | 78 | (attribute 79 | name: (atom) @keyword 80 | (arguments 81 | . 82 | [ 83 | (atom) @constant 84 | (variable) @constant 85 | (call 86 | function: 87 | [(variable) (atom)] @keyword.directive) 88 | ]) 89 | (#eq? @keyword "define")) 90 | 91 | (attribute 92 | name: (atom) @keyword 93 | (arguments 94 | (_) @keyword.directive) 95 | (#any-of? @keyword "ifndef" "ifdef")) 96 | 97 | (attribute 98 | name: (atom) @keyword 99 | module: (atom) @namespace 100 | (#any-of? @keyword "spec" "callback")) 101 | 102 | (attribute 103 | name: (atom) @keyword 104 | (arguments [ 105 | (string) 106 | (sigil) 107 | ] @comment.block.documentation) 108 | (#any-of? @keyword "doc" "moduledoc")) 109 | 110 | ; Functions 111 | (function_clause name: (atom) @function) 112 | (call module: (atom) @namespace) 113 | (call function: (atom) @function) 114 | (stab_clause name: (atom) @function) 115 | (function_capture module: (atom) @namespace) 116 | (function_capture function: (atom) @function) 117 | 118 | ; Ignored variables 119 | ((variable) @comment.discard 120 | (#match? @comment.discard "^_")) 121 | 122 | ; Macros 123 | (macro 124 | "?"+ @constant 125 | name: (_) @constant 126 | !arguments) 127 | 128 | (macro 129 | "?"+ @keyword.directive 130 | name: (_) @keyword.directive) 131 | 132 | ; Parameters 133 | ; specs 134 | ((attribute 135 | name: (atom) @keyword 136 | (stab_clause 137 | pattern: (arguments (variable)? @variable.parameter) 138 | body: (variable)? @variable.parameter)) 139 | (#match? @keyword "(spec|callback)")) 140 | ; functions 141 | (function_clause pattern: (arguments (variable) @variable.parameter)) 142 | ; anonymous functions 143 | (stab_clause pattern: (arguments (variable) @variable.parameter)) 144 | ; parametric types 145 | ((attribute 146 | name: (atom) @keyword 147 | (arguments 148 | (binary_operator 149 | left: (call (arguments (variable) @variable.parameter)) 150 | operator: "::"))) 151 | (#match? @keyword "(type|opaque)")) 152 | ; macros 153 | ((attribute 154 | name: (atom) @keyword 155 | (arguments 156 | (call (arguments (variable) @variable.parameter)))) 157 | (#eq? @keyword "define")) 158 | 159 | ; Records 160 | (record_content 161 | (binary_operator 162 | left: (atom) @variable.other.member 163 | operator: "=")) 164 | 165 | (record field: (atom) @variable.other.member) 166 | (record name: (atom) @type) 167 | -------------------------------------------------------------------------------- /test-grammars/erlang/injections.scm: -------------------------------------------------------------------------------- 1 | ((line_comment (comment_content) @injection.content) 2 | (#set! injection.language "edoc") 3 | (#set! injection.include-children) 4 | (#set! injection.combined)) 5 | 6 | ((comment (comment_content) @injection.content) 7 | (#set! injection.language "comment")) 8 | 9 | ; EEP-59 doc attributes use markdown by default. 10 | (attribute 11 | name: (atom) @_attribute 12 | (arguments [ 13 | (string (quoted_content) @injection.content) 14 | (sigil (quoted_content) @injection.content) 15 | ]) 16 | (#set! injection.language "markdown") 17 | (#any-of? @_attribute "doc" "moduledoc")) 18 | -------------------------------------------------------------------------------- /test-grammars/erlang/locals.scm: -------------------------------------------------------------------------------- 1 | ; Specs and Callbacks 2 | (attribute 3 | (stab_clause 4 | pattern: (arguments (variable)? @local.definition.variable.parameter) 5 | ; If a spec uses a variable as the return type (and later a `when` clause to type it): 6 | body: (variable)? @local.definition.variable.parameter)) @local.scope 7 | 8 | ; parametric `-type`s 9 | ((attribute 10 | name: (atom) @_type 11 | (arguments 12 | (binary_operator 13 | left: (call (arguments (variable) @local.definition.variable.parameter)) 14 | operator: "::") @local.scope)) 15 | (#match? @_type "(type|opaque)")) 16 | 17 | ; `fun`s 18 | (anonymous_function (stab_clause pattern: (arguments (variable) @local.definition.variable.parameter))) @local.scope 19 | 20 | ; Ordinary functions 21 | (function_clause pattern: (arguments (variable) @local.definition.variable.parameter)) @local.scope 22 | 23 | (variable) @local.reference 24 | -------------------------------------------------------------------------------- /test-grammars/erlang/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "repo": "https://github.com/the-mikedavis/tree-sitter-erlang", 3 | "rev": "33a3e4f1fa77a3e1a2736813f4b27c358f6c0b63", 4 | "license": "Apache-2.0", 5 | "compressed": true 6 | } -------------------------------------------------------------------------------- /test-grammars/erlang/src/grammar.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/helix-editor/tree-house/959309a5e86e71cd1140af9c35d0148eed6a4b12/test-grammars/erlang/src/grammar.json -------------------------------------------------------------------------------- /test-grammars/erlang/src/parser.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/helix-editor/tree-house/959309a5e86e71cd1140af9c35d0148eed6a4b12/test-grammars/erlang/src/parser.c -------------------------------------------------------------------------------- /test-grammars/erlang/src/tree_sitter/parser.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_PARSER_H_ 2 | #define TREE_SITTER_PARSER_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #define ts_builtin_sym_error ((TSSymbol)-1) 13 | #define ts_builtin_sym_end 0 14 | #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 15 | 16 | typedef uint16_t TSStateId; 17 | 18 | #ifndef TREE_SITTER_API_H_ 19 | typedef uint16_t TSSymbol; 20 | typedef uint16_t TSFieldId; 21 | typedef struct TSLanguage TSLanguage; 22 | #endif 23 | 24 | typedef struct { 25 | TSFieldId field_id; 26 | uint8_t child_index; 27 | bool inherited; 28 | } TSFieldMapEntry; 29 | 30 | typedef struct { 31 | uint16_t index; 32 | uint16_t length; 33 | } TSFieldMapSlice; 34 | 35 | typedef struct { 36 | bool visible; 37 | bool named; 38 | bool supertype; 39 | } TSSymbolMetadata; 40 | 41 | typedef struct TSLexer TSLexer; 42 | 43 | struct TSLexer { 44 | int32_t lookahead; 45 | TSSymbol result_symbol; 46 | void (*advance)(TSLexer *, bool); 47 | void (*mark_end)(TSLexer *); 48 | uint32_t (*get_column)(TSLexer *); 49 | bool (*is_at_included_range_start)(const TSLexer *); 50 | bool (*eof)(const TSLexer *); 51 | }; 52 | 53 | typedef enum { 54 | TSParseActionTypeShift, 55 | TSParseActionTypeReduce, 56 | TSParseActionTypeAccept, 57 | TSParseActionTypeRecover, 58 | } TSParseActionType; 59 | 60 | typedef union { 61 | struct { 62 | uint8_t type; 63 | TSStateId state; 64 | bool extra; 65 | bool repetition; 66 | } shift; 67 | struct { 68 | uint8_t type; 69 | uint8_t child_count; 70 | TSSymbol symbol; 71 | int16_t dynamic_precedence; 72 | uint16_t production_id; 73 | } reduce; 74 | uint8_t type; 75 | } TSParseAction; 76 | 77 | typedef struct { 78 | uint16_t lex_state; 79 | uint16_t external_lex_state; 80 | } TSLexMode; 81 | 82 | typedef union { 83 | TSParseAction action; 84 | struct { 85 | uint8_t count; 86 | bool reusable; 87 | } entry; 88 | } TSParseActionEntry; 89 | 90 | struct TSLanguage { 91 | uint32_t version; 92 | uint32_t symbol_count; 93 | uint32_t alias_count; 94 | uint32_t token_count; 95 | uint32_t external_token_count; 96 | uint32_t state_count; 97 | uint32_t large_state_count; 98 | uint32_t production_id_count; 99 | uint32_t field_count; 100 | uint16_t max_alias_sequence_length; 101 | const uint16_t *parse_table; 102 | const uint16_t *small_parse_table; 103 | const uint32_t *small_parse_table_map; 104 | const TSParseActionEntry *parse_actions; 105 | const char * const *symbol_names; 106 | const char * const *field_names; 107 | const TSFieldMapSlice *field_map_slices; 108 | const TSFieldMapEntry *field_map_entries; 109 | const TSSymbolMetadata *symbol_metadata; 110 | const TSSymbol *public_symbol_map; 111 | const uint16_t *alias_map; 112 | const TSSymbol *alias_sequences; 113 | const TSLexMode *lex_modes; 114 | bool (*lex_fn)(TSLexer *, TSStateId); 115 | bool (*keyword_lex_fn)(TSLexer *, TSStateId); 116 | TSSymbol keyword_capture_token; 117 | struct { 118 | const bool *states; 119 | const TSSymbol *symbol_map; 120 | void *(*create)(void); 121 | void (*destroy)(void *); 122 | bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist); 123 | unsigned (*serialize)(void *, char *); 124 | void (*deserialize)(void *, const char *, unsigned); 125 | } external_scanner; 126 | const TSStateId *primary_state_ids; 127 | }; 128 | 129 | /* 130 | * Lexer Macros 131 | */ 132 | 133 | #define START_LEXER() \ 134 | bool result = false; \ 135 | bool skip = false; \ 136 | bool eof = false; \ 137 | int32_t lookahead; \ 138 | goto start; \ 139 | next_state: \ 140 | lexer->advance(lexer, skip); \ 141 | start: \ 142 | skip = false; \ 143 | lookahead = lexer->lookahead; 144 | 145 | #define ADVANCE(state_value) \ 146 | { \ 147 | state = state_value; \ 148 | goto next_state; \ 149 | } 150 | 151 | #define SKIP(state_value) \ 152 | { \ 153 | skip = true; \ 154 | state = state_value; \ 155 | goto next_state; \ 156 | } 157 | 158 | #define ACCEPT_TOKEN(symbol_value) \ 159 | result = true; \ 160 | lexer->result_symbol = symbol_value; \ 161 | lexer->mark_end(lexer); 162 | 163 | #define END_STATE() return result; 164 | 165 | /* 166 | * Parse Table Macros 167 | */ 168 | 169 | #define SMALL_STATE(id) id - LARGE_STATE_COUNT 170 | 171 | #define STATE(id) id 172 | 173 | #define ACTIONS(id) id 174 | 175 | #define SHIFT(state_value) \ 176 | {{ \ 177 | .shift = { \ 178 | .type = TSParseActionTypeShift, \ 179 | .state = state_value \ 180 | } \ 181 | }} 182 | 183 | #define SHIFT_REPEAT(state_value) \ 184 | {{ \ 185 | .shift = { \ 186 | .type = TSParseActionTypeShift, \ 187 | .state = state_value, \ 188 | .repetition = true \ 189 | } \ 190 | }} 191 | 192 | #define SHIFT_EXTRA() \ 193 | {{ \ 194 | .shift = { \ 195 | .type = TSParseActionTypeShift, \ 196 | .extra = true \ 197 | } \ 198 | }} 199 | 200 | #define REDUCE(symbol_val, child_count_val, ...) \ 201 | {{ \ 202 | .reduce = { \ 203 | .type = TSParseActionTypeReduce, \ 204 | .symbol = symbol_val, \ 205 | .child_count = child_count_val, \ 206 | __VA_ARGS__ \ 207 | }, \ 208 | }} 209 | 210 | #define RECOVER() \ 211 | {{ \ 212 | .type = TSParseActionTypeRecover \ 213 | }} 214 | 215 | #define ACCEPT_INPUT() \ 216 | {{ \ 217 | .type = TSParseActionTypeAccept \ 218 | }} 219 | 220 | #ifdef __cplusplus 221 | } 222 | #endif 223 | 224 | #endif // TREE_SITTER_PARSER_H_ 225 | -------------------------------------------------------------------------------- /test-grammars/erlang/textobjects.scm: -------------------------------------------------------------------------------- 1 | (function_clause 2 | pattern: (arguments (_)? @parameter.inside) 3 | body: (_) @function.inside) @function.around 4 | 5 | (anonymous_function 6 | (stab_clause body: (_) @function.inside)) @function.around 7 | 8 | (comment (comment_content) @comment.inside) @comment.around 9 | 10 | ; EUnit test names. 11 | ; (CommonTest cases are not recognizable by syntax alone.) 12 | ((function_clause 13 | name: (atom) @_name 14 | pattern: (arguments (_)? @parameter.inside) 15 | body: (_) @test.inside) @test.around 16 | (#match? @_name "_test$")) 17 | -------------------------------------------------------------------------------- /test-grammars/html/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Max Brunsfeld 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /test-grammars/html/highlights.scm: -------------------------------------------------------------------------------- 1 | (tag_name) @tag 2 | (erroneous_end_tag_name) @error 3 | (doctype) @constant 4 | (attribute_name) @attribute 5 | 6 | (attribute [(attribute_value) (quoted_attribute_value)] @string) 7 | 8 | ((attribute 9 | (attribute_name) @attribute 10 | (quoted_attribute_value (attribute_value) @markup.link.url)) 11 | (#any-of? @attribute "href" "src")) 12 | 13 | ((element 14 | (start_tag 15 | (tag_name) @_tag) 16 | (text) @markup.link.label) 17 | (#eq? @_tag "a")) 18 | 19 | ((element 20 | (start_tag 21 | (tag_name) @_tag) 22 | (text) @markup.bold) 23 | (#any-of? @_tag "strong" "b")) 24 | 25 | ((element 26 | (start_tag 27 | (tag_name) @_tag) 28 | (text) @markup.italic) 29 | (#any-of? @_tag "em" "i")) 30 | 31 | ((element 32 | (start_tag 33 | (tag_name) @_tag) 34 | (text) @markup.strikethrough) 35 | (#any-of? @_tag "s" "del")) 36 | 37 | [ 38 | "<" 39 | ">" 40 | "" 42 | " 2 | #include 3 | 4 | using std::string; 5 | using std::map; 6 | 7 | enum TagType { 8 | AREA, 9 | BASE, 10 | BASEFONT, 11 | BGSOUND, 12 | BR, 13 | COL, 14 | COMMAND, 15 | EMBED, 16 | FRAME, 17 | HR, 18 | IMAGE, 19 | IMG, 20 | INPUT, 21 | ISINDEX, 22 | KEYGEN, 23 | LINK, 24 | MENUITEM, 25 | META, 26 | NEXTID, 27 | PARAM, 28 | SOURCE, 29 | TRACK, 30 | WBR, 31 | END_OF_VOID_TAGS, 32 | 33 | A, 34 | ABBR, 35 | ADDRESS, 36 | ARTICLE, 37 | ASIDE, 38 | AUDIO, 39 | B, 40 | BDI, 41 | BDO, 42 | BLOCKQUOTE, 43 | BODY, 44 | BUTTON, 45 | CANVAS, 46 | CAPTION, 47 | CITE, 48 | CODE, 49 | COLGROUP, 50 | DATA, 51 | DATALIST, 52 | DD, 53 | DEL, 54 | DETAILS, 55 | DFN, 56 | DIALOG, 57 | DIV, 58 | DL, 59 | DT, 60 | EM, 61 | FIELDSET, 62 | FIGCAPTION, 63 | FIGURE, 64 | FOOTER, 65 | FORM, 66 | H1, 67 | H2, 68 | H3, 69 | H4, 70 | H5, 71 | H6, 72 | HEAD, 73 | HEADER, 74 | HGROUP, 75 | HTML, 76 | I, 77 | IFRAME, 78 | INS, 79 | KBD, 80 | LABEL, 81 | LEGEND, 82 | LI, 83 | MAIN, 84 | MAP, 85 | MARK, 86 | MATH, 87 | MENU, 88 | METER, 89 | NAV, 90 | NOSCRIPT, 91 | OBJECT, 92 | OL, 93 | OPTGROUP, 94 | OPTION, 95 | OUTPUT, 96 | P, 97 | PICTURE, 98 | PRE, 99 | PROGRESS, 100 | Q, 101 | RB, 102 | RP, 103 | RT, 104 | RTC, 105 | RUBY, 106 | S, 107 | SAMP, 108 | SCRIPT, 109 | SECTION, 110 | SELECT, 111 | SLOT, 112 | SMALL, 113 | SPAN, 114 | STRONG, 115 | STYLE, 116 | SUB, 117 | SUMMARY, 118 | SUP, 119 | SVG, 120 | TABLE, 121 | TBODY, 122 | TD, 123 | TEMPLATE, 124 | TEXTAREA, 125 | TFOOT, 126 | TH, 127 | THEAD, 128 | TIME, 129 | TITLE, 130 | TR, 131 | U, 132 | UL, 133 | VAR, 134 | VIDEO, 135 | 136 | CUSTOM, 137 | }; 138 | 139 | 140 | static const map get_tag_map() { 141 | map result; 142 | #define TAG(name) result[#name] = name 143 | TAG(AREA); 144 | TAG(BASE); 145 | TAG(BASEFONT); 146 | TAG(BGSOUND); 147 | TAG(BR); 148 | TAG(COL); 149 | TAG(COMMAND); 150 | TAG(EMBED); 151 | TAG(FRAME); 152 | TAG(HR); 153 | TAG(IMAGE); 154 | TAG(IMG); 155 | TAG(INPUT); 156 | TAG(ISINDEX); 157 | TAG(KEYGEN); 158 | TAG(LINK); 159 | TAG(MENUITEM); 160 | TAG(META); 161 | TAG(NEXTID); 162 | TAG(PARAM); 163 | TAG(SOURCE); 164 | TAG(TRACK); 165 | TAG(WBR); 166 | TAG(A); 167 | TAG(ABBR); 168 | TAG(ADDRESS); 169 | TAG(ARTICLE); 170 | TAG(ASIDE); 171 | TAG(AUDIO); 172 | TAG(B); 173 | TAG(BDI); 174 | TAG(BDO); 175 | TAG(BLOCKQUOTE); 176 | TAG(BODY); 177 | TAG(BUTTON); 178 | TAG(CANVAS); 179 | TAG(CAPTION); 180 | TAG(CITE); 181 | TAG(CODE); 182 | TAG(COLGROUP); 183 | TAG(DATA); 184 | TAG(DATALIST); 185 | TAG(DD); 186 | TAG(DEL); 187 | TAG(DETAILS); 188 | TAG(DFN); 189 | TAG(DIALOG); 190 | TAG(DIV); 191 | TAG(DL); 192 | TAG(DT); 193 | TAG(EM); 194 | TAG(FIELDSET); 195 | TAG(FIGCAPTION); 196 | TAG(FIGURE); 197 | TAG(FOOTER); 198 | TAG(FORM); 199 | TAG(H1); 200 | TAG(H2); 201 | TAG(H3); 202 | TAG(H4); 203 | TAG(H5); 204 | TAG(H6); 205 | TAG(HEAD); 206 | TAG(HEADER); 207 | TAG(HGROUP); 208 | TAG(HTML); 209 | TAG(I); 210 | TAG(IFRAME); 211 | TAG(INS); 212 | TAG(KBD); 213 | TAG(LABEL); 214 | TAG(LEGEND); 215 | TAG(LI); 216 | TAG(MAIN); 217 | TAG(MAP); 218 | TAG(MARK); 219 | TAG(MATH); 220 | TAG(MENU); 221 | TAG(METER); 222 | TAG(NAV); 223 | TAG(NOSCRIPT); 224 | TAG(OBJECT); 225 | TAG(OL); 226 | TAG(OPTGROUP); 227 | TAG(OPTION); 228 | TAG(OUTPUT); 229 | TAG(P); 230 | TAG(PICTURE); 231 | TAG(PRE); 232 | TAG(PROGRESS); 233 | TAG(Q); 234 | TAG(RB); 235 | TAG(RP); 236 | TAG(RT); 237 | TAG(RTC); 238 | TAG(RUBY); 239 | TAG(S); 240 | TAG(SAMP); 241 | TAG(SCRIPT); 242 | TAG(SECTION); 243 | TAG(SELECT); 244 | TAG(SLOT); 245 | TAG(SMALL); 246 | TAG(SPAN); 247 | TAG(STRONG); 248 | TAG(STYLE); 249 | TAG(SUB); 250 | TAG(SUMMARY); 251 | TAG(SUP); 252 | TAG(SVG); 253 | TAG(TABLE); 254 | TAG(TBODY); 255 | TAG(TD); 256 | TAG(TEMPLATE); 257 | TAG(TEXTAREA); 258 | TAG(TFOOT); 259 | TAG(TH); 260 | TAG(THEAD); 261 | TAG(TIME); 262 | TAG(TITLE); 263 | TAG(TR); 264 | TAG(U); 265 | TAG(UL); 266 | TAG(VAR); 267 | TAG(VIDEO); 268 | #undef TAG 269 | return result; 270 | } 271 | 272 | static const map TAG_TYPES_BY_TAG_NAME = get_tag_map(); 273 | 274 | static const TagType TAG_TYPES_NOT_ALLOWED_IN_PARAGRAPHS[] = { 275 | ADDRESS, 276 | ARTICLE, 277 | ASIDE, 278 | BLOCKQUOTE, 279 | DETAILS, 280 | DIV, 281 | DL, 282 | FIELDSET, 283 | FIGCAPTION, 284 | FIGURE, 285 | FOOTER, 286 | FORM, 287 | H1, 288 | H2, 289 | H3, 290 | H4, 291 | H5, 292 | H6, 293 | HEADER, 294 | HR, 295 | MAIN, 296 | NAV, 297 | OL, 298 | P, 299 | PRE, 300 | SECTION, 301 | }; 302 | 303 | static const TagType *TAG_TYPES_NOT_ALLOWED_IN_PARAGRAPHS_END = ( 304 | TAG_TYPES_NOT_ALLOWED_IN_PARAGRAPHS + 305 | sizeof(TAG_TYPES_NOT_ALLOWED_IN_PARAGRAPHS) / 306 | sizeof(TagType) 307 | ); 308 | 309 | struct Tag { 310 | TagType type; 311 | string custom_tag_name; 312 | 313 | // This default constructor is used in the case where there is not enough space 314 | // in the serialization buffer to store all of the tags. In that case, tags 315 | // that cannot be serialized will be treated as having an unknown type. These 316 | // tags will be closed via implicit end tags regardless of the next closing 317 | // tag is encountered. 318 | Tag() : type(END_OF_VOID_TAGS) {} 319 | 320 | Tag(TagType type, const string &name) : type(type), custom_tag_name(name) {} 321 | 322 | bool operator==(const Tag &other) const { 323 | if (type != other.type) return false; 324 | if (type == CUSTOM && custom_tag_name != other.custom_tag_name) return false; 325 | return true; 326 | } 327 | 328 | inline bool is_void() const { 329 | return type < END_OF_VOID_TAGS; 330 | } 331 | 332 | inline bool can_contain(const Tag &tag) { 333 | TagType child = tag.type; 334 | 335 | switch (type) { 336 | case LI: return child != LI; 337 | 338 | case DT: 339 | case DD: 340 | return child != DT && child != DD; 341 | 342 | case P: 343 | return std::find( 344 | TAG_TYPES_NOT_ALLOWED_IN_PARAGRAPHS, 345 | TAG_TYPES_NOT_ALLOWED_IN_PARAGRAPHS_END, 346 | tag.type 347 | ) == TAG_TYPES_NOT_ALLOWED_IN_PARAGRAPHS_END; 348 | 349 | case COLGROUP: 350 | return child == COL; 351 | 352 | case RB: 353 | case RT: 354 | case RP: 355 | return child != RB && child != RT && child != RP; 356 | 357 | case OPTGROUP: 358 | return child != OPTGROUP; 359 | 360 | case TR: 361 | return child != TR; 362 | 363 | case TD: 364 | case TH: 365 | return child != TD && child != TH && child != TR; 366 | 367 | default: 368 | return true; 369 | } 370 | } 371 | 372 | static inline Tag for_name(const string &name) { 373 | map::const_iterator type = TAG_TYPES_BY_TAG_NAME.find(name); 374 | if (type != TAG_TYPES_BY_TAG_NAME.end()) { 375 | return Tag(type->second, string()); 376 | } else { 377 | return Tag(CUSTOM, name); 378 | } 379 | } 380 | }; 381 | -------------------------------------------------------------------------------- /test-grammars/html/src/tree_sitter/parser.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_PARSER_H_ 2 | #define TREE_SITTER_PARSER_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #define ts_builtin_sym_error ((TSSymbol)-1) 13 | #define ts_builtin_sym_end 0 14 | #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 15 | 16 | typedef uint16_t TSStateId; 17 | 18 | #ifndef TREE_SITTER_API_H_ 19 | typedef uint16_t TSSymbol; 20 | typedef uint16_t TSFieldId; 21 | typedef struct TSLanguage TSLanguage; 22 | #endif 23 | 24 | typedef struct { 25 | TSFieldId field_id; 26 | uint8_t child_index; 27 | bool inherited; 28 | } TSFieldMapEntry; 29 | 30 | typedef struct { 31 | uint16_t index; 32 | uint16_t length; 33 | } TSFieldMapSlice; 34 | 35 | typedef struct { 36 | bool visible; 37 | bool named; 38 | bool supertype; 39 | } TSSymbolMetadata; 40 | 41 | typedef struct TSLexer TSLexer; 42 | 43 | struct TSLexer { 44 | int32_t lookahead; 45 | TSSymbol result_symbol; 46 | void (*advance)(TSLexer *, bool); 47 | void (*mark_end)(TSLexer *); 48 | uint32_t (*get_column)(TSLexer *); 49 | bool (*is_at_included_range_start)(const TSLexer *); 50 | bool (*eof)(const TSLexer *); 51 | }; 52 | 53 | typedef enum { 54 | TSParseActionTypeShift, 55 | TSParseActionTypeReduce, 56 | TSParseActionTypeAccept, 57 | TSParseActionTypeRecover, 58 | } TSParseActionType; 59 | 60 | typedef union { 61 | struct { 62 | uint8_t type; 63 | TSStateId state; 64 | bool extra; 65 | bool repetition; 66 | } shift; 67 | struct { 68 | uint8_t type; 69 | uint8_t child_count; 70 | TSSymbol symbol; 71 | int16_t dynamic_precedence; 72 | uint16_t production_id; 73 | } reduce; 74 | uint8_t type; 75 | } TSParseAction; 76 | 77 | typedef struct { 78 | uint16_t lex_state; 79 | uint16_t external_lex_state; 80 | } TSLexMode; 81 | 82 | typedef union { 83 | TSParseAction action; 84 | struct { 85 | uint8_t count; 86 | bool reusable; 87 | } entry; 88 | } TSParseActionEntry; 89 | 90 | struct TSLanguage { 91 | uint32_t version; 92 | uint32_t symbol_count; 93 | uint32_t alias_count; 94 | uint32_t token_count; 95 | uint32_t external_token_count; 96 | uint32_t state_count; 97 | uint32_t large_state_count; 98 | uint32_t production_id_count; 99 | uint32_t field_count; 100 | uint16_t max_alias_sequence_length; 101 | const uint16_t *parse_table; 102 | const uint16_t *small_parse_table; 103 | const uint32_t *small_parse_table_map; 104 | const TSParseActionEntry *parse_actions; 105 | const char * const *symbol_names; 106 | const char * const *field_names; 107 | const TSFieldMapSlice *field_map_slices; 108 | const TSFieldMapEntry *field_map_entries; 109 | const TSSymbolMetadata *symbol_metadata; 110 | const TSSymbol *public_symbol_map; 111 | const uint16_t *alias_map; 112 | const TSSymbol *alias_sequences; 113 | const TSLexMode *lex_modes; 114 | bool (*lex_fn)(TSLexer *, TSStateId); 115 | bool (*keyword_lex_fn)(TSLexer *, TSStateId); 116 | TSSymbol keyword_capture_token; 117 | struct { 118 | const bool *states; 119 | const TSSymbol *symbol_map; 120 | void *(*create)(void); 121 | void (*destroy)(void *); 122 | bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist); 123 | unsigned (*serialize)(void *, char *); 124 | void (*deserialize)(void *, const char *, unsigned); 125 | } external_scanner; 126 | }; 127 | 128 | /* 129 | * Lexer Macros 130 | */ 131 | 132 | #define START_LEXER() \ 133 | bool result = false; \ 134 | bool skip = false; \ 135 | bool eof = false; \ 136 | int32_t lookahead; \ 137 | goto start; \ 138 | next_state: \ 139 | lexer->advance(lexer, skip); \ 140 | start: \ 141 | skip = false; \ 142 | lookahead = lexer->lookahead; 143 | 144 | #define ADVANCE(state_value) \ 145 | { \ 146 | state = state_value; \ 147 | goto next_state; \ 148 | } 149 | 150 | #define SKIP(state_value) \ 151 | { \ 152 | skip = true; \ 153 | state = state_value; \ 154 | goto next_state; \ 155 | } 156 | 157 | #define ACCEPT_TOKEN(symbol_value) \ 158 | result = true; \ 159 | lexer->result_symbol = symbol_value; \ 160 | lexer->mark_end(lexer); 161 | 162 | #define END_STATE() return result; 163 | 164 | /* 165 | * Parse Table Macros 166 | */ 167 | 168 | #define SMALL_STATE(id) id - LARGE_STATE_COUNT 169 | 170 | #define STATE(id) id 171 | 172 | #define ACTIONS(id) id 173 | 174 | #define SHIFT(state_value) \ 175 | {{ \ 176 | .shift = { \ 177 | .type = TSParseActionTypeShift, \ 178 | .state = state_value \ 179 | } \ 180 | }} 181 | 182 | #define SHIFT_REPEAT(state_value) \ 183 | {{ \ 184 | .shift = { \ 185 | .type = TSParseActionTypeShift, \ 186 | .state = state_value, \ 187 | .repetition = true \ 188 | } \ 189 | }} 190 | 191 | #define SHIFT_EXTRA() \ 192 | {{ \ 193 | .shift = { \ 194 | .type = TSParseActionTypeShift, \ 195 | .extra = true \ 196 | } \ 197 | }} 198 | 199 | #define REDUCE(symbol_val, child_count_val, ...) \ 200 | {{ \ 201 | .reduce = { \ 202 | .type = TSParseActionTypeReduce, \ 203 | .symbol = symbol_val, \ 204 | .child_count = child_count_val, \ 205 | __VA_ARGS__ \ 206 | }, \ 207 | }} 208 | 209 | #define RECOVER() \ 210 | {{ \ 211 | .type = TSParseActionTypeRecover \ 212 | }} 213 | 214 | #define ACCEPT_INPUT() \ 215 | {{ \ 216 | .type = TSParseActionTypeAccept \ 217 | }} 218 | 219 | #ifdef __cplusplus 220 | } 221 | #endif 222 | 223 | #endif // TREE_SITTER_PARSER_H_ 224 | -------------------------------------------------------------------------------- /test-grammars/markdown-inline/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Matthias Deiml 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /test-grammars/markdown-inline/highlights.scm: -------------------------------------------------------------------------------- 1 | ;; From nvim-treesitter/nvim-treesitter 2 | [ 3 | (code_span) 4 | (link_title) 5 | ] @markup.raw.inline 6 | 7 | [ 8 | (emphasis_delimiter) 9 | (code_span_delimiter) 10 | ] @punctuation.bracket 11 | 12 | (emphasis) @markup.italic 13 | 14 | (strong_emphasis) @markup.bold 15 | 16 | (strikethrough) @markup.strikethrough 17 | 18 | [ 19 | (link_destination) 20 | (uri_autolink) 21 | ] @markup.link.url 22 | 23 | [ 24 | (link_text) 25 | (image_description) 26 | ] @markup.link.text 27 | 28 | (link_label) @markup.link.label 29 | 30 | [ 31 | (backslash_escape) 32 | (hard_line_break) 33 | ] @constant.character.escape 34 | 35 | (image ["[" "]" "(" ")"] @punctuation.bracket) 36 | (image "!" @punctuation.special) 37 | (inline_link ["[" "]" "(" ")"] @punctuation.bracket) 38 | (shortcut_link ["[" "]"] @punctuation.bracket) 39 | 40 | -------------------------------------------------------------------------------- /test-grammars/markdown-inline/injections.scm: -------------------------------------------------------------------------------- 1 | 2 | ((html_tag) @injection.content 3 | (#set! injection.language "html") 4 | (#set! injection.include-unnamed-children) 5 | (#set! injection.combined)) 6 | 7 | ((latex_block) @injection.content (#set! injection.language "latex") (#set! injection.include-unnamed-children)) 8 | -------------------------------------------------------------------------------- /test-grammars/markdown-inline/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "repo": "https://github.com/tree-sitter-grammars/tree-sitter-markdown", 3 | "rev": "62516e8c78380e3b51d5b55727995d2c511436d8", 4 | "license": "MIT", 5 | "compressed": true 6 | } -------------------------------------------------------------------------------- /test-grammars/markdown-inline/src/grammar.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/helix-editor/tree-house/959309a5e86e71cd1140af9c35d0148eed6a4b12/test-grammars/markdown-inline/src/grammar.json -------------------------------------------------------------------------------- /test-grammars/markdown-inline/src/parser.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/helix-editor/tree-house/959309a5e86e71cd1140af9c35d0148eed6a4b12/test-grammars/markdown-inline/src/parser.c -------------------------------------------------------------------------------- /test-grammars/markdown-inline/src/tree_sitter/alloc.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_ALLOC_H_ 2 | #define TREE_SITTER_ALLOC_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | // Allow clients to override allocation functions 13 | #ifdef TREE_SITTER_REUSE_ALLOCATOR 14 | 15 | extern void *(*ts_current_malloc)(size_t); 16 | extern void *(*ts_current_calloc)(size_t, size_t); 17 | extern void *(*ts_current_realloc)(void *, size_t); 18 | extern void (*ts_current_free)(void *); 19 | 20 | #ifndef ts_malloc 21 | #define ts_malloc ts_current_malloc 22 | #endif 23 | #ifndef ts_calloc 24 | #define ts_calloc ts_current_calloc 25 | #endif 26 | #ifndef ts_realloc 27 | #define ts_realloc ts_current_realloc 28 | #endif 29 | #ifndef ts_free 30 | #define ts_free ts_current_free 31 | #endif 32 | 33 | #else 34 | 35 | #ifndef ts_malloc 36 | #define ts_malloc malloc 37 | #endif 38 | #ifndef ts_calloc 39 | #define ts_calloc calloc 40 | #endif 41 | #ifndef ts_realloc 42 | #define ts_realloc realloc 43 | #endif 44 | #ifndef ts_free 45 | #define ts_free free 46 | #endif 47 | 48 | #endif 49 | 50 | #ifdef __cplusplus 51 | } 52 | #endif 53 | 54 | #endif // TREE_SITTER_ALLOC_H_ 55 | -------------------------------------------------------------------------------- /test-grammars/markdown-inline/src/tree_sitter/parser.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_PARSER_H_ 2 | #define TREE_SITTER_PARSER_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #define ts_builtin_sym_error ((TSSymbol)-1) 13 | #define ts_builtin_sym_end 0 14 | #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 15 | 16 | #ifndef TREE_SITTER_API_H_ 17 | typedef uint16_t TSStateId; 18 | typedef uint16_t TSSymbol; 19 | typedef uint16_t TSFieldId; 20 | typedef struct TSLanguage TSLanguage; 21 | #endif 22 | 23 | typedef struct { 24 | TSFieldId field_id; 25 | uint8_t child_index; 26 | bool inherited; 27 | } TSFieldMapEntry; 28 | 29 | typedef struct { 30 | uint16_t index; 31 | uint16_t length; 32 | } TSFieldMapSlice; 33 | 34 | typedef struct { 35 | bool visible; 36 | bool named; 37 | bool supertype; 38 | } TSSymbolMetadata; 39 | 40 | typedef struct TSLexer TSLexer; 41 | 42 | struct TSLexer { 43 | int32_t lookahead; 44 | TSSymbol result_symbol; 45 | void (*advance)(TSLexer *, bool); 46 | void (*mark_end)(TSLexer *); 47 | uint32_t (*get_column)(TSLexer *); 48 | bool (*is_at_included_range_start)(const TSLexer *); 49 | bool (*eof)(const TSLexer *); 50 | }; 51 | 52 | typedef enum { 53 | TSParseActionTypeShift, 54 | TSParseActionTypeReduce, 55 | TSParseActionTypeAccept, 56 | TSParseActionTypeRecover, 57 | } TSParseActionType; 58 | 59 | typedef union { 60 | struct { 61 | uint8_t type; 62 | TSStateId state; 63 | bool extra; 64 | bool repetition; 65 | } shift; 66 | struct { 67 | uint8_t type; 68 | uint8_t child_count; 69 | TSSymbol symbol; 70 | int16_t dynamic_precedence; 71 | uint16_t production_id; 72 | } reduce; 73 | uint8_t type; 74 | } TSParseAction; 75 | 76 | typedef struct { 77 | uint16_t lex_state; 78 | uint16_t external_lex_state; 79 | } TSLexMode; 80 | 81 | typedef union { 82 | TSParseAction action; 83 | struct { 84 | uint8_t count; 85 | bool reusable; 86 | } entry; 87 | } TSParseActionEntry; 88 | 89 | struct TSLanguage { 90 | uint32_t version; 91 | uint32_t symbol_count; 92 | uint32_t alias_count; 93 | uint32_t token_count; 94 | uint32_t external_token_count; 95 | uint32_t state_count; 96 | uint32_t large_state_count; 97 | uint32_t production_id_count; 98 | uint32_t field_count; 99 | uint16_t max_alias_sequence_length; 100 | const uint16_t *parse_table; 101 | const uint16_t *small_parse_table; 102 | const uint32_t *small_parse_table_map; 103 | const TSParseActionEntry *parse_actions; 104 | const char * const *symbol_names; 105 | const char * const *field_names; 106 | const TSFieldMapSlice *field_map_slices; 107 | const TSFieldMapEntry *field_map_entries; 108 | const TSSymbolMetadata *symbol_metadata; 109 | const TSSymbol *public_symbol_map; 110 | const uint16_t *alias_map; 111 | const TSSymbol *alias_sequences; 112 | const TSLexMode *lex_modes; 113 | bool (*lex_fn)(TSLexer *, TSStateId); 114 | bool (*keyword_lex_fn)(TSLexer *, TSStateId); 115 | TSSymbol keyword_capture_token; 116 | struct { 117 | const bool *states; 118 | const TSSymbol *symbol_map; 119 | void *(*create)(void); 120 | void (*destroy)(void *); 121 | bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist); 122 | unsigned (*serialize)(void *, char *); 123 | void (*deserialize)(void *, const char *, unsigned); 124 | } external_scanner; 125 | const TSStateId *primary_state_ids; 126 | }; 127 | 128 | /* 129 | * Lexer Macros 130 | */ 131 | 132 | #ifdef _MSC_VER 133 | #define UNUSED __pragma(warning(suppress : 4101)) 134 | #else 135 | #define UNUSED __attribute__((unused)) 136 | #endif 137 | 138 | #define START_LEXER() \ 139 | bool result = false; \ 140 | bool skip = false; \ 141 | UNUSED \ 142 | bool eof = false; \ 143 | int32_t lookahead; \ 144 | goto start; \ 145 | next_state: \ 146 | lexer->advance(lexer, skip); \ 147 | start: \ 148 | skip = false; \ 149 | lookahead = lexer->lookahead; 150 | 151 | #define ADVANCE(state_value) \ 152 | { \ 153 | state = state_value; \ 154 | goto next_state; \ 155 | } 156 | 157 | #define SKIP(state_value) \ 158 | { \ 159 | skip = true; \ 160 | state = state_value; \ 161 | goto next_state; \ 162 | } 163 | 164 | #define ACCEPT_TOKEN(symbol_value) \ 165 | result = true; \ 166 | lexer->result_symbol = symbol_value; \ 167 | lexer->mark_end(lexer); 168 | 169 | #define END_STATE() return result; 170 | 171 | /* 172 | * Parse Table Macros 173 | */ 174 | 175 | #define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT) 176 | 177 | #define STATE(id) id 178 | 179 | #define ACTIONS(id) id 180 | 181 | #define SHIFT(state_value) \ 182 | {{ \ 183 | .shift = { \ 184 | .type = TSParseActionTypeShift, \ 185 | .state = (state_value) \ 186 | } \ 187 | }} 188 | 189 | #define SHIFT_REPEAT(state_value) \ 190 | {{ \ 191 | .shift = { \ 192 | .type = TSParseActionTypeShift, \ 193 | .state = (state_value), \ 194 | .repetition = true \ 195 | } \ 196 | }} 197 | 198 | #define SHIFT_EXTRA() \ 199 | {{ \ 200 | .shift = { \ 201 | .type = TSParseActionTypeShift, \ 202 | .extra = true \ 203 | } \ 204 | }} 205 | 206 | #define REDUCE(symbol_val, child_count_val, ...) \ 207 | {{ \ 208 | .reduce = { \ 209 | .type = TSParseActionTypeReduce, \ 210 | .symbol = symbol_val, \ 211 | .child_count = child_count_val, \ 212 | __VA_ARGS__ \ 213 | }, \ 214 | }} 215 | 216 | #define RECOVER() \ 217 | {{ \ 218 | .type = TSParseActionTypeRecover \ 219 | }} 220 | 221 | #define ACCEPT_INPUT() \ 222 | {{ \ 223 | .type = TSParseActionTypeAccept \ 224 | }} 225 | 226 | #ifdef __cplusplus 227 | } 228 | #endif 229 | 230 | #endif // TREE_SITTER_PARSER_H_ 231 | -------------------------------------------------------------------------------- /test-grammars/markdown/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Matthias Deiml 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /test-grammars/markdown/highlights.scm: -------------------------------------------------------------------------------- 1 | 2 | (setext_heading (paragraph) @markup.heading.1 (setext_h1_underline) @markup.heading.marker) 3 | (setext_heading (paragraph) @markup.heading.2 (setext_h2_underline) @markup.heading.marker) 4 | 5 | (atx_heading (atx_h1_marker) @markup.heading.marker) @markup.heading.1 6 | (atx_heading (atx_h2_marker) @markup.heading.marker) @markup.heading.2 7 | (atx_heading (atx_h3_marker) @markup.heading.marker) @markup.heading.3 8 | (atx_heading (atx_h4_marker) @markup.heading.marker) @markup.heading.4 9 | (atx_heading (atx_h5_marker) @markup.heading.marker) @markup.heading.5 10 | (atx_heading (atx_h6_marker) @markup.heading.marker) @markup.heading.6 11 | 12 | [ 13 | (indented_code_block) 14 | (fenced_code_block) 15 | ] @markup.raw.block 16 | 17 | (info_string) @label 18 | 19 | [ 20 | (fenced_code_block_delimiter) 21 | ] @punctuation.bracket 22 | 23 | [ 24 | (link_destination) 25 | ] @markup.link.url 26 | 27 | [ 28 | (link_label) 29 | ] @markup.link.label 30 | 31 | [ 32 | (list_marker_plus) 33 | (list_marker_minus) 34 | (list_marker_star) 35 | ] @markup.list.unnumbered 36 | 37 | [ 38 | (list_marker_dot) 39 | (list_marker_parenthesis) 40 | ] @markup.list.numbered 41 | 42 | (task_list_marker_checked) @markup.list.checked 43 | (task_list_marker_unchecked) @markup.list.unchecked 44 | 45 | (thematic_break) @punctuation.special 46 | 47 | [ 48 | (block_continuation) 49 | (block_quote_marker) 50 | ] @punctuation.special 51 | 52 | [ 53 | (backslash_escape) 54 | ] @string.escape 55 | 56 | (block_quote) @markup.quote 57 | 58 | (pipe_table_row 59 | "|" @punctuation.special) 60 | (pipe_table_header 61 | "|" @punctuation.special) 62 | (pipe_table_delimiter_row) @punctuation.special 63 | -------------------------------------------------------------------------------- /test-grammars/markdown/injections.scm: -------------------------------------------------------------------------------- 1 | ; From nvim-treesitter/nvim-treesitter 2 | 3 | (fenced_code_block 4 | (code_fence_content) @injection.shebang @injection.content 5 | (#set! injection.include-unnamed-children)) 6 | 7 | (fenced_code_block 8 | (info_string 9 | (language) @injection.language) 10 | (code_fence_content) @injection.content (#set! injection.include-unnamed-children)) 11 | 12 | ((html_block) @injection.content 13 | (#set! injection.language "html") 14 | (#set! injection.include-unnamed-children) 15 | (#set! injection.combined)) 16 | 17 | ((pipe_table_cell) @injection.content (#set! injection.language "markdown-inline") (#set! injection.include-unnamed-children)) 18 | 19 | ((minus_metadata) @injection.content (#set! injection.language "yaml") (#set! injection.include-unnamed-children)) 20 | ((plus_metadata) @injection.content (#set! injection.language "toml") (#set! injection.include-unnamed-children)) 21 | 22 | ((inline) @injection.content (#set! injection.language "markdown-inline") (#set! injection.include-unnamed-children)) 23 | -------------------------------------------------------------------------------- /test-grammars/markdown/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "repo": "https://github.com/tree-sitter-grammars/tree-sitter-markdown", 3 | "rev": "62516e8c78380e3b51d5b55727995d2c511436d8", 4 | "license": "MIT", 5 | "compressed": true 6 | } -------------------------------------------------------------------------------- /test-grammars/markdown/src/grammar.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/helix-editor/tree-house/959309a5e86e71cd1140af9c35d0148eed6a4b12/test-grammars/markdown/src/grammar.json -------------------------------------------------------------------------------- /test-grammars/markdown/src/parser.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/helix-editor/tree-house/959309a5e86e71cd1140af9c35d0148eed6a4b12/test-grammars/markdown/src/parser.c -------------------------------------------------------------------------------- /test-grammars/markdown/src/tree_sitter/alloc.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_ALLOC_H_ 2 | #define TREE_SITTER_ALLOC_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | // Allow clients to override allocation functions 13 | #ifdef TREE_SITTER_REUSE_ALLOCATOR 14 | 15 | extern void *(*ts_current_malloc)(size_t); 16 | extern void *(*ts_current_calloc)(size_t, size_t); 17 | extern void *(*ts_current_realloc)(void *, size_t); 18 | extern void (*ts_current_free)(void *); 19 | 20 | #ifndef ts_malloc 21 | #define ts_malloc ts_current_malloc 22 | #endif 23 | #ifndef ts_calloc 24 | #define ts_calloc ts_current_calloc 25 | #endif 26 | #ifndef ts_realloc 27 | #define ts_realloc ts_current_realloc 28 | #endif 29 | #ifndef ts_free 30 | #define ts_free ts_current_free 31 | #endif 32 | 33 | #else 34 | 35 | #ifndef ts_malloc 36 | #define ts_malloc malloc 37 | #endif 38 | #ifndef ts_calloc 39 | #define ts_calloc calloc 40 | #endif 41 | #ifndef ts_realloc 42 | #define ts_realloc realloc 43 | #endif 44 | #ifndef ts_free 45 | #define ts_free free 46 | #endif 47 | 48 | #endif 49 | 50 | #ifdef __cplusplus 51 | } 52 | #endif 53 | 54 | #endif // TREE_SITTER_ALLOC_H_ 55 | -------------------------------------------------------------------------------- /test-grammars/markdown/src/tree_sitter/parser.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_PARSER_H_ 2 | #define TREE_SITTER_PARSER_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #define ts_builtin_sym_error ((TSSymbol)-1) 13 | #define ts_builtin_sym_end 0 14 | #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 15 | 16 | #ifndef TREE_SITTER_API_H_ 17 | typedef uint16_t TSStateId; 18 | typedef uint16_t TSSymbol; 19 | typedef uint16_t TSFieldId; 20 | typedef struct TSLanguage TSLanguage; 21 | #endif 22 | 23 | typedef struct { 24 | TSFieldId field_id; 25 | uint8_t child_index; 26 | bool inherited; 27 | } TSFieldMapEntry; 28 | 29 | typedef struct { 30 | uint16_t index; 31 | uint16_t length; 32 | } TSFieldMapSlice; 33 | 34 | typedef struct { 35 | bool visible; 36 | bool named; 37 | bool supertype; 38 | } TSSymbolMetadata; 39 | 40 | typedef struct TSLexer TSLexer; 41 | 42 | struct TSLexer { 43 | int32_t lookahead; 44 | TSSymbol result_symbol; 45 | void (*advance)(TSLexer *, bool); 46 | void (*mark_end)(TSLexer *); 47 | uint32_t (*get_column)(TSLexer *); 48 | bool (*is_at_included_range_start)(const TSLexer *); 49 | bool (*eof)(const TSLexer *); 50 | }; 51 | 52 | typedef enum { 53 | TSParseActionTypeShift, 54 | TSParseActionTypeReduce, 55 | TSParseActionTypeAccept, 56 | TSParseActionTypeRecover, 57 | } TSParseActionType; 58 | 59 | typedef union { 60 | struct { 61 | uint8_t type; 62 | TSStateId state; 63 | bool extra; 64 | bool repetition; 65 | } shift; 66 | struct { 67 | uint8_t type; 68 | uint8_t child_count; 69 | TSSymbol symbol; 70 | int16_t dynamic_precedence; 71 | uint16_t production_id; 72 | } reduce; 73 | uint8_t type; 74 | } TSParseAction; 75 | 76 | typedef struct { 77 | uint16_t lex_state; 78 | uint16_t external_lex_state; 79 | } TSLexMode; 80 | 81 | typedef union { 82 | TSParseAction action; 83 | struct { 84 | uint8_t count; 85 | bool reusable; 86 | } entry; 87 | } TSParseActionEntry; 88 | 89 | struct TSLanguage { 90 | uint32_t version; 91 | uint32_t symbol_count; 92 | uint32_t alias_count; 93 | uint32_t token_count; 94 | uint32_t external_token_count; 95 | uint32_t state_count; 96 | uint32_t large_state_count; 97 | uint32_t production_id_count; 98 | uint32_t field_count; 99 | uint16_t max_alias_sequence_length; 100 | const uint16_t *parse_table; 101 | const uint16_t *small_parse_table; 102 | const uint32_t *small_parse_table_map; 103 | const TSParseActionEntry *parse_actions; 104 | const char * const *symbol_names; 105 | const char * const *field_names; 106 | const TSFieldMapSlice *field_map_slices; 107 | const TSFieldMapEntry *field_map_entries; 108 | const TSSymbolMetadata *symbol_metadata; 109 | const TSSymbol *public_symbol_map; 110 | const uint16_t *alias_map; 111 | const TSSymbol *alias_sequences; 112 | const TSLexMode *lex_modes; 113 | bool (*lex_fn)(TSLexer *, TSStateId); 114 | bool (*keyword_lex_fn)(TSLexer *, TSStateId); 115 | TSSymbol keyword_capture_token; 116 | struct { 117 | const bool *states; 118 | const TSSymbol *symbol_map; 119 | void *(*create)(void); 120 | void (*destroy)(void *); 121 | bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist); 122 | unsigned (*serialize)(void *, char *); 123 | void (*deserialize)(void *, const char *, unsigned); 124 | } external_scanner; 125 | const TSStateId *primary_state_ids; 126 | }; 127 | 128 | /* 129 | * Lexer Macros 130 | */ 131 | 132 | #ifdef _MSC_VER 133 | #define UNUSED __pragma(warning(suppress : 4101)) 134 | #else 135 | #define UNUSED __attribute__((unused)) 136 | #endif 137 | 138 | #define START_LEXER() \ 139 | bool result = false; \ 140 | bool skip = false; \ 141 | UNUSED \ 142 | bool eof = false; \ 143 | int32_t lookahead; \ 144 | goto start; \ 145 | next_state: \ 146 | lexer->advance(lexer, skip); \ 147 | start: \ 148 | skip = false; \ 149 | lookahead = lexer->lookahead; 150 | 151 | #define ADVANCE(state_value) \ 152 | { \ 153 | state = state_value; \ 154 | goto next_state; \ 155 | } 156 | 157 | #define SKIP(state_value) \ 158 | { \ 159 | skip = true; \ 160 | state = state_value; \ 161 | goto next_state; \ 162 | } 163 | 164 | #define ACCEPT_TOKEN(symbol_value) \ 165 | result = true; \ 166 | lexer->result_symbol = symbol_value; \ 167 | lexer->mark_end(lexer); 168 | 169 | #define END_STATE() return result; 170 | 171 | /* 172 | * Parse Table Macros 173 | */ 174 | 175 | #define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT) 176 | 177 | #define STATE(id) id 178 | 179 | #define ACTIONS(id) id 180 | 181 | #define SHIFT(state_value) \ 182 | {{ \ 183 | .shift = { \ 184 | .type = TSParseActionTypeShift, \ 185 | .state = (state_value) \ 186 | } \ 187 | }} 188 | 189 | #define SHIFT_REPEAT(state_value) \ 190 | {{ \ 191 | .shift = { \ 192 | .type = TSParseActionTypeShift, \ 193 | .state = (state_value), \ 194 | .repetition = true \ 195 | } \ 196 | }} 197 | 198 | #define SHIFT_EXTRA() \ 199 | {{ \ 200 | .shift = { \ 201 | .type = TSParseActionTypeShift, \ 202 | .extra = true \ 203 | } \ 204 | }} 205 | 206 | #define REDUCE(symbol_val, child_count_val, ...) \ 207 | {{ \ 208 | .reduce = { \ 209 | .type = TSParseActionTypeReduce, \ 210 | .symbol = symbol_val, \ 211 | .child_count = child_count_val, \ 212 | __VA_ARGS__ \ 213 | }, \ 214 | }} 215 | 216 | #define RECOVER() \ 217 | {{ \ 218 | .type = TSParseActionTypeRecover \ 219 | }} 220 | 221 | #define ACCEPT_INPUT() \ 222 | {{ \ 223 | .type = TSParseActionTypeAccept \ 224 | }} 225 | 226 | #ifdef __cplusplus 227 | } 228 | #endif 229 | 230 | #endif // TREE_SITTER_PARSER_H_ 231 | -------------------------------------------------------------------------------- /test-grammars/rust/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 Maxim Sokolov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /test-grammars/rust/injections.scm: -------------------------------------------------------------------------------- 1 | ([(line_comment !doc) (block_comment !doc)] @injection.content 2 | (#set! injection.language "comment")) 3 | 4 | ((doc_comment) @injection.content 5 | (#set! injection.language "markdown") 6 | (#set! injection.combined)) 7 | 8 | ((macro_invocation 9 | macro: 10 | [ 11 | (scoped_identifier 12 | name: (_) @_macro_name) 13 | (identifier) @_macro_name 14 | ] 15 | (token_tree) @injection.content) 16 | (#eq? @_macro_name "html") 17 | (#set! injection.language "html") 18 | (#set! injection.include-children)) 19 | 20 | ((macro_invocation 21 | macro: 22 | [ 23 | (scoped_identifier 24 | name: (_) @_macro_name) 25 | (identifier) @_macro_name 26 | ] 27 | (token_tree) @injection.content) 28 | (#eq? @_macro_name "slint") 29 | (#set! injection.language "slint") 30 | (#set! injection.include-children)) 31 | 32 | ((macro_invocation 33 | (token_tree) @injection.content) 34 | (#set! injection.language "rust") 35 | (#set! injection.include-children)) 36 | 37 | ((macro_rule 38 | (token_tree) @injection.content) 39 | (#set! injection.language "rust") 40 | (#set! injection.include-children)) 41 | 42 | (call_expression 43 | function: (scoped_identifier 44 | path: (identifier) @_regex (#eq? @_regex "Regex") 45 | name: (identifier) @_new (#eq? @_new "new")) 46 | arguments: (arguments (raw_string_literal) @injection.content) 47 | (#set! injection.language "regex")) 48 | 49 | (call_expression 50 | function: (scoped_identifier 51 | path: (scoped_identifier (identifier) @_regex (#eq? @_regex "Regex") .) 52 | name: (identifier) @_new (#eq? @_new "new")) 53 | arguments: (arguments (raw_string_literal) @injection.content) 54 | (#set! injection.language "regex")) 55 | 56 | ; Highlight SQL in `sqlx::query!()`, `sqlx::query_scalar!()`, and `sqlx::query_scalar_unchecked!()` 57 | (macro_invocation 58 | macro: (scoped_identifier 59 | path: (identifier) @_sqlx (#eq? @_sqlx "sqlx") 60 | name: (identifier) @_query (#match? @_query "^query(_scalar|_scalar_unchecked)?$")) 61 | (token_tree 62 | ; Only the first argument is SQL 63 | . 64 | [(string_literal) (raw_string_literal)] @injection.content 65 | ) 66 | (#set! injection.language "sql")) 67 | 68 | ; Highlight SQL in `sqlx::query_as!()` and `sqlx::query_as_unchecked!()` 69 | (macro_invocation 70 | macro: (scoped_identifier 71 | path: (identifier) @_sqlx (#eq? @_sqlx "sqlx") 72 | name: (identifier) @_query_as (#match? @_query_as "^query_as(_unchecked)?$")) 73 | (token_tree 74 | ; Only the second argument is SQL 75 | . 76 | ; Allow anything as the first argument in case the user has lower case type 77 | ; names for some reason 78 | (_) 79 | [(string_literal) (raw_string_literal)] @injection.content 80 | ) 81 | (#set! injection.language "sql")) 82 | -------------------------------------------------------------------------------- /test-grammars/rust/locals.scm: -------------------------------------------------------------------------------- 1 | ; Scopes 2 | 3 | [ 4 | (function_item) 5 | (struct_item) 6 | (enum_item) 7 | (union_item) 8 | (type_item) 9 | (trait_item) 10 | (impl_item) 11 | (closure_expression) 12 | (block) 13 | ] @local.scope 14 | 15 | ; Definitions 16 | 17 | (parameter 18 | (identifier) @local.definition.variable.parameter) 19 | 20 | (closure_parameters (identifier) @local.definition.variable.parameter) 21 | 22 | ; References 23 | (identifier) @local.reference 24 | (type_identifier) @local.reference 25 | -------------------------------------------------------------------------------- /test-grammars/rust/metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "repo": "https://github.com/tree-sitter/tree-sitter-rust", 3 | "rev": "1f63b33efee17e833e0ea29266dd3d713e27e321", 4 | "license": "MIT", 5 | "compressed": true 6 | } -------------------------------------------------------------------------------- /test-grammars/rust/src/grammar.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/helix-editor/tree-house/959309a5e86e71cd1140af9c35d0148eed6a4b12/test-grammars/rust/src/grammar.json -------------------------------------------------------------------------------- /test-grammars/rust/src/parser.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/helix-editor/tree-house/959309a5e86e71cd1140af9c35d0148eed6a4b12/test-grammars/rust/src/parser.c -------------------------------------------------------------------------------- /test-grammars/rust/src/tree_sitter/alloc.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_ALLOC_H_ 2 | #define TREE_SITTER_ALLOC_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | // Allow clients to override allocation functions 13 | #ifdef TREE_SITTER_REUSE_ALLOCATOR 14 | 15 | extern void *(*ts_current_malloc)(size_t size); 16 | extern void *(*ts_current_calloc)(size_t count, size_t size); 17 | extern void *(*ts_current_realloc)(void *ptr, size_t size); 18 | extern void (*ts_current_free)(void *ptr); 19 | 20 | #ifndef ts_malloc 21 | #define ts_malloc ts_current_malloc 22 | #endif 23 | #ifndef ts_calloc 24 | #define ts_calloc ts_current_calloc 25 | #endif 26 | #ifndef ts_realloc 27 | #define ts_realloc ts_current_realloc 28 | #endif 29 | #ifndef ts_free 30 | #define ts_free ts_current_free 31 | #endif 32 | 33 | #else 34 | 35 | #ifndef ts_malloc 36 | #define ts_malloc malloc 37 | #endif 38 | #ifndef ts_calloc 39 | #define ts_calloc calloc 40 | #endif 41 | #ifndef ts_realloc 42 | #define ts_realloc realloc 43 | #endif 44 | #ifndef ts_free 45 | #define ts_free free 46 | #endif 47 | 48 | #endif 49 | 50 | #ifdef __cplusplus 51 | } 52 | #endif 53 | 54 | #endif // TREE_SITTER_ALLOC_H_ 55 | --------------------------------------------------------------------------------