├── Dockerfile ├── README.md ├── run-in-docker.sh ├── server.js └── treesittervim.vim /Dockerfile: -------------------------------------------------------------------------------- 1 | # build with: docker build -t vim-treesitter:v1 . 2 | 3 | FROM node:latest 4 | 5 | RUN npm install tree-sitter tree-sitter-javascript tree-sitter-php tree-sitter-bash tree-sitter-json 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Vim async coloring experiment 2 | 3 | This is an experiment to see if it might be practical to do vim syntax coloring asynchronously. 4 | 5 | In this experiment, vim communicates over a network socket, sending buffer contents. The server runs the buffer contents through treesitter, and replies with coloring instructions. 6 | 7 | 8 | ## Setup 9 | 10 | 1. Build the docker image for the server: `docker build -t vim-treesitter:v1 .` 11 | 12 | 2. Copy `treesittervim.vim` into `~/.vim/autoload/` (create that directory if needed) 13 | 14 | 15 | ## Try it 16 | 17 | 1. Run the server: ./run-in-docker.sh 18 | 19 | 2. Open a javascript file, or paste some javascript code into an empty buffer 20 | 21 | 3. Disable syntax highlighting if there is any, eg by running `:set ft=txt` 22 | 23 | 4. In vim, do: `:call treesittervim#main()` 24 | 25 | 26 | ## Status 27 | 28 | Maturity: experiment. 29 | 30 | Currently it does the simplest thing that could possibly work. It simply sends the entire buffer contents to the server, then executes the coloring instructions for the whole buffer. When the buffer is large, there is a considerable delay (non-responsive interface) while the vimscript applies the coloring instructions. This should be fixable at least for common use cases (eg by not applying the colors immediately to lines which are not visible). 31 | -------------------------------------------------------------------------------- /run-in-docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | exec docker run -p 127.0.0.1:33039:33039 -t -i --rm -v "$(readlink -f "$(dirname "$0")")/server.js":/opt/server.js:ro -w /opt/pwd vim-treesitter:v1 node "/opt/server.js" 4 | -------------------------------------------------------------------------------- /server.js: -------------------------------------------------------------------------------- 1 | // © 2020 Jason Woofenden CC0 2 | 3 | "use strict"; 4 | 5 | // be a good little process (needed for Docker) 6 | process.on('SIGINT', () => { 7 | console.log('received SIGINT, exiting'); 8 | process.exit(); 9 | }); 10 | process.on('SIGTERM', () => { 11 | console.log('received SIGTERM, exiting'); 12 | process.exit(); 13 | }); 14 | 15 | const PORT = "33039"; 16 | 17 | const net = require('net'); 18 | const TreeSitter = require('tree-sitter'); 19 | const JavaScript = require('tree-sitter-javascript'); 20 | 21 | // some types that vim can color: https://github.com/crusoexia/vim-monokai/blob/master/colors/monokai.vim 22 | // sent to vim over the wire 23 | const EOL = 0; 24 | const PLAIN = 1; 25 | const SYMBOL = 2; 26 | const KEYWORD = 3; 27 | const IDENTIFIER = 4; 28 | const SPECIAL_CHAR = 5; 29 | const STRING = 6; 30 | const NUMBER = 7; 31 | const ERROR = 8; 32 | const COMMENT = 9; 33 | 34 | const keywords = new Set('class this else return var const let for while if try throw catch function next continue break of in new'.split(' ')); 35 | 36 | const symbols = new Set('% ( ) [ ] { } , - + ; . / = == === && & | || < != <= <= >= <=> => " \' ` ${ ~ ^ * ** + - % ! '.split(' ')); 37 | 38 | 39 | 40 | // notes 41 | // treesitter indexes are zero based and non-inclusive 42 | // types we care about: 43 | // number 44 | // identifier (eg function/variable name) 45 | // var (the actual "var") 46 | // can ignore 47 | // variable_declarator (variable name after "var") 48 | // because this span is also flagged "identifier" 49 | // (, ), ;, etc 50 | // SyntaxNode has types like "(" and ";" 51 | 52 | class Colorizer { 53 | constructor (row, column) { 54 | this.row = row; 55 | this.column = column; 56 | this.colors = [null]; // backwards 57 | this.line = []; // backwards 58 | this.lines = []; 59 | } 60 | // distance must be > 0 or EOL 61 | extend_line (distance) { 62 | if (this.line.length === 0) { 63 | this.lines.push(this.line); 64 | } 65 | if (this.line.length > 0 && this.line[0].color == this.colors[0]) { 66 | // extend existing/current same-color section 67 | if (distance === EOL) { 68 | this.line[0].distance = EOL; 69 | } else { 70 | this.line[0].distance += distance; 71 | } 72 | } else { 73 | this.line.unshift({distance, color: this.colors[0]}); 74 | } 75 | if (distance === EOL) { 76 | this.line = []; 77 | } 78 | } 79 | advance_to (row, column) { 80 | // Handle line wraps within colored area 81 | while (row > this.row) { 82 | this.extend_line(EOL); 83 | this.row += 1; 84 | this.column = 0; 85 | } 86 | if (column > this.column) { 87 | this.extend_line(column - this.column); 88 | this.column = column; 89 | } 90 | } 91 | start (color, row, column) { 92 | this.advance_to(row, column); 93 | this.colors.unshift(color); 94 | } 95 | end (row, column) { 96 | this.advance_to(row, column); 97 | this.colors.shift(); 98 | } 99 | debug_render () { 100 | console.log(this.colors); 101 | console.log(this.lines); 102 | console.log(this.line); 103 | // return this.commands.map(c => c.join(',')).join(' '); 104 | return this.lines.map(line => { 105 | const ret = []; 106 | for (let i = line.length - 1; i >= 0; --i) { 107 | const {color, distance} = line[i]; 108 | ret.push(" *kicsn !"[color]); 109 | if (distance === EOL) { 110 | ret.push('$'); 111 | } else if (distance > 1) { 112 | ret.push('.'.repeat(distance - 1)); 113 | } 114 | } 115 | return ret.join(''); 116 | }).join(' '); 117 | } 118 | render () { 119 | // return this.debug_render(); 120 | return this.lines.map((line) => line.reduceRight((ret, chunk) => { 121 | ret.push(chunk.color, chunk.distance); 122 | return ret; 123 | }, [])); 124 | } 125 | } 126 | 127 | 128 | const source_to_colors = (source) => { 129 | const parser = new TreeSitter(); 130 | parser.setLanguage(JavaScript); 131 | const tree = parser.parse(source); 132 | const root = tree.rootNode; 133 | // example node: 134 | // ProgramNode { 135 | // type: program, 136 | // startPosition: {row: 0, column: 0}, 137 | // endPosition: {row: 1, column: 0}, 138 | // childCount: 1, 139 | // } 140 | 141 | // traverse the parse tree and encode the relevant bits for vim 142 | // FIXME just recurse like a normal person. this loop is not helpful 143 | const colorizer = new Colorizer(root.startPosition.row, root.startPosition.column); 144 | const types = []; 145 | (function process_node(node) { 146 | const type = node.type; 147 | types.push(type); 148 | let color = null; 149 | // start of node 150 | if (type === 'program' || type === 'template_substitution') { 151 | color = PLAIN; 152 | } else if (type === 'number') { 153 | color = NUMBER; 154 | } else if (keywords.has(type)) { 155 | color = KEYWORD; 156 | } else if (symbols.has(type)) { 157 | color = SYMBOL; 158 | } else if (type === 'string' || type === 'template_string') { 159 | color = STRING; 160 | } else if (type === 'identifier' || type === 'property_identifier') { 161 | color = IDENTIFIER; 162 | } else if (type === 'escape_sequence') { 163 | color = SPECIAL_CHAR; 164 | } else if (type === 'ERROR') { 165 | color = ERROR; 166 | } else if (type === 'comment') { 167 | color = COMMENT; 168 | } 169 | 170 | if (color !== null) { 171 | colorizer.start(color, node.startPosition.row, node.startPosition.column); 172 | } 173 | 174 | for (let i = 0; i < node.childCount; ++i) { 175 | process_node(node.child(i)); 176 | } 177 | 178 | types.push(`/${type}`); 179 | 180 | if (color !== null) { 181 | colorizer.end(node.endPosition.row, node.endPosition.column); 182 | } 183 | })(root); 184 | //console.log(types.join(' ')); 185 | return colorizer.render(); 186 | }; 187 | 188 | 189 | const server = net.createServer((connection) => { 190 | console.log('connected'); 191 | let buf = null; 192 | let used = 0; 193 | connection.on('data', (data) => { 194 | console.log(`got data of length ${data.length}`); 195 | //console.log(`got data: ${data}`); 196 | // check for the common case 197 | if (buf === null) { 198 | buf = data; 199 | } else { 200 | console.log('merging into existing buffer'); 201 | buf = Buffer.concat([buf, data]); 202 | } 203 | let eol = 0; 204 | while ((eol = buf.indexOf("\n", used, 'binary')) > -1) { 205 | console.log('found newline'); 206 | const line = buf.toString('utf8', used, eol); 207 | let id = null, source = null; 208 | try { 209 | [id, source] = JSON.parse(line); 210 | } catch (e) { 211 | // json parsing failed, there must be more data coming 212 | console.log('json parsing failed'); 213 | } 214 | if (id !== null) { 215 | const reply = source_to_colors(source); 216 | connection.write(JSON.stringify([id, reply])); 217 | console.log('replied'); 218 | } 219 | used = eol + 1; 220 | } 221 | if (used === buf.length) { 222 | console.log('emptying buffer'); 223 | buf = null; 224 | used = 0; 225 | } 226 | // TODO if used is big, then clip it off buffer 227 | }); 228 | connection.on('end', () => { 229 | console.log('disconnected'); 230 | }); 231 | }).on('error', (err) => { 232 | throw err; 233 | }) 234 | server.listen(PORT, () => { 235 | console.log(`listening on port ${PORT}`); 236 | }) 237 | -------------------------------------------------------------------------------- /treesittervim.vim: -------------------------------------------------------------------------------- 1 | " © 2020 Jason Woofenden CC0 2 | 3 | " 1 is plain 4 | call prop_type_add('tsv_2', {'highlight': 'Operator'}) 5 | call prop_type_add('tsv_3', {'highlight': 'Keyword'}) 6 | call prop_type_add('tsv_4', {'highlight': 'Identifier'}) 7 | call prop_type_add('tsv_5', {'highlight': 'SpecialChar'}) 8 | call prop_type_add('tsv_6', {'highlight': 'String'}) 9 | call prop_type_add('tsv_7', {'highlight': 'Number'}) 10 | call prop_type_add('tsv_8', {'highlight': 'Error'}) 11 | call prop_type_add('tsv_9', {'highlight': 'Comment'}) 12 | 13 | " called when recieving a message from the socket 14 | func treesittervim#sochandler(soc, msg) 15 | let line_number = 0 16 | for line in a:msg 17 | let line_number += 1 18 | let col = 1 19 | let i = 0 20 | while i < len(line) 21 | " loop stuff 22 | let color = line[i] 23 | let span = line[i+1] 24 | let i += 2 25 | " loop body 26 | if color != 1 27 | call prop_add(line_number, col, {'length': span, 'type': 'tsv_' . color}) 28 | endif 29 | let col += span 30 | endwhile 31 | endfor 32 | endfunc 33 | 34 | " call this in a uncolored buffor with javascript in it 35 | function treesittervim#main() 36 | " I don't think this callback works 37 | let soc = ch_open('localhost:33039', {'callback': "treesittervim#sochandler"}) 38 | " this one does get called: 39 | call ch_sendexpr(soc, join(getline(1, '$'), "\n"), {'callback': "treesittervim#sochandler"}) 40 | endfunction 41 | --------------------------------------------------------------------------------