├── .gitignore ├── LICENSE ├── README.md ├── example ├── binding.c ├── binding.gyp ├── example.js └── package.json ├── index.js ├── package.json ├── parse.js ├── require.js ├── string.js └── test ├── compile.js └── parse.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | build 3 | sandbox.js 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Mathias Buus 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # shared-structs 2 | 3 | Share a struct backed by the same underlying buffer between C and JavaScript 4 | 5 | ``` 6 | npm install shared-structs 7 | ``` 8 | 9 | Useful for doing bulk updates of data in native modules with no context switching cost. 10 | 11 | ## Usage 12 | 13 | ``` js 14 | const sharedStructs = require('shared-structs') 15 | 16 | const structs = sharedStructs(` 17 | struct aStruct { 18 | int32_t i; 19 | char buf[1024]; 20 | char someChar; 21 | int someInt; 22 | } 23 | `) 24 | 25 | const struct = structs.aStruct() 26 | 27 | struct.i = 42 28 | struct.buf[0] = 42 29 | 30 | // pass this to c, and it will be able to parse it 31 | console.log(struct.rawBuffer) 32 | ``` 33 | 34 | Also supports nested structs, multidimensional arrays, defines, constant expressions, and most other things you'd normally use in c! 35 | 36 | See [example/example.js](example/example.js) for more. 37 | 38 | ## API 39 | 40 | #### `structs = sharedStructs(src, [options])` 41 | 42 | Parses the structs specified in the global scope of src 43 | and returns JavaScript implementations of each. 44 | 45 | Each property is exposed as a normal JavaScript property you can 46 | get/set. 47 | 48 | All changes are reflected in `.rawBuffer` which you can pass to a c program 49 | and parse with the same struct. 50 | 51 | If you want to pass a nested struct to c, use the `.rawBufferSlice` to get a pointer 52 | directly to this struct instead of `.rawBuffer`. 53 | 54 | If you are using this with a native module, make sure to keep a reference to the allocated 55 | struct in JavaScript (unless you know what you are doing) to avoid the buffer getting garbage 56 | collected, while you are still using it in your native code. 57 | 58 | If you are compiling a struct, that has a field that `shared-structs` cannot determine the size and alignment of deterministicly, it will throw an error. If you use the [napi-macros](https://github.com/mafintosh/napi-macros) module, you can easily export these from your native code using the `NAPI_EXPORT_SIZEOF` and `NAPI_EXPORT_ALIGNMENT` macros and pass them in as options. 59 | 60 | Options include: 61 | 62 | ```js 63 | { 64 | defines: { 65 | CUSTOM_DEFINE_HERE: 42 // add a custom define is defined elsewhere 66 | }, 67 | sizes: { 68 | foo: 1024 // set the size of struct foo if defined elsewhere 69 | }, 70 | alignment: { 71 | foo: 8 // set the alignment of struct foo if defined elsewhere 72 | } 73 | } 74 | ``` 75 | 76 | ## Writing strings 77 | 78 | There is a small helper included in `require('shared-structs/string')` that 79 | allows you to encode/decode c style strings into char buffers 80 | 81 | ```js 82 | const strings = require('shared-structs/string') 83 | 84 | // encode 85 | strings.encode('hello world', struct.buf) 86 | 87 | // decode 88 | console.log(strings.decode(struct.buf)) 89 | ``` 90 | 91 | ## Requiring .h files 92 | 93 | If you have your structs defined in a .h (or any file) there is a 94 | helper included in `require('shared-structs/require')` that can require 95 | these and parse them then as you would any other .js file 96 | 97 | ```js 98 | const structs = require('shared-structs/require')('file.h') 99 | 100 | console.log(structs) // same as loading the src of file.h and parsing it 101 | ``` 102 | 103 | If you want to pass options, pass them after the filename. 104 | 105 | ## License 106 | 107 | MIT 108 | -------------------------------------------------------------------------------- /example/binding.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | struct things { 6 | uint64_t a_number; 7 | char a_char; 8 | char input[61]; 9 | char output[61]; 10 | int operations; 11 | }; 12 | 13 | NAPI_METHOD(tick) { 14 | NAPI_ARGV(1) 15 | NAPI_ARGV_BUFFER_CAST(struct things *, t, 0) 16 | 17 | t->operations++; 18 | 19 | return NULL; 20 | } 21 | 22 | NAPI_METHOD(copy_string) { 23 | NAPI_ARGV(1) 24 | NAPI_ARGV_BUFFER_CAST(struct things *, t, 0) 25 | 26 | t->operations++; 27 | memcpy(t->output, t->input, 61); 28 | 29 | return NULL; 30 | } 31 | 32 | NAPI_INIT() { 33 | NAPI_EXPORT_FUNCTION(copy_string) 34 | NAPI_EXPORT_FUNCTION(tick) 35 | } 36 | -------------------------------------------------------------------------------- /example/binding.gyp: -------------------------------------------------------------------------------- 1 | { 2 | "targets": [{ 3 | "target_name": "pat", 4 | "include_dirs": [ 5 | "=12" 21 | }, 22 | "author": "Mathias Buus (@mafintosh)", 23 | "license": "MIT", 24 | "bugs": { 25 | "url": "https://github.com/mafintosh/shared-structs/issues" 26 | }, 27 | "homepage": "https://github.com/mafintosh/shared-structs" 28 | } 29 | -------------------------------------------------------------------------------- /parse.js: -------------------------------------------------------------------------------- 1 | const SIZEOF_PTR = process.arch === 'x64' ? 8 : 4 2 | const PRIMITIVE_SIZES = { 3 | uint: 4, 4 | uint8_t: 1, 5 | uint16_t: 2, 6 | uint32_t: 4, 7 | uint64_t: 8, 8 | int: 4, 9 | int8_t: 1, 10 | int16_t: 2, 11 | int32_t: 4, 12 | int64_t: 8, 13 | char: 1, 14 | byte: 1, 15 | bool: 1, 16 | float: 4, 17 | double: 8 18 | } 19 | 20 | module.exports = parse 21 | 22 | function align (n, a) { 23 | const rem = n & (a - 1) 24 | if (!rem) return n 25 | return n + (a - rem) 26 | } 27 | 28 | function filter (str) { 29 | const tokens = [ 30 | {start: '"', end: '"', index: -1, replace: "''"}, 31 | {start: '/*', end: '*/', index: -1, replace: ''}, 32 | {start: '//', end: '\n', index: -1, replace: '\n'} 33 | ] 34 | 35 | while (true) { 36 | const token = nextToken() 37 | if (!token) return str 38 | str = str.slice(0, token.index) + token.replace + str.slice(nextEnd(token)) 39 | } 40 | 41 | function nextEnd (token) { 42 | var index = token.index + token.start.length 43 | 44 | while (true) { 45 | index = str.indexOf(token.end, index) 46 | if (index === -1) return str.length 47 | if (str[index - 1] !== '\\') return index + token.end.length 48 | index++ 49 | } 50 | } 51 | 52 | function nextToken () { 53 | var min = null 54 | 55 | tokens.forEach(function (token) { 56 | token.index = str.indexOf(token.start) 57 | if (token.index > -1 && (!min || min.index > token.index)) min = token 58 | }) 59 | 60 | return min 61 | } 62 | } 63 | 64 | function assign (def, obj) { 65 | if (!obj) return def 66 | return Object.assign({}, def, obj) 67 | } 68 | 69 | function parse (str, opts) { 70 | if (!opts) opts = {} 71 | 72 | const alignments = opts.alignments || {} 73 | const defines = assign({}, opts.defines) 74 | const sizes = assign(PRIMITIVE_SIZES, opts.sizes) 75 | const tokens = resolveStatic(filterDefines(filter(str), defines), defines) 76 | .split(/(;|\s+)/gm) 77 | .filter(s => !/^(;|\s)*$/.test(s)) 78 | .reverse() 79 | 80 | const structs = [] 81 | while (tokens.length) parseNext() 82 | structs.forEach(postProcess) 83 | return structs 84 | 85 | function pop () { 86 | const top = tokens.pop() 87 | if (top === '()' || top === '(' || top === ')') return pop() 88 | if (top.length > 1 && top[0] === '*') { 89 | tokens.push(top.slice(1)) 90 | return '*' 91 | } 92 | if (top.length > 1 && top[top.length - 1] === '*') { 93 | tokens.push(top.slice(0, -1)) 94 | return '*' 95 | } 96 | if (top === '{}') { 97 | tokens.push('}') 98 | return '{' 99 | } 100 | return top 101 | } 102 | 103 | function parseNext () { 104 | const next = pop() 105 | 106 | if (next === 'struct') return parseStruct() 107 | if (next === 'typedef') return parseTypedef() 108 | if (next === '{') return skipBlock() 109 | 110 | return null 111 | } 112 | 113 | function skipBlock () { 114 | var depth = 1 115 | while (tokens.length && depth) { 116 | const next = pop() 117 | if (next === '{') depth++ 118 | else if (next === '}') depth-- 119 | } 120 | return null 121 | } 122 | 123 | function parseStruct () { 124 | const result = {node: 'struct', name: null, size: 0, alignment: 1, fields: []} 125 | const name = pop() 126 | 127 | if (name !== '{') { 128 | result.name = name 129 | if (pop() !== '{') return null 130 | } 131 | 132 | var field = null 133 | while ((field = parseStructField()) !== null) result.fields.push(field) 134 | structs.push(result) 135 | 136 | return result 137 | } 138 | 139 | function parseStructField () { 140 | const field = {node: 'field', type: pop(), name: null, size: 0, offset: 0, array: null, struct: false, pointer: false, alignment: 1} 141 | 142 | if (field.type === '}') return null 143 | 144 | if (field.type === 'struct') { 145 | field.struct = true 146 | field.type = pop() 147 | } 148 | 149 | if (!validId(field.type)) throw new Error('Invalid struct field type: ' + field.type) 150 | 151 | field.name = pop() 152 | 153 | if (field.name === '*') { 154 | field.pointer = true 155 | field.name = pop() 156 | } 157 | 158 | var index = field.name.length 159 | while ((index = field.name.lastIndexOf('[', index)) > -1) { 160 | const end = field.name.indexOf(']', index) 161 | if (end === -1) throw new Error('Invalid struct field array: ' + field.name) 162 | const val = field.name.slice(index + 1, end) 163 | field.name = field.name.slice(0, index) 164 | index-- 165 | 166 | if (!field.array) field.array = [] 167 | field.array.unshift(resolveUint(val)) 168 | } 169 | 170 | if (!validId(field.name)) throw new Error('Invalid struct field name: ' + field.name) 171 | 172 | return field 173 | } 174 | 175 | function parseTypedef () { 176 | const value = parseNext() 177 | const name = pop() 178 | if (!validId(name)) throw new Error('Invalid typedef name: ' + name) 179 | if (value.node === 'struct') value.name = name 180 | return value 181 | } 182 | 183 | function postProcess (v) { 184 | if (v.size > 0) return v.size 185 | 186 | if (v.node === 'struct') { 187 | var offset = 0 188 | for (var i = 0; i < v.fields.length; i++) { 189 | const f = v.fields[i] 190 | postProcess(f) 191 | v.alignment = Math.max(f.alignment, v.alignment) 192 | offset = align(offset, f.alignment) 193 | f.offset = offset 194 | offset += f.size 195 | } 196 | if (alignments[v.type]) v.alignment = alignments[v.type] 197 | v.size = sizes[v.type] || align(offset, v.alignment) 198 | return 199 | } 200 | 201 | if (v.node !== 'field') return 202 | 203 | var size = 0 204 | if (v.pointer) { 205 | v.alignment = size = sizes['*'] || SIZEOF_PTR 206 | } else if (sizes[v.type]) { 207 | size = sizes[v.type] 208 | v.alignment = alignments[v.type] || size 209 | } else { 210 | const struct = lookupStruct(v.type) 211 | v.alignment = struct.alignment 212 | size = struct.size 213 | v.struct = true 214 | } 215 | 216 | if (v.array) v.size = v.array.reduce(times) * size 217 | else v.size = size 218 | } 219 | 220 | function lookupStruct (type) { 221 | for (var i = 0; i < structs.length; i++) { 222 | if (structs[i].name === type) return structs[i] 223 | } 224 | throw new Error('Unknown struct: ' + type) 225 | } 226 | } 227 | 228 | function resolveStatic (src, defines) { 229 | if (!/^\w+$/.test(src) || defines.hasOwnProperty(src)) { 230 | const keys = Object.keys(defines) 231 | for (var i = 0; i < keys.length; i++) { 232 | const key = keys[i] 233 | const reg = new RegExp('([^\\w])' + key + '([^\\w])', 'g') 234 | src = src.replace(reg, function (_, start, end) { 235 | return start + resolveStatic(defines[key], defines) + end 236 | }) 237 | } 238 | } 239 | 240 | src = src.replace(/\[([^\]]+)\]/g, function (_, num) { 241 | if (/^\d+$/.test(num)) return _ 242 | if (/^[0-9+\-*/>< ()&]+$/.test(num)) return '[' + evalNumber(num) + ']' 243 | return _ 244 | }) 245 | 246 | return src 247 | } 248 | 249 | function resolveUint (name) { 250 | if (/^\d+$/.test(name)) return Number(name) 251 | throw new Error('Expected ' + name + ' to be an unsigned integer') 252 | } 253 | 254 | function filterDefines (src, defines) { 255 | return src.replace(/#define\s+(\S+)+\s+(\S+)/g, function (_, name, val) { 256 | if (!defines.hasOwnProperty(name)) defines[name] = val 257 | return '' 258 | }) 259 | } 260 | 261 | function times (a, b) { 262 | return a * b 263 | } 264 | 265 | function validId (n) { 266 | return /^[a-z_]([a-z0-9_.])*$/i.test(n) 267 | } 268 | 269 | function evalNumber (expr) { 270 | return new Function('return (' + expr + ')')() 271 | } 272 | -------------------------------------------------------------------------------- /require.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs') 2 | const path = require('path') 3 | const compile = require('./') 4 | 5 | delete require.cache[__filename] 6 | 7 | module.exports = function (files, opts) { 8 | const dirname = path.dirname(module.parent.filename) 9 | const src = (Array.isArray(files) ? files : [files]) 10 | .map(name => fs.readFileSync(path.join(dirname, name), 'utf-8')) 11 | .join('\n') 12 | 13 | return compile(src, opts) 14 | } 15 | -------------------------------------------------------------------------------- /string.js: -------------------------------------------------------------------------------- 1 | exports.encode = function (s, buf, offset) { 2 | if (!buf) buf = Buffer.alloc(exports.encodingLength(s)) 3 | if (!offset) offset = 0 4 | const oldOffset = offset 5 | offset += buf.write(s, offset) 6 | buf[offset++] = 0 7 | exports.encode.bytes = oldOffset - offset 8 | return buf 9 | } 10 | 11 | exports.encodingLength = function (s) { 12 | return Buffer.byteLength(s) + 1 13 | } 14 | 15 | exports.decode = function (buf, offset) { 16 | if (offset) buf = buf.slice(offset) 17 | var i = buf.indexOf(0) 18 | if (i === -1) i = buf.length 19 | exports.bytes = i + 1 20 | return buf.toString('utf-8', 0, i) 21 | } 22 | -------------------------------------------------------------------------------- /test/compile.js: -------------------------------------------------------------------------------- 1 | const tape = require('tape') 2 | const compile = require('../') 3 | 4 | tape('basic', function (t) { 5 | const structs = compile(` 6 | struct foo { 7 | int32_t a; 8 | } 9 | `) 10 | 11 | const foo = structs.foo() 12 | 13 | foo.a = 42 14 | t.same(foo.a, 42) 15 | t.same(foo.rawBuffer.length, 4) 16 | t.notSame(foo.rawBuffer, Buffer.alloc(4)) 17 | 18 | const fooClone = structs.foo(foo.rawBuffer) 19 | 20 | t.same(fooClone.a, 42) 21 | t.ok(fooClone.rawBuffer === foo.rawBuffer) 22 | 23 | t.end() 24 | }) 25 | 26 | tape('complex', function (t) { 27 | const structs = compile(` 28 | #define BUF_SIZE 2001 29 | 30 | typedef struct { 31 | char buf[BUF_SIZE]; 32 | } bar; 33 | 34 | struct foo { 35 | char a; 36 | double b[10][12]; 37 | bar c[10]; 38 | bar d[1][2][3]; 39 | int e; 40 | int64_t i64; 41 | uint64_t u64; 42 | }; 43 | `) 44 | 45 | const foo = structs.foo() 46 | 47 | foo.a = 1 48 | t.same(foo.a, 1) 49 | 50 | foo.e = 42 51 | t.same(foo.e, 42) 52 | 53 | foo.b[0][10] = 0.1 54 | t.same(foo.b[0][10], 0.1) 55 | 56 | foo.c[0].buf[42] = 10 57 | t.same(foo.c[0].buf[42], 10) 58 | 59 | foo.d[0][1][1].buf[100] = 11 60 | t.same(foo.d[0][1][1].buf[100], 11) 61 | 62 | foo.i64 = -755n; 63 | t.same(foo.i64, -755n); 64 | 65 | foo.u64 = 777n; 66 | t.same(foo.u64, 777n); 67 | 68 | t.same(foo.rawBuffer.length, 33008) 69 | t.notSame(foo.rawBuffer, Buffer.alloc(33008)) 70 | 71 | const fooClone = structs.foo(foo.rawBuffer) 72 | 73 | t.same(fooClone.a, 1) 74 | t.same(fooClone.e, 42) 75 | t.same(fooClone.b[0][10], 0.1) 76 | t.same(fooClone.c[0].buf[42], 10) 77 | t.same(fooClone.d[0][1][1].buf[100], 11) 78 | t.same(fooClone.i64, -755n); 79 | t.same(fooClone.u64, 777n); 80 | t.same(fooClone.rawBuffer.length, 33008) 81 | 82 | t.ok(fooClone.rawBuffer === foo.rawBuffer) 83 | 84 | t.end() 85 | }) 86 | 87 | tape('view parse', function (t) { 88 | const structs = compile(` 89 | struct foo { 90 | uint32_t magic1; 91 | uint32_t magic2; 92 | uint32_t magic3; 93 | uint32_t magic4; 94 | uint32_t magic5; 95 | }; 96 | `) 97 | 98 | const struct1 = structs.foo() 99 | 100 | struct1.magic1 = 1 101 | struct1.magic2 = 2 102 | struct1.magic3 = 3 103 | struct1.magic4 = 4 104 | struct1.magic5 = 5 105 | 106 | const tmp = Buffer.allocUnsafe(1) // to force unalignment of the next buf 107 | const buf = Buffer.allocUnsafe(struct1.rawBuffer.length) 108 | struct1.rawBuffer.copy(buf, 0) 109 | 110 | let struct2 = structs.foo(buf) 111 | 112 | t.same(struct1.magic1, struct2.magic1) 113 | t.same(struct1.magic2, struct2.magic2) 114 | t.same(struct1.magic3, struct2.magic3) 115 | t.same(struct1.magic4, struct2.magic4) 116 | t.same(struct1.magic5, struct2.magic5) 117 | 118 | t.end() 119 | }) 120 | -------------------------------------------------------------------------------- /test/parse.js: -------------------------------------------------------------------------------- 1 | const tape = require('tape') 2 | const parse = require('../parse') 3 | 4 | tape('parse basic', function (t) { 5 | const structs = parse(` 6 | struct foo { 7 | int32_t i; 8 | }; 9 | `) 10 | 11 | t.same(structs, [{ 12 | node: 'struct', 13 | name: 'foo', 14 | alignment: 4, 15 | size: 4, 16 | fields: [{ 17 | node: 'field', 18 | type: 'int32_t', 19 | name: 'i', 20 | struct: false, 21 | pointer: false, 22 | array: null, 23 | size: 4, 24 | offset: 0, 25 | alignment: 4 26 | }] 27 | }]) 28 | 29 | t.end() 30 | }) 31 | 32 | tape('parse basic with comments and noise', function (t) { 33 | const structs = parse(` 34 | #include "baz /* struct main {" 35 | 36 | // ' and stuff struct foobar {} 37 | /* 38 | struct bar {} 39 | */ 40 | 41 | struct foo { 42 | int32_t i; 43 | }; 44 | 45 | void main () { 46 | printf("struct baz {}"); 47 | printf("\\"struct baa {}"); 48 | } 49 | 50 | struct bax method () { 51 | struct inline { 52 | int tmp; 53 | } 54 | } 55 | `) 56 | 57 | t.same(structs, [{ 58 | node: 'struct', 59 | name: 'foo', 60 | alignment: 4, 61 | size: 4, 62 | fields: [{ 63 | node: 'field', 64 | type: 'int32_t', 65 | name: 'i', 66 | struct: false, 67 | pointer: false, 68 | array: null, 69 | size: 4, 70 | offset: 0, 71 | alignment: 4 72 | }] 73 | }]) 74 | 75 | t.end() 76 | }) 77 | 78 | tape('multi dim array', function (t) { 79 | const structs = parse(` 80 | struct foo { 81 | char buf[10][12] 82 | } 83 | `) 84 | 85 | t.same(structs, [{ 86 | node: 'struct', 87 | name: 'foo', 88 | alignment: 1, 89 | size: 10 * 12, 90 | fields: [{ 91 | node: 'field', 92 | type: 'char', 93 | name: 'buf', 94 | struct: false, 95 | pointer: false, 96 | array: [10, 12], 97 | size: 10 * 12, 98 | offset: 0, 99 | alignment: 1 100 | }] 101 | }]) 102 | 103 | t.end() 104 | }) 105 | --------------------------------------------------------------------------------