├── .gitignore ├── LICENCE.MD ├── README.md ├── lerna.json ├── package-lock.json ├── package.json ├── packages ├── ts-binary-types │ ├── README.MD │ ├── __tests__ │ │ └── lib.test.ts │ ├── example.tsconfig.json │ ├── examples │ │ ├── main.ts │ │ ├── message.ts │ │ └── worker.ts │ ├── jest.config.js │ ├── package-lock.json │ ├── package.json │ ├── src │ │ ├── core.ts │ │ ├── experiments │ │ │ ├── bench.ts │ │ │ ├── example.ts │ │ │ ├── tes-types.ts │ │ │ └── traverse.ts │ │ ├── index.ts │ │ ├── retype.ts │ │ └── types │ │ │ ├── bool.ts │ │ │ ├── enum.ts │ │ │ ├── nullable.ts │ │ │ ├── numbers.ts │ │ │ ├── optional.ts │ │ │ ├── str.ts │ │ │ ├── struct.ts │ │ │ ├── tuple.ts │ │ │ ├── union.ts │ │ │ └── vector.ts │ ├── tsconfig.bench.json │ ├── tsconfig.build.json │ ├── tsconfig.examples.json │ └── tsconfig.json ├── ts-binary │ ├── .prettierrc │ ├── README.MD │ ├── __tests__ │ │ ├── serializeAndRestore.test.ts │ │ └── writeU64.test.ts │ ├── jest.config.js │ ├── package-lock.json │ ├── package.json │ ├── src │ │ ├── index.ts │ │ └── legacy.ts │ ├── tsconfig.build.json │ └── tsconfig.json ├── ts-rust-bridge-codegen │ ├── .prettierrc │ ├── README.MD │ ├── __tests__ │ │ ├── gen_testSchema.ts │ │ ├── generated │ │ │ ├── types.g.ts │ │ │ └── types.serde.g.ts │ │ └── serializeAndRestore.test.ts │ ├── bench_data.json │ ├── examples │ │ ├── benchmark.ts │ │ ├── for_docs │ │ │ ├── gen_docs_example.ts │ │ │ ├── schema.rs │ │ │ ├── schema.ts │ │ │ ├── schema_serde.ts │ │ │ └── usage.ts │ │ ├── gen_simple.schema.ts │ │ ├── generated │ │ │ ├── simple.g.rs │ │ │ ├── simple.g.ts │ │ │ └── simple_serde.g.ts │ │ ├── runExample.ts │ │ └── simple.schema.ts │ ├── jest.config.js │ ├── package-lock.json │ ├── package.json │ ├── src │ │ ├── index.ts │ │ ├── rust │ │ │ └── schema2rust.ts │ │ ├── schema.ts │ │ ├── serde │ │ │ ├── schema2deserializers.ts │ │ │ ├── schema2serializers.ts │ │ │ ├── sharedPieces.ts │ │ │ └── topologicalSort.ts │ │ └── ts │ │ │ ├── ast.ts │ │ │ ├── ast2ts.ts │ │ │ └── schema2ast.ts │ ├── tsconfig.build.json │ ├── tsconfig.json │ ├── types.rs │ └── types.ts └── workspace.code-workspace ├── tsconfig.build.json └── tsconfig.json /.gitignore: -------------------------------------------------------------------------------- 1 | /packages/*/node_modules 2 | /packages/*/pkg 3 | /packages/*/examples_built 4 | /node_modules 5 | lerna-debug.log 6 | todo/ 7 | .DS_Store 8 | *.log 9 | *.tsbuildinfo 10 | /.nyc_output 11 | /coverage 12 | /dist-* 13 | /dist 14 | /dist-debug 15 | /artifacts 16 | /updates 17 | .vs 18 | *.msi 19 | *.nupkg 20 | /tmp/ 21 | .idea 22 | .pnp 23 | .vscode/ -------------------------------------------------------------------------------- /LICENCE.MD: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Simon Korzunov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ts-rust-bridge 2 | 3 | A collection of libraries for efficient communication between rust and typescript ( + other languages in the future). 4 | 5 | The project is structure as a monorepo. Look at the docs for each to find more details: 6 | 7 | Code generation library: [ts-rust-bridge-codegen](https://github.com/twop/ts-rust-bridge/tree/master/packages/ts-rust-bridge-codegen) 8 | 9 | Utilities to serialize/deserialize data in binary form: [ts-binary](https://github.com/twop/ts-rust-bridge/tree/master/packages/ts-binary) 10 | 11 | # WIP 12 | 13 | WARNING: The tool is far from being ready: not enough documentation + missing features. That said, you are welcome to take a look and give feedback. 14 | 15 | ## Install 16 | 17 | `npm install ts-rust-bridge-codegen --save-dev` 18 | 19 | If you want to use binary serialization/deserialization: 20 | 21 | `npm install ts-binary --save` 22 | 23 | ## Example 24 | 25 | Define AST(ish) structure in typescript. Note that it is a small subset of `serde` types from rust ecosystem. 26 | 27 | ```ts 28 | import { Type } from "ts-rust-bridge-codegen"; 29 | 30 | const { Enum, Struct, Str, F32 } = Type; 31 | 32 | const Size = Enum("S", "M", "L"); 33 | const Shirt = Struct({ size: Size, color: Str, price: F32 }); 34 | const schema = { Size, Shirt }; 35 | ``` 36 | 37 | Then codegen typescript and rust: 38 | 39 | ```ts 40 | import { schema2rust, schema2ts } from "ts-rust-bridge-codegen"; 41 | 42 | const tsCode = schema2ts(schema).join("\n\n"); 43 | 44 | const rustCode = ` 45 | use serde::{Deserialize, Serialize}; 46 | 47 | ${schema2rust(schema).join("\n")} 48 | `; 49 | ``` 50 | 51 | And here is the result: 52 | 53 | rust 54 | 55 | ```rust 56 | // schema.rs 57 | use serde::{Deserialize, Serialize}; 58 | #[derive(Deserialize, Serialize, Debug, Clone)] 59 | pub struct Shirt { 60 | pub size: Size, 61 | pub color: String, 62 | pub price: f32, 63 | } 64 | 65 | 66 | #[derive(Deserialize, Serialize, Debug, Clone)] 67 | pub enum Size { 68 | S, 69 | M, 70 | L, 71 | } 72 | ``` 73 | 74 | typescript 75 | 76 | ```ts 77 | // schema.ts after prettier 78 | export interface Shirt { 79 | size: Size; 80 | color: string; 81 | price: number; 82 | } 83 | 84 | export enum Size { 85 | S = "S", 86 | M = "M", 87 | L = "L" 88 | } 89 | ``` 90 | 91 | ## How to use it 92 | 93 | You can use `ts-rust-bridge-codegen` as a standalone tool. It is designed to be run manually or build time, so it should be a dev dependency. 94 | 95 | What it can do: 96 | 97 | 1. Define a type schema that can be used to generate typescript and/or rust type definitions. 98 | 2. After that you can just use JSON as a format to communicate between the two runtimes. 99 | 100 | If you want to be more efficient than JSON (it is CPU + memory intensive) you can use a binary serialization format compatible with [bincode](https://github.com/TyOverby/bincode) 101 | 102 | 3. Generate binary type serializers/deserializers based on the schema. 103 | 4. Profit! 104 | 105 | Look at [examples](https://github.com/twop/ts-rust-bridge/tree/master/packages/ts-rust-bridge-codegen/examples) dir for more information how to use the library. 106 | 107 | ## Simple benchmarks 108 | 109 | I just copypasted generated code from examples and tried to construct a simple benchmark. 110 | 111 | Code 112 | https://stackblitz.com/edit/ts-binaray-benchmark?file=index.ts 113 | 114 | Version to try 115 | https://ts-binaray-benchmark.stackblitz.io 116 | 117 | On complex data structure: 118 | 119 | | Method | Serialization | Deserialization | 120 | | --------- | :-----------: | --------------: | 121 | | ts-binary | 74 ms | 91 ms | 122 | | JSON | 641 ms | 405 ms | 123 | 124 | Simple data structure: 125 | 126 | | Method | Serialization | Deserialization | 127 | | --------- | :-----------: | --------------: | 128 | | ts-binary | 2 ms | 1 ms | 129 | | JSON | 6 ms | 5 ms | 130 | 131 | That was measured on latest Safari version. 132 | 133 | Note you can run the benchmark yourself cloning the repo + running npm scripts 134 | 135 | ## License 136 | 137 | MIT 138 | -------------------------------------------------------------------------------- /lerna.json: -------------------------------------------------------------------------------- 1 | { 2 | "packages": ["packages/*"], 3 | "version": "0.8.0" 4 | } 5 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "root", 3 | "private": true, 4 | "license": "MIT", 5 | "scripts": { 6 | "version": "lerna exec npm version", 7 | "build": "lerna run build", 8 | "publish": "lerna publish from-package --contents pkg", 9 | "publish:canary": "lerna publish from-package --contents pkg --canary", 10 | "bootstrap": "lerna bootstrap", 11 | "nuke": "lerna exec \"rm -f package-lock.json npm-shrinkwrap.json\" && lerna clean --yes && lerna bootstrap && lerna exec --stream -- \"test -f package-lock.json || npm install --package-lock-only\"" 12 | }, 13 | "devDependencies": { 14 | "lerna": "^3.19.0" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /packages/ts-binary-types/README.MD: -------------------------------------------------------------------------------- 1 | # ts-binary-types 2 | 3 | A collection of runtypes to serialize/deserialize data structures in typescript. 4 | 5 | ## Install 6 | 7 | `npm install ts-binary-types --save` 8 | 9 | ## License 10 | 11 | MIT. 12 | -------------------------------------------------------------------------------- /packages/ts-binary-types/__tests__/lib.test.ts: -------------------------------------------------------------------------------- 1 | import { Sink } from "ts-binary"; 2 | import { BinType, bindesc, Static } from "../src/core"; 3 | import { U8, U16, U32, I32, F32, F64 } from "../src/types/numbers"; 4 | import { Bool } from "../src/types/bool"; 5 | import { Str } from "../src/types/str"; 6 | import { Optional } from "../src/types/optional"; 7 | import { Enum } from "../src/types/enum"; 8 | import { Vec } from "../src/types/vector"; 9 | import { Tuple } from "../src"; 10 | import { Struct } from "../src/types/struct"; 11 | import { Union } from "../src/types/union"; 12 | import { Nullable } from "../src/types/nullable"; 13 | 14 | const saveAndRestore = ( 15 | val: T, 16 | bintype: BinType 17 | // deser: Deserializer, 18 | // ser: Serializer 19 | ): T => { 20 | const sink = bintype[bindesc].write(Sink(new ArrayBuffer(1)), val); 21 | sink.pos = 0; 22 | return bintype[bindesc].read(sink); 23 | }; 24 | 25 | describe("numbers", () => { 26 | it("can save and restore U8", () => { 27 | expect(saveAndRestore(5, U8)).toBe(5); 28 | // out of 0-255 29 | expect(saveAndRestore(255 + 2, U8)).toBe(1); 30 | }); 31 | 32 | it("can save and restore U16", () => { 33 | expect(saveAndRestore(257, U16)).toBe(257); 34 | // out of U16 35 | expect(saveAndRestore(2 ** 16 + 1, U16)).toBe(1); 36 | }); 37 | 38 | it("can save and restore U32", () => { 39 | expect(saveAndRestore(2 ** 31, U32)).toBe(2 ** 31); 40 | // out of U32 41 | expect(saveAndRestore(2 ** 32 + 1, U32)).toBe(1); 42 | }); 43 | 44 | it("can save and restore I32", () => { 45 | expect(saveAndRestore(-(2 ** 30), I32)).toBe(-(2 ** 30)); 46 | }); 47 | 48 | it("can save and restore F32", () => { 49 | // FLoat32 are not exactly js numbers. They are being rounded a bit 50 | expect(saveAndRestore(-100.6, F32) === -100.6).toBe(false); 51 | 52 | const arr = new Float32Array(1); 53 | arr[0] = -100.6; 54 | const f32 = arr[0]; 55 | 56 | expect(saveAndRestore(-100.6, F32)).toBe(f32); 57 | }); 58 | 59 | it("can save and restore F64", () => { 60 | expect(saveAndRestore(Number.MAX_VALUE, F64)).toBe(Number.MAX_VALUE); 61 | }); 62 | }); 63 | 64 | test("can save and restore Bool", () => { 65 | expect(saveAndRestore(true, Bool)).toBe(true); 66 | expect(saveAndRestore(false, Bool)).toBe(false); 67 | }); 68 | 69 | test("can save and restore Str", () => { 70 | expect(saveAndRestore("", Str)).toBe(""); 71 | expect(saveAndRestore("abc", Str)).toBe("abc"); 72 | }); 73 | 74 | test("can save and restore Option(Str)", () => { 75 | const OptStr = Optional(Str); 76 | expect(saveAndRestore(undefined, OptStr)).toBe(undefined); 77 | expect(saveAndRestore("abc", OptStr)).toBe("abc"); 78 | }); 79 | 80 | test("can save and restore Nullable(Str)", () => { 81 | const NullableStr = Nullable(Str); 82 | expect(saveAndRestore(null, NullableStr)).toBe(null); 83 | expect(saveAndRestore("abc", NullableStr)).toBe("abc"); 84 | }); 85 | 86 | test("can save and restore Enum", () => { 87 | const AorB = Enum("A", "B"); 88 | expect(saveAndRestore(AorB.A, AorB)).toBe("A"); 89 | expect(saveAndRestore(AorB.B, AorB)).toBe("B"); 90 | }); 91 | 92 | test("can save and restore Vector(Bool)", () => { 93 | const VecBool = Vec(Bool); 94 | expect(saveAndRestore([true, false], VecBool)).toEqual([true, false]); 95 | }); 96 | 97 | describe("Tuple", () => { 98 | it("can save and restore Tuple of 2", () => { 99 | const BoolStr = Tuple(Bool, Str); 100 | expect(saveAndRestore(BoolStr(true, "a"), BoolStr)).toEqual([true, "a"]); 101 | }); 102 | 103 | it("can save and restore Tuple of 3", () => { 104 | const BoolStrU32 = Tuple(Bool, Str, U32); 105 | expect(saveAndRestore(BoolStrU32(true, "a", 100500), BoolStrU32)).toEqual([ 106 | true, 107 | "a", 108 | 100500 109 | ]); 110 | }); 111 | 112 | it("can save and restore Tuple of 5", () => { 113 | const FiveU32 = Tuple(U32, U32, U32, U32, U32); 114 | expect(saveAndRestore(FiveU32(1, 2, 3, 4, 5), FiveU32)).toEqual([ 115 | 1, 116 | 2, 117 | 3, 118 | 4, 119 | 5 120 | ]); 121 | }); 122 | }); 123 | 124 | test("can save and restore Struct", () => { 125 | const S = Struct({ bool: Bool, str: Str }); 126 | const val: Static = { bool: true, str: "a" }; 127 | expect(saveAndRestore(val, S)).toEqual({ ...val }); 128 | }); 129 | 130 | test("can save and restore Struct", () => { 131 | const U = Union({ 132 | Unit: null, 133 | B: Bool, 134 | S: Str 135 | }); 136 | 137 | expect(saveAndRestore(U.Unit, U)).toEqual({ tag: "Unit" }); 138 | expect(saveAndRestore(U.B(false), U)).toEqual({ tag: "B", val: false }); 139 | expect(saveAndRestore(U.S("a"), U)).toEqual({ tag: "S", val: "a" }); 140 | }); 141 | -------------------------------------------------------------------------------- /packages/ts-binary-types/example.tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | 4 | "include": ["examples/**/*"], 5 | "compilerOptions": { 6 | "target": "esnext", 7 | "module": "esnext", 8 | "declaration": false, 9 | "sourceMap": false 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /packages/ts-binary-types/examples/main.ts: -------------------------------------------------------------------------------- 1 | import { Worker } from "worker_threads"; 2 | import { join } from "path"; 3 | import { 4 | genMessage, 5 | Msg, 6 | writeMessage, 7 | readMessage, 8 | WorkerMsg, 9 | printExecTime 10 | } from "./message"; 11 | 12 | // import { Sink, Serializer, bindesc } from "../src/index"; 13 | 14 | // const log = console.log; 15 | // const measure = (name: string, func: () => void) => { 16 | // //log(`\n${' '.repeat(4)}${name}`); 17 | 18 | // // let fastest = 100500; 19 | 20 | // const numberOfRuns = 1; 21 | // const takeTop = 1; 22 | 23 | // let runs: number[] = []; 24 | // for (let i = 0; i < numberOfRuns; i++) { 25 | // const hrstart = process.hrtime(); 26 | // func(); 27 | // const hrend = process.hrtime(hrstart); 28 | 29 | // const current = hrend[1] / 1000000; 30 | 31 | // runs.push(current); 32 | 33 | // // fastest = Math.min(fastest, current); 34 | // } 35 | 36 | // const result = runs 37 | // .sort((a, b) => a - b) 38 | // .slice(numberOfRuns - takeTop, numberOfRuns) 39 | // .reduce((s, v) => s + v, 0); 40 | 41 | // log(`${name}: ${result.toFixed(2)} ms`); 42 | // }; 43 | 44 | const COUNT = 100; 45 | const stopGC: any[] = []; 46 | 47 | const messages = Array.from({ length: COUNT }, () => genMessage(10)); 48 | 49 | const worker = new Worker(join(__dirname, "worker.js")); 50 | 51 | const sendMsgObj = (msg: Msg) => 52 | worker.postMessage({ tag: "json", val: msg } as WorkerMsg); 53 | 54 | const measureJson = () => { 55 | let i = 0; 56 | const recieved = new Array(messages.length); 57 | stopGC.push(recieved); 58 | const hrstart = process.hrtime(); 59 | 60 | worker.on("message", (msg: Msg) => { 61 | recieved[i] = msg; 62 | i++; 63 | if (i < messages.length) { 64 | sendMsgObj(messages[i]); 65 | } else { 66 | worker.removeAllListeners("message"); 67 | 68 | const hrend = process.hrtime(hrstart); 69 | 70 | printExecTime("native", hrend); 71 | 72 | measureBinary(); 73 | } 74 | }); 75 | 76 | sendMsgObj(messages[0]); 77 | }; 78 | 79 | const sendMsgBin = (msg: Msg, arr: Uint8Array) => { 80 | const toSend = writeMessage(msg, arr).buffer; 81 | const workerMsg: WorkerMsg = { tag: "msg_arr", val: toSend }; 82 | worker.postMessage(workerMsg, [toSend]); 83 | // return worker.postMessage(toSend, [toSend]); 84 | }; 85 | 86 | const measureBinary = () => { 87 | let i = 0; 88 | const recieved = new Array(messages.length); 89 | stopGC.push(recieved); 90 | 91 | const hrstart = process.hrtime(); 92 | 93 | let ping_time = process.hrtime(); 94 | let pong_time = process.hrtime(); 95 | 96 | worker.on("message", (buffer: ArrayBuffer) => { 97 | const arr = new Uint8Array(buffer); 98 | const msg = readMessage(arr); 99 | pong_time = process.hrtime(ping_time); 100 | printExecTime("ping-pong", pong_time); 101 | // console.log("got arr.len", arr.byteLength); 102 | recieved[i] = msg; 103 | i++; 104 | if (i < messages.length) { 105 | sendMsgBin(messages[i], arr); 106 | ping_time = process.hrtime(); 107 | } else { 108 | worker.removeAllListeners("message"); 109 | 110 | const hrend = process.hrtime(hrstart); 111 | 112 | printExecTime("binary", hrend); 113 | worker.terminate(); 114 | } 115 | }); 116 | sendMsgBin(messages[0], new Uint8Array(1024)); 117 | }; 118 | 119 | // const measureSharedArrayBuff = () => { 120 | // let i = 0; 121 | // const recieved: Msg[] = []; 122 | 123 | // // const sharedArr = new SharedArrayBuffer() 124 | // const hrstart = process.hrtime(); 125 | 126 | // worker.on("message", (arr: ArrayBuffer) => { 127 | // const msg = readMessage(arr); 128 | // // console.log("read", msg); 129 | // recieved.push(msg); 130 | // i++; 131 | // if (i < messages.length) { 132 | // sendMsgBin(messages[i], arr); 133 | // } else { 134 | // worker.removeAllListeners("message"); 135 | 136 | // const hrend = process.hrtime(hrstart); 137 | 138 | // const tookMs = hrend[1] / 1000000; 139 | 140 | // console.log(`binary: took ${tookMs}ms`); 141 | // } 142 | // }); 143 | // sendMsgBin(messages[0], new Uint8Array(1024).buffer); 144 | // }; 145 | 146 | measureJson(); 147 | 148 | // if (isMainThread) { 149 | // // This code is executed in the main thread and not in the worker. 150 | // // Create the worker. 151 | // } else { 152 | // // This code is executed in the worker and not in the main thread. 153 | 154 | // // Send a message to the main thread. 155 | // parentPort!.postMessage("Hello world!" + JSON.stringify(genMessage(10))); 156 | // } 157 | -------------------------------------------------------------------------------- /packages/ts-binary-types/examples/message.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Struct, 3 | Vec, 4 | Tuple, 5 | Bool, 6 | // Str, 7 | F64, 8 | Static, 9 | bindesc, 10 | Union, 11 | I32, 12 | Sink 13 | } from "../src/index"; 14 | 15 | const BoolStr = Tuple(Bool, I32); 16 | // const BoolStr = Tuple(Bool, Str); 17 | 18 | const Inner = Union({ 19 | B: Bool, 20 | S: Tuple(Bool, F64), 21 | // S: Struct({ bool: Bool, f64: F64 }), 22 | Color: Tuple(F64, F64, F64) 23 | }); 24 | 25 | // export const Msg = Tuple(Vec(BoolStr), Vec(F64), Vec(Inner)); 26 | export const Msg = Struct({ 27 | vec: Vec(BoolStr), 28 | nums: Vec(F64), 29 | unions: Vec(Inner) 30 | }); 31 | 32 | export type Msg = Static; 33 | 34 | const genF64 = () => Math.random() * 1000000; 35 | const genI32 = () => 36 | Math.floor(Math.random() * 1000000 * (Math.random() > 0.5 ? 1 : -1)); 37 | 38 | function randomStr(length: number): string { 39 | var text = ""; 40 | var possible = "авыджалдллтЛОЫФДЛВдфульсвдыолдо"; 41 | 42 | for (var i = 0; i < length; i++) 43 | text += possible.charAt(Math.floor(Math.random() * possible.length)); 44 | 45 | return text; 46 | } 47 | randomStr; 48 | // export const genMessage = (vecSize: number): Msg => [ 49 | // Array.from({ length: vecSize }, () => BoolStr(genBool(), genF64())), 50 | // Array.from({ length: vecSize }, genF64), 51 | // Array.from({ length: vecSize }, () => { 52 | // const seed = Math.random(); 53 | 54 | // return seed < 0.33 55 | // ? Inner.B(genBool()) 56 | // : seed < 0.66 57 | // ? Inner.S([genBool(), genF64()]) 58 | // : // ? Inner.S({ bool: genBool(), f64: genF64() }) 59 | // Inner.Color([genF64(), genF64(), genF64()]); 60 | // }) 61 | // ]; 62 | export const genMessage = (vecSize: number): Msg => ({ 63 | vec: Array.from( 64 | { length: vecSize }, 65 | // () => BoolStr(genBool(), genF64()) 66 | // () => BoolStr(genBool(), randomStr(vecSize)) 67 | () => BoolStr(genBool(), genI32()) 68 | ), 69 | nums: Array.from({ length: vecSize }, genF64), 70 | unions: Array.from({ length: vecSize }, () => { 71 | const seed = Math.random(); 72 | 73 | return seed < 0.33 74 | ? Inner.B(genBool()) 75 | : seed < 0.66 76 | ? Inner.S([genBool(), genF64()]) 77 | : // ? Inner.S({ bool: genBool(), f64: genF64() }) 78 | Inner.Color([genF64(), genF64(), genF64()]); 79 | }) 80 | }); 81 | 82 | export const writeMessage = (msg: Msg, arr: Uint8Array): Uint8Array => 83 | Msg[bindesc].write({ arr, pos: 0 }, msg).arr; 84 | 85 | export const readMessage = (arr: Uint8Array): Msg => { 86 | const sink: Sink = { arr, pos: 0 }; 87 | const res = Msg[bindesc].read(sink); 88 | // console.log(`read ${sink.pos} bytes`); 89 | return res; 90 | }; 91 | 92 | function genBool(): boolean { 93 | return Math.random() > 0.5; 94 | } 95 | 96 | export type WorkerMsg = 97 | | { tag: "json"; val: Msg } 98 | | { tag: "msg_arr"; val: ArrayBuffer } 99 | | { tag: "shared_arr"; val: SharedArrayBuffer }; 100 | 101 | export const printExecTime = (name: string, hrtime: [number, number]) => 102 | console.info(name + " took (hr): %ds %dms", hrtime[0], hrtime[1] / 1000000); 103 | -------------------------------------------------------------------------------- /packages/ts-binary-types/examples/worker.ts: -------------------------------------------------------------------------------- 1 | import { parentPort } from "worker_threads"; 2 | import { writeMessage, readMessage, WorkerMsg, printExecTime } from "./message"; 3 | 4 | parentPort!.on("message", (msg: WorkerMsg) => { 5 | switch (msg.tag) { 6 | case "json": 7 | parentPort!.postMessage(msg); 8 | break; 9 | case "msg_arr": { 10 | const arr = new Uint8Array(msg.val); 11 | const hrstart = process.hrtime(); 12 | const toSend = writeMessage(readMessage(arr), arr).buffer; 13 | const hrend = process.hrtime(hrstart); 14 | parentPort!.postMessage(toSend, [toSend]); 15 | 16 | printExecTime("worker-serde", hrend); 17 | } 18 | } 19 | }); 20 | -------------------------------------------------------------------------------- /packages/ts-binary-types/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: 'ts-jest', 3 | testEnvironment: 'node', 4 | }; -------------------------------------------------------------------------------- /packages/ts-binary-types/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ts-binary-types", 3 | "version": "0.10.0", 4 | "description": "A collection of runtype types to serialize data structures in typescript to binary format (rust bincode compatible)", 5 | "license": "MIT", 6 | "type": "module", 7 | "repository": { 8 | "type": "git", 9 | "url": "https://github.com/twop/ts-rust-bridge.git" 10 | }, 11 | "publishConfig": { 12 | "access": "public" 13 | }, 14 | "@pika/pack": { 15 | "pipeline": [ 16 | [ 17 | "@pika/plugin-ts-standard-pkg", 18 | { 19 | "tsconfig": "./tsconfig.build.json", 20 | "exclude": [ 21 | "__tests__/**/*.*", 22 | "examples/**/*.*" 23 | ] 24 | } 25 | ], 26 | [ 27 | "@pika/plugin-build-node" 28 | ], 29 | [ 30 | "@pika/plugin-build-web" 31 | ] 32 | ] 33 | }, 34 | "scripts": { 35 | "build": "pika build", 36 | "test": "jest", 37 | "bench": "ts-node --project ./tsconfig.bench.json src/experiments/bench.ts", 38 | "bench-trace": "node -r esm -r ts-node/register --trace-opt --trace-deopt src/experiments/bench.ts >> bench.log", 39 | "run-example": "npm run tsc-example && node --experimental-modules examples_built/examples/main.js", 40 | "tsc-example": "tsc --build tsconfig.examples.json" 41 | }, 42 | "dependencies": { 43 | "ts-binary": "^0.8.0" 44 | }, 45 | "devDependencies": { 46 | "@babel/plugin-transform-modules-commonjs": "^7.6.0", 47 | "@babel/preset-env": "^7.6.0", 48 | "@babel/preset-typescript": "^7.6.0", 49 | "@pika/pack": "^0.5.0", 50 | "@pika/plugin-build-node": "^0.9.2", 51 | "@pika/plugin-build-web": "^0.9.2", 52 | "@pika/plugin-ts-standard-pkg": "^0.9.2", 53 | "@types/jest": "^24.0.13", 54 | "jest": "^24.8.0", 55 | "ts-jest": "^24.0.2", 56 | "esm": "3.2.17", 57 | "typescript": "3.9.5", 58 | "ts-node": "^8.4.1" 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /packages/ts-binary-types/src/core.ts: -------------------------------------------------------------------------------- 1 | import { Deserializer, Serializer } from "ts-binary"; 2 | 3 | export const bindesc = Symbol("bindesc"); 4 | 5 | export interface BinTypeDesc { 6 | read: Deserializer; 7 | write: Serializer; 8 | tag: TTag; 9 | _phantomType: T; 10 | } 11 | 12 | export type AnyBinType = BinType; 13 | 14 | export interface BinType { 15 | [bindesc]: BinTypeDesc & Extra; 16 | } 17 | 18 | export type Static< 19 | A extends BinType 20 | > = A[typeof bindesc]["_phantomType"]; 21 | 22 | export const createBinType = >( 23 | read: Deserializer>, 24 | write: Serializer>, 25 | tag: R[typeof bindesc]["tag"], 26 | extra: Omit, 27 | baseObj: Omit 28 | ): R => { 29 | (baseObj as any)[bindesc] = Object.assign(extra, { read, write, tag }); 30 | return baseObj as any; 31 | }; 32 | 33 | export enum TypeTag { 34 | Bool = "Bool", 35 | Str = "Str", 36 | U32 = "U32", 37 | F32 = "F32", 38 | F64 = "F64", 39 | U16 = "U16", 40 | I32 = "I32", 41 | U8 = "U8", 42 | Enum = "Enum", 43 | Struct = "Struct", 44 | Tuple = "Tuple", 45 | Union = "Union", 46 | Optional = "Optional", 47 | Nullable = "Nullable", 48 | Vec = "Vec" 49 | } 50 | -------------------------------------------------------------------------------- /packages/ts-binary-types/src/experiments/bench.ts: -------------------------------------------------------------------------------- 1 | import { Message, Container, Figure, Color, Vec3 } from "./tes-types"; 2 | import { Serializer, Sink, Deserializer } from "ts-binary"; 3 | import { bindesc } from "../core"; 4 | // import { bintypeToBinAst, binAst2bintype } from "./traverse"; 5 | // import { writeMessage, writeContainer } from "./ser"; 6 | // import { readMessage, readContainer } from "./deser"; 7 | 8 | let logged: any[] = []; 9 | const log = (...args: any[]) => { 10 | logged.push(args); 11 | console.log(...args); 12 | }; 13 | 14 | const bench = () => { 15 | const measure = (name: string, func: () => void) => { 16 | //log(`\n${' '.repeat(4)}${name}`); 17 | 18 | // let fastest = 100500; 19 | 20 | const numberOfRuns = 1; 21 | const takeTop = 1; 22 | 23 | let runs: number[] = []; 24 | for (let i = 0; i < numberOfRuns; i++) { 25 | const hrstart = Date.now(); 26 | func(); 27 | const hrend = Date.now(); 28 | 29 | const current = hrend - hrstart; 30 | 31 | runs.push(current); 32 | 33 | // fastest = Math.min(fastest, current); 34 | } 35 | 36 | const result = runs 37 | .sort((a, b) => a - b) 38 | .slice(numberOfRuns - takeTop, numberOfRuns) 39 | .reduce((s, v) => s + v, 0); 40 | 41 | log(`${name}: ${result.toFixed(2)} ms`); 42 | }; 43 | 44 | const COUNT = 10000; 45 | 46 | function randStr(length: number): string { 47 | var text = ""; 48 | var possible = "выфвпаывпцукждслчмДЛОДЛТДЛОЖЖЩШЛДЙТЦУЗЧЖСДЛ12389050-5435"; 49 | // 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; 50 | 51 | for (var i = 0; i < length; i++) 52 | text += possible.charAt(Math.floor(Math.random() * possible.length)); 53 | 54 | return text; 55 | } 56 | 57 | const randu8 = () => Math.floor(Math.random() * 256); 58 | const randi32 = () => 59 | Math.floor(Math.random() * 1000) * (randBool() ? 1 : -1); 60 | const randf32 = () => Math.random() * 1000; 61 | const randBool = () => Math.random() > 0.5; 62 | 63 | const ctors: (() => Message)[] = [ 64 | () => Message.Unit, 65 | () => Message.One(Math.random() * 10000), 66 | () => 67 | Message.Two([ 68 | Math.random() > 0.5 ? undefined : Math.random() > 0.5, 69 | Math.floor(Math.random() * 1000) 70 | ]), 71 | 72 | () => 73 | Message.VStruct({ 74 | id: randStr(Math.random() * 300), 75 | data: randStr(Math.random() * 300) 76 | }), 77 | () => 78 | Message.SmallStruct({ 79 | bool: randBool(), 80 | f64: randf32(), 81 | maybeI32: randBool() ? undefined : randi32() 82 | }) 83 | ]; 84 | log("----Simple(START)---"); 85 | const messages: Message[] = Array.from( 86 | { length: COUNT }, 87 | // () => ctors[2]() 88 | () => ctors[3]() // strings 89 | // () => ctors[4]() 90 | // () => ctors[Math.floor(Math.random() * 5)]() 91 | ); 92 | log("----Simple(END)---"); 93 | 94 | const genArray = (f: () => T): T[] => 95 | Array.from({ length: Math.floor(Math.random() * 30) }, f); 96 | 97 | const genColor = (): Color => Color(randu8(), randu8(), randu8()); 98 | const genVec3 = (): Vec3 => Vec3(randf32(), randf32(), randf32()); 99 | 100 | const genFigure = (): Figure => ({ 101 | dots: genArray(genVec3), 102 | colors: genArray(genColor) 103 | }); 104 | 105 | const genContainer = (): Container => { 106 | const seed = Math.random(); 107 | 108 | return seed < 0.33 109 | ? Container.Units 110 | : seed < 0.66 111 | ? Container.JustNumber(randu8()) 112 | : Container.Figures(genArray(genFigure)); 113 | }; 114 | 115 | log("----Complex(START)---"); 116 | const containers: Container[] = Array.from({ length: COUNT }, genContainer); 117 | log("----Complex(END)----"); 118 | containers; 119 | 120 | let sink: Sink = Sink(new ArrayBuffer(1000)); 121 | 122 | // const writeAThingToNothing = (thing: T, ser: Serializer): void => { 123 | // sink.pos = 0; 124 | // sink = ser(sink, thing); 125 | // }; 126 | 127 | const writeAThingToSlice = (thing: T, ser: Serializer): ArrayBuffer => { 128 | sink.pos = 0; 129 | sink = ser(sink, thing); 130 | return new Uint8Array(sink.view.buffer).slice(0, sink.pos).buffer; 131 | }; 132 | 133 | function runbench( 134 | benchName: string, 135 | data: T[], 136 | serFun: Serializer, 137 | deserializer: Deserializer 138 | ) { 139 | const strings = Array.from({ length: COUNT }, () => ""); 140 | 141 | log(" "); 142 | log(benchName.toUpperCase()); 143 | log("----Serialization----"); 144 | // measure("bincode + allocating a new buf", () => { 145 | // data.forEach(d => writeAThingToSlice(d, serFun)); 146 | // }); 147 | measure("json", () => { 148 | data.forEach((d, i) => (strings[i] = JSON.stringify(d))); 149 | }); 150 | 151 | const buffers = data.map(d => writeAThingToSlice(d, serFun)); 152 | // data.map(d => writeAThingToSlice(d, serFun)); 153 | measure("bincode", () => { 154 | // data.map(d => writeAThingToSlice(d, serFun)); 155 | // const buffers = data.map(d => writeAThingToSlice(d, serFun)); 156 | data.forEach((d, i) => (buffers[i] = writeAThingToSlice(d, serFun))); 157 | }); 158 | 159 | const res = [...data]; // just a copy 160 | 161 | log("----Deserialization----"); 162 | measure("D: bincode", () => { 163 | buffers.forEach((b, i) => (res[i] = deserializer(Sink(b)))); 164 | }); 165 | 166 | // res.forEach((d, i) => { 167 | // if (JSON.stringify(d) !== strings[i]) { 168 | // console.error('mismatch', { 169 | // expected: strings[i], 170 | // actual: JSON.stringify(d) 171 | // }); 172 | // } 173 | // }); 174 | 175 | measure("D: json", () => { 176 | strings.forEach((s, i) => (res[i] = JSON.parse(s))); 177 | }); 178 | } 179 | // }, 1000 * 20); 180 | 181 | // runbench( 182 | // 'complex', 183 | // containers, 184 | // Container[runtype].write, 185 | // Container[runtype].read 186 | // ); 187 | runbench("simple", messages, Message[bindesc].write, Message[bindesc].read); 188 | }; 189 | 190 | export const runBench = (): string[] => { 191 | bench(); 192 | const res = logged; 193 | logged = []; 194 | 195 | return res; 196 | }; 197 | 198 | runBench(); 199 | 200 | // console.log(bintypeToBinAst(Message)); 201 | // console.log(bintypeToBinAst(Container)); 202 | 203 | // const messageAst = bintypeToBinAst(Message); 204 | // const messageRestoredBintype = binAst2bintype(messageAst); 205 | 206 | // const msg = Message.VStruct({ id: "id", data: "агф1!" }); 207 | 208 | // const serialize = (thing: T, ser: Serializer): Uint8Array => { 209 | // let sink: Sink = { 210 | // arr: new Uint8Array(1), // small on purpose 211 | // pos: 0 212 | // }; 213 | 214 | // sink = ser(sink, thing); 215 | // return sink.arr.slice(0, sink.pos); 216 | // }; 217 | 218 | // const restore = (deserializer: Deserializer, from: Uint8Array): T => 219 | // deserializer({ arr: from, pos: 0 }); 220 | 221 | // console.log(msg); 222 | // console.log( 223 | // restore( 224 | // messageRestoredBintype[bindesc].read, 225 | // serialize(msg, Message[bindesc].write) 226 | // ) 227 | // ); 228 | 229 | // console.log(runBench()); 230 | 231 | // res.innerHTML = logged.map(i => `

${i.toString()}

`).join(""); 232 | // const res = document.createElement("div"); 233 | // // Write TypeScript code! 234 | // const appDiv: HTMLElement = document.getElementById("app"); 235 | 236 | // const btn = document.createElement("button"); 237 | // btn.textContent = "start"; 238 | // btn.onclick = run; 239 | // appDiv.appendChild(btn); 240 | // appDiv.appendChild(res); 241 | -------------------------------------------------------------------------------- /packages/ts-binary-types/src/experiments/example.ts: -------------------------------------------------------------------------------- 1 | import { Static } from "../core"; 2 | import { U8, F32 } from "../types/numbers"; 3 | import { Tuple } from "../types/tuple"; 4 | import { Struct } from "../types/struct"; 5 | import { Enum } from "../types/enum"; 6 | import { Union } from "../types/union"; 7 | import { U32 } from "../types/numbers"; 8 | import { retype } from "../retype"; 9 | // import { Bool } from '../types/bool'; 10 | // import { Str } from '../types/str'; 11 | // import { Bool } from './bool'; 12 | 13 | const Color_ = Tuple(U8, U8, U8); 14 | export interface Color extends Static {} 15 | export const Color = retype(Color_).as(); 16 | 17 | const Vec3_ = Tuple(F32, F32, F32); 18 | export interface Vec3 extends Static {} 19 | export const Vec3 = retype(Vec3_).as(); 20 | 21 | const S_ = Struct({ 22 | color: Color, 23 | vec3: Vec3 24 | }); 25 | 26 | export interface S extends Static {} 27 | export const S = retype(S_).as(); 28 | 29 | const E = Enum("A", "B", "C"); 30 | E; 31 | 32 | type E = Static; 33 | 34 | const U = Union({ 35 | A: U32, 36 | B: S, 37 | C: null, 38 | Col: Color 39 | }); 40 | 41 | // U. 42 | 43 | type U = Static; 44 | U.Col(Color(1, 2, 3)); 45 | 46 | // type BoxedBool = { 47 | // bool: boolean; 48 | // str: string; 49 | // }; 50 | 51 | // export interface S extends Static {} 52 | -------------------------------------------------------------------------------- /packages/ts-binary-types/src/experiments/tes-types.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Struct, 3 | Str, 4 | Union, 5 | F32, 6 | Tuple, 7 | Bool, 8 | U32, 9 | Static, 10 | U8, 11 | Vec, 12 | Optional, 13 | F64 14 | } from ".."; 15 | import { I32 } from "../types/numbers"; 16 | 17 | // import { Struct } from './types/struct'; 18 | // import { Static } from './core'; 19 | // import { Str } from './types/str'; 20 | // import { Union } from './types/union'; 21 | // import { Tuple } from './types/tuple'; 22 | // import { Option } from './types/option'; 23 | // import { Bool } from './types/bool'; 24 | // import { U32, F32, U8 } from './types/numbers'; 25 | // import { Vec } from './types/vector'; 26 | 27 | // import { 28 | // Union, 29 | // F32, 30 | // Record, 31 | // Tuple, 32 | // Str, 33 | // Option, 34 | // Bool, 35 | // U32, 36 | // Static, 37 | // U8, 38 | // Vec 39 | // } from './lib'; 40 | 41 | // pub enum Message { 42 | // Unit, 43 | // One(f32), 44 | // Two(Option, u32), 45 | // VStruct { id: String, data: String }, 46 | // } 47 | 48 | export const SmallStruct = Struct({ 49 | bool: Bool, 50 | f64: F64, 51 | maybeI32: Optional(I32) 52 | }); 53 | 54 | export const VStruct = Struct({ 55 | id: Str, 56 | data: Str 57 | }); 58 | 59 | export const Message = Union({ 60 | Unit: null, 61 | One: F32, 62 | Two: Tuple(Optional(Bool), U32), 63 | VStruct, 64 | SmallStruct 65 | }); 66 | 67 | export interface VStruct extends Static {} 68 | export type Message = Static; 69 | 70 | export interface Color extends Static {} 71 | export const Color = Tuple(U8, U8, U8); 72 | export interface Vec3 extends Static {} 73 | 74 | export const Vec3 = Tuple(F32, F32, F32); 75 | 76 | export const Figure = Struct({ 77 | dots: Vec(Vec3), 78 | colors: Vec(Color) 79 | }); 80 | 81 | export interface Figure extends Static {} 82 | 83 | export const Container = Union({ 84 | Units: null, 85 | JustNumber: U32, 86 | Figures: Vec(Figure) 87 | }); 88 | 89 | export type Container = Static; 90 | 91 | // #[derive(Deserialize, Serialize, Debug, Clone)] 92 | // pub enum Container { 93 | // Units, 94 | // JustNumber(u32), 95 | // Figures(Vec
), 96 | // } 97 | 98 | // #[derive(Deserialize, Serialize, Debug, Clone)] 99 | // pub struct Color(pub u8, pub u8, pub u8); 100 | 101 | // #[derive(Deserialize, Serialize, Debug, Clone)] 102 | // pub struct Figure { 103 | // pub dots: Vec, 104 | // pub colors: Vec, 105 | // } 106 | 107 | // #[derive(Deserialize, Serialize, Debug, Clone)] 108 | // pub struct Vec3(pub f32, pub f32, pub f32); 109 | 110 | // export type Message = 111 | // | { tag: "Unit" } 112 | // | { tag: "One"; value: number } 113 | // | { tag: "Two"; value: Message_Two } 114 | // | { tag: "VStruct"; value: Message_VStruct }; 115 | 116 | // export interface Message_Two { 117 | // 0: (boolean) | undefined; 118 | // 1: number; 119 | // length: 2; 120 | // } 121 | 122 | // export interface Message_VStruct { 123 | // id: string; 124 | // data: string; 125 | // } 126 | 127 | // export module Message { 128 | // export const Unit: Message = { tag: "Unit" }; 129 | 130 | // export const One = (value: number): Message => ({ tag: "One", value }); 131 | 132 | // export const Two = (p0: (boolean) | undefined, p1: number): Message => ({ 133 | // tag: "Two", 134 | // value: [p0, p1] 135 | // }); 136 | 137 | // export const VStruct = (value: Message_VStruct): Message => ({ 138 | // tag: "VStruct", 139 | // value 140 | // }); 141 | // } 142 | 143 | // export type NType = number & { type: "NType" }; 144 | 145 | // export const NType = (val: number): number & { type: "NType" } => val as any; 146 | 147 | // export type Container = 148 | // | { tag: "Units" } 149 | // | { tag: "JustNumber"; value: number } 150 | // | { tag: "Figures"; value: Array
}; 151 | 152 | // export module Container { 153 | // export const Units: Container = { tag: "Units" }; 154 | 155 | // export const JustNumber = (value: number): Container => ({ 156 | // tag: "JustNumber", 157 | // value 158 | // }); 159 | 160 | // export const Figures = (value: Array
): Container => ({ 161 | // tag: "Figures", 162 | // value 163 | // }); 164 | // } 165 | 166 | // export interface Color { 167 | // 0: number; 168 | // 1: number; 169 | // 2: number; 170 | // length: 3; 171 | // } 172 | 173 | // export const Color = (p0: number, p1: number, p2: number): Color => [ 174 | // p0, 175 | // p1, 176 | // p2 177 | // ]; 178 | 179 | // export interface Figure { 180 | // dots: Array; 181 | // colors: Array; 182 | // } 183 | 184 | // export interface Vec3 { 185 | // 0: number; 186 | // 1: number; 187 | // 2: number; 188 | // length: 3; 189 | // } 190 | 191 | // export const Vec3 = (p0: number, p1: number, p2: number): Vec3 => [p0, p1, p2]; 192 | 193 | // export interface NormalStruct { 194 | // a: number; 195 | // tuple: Tuple; 196 | // } 197 | 198 | // export enum Enum { 199 | // ONE = "ONE", 200 | // TWO = "TWO", 201 | // THREE = "THREE" 202 | // } 203 | 204 | // export interface Tuple { 205 | // 0: (boolean) | undefined; 206 | // 1: Array; 207 | // length: 2; 208 | // } 209 | 210 | // export const Tuple = (p0: (boolean) | undefined, p1: Array): Tuple => [ 211 | // p0, 212 | // p1 213 | // ]; 214 | 215 | // export type Aha = Array<(Array) | undefined>; 216 | -------------------------------------------------------------------------------- /packages/ts-binary-types/src/experiments/traverse.ts: -------------------------------------------------------------------------------- 1 | import { Optional } from "../types/optional"; 2 | import { Nullable } from "../types/nullable"; 3 | import { Str } from "../types/str"; 4 | import { Struct } from "../types/struct"; 5 | import { Tuple } from "../types/tuple"; 6 | import { Union } from "../types/union"; 7 | import { Vec } from "../types/vector"; 8 | import { bindesc, TypeTag, BinType } from "../core"; 9 | import { Bool } from "../types/bool"; 10 | import { F32, I32, U32, U16, U8, F64 } from "../types/numbers"; 11 | import { Enum } from "../types/enum"; 12 | 13 | export type BuiltInType = 14 | | Bool 15 | | Str 16 | | F32 17 | | F64 18 | | I32 19 | | U32 20 | | U16 21 | | U8 22 | | Enum 23 | | Optional 24 | | Nullable 25 | | Vec 26 | | Tuple 27 | | Struct 28 | | Union; 29 | 30 | module AST { 31 | export type BinNode = 32 | | TypeTag.Bool 33 | | TypeTag.Str 34 | | TypeTag.F32 35 | | TypeTag.F64 36 | | TypeTag.I32 37 | | TypeTag.U32 38 | | TypeTag.U16 39 | | TypeTag.U8 40 | | { tag: TypeTag.Optional | TypeTag.Vec | TypeTag.Nullable; type: BinNode } 41 | | { tag: TypeTag.Struct; fields: ([string, BinNode])[] } 42 | | { tag: TypeTag.Union; variants: { [key: string]: BinNode | null } } 43 | | { tag: TypeTag.Enum; variants: string[] } 44 | | { tag: TypeTag.Tuple; types: [BinNode, BinNode, ...BinNode[]] }; 45 | } 46 | 47 | export const bintypeToBinAst = (bintype: BuiltInType): AST.BinNode => { 48 | const data = bintype[bindesc]; 49 | switch (data.tag) { 50 | case TypeTag.Bool: 51 | case TypeTag.Str: 52 | case TypeTag.F32: 53 | case TypeTag.F64: 54 | case TypeTag.I32: 55 | case TypeTag.U8: 56 | case TypeTag.U16: 57 | case TypeTag.U32: 58 | return data.tag; 59 | 60 | case TypeTag.Optional: 61 | case TypeTag.Nullable: 62 | case TypeTag.Vec: 63 | return { tag: data.tag, type: bintypeToBinAst(data.type) }; 64 | 65 | case TypeTag.Tuple: 66 | return { 67 | tag: TypeTag.Tuple, 68 | types: data.components.map(bintypeToBinAst) 69 | }; 70 | 71 | case TypeTag.Enum: 72 | return { 73 | tag: TypeTag.Enum, 74 | variants: data.variants 75 | }; 76 | 77 | case TypeTag.Struct: { 78 | const fields = data.fields as { 79 | [_: string]: BinType; 80 | }; 81 | 82 | return { 83 | tag: TypeTag.Struct, 84 | fields: Object.entries(fields).map( 85 | ([fieldName, bintype]): [string, AST.BinNode] => [ 86 | fieldName, 87 | bintypeToBinAst(bintype) 88 | ] 89 | ) 90 | }; 91 | } 92 | case TypeTag.Union: { 93 | const variants = data.variants as { 94 | [_: string]: BinType | null; 95 | }; 96 | return { 97 | tag: TypeTag.Union, 98 | variants: Object.entries(variants).reduce( 99 | (v, [variantName, bintype]) => { 100 | v[variantName] = bintype === null ? null : bintypeToBinAst(bintype); 101 | return v; 102 | }, 103 | {} as { [_: string]: AST.BinNode | null } 104 | ) 105 | }; 106 | } 107 | } 108 | 109 | const shouldHaveChecked: never = data; 110 | shouldHaveChecked; 111 | 112 | throw new Error("uknown type " + (data as any).tag); 113 | }; 114 | 115 | export const binAst2bintype = (node: AST.BinNode): BuiltInType => { 116 | if (node === TypeTag.Bool) return Bool; 117 | if (node === TypeTag.Str) return Str; 118 | if (node === TypeTag.F32) return F32; 119 | if (node === TypeTag.F64) return F64; 120 | if (node === TypeTag.I32) return I32; 121 | if (node === TypeTag.U32) return U32; 122 | if (node === TypeTag.U16) return U16; 123 | if (node === TypeTag.U8) return U8; 124 | 125 | switch (node.tag) { 126 | case TypeTag.Optional: 127 | return Optional(binAst2bintype(node.type)); 128 | case TypeTag.Nullable: 129 | return Nullable(binAst2bintype(node.type)); 130 | case TypeTag.Vec: 131 | return Vec(binAst2bintype(node.type)); 132 | 133 | case TypeTag.Tuple: 134 | return Tuple( 135 | ...(node.types.map(binAst2bintype) as [ 136 | BuiltInType, 137 | BuiltInType, 138 | ...BuiltInType[] 139 | ]) 140 | ); 141 | 142 | case TypeTag.Enum: 143 | return Enum(...node.variants); 144 | 145 | case TypeTag.Struct: { 146 | const { fields } = node; 147 | 148 | const fieldsObj = fields.reduce( 149 | (obj, [fieldName, binNode]) => { 150 | obj[fieldName] = binAst2bintype(binNode); 151 | return obj; 152 | }, 153 | {} as { [_: string]: BuiltInType } 154 | ); 155 | 156 | return Struct(fieldsObj); 157 | } 158 | 159 | case TypeTag.Union: { 160 | const { variants } = node; 161 | 162 | const variantsObj = Object.keys(variants).reduce( 163 | (obj, variantName) => { 164 | const binNode = variants[variantName]; 165 | obj[variantName] = binNode === null ? null : binAst2bintype(binNode); 166 | return obj; 167 | }, 168 | {} as { [_: string]: BuiltInType | null } 169 | ); 170 | 171 | return Union(variantsObj); 172 | } 173 | } 174 | 175 | throw new Error("uknown type " + JSON.stringify(node)); 176 | }; 177 | -------------------------------------------------------------------------------- /packages/ts-binary-types/src/index.ts: -------------------------------------------------------------------------------- 1 | export { Bool } from "./types/bool"; 2 | export { Enum } from "./types/enum"; 3 | export { I32, F32, F64, U8, U16, U32 } from "./types/numbers"; 4 | export { Optional } from "./types/optional"; 5 | export { Nullable } from "./types/nullable"; 6 | export { Str } from "./types/str"; 7 | export { Struct, RStruct } from "./types/struct"; 8 | export { Tuple, RTuple } from "./types/tuple"; 9 | export { Union, ConstTag, Tagged } from "./types/union"; 10 | export { Vec } from "./types/vector"; 11 | 12 | export { retype } from "./retype"; 13 | export { 14 | AnyBinType, 15 | Static, 16 | BinType, 17 | BinTypeDesc, 18 | TypeTag, 19 | bindesc, 20 | createBinType 21 | } from "./core"; 22 | 23 | export * from "ts-binary"; 24 | -------------------------------------------------------------------------------- /packages/ts-binary-types/src/retype.ts: -------------------------------------------------------------------------------- 1 | import { Tuple, RTuple } from './types/tuple'; 2 | import { Static } from './core'; 3 | import { RStruct, Struct } from './types/struct'; 4 | 5 | export const retype = | Struct>( 6 | bintype: TBin 7 | ) => ({ 8 | as: >(): Static extends T 9 | ? TBin extends Tuple 10 | ? RTuple 11 | : TBin extends Struct 12 | ? RStruct 13 | : never 14 | : never => bintype as any 15 | }); 16 | -------------------------------------------------------------------------------- /packages/ts-binary-types/src/types/bool.ts: -------------------------------------------------------------------------------- 1 | import { read_bool, write_bool } from "ts-binary"; 2 | import { createBinType, BinType, TypeTag } from "../core"; 3 | 4 | export interface Bool extends BinType {} 5 | 6 | export const Bool = createBinType( 7 | read_bool, 8 | write_bool, 9 | TypeTag.Bool, 10 | {}, 11 | {} 12 | ); 13 | -------------------------------------------------------------------------------- /packages/ts-binary-types/src/types/enum.ts: -------------------------------------------------------------------------------- 1 | import { Sink, read_u32, Serializer, write_u32 } from "ts-binary"; 2 | import { BinType, createBinType, TypeTag } from "../core"; 3 | 4 | type Variants = { [K in V[number]]: K }; 5 | 6 | interface EnumBinType 7 | extends BinType {} 8 | 9 | export type Enum = EnumBinType & Variants; 10 | 11 | export const Enum = ( 12 | ...variants: V 13 | ): Enum & Variants => 14 | createBinType>( 15 | (sink: Sink) => variants[read_u32(sink)], 16 | createEnumSerializer(variants), 17 | TypeTag.Enum, 18 | { variants } as any, 19 | variants.reduce((map, variant) => ({ ...map, [variant]: variant }), {} as { 20 | [key: string]: string; 21 | }) as any 22 | ); 23 | 24 | const createEnumSerializer = ( 25 | variants: V 26 | ): Serializer => { 27 | const valToIndex = variants.reduce( 28 | (map, v, i) => ({ ...map, [v]: i }), 29 | {} as { [key: string]: number } 30 | ); 31 | 32 | return (sink: Sink, val) => write_u32(sink, valToIndex[val]); 33 | }; 34 | -------------------------------------------------------------------------------- /packages/ts-binary-types/src/types/nullable.ts: -------------------------------------------------------------------------------- 1 | import { BinType, Static, bindesc, createBinType, TypeTag } from "../core"; 2 | import { write_u8, read_u8 } from "ts-binary"; 3 | 4 | export interface Nullable> 5 | extends BinType | null, { type: T }> {} 6 | 7 | export const Nullable = >(type: T) => { 8 | const { read, write } = type[bindesc]; 9 | 10 | return createBinType>( 11 | sink => (read_u8(sink) === 1 ? read(sink) : null), 12 | (sink, val) => 13 | val === null ? write_u8(sink, 0) : write(write_u8(sink, 1), val), 14 | TypeTag.Nullable, 15 | { type }, 16 | {} 17 | ); 18 | }; 19 | -------------------------------------------------------------------------------- /packages/ts-binary-types/src/types/numbers.ts: -------------------------------------------------------------------------------- 1 | import { BinType, createBinType, TypeTag } from "../core"; 2 | import { 3 | read_u8, 4 | write_u8, 5 | read_f32, 6 | write_f32, 7 | read_u32, 8 | write_u32, 9 | read_u16, 10 | write_u16, 11 | read_i32, 12 | write_i32, 13 | write_f64, 14 | read_f64 15 | } from "ts-binary"; 16 | 17 | export interface U32 extends BinType {} 18 | export const U32 = createBinType(read_u32, write_u32, TypeTag.U32, {}, {}); 19 | 20 | export interface U8 extends BinType {} 21 | export const U8 = createBinType(read_u8, write_u8, TypeTag.U8, {}, {}); 22 | 23 | export interface U16 extends BinType {} 24 | export const U16 = createBinType(read_u16, write_u16, TypeTag.U16, {}, {}); 25 | 26 | export interface F32 extends BinType {} 27 | export const F32 = createBinType(read_f32, write_f32, TypeTag.F32, {}, {}); 28 | 29 | export interface F64 extends BinType {} 30 | export const F64 = createBinType(read_f64, write_f64, TypeTag.F64, {}, {}); 31 | 32 | export interface I32 extends BinType {} 33 | export const I32 = createBinType(read_i32, write_i32, TypeTag.I32, {}, {}); 34 | -------------------------------------------------------------------------------- /packages/ts-binary-types/src/types/optional.ts: -------------------------------------------------------------------------------- 1 | import { BinType, Static, bindesc, createBinType, TypeTag } from "../core"; 2 | import { opt_reader, opt_writer } from "ts-binary"; 3 | 4 | export interface Optional> 5 | extends BinType | undefined, { type: T }> {} 6 | 7 | export const Optional = >(type: T) => { 8 | const { read, write } = type[bindesc]; 9 | return createBinType>( 10 | opt_reader(read), 11 | opt_writer(write), 12 | TypeTag.Optional, 13 | { type }, 14 | {} 15 | ); 16 | }; 17 | -------------------------------------------------------------------------------- /packages/ts-binary-types/src/types/str.ts: -------------------------------------------------------------------------------- 1 | import { read_str, write_str } from "ts-binary"; 2 | import { createBinType, BinType, TypeTag } from "../core"; 3 | 4 | export interface Str extends BinType {} 5 | export const Str = createBinType(read_str, write_str, TypeTag.Str, {}, {}); 6 | -------------------------------------------------------------------------------- /packages/ts-binary-types/src/types/struct.ts: -------------------------------------------------------------------------------- 1 | import { BinType, Static, createBinType, bindesc, TypeTag } from "../core"; 2 | import { Serializer, Sink, Deserializer } from "ts-binary"; 3 | 4 | type StructDefs = { [_: string]: BinType }; 5 | 6 | type StructStaticType = { 7 | [K in keyof SD]: Static 8 | }; 9 | 10 | export interface Struct 11 | extends BinType< 12 | TypeTag.Struct, 13 | StructStaticType, 14 | { fields: Fields } 15 | > {} 16 | 17 | export interface RStruct 18 | extends BinType {} 19 | 20 | export const Struct = ( 21 | fields: Fields 22 | ): Struct => 23 | createBinType>( 24 | createStructDeserializer(fields), 25 | createStructSerializer(fields), 26 | TypeTag.Struct, 27 | { fields }, 28 | {} 29 | ); 30 | 31 | const createStructSerializer = ( 32 | fields: O 33 | ): Serializer> => { 34 | return buildStructSerializer(undefined, 0, Object.keys(fields), fields); 35 | 36 | // const structSer = Object.keys(fields) 37 | // .map(k => ({ k, w: fields[k][bindesc].write })) 38 | // .reduce>>( 39 | // (ser, { k, w }) => writeField(r => r[k], w, ser), 40 | // (s, _) => s 41 | // ); 42 | // return structSer; 43 | }; 44 | 45 | const buildStructSerializer = ( 46 | existingSer: Serializer | undefined, 47 | i: number, 48 | keys: string[], 49 | fields: { [_: string]: BinType } 50 | ): Serializer => { 51 | if (i >= keys.length) { 52 | return existingSer ? existingSer : (s, _r) => s; 53 | } 54 | 55 | const key = keys[i]; 56 | const writeField = fields[key][bindesc].write; 57 | 58 | const ser: Serializer = existingSer 59 | ? (s, r) => writeField(existingSer(s, r), (r as any)[key]) 60 | : (s, r) => writeField(s, (r as any)[key]); 61 | 62 | return buildStructSerializer(ser, i + 1, keys, fields); 63 | }; 64 | 65 | // const writeField = ( 66 | // getA: (r: Rec) => A, 67 | // serA: Serializer, 68 | // ser: Serializer 69 | // ): Serializer => { 70 | // return (sink: Sink, r: Rec) => serA(ser(sink, r), getA(r)); 71 | // }; 72 | 73 | const createStructDeserializer = ( 74 | fields: O 75 | ): Deserializer> => { 76 | const keys = Object.keys(fields); 77 | const deserializers = keys.map(k => fields[k][bindesc].read); 78 | 79 | return function structDes(sink: Sink) { 80 | const rec: { [_: string]: any } = {}; 81 | 82 | for (let i = 0; i < keys.length; i++) { 83 | rec[keys[i]] = deserializers[i](sink); 84 | } 85 | 86 | return rec as any; 87 | }; 88 | }; 89 | -------------------------------------------------------------------------------- /packages/ts-binary-types/src/types/tuple.ts: -------------------------------------------------------------------------------- 1 | import { 2 | BinType, 3 | Static, 4 | bindesc, 5 | createBinType, 6 | TypeTag, 7 | AnyBinType 8 | } from "../core"; 9 | import { Serializer, Deserializer } from "ts-binary"; 10 | 11 | type ValuesOf = { 12 | [K in keyof C]: C[K] extends AnyBinType ? Static : C[K] 13 | }; 14 | 15 | export interface Tuple 16 | extends BinType, { components: C }> { 17 | (...t: ValuesOf): ValuesOf; 18 | } 19 | 20 | export interface RTuple< 21 | C extends [AnyBinType, AnyBinType, ...AnyBinType[]], 22 | Alias 23 | > extends BinType { 24 | (...t: ValuesOf): Alias; 25 | } 26 | 27 | export function Tuple( 28 | ...c: C 29 | ): Tuple { 30 | let ctor: (...args: any[]) => any; 31 | 32 | const components = c; 33 | const serializers = components.map(c => c[bindesc].write); 34 | const deserializers = components.map(c => c[bindesc].read); 35 | 36 | let write: Serializer> | undefined; 37 | let read: Deserializer> | undefined; 38 | 39 | switch (components.length) { 40 | case 2: { 41 | ctor = (a, b) => ([a, b] as unknown) as ValuesOf; 42 | 43 | const [s0, s1] = serializers; 44 | write = (sink, [a, b]) => s1(s0(sink, a), b); 45 | 46 | const [d0, d1] = deserializers; 47 | read = sink => ctor(d0(sink), d1(sink)); 48 | break; 49 | } 50 | case 3: { 51 | ctor = (a, b, c) => ([a, b, c] as unknown) as ValuesOf; 52 | 53 | const [s0, s1, s2] = serializers; 54 | write = (sink, [a, b, c]) => s2(s1(s0(sink, a), b), c); 55 | 56 | const [d0, d1, d2] = deserializers; 57 | read = sink => ctor(d0(sink), d1(sink), d2(sink)); 58 | break; 59 | } 60 | default: { 61 | ctor = (...args) => args; 62 | 63 | write = (sink, tuple) => 64 | tuple.reduce((s, val, i) => serializers[i](s, val), sink); 65 | 66 | read = sink => deserializers.map(d => d(sink)) as any; 67 | break; 68 | } 69 | } 70 | 71 | return createBinType(read, write, TypeTag.Tuple, { components }, ctor); 72 | } 73 | -------------------------------------------------------------------------------- /packages/ts-binary-types/src/types/union.ts: -------------------------------------------------------------------------------- 1 | import { Static, BinType, createBinType, bindesc, TypeTag } from "../core"; 2 | import { Serializer, write_u32, Sink, Deserializer, read_u32 } from "ts-binary"; 3 | // import { Tuple2, Tuple3 } from './tuple'; 4 | 5 | // type VariantDef = Runtype | null | Inline; 6 | type VariantDefs = { [_: string]: BinType | null }; 7 | 8 | // type Inline | Tuple3> = { 9 | // inlined: 'tuple'; 10 | // tuple: T; 11 | // }; 12 | 13 | // export const inline = | Tuple3>( 14 | // tuple: T 15 | // ): Inline => ({ tuple, inlined: 'tuple' }); 16 | 17 | export interface Tagged { 18 | tag: T; 19 | val: Val; 20 | } 21 | 22 | export interface ConstTag { 23 | tag: T; 24 | val: undefined; 25 | } 26 | 27 | type Unify = { 28 | [K in keyof Record]: null extends Record[K] 29 | ? ConstTag 30 | : Record[K] extends BinType 31 | ? Tagged, K> //{ tag: K; val: Static } 32 | : never 33 | }; 34 | 35 | type Contsructors = { 36 | [K in keyof V]: null extends V[K] 37 | ? UnionVal 38 | : V[K] extends BinType 39 | ? (val: Static) => UnionVal 40 | : never 41 | }; 42 | 43 | type JustValues< 44 | TaggedRecord extends { [_: string]: any } 45 | > = TaggedRecord[keyof TaggedRecord]; 46 | 47 | type UnionVal = JustValues>; 48 | 49 | interface UnionRuntype 50 | extends BinType, { variants: V }> {} 51 | 52 | export type Union = UnionRuntype & Contsructors; 53 | 54 | // TODO have a little more typesafety than 5 `as any` out of 5 55 | export const Union = (variants: V): Union => 56 | createBinType>( 57 | createUnionDeserializer(variants) as any, 58 | createUnionSerializer(variants), 59 | TypeTag.Union as any, 60 | { variants } as any, 61 | createUnionConstructors(variants) as any 62 | ); 63 | 64 | const createUnionSerializer = ( 65 | variants: V 66 | ): Serializer> => { 67 | const serializersMap = {} as { 68 | [key: string]: Serializer; 69 | }; 70 | 71 | Object.keys(variants).forEach((key, i) => { 72 | const variantType = variants[key]; 73 | 74 | if (variantType === null) { 75 | serializersMap[key] = (sink: Sink, _val: undefined) => write_u32(sink, i); 76 | } else { 77 | const serializer = variantType![bindesc].write; 78 | serializersMap[key] = (sink: Sink, val: any) => 79 | serializer(write_u32(sink, i), val); 80 | } 81 | }); 82 | 83 | return ((sink: Sink, { tag, val }: { tag: string; val: any }) => 84 | serializersMap[tag](sink, val)) as any; 85 | }; 86 | 87 | const createUnionDeserializer = ( 88 | variants: V 89 | ): Deserializer> => { 90 | const keys = Object.keys(variants); 91 | 92 | const deserializers = keys.map(key => { 93 | const variantType = variants[key]; 94 | return variantType && variantType[bindesc].read; 95 | }); 96 | 97 | const unitValues = keys.map(key => { 98 | const bintype = variants[key]; 99 | return bintype ? null : { tag: key, val: undefined }; 100 | }); 101 | 102 | return (sink: Sink): any => { 103 | const index = read_u32(sink); 104 | const deserializer = deserializers[index]; 105 | const tag = keys[index]; 106 | return deserializer ? { tag, val: deserializer(sink) } : unitValues[index]; 107 | }; 108 | }; 109 | 110 | const createUnionConstructors = ( 111 | variants: V 112 | ): Contsructors => 113 | Object.keys(variants).reduce( 114 | (map, tag) => { 115 | const typeDef = variants[tag]; 116 | (map as any)[tag] = 117 | typeDef === null 118 | ? { tag, val: undefined } 119 | : (val: any) => ({ tag, val }); 120 | return map; 121 | }, 122 | {} as Contsructors 123 | ); 124 | -------------------------------------------------------------------------------- /packages/ts-binary-types/src/types/vector.ts: -------------------------------------------------------------------------------- 1 | import { BinType, Static, bindesc, createBinType, TypeTag } from "../core"; 2 | import { seq_reader, seq_writer } from "ts-binary"; 3 | 4 | export interface Vec> 5 | extends BinType[], { type: T }> {} 6 | 7 | export const Vec = >(type: T) => { 8 | const { read, write } = type[bindesc]; 9 | return createBinType>( 10 | seq_reader(read), 11 | seq_writer(write), 12 | TypeTag.Vec, 13 | { type }, 14 | {} 15 | ); 16 | }; 17 | -------------------------------------------------------------------------------- /packages/ts-binary-types/tsconfig.bench.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.build.json", 3 | 4 | "compilerOptions": { 5 | "rootDir": "./src", 6 | "module": "commonjs" 7 | }, 8 | 9 | "references": [ 10 | { 11 | "path": "../ts-binary/tsconfig.build.json" 12 | } 13 | ], 14 | 15 | "include": ["src/**/*"] 16 | } 17 | -------------------------------------------------------------------------------- /packages/ts-binary-types/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.build.json", 3 | 4 | "compilerOptions": { 5 | "outDir": "./dist", 6 | "rootDir": "./src", 7 | "composite": true 8 | }, 9 | 10 | "references": [ 11 | { 12 | "path": "../ts-binary/tsconfig.build.json" 13 | } 14 | ], 15 | 16 | "include": ["src/**/*"] 17 | } 18 | -------------------------------------------------------------------------------- /packages/ts-binary-types/tsconfig.examples.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.build.json", 3 | 4 | "compilerOptions": { 5 | "outDir": "./examples_built", 6 | "rootDir": "./", 7 | "composite": true, 8 | "module": "commonjs" 9 | }, 10 | 11 | "references": [ 12 | { 13 | "path": "../ts-binary/tsconfig.build.json" 14 | } 15 | ], 16 | 17 | "include": ["examples/**/*", "src/**/*"] 18 | } 19 | -------------------------------------------------------------------------------- /packages/ts-binary-types/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | 4 | "include": ["src/**/*", "examples/**/*"] 5 | } 6 | -------------------------------------------------------------------------------- /packages/ts-binary/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true 3 | } 4 | -------------------------------------------------------------------------------- /packages/ts-binary/README.MD: -------------------------------------------------------------------------------- 1 | # ts-binary 2 | 3 | A collection of helper functions to serialize/deserialize primitive types in typescript. It is designed to be a building block rather than a standalone library. 4 | 5 | ## Install 6 | 7 | `npm install ts-binary --save` 8 | 9 | ## Example 10 | 11 | ```ts 12 | import { Sink, write_str, read_str, write_f32, read_f32 } from 'ts-binary'; 13 | 14 | let sink: Sink = Sink(new ArrayBuffer(100)); 15 | // Sink is just a convince function not a class ctor 16 | // note that it resizes automatically if needed 17 | 18 | sink = write_str(sink, 'abc'); 19 | sink = write_f32(sink, 3.14); 20 | 21 | sink.pos = 0; // reset position to read from the beginning 22 | const str = read_str(sink); // 'abc' 23 | const num = read_f32(sink); // 3.14, actually 3.140000104904175 :) 24 | ``` 25 | 26 | ## Why 27 | 28 | The project was created as a way to communicate between rust and typescript (checkout https://github.com/cztomsik/stain). For a while we used json, but it wasn't efficient enough (plus there were some pain points to maintain type compatibility between rust and typescript). Rust already had a standard way to serialize stuff: https://serde.rs/ + https://github.com/TyOverby/bincode. So the solution was to adopt serde type system + binary layout from bincode. 29 | 30 | The idea: 31 | 32 | 1. Codegen contract types based on the schema + codegen serializers/deserializers for them. 33 | 2. Write a small library to serialize/deserialize primitive types (string, boolean, optional, sequence, f32, u32, ...) 34 | 35 | This library is an implementation of 2) 36 | 37 | ## Design goals 38 | 39 | 1. Be compatible with bincode and serde (https://github.com/TyOverby/bincode) 40 | 2. Fast -> try to be JIT friendly if possible 41 | 3. Small -> optimized for js minification (it is just a collection of functions) 42 | 4. Designed as a building block -> it doesn't do much outside of serializing/deserializing primitive types. 43 | 5. Should work across all modern runtimes (browsers + node) 44 | 45 | ## Api 46 | 47 | the api is based on three concepts: 48 | 49 | ```ts 50 | export type Sink = { 51 | pos: number; 52 | view: DataView; 53 | littleEndian: boolean; 54 | }; 55 | 56 | type Serializer = (sink: Sink, val: T) => Sink; 57 | type Deserializer = (sink: Sink) => T; 58 | ``` 59 | 60 | `Sink` is used for both serialization and deserialization. It is just a buffer with a current position to read/write from. Sink instance is designed to be reused (normally you will have a single buffer/sink to read and write from). 61 | 62 | `Serializer` is a function that writes a value of type T to the `sink` starting from `sink.pos` (moves `pos` in the process). It is assumed that it will resize the buffer if it needs more space (important if you want to implement custom serializers). Note that it always returns a `Sink` which can be a brand new instance. You **cannot** rely on that the initial sink will be mutated, always use the returned value instead. 63 | 64 | `Deserializer` is a function that reads a value of type T from the `sink` starting from `sink.pos` (moves `pos` in the process). Deserializers in this library **don't** check for out of boundary cases. It is assumed that the correct data is just there. 65 | 66 | And because these are just types/conventions it is easy to support custom types. Which is exactly how more complex data structures are handled in [ts-rust-bridge-codegen](https://github.com/twop/ts-rust-bridge/tree/master/packages/ts-rust-bridge-codegen) 67 | 68 | ### Supported primitives 69 | 70 | At the moment only the list of supported types: 71 | 72 | `u8, u16, u32, u64, i32, f32, f64, string, boolean, Option, Sequence`. 73 | 74 | Caveat: u64 is serialized as 8 bytes but only 4 bytes are actually being used. Which means that technically only u32 values are supported (but nothing stops you from implementing that yourself :) ). 75 | 76 | ### Numbers: 77 | 78 | ```ts 79 | const write_u8: Serializer; 80 | const write_u16: Serializer; 81 | const write_u32: Serializer; 82 | const write_u64: Serializer; 83 | const write_f32: Serializer; 84 | const write_f64: Serializer; 85 | const write_i32: Serializer; 86 | 87 | const read_u8: Deserializer; 88 | const read_u16: Deserializer; 89 | const read_u32: Deserializer; 90 | const read_u64: Deserializer; 91 | const read_f32: Deserializer; 92 | const read_f64: Deserializer; 93 | const read_i32: Deserializer; 94 | ``` 95 | 96 | ### Strings 97 | 98 | Note that strings are stored as UTF8 via `TextEncoder` class (and `TextDecoder` on the way back). It is serialized as u64 (number of bytes encoded) + UTF8 encoded string. 99 | 100 | ```ts 101 | const write_str: Serializer; 102 | const read_str: Deserializer; 103 | ``` 104 | 105 | ### Bool 106 | 107 | Encoded as 1 or 0 in a single byte. 108 | 109 | ```ts 110 | const write_bool: Serializer; 111 | const read_bool: Deserializer; 112 | ``` 113 | 114 | ### Sequence 115 | 116 | You can make new serializer and deserializer for `T[]` out of serializer/deserializer of type `T`. It encodes a sequence as u64 + serialized elements. 117 | 118 | ```ts 119 | const seq_writer: (serEl: Serializer) => Serializer; 120 | const seq_reader: (readEl: Deserializer) => Deserializer; 121 | ``` 122 | 123 | ### Optional 124 | 125 | Optionals are described as `T | undefined` (but not `null`). Encoded as 1 (one byte) followed up by a serialized value T if !== undefined or just 0 (one byte). 126 | 127 | ```ts 128 | // similar to sequence they produce new serializer/deserializer 129 | const opt_writer: (serEl: Serializer) => Serializer; 130 | const opt_reader: (readEl: Deserializer) => Deserializer; 131 | ``` 132 | 133 | ### Nullable 134 | 135 | In case you need to deal with nulls, there is also a nullable version that is described as `T | null` (but not `undefined`). Encoded as 1 (one byte) followed up by a serialized value T if !== null or just 0 (one byte). 136 | 137 | ```ts 138 | const nullable_writer: (serEl: Serializer) => Serializer; 139 | const nullable_reader: (readEl: Deserializer) => Deserializer; 140 | ``` 141 | 142 | Caveat: `Nullable>` equals to just `Nullable` in typescript, but in rust this is technically not true. 143 | 144 | ## Simple benchmarks 145 | 146 | I just copypasted generated code from examples and tried to construct a simple benchmark. 147 | 148 | Code 149 | https://stackblitz.com/edit/ts-binaray-benchmark?file=index.ts 150 | 151 | Version to try 152 | https://ts-binaray-benchmark.stackblitz.io 153 | 154 | On complex data structure: 155 | 156 | | Method | Serialization | Deserialization | 157 | | --------- | :-----------: | --------------: | 158 | | ts-binary | 74 ms | 91 ms | 159 | | JSON | 641 ms | 405 ms | 160 | 161 | Simple data structure: 162 | 163 | | Method | Serialization | Deserialization | 164 | | --------- | :-----------: | --------------: | 165 | | ts-binary | 2 ms | 1 ms | 166 | | JSON | 6 ms | 5 ms | 167 | 168 | That was measured on latest Safari version. 169 | 170 | Note you can run the benchmark yourself cloning the repo + running npm scripts 171 | 172 | ## FAQ 173 | 174 | ### Why `_` in function names? 175 | 176 | I wanted something that would signal if it is a library serializer vs custom in my generated code (I use camel casing for custom ones). Plus I didn't really like how `writeF32` looked :) 177 | 178 | ## License 179 | 180 | MIT. 181 | -------------------------------------------------------------------------------- /packages/ts-binary/__tests__/serializeAndRestore.test.ts: -------------------------------------------------------------------------------- 1 | import { 2 | write_str, 3 | seq_writer, 4 | write_u32, 5 | seq_reader, 6 | read_u32, 7 | read_str, 8 | Sink, 9 | Serializer, 10 | Deserializer, 11 | read_u16, 12 | write_u16, 13 | read_bool, 14 | write_bool, 15 | read_u8, 16 | write_u8, 17 | opt_writer, 18 | opt_reader, 19 | nullable_reader, 20 | nullable_writer, 21 | read_f32, 22 | write_f32, 23 | write_i32, 24 | read_i32, 25 | write_f64, 26 | read_f64, 27 | } from '../src/index'; 28 | 29 | const serializeAndRestore = ( 30 | thing: T, 31 | serializer: Serializer, 32 | deserializer: Deserializer 33 | ): T => { 34 | let sink = Sink(new ArrayBuffer(1)); 35 | sink = serializer(sink, thing); 36 | sink.pos = 0; 37 | return deserializer(sink); 38 | }; 39 | 40 | test('it reads and writes string', () => { 41 | const str = 'abc + абс'; 42 | expect(serializeAndRestore(str, write_str, read_str)).toBe(str); 43 | }); 44 | 45 | test('it reads and writes u32', () => { 46 | const u32 = 100500; 47 | expect(serializeAndRestore(u32, write_u32, read_u32)).toBe(u32); 48 | }); 49 | 50 | test('it reads and writes f32', () => { 51 | const f32 = 100.5; 52 | expect(serializeAndRestore(f32, write_f32, read_f32)).toBe(f32); 53 | }); 54 | 55 | test('it reads and writes f64', () => { 56 | const f64 = Number.MAX_VALUE; 57 | expect(serializeAndRestore(f64, write_f64, read_f64)).toBe(f64); 58 | }); 59 | 60 | test('it reads and writes i32', () => { 61 | const i32 = -100; 62 | expect(serializeAndRestore(i32, write_i32, read_i32)).toBe(i32); 63 | }); 64 | 65 | test('it reads and writes u16', () => { 66 | const u16 = 253 * 4; // will require two bytes to store 67 | expect(serializeAndRestore(u16, write_u16, read_u16)).toBe(u16); 68 | }); 69 | 70 | test('it reads and writes bool', () => { 71 | const bool = false; 72 | expect(serializeAndRestore(bool, write_bool, read_bool)).toBe(bool); 73 | }); 74 | 75 | test('it reads and writes u8', () => { 76 | const u8 = 173; 77 | expect(serializeAndRestore(u8, write_u8, read_u8)).toBe(u8); 78 | }); 79 | 80 | test('it reads and writes sequence of strings', () => { 81 | const seq = ['abc', 'бла', 'some other str']; 82 | 83 | const writeStrings = seq_writer(write_str); 84 | 85 | const readStrings = seq_reader(read_str); 86 | 87 | expect(serializeAndRestore(seq, writeStrings, readStrings)).toEqual(seq); 88 | }); 89 | 90 | test('it reads and writes optional string', () => { 91 | const writeOptString = opt_writer(write_str); 92 | const readOptString = opt_reader(read_str); 93 | 94 | const str = 'some str'; 95 | expect(serializeAndRestore(str, writeOptString, readOptString)).toEqual(str); 96 | 97 | expect(serializeAndRestore(undefined, writeOptString, readOptString)).toEqual( 98 | undefined 99 | ); 100 | }); 101 | 102 | test('it reads and writes nullable string', () => { 103 | const writeNullableString = nullable_writer(write_str); 104 | const readNullableString = nullable_reader(read_str); 105 | 106 | const str = 'some str'; 107 | expect( 108 | serializeAndRestore(str, writeNullableString, readNullableString) 109 | ).toEqual(str); 110 | 111 | expect( 112 | serializeAndRestore(null, writeNullableString, readNullableString) 113 | ).toEqual(null); 114 | }); 115 | -------------------------------------------------------------------------------- /packages/ts-binary/__tests__/writeU64.test.ts: -------------------------------------------------------------------------------- 1 | import { Sink, write_u64, write_u8, read_u8 } from '../src/index'; 2 | 3 | test('it writes 8 bytes', () => { 4 | // Fill with 2s. 5 | let sink = Sink(new ArrayBuffer(8)); 6 | for (let i = 0; i < 8; i++) { 7 | sink = write_u8(sink, 2); 8 | } 9 | 10 | // Should write out a single 1 and 7 0's, no matter the endianness. 11 | sink.pos = 0; 12 | sink = write_u64(sink, 1); 13 | 14 | // No byte from the original write should remain. 15 | sink.pos = 0; 16 | for (let i = 0; i < 8; i++) { 17 | expect(read_u8(sink)).toBeLessThan(2); 18 | } 19 | }); 20 | -------------------------------------------------------------------------------- /packages/ts-binary/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: 'ts-jest', 3 | testEnvironment: 'node', 4 | }; -------------------------------------------------------------------------------- /packages/ts-binary/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ts-binary", 3 | "version": "0.11.0", 4 | "description": "A collection of helper functions to serialize data structures in typescript to rust bincode format", 5 | "license": "MIT", 6 | "repository": { 7 | "type": "git", 8 | "url": "https://github.com/twop/ts-rust-bridge.git" 9 | }, 10 | "publishConfig": { 11 | "access": "public" 12 | }, 13 | "@pika/pack": { 14 | "pipeline": [ 15 | [ 16 | "@pika/plugin-ts-standard-pkg", 17 | { 18 | "tsconfig": "./tsconfig.build.json", 19 | "exclude": [ 20 | "__tests__/**/*.*", 21 | "examples/**/*.*" 22 | ] 23 | } 24 | ], 25 | [ 26 | "@pika/plugin-build-node" 27 | ], 28 | [ 29 | "@pika/plugin-build-web" 30 | ] 31 | ] 32 | }, 33 | "scripts": { 34 | "build": "pika build", 35 | "test": "jest" 36 | }, 37 | "dependencies": {}, 38 | "devDependencies": { 39 | "@babel/plugin-transform-modules-commonjs": "^7.6.0", 40 | "@babel/preset-env": "^7.6.0", 41 | "@babel/preset-typescript": "^7.6.0", 42 | "@pika/pack": "^0.5.0", 43 | "@pika/plugin-build-node": "^0.9.2", 44 | "@pika/plugin-build-web": "^0.9.2", 45 | "@pika/plugin-ts-standard-pkg": "^0.9.2", 46 | "@types/jest": "^24.0.11", 47 | "jest": "^24.7.1", 48 | "ts-jest": "^24.0.2", 49 | "esm": "3.2.17", 50 | "typescript": "3.9.5" 51 | }, 52 | "types": "pkg/dist-types/index.d.ts", 53 | "main": "pkg/dist-node/index.js", 54 | "module": "pkg/dist-web/index.js" 55 | } 56 | -------------------------------------------------------------------------------- /packages/ts-binary/src/index.ts: -------------------------------------------------------------------------------- 1 | export type Sink = { 2 | pos: number; 3 | view: DataView; 4 | littleEndian: boolean; 5 | }; 6 | 7 | export const Sink = ( 8 | arr: ArrayBuffer, 9 | pos: number = 0, 10 | littleEndian: boolean = true 11 | ): Sink => ({ 12 | pos, 13 | littleEndian, 14 | view: new DataView(arr), 15 | }); 16 | 17 | export type Serializer = (sink: Sink, val: T) => Sink; 18 | export type Deserializer = (sink: Sink) => T; 19 | 20 | const reserve = (sink: Sink, numberOfBytes: number): Sink => { 21 | const { 22 | view: { buffer }, 23 | pos, 24 | } = sink; 25 | 26 | const curLen = buffer.byteLength; 27 | if (curLen - pos > numberOfBytes) return sink; 28 | 29 | const newLen = Math.max(curLen * 2, curLen + numberOfBytes); 30 | const newArr = new Uint8Array(newLen); 31 | newArr.set(new Uint8Array(buffer)); 32 | return Sink(newArr.buffer, pos, sink.littleEndian); 33 | }; 34 | 35 | export const write_u8: Serializer = (sink, val) => { 36 | sink = reserve(sink, 1); 37 | const { view, pos } = sink; 38 | view.setUint8(pos, val); 39 | return (sink.pos += 1), sink; 40 | }; 41 | 42 | export const write_u32: Serializer = (sink, val) => { 43 | sink = reserve(sink, 4); 44 | const { view, pos, littleEndian } = sink; 45 | view.setUint32(pos, val, littleEndian); 46 | return (sink.pos += 4), sink; 47 | }; 48 | 49 | export const write_u16: Serializer = (sink, val) => { 50 | sink = reserve(sink, 2); 51 | const { view, pos, littleEndian } = sink; 52 | view.setUint16(pos, val, littleEndian); 53 | return (sink.pos += 2), sink; 54 | }; 55 | 56 | function write_u64_unchecked(sink: Sink, val: number) { 57 | const { view, pos, littleEndian } = sink; 58 | // Even though we only support writing 4 byte values from JS, it's important 59 | // to write 8 bytes in case the buffer is not filled with 0. Otherwise, other 60 | // languages that can support 64 bit values (e.g. rust) risk reading garbage 61 | // in the other 4 bytes and getting an incorrect value. 62 | // 63 | // We could require that the Sink buffer be zeroed as part of the API, but 64 | // that's easy to forget (and it might be less efficient - it may require the 65 | // caller to zero out the entire buffer when that may be mostly unnecessary). 66 | view.setUint32(littleEndian ? pos : pos + 4, val, littleEndian); 67 | view.setUint32(littleEndian ? pos + 4 : pos, 0, littleEndian); 68 | return (sink.pos += 8), sink; 69 | } 70 | 71 | export const write_u64: Serializer = (sink, val) => 72 | write_u64_unchecked(reserve(sink, 8), val); 73 | 74 | export const write_f32: Serializer = (sink, val) => { 75 | sink = reserve(sink, 4); 76 | const { view, pos, littleEndian } = sink; 77 | view.setFloat32(pos, val, littleEndian); 78 | return (sink.pos += 4), sink; 79 | }; 80 | 81 | export const write_f64: Serializer = (sink, val) => { 82 | sink = reserve(sink, 8); 83 | const { view, pos, littleEndian } = sink; 84 | view.setFloat64(pos, val, littleEndian); 85 | return (sink.pos += 8), sink; 86 | }; 87 | 88 | export const write_i32: Serializer = (sink, val) => { 89 | sink = reserve(sink, 4); 90 | const { view, pos, littleEndian } = sink; 91 | view.setInt32(pos, val, littleEndian); 92 | return (sink.pos += 4), sink; 93 | }; 94 | 95 | const encoder = new TextEncoder(); 96 | 97 | const encodeStrInto: (str: string, resArr: Uint8Array) => number = 98 | 'encodeInto' in encoder 99 | ? (str, arr) => encoder.encodeInto(str, arr).written! 100 | : (str, arr) => { 101 | const bytes: Uint8Array = (encoder as any).encode(str); 102 | arr.set(bytes); 103 | return bytes.length; 104 | }; 105 | 106 | export const write_str: Serializer = (sink, val) => { 107 | // reserve 8 bytes for the u64 len 108 | sink = reserve(sink, val.length * 3 + 8); 109 | const bytesWritten = encodeStrInto( 110 | val, 111 | new Uint8Array(sink.view.buffer, sink.pos + 8) 112 | ); 113 | sink = write_u64_unchecked(sink, bytesWritten); 114 | sink.pos += bytesWritten; 115 | return sink; 116 | }; 117 | 118 | export const write_bool: Serializer = (sink, val) => 119 | write_u8(sink, val ? 1 : 0); 120 | 121 | export const seq_writer = (serEl: Serializer): Serializer => ( 122 | sink, 123 | seq: T[] 124 | ) => seq.reduce(serEl, write_u64(sink, seq.length)); 125 | 126 | export const opt_writer = ( 127 | serEl: Serializer 128 | ): Serializer => (sink: Sink, val: T | undefined) => 129 | val === undefined ? write_u8(sink, 0) : serEl(write_u8(sink, 1), val); 130 | 131 | export const nullable_writer = ( 132 | serEl: Serializer 133 | ): Serializer => (sink: Sink, val: T | null) => 134 | val === null ? write_u8(sink, 0) : serEl(write_u8(sink, 1), val); 135 | 136 | // -------- Deserialization ---------- 137 | 138 | export const read_u8: Deserializer = (sink) => { 139 | const { pos, view } = sink; 140 | return (sink.pos += 1), view.getUint8(pos); 141 | }; 142 | 143 | export const read_u16: Deserializer = (sink) => { 144 | const { pos, view, littleEndian } = sink; 145 | return (sink.pos += 2), view.getUint16(pos, littleEndian); 146 | }; 147 | 148 | export const read_u32: Deserializer = (sink) => { 149 | const { pos, view, littleEndian } = sink; 150 | return (sink.pos += 4), view.getUint32(pos, littleEndian); 151 | }; 152 | 153 | export const read_u64: Deserializer = (sink) => { 154 | const { view, pos, littleEndian } = sink; 155 | 156 | // we don't support numbers more than u32 (yet) 157 | return ( 158 | (sink.pos += 8), view.getUint32(littleEndian ? pos : pos + 4, littleEndian) 159 | ); 160 | }; 161 | 162 | export const read_f32: Deserializer = (sink) => { 163 | const { pos, view, littleEndian } = sink; 164 | return (sink.pos += 4), view.getFloat32(pos, littleEndian); 165 | }; 166 | 167 | export const read_f64: Deserializer = (sink) => { 168 | const { pos, view, littleEndian } = sink; 169 | return (sink.pos += 8), view.getFloat64(pos, littleEndian); 170 | }; 171 | 172 | export const read_i32: Deserializer = (sink) => { 173 | const { pos, view, littleEndian } = sink; 174 | return (sink.pos += 4), view.getInt32(pos, littleEndian); 175 | }; 176 | 177 | export const read_bool: Deserializer = (sink) => read_u8(sink) === 1; 178 | 179 | export const opt_reader = ( 180 | readEl: Deserializer 181 | ): Deserializer => (sink) => 182 | read_u8(sink) === 1 ? readEl(sink) : undefined; 183 | 184 | export const nullable_reader = ( 185 | readEl: Deserializer 186 | ): Deserializer => (sink) => 187 | read_u8(sink) === 1 ? readEl(sink) : null; 188 | 189 | export const seq_reader = (readEl: Deserializer): Deserializer => ( 190 | sink 191 | ) => { 192 | const count = read_u64(sink); 193 | 194 | // Note it doesn't make sense to set capacity here 195 | // because it will mess up shapes 196 | const res = new Array(); 197 | 198 | for (let i = 0; i < count; i++) { 199 | res.push(readEl(sink)); 200 | } 201 | 202 | return res; 203 | }; 204 | 205 | const decoder = new TextDecoder(); 206 | 207 | export const read_str: Deserializer = (sink) => { 208 | const len = read_u64(sink); 209 | const { 210 | pos, 211 | view: { buffer }, 212 | } = sink; 213 | const str = decoder.decode(new Uint8Array(buffer, pos, len)); 214 | return (sink.pos += len), str; 215 | }; 216 | -------------------------------------------------------------------------------- /packages/ts-binary/src/legacy.ts: -------------------------------------------------------------------------------- 1 | export type Sink = { 2 | pos: number; 3 | arr: Uint8Array; 4 | }; 5 | 6 | export const Sink = (arr: ArrayBuffer): Sink => ({ 7 | arr: new Uint8Array(arr), 8 | pos: 0 9 | }); 10 | 11 | export type Serializer = (sink: Sink, val: T) => Sink; 12 | export type Deserializer = (sink: Sink) => T; 13 | 14 | // note that all of them look at the same memory (same 4 bytes) 15 | const au32 = new Uint32Array(2); 16 | const au16 = new Uint16Array(au32.buffer); 17 | const au8 = new Uint8Array(au32.buffer); 18 | const af32 = new Float32Array(au32.buffer); 19 | const af64 = new Float64Array(au32.buffer); 20 | const ai32 = new Int32Array(au32.buffer); 21 | 22 | const reserve = (sink: Sink, numberOfBytes: number): Sink => { 23 | const { arr, pos } = sink; 24 | 25 | if (arr.length - pos > numberOfBytes) return sink; 26 | 27 | const newLen = Math.max(arr.length * 2, arr.length + numberOfBytes); 28 | const newArr = new Uint8Array(newLen); 29 | newArr.set(arr, 0); 30 | return { arr: newArr, pos }; 31 | }; 32 | 33 | // write a byte without any checks 34 | const wb = (sink: Sink, byte: number): Sink => { 35 | const { arr, pos } = sink; 36 | arr[pos] = byte; 37 | sink.pos += 1; 38 | return sink; 39 | }; 40 | 41 | export const write_u8: Serializer = (sink, val) => 42 | wb(reserve(sink, 1), val); 43 | 44 | export const write_u32: Serializer = (sink, val) => 45 | wb(wb(wb(wb(reserve(sink, 4), val), val >> 8), val >> 16), val >> 24); 46 | 47 | export const write_u16: Serializer = (sink, val) => 48 | wb(wb(reserve(sink, 2), val), val >> 8); 49 | 50 | export const write_u64: Serializer = (sink, val) => 51 | write_u32(write_u32(reserve(sink, 8), val), 0); 52 | 53 | export const write_f32: Serializer = (sink, val) => { 54 | af32[0] = val; // just translate the bytes from float to u32 55 | return write_u32(reserve(sink, 4), au32[0]); 56 | }; 57 | 58 | export const write_f64: Serializer = (sink, val) => { 59 | af64[0] = val; // just translate the bytes from float64 to u32 60 | return write_u32(write_u32(reserve(sink, 8), au32[0]), au32[1]); 61 | }; 62 | 63 | export const write_i32: Serializer = (sink, val) => { 64 | ai32[0] = val; // just translate the bytes from i32 to u32 65 | return write_u32(reserve(sink, 4), au32[0]); 66 | }; 67 | 68 | const encoder = new TextEncoder(); 69 | 70 | const encodeStrInto: (str: string, resArr: Uint8Array, pos: number) => number = 71 | 'encodeInto' in encoder 72 | ? (str, arr, pos) => 73 | encoder.encodeInto(str, new Uint8Array(arr.buffer, pos)).written! 74 | : (str, arr, pos) => { 75 | const bytes: Uint8Array = (encoder as any).encode(str); 76 | arr.set(bytes, pos); 77 | return bytes.length; 78 | }; 79 | 80 | export const write_str: Serializer = (sink, val) => { 81 | // reserve 8 bytes for the u64 len 82 | sink = reserve(sink, val.length * 3 + 8); 83 | const bytesWritten = encodeStrInto(val, sink.arr, sink.pos + 8); 84 | sink = write_u64(sink, bytesWritten); 85 | sink.pos += bytesWritten; 86 | return sink; 87 | }; 88 | 89 | export const write_bool: Serializer = (sink, val) => 90 | write_u8(sink, val ? 1 : 0); 91 | 92 | export const seq_writer = (serEl: Serializer): Serializer => ( 93 | sink, 94 | seq: T[] 95 | ) => seq.reduce(serEl, write_u64(sink, seq.length)); 96 | 97 | export const opt_writer = ( 98 | serEl: Serializer 99 | ): Serializer => (sink: Sink, val: T | undefined) => 100 | val === undefined ? write_u8(sink, 0) : serEl(write_u8(sink, 1), val); 101 | 102 | // -------- Deserialization ---------- 103 | 104 | export const read_u8: Deserializer = sink => sink.arr[sink.pos++]; 105 | 106 | // read 1 byte into a number + move pos in sink by 1 107 | const rb = read_u8; 108 | 109 | export const read_u32: Deserializer = sink => { 110 | au8[0] = rb(sink); 111 | au8[1] = rb(sink); 112 | au8[2] = rb(sink); 113 | au8[3] = rb(sink); 114 | return au32[0]; 115 | }; 116 | 117 | export const read_u16: Deserializer = sink => { 118 | au8[0] = rb(sink); 119 | au8[1] = rb(sink); 120 | return au16[0]; 121 | }; 122 | 123 | export const read_u64: Deserializer = sink => { 124 | // we don't support numbers more than u32 (yet) 125 | const val = read_u32(sink); 126 | sink.pos += 4; 127 | return val; 128 | }; 129 | 130 | export const read_f32: Deserializer = sink => { 131 | au8[0] = rb(sink); 132 | au8[1] = rb(sink); 133 | au8[2] = rb(sink); 134 | au8[3] = rb(sink); 135 | return af32[0]; 136 | }; 137 | 138 | export const read_f64: Deserializer = sink => { 139 | for (let i = 0; i < 8; i++) au8[i] = rb(sink); 140 | return af64[0]; 141 | }; 142 | 143 | export const read_i32: Deserializer = sink => { 144 | au8[0] = rb(sink); 145 | au8[1] = rb(sink); 146 | au8[2] = rb(sink); 147 | au8[3] = rb(sink); 148 | return ai32[0]; 149 | }; 150 | 151 | export const read_bool: Deserializer = sink => rb(sink) === 1; 152 | 153 | export const opt_reader = ( 154 | readEl: Deserializer 155 | ): Deserializer => sink => 156 | rb(sink) === 1 ? readEl(sink) : undefined; 157 | 158 | export const seq_reader = ( 159 | readEl: Deserializer 160 | ): Deserializer => sink => { 161 | const count = read_u64(sink); 162 | 163 | // Note it doesn't make sense to set capacity here 164 | // because it will mess up shapes 165 | const res = new Array(); 166 | 167 | for (let i = 0; i < count; i++) { 168 | res.push(readEl(sink)); 169 | } 170 | 171 | return res; 172 | }; 173 | 174 | const decoder = new TextDecoder(); 175 | 176 | export const read_str: Deserializer = sink => { 177 | const len = read_u64(sink); 178 | const str = decoder.decode(new Uint8Array(sink.arr.buffer, sink.pos, len)); 179 | sink.pos += len; 180 | return str; 181 | }; 182 | -------------------------------------------------------------------------------- /packages/ts-binary/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.build.json", 3 | 4 | "compilerOptions": { 5 | "outDir": "./dist", 6 | "rootDir": "./src", 7 | "composite": true 8 | }, 9 | 10 | "include": ["src/**/*"] 11 | } 12 | -------------------------------------------------------------------------------- /packages/ts-binary/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | 4 | "include": ["src/**/*"] 5 | } 6 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true 3 | } 4 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/README.MD: -------------------------------------------------------------------------------- 1 | # ts-rust-bridge-codegen 2 | 3 | Code generation library for efficient communication between rust and typescript. 4 | 5 | # WIP 6 | 7 | WARNING: The tool is far from being ready: not enough documentation + missing features. That said, you are welcome to take a look and give feedback. 8 | 9 | ## Install 10 | 11 | `npm install ts-rust-bridge-codegen --save-dev` 12 | 13 | If you want to use binary serialization/deserialization: 14 | 15 | `npm install ts-binary --save` 16 | 17 | ## Goal 18 | 19 | The goal of the this project is to provide a toolset to build efficient communication between rust and typescript. 20 | 21 | ## Example 22 | 23 | Define schema in typescript DSL (Domain specific language). Note that it is a small subset of `serde` types from rust ecosystem. 24 | 25 | ```ts 26 | import { schema2ts, schema2rust, Type } from 'ts-rust-bridge-codegen'; 27 | import * as fs from 'fs'; 28 | 29 | const { Enum, Struct, Str, F32 } = Type; 30 | 31 | const Size = Enum('S', 'M', 'L'); 32 | const Shirt = Struct({ size: Size, color: Str, price: F32 }); 33 | const schema = { Size, Shirt }; 34 | 35 | const tsCode = schema2ts(schema).join('\n\n'); 36 | 37 | const rustCode = ` 38 | use serde::{Deserialize, Serialize}; 39 | 40 | ${schema2rust(schema).join('\n')} 41 | `; 42 | 43 | // save to disc 44 | fs.writeFileSync('schema.ts', tsCode); 45 | fs.writeFileSync('schema.rs', rustCode); 46 | ``` 47 | 48 | And here is the result: 49 | 50 | rust 51 | 52 | ```rust 53 | // schema.rs 54 | use serde::{Deserialize, Serialize}; 55 | #[derive(Deserialize, Serialize, Debug, Clone)] 56 | pub struct Shirt { 57 | pub size: Size, 58 | pub color: String, 59 | pub price: f32, 60 | } 61 | 62 | 63 | #[derive(Deserialize, Serialize, Debug, Clone)] 64 | pub enum Size { 65 | S, 66 | M, 67 | L, 68 | } 69 | ``` 70 | 71 | typescript 72 | 73 | ```ts 74 | // schema.ts after prettier 75 | export interface Shirt { 76 | size: Size; 77 | color: string; 78 | price: number; 79 | } 80 | 81 | export enum Size { 82 | S = 'S', 83 | M = 'M', 84 | L = 'L' 85 | } 86 | ``` 87 | 88 | Now you can serialize them as JSON or as binary. 89 | 90 | ## Bincode 91 | 92 | Now you can serialize your data structures to binary format called `bincode`! You can find more detail about the format here: https://github.com/servo/bincode. 93 | 94 | In short: it is a very efficient way to represent data structures that native to rust in a binary form. 95 | 96 | ### Why? 97 | 98 | There are three potential usecases: 99 | 100 | 1. Communicate between typescript (js) and rust code. In case of our initial motivation project (https://github.com/cztomsik/stain) it is node -> rust native via `ffi` module. 101 | 2. Communicate between WebAssembly module written in rust and typecript. 102 | 3. Communicate between WebWorker(ts/js) and main thread(ts/js). That's right, you can just use typescript serializers/deserializers without rust code :) 103 | 104 | Any combination of the above: WASM module running in a WebWorker that talks to a rust backend? ^\_^ 105 | 106 | ### How to generate code serializers/deserializers for typescript 107 | 108 | Note: bincode serialization relies on read/write api to a ArrayBuffer provided by `ts-binary` package (it is located in the neighbor folder in this repo). 109 | 110 | ```ts 111 | import { 112 | schema2ts, 113 | schema2rust, 114 | schema2serde, 115 | Type 116 | } from 'ts-rust-bridge-codegen'; 117 | import * as fs from 'fs'; 118 | 119 | const { Enum, Struct, Str, F32 } = Type; 120 | 121 | const Size = Enum('S', 'M', 'L'); 122 | const Shirt = Struct({ size: Size, color: Str, price: F32 }); 123 | const schema = { Size, Shirt }; 124 | 125 | const tsCode = schema2ts(schema).join('\n\n'); 126 | 127 | const rustCode = ` 128 | use serde::{Deserialize, Serialize}; 129 | 130 | ${schema2rust(schema).join('\n')} 131 | `; 132 | 133 | const tsSerDeCode = ` 134 | ${schema2serde({ 135 | schema: schema, 136 | typesDeclarationFile: `./schema` 137 | }).join('\n\n')} 138 | `; 139 | 140 | // save to disc 141 | fs.writeFileSync('schema.ts', tsCode); 142 | fs.writeFileSync('schema.rs', rustCode); 143 | fs.writeFileSync('schema_serde.ts', tsSerDeCode); 144 | ``` 145 | 146 | In addition to type definitions files it will generate human readable serializers + deserializers. 147 | 148 | ```ts 149 | // schema_serde.ts after prettier 150 | import { Size, Shirt } from './schema'; 151 | 152 | import { 153 | write_u32, 154 | write_str, 155 | write_f32, 156 | Sink, 157 | read_u32, 158 | read_str, 159 | read_f32 160 | } from 'ts-binary'; 161 | 162 | // Serializers 163 | 164 | const SizeMap: { [key: string]: number } = { S: 0, M: 1, L: 2 }; 165 | 166 | export const writeSize = (sink: Sink, val: Size): Sink => 167 | write_u32(sink, SizeMap[val]); 168 | 169 | export const writeShirt = (sink: Sink, { size, color, price }: Shirt): Sink => 170 | write_f32(write_str(writeSize(sink, size), color), price); 171 | 172 | // Deserializers 173 | 174 | const SizeReverseMap: Size[] = [Size.S, Size.M, Size.L]; 175 | 176 | export const readSize = (sink: Sink): Size => SizeReverseMap[read_u32(sink)]; 177 | 178 | export const readShirt = (sink: Sink): Shirt => { 179 | const size = readSize(sink); 180 | const color = read_str(sink); 181 | const price = read_f32(sink); 182 | return { size, color, price }; 183 | }; 184 | ``` 185 | 186 | ### How to use it 187 | 188 | ```ts 189 | // usage.ts 190 | import { writeShirt, readShirt } from './schema_serde'; 191 | import { Shirt, Size } from './schema'; 192 | import { Sink } from 'ts-binary'; 193 | 194 | const shirt: Shirt = { color: 'red', price: 10, size: Size.L }; 195 | 196 | let sink = writeShirt(Sink(new ArrayBuffer(100)), shirt); 197 | 198 | console.log('bytes:', new Uint8Array(sink.view.buffer, 0, sink.pos)); 199 | // bytes: Uint8Array 200 | // [ 201 | // 2, 0, 0, 0, <-- Size.L 202 | // 3, 0, 0, 0, 0, 0, 0, 0, <- number of bytes for 'red' in utf-8 203 | // 114, 101, 100, <-- 'r', 'e', 'd' 204 | // 0, 0, 32, 65 <-- 10 as float 32 byte representation 205 | // ] 206 | 207 | // reset pos to read value back 208 | sink.pos = 0; 209 | const restoredShirt = readShirt(sink); 210 | console.log('restored:', restoredShirt); 211 | // restored: { size: 'L', color: 'red', price: 10 } 212 | ``` 213 | 214 | Look at [examples](https://github.com/twop/ts-rust-bridge/tree/master/packages/ts-rust-bridge-codegen/examples) dir for more information how to use the library. 215 | 216 | ## API 217 | 218 | TODO. 219 | 220 | ## Simple benchmarks 221 | 222 | I just copypasted generated code from examples and tried to construct a simple benchmark. 223 | 224 | Code 225 | https://stackblitz.com/edit/ts-binaray-benchmark?file=index.ts 226 | 227 | Version to try 228 | https://ts-binaray-benchmark.stackblitz.io 229 | 230 | On complex data structure: 231 | 232 | | Method | Serialization | Deserialization | 233 | | --------- | :-----------: | --------------: | 234 | | ts-binary | 74 ms | 91 ms | 235 | | JSON | 641 ms | 405 ms | 236 | 237 | Simple data structure: 238 | 239 | | Method | Serialization | Deserialization | 240 | | --------- | :-----------: | --------------: | 241 | | ts-binary | 2 ms | 1 ms | 242 | | JSON | 6 ms | 5 ms | 243 | 244 | That was measured on latest Safari version. 245 | 246 | Note you can run the benchmark yourself cloning the repo + running npm scripts 247 | 248 | ## License 249 | 250 | MIT 251 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/__tests__/gen_testSchema.ts: -------------------------------------------------------------------------------- 1 | import { schema2ts, schema2serde, schema2rust } from '../src/index'; 2 | 3 | import { format } from 'prettier'; 4 | import * as fs from 'fs'; 5 | 6 | import { Type } from '../src/schema'; 7 | 8 | const { 9 | Alias, 10 | Enum, 11 | Tuple, 12 | Struct, 13 | Union, 14 | Newtype, 15 | Option, 16 | Nullable, 17 | Vec, 18 | Str, 19 | U32, 20 | Bool, 21 | U8, 22 | F32 23 | } = Type; 24 | 25 | const Size = Enum('S', 'M', 'L'); 26 | const Shirt = Struct({ size: Size, color: Str, price: F32 }); 27 | 28 | const MyTuple = Tuple(Nullable(Bool), Vec(Str)); 29 | 30 | const JustAStruct = Struct({ 31 | u8: U8, 32 | myTuple: MyTuple 33 | }); 34 | 35 | const SimpleUnion = Union({ 36 | Unit: null, 37 | Float32: F32, 38 | BoolAndU32: [Option(Bool), U32], 39 | StructVariant: { id: Str, tuple: MyTuple } 40 | }); 41 | 42 | const MyEnum = Enum('One', 'Two', 'Three'); 43 | const NewTypeU32 = Newtype(U32); 44 | const AliasToStr = Alias(Str); 45 | 46 | const typesToCodegen = { 47 | MyEnum, 48 | NewTypeU32, 49 | AliasToStr, 50 | SimpleUnion, 51 | JustAStruct, 52 | MyTuple 53 | }; 54 | 55 | const tsFile = __dirname + '/generated/types.g.ts'; 56 | const tsSerFile = __dirname + '/generated/types.serde.g.ts'; 57 | 58 | const tsSerDeContent = ` 59 | ${schema2serde({ 60 | schema: typesToCodegen, 61 | typesDeclarationFile: `./types.g`, 62 | pathToBincodeLib: `../../../ts-binary/src/index` 63 | }).join('\n\n')} 64 | `; 65 | 66 | const prettierOptions = JSON.parse( 67 | fs.readFileSync(__dirname + '/../.prettierrc').toString() 68 | ); 69 | 70 | const pretty = (content: string) => 71 | format(content, { 72 | ...prettierOptions, 73 | parser: 'typescript' 74 | }); 75 | // const pretty = (content: string) => content; 76 | 77 | fs.writeFileSync(tsFile, pretty(schema2ts(typesToCodegen).join('\n\n'))); 78 | fs.writeFileSync('types.ts', pretty(schema2ts({ Shirt, Size }).join('\n'))); 79 | fs.writeFileSync(tsSerFile, pretty(tsSerDeContent)); 80 | fs.writeFileSync('types.rs', schema2rust({ Shirt, Size }).join('\n')); 81 | 82 | console.log('\n\n', JSON.stringify(typesToCodegen)); 83 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/__tests__/generated/types.g.ts: -------------------------------------------------------------------------------- 1 | export enum MyEnum { 2 | One = 'One', 3 | Two = 'Two', 4 | Three = 'Three' 5 | } 6 | 7 | export type NewTypeU32 = number & { type: 'NewTypeU32' }; 8 | 9 | export const NewTypeU32 = (val: number): number & { type: 'NewTypeU32' } => 10 | val as any; 11 | 12 | export type AliasToStr = string; 13 | 14 | export type SimpleUnion = 15 | | { tag: 'Unit' } 16 | | { tag: 'Float32'; value: number } 17 | | { tag: 'BoolAndU32'; value: SimpleUnion_BoolAndU32 } 18 | | { tag: 'StructVariant'; value: SimpleUnion_StructVariant }; 19 | 20 | export interface SimpleUnion_BoolAndU32 { 21 | 0: (boolean) | undefined; 22 | 1: number; 23 | length: 2; 24 | } 25 | 26 | export interface SimpleUnion_StructVariant { 27 | id: string; 28 | tuple: MyTuple; 29 | } 30 | 31 | export module SimpleUnion { 32 | export const Unit: SimpleUnion = { tag: 'Unit' }; 33 | 34 | export const Float32 = (value: number): SimpleUnion => ({ 35 | tag: 'Float32', 36 | value 37 | }); 38 | 39 | export const BoolAndU32 = ( 40 | p0: (boolean) | undefined, 41 | p1: number 42 | ): SimpleUnion => ({ tag: 'BoolAndU32', value: [p0, p1] }); 43 | 44 | export const StructVariant = ( 45 | value: SimpleUnion_StructVariant 46 | ): SimpleUnion => ({ tag: 'StructVariant', value }); 47 | } 48 | 49 | export interface JustAStruct { 50 | u8: number; 51 | myTuple: MyTuple; 52 | } 53 | 54 | export interface MyTuple { 55 | 0: (boolean) | null; 56 | 1: Array; 57 | length: 2; 58 | } 59 | 60 | export const MyTuple = (p0: (boolean) | null, p1: Array): MyTuple => [ 61 | p0, 62 | p1 63 | ]; 64 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/__tests__/generated/types.serde.g.ts: -------------------------------------------------------------------------------- 1 | import { 2 | MyEnum, 3 | NewTypeU32, 4 | AliasToStr, 5 | SimpleUnion, 6 | SimpleUnion_BoolAndU32, 7 | MyTuple, 8 | SimpleUnion_StructVariant, 9 | JustAStruct 10 | } from './types.g'; 11 | 12 | import { 13 | write_u32, 14 | write_str, 15 | write_f32, 16 | opt_writer, 17 | write_bool, 18 | write_u8, 19 | nullable_writer, 20 | seq_writer, 21 | Sink, 22 | Serializer, 23 | read_u32, 24 | read_str, 25 | read_f32, 26 | opt_reader, 27 | read_bool, 28 | read_u8, 29 | nullable_reader, 30 | seq_reader, 31 | Deserializer 32 | } from '../../../ts-binary/src/index'; 33 | 34 | // Serializers 35 | 36 | const writeOptBool: Serializer<(boolean) | undefined> = opt_writer(write_bool); 37 | 38 | const writeNullableBool: Serializer<(boolean) | null> = nullable_writer( 39 | write_bool 40 | ); 41 | 42 | const writeVecStr: Serializer> = seq_writer(write_str); 43 | 44 | const MyEnumMap: { [key: string]: number } = { One: 0, Two: 1, Three: 2 }; 45 | 46 | export const writeMyEnum = (sink: Sink, val: MyEnum): Sink => 47 | write_u32(sink, MyEnumMap[val]); 48 | 49 | export const writeNewTypeU32: Serializer = write_u32; 50 | 51 | export const writeAliasToStr: Serializer = write_str; 52 | 53 | export const writeMyTuple = (sink: Sink, val: MyTuple): Sink => 54 | writeVecStr(writeNullableBool(sink, val[0]), val[1]); 55 | 56 | const writeSimpleUnion_BoolAndU32 = ( 57 | sink: Sink, 58 | val: SimpleUnion_BoolAndU32 59 | ): Sink => write_u32(writeOptBool(sink, val[0]), val[1]); 60 | 61 | const writeSimpleUnion_StructVariant = ( 62 | sink: Sink, 63 | { id, tuple }: SimpleUnion_StructVariant 64 | ): Sink => writeMyTuple(write_str(sink, id), tuple); 65 | 66 | export const writeSimpleUnion = (sink: Sink, val: SimpleUnion): Sink => { 67 | switch (val.tag) { 68 | case 'Unit': 69 | return write_u32(sink, 0); 70 | case 'Float32': 71 | return write_f32(write_u32(sink, 1), val.value); 72 | case 'BoolAndU32': 73 | return writeSimpleUnion_BoolAndU32(write_u32(sink, 2), val.value); 74 | case 'StructVariant': 75 | return writeSimpleUnion_StructVariant(write_u32(sink, 3), val.value); 76 | } 77 | }; 78 | 79 | export const writeJustAStruct = ( 80 | sink: Sink, 81 | { u8, myTuple }: JustAStruct 82 | ): Sink => writeMyTuple(write_u8(sink, u8), myTuple); 83 | 84 | // Deserializers 85 | 86 | const readOptBool: Deserializer<(boolean) | undefined> = opt_reader(read_bool); 87 | 88 | const readNullableBool: Deserializer<(boolean) | null> = nullable_reader( 89 | read_bool 90 | ); 91 | 92 | const readVecStr: Deserializer> = seq_reader(read_str); 93 | 94 | const MyEnumReverseMap: MyEnum[] = [MyEnum.One, MyEnum.Two, MyEnum.Three]; 95 | 96 | export const readMyEnum = (sink: Sink): MyEnum => 97 | MyEnumReverseMap[read_u32(sink)]; 98 | 99 | export const readNewTypeU32 = (sink: Sink): NewTypeU32 => 100 | NewTypeU32(read_u32(sink)); 101 | 102 | export const readAliasToStr: Deserializer = read_str; 103 | 104 | export const readMyTuple = (sink: Sink): MyTuple => 105 | MyTuple(readNullableBool(sink), readVecStr(sink)); 106 | 107 | export const readSimpleUnion = (sink: Sink): SimpleUnion => { 108 | switch (read_u32(sink)) { 109 | case 0: 110 | return SimpleUnion.Unit; 111 | case 1: 112 | return SimpleUnion.Float32(read_f32(sink)); 113 | case 2: 114 | return SimpleUnion.BoolAndU32(readOptBool(sink), read_u32(sink)); 115 | case 3: 116 | return SimpleUnion.StructVariant(readSimpleUnion_StructVariant(sink)); 117 | } 118 | throw new Error('bad variant index for SimpleUnion'); 119 | }; 120 | 121 | const readSimpleUnion_StructVariant = ( 122 | sink: Sink 123 | ): SimpleUnion_StructVariant => { 124 | const id = read_str(sink); 125 | const tuple = readMyTuple(sink); 126 | return { id, tuple }; 127 | }; 128 | 129 | export const readJustAStruct = (sink: Sink): JustAStruct => { 130 | const u8 = read_u8(sink); 131 | const myTuple = readMyTuple(sink); 132 | return { u8, myTuple }; 133 | }; 134 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/__tests__/serializeAndRestore.test.ts: -------------------------------------------------------------------------------- 1 | import { Sink, Deserializer, Serializer } from '../../ts-binary/src/index'; 2 | 3 | import * as t from './generated/types.g'; 4 | import * as sd from './generated/types.serde.g'; 5 | 6 | const serde = (val: T, ser: Serializer, deser: Deserializer): T => { 7 | let sink = Sink(new ArrayBuffer(1)); 8 | 9 | sink = ser(sink, val); 10 | sink.pos = 0; 11 | 12 | return deser(sink); 13 | }; 14 | 15 | test('it reads and writes Enum', () => { 16 | const val = t.MyEnum.Three; 17 | expect(serde(val, sd.writeMyEnum, sd.readMyEnum)).toBe(val); 18 | }); 19 | 20 | test('it reads and writes Tuple', () => { 21 | const val = t.MyTuple(false, ['a', 'b', 'ccs']); 22 | expect(serde(val, sd.writeMyTuple, sd.readMyTuple)).toEqual(val); 23 | const val2 = t.MyTuple(null, ['a']); 24 | expect(serde(val2, sd.writeMyTuple, sd.readMyTuple)).toEqual(val2); 25 | }); 26 | 27 | test('it reads and writes NewType', () => { 28 | const val = t.NewTypeU32(3); 29 | expect(serde(val, sd.writeNewTypeU32, sd.readNewTypeU32)).toBe(val); 30 | }); 31 | 32 | test('it reads and writes Alias', () => { 33 | const val: t.AliasToStr = 'str'; 34 | expect(serde(val, sd.writeAliasToStr, sd.readAliasToStr)).toBe(val); 35 | }); 36 | 37 | test('it reads and writes Structs', () => { 38 | const val: t.JustAStruct = { u8: 5, myTuple: t.MyTuple(true, ['aha!']) }; 39 | expect(serde(val, sd.writeJustAStruct, sd.readJustAStruct)).toEqual(val); 40 | }); 41 | 42 | test('it reads and writes Union variants', () => { 43 | const { BoolAndU32, Float32, Unit, StructVariant } = t.SimpleUnion; 44 | 45 | const f32Arr = new Float32Array(1); 46 | f32Arr[0] = 4.1; // this is needed because of issues like 4.1 -> 4.099999999998 47 | const values: t.SimpleUnion[] = [ 48 | Unit, 49 | Float32(f32Arr[0]), 50 | BoolAndU32(false, 445), 51 | StructVariant({ id: 'str', tuple: t.MyTuple(false, ['bla']) }) 52 | ]; 53 | 54 | values.forEach(val => 55 | expect(serde(val, sd.writeSimpleUnion, sd.readSimpleUnion)).toEqual(val) 56 | ); 57 | }); 58 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/examples/benchmark.ts: -------------------------------------------------------------------------------- 1 | import { Message, Container, Figure, Color, Vec3 } from './generated/simple.g'; 2 | import { Serializer, Sink, Deserializer } from '../../ts-binary/src/index'; 3 | import { 4 | readMessage, 5 | readContainer, 6 | writeMessage, 7 | writeContainer 8 | } from './generated/simple_serde.g'; 9 | import * as fs from 'fs'; 10 | 11 | const L = console.log; 12 | const measure = (name: string, func: () => void) => { 13 | //log(`\n${' '.repeat(4)}${name}`); 14 | 15 | // let fastest = 100500; 16 | 17 | const numberOfRuns = 4; 18 | const takeTop = 1; 19 | 20 | let runs: number[] = []; 21 | for (let i = 0; i < numberOfRuns; i++) { 22 | const hrstart = process.hrtime(); 23 | func(); 24 | const hrend = process.hrtime(hrstart); 25 | 26 | const current = hrend[1] / 1000000; 27 | 28 | runs.push(current); 29 | 30 | // fastest = Math.min(fastest, current); 31 | } 32 | 33 | const result = runs 34 | .sort((a, b) => a - b) 35 | .slice(numberOfRuns - takeTop, numberOfRuns) 36 | .reduce((s, v) => s + v, 0); 37 | 38 | L(`${name}: ${result.toFixed(2)} ms`); 39 | }; 40 | 41 | const COUNT = 10000; 42 | 43 | function randomStr(length: number): string { 44 | var text = ''; 45 | var possible = 'ываыафяДЛОАВЫЛОАВУКЦДЛСВЫФзвфыджл0123456789'; 46 | 47 | for (var i = 0; i < length; i++) 48 | text += possible.charAt(Math.floor(Math.random() * possible.length)); 49 | 50 | return text; 51 | } 52 | 53 | const ctors: (() => Message)[] = [ 54 | () => Message.Unit, 55 | () => Message.One(Math.random() * 10000), 56 | () => 57 | Message.Two( 58 | Math.random() > 0.5 ? undefined : Math.random() > 0.5, 59 | Math.floor(Math.random() * 1000) 60 | ), 61 | () => 62 | Message.Two( 63 | Math.random() > 0.5 ? undefined : Math.random() > 0.5, 64 | Math.floor(Math.random() * 1000) 65 | ), 66 | () => 67 | Message.VStruct({ 68 | id: randomStr(Math.random() * 20), 69 | data: randomStr(Math.random() * 20) 70 | }) 71 | ]; 72 | 73 | const genArray = (f: () => T): T[] => 74 | Array.from({ length: Math.floor(Math.random() * 30) }, f); 75 | 76 | const randu8 = () => Math.floor(Math.random() * 256); 77 | const randf32 = () => Math.random() * 1000; 78 | 79 | const genColor = (): Color => Color(randu8(), randu8(), randu8()); 80 | const genVec3 = (): Vec3 => Vec3(randf32(), randf32(), randf32()); 81 | 82 | const genFigure = (): Figure => ({ 83 | dots: genArray(genVec3), 84 | colors: genArray(genColor) 85 | }); 86 | 87 | const genContainer = (): Container => { 88 | const seed = Math.random(); 89 | 90 | return seed < 0.33 91 | ? Container.Units 92 | : seed < 0.66 93 | ? Container.JustNumber(randu8()) 94 | : Container.Figures(genArray(genFigure)); 95 | }; 96 | 97 | type Data = { 98 | len: number; 99 | containers: Container[]; 100 | messages: Message[]; 101 | }; 102 | 103 | const genData = (): Data => ({ 104 | containers: Array.from({ length: COUNT }, genContainer), 105 | len: COUNT, 106 | messages: Array.from( 107 | { length: COUNT }, 108 | // () => ctors[4]() 109 | () => ctors[Math.floor(Math.random() * 4)]() 110 | ) 111 | }); 112 | let data: Data | undefined = undefined; 113 | 114 | try { 115 | data = JSON.parse(fs.readFileSync('bench_data.json').toString()); 116 | if (data!.len !== COUNT) { 117 | data = undefined; 118 | } 119 | } catch {} 120 | 121 | if (!data) { 122 | data = genData(); 123 | L('regenerated data'); 124 | fs.writeFileSync('bench_data.json', JSON.stringify(data)); 125 | } else { 126 | L('data read from cache'); 127 | } 128 | 129 | const { messages, containers } = data; 130 | // fs.writeFileSync(); 131 | 132 | // let sink: Sink = { 133 | // arr: new Uint8Array(1000), 134 | // pos: 0 135 | // }; 136 | let sink = Sink(new ArrayBuffer(2000)); 137 | 138 | const writeAThingToNothing = (thing: T, ser: Serializer): void => { 139 | sink.pos = 0; 140 | sink = ser(sink, thing); 141 | }; 142 | 143 | const writeAThingToSlice = (thing: T, ser: Serializer): ArrayBuffer => { 144 | const s = ser(Sink(new ArrayBuffer(1000)), thing); 145 | // const slice = new Uint8Array(s.arr.buffer).slice(0, s.pos).buffer; 146 | const slice = new Uint8Array(s.view.buffer).slice(0, s.pos).buffer; 147 | 148 | // if (s.arr.buffer === slice) { 149 | if (s.view.buffer === slice) { 150 | throw new Error('Aha!'); 151 | } 152 | 153 | return slice; 154 | }; 155 | 156 | runbench('containers', containers, writeContainer, readContainer); 157 | runbench('messages', messages, writeMessage, readMessage); 158 | 159 | containers; 160 | readContainer; 161 | writeContainer; 162 | 163 | function runbench( 164 | benchName: string, 165 | data: T[], 166 | serialize: Serializer, 167 | deserialize: Deserializer 168 | ) { 169 | L(' '); 170 | L(benchName.toUpperCase()); 171 | L('----Serialization----'); 172 | measure('just bincode', () => { 173 | data.forEach(d => writeAThingToNothing(d, serialize)); 174 | }); 175 | 176 | measure('json', () => { 177 | data.forEach(d => JSON.stringify(d)); 178 | }); 179 | L('----Deserialization----'); 180 | 181 | const sinks = data.map(d => Sink(writeAThingToSlice(d, serialize))); 182 | 183 | sinks.forEach(s => { 184 | if (s.pos !== 0) { 185 | throw 'a'; 186 | } 187 | }); 188 | const strings = data.map(d => JSON.stringify(d)); 189 | 190 | const res = [...data]; // just a copy 191 | 192 | measure('D: bincode', () => { 193 | sinks.forEach((s, i) => { 194 | // if (s.pos !== 0) { 195 | // throw 'b'; 196 | // } 197 | // if (i === 0) L('!!', i, s.pos, s.view.buffer.byteLength); 198 | s.pos = 0; 199 | res[i] = deserialize(s); 200 | }); 201 | }); 202 | 203 | // res.forEach((d, i) => { 204 | // if (JSON.stringify(d) !== strings[i]) { 205 | // console.error('mismatch', { 206 | // expected: strings[i], 207 | // actual: JSON.stringify(d) 208 | // }); 209 | // } 210 | // }); 211 | 212 | measure('D: json', () => { 213 | strings.forEach((s, i) => (res[i] = JSON.parse(s))); 214 | }); 215 | } 216 | // }, 1000 * 20); 217 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/examples/for_docs/gen_docs_example.ts: -------------------------------------------------------------------------------- 1 | import { schema2ts, schema2rust, schema2serde, Type } from '../../src/index'; 2 | import * as fs from 'fs'; 3 | 4 | const { Enum, Struct, Str, F32 } = Type; 5 | 6 | const Size = Enum('S', 'M', 'L'); 7 | const Shirt = Struct({ size: Size, color: Str, price: F32 }); 8 | const schema = { Size, Shirt }; 9 | 10 | const tsCode = schema2ts(schema).join('\n\n'); 11 | 12 | const rustCode = ` 13 | use serde::{Deserialize, Serialize}; 14 | 15 | ${schema2rust(schema).join('\n')} 16 | `; 17 | 18 | const tsSerDeCode = ` 19 | ${schema2serde({ 20 | schema: schema, 21 | typesDeclarationFile: `./schema` 22 | }).join('\n\n')} 23 | `; 24 | 25 | // save to disc 26 | fs.writeFileSync('schema.ts', tsCode); 27 | fs.writeFileSync('schema.rs', rustCode); 28 | fs.writeFileSync('schema_serde.ts', tsSerDeCode); 29 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/examples/for_docs/schema.rs: -------------------------------------------------------------------------------- 1 | 2 | use serde::{Deserialize, Serialize}; 3 | 4 | 5 | #[derive(Deserialize, Serialize, Debug, Clone)] 6 | pub enum Size { 7 | S, 8 | M, 9 | L, 10 | } 11 | 12 | 13 | #[derive(Deserialize, Serialize, Debug, Clone)] 14 | pub struct Shirt { 15 | pub size: Size, 16 | pub color: String, 17 | pub price: f32, 18 | } 19 | 20 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/examples/for_docs/schema.ts: -------------------------------------------------------------------------------- 1 | export enum Size { 2 | S = 'S', 3 | M = 'M', 4 | L = 'L' 5 | } 6 | 7 | export interface Shirt { 8 | size: Size; 9 | color: string; 10 | price: number; 11 | } 12 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/examples/for_docs/schema_serde.ts: -------------------------------------------------------------------------------- 1 | import { Size, Shirt } from './schema'; 2 | 3 | import { 4 | write_u32, 5 | write_str, 6 | write_f32, 7 | Sink, 8 | read_u32, 9 | read_str, 10 | read_f32 11 | } from '../../../ts-binary/src/index'; 12 | 13 | // Serializers 14 | 15 | const SizeMap: { [key: string]: number } = { S: 0, M: 1, L: 2 }; 16 | 17 | export const writeSize = (sink: Sink, val: Size): Sink => 18 | write_u32(sink, SizeMap[val]); 19 | 20 | export const writeShirt = (sink: Sink, { size, color, price }: Shirt): Sink => 21 | write_f32(write_str(writeSize(sink, size), color), price); 22 | 23 | // Deserializers 24 | 25 | const SizeReverseMap: Size[] = [Size.S, Size.M, Size.L]; 26 | 27 | export const readSize = (sink: Sink): Size => SizeReverseMap[read_u32(sink)]; 28 | 29 | export const readShirt = (sink: Sink): Shirt => { 30 | const size = readSize(sink); 31 | const color = read_str(sink); 32 | const price = read_f32(sink); 33 | return { size, color, price }; 34 | }; 35 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/examples/for_docs/usage.ts: -------------------------------------------------------------------------------- 1 | import { writeShirt, readShirt } from './schema_serde'; 2 | import { Shirt, Size } from './schema'; 3 | import { Sink } from '../../../ts-binary/src/index'; 4 | 5 | const shirt: Shirt = { color: 'red', price: 10, size: Size.L }; 6 | 7 | let sink = writeShirt(Sink(new ArrayBuffer(100)), shirt); 8 | 9 | console.log('bytes:', new Uint8Array(sink.view.buffer, 0, sink.pos)); 10 | // bytes: Uint8Array 11 | // [ 12 | // 2, 0, 0, 0, <- Size.L 13 | // 3, 0, 0, 0, 0, 0, 0, 0, <- number of bytes for 'red' in utf-8 14 | // 114, 101, 100, <-- 'r', 'e', 'd' 15 | // 0, 0, 32, 65 <-- 10 as float 32 byte representation 16 | // ] 17 | 18 | // reset pos to read value back 19 | sink.pos = 0; 20 | const restoredShirt = readShirt(sink); 21 | console.log('restored:', restoredShirt); 22 | // restored: { size: 'L', color: 'red', price: 10 } 23 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/examples/gen_simple.schema.ts: -------------------------------------------------------------------------------- 1 | import { schema2rust, schema2ts, schema2serde } from '../src/index'; 2 | import { exampleSchema } from './simple.schema'; 3 | import { format } from 'prettier'; 4 | import * as fs from 'fs'; 5 | 6 | const tsFile = __dirname + '/generated/simple.g.ts'; 7 | const tsSerDeFile = __dirname + '/generated/simple_serde.g.ts'; 8 | const testRustFile = __dirname + '/generated/simple.g.rs'; 9 | 10 | const rustContent = ` 11 | use serde::Deserialize; 12 | 13 | ${schema2rust(exampleSchema, { 14 | MyEnum: { derive: ['Clone', 'Copy'] } 15 | }).join('\n')} 16 | `; 17 | 18 | const tsSerDeContent = ` 19 | ${schema2serde({ 20 | schema: exampleSchema, 21 | typesDeclarationFile: `./simple.g`, 22 | pathToBincodeLib: `../../../ts-binary/src/index` 23 | }).join('\n\n')} 24 | `; 25 | 26 | const prettierOptions = JSON.parse( 27 | fs.readFileSync(__dirname + '/../.prettierrc').toString() 28 | ); 29 | 30 | const pretty = (content: string) => 31 | format(content, { 32 | ...prettierOptions, 33 | parser: 'typescript' 34 | }); 35 | // const pretty = (content: string) => content; 36 | 37 | fs.writeFileSync(testRustFile, rustContent); 38 | fs.writeFileSync(tsFile, pretty(schema2ts(exampleSchema).join('\n\n'))); 39 | fs.writeFileSync(tsSerDeFile, pretty(tsSerDeContent)); 40 | 41 | console.log('\n\n', JSON.stringify(exampleSchema)); 42 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/examples/generated/simple.g.rs: -------------------------------------------------------------------------------- 1 | 2 | use serde::Deserialize; 3 | 4 | 5 | #[derive(Deserialize, Serialize)] 6 | pub enum Message { 7 | Unit, 8 | One(f32), 9 | Two(Option, u32), 10 | VStruct { id: String, data: String }, 11 | } 12 | 13 | 14 | #[derive(Deserialize, Serialize)] 15 | pub struct NType(pub u32); 16 | 17 | 18 | #[derive(Deserialize, Serialize)] 19 | pub enum Container { 20 | Units, 21 | JustNumber(u32), 22 | Figures(Vec
), 23 | } 24 | 25 | 26 | #[derive(Deserialize, Serialize)] 27 | pub struct Color(pub u8, pub u8, pub u8); 28 | 29 | 30 | #[derive(Deserialize, Serialize)] 31 | pub struct Figure { 32 | pub dots: Vec, 33 | pub colors: Vec, 34 | } 35 | 36 | 37 | #[derive(Deserialize, Serialize)] 38 | pub struct Vec3(pub f32, pub f32, pub f32); 39 | 40 | 41 | pub type NewtypeAlias = NType; 42 | 43 | 44 | #[derive(Deserialize, Serialize)] 45 | pub struct NormalStruct { 46 | pub a: u8, 47 | pub tuple: MyTuple, 48 | } 49 | 50 | 51 | #[derive(Deserialize, Serialize, Clone, Copy)] 52 | pub enum MyEnum { 53 | ONE, 54 | TWO, 55 | THREE, 56 | } 57 | 58 | 59 | #[derive(Deserialize, Serialize)] 60 | pub struct MyTuple(pub Option, pub Vec); 61 | 62 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/examples/generated/simple.g.ts: -------------------------------------------------------------------------------- 1 | export type Message = 2 | | { tag: 'Unit' } 3 | | { tag: 'One'; value: number } 4 | | { tag: 'Two'; value: Message_Two } 5 | | { tag: 'VStruct'; value: Message_VStruct }; 6 | 7 | export interface Message_Two { 8 | 0: (boolean) | undefined; 9 | 1: number; 10 | length: 2; 11 | } 12 | 13 | export interface Message_VStruct { 14 | id: string; 15 | data: string; 16 | } 17 | 18 | export module Message { 19 | export const Unit: Message = { tag: 'Unit' }; 20 | 21 | export const One = (value: number): Message => ({ tag: 'One', value }); 22 | 23 | export const Two = (p0: (boolean) | undefined, p1: number): Message => ({ 24 | tag: 'Two', 25 | value: [p0, p1] 26 | }); 27 | 28 | export const VStruct = (value: Message_VStruct): Message => ({ 29 | tag: 'VStruct', 30 | value 31 | }); 32 | } 33 | 34 | export type NType = number & { type: 'NType' }; 35 | 36 | export const NType = (val: number): number & { type: 'NType' } => val as any; 37 | 38 | export type Container = 39 | | { tag: 'Units' } 40 | | { tag: 'JustNumber'; value: number } 41 | | { tag: 'Figures'; value: Array
}; 42 | 43 | export module Container { 44 | export const Units: Container = { tag: 'Units' }; 45 | 46 | export const JustNumber = (value: number): Container => ({ 47 | tag: 'JustNumber', 48 | value 49 | }); 50 | 51 | export const Figures = (value: Array
): Container => ({ 52 | tag: 'Figures', 53 | value 54 | }); 55 | } 56 | 57 | export interface Color { 58 | 0: number; 59 | 1: number; 60 | 2: number; 61 | length: 3; 62 | } 63 | 64 | export const Color = (p0: number, p1: number, p2: number): Color => [ 65 | p0, 66 | p1, 67 | p2 68 | ]; 69 | 70 | export interface Figure { 71 | dots: Array; 72 | colors: Array; 73 | } 74 | 75 | export interface Vec3 { 76 | 0: number; 77 | 1: number; 78 | 2: number; 79 | length: 3; 80 | } 81 | 82 | export const Vec3 = (p0: number, p1: number, p2: number): Vec3 => [p0, p1, p2]; 83 | 84 | export type NewtypeAlias = NType; 85 | 86 | export interface NormalStruct { 87 | a: number; 88 | tuple: MyTuple; 89 | } 90 | 91 | export enum MyEnum { 92 | ONE = 'ONE', 93 | TWO = 'TWO', 94 | THREE = 'THREE' 95 | } 96 | 97 | export interface MyTuple { 98 | 0: (boolean) | null; 99 | 1: Array; 100 | length: 2; 101 | } 102 | 103 | export const MyTuple = (p0: (boolean) | null, p1: Array): MyTuple => [ 104 | p0, 105 | p1 106 | ]; 107 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/examples/generated/simple_serde.g.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Message, 3 | Message_Two, 4 | Message_VStruct, 5 | NType, 6 | Container, 7 | Figure, 8 | Color, 9 | Vec3, 10 | NewtypeAlias, 11 | NormalStruct, 12 | MyTuple, 13 | MyEnum 14 | } from './simple.g'; 15 | 16 | import { 17 | write_u32, 18 | write_f32, 19 | opt_writer, 20 | write_bool, 21 | write_str, 22 | seq_writer, 23 | write_u8, 24 | nullable_writer, 25 | Sink, 26 | Serializer, 27 | read_u32, 28 | read_f32, 29 | opt_reader, 30 | read_bool, 31 | read_str, 32 | seq_reader, 33 | read_u8, 34 | nullable_reader, 35 | Deserializer 36 | } from '../../../ts-binary/src/index'; 37 | 38 | // Serializers 39 | 40 | const writeOptBool: Serializer<(boolean) | undefined> = opt_writer(write_bool); 41 | 42 | export const writeVec3 = (sink: Sink, val: Vec3): Sink => 43 | write_f32(write_f32(write_f32(sink, val[0]), val[1]), val[2]); 44 | 45 | const writeVecVec3: Serializer> = seq_writer(writeVec3); 46 | 47 | export const writeColor = (sink: Sink, val: Color): Sink => 48 | write_u8(write_u8(write_u8(sink, val[0]), val[1]), val[2]); 49 | 50 | const writeVecColor: Serializer> = seq_writer(writeColor); 51 | 52 | export const writeFigure = (sink: Sink, { dots, colors }: Figure): Sink => 53 | writeVecColor(writeVecVec3(sink, dots), colors); 54 | 55 | const writeVecFigure: Serializer> = seq_writer(writeFigure); 56 | 57 | const writeNullableBool: Serializer<(boolean) | null> = nullable_writer( 58 | write_bool 59 | ); 60 | 61 | const writeVecStr: Serializer> = seq_writer(write_str); 62 | 63 | const writeMessage_Two = (sink: Sink, val: Message_Two): Sink => 64 | write_u32(writeOptBool(sink, val[0]), val[1]); 65 | 66 | const writeMessage_VStruct = ( 67 | sink: Sink, 68 | { id, data }: Message_VStruct 69 | ): Sink => write_str(write_str(sink, id), data); 70 | 71 | export const writeMessage = (sink: Sink, val: Message): Sink => { 72 | switch (val.tag) { 73 | case 'Unit': 74 | return write_u32(sink, 0); 75 | case 'One': 76 | return write_f32(write_u32(sink, 1), val.value); 77 | case 'Two': 78 | return writeMessage_Two(write_u32(sink, 2), val.value); 79 | case 'VStruct': 80 | return writeMessage_VStruct(write_u32(sink, 3), val.value); 81 | } 82 | }; 83 | 84 | export const writeNType: Serializer = write_u32; 85 | 86 | export const writeContainer = (sink: Sink, val: Container): Sink => { 87 | switch (val.tag) { 88 | case 'Units': 89 | return write_u32(sink, 0); 90 | case 'JustNumber': 91 | return write_u32(write_u32(sink, 1), val.value); 92 | case 'Figures': 93 | return writeVecFigure(write_u32(sink, 2), val.value); 94 | } 95 | }; 96 | 97 | export const writeNewtypeAlias: Serializer = writeNType; 98 | 99 | export const writeMyTuple = (sink: Sink, val: MyTuple): Sink => 100 | writeVecStr(writeNullableBool(sink, val[0]), val[1]); 101 | 102 | export const writeNormalStruct = ( 103 | sink: Sink, 104 | { a, tuple }: NormalStruct 105 | ): Sink => writeMyTuple(write_u8(sink, a), tuple); 106 | 107 | const MyEnumMap: { [key: string]: number } = { ONE: 0, TWO: 1, THREE: 2 }; 108 | 109 | export const writeMyEnum = (sink: Sink, val: MyEnum): Sink => 110 | write_u32(sink, MyEnumMap[val]); 111 | 112 | // Deserializers 113 | 114 | const readOptBool: Deserializer<(boolean) | undefined> = opt_reader(read_bool); 115 | 116 | export const readVec3 = (sink: Sink): Vec3 => 117 | Vec3(read_f32(sink), read_f32(sink), read_f32(sink)); 118 | 119 | const readVecVec3: Deserializer> = seq_reader(readVec3); 120 | 121 | export const readColor = (sink: Sink): Color => 122 | Color(read_u8(sink), read_u8(sink), read_u8(sink)); 123 | 124 | const readVecColor: Deserializer> = seq_reader(readColor); 125 | 126 | export const readFigure = (sink: Sink): Figure => { 127 | const dots = readVecVec3(sink); 128 | const colors = readVecColor(sink); 129 | return { dots, colors }; 130 | }; 131 | 132 | const readVecFigure: Deserializer> = seq_reader(readFigure); 133 | 134 | const readNullableBool: Deserializer<(boolean) | null> = nullable_reader( 135 | read_bool 136 | ); 137 | 138 | const readVecStr: Deserializer> = seq_reader(read_str); 139 | 140 | export const readMessage = (sink: Sink): Message => { 141 | switch (read_u32(sink)) { 142 | case 0: 143 | return Message.Unit; 144 | case 1: 145 | return Message.One(read_f32(sink)); 146 | case 2: 147 | return Message.Two(readOptBool(sink), read_u32(sink)); 148 | case 3: 149 | return Message.VStruct(readMessage_VStruct(sink)); 150 | } 151 | throw new Error('bad variant index for Message'); 152 | }; 153 | 154 | const readMessage_VStruct = (sink: Sink): Message_VStruct => { 155 | const id = read_str(sink); 156 | const data = read_str(sink); 157 | return { id, data }; 158 | }; 159 | 160 | export const readNType = (sink: Sink): NType => NType(read_u32(sink)); 161 | 162 | export const readContainer = (sink: Sink): Container => { 163 | switch (read_u32(sink)) { 164 | case 0: 165 | return Container.Units; 166 | case 1: 167 | return Container.JustNumber(read_u32(sink)); 168 | case 2: 169 | return Container.Figures(readVecFigure(sink)); 170 | } 171 | throw new Error('bad variant index for Container'); 172 | }; 173 | 174 | export const readNewtypeAlias: Deserializer = readNType; 175 | 176 | export const readMyTuple = (sink: Sink): MyTuple => 177 | MyTuple(readNullableBool(sink), readVecStr(sink)); 178 | 179 | export const readNormalStruct = (sink: Sink): NormalStruct => { 180 | const a = read_u8(sink); 181 | const tuple = readMyTuple(sink); 182 | return { a, tuple }; 183 | }; 184 | 185 | const MyEnumReverseMap: MyEnum[] = [MyEnum.ONE, MyEnum.TWO, MyEnum.THREE]; 186 | 187 | export const readMyEnum = (sink: Sink): MyEnum => 188 | MyEnumReverseMap[read_u32(sink)]; 189 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/examples/runExample.ts: -------------------------------------------------------------------------------- 1 | import { 2 | writeMessage, 3 | writeMyEnum, 4 | writeNormalStruct, 5 | readMessage, 6 | readMyEnum, 7 | readNormalStruct 8 | } from './generated/simple_serde.g'; 9 | import { 10 | Sink, 11 | Serializer, 12 | write_str, 13 | Deserializer 14 | } from '../../ts-binary/src/index'; 15 | 16 | import { NormalStruct, Message, MyEnum, MyTuple } from './generated/simple.g'; 17 | 18 | let sink = Sink(new ArrayBuffer(100)); 19 | 20 | const writeAThing = (thing: T, ser: Serializer): Uint8Array => { 21 | sink.pos = 0; 22 | sink = ser(sink, thing); 23 | return new Uint8Array(sink.view.buffer).slice(0, sink.pos); 24 | }; 25 | 26 | const readAThing = (arr: ArrayBuffer, deser: Deserializer): T => 27 | deser(Sink(arr)); 28 | 29 | const str = 'AnotherUnit'; 30 | console.log('tag', JSON.stringify(str), writeAThing(str, write_str)); 31 | 32 | // const msg = Message.One(7); 33 | // [3, 0, 0, 0, 0, 0, 0, 0, 84, 119, 111, 1, 1, 7, 0, 0, 0] 34 | // [ 3, 0, 0, 0, 1, 1, 7, 0, 0, 0 ] 35 | // [ 3, 0, 0, 0, 1, 1, 7, 0, 0, 0 ] 36 | const msg = Message.VStruct({ id: 'some id', data: 'some data' }); 37 | 38 | // with tag [11, 0, 0, 0, 0, 0, 0, 0, 65, 110, 111, 116, 104, 101, 114, 85, 110, 105, 116] 39 | // no tag [ 1, 0, 0, 0 ] 40 | // const msg = Message.AnotherUnit; 41 | 42 | const bytes = writeAThing(msg, writeMessage); 43 | const restoredMessage = readAThing(bytes.buffer, readMessage); 44 | 45 | console.log( 46 | `are equals = ${JSON.stringify(msg) === 47 | JSON.stringify(restoredMessage)}, json =`, 48 | JSON.stringify(msg) 49 | ); 50 | 51 | const en = MyEnum.ONE; 52 | console.log('enum', JSON.stringify(en), writeAThing(en, writeMyEnum)); 53 | console.log('D: enum', readAThing(sink.view.buffer, readMyEnum)); 54 | 55 | const struct: NormalStruct = { a: 13, tuple: MyTuple(true, ['ab', 'c']) }; 56 | console.log( 57 | 'struct', 58 | JSON.stringify(struct), 59 | writeAThing(struct, writeNormalStruct) 60 | ); 61 | console.log('D: struct', readAThing(sink.view.buffer, readNormalStruct)); 62 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/examples/simple.schema.ts: -------------------------------------------------------------------------------- 1 | import { Type } from '../src/schema'; 2 | 3 | const { 4 | Alias, 5 | Enum, 6 | Tuple, 7 | Struct, 8 | Union, 9 | Newtype, 10 | Vec, 11 | Str, 12 | Bool, 13 | Option, 14 | Nullable, 15 | F32, 16 | U8, 17 | U32 18 | } = Type; 19 | 20 | export const MyEnum = Enum('ONE', 'TWO', 'THREE'); 21 | 22 | const MyTuple = Tuple(Nullable(Bool), Vec(Str)); 23 | 24 | const NormalStruct = Struct({ 25 | a: U8, 26 | tuple: MyTuple 27 | }); 28 | 29 | const Message = Union({ 30 | Unit: null, 31 | One: F32, 32 | Two: [Option(Bool), U32], 33 | VStruct: { id: Str, data: Str } 34 | }); 35 | 36 | const Vec3 = Tuple(F32, F32, F32); 37 | const Color = Tuple(U8, U8, U8); 38 | const Figure = Struct({ dots: Vec(Vec3), colors: Vec(Color) }); 39 | 40 | const Container = Union({ 41 | Units: null, 42 | JustNumber: U32, 43 | Figures: Vec(Figure) 44 | }); 45 | 46 | const NType = Newtype(U32); 47 | const NewtypeAlias = Alias(NType); 48 | 49 | export const exampleSchema = { 50 | Message, 51 | NType, 52 | Container, 53 | Color, 54 | Figure, 55 | Vec3, 56 | NewtypeAlias, 57 | NormalStruct, 58 | MyEnum, 59 | MyTuple 60 | }; 61 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: 'ts-jest', 3 | testEnvironment: 'node', 4 | testMatch: ['**/?(*.)+(spec|test).[jt]s?(x)'] 5 | }; 6 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ts-rust-bridge-codegen", 3 | "version": "0.11.0", 4 | "description": "A toolset to build efficient communication between rust and typescript", 5 | "license": "MIT", 6 | "repository": { 7 | "type": "git", 8 | "url": "https://github.com/twop/ts-rust-bridge.git" 9 | }, 10 | "publishConfig": { 11 | "access": "public" 12 | }, 13 | "@pika/pack": { 14 | "pipeline": [ 15 | [ 16 | "@pika/plugin-ts-standard-pkg", 17 | { 18 | "tsconfig": "./tsconfig.build.json", 19 | "exclude": [ 20 | "__tests__/**/*.*", 21 | "examples/**/*.*" 22 | ] 23 | } 24 | ], 25 | [ 26 | "@pika/plugin-build-node" 27 | ], 28 | [ 29 | "@pika/plugin-build-web" 30 | ] 31 | ] 32 | }, 33 | "scripts": { 34 | "build": "pika build", 35 | "gen:example-code": "ts-node -O \"{\\\"module\\\": \\\"CommonJS\\\"}\" examples/gen_simple.schema.ts", 36 | "gen:docs-code": "ts-node -O \"{\\\"module\\\": \\\"CommonJS\\\"}\" examples/for_docs/gen_docs_example.ts", 37 | "run:docs-code": "ts-node -O \"{\\\"module\\\": \\\"CommonJS\\\"}\" examples/for_docs/usage.ts", 38 | "run:example": "ts-node -O \"{\\\"module\\\": \\\"CommonJS\\\"}\" examples/runExample.ts", 39 | "gen:test-code": "ts-node -O \"{\\\"module\\\": \\\"CommonJS\\\"}\" __tests__/gen_testSchema.ts", 40 | "test": "jest", 41 | "benchmark": "ts-node -O \"{\\\"module\\\": \\\"CommonJS\\\"}\" examples/benchmark.ts" 42 | }, 43 | "dependencies": { 44 | "ts-union": "^2.2.1" 45 | }, 46 | "devDependencies": { 47 | "@babel/plugin-transform-modules-commonjs": "^7.4.4", 48 | "@babel/preset-env": "^7.4.5", 49 | "@babel/preset-typescript": "^7.3.3", 50 | "@pika/pack": "^0.5.0", 51 | "@pika/plugin-build-node": "^0.9.2", 52 | "@pika/plugin-build-web": "^0.9.2", 53 | "@pika/plugin-ts-standard-pkg": "^0.9.2", 54 | "@types/jest": "^24.0.13", 55 | "@types/node": "^12.0.4", 56 | "@types/prettier": "^1.16.1", 57 | "jest": "^26.0.1", 58 | "prettier": "^1.18.2", 59 | "ts-binary": "^0.11.0", 60 | "ts-jest": "^26.1.0", 61 | "ts-node": "^8.4.1", 62 | "typescript": "^3.9.5" 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/src/index.ts: -------------------------------------------------------------------------------- 1 | import { Schema, FileBlock } from './schema'; 2 | import { ast2ts } from './ts/ast2ts'; 3 | import { schema2ast } from './ts/schema2ast'; 4 | import { schema2serializersAST } from './serde/schema2serializers'; 5 | import { schema2deserializersAST } from './serde/schema2deserializers'; 6 | import { SerDeCodeGenInput } from './serde/sharedPieces'; 7 | import { TsFileBlock, Code } from './ts/ast'; 8 | 9 | export * from './schema'; 10 | 11 | export { 12 | schema2rust, 13 | SchemaDelcarationObject, 14 | SchemaRustOptions, 15 | RustTypeOptions 16 | } from './rust/schema2rust'; 17 | 18 | export { SerDeCodeGenInput }; 19 | export const schema2ts = (schema: Schema): FileBlock[] => 20 | ast2ts(schema2ast(schema)); 21 | 22 | export const schema2serde = (input: SerDeCodeGenInput): FileBlock[] => { 23 | const serAst = schema2serializersAST(input); 24 | const deserAst = schema2deserializersAST(input); 25 | return ast2ts(mergeTsBlocks(serAst, deserAst)); 26 | }; 27 | 28 | const mergeTsBlocks = ( 29 | ser: TsFileBlock[], 30 | deser: TsFileBlock[] 31 | ): TsFileBlock[] => { 32 | const { imports, rest } = [ 33 | TsFileBlock.LineComment('Serializers'), 34 | ...ser, 35 | TsFileBlock.LineComment('Deserializers'), 36 | ...deser 37 | ].reduce<{ imports: Code.Import[]; rest: TsFileBlock[] }>( 38 | ({ imports, rest }, cur) => 39 | TsFileBlock.match(cur, { 40 | Import: i => ({ imports: append(imports, i), rest }), 41 | default: block => ({ imports, rest: append(rest, block) }) 42 | }), 43 | { imports: [], rest: [] } 44 | ); 45 | 46 | return smooshImports(imports) 47 | .map(i => TsFileBlock.Import(i)) 48 | .concat(rest); 49 | }; 50 | 51 | function smooshImports(imports: Code.Import[]): Code.Import[] { 52 | const fromToImportNames = imports.reduce( 53 | (map, { from, names }) => map.set(from, append(map.get(from), ...names)), 54 | new Map() 55 | ); 56 | 57 | return Array.from(fromToImportNames.entries()).map( 58 | ([from, names]): Code.Import => ({ 59 | from, 60 | names: Array.from(new Set(names)) 61 | }) 62 | ); 63 | } 64 | 65 | function append(arr: T[] = [], ...values: T[]): T[] { 66 | arr.push(...values); 67 | return arr; 68 | } 69 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/src/rust/schema2rust.ts: -------------------------------------------------------------------------------- 1 | import { 2 | EnumVariants, 3 | StructMembers, 4 | Scalar, 5 | Type, 6 | SchemaElement, 7 | TypeTag, 8 | Variant, 9 | FileBlock, 10 | UnionOptions, 11 | LookupName, 12 | createLookupName, 13 | matchSchemaElement 14 | } from '../schema'; 15 | 16 | export type RustTypeOptions = { 17 | derive: [string, ...string[]]; 18 | }; 19 | 20 | // const mapValues = ( 21 | // obj: { [name: string]: A }, 22 | // f: (a: A) => B 23 | // ): { [name: string]: B } => 24 | // Object.fromEntries( 25 | // Object.entries(obj).map(([key, val]): [string, B] => [key, f(val)]) 26 | // ); 27 | 28 | // export type ElementWithRustSettings = [SchemaElement, RustTypeOptions]; 29 | 30 | // const hasOptions = ( 31 | // elem: SchemaElement | ElementWith 32 | // ): elem is ElementWithRustSettings => Array.isArray(elem); 33 | 34 | export type SchemaDelcarationObject = { [name: string]: SchemaElement }; 35 | export type SchemaRustOptions = Partial< 36 | { [K in keyof T]: RustTypeOptions } 37 | >; 38 | 39 | export const schema2rust = ( 40 | entries: T, 41 | options?: SchemaRustOptions 42 | ): FileBlock[] => { 43 | const lookup = createLookupName(entries); 44 | 45 | return Object.entries(entries).map(([name, entry]) => { 46 | const opt = options && options[name]; 47 | 48 | return matchSchemaElement(entry, { 49 | Alias: t => aliasToAlias(name, t, lookup), 50 | Enum: variants => enumToEnum(name, variants, opt), 51 | Newtype: t => newtypeToStruct(name, t, lookup, opt), 52 | Tuple: fields => tupleToStruct(name, fields, lookup, opt), 53 | Struct: members => structToStruct(name, members, lookup, opt), 54 | Union: (variants, options) => 55 | unionToEnum(name, variants, options, lookup, opt) 56 | }); 57 | }); 58 | }; 59 | 60 | const deriveBlock = (opt: RustTypeOptions | undefined): string => 61 | opt 62 | ? `#[derive(Deserialize, Serialize, ${opt.derive.join(', ')})]` 63 | : `#[derive(Deserialize, Serialize)]`; 64 | 65 | const aliasToAlias = (name: string, type: Type, lookup: LookupName) => ` 66 | pub type ${name} = ${typeToString(type, lookup)}; 67 | `; 68 | 69 | const enumToEnum = ( 70 | name: string, 71 | { variants }: EnumVariants, 72 | opt: RustTypeOptions | undefined 73 | ): string => ` 74 | ${deriveBlock(opt)} 75 | pub enum ${name} { 76 | ${variants.map(v => ` ${v},`).join('\n')} 77 | } 78 | `; 79 | 80 | const newtypeToStruct = ( 81 | name: string, 82 | type: Type, 83 | lookup: LookupName, 84 | opt: RustTypeOptions | undefined 85 | ): string => ` 86 | ${deriveBlock(opt)} 87 | pub struct ${name}(pub ${typeToString(type, lookup)}); 88 | `; 89 | 90 | const tupleToStruct = ( 91 | name: string, 92 | fields: Type[], 93 | lookup: LookupName, 94 | opt: RustTypeOptions | undefined 95 | ): string => ` 96 | ${deriveBlock(opt)} 97 | pub struct ${name}(${fields 98 | .map(t => `pub ${typeToString(t, lookup)}`) 99 | .join(', ')}); 100 | `; 101 | const structToStruct = ( 102 | name: string, 103 | members: StructMembers, 104 | lookup: LookupName, 105 | opt: RustTypeOptions | undefined 106 | ): string => ` 107 | ${deriveBlock(opt)} 108 | pub struct ${name} { 109 | ${Object.keys(members) 110 | .map(n => { 111 | const snakeName = camelToSnakeCase(n); 112 | const field = `pub ${snakeName}: ${typeToString(members[n], lookup)}`; 113 | return snakeName === n 114 | ? ` ${field},` 115 | : ` #[serde(rename = "${n}")]\n ${field},\n`; 116 | }) 117 | .join('\n')} 118 | } 119 | `; 120 | 121 | const unionToEnum = ( 122 | name: string, 123 | variants: Variant[], 124 | { tagAnnotation }: UnionOptions, 125 | lookup: LookupName, 126 | opt: RustTypeOptions | undefined 127 | ): string => ` 128 | ${deriveBlock(opt)}${ 129 | tagAnnotation ? '\n#[serde(tag = "tag", content = "value")]' : '' 130 | } 131 | pub enum ${name} { 132 | ${variants.map(v => ` ${variantStr(v, lookup)},`).join('\n')} 133 | } 134 | `; 135 | 136 | const variantStr = (v: Variant, lookup: LookupName) => 137 | Variant.match(v, { 138 | Unit: name => name, 139 | NewType: (name, type) => `${name}(${typeToString(type, lookup)})`, 140 | Tuple: (name, fields) => 141 | `${name}(${fields.map(f => typeToString(f, lookup)).join(', ')})`, 142 | Struct: (name, members) => 143 | `${name} { ${Object.keys(members) 144 | .map(n => { 145 | const snakeName = camelToSnakeCase(n); 146 | 147 | return `${ 148 | snakeName === n ? '' : `#[serde(rename = "${n}")] ` 149 | }${camelToSnakeCase(n)}: ${typeToString(members[n], lookup)}`; 150 | }) 151 | .join(', ')} }` 152 | }); 153 | 154 | const scalarToString = (scalar: Scalar): string => { 155 | switch (scalar) { 156 | case Scalar.Bool: 157 | return 'bool'; 158 | case Scalar.F32: 159 | return 'f32'; 160 | case Scalar.F64: 161 | return 'f64'; 162 | case Scalar.U8: 163 | return 'u8'; 164 | case Scalar.U16: 165 | return 'u16'; 166 | case Scalar.U32: 167 | return 'u32'; 168 | case Scalar.USIZE: 169 | return 'usize'; 170 | case Scalar.I32: 171 | return 'i32'; 172 | case Scalar.Str: 173 | return 'String'; 174 | } 175 | }; 176 | 177 | const typeToString = (type: Type, lookup: LookupName): string => { 178 | switch (type.tag) { 179 | case TypeTag.Nullable: 180 | case TypeTag.Option: 181 | return `Option<${typeToString(type.value, lookup)}>`; 182 | case TypeTag.Scalar: 183 | return scalarToString(type.value); 184 | case TypeTag.Vec: 185 | return `Vec<${typeToString(type.value, lookup)}>`; 186 | } 187 | 188 | return lookup(type); 189 | }; 190 | 191 | const camelToSnakeCase = (str: string) => 192 | str.replace(/[A-Z]/g, letter => `_${letter.toLowerCase()}`); 193 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/src/schema.ts: -------------------------------------------------------------------------------- 1 | import { Union, of } from 'ts-union'; 2 | 3 | export enum Scalar { 4 | U8 = 'U8', 5 | U16 = 'U16', 6 | U32 = 'U32', 7 | I32 = 'I32', 8 | USIZE = 'USIZE', 9 | F32 = 'F32', 10 | F64 = 'F64', 11 | Str = 'Str', 12 | Bool = 'Bool' 13 | } 14 | 15 | export type EnumVariants = { 16 | variants: string[]; 17 | }; 18 | 19 | export type StructMembers = { 20 | [prop: string]: Type; 21 | }; 22 | 23 | export type UnionOptions = { 24 | tagAnnotation: boolean; 25 | }; 26 | 27 | export type SchemaElement = 28 | | { tag: 'Alias'; val: Type } 29 | | { tag: 'Struct'; val: StructMembers } 30 | | { tag: 'Enum'; val: EnumVariants } 31 | | { tag: 'Tuple'; val: Type[] } 32 | | { tag: 'Newtype'; val: Type } 33 | | { tag: 'Union'; val: [Variant[], UnionOptions] }; 34 | 35 | export type Schema = { [name: string]: SchemaElement }; 36 | 37 | const possibleSchemaElementTags = [ 38 | 'Alias', 39 | 'Struct', 40 | 'Enum', 41 | 'Tuple', 42 | 'Newtype', 43 | 'Union' 44 | ]; 45 | 46 | export const isSchemaElement = (val: unknown): val is SchemaElement => 47 | typeof val === 'object' && 48 | val !== null && 49 | 'tag' in val && 50 | possibleSchemaElementTags.includes((val as any).tag); 51 | 52 | type MatchSchemaElement = { 53 | Alias: (val: Type) => Res; 54 | Struct: (val: StructMembers) => Res; 55 | Enum: (val: EnumVariants) => Res; 56 | Tuple: (val: Type[]) => Res; 57 | Newtype: (val: Type) => Res; 58 | Union: (variants: Variant[], options: UnionOptions) => Res; 59 | }; 60 | 61 | export const matchSchemaElement = ( 62 | entry: SchemaElement, 63 | m: MatchSchemaElement 64 | ): Res => { 65 | switch (entry.tag) { 66 | case 'Alias': 67 | return m.Alias(entry.val); 68 | case 'Struct': 69 | return m.Struct(entry.val); 70 | case 'Enum': 71 | return m.Enum(entry.val); 72 | case 'Tuple': 73 | return m.Tuple(entry.val); 74 | case 'Newtype': 75 | return m.Newtype(entry.val); 76 | case 'Union': 77 | return m.Union(entry.val[0], entry.val[1]); 78 | } 79 | }; 80 | 81 | export type LookupName = (element: SchemaElement) => string; 82 | 83 | export const createLookupName = (schema: Schema): LookupName => { 84 | const mapping = Object.entries(schema).reduce( 85 | (map, [name, entry]) => map.set(entry, name), 86 | new Map() 87 | ); 88 | 89 | return element => { 90 | const name = mapping.get(element); 91 | if (name === undefined) { 92 | throw new Error( 93 | 'not found name for:\n' + JSON.stringify(element, undefined, 2) 94 | ); 95 | } 96 | 97 | return name; 98 | }; 99 | }; 100 | 101 | // export type Entry = Entry2; //typeof Entry.T; 102 | // export type Entry = UnionOf; //typeof Entry.T; 103 | 104 | export const Variant = Union({ 105 | Unit: of(), 106 | Tuple: of(), 107 | NewType: of(), 108 | Struct: of() 109 | }); 110 | 111 | export enum TypeTag { 112 | Scalar = 'Scalar', 113 | Vec = 'Vec', 114 | Option = 'Option', 115 | Nullable = 'Nullable' 116 | // RefTo = 'RefTo' 117 | } 118 | 119 | const scalarsToType: { [K in Scalar]: { tag: TypeTag.Scalar; value: K } } = { 120 | [Scalar.U8]: { tag: TypeTag.Scalar, value: Scalar.U8 }, 121 | [Scalar.U16]: { tag: TypeTag.Scalar, value: Scalar.U16 }, 122 | [Scalar.U32]: { tag: TypeTag.Scalar, value: Scalar.U32 }, 123 | [Scalar.I32]: { tag: TypeTag.Scalar, value: Scalar.I32 }, 124 | [Scalar.USIZE]: { tag: TypeTag.Scalar, value: Scalar.USIZE }, 125 | [Scalar.F32]: { tag: TypeTag.Scalar, value: Scalar.F32 }, 126 | [Scalar.F64]: { tag: TypeTag.Scalar, value: Scalar.F64 }, 127 | [Scalar.Str]: { tag: TypeTag.Scalar, value: Scalar.Str }, 128 | [Scalar.Bool]: { tag: TypeTag.Scalar, value: Scalar.Bool } 129 | }; 130 | 131 | export type Type = 132 | | { tag: TypeTag.Scalar; value: Scalar } 133 | | { tag: TypeTag.Vec; value: Type } 134 | | { tag: TypeTag.Option; value: Type } 135 | | { tag: TypeTag.Nullable; value: Type } 136 | | SchemaElement; 137 | 138 | const isTypeDefinition = (val: unknown): val is Type => { 139 | if (isSchemaElement(val)) return true; 140 | 141 | const possibleTypeTags = [ 142 | TypeTag.Scalar, 143 | TypeTag.Vec, 144 | TypeTag.Option, 145 | TypeTag.Nullable 146 | ]; 147 | if (typeof val === 'object' && val != null && 'tag' in val) { 148 | return possibleTypeTags.includes((val as any).tag); 149 | } 150 | return false; 151 | }; 152 | 153 | type UnionVariantValue = 154 | | null 155 | | Type 156 | | [Type, Type, ...Type[]] 157 | | { [field: string]: Type }; 158 | 159 | type UnionDef = { [variant: string]: UnionVariantValue }; 160 | const isTupleVariant = ( 161 | val: UnionVariantValue 162 | ): val is [Type, Type, ...Type[]] => Array.isArray(val); 163 | 164 | const unionDefToVariants = (def: UnionDef): Variant[] => 165 | Object.entries(def).map(([field, val]) => { 166 | if (val === null) return Variant.Unit(field); 167 | if (isTupleVariant(val)) return Variant.Tuple(field, val); 168 | if (isTypeDefinition(val)) return Variant.NewType(field, val); 169 | 170 | return Variant.Struct(field, val); 171 | }); 172 | 173 | const getName = (s: string) => s; 174 | export const Type = { 175 | ...scalarsToType, 176 | Vec: (value: Type): Type => ({ tag: TypeTag.Vec, value }), 177 | Option: (value: Type): Type => ({ tag: TypeTag.Option, value }), 178 | Nullable: (value: Type): Type => ({ tag: TypeTag.Nullable, value }), 179 | Alias: (val: Type): SchemaElement => ({ tag: 'Alias', val }), 180 | Struct: (val: StructMembers): SchemaElement => ({ tag: 'Struct', val }), 181 | Enum: (...variants: string[]): SchemaElement => ({ 182 | tag: 'Enum', 183 | val: { variants } 184 | }), 185 | Tuple: (...val: Type[]): SchemaElement => ({ tag: 'Tuple', val }), 186 | Newtype: (val: Type): SchemaElement => ({ tag: 'Newtype', val }), 187 | Union: ( 188 | def: UnionDef, 189 | options: UnionOptions = { tagAnnotation: false } 190 | ): SchemaElement => ({ 191 | tag: 'Union', 192 | val: [unionDefToVariants(def), options] 193 | }) 194 | }; 195 | 196 | export const getVariantName = Variant.match({ 197 | Struct: getName, 198 | Tuple: getName, 199 | NewType: getName, 200 | Unit: getName 201 | }); 202 | 203 | // export type EntryType = typeof Entry.T; 204 | export type Variant = typeof Variant.T; 205 | 206 | export type FileBlock = string; 207 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/src/serde/schema2deserializers.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Scalar, 3 | SchemaElement, 4 | Type, 5 | TypeTag, 6 | Variant, 7 | LookupName, 8 | matchSchemaElement, 9 | createLookupName 10 | } from '../schema'; 11 | 12 | import { TsFileBlock, TsFileBlock as ts } from '../ts/ast'; 13 | import { variantPayloadTypeName } from '../ts/schema2ast'; 14 | import { 15 | BincodeLibTypes, 16 | traverseType, 17 | chainName, 18 | RequiredImport, 19 | flatMap, 20 | ReadOrWrite, 21 | collectRequiredImports, 22 | enumerateStructFields, 23 | CodePiece, 24 | TypeSerDe, 25 | schema2tsBlocks, 26 | SerDeCodeGenInput 27 | } from './sharedPieces'; 28 | 29 | const ReadFuncs: ReadOrWrite = { 30 | [Scalar.Bool]: 'read_bool', 31 | [Scalar.Str]: 'read_str', 32 | [Scalar.F32]: 'read_f32', 33 | [Scalar.F64]: 'read_f64', 34 | [Scalar.I32]: 'read_i32', 35 | [Scalar.U8]: 'read_u8', 36 | [Scalar.U16]: 'read_u16', 37 | [Scalar.U32]: 'read_u32', 38 | [Scalar.USIZE]: 'read_u64', 39 | Opt: 'opt_reader', 40 | Seq: 'seq_reader', 41 | Nullable: 'nullable_reader' 42 | }; 43 | 44 | const deserializerType = (typeStr: string) => 45 | `${BincodeLibTypes.Deserializer}<${typeStr}>`; 46 | const enumMappingArrayName = (enumName: string) => `${enumName}ReverseMap`; 47 | const deserFuncName = (typeName: string) => `read${typeName}`; 48 | 49 | const deserializerChainName = (types: Type[], lookup: LookupName): string => 50 | chainName(types, ReadFuncs, deserFuncName, lookup); 51 | 52 | const deserializerNameFor = (type: Type, lookup: LookupName): string => 53 | deserializerChainName(traverseType(type), lookup); 54 | 55 | const { fromLibrary, fromTypesDeclaration } = RequiredImport; 56 | 57 | const entry2DeserBlocks = ( 58 | name: string, 59 | entry: SchemaElement, 60 | lookup: LookupName 61 | ) => 62 | matchSchemaElement(entry, { 63 | Enum: ({ variants }): CodePiece => ({ 64 | name, 65 | requiredImports: [ 66 | fromTypesDeclaration(name), 67 | fromLibrary(ReadFuncs[Scalar.U32]) 68 | ], 69 | blocks: [ 70 | genEnumIndexMapping(name, variants), 71 | generateEnumDeserializer(name) 72 | ], 73 | serdes: [] 74 | }), 75 | // default: (): Piece => ({ 76 | 77 | // requiredImports: [], 78 | // blocks: [], 79 | // typeDeserializers: [] 80 | // }), 81 | 82 | Alias: (type): CodePiece => ({ 83 | name, 84 | requiredImports: [ 85 | fromTypesDeclaration(name), 86 | ...collectRequiredImports(type, ReadFuncs, lookup) 87 | ], 88 | serdes: generateTypesDeserializers(type, lookup), 89 | blocks: [ 90 | ts.ConstVar({ 91 | name: deserFuncName(name), 92 | type: deserializerType(name), 93 | expression: deserializerNameFor(type, lookup) 94 | }) 95 | ], 96 | dependsOn: [deserializerNameFor(type, lookup)] 97 | }), 98 | 99 | Newtype: (type): CodePiece => ({ 100 | name, 101 | requiredImports: [ 102 | fromTypesDeclaration(name), 103 | ...collectRequiredImports(type, ReadFuncs, lookup) 104 | ], 105 | serdes: generateTypesDeserializers(type, lookup), 106 | blocks: [ 107 | ts.ArrowFunc({ 108 | name: deserFuncName(name), 109 | returnType: name, 110 | body: `${name}(${deserializerNameFor(type, lookup)}(sink))`, 111 | params: [{ name: 'sink', type: BincodeLibTypes.Sink }] 112 | }) 113 | ], 114 | dependsOn: [deserializerNameFor(type, lookup)] 115 | }), 116 | 117 | Tuple: (types): CodePiece => ({ 118 | name, 119 | requiredImports: [ 120 | fromTypesDeclaration(name), 121 | ...flatMap(types, t => collectRequiredImports(t, ReadFuncs, lookup)) 122 | ], 123 | blocks: [ 124 | generateTupleDeserializer( 125 | name, 126 | types, 127 | args => `${name}(${args})`, 128 | true, 129 | lookup 130 | ) 131 | ], 132 | serdes: flatMap(types, t => generateTypesDeserializers(t, lookup)), 133 | dependsOn: types.map(t => deserializerNameFor(t, lookup)) 134 | }), 135 | 136 | Struct: (members): CodePiece => { 137 | const fields = enumerateStructFields(members); 138 | 139 | return { 140 | name, 141 | requiredImports: [ 142 | fromTypesDeclaration(name), 143 | ...flatMap(fields, f => 144 | collectRequiredImports(f.type, ReadFuncs, lookup) 145 | ) 146 | ], 147 | blocks: [generateStructDeserializer(name, fields, true, lookup)], 148 | serdes: flatMap(fields, f => 149 | generateTypesDeserializers(f.type, lookup) 150 | ), 151 | dependsOn: fields.map(f => deserializerNameFor(f.type, lookup)) 152 | }; 153 | }, 154 | 155 | Union: (variants): CodePiece => ({ 156 | // this can be potentially sharable? 157 | name: name, 158 | requiredImports: [ 159 | fromTypesDeclaration(name), 160 | fromLibrary(ReadFuncs[Scalar.U32]), 161 | ...flatMap( 162 | variants, 163 | Variant.match({ 164 | Unit: () => [] as RequiredImport[], 165 | NewType: (_, type) => 166 | collectRequiredImports(type, ReadFuncs, lookup), 167 | Struct: (variantName, members) => 168 | flatMap(enumerateStructFields(members), m => 169 | collectRequiredImports(m.type, ReadFuncs, lookup) 170 | ).concat( 171 | fromTypesDeclaration(variantPayloadTypeName(name, variantName)) 172 | ), 173 | Tuple: (_, types) => 174 | flatMap(types, t => collectRequiredImports(t, ReadFuncs, lookup)) 175 | }) 176 | ) 177 | ], 178 | blocks: [ 179 | ts.ArrowFunc({ 180 | name: deserFuncName(name), 181 | body: genUnionDeserializers(name, variants, 'sink', lookup), 182 | returnType: name, 183 | params: [{ name: 'sink', type: BincodeLibTypes.Sink }] 184 | }), 185 | ...flatMap( 186 | variants, 187 | Variant.match({ 188 | Struct: (variantName, members) => [ 189 | generateStructDeserializer( 190 | variantPayloadTypeName(name, variantName), 191 | enumerateStructFields(members), 192 | false, 193 | lookup 194 | ) 195 | ], 196 | default: () => [] as TsFileBlock[] 197 | }) 198 | ) 199 | ], 200 | serdes: flatMap( 201 | variants, 202 | Variant.match({ 203 | Unit: () => [] as TypeSerDe[], 204 | NewType: (_, type) => generateTypesDeserializers(type, lookup), 205 | Struct: (_, members) => 206 | flatMap(enumerateStructFields(members), m => 207 | generateTypesDeserializers(m.type, lookup) 208 | ), 209 | Tuple: (_, types) => 210 | flatMap(types, t => generateTypesDeserializers(t, lookup)) 211 | }) 212 | ), 213 | dependsOn: flatMap( 214 | variants, 215 | Variant.match({ 216 | Unit: () => [] as string[], 217 | NewType: (_, type) => [deserializerNameFor(type, lookup)], 218 | Struct: (_, members) => 219 | enumerateStructFields(members).map(m => 220 | deserializerNameFor(m.type, lookup) 221 | ), 222 | Tuple: (_, types) => types.map(t => deserializerNameFor(t, lookup)) 223 | }) 224 | ) 225 | }) 226 | }); 227 | 228 | export const schema2deserializersAST = ({ 229 | schema, 230 | typesDeclarationFile, 231 | pathToBincodeLib 232 | }: SerDeCodeGenInput): TsFileBlock[] => { 233 | const lookup = createLookupName(schema); 234 | return schema2tsBlocks({ 235 | pieces: Object.entries(schema).map(([name, element]) => 236 | entry2DeserBlocks(name, element, lookup) 237 | ), 238 | serdeName: deserFuncName, 239 | serdeType: deserializerType, 240 | serdeChainName: types => deserializerChainName(types, lookup), 241 | lookup, 242 | libImports: [BincodeLibTypes.Deserializer], 243 | pathToBincodeLib, 244 | typesDeclarationFile, 245 | readOrWrite: ReadFuncs 246 | }); 247 | }; 248 | 249 | const genEnumIndexMapping = (enumName: string, variants: string[]) => 250 | ts.ConstVar({ 251 | name: enumMappingArrayName(enumName), 252 | dontExport: true, 253 | expression: `[${variants.map(v => `${enumName}.${v}`).join(', ')}]`, 254 | type: `${enumName}[]` 255 | }); 256 | 257 | const genUnionDeserializers = ( 258 | unionName: string, 259 | variants: Variant[], 260 | sinkArg: string, 261 | lookup: LookupName 262 | ) => { 263 | const unionCtor = (variantName: string) => `${unionName}.${variantName}`; 264 | return `{ 265 | switch (${ReadFuncs[Scalar.U32]}(${sinkArg})) { 266 | ${variants 267 | .map(v => ({ 268 | exp: Variant.match(v, { 269 | Unit: unionCtor, 270 | Struct: name => 271 | `${unionCtor(name)}(${deserFuncName( 272 | variantPayloadTypeName(unionName, name) 273 | )}(${sinkArg}))`, 274 | NewType: (name, type) => 275 | `${unionCtor(name)}(${deserializerNameFor( 276 | type, 277 | lookup 278 | )}(${sinkArg}))`, 279 | Tuple: (name, types) => 280 | `${unionCtor(name)}(${types 281 | .map(type => `${deserializerNameFor(type, lookup)}(${sinkArg})`) 282 | .join(', ')})` 283 | }) 284 | })) 285 | .map(({ exp }, i) => `case ${i}: return ${exp};`) 286 | .join('\n')} 287 | }; 288 | throw new Error("bad variant index for ${unionName}"); 289 | }`; 290 | }; 291 | 292 | const generateTypesDeserializers = ( 293 | type: Type, 294 | lookup: LookupName, 295 | typeDeserializers: TypeSerDe[] = [] 296 | ): TypeSerDe[] => { 297 | switch (type.tag) { 298 | case TypeTag.Scalar: 299 | // skip scalars 300 | return typeDeserializers; 301 | 302 | case TypeTag.Vec: 303 | case TypeTag.Nullable: 304 | case TypeTag.Option: 305 | return generateTypesDeserializers( 306 | type.value, 307 | lookup, 308 | typeDeserializers.concat({ 309 | typeChain: traverseType(type), 310 | toOrFrom: type, 311 | body: `${ 312 | type.tag === TypeTag.Option 313 | ? ReadFuncs.Opt 314 | : type.tag === TypeTag.Nullable 315 | ? ReadFuncs.Nullable 316 | : ReadFuncs.Seq 317 | }(${deserializerChainName(traverseType(type.value), lookup)})` 318 | }) 319 | ); 320 | } 321 | // skip direct references 322 | return typeDeserializers; 323 | }; 324 | 325 | const generateEnumDeserializer = (enumName: string): TsFileBlock => 326 | ts.ArrowFunc({ 327 | name: deserFuncName(enumName), 328 | returnType: enumName, 329 | params: [{ name: 'sink', type: BincodeLibTypes.Sink }], 330 | body: `${enumMappingArrayName(enumName)}[${ReadFuncs[Scalar.U32]}(sink)]` 331 | }); 332 | 333 | const generateStructDeserializer = ( 334 | name: string, 335 | fields: { name: string; type: Type }[], 336 | shouldExport: boolean, 337 | lookup: LookupName 338 | ): TsFileBlock => 339 | ts.ArrowFunc({ 340 | name: deserFuncName(name), 341 | wrappedInBraces: true, 342 | returnType: name, 343 | dontExport: !shouldExport || undefined, 344 | params: [{ name: 'sink', type: BincodeLibTypes.Sink }], 345 | body: `${fields 346 | .map( 347 | f => `const ${f.name} = ${deserializerNameFor(f.type, lookup)}(sink);` 348 | ) 349 | .join('\n')} 350 | return {${fields.map(f => f.name).join(', ')}}; 351 | ` 352 | }); 353 | 354 | const generateTupleDeserializer = ( 355 | tupleName: string, 356 | types: Type[], 357 | tupleCtorFunc: (argsStr: string) => string, 358 | shouldExport: boolean, 359 | lookup: LookupName 360 | ): TsFileBlock => 361 | ts.ArrowFunc({ 362 | name: deserFuncName(tupleName), 363 | returnType: tupleName, 364 | body: tupleCtorFunc( 365 | `${types 366 | .map(type => `${deserializerNameFor(type, lookup)}(sink)`) 367 | .join(', ')}` 368 | ), 369 | dontExport: !shouldExport || undefined, 370 | params: [{ name: 'sink', type: BincodeLibTypes.Sink }] 371 | }); 372 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/src/serde/schema2serializers.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Scalar, 3 | SchemaElement, 4 | matchSchemaElement, 5 | Type, 6 | TypeTag, 7 | Variant, 8 | getVariantName, 9 | LookupName, 10 | createLookupName 11 | } from '../schema'; 12 | 13 | import { TsFileBlock, TsFileBlock as ts } from '../ts/ast'; 14 | import { variantPayloadTypeName } from '../ts/schema2ast'; 15 | import { 16 | BincodeLibTypes, 17 | traverseType, 18 | chainName, 19 | enumerateStructFields, 20 | RequiredImport, 21 | flatMap, 22 | ReadOrWrite, 23 | collectRequiredImports, 24 | TypeSerDe, 25 | CodePiece, 26 | schema2tsBlocks, 27 | SerDeCodeGenInput 28 | } from './sharedPieces'; 29 | // import { typeToString } from './schema2ast'; 30 | 31 | const WriteFuncs: ReadOrWrite = { 32 | [Scalar.Bool]: 'write_bool', 33 | [Scalar.Str]: 'write_str', 34 | [Scalar.F32]: 'write_f32', 35 | [Scalar.F64]: 'write_f64', 36 | [Scalar.U8]: 'write_u8', 37 | [Scalar.U16]: 'write_u16', 38 | [Scalar.U32]: 'write_u32', 39 | [Scalar.USIZE]: 'write_u64', 40 | [Scalar.I32]: 'write_i32', 41 | Opt: 'opt_writer', 42 | Seq: 'seq_writer', 43 | Nullable: 'nullable_writer' 44 | }; 45 | 46 | const serializerType = (typeStr: string) => 47 | `${BincodeLibTypes.Serializer}<${typeStr}>`; 48 | const enumMappingName = (enumName: string) => `${enumName}Map`; 49 | const serFuncName = (typeName: string) => `write${typeName}`; 50 | 51 | const serializerChainName = (types: Type[], lookup: LookupName): string => 52 | chainName(types, WriteFuncs, serFuncName, lookup); 53 | 54 | const serializerNameFor = (type: Type, lookup: LookupName): string => 55 | serializerChainName(traverseType(type), lookup); 56 | 57 | const { fromLibrary, fromTypesDeclaration } = RequiredImport; 58 | 59 | const entry2SerPiece = ( 60 | entryName: string, 61 | entry: SchemaElement, 62 | lookup: LookupName 63 | ): CodePiece => 64 | matchSchemaElement(entry, { 65 | Enum: ({ variants }): CodePiece => ({ 66 | requiredImports: [ 67 | fromTypesDeclaration(entryName), 68 | fromLibrary(WriteFuncs[Scalar.U32]) 69 | ], 70 | blocks: [ 71 | genEnumIndexMapping(entryName, variants), 72 | generateEnumSerializer(entryName) 73 | ], 74 | serdes: [], 75 | name: entryName 76 | }), 77 | 78 | Alias: (type): CodePiece => ({ 79 | requiredImports: [ 80 | fromTypesDeclaration(entryName), 81 | ...collectRequiredImports(type, WriteFuncs, lookup) 82 | ], 83 | serdes: generateTypesSerializers(type, lookup), 84 | blocks: [ 85 | ts.ConstVar({ 86 | name: serFuncName(entryName), 87 | type: serializerType(entryName), 88 | expression: serializerNameFor(type, lookup) 89 | }) 90 | ], 91 | dependsOn: [serializerNameFor(type, lookup)], 92 | name: entryName 93 | }), 94 | 95 | Newtype: (type): CodePiece => ({ 96 | requiredImports: [ 97 | fromTypesDeclaration(entryName), 98 | ...collectRequiredImports(type, WriteFuncs, lookup) 99 | ], 100 | serdes: generateTypesSerializers(type, lookup), 101 | blocks: [ 102 | ts.ConstVar({ 103 | name: serFuncName(entryName), 104 | type: serializerType(entryName), 105 | expression: serializerNameFor(type, lookup) 106 | }) 107 | ], 108 | dependsOn: [serializerNameFor(type, lookup)], 109 | name: entryName 110 | }), 111 | 112 | Tuple: (types): CodePiece => ({ 113 | requiredImports: [ 114 | fromTypesDeclaration(entryName), 115 | ...flatMap(types, t => collectRequiredImports(t, WriteFuncs, lookup)) 116 | ], 117 | blocks: [generateTupleSerializer(entryName, types, true, lookup)], 118 | serdes: flatMap(types, t => generateTypesSerializers(t, lookup)), 119 | dependsOn: types.map(t => serializerNameFor(t, lookup)), 120 | name: entryName 121 | }), 122 | 123 | Struct: (members): CodePiece => { 124 | const fields = enumerateStructFields(members); 125 | 126 | return { 127 | requiredImports: [ 128 | fromTypesDeclaration(entryName), 129 | ...flatMap(fields, f => 130 | collectRequiredImports(f.type, WriteFuncs, lookup) 131 | ) 132 | ], 133 | blocks: [generateStructSerializer(entryName, fields, true, lookup)], 134 | serdes: flatMap(fields, f => generateTypesSerializers(f.type, lookup)), 135 | dependsOn: fields.map(f => serializerNameFor(f.type, lookup)), 136 | name: entryName 137 | }; 138 | }, 139 | 140 | Union: (variants): CodePiece => ({ 141 | requiredImports: [ 142 | fromTypesDeclaration(entryName), 143 | fromLibrary(WriteFuncs[Scalar.U32]), 144 | ...flatMap( 145 | variants, 146 | Variant.match({ 147 | Unit: () => [] as RequiredImport[], 148 | NewType: (_, type) => 149 | collectRequiredImports(type, WriteFuncs, lookup), 150 | Struct: (variantName, members) => 151 | flatMap(enumerateStructFields(members), m => 152 | collectRequiredImports(m.type, WriteFuncs, lookup) 153 | ).concat( 154 | fromTypesDeclaration( 155 | variantPayloadTypeName(entryName, variantName) 156 | ) 157 | ), 158 | Tuple: (variantName, types) => 159 | flatMap(types, t => 160 | collectRequiredImports(t, WriteFuncs, lookup) 161 | ).concat( 162 | fromTypesDeclaration( 163 | variantPayloadTypeName(entryName, variantName) 164 | ) 165 | ) 166 | }) 167 | ) 168 | ], 169 | blocks: [ 170 | // genEnumIndexMapping(unionName, variants.map(getVariantName)), 171 | ...flatMap( 172 | variants, 173 | Variant.match({ 174 | Tuple: (variantName, types) => [ 175 | generateTupleSerializer( 176 | variantPayloadTypeName(entryName, variantName), 177 | types, 178 | false, 179 | lookup 180 | ) 181 | ], 182 | Struct: (variantName, members) => [ 183 | generateStructSerializer( 184 | variantPayloadTypeName(entryName, variantName), 185 | enumerateStructFields(members), 186 | false, 187 | lookup 188 | ) 189 | ], 190 | default: () => [] as TsFileBlock[] 191 | }) 192 | ), 193 | ts.ArrowFunc({ 194 | name: serFuncName(entryName), 195 | body: genUnionSerializers(entryName, variants, 'sink', lookup), 196 | returnType: BincodeLibTypes.Sink, 197 | params: [ 198 | { name: 'sink', type: BincodeLibTypes.Sink }, 199 | { name: 'val', type: entryName } 200 | ] 201 | }) 202 | ], 203 | serdes: flatMap( 204 | variants, 205 | Variant.match({ 206 | Unit: () => [] as TypeSerDe[], 207 | NewType: (_, type) => generateTypesSerializers(type, lookup), 208 | Struct: (_, members) => 209 | flatMap(enumerateStructFields(members), m => 210 | generateTypesSerializers(m.type, lookup) 211 | ), 212 | Tuple: (_, types) => 213 | flatMap(types, t => generateTypesSerializers(t, lookup)) 214 | }) 215 | ), 216 | dependsOn: flatMap( 217 | variants, 218 | Variant.match({ 219 | Unit: () => [] as string[], 220 | NewType: (_, type) => [serializerNameFor(type, lookup)], 221 | Struct: (_, members) => 222 | enumerateStructFields(members).map(m => 223 | serializerNameFor(m.type, lookup) 224 | ), 225 | Tuple: (_, types) => types.map(t => serializerNameFor(t, lookup)) 226 | }) 227 | ), 228 | name: entryName 229 | }) 230 | }); 231 | 232 | // aka write_u32(write_u32(sink, 0), val) 233 | const composeTypeSerializers = ( 234 | params: { name: string; type: Type }[], 235 | sinkArg: string, 236 | lookup: LookupName 237 | ): string => { 238 | if (params.length === 0) { 239 | return sinkArg; 240 | } 241 | 242 | const [{ name, type }, ...rest] = params; 243 | 244 | return composeTypeSerializers( 245 | rest, 246 | `${serializerNameFor(type, lookup)}(${sinkArg}, ${name})`, 247 | lookup 248 | ); 249 | }; 250 | 251 | export const schema2serializersAST = ({ 252 | schema, 253 | typesDeclarationFile, 254 | pathToBincodeLib 255 | }: SerDeCodeGenInput): TsFileBlock[] => { 256 | const lookup = createLookupName(schema); 257 | return schema2tsBlocks({ 258 | pieces: Object.entries(schema).map(([name, element]) => 259 | entry2SerPiece(name, element, lookup) 260 | ), 261 | serdeChainName: types => serializerChainName(types, lookup), 262 | lookup, 263 | serdeType: serializerType, 264 | serdeName: serFuncName, 265 | libImports: [BincodeLibTypes.Serializer], 266 | pathToBincodeLib, 267 | typesDeclarationFile, 268 | readOrWrite: WriteFuncs 269 | }); 270 | }; 271 | 272 | const genEnumIndexMapping = (enumName: string, variants: string[]) => 273 | ts.ConstVar({ 274 | name: enumMappingName(enumName), 275 | dontExport: true, 276 | expression: `{${variants.map((v, i) => ` ${v}: ${i}`).join(',\n')}}`, 277 | type: '{[key: string]:number}' 278 | }); 279 | 280 | const genUnionSerializers = ( 281 | unionName: string, 282 | variants: Variant[], 283 | sinkArg: string, 284 | lookup: LookupName 285 | ) => 286 | `{ 287 | switch (val.tag) { 288 | ${variants 289 | // .map(v => ({ 290 | // v, 291 | // tag: getVariantName(v), 292 | // sink: `${WriteScalar[Scalar.Str]}(${sinkArg}, "${getVariantName(v)}")` 293 | // })) 294 | .map((v, i) => ({ 295 | v, 296 | tag: getVariantName(v), 297 | sink: `${WriteFuncs[Scalar.U32]}(${sinkArg}, ${i})` 298 | })) 299 | .map(({ v, tag, sink }) => ({ 300 | exp: Variant.match(v, { 301 | Unit: () => sink, 302 | Struct: name => 303 | `${serFuncName( 304 | variantPayloadTypeName(unionName, name) 305 | )}(${sink}, val.value)`, 306 | NewType: (_, type) => 307 | `${serializerNameFor(type, lookup)}(${sink}, val.value)`, 308 | Tuple: name => 309 | `${serFuncName( 310 | variantPayloadTypeName(unionName, name) 311 | )}(${sink}, val.value)` 312 | }), 313 | tag 314 | })) 315 | .map(({ tag, exp }) => `case "${tag}": return ${exp};`) 316 | .join('\n')} 317 | }; 318 | }`; 319 | 320 | const generateTypesSerializers = ( 321 | type: Type, 322 | lookup: LookupName, 323 | descriptions: TypeSerDe[] = [] 324 | ): TypeSerDe[] => { 325 | switch (type.tag) { 326 | case TypeTag.Scalar: 327 | // skip scalars 328 | return descriptions; 329 | 330 | case TypeTag.Vec: 331 | case TypeTag.Nullable: 332 | case TypeTag.Option: 333 | return generateTypesSerializers(type.value, lookup, [ 334 | { 335 | typeChain: traverseType(type), 336 | toOrFrom: type, 337 | body: `${ 338 | type.tag === TypeTag.Option 339 | ? WriteFuncs.Opt 340 | : type.tag === TypeTag.Nullable 341 | ? WriteFuncs.Nullable 342 | : WriteFuncs.Seq 343 | }(${serializerChainName(traverseType(type.value), lookup)})` 344 | }, 345 | ...descriptions 346 | ]); 347 | } 348 | // skip direct type reference 349 | return descriptions; 350 | }; 351 | 352 | const generateEnumSerializer = (name: string): TsFileBlock => 353 | ts.ArrowFunc({ 354 | name: serFuncName(name), 355 | returnType: BincodeLibTypes.Sink, 356 | params: [ 357 | { name: 'sink', type: BincodeLibTypes.Sink }, 358 | { name: 'val', type: name } 359 | ], 360 | body: `${WriteFuncs[Scalar.U32]}(sink, ${enumMappingName(name)}[val])` 361 | }); 362 | 363 | const generateStructSerializer = ( 364 | name: string, 365 | fields: { name: string; type: Type }[], 366 | shouldExport: boolean, 367 | lookup: LookupName 368 | ): TsFileBlock => 369 | ts.ArrowFunc({ 370 | name: serFuncName(name), 371 | returnType: BincodeLibTypes.Sink, 372 | body: composeTypeSerializers(fields, 'sink', lookup), 373 | dontExport: !shouldExport || undefined, 374 | params: [ 375 | { name: 'sink', type: BincodeLibTypes.Sink }, 376 | { name: `{${fields.map(f => f.name).join(', ')}}`, type: name } 377 | ] 378 | }); 379 | 380 | const generateTupleSerializer = ( 381 | tupleName: string, 382 | types: Type[], 383 | shouldExport: boolean, 384 | lookup: LookupName 385 | ): TsFileBlock => 386 | ts.ArrowFunc({ 387 | name: serFuncName(tupleName), 388 | returnType: BincodeLibTypes.Sink, 389 | body: composeTypeSerializers( 390 | types.map((type, i) => ({ type, name: `val[${i}]` })), 391 | 'sink', 392 | lookup 393 | ), 394 | dontExport: !shouldExport || undefined, 395 | params: [ 396 | { name: 'sink', type: BincodeLibTypes.Sink }, 397 | { name: 'val', type: tupleName } 398 | ] 399 | }); 400 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/src/serde/sharedPieces.ts: -------------------------------------------------------------------------------- 1 | import { 2 | TypeTag, 3 | Type, 4 | Scalar, 5 | StructMembers, 6 | LookupName, 7 | isSchemaElement, 8 | Schema 9 | } from '../schema'; 10 | import { Union, of } from 'ts-union'; 11 | import { TsFileBlock, TsFileBlock as ts } from '../ts/ast'; 12 | import { findOrder } from './topologicalSort'; 13 | import { typeToString } from '../ts/schema2ast'; 14 | 15 | export type SerDeCodeGenInput = { 16 | schema: Schema; 17 | typesDeclarationFile: string; 18 | pathToBincodeLib?: string; 19 | }; 20 | 21 | export const enum BincodeLibTypes { 22 | Sink = 'Sink', 23 | Deserializer = 'Deserializer', 24 | Serializer = 'Serializer' 25 | } 26 | 27 | export const traverseType = (type: Type, parts: Type[] = []): Type[] => { 28 | switch (type.tag) { 29 | case TypeTag.Scalar: 30 | return parts.concat(type); 31 | 32 | case TypeTag.Vec: 33 | case TypeTag.Option: 34 | case TypeTag.Nullable: 35 | return traverseType(type.value, parts.concat(type)); 36 | } 37 | return parts.concat(type); 38 | }; 39 | 40 | const nameOfTopLevelTypeOnly = (type: Type, lookup: LookupName): string => { 41 | switch (type.tag) { 42 | case TypeTag.Scalar: 43 | return type.value; 44 | case TypeTag.Vec: 45 | return 'Vec'; 46 | case TypeTag.Option: 47 | return 'Opt'; 48 | case TypeTag.Nullable: 49 | return 'Nullable'; 50 | } 51 | return lookup(type); 52 | }; 53 | 54 | export type ReadOrWrite = { [K in Scalar]: string } & { 55 | Seq: string; 56 | Opt: string; 57 | Nullable: string; 58 | }; 59 | 60 | export const chainName = ( 61 | types: Type[], 62 | readOrWrite: ReadOrWrite, 63 | makeFuncName: (typeName: string) => string, 64 | lookup: LookupName 65 | ): string => { 66 | if (types.length === 1) { 67 | const type = types[0]; 68 | 69 | if (type.tag === TypeTag.Scalar) { 70 | return readOrWrite[type.value]; 71 | } else if (isSchemaElement(type)) { 72 | return `${makeFuncName(lookup(type))}`; 73 | } 74 | throw new Error('incomplete chain'); 75 | } 76 | 77 | return makeFuncName( 78 | types.map(t => nameOfTopLevelTypeOnly(t, lookup)).join('') 79 | ); 80 | }; 81 | 82 | export const enumerateStructFields = (members: StructMembers) => 83 | Object.keys(members).map(name => ({ 84 | name, 85 | type: members[name] 86 | })); 87 | 88 | export const RequiredImport = Union({ 89 | fromLibrary: of(), 90 | fromTypesDeclaration: of() 91 | }); 92 | 93 | export type RequiredImport = typeof RequiredImport.T; 94 | 95 | export const unique = (arr: T[], key: (el: T) => U): T[] => 96 | arr.reduce( 97 | ({ res, set }, el) => 98 | set.has(key(el)) 99 | ? { res, set } 100 | : { res: res.concat(el), set: set.add(key(el)) }, 101 | { res: [] as T[], set: new Set() } 102 | ).res; 103 | 104 | export const flatMap = (a: T[], f: (t: T) => U[]): U[] => 105 | a.reduce((s, el) => s.concat(f(el)), [] as U[]); 106 | 107 | const { fromLibrary, fromTypesDeclaration } = RequiredImport; 108 | 109 | export const collectRequiredImports = ( 110 | type: Type, 111 | readOrWrite: ReadOrWrite, 112 | lookup: LookupName, 113 | imports: RequiredImport[] = [] 114 | ): RequiredImport[] => { 115 | switch (type.tag) { 116 | case TypeTag.Scalar: 117 | return imports.concat(fromLibrary(readOrWrite[type.value])); 118 | case TypeTag.Vec: 119 | return imports.concat( 120 | fromLibrary(readOrWrite.Seq), 121 | ...collectRequiredImports(type.value, readOrWrite, lookup) 122 | ); 123 | case TypeTag.Option: 124 | return imports.concat( 125 | fromLibrary(readOrWrite.Opt), 126 | ...collectRequiredImports(type.value, readOrWrite, lookup) 127 | ); 128 | case TypeTag.Nullable: 129 | return imports.concat( 130 | fromLibrary(readOrWrite.Nullable), 131 | ...collectRequiredImports(type.value, readOrWrite, lookup) 132 | ); 133 | } 134 | return imports.concat(fromTypesDeclaration(lookup(type))); 135 | }; 136 | 137 | export type TypeSerDe = { 138 | typeChain: Type[]; 139 | body: string; 140 | toOrFrom: Type; 141 | }; 142 | 143 | export type CodePiece = { 144 | requiredImports: RequiredImport[]; 145 | serdes: TypeSerDe[]; 146 | blocks: TsFileBlock[]; 147 | dependsOn?: string[]; 148 | name: string; 149 | }; 150 | 151 | type PieceToSort = { 152 | dependsOn: string[]; 153 | blocks: TsFileBlock[]; 154 | id: string; 155 | }; 156 | 157 | const sortPiecesByDependencies = ( 158 | pieces: PieceToSort[], 159 | readOrWrite: ReadOrWrite 160 | ): PieceToSort[] => { 161 | const allFuncNames = pieces.map(p => p.id); 162 | 163 | const primitive = new Set(Object.values(readOrWrite)); 164 | 165 | // console.log({ primitiveSerializers }); 166 | 167 | const dependencyEdges = flatMap(pieces, p => 168 | p.dependsOn 169 | .filter(dep => !primitive.has(dep)) 170 | .map((dep): [string, string] => [p.id, dep]) 171 | ); 172 | 173 | const sorted = findOrder(allFuncNames, dependencyEdges); 174 | 175 | // console.log({ sorted }); 176 | // console.log({ pieces }); 177 | 178 | return sorted.map(i => pieces.find(p => p.id === i)!); 179 | }; 180 | 181 | export const schema2tsBlocks = ({ 182 | serdeName, 183 | serdeType, 184 | serdeChainName, 185 | readOrWrite, 186 | pieces, 187 | typesDeclarationFile, 188 | libImports, 189 | pathToBincodeLib = 'ts-binary', 190 | lookup 191 | }: { 192 | serdeName: (_: string) => string; 193 | serdeType: (_: string) => string; 194 | serdeChainName: (_: Type[]) => string; 195 | readOrWrite: ReadOrWrite; 196 | pieces: CodePiece[]; 197 | typesDeclarationFile: string; 198 | pathToBincodeLib?: string; 199 | libImports: string[]; 200 | lookup: LookupName; 201 | }): TsFileBlock[] => { 202 | // const pieces = entries.map(entry2SerPiece); 203 | 204 | // TODO cleanup 205 | const { lib, decl } = flatMap(pieces, p => p.requiredImports).reduce( 206 | ({ lib, decl }, imp) => 207 | RequiredImport.match(imp, { 208 | fromTypesDeclaration: s => ({ lib, decl: decl.concat(s) }), 209 | fromLibrary: s => ({ lib: lib.concat(s), decl }) 210 | }), 211 | { lib: [] as string[], decl: [] as string[] } 212 | ); 213 | 214 | const typeSerdesToSort: PieceToSort[] = unique( 215 | flatMap(pieces, p => p.serdes), 216 | s => serdeChainName(s.typeChain) 217 | ).map(({ typeChain, body, toOrFrom: fromType }) => { 218 | const name = serdeChainName(typeChain); 219 | const sourceTypeAsString = typeToString(fromType, lookup); 220 | 221 | const dependsOnTypes = typeChain.slice(1); 222 | const dependsOn = 223 | dependsOnTypes.length > 0 ? [serdeChainName(dependsOnTypes)] : []; 224 | 225 | return { 226 | id: name, 227 | dependsOn, 228 | 229 | blocks: [ 230 | ts.ConstVar({ 231 | name, 232 | expression: `${body}`, 233 | dontExport: true, 234 | type: serdeType(sourceTypeAsString) 235 | }) 236 | ] 237 | }; 238 | }); 239 | 240 | const codePiecesToSort: PieceToSort[] = pieces.map( 241 | ({ blocks, dependsOn = [], name }): PieceToSort => ({ 242 | blocks, 243 | dependsOn, 244 | id: serdeName(name) 245 | }) 246 | ); 247 | 248 | // console.log( 249 | // 'typeSerializersToSort', 250 | // // JSON.stringify( 251 | // typeSerializersToSort.map(({ dependsOn, id }) => ({ id, dependsOn })) 252 | // // ) 253 | // ); 254 | // console.log( 255 | // 'piecesToSort', 256 | // codePiecesToSort.map(({ dependsOn, id }) => ({ id, dependsOn })) 257 | // ); 258 | 259 | // console.log( 260 | // 'typeSerdesToSort', 261 | // typeSerdesToSort.map(({ dependsOn, id }) => ({ id, dependsOn })) 262 | // ); 263 | 264 | // console.log(JSON.stringify(codePiecesToSort)); 265 | 266 | return [ 267 | ts.Import({ names: unique(decl, s => s), from: typesDeclarationFile }), 268 | ts.Import({ 269 | names: unique( 270 | lib.concat(BincodeLibTypes.Sink).concat(libImports), 271 | s => s 272 | ), 273 | from: pathToBincodeLib 274 | }), 275 | 276 | ...flatMap( 277 | sortPiecesByDependencies( 278 | typeSerdesToSort.concat(codePiecesToSort), 279 | readOrWrite 280 | ), 281 | p => p.blocks 282 | ) 283 | ]; 284 | }; 285 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/src/serde/topologicalSort.ts: -------------------------------------------------------------------------------- 1 | // ### Inspired by: 2 | 3 | // https://leetcode.com/problems/course-schedule-ii/description/ 4 | 5 | // There are a total of n courses you have to take, labeled from 0 to n-1. 6 | 7 | // Some courses may have prerequisites, for example to take course 0 you have to first take course 1, which is expressed as a pair: [0,1] 8 | 9 | // Given the total number of courses and a list of prerequisite pairs, return the ordering of courses you should take to finish all courses. 10 | 11 | // There may be multiple correct orders, you just need to return one of them. If it is impossible to finish all courses, return an empty array. 12 | 13 | // Example 1: 14 | 15 | // Input: 2, [[1,0]] 16 | // Output: [0,1] 17 | // Explanation: There are a total of 2 courses to take. To take course 1 you should have finished 18 | // course 0. So the correct course order is [0,1] . 19 | 20 | // Example 2: 21 | 22 | // Input: 4, [[1,0],[2,0],[3,1],[3,2]] 23 | // Output: [0,1,2,3] or [0,2,1,3] 24 | // Explanation: There are a total of 4 courses to take. To take course 3 you should have finished both 25 | // courses 1 and 2. Both courses 1 and 2 should be taken after you finished course 0. 26 | // So one correct course order is [0,1,2,3]. Another correct ordering is [0,2,1,3] . 27 | 28 | // ### Note this is an adaptation 29 | type Edge = [string, string]; 30 | type DependencyMap = Map>; 31 | type Computation = { res: boolean; seq: Set }; 32 | export const findOrder = ( 33 | allElements: string[], 34 | dependencies: Edge[] 35 | ): string[] => { 36 | const courseMap = dependencies.reduce( 37 | (m, [toTake, required]) => 38 | m.set(toTake, (m.get(toTake) || new Set()).add(required)), 39 | allElements.reduce( 40 | (m, c) => m.set(c, new Set()), 41 | new Map>() 42 | ) 43 | ); 44 | 45 | return Array.from( 46 | Array.from(courseMap.keys()).reduce( 47 | aggregate(courseMap, new Set()), 48 | { res: true, seq: new Set() } 49 | ).seq 50 | ); 51 | }; 52 | 53 | const aggregate = (dependencyMap: DependencyMap, pending: Set) => ( 54 | { res, seq }: Computation, 55 | element: string 56 | ) => 57 | res 58 | ? seq.has(element) 59 | ? { res, seq } 60 | : completeElementTraversal(element, dependencyMap, pending, seq) 61 | : { res: false, seq: new Set() }; 62 | 63 | const completeElementTraversal = ( 64 | element: string, 65 | dependencyMap: DependencyMap, 66 | pending: Set, 67 | seq: Set 68 | ): Computation => { 69 | const deps = dependencyMap.get(element); 70 | if (!deps) { 71 | return { res: true, seq: seq.add(element) }; 72 | } 73 | 74 | if (pending.has(element)) { 75 | return { res: false, seq }; 76 | } 77 | 78 | pending.add(element); 79 | 80 | const { res, seq: s } = Array.from(deps.values()).reduce( 81 | aggregate(dependencyMap, pending), 82 | { res: true, seq } 83 | ); 84 | 85 | pending.delete(element); 86 | dependencyMap.delete(element); 87 | 88 | return { res, seq: s.add(element) }; 89 | }; 90 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/src/ts/ast.ts: -------------------------------------------------------------------------------- 1 | import { Union, of } from 'ts-union'; 2 | 3 | export module Code { 4 | export type StringEnum = { 5 | name: string; 6 | variants: [string, string][]; 7 | }; 8 | 9 | export type Field = { 10 | name: string; 11 | type: string; 12 | optional?: true; 13 | }; 14 | 15 | export type Interface = { 16 | name: string; 17 | fields: Field[]; 18 | }; 19 | 20 | export type Union = { 21 | name: string; 22 | tagField: string; 23 | valueField: string; 24 | variants: { tag: string; valueType?: string }[]; 25 | }; 26 | 27 | export type ArrowFunc = { 28 | name: string; 29 | params: Field[]; 30 | returnType?: string; 31 | wrappedInBraces?: true; 32 | body: string; 33 | dontExport?: true; 34 | }; 35 | 36 | export type Alias = { 37 | name: string; 38 | toType: string; 39 | }; 40 | 41 | export type ConstVar = { 42 | name: string; 43 | type?: string; 44 | dontExport?: true; 45 | expression: string; 46 | }; 47 | 48 | export type Import = { 49 | names: string[]; 50 | from: string; 51 | }; 52 | } 53 | 54 | export const TsFileBlock = Union({ 55 | StringEnum: of(), 56 | Interface: of(), 57 | Union: of(), 58 | LineComment: of(), 59 | ArrowFunc: of(), 60 | Alias: of(), 61 | ConstVar: of(), 62 | Import: of() 63 | }); 64 | 65 | export type TsFileBlock = typeof TsFileBlock.T; 66 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/src/ts/ast2ts.ts: -------------------------------------------------------------------------------- 1 | import { TsFileBlock, Code } from './ast'; 2 | import { FileBlock } from '../schema'; 3 | import { Module, isModule } from './schema2ast'; 4 | 5 | export const ast2ts = (tsBlocks: (TsFileBlock | Module)[]): FileBlock[] => 6 | tsBlocks.reduce( 7 | (blocks, tsBlock) => 8 | isModule(tsBlock) 9 | ? blocks 10 | .concat(startModule(tsBlock.name)) 11 | .concat(ast2ts(tsBlock.blocks)) 12 | .concat(endModule()) 13 | : blocks.concat( 14 | TsFileBlock.match(tsBlock, { 15 | StringEnum: genStringEnum, 16 | Interface: genInterface, 17 | ArrowFunc: genArrowFunc, 18 | Union: genUnion, 19 | Alias: genAlias, 20 | LineComment: comment => `// ${comment}`, 21 | ConstVar: genConstVariable, 22 | Import: genImport 23 | }) 24 | ), 25 | [] as FileBlock[] 26 | ); 27 | 28 | const startModule = (name: string): FileBlock => `export module ${name} { 29 | `; 30 | 31 | const endModule = (): FileBlock => ` 32 | }`; 33 | 34 | const genStringEnum = ({ name, variants }: Code.StringEnum): FileBlock => 35 | `export enum ${name} { 36 | ${variants.map(v => ` ${v[0]} = "${v[1]}"`).join(',\n')} 37 | } 38 | `; 39 | 40 | const genInterface = ({ 41 | name: interfaceName, 42 | fields 43 | }: Code.Interface): FileBlock => 44 | `export interface ${interfaceName} { 45 | ${fields 46 | .map( 47 | ({ name, type, optional }) => ` ${name}${optional ? '?' : ''}: ${type}` 48 | ) 49 | .join(';\n')} 50 | } 51 | `; 52 | 53 | const genArrowFunc = ({ 54 | name, 55 | params, 56 | returnType, 57 | wrappedInBraces, 58 | body, 59 | dontExport 60 | }: Code.ArrowFunc): FileBlock => 61 | `${dontExport ? '' : 'export '}const ${name} = (${params 62 | .map(({ name: n, type }) => `${n}: ${type}`) 63 | .join(', ')})${returnType ? `: ${returnType}` : ''} => ${ 64 | wrappedInBraces 65 | ? `{ 66 | ${body}; 67 | }` 68 | : `${body};` 69 | }`; 70 | 71 | const genUnion = ({ 72 | name: unionName, 73 | tagField, 74 | valueField, 75 | variants 76 | }: Code.Union): FileBlock => 77 | `export type ${unionName} = 78 | ${variants 79 | .map( 80 | ({ tag, valueType }) => 81 | ` | { ${tagField}: "${tag}"${ 82 | valueType ? `, ${valueField}: ${valueType}` : '' 83 | }}` 84 | ) 85 | .join('\n')} 86 | `; 87 | 88 | const genAlias = ({ name, toType }: Code.Alias): FileBlock => 89 | `export type ${name} = ${toType}`; 90 | 91 | const genConstVariable = ({ 92 | name, 93 | type, 94 | expression, 95 | dontExport 96 | }: Code.ConstVar): FileBlock => 97 | `${dontExport ? '' : 'export '}const ${name}${ 98 | type ? `: ${type}` : '' 99 | } = ${expression};`; 100 | 101 | const genImport = ({ names, from }: Code.Import): FileBlock => 102 | `import { ${names.join(', ')} } from "${from}";`; 103 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/src/ts/schema2ast.ts: -------------------------------------------------------------------------------- 1 | import { 2 | StructMembers, 3 | Scalar, 4 | Type, 5 | SchemaElement, 6 | TypeTag, 7 | Variant, 8 | getVariantName, 9 | matchSchemaElement, 10 | LookupName, 11 | createLookupName 12 | } from '../schema'; 13 | 14 | import { TsFileBlock, TsFileBlock as ts, Code } from './ast'; 15 | 16 | export type Module = { 17 | type: 'ts-file-module'; 18 | name: string; 19 | blocks: TsFileBlock[]; 20 | }; 21 | 22 | const Module = (name: string, blocks: TsFileBlock[]): Module => ({ 23 | name, 24 | blocks, 25 | type: 'ts-file-module' 26 | }); 27 | 28 | export const isModule = (block: Module | TsFileBlock): block is Module => 29 | (block as Module).type === 'ts-file-module'; 30 | 31 | const entryToBlocks = ( 32 | name: string, 33 | entry: SchemaElement, 34 | lookup: LookupName 35 | ): (TsFileBlock | Module)[] => 36 | matchSchemaElement(entry, { 37 | Alias: type => [aliasToAlias(name, type, lookup)], 38 | Struct: members => [structToInterface(name, members, lookup)], 39 | Enum: ({ variants }) => [enumToStringEnum(name, variants)], 40 | Union: variants => [ 41 | unionToTaggedUnion(name, variants, lookup), 42 | ...unionToPayloadInterfaces(name, variants, lookup), 43 | Module(name, unionToConstructors(name, variants, lookup)) 44 | ], 45 | Tuple: fields => [ 46 | tupleToInterface(name, fields, lookup), 47 | tupleToConstructor(name, fields, lookup) 48 | ], 49 | Newtype: type => [ 50 | newtypeToTypeAlias(name, type, lookup), 51 | newtypeToConstructor(name, type, lookup) 52 | ] 53 | }); 54 | 55 | export const schema2ast = (entries: { [name: string]: SchemaElement }) => 56 | Object.entries(entries).reduce( 57 | (blocks, [name, entry]) => 58 | blocks.concat(entryToBlocks(name, entry, createLookupName(entries))), 59 | [] as (TsFileBlock | Module)[] 60 | ); 61 | 62 | const aliasToAlias = ( 63 | name: string, 64 | type: Type, 65 | lookup: LookupName 66 | ): TsFileBlock => ts.Alias({ name, toType: typeToString(type, lookup) }); 67 | 68 | const enumToStringEnum = (name: string, variants: string[]): TsFileBlock => 69 | ts.StringEnum({ 70 | name, 71 | variants: variants.map((v): [string, string] => [v, v]) 72 | }); 73 | 74 | const unionToTaggedUnion = ( 75 | name: string, 76 | variants: Variant[], 77 | lookup: LookupName 78 | ): TsFileBlock => 79 | ts.Union({ 80 | name, 81 | tagField: 'tag', 82 | valueField: 'value', 83 | variants: variants.map(v => ({ 84 | tag: getVariantName(v), 85 | valueType: variantPayloadType(name, v, lookup) 86 | })) 87 | }); 88 | 89 | const newtypeToTypeAlias = ( 90 | name: string, 91 | type: Type, 92 | lookup: LookupName 93 | ): TsFileBlock => 94 | ts.Alias({ 95 | name, 96 | toType: newtypeToTypeStr(type, name, lookup) 97 | }); 98 | 99 | const newtypeToConstructor = ( 100 | name: string, 101 | type: Type, 102 | lookup: LookupName 103 | ): TsFileBlock => 104 | ts.ArrowFunc({ 105 | name, 106 | params: [ 107 | { 108 | name: 'val', 109 | type: typeToString(type, lookup) 110 | } 111 | ], 112 | body: `(val as any)`, 113 | returnType: newtypeToTypeStr(type, name, lookup) 114 | }); 115 | 116 | const unionToPayloadInterfaces = ( 117 | unionName: string, 118 | variants: Variant[], 119 | lookup: LookupName 120 | ): TsFileBlock[] => 121 | variants.reduce( 122 | (interfaces, v) => 123 | Variant.match(v, { 124 | Struct: (structName, members) => 125 | interfaces.concat( 126 | structToInterface( 127 | variantPayloadTypeName(unionName, structName), 128 | members, 129 | lookup 130 | ) 131 | ), 132 | Tuple: (tupleName, types) => 133 | interfaces.concat( 134 | tupleToInterface( 135 | variantPayloadTypeName(unionName, tupleName), 136 | types, 137 | lookup 138 | ) 139 | ), 140 | default: () => interfaces 141 | }), 142 | [] as TsFileBlock[] 143 | ); 144 | 145 | const unionToConstructors = ( 146 | unionName: string, 147 | variants: Variant[], 148 | lookup: LookupName 149 | ): TsFileBlock[] => 150 | variants.map(v => { 151 | const params = variantToCtorParameters(unionName, v, lookup); 152 | const name = getVariantName(v); 153 | return params.length > 0 154 | ? ts.ArrowFunc({ 155 | name, 156 | params, 157 | body: `(${variantToCtorBody(v)})`, 158 | returnType: unionName 159 | }) 160 | : ts.ConstVar({ 161 | name, 162 | type: unionName, 163 | expression: variantToCtorBody(v) 164 | }); 165 | }); 166 | 167 | const variantToCtorParameters = ( 168 | unionName: string, 169 | v: Variant, 170 | lookup: LookupName 171 | ): Code.Field[] => 172 | Variant.match(v, { 173 | Struct: name => [ 174 | { name: 'value', type: variantPayloadTypeName(unionName, name) } 175 | ], 176 | Unit: () => [], 177 | NewType: (_, type) => [{ name: 'value', type: typeToString(type, lookup) }], 178 | Tuple: (_, fields) => 179 | fields.map((f, i) => ({ name: `p${i}`, type: typeToString(f, lookup) })) 180 | }); 181 | 182 | const variantToCtorBody = Variant.match({ 183 | Struct: name => `{ tag: "${name}", value}`, 184 | Unit: name => `{ tag: "${name}"}`, 185 | NewType: name => `{ tag: "${name}", value}`, 186 | Tuple: (name, fields) => 187 | `{ tag: "${name}", value: [${fields.map((_, i) => `p${i}`)}]}` 188 | }); 189 | 190 | const variantPayloadType = ( 191 | unionName: string, 192 | v: Variant, 193 | lookup: LookupName 194 | ): string | undefined => 195 | Variant.match(v, { 196 | Struct: name => variantPayloadTypeName(unionName, name), 197 | Tuple: name => variantPayloadTypeName(unionName, name), 198 | Unit: () => undefined, 199 | NewType: (_, type) => typeToString(type, lookup) 200 | // Tuple: (_, fields) => `[${fields.map(typeToString).join(', ')}]` 201 | }); 202 | 203 | const structToInterface = ( 204 | name: string, 205 | members: StructMembers, 206 | lookup: LookupName 207 | ): TsFileBlock => 208 | ts.Interface({ 209 | name, 210 | fields: Object.keys(members).map( 211 | (name): Code.Field => ({ 212 | name, 213 | type: typeToString(members[name], lookup) 214 | }) 215 | ) 216 | }); 217 | 218 | const tupleToInterface = ( 219 | name: string, 220 | fields: Type[], 221 | lookup: LookupName 222 | ): TsFileBlock => 223 | ts.Interface({ 224 | name, 225 | fields: fields 226 | .map( 227 | (field, i): Code.Field => ({ 228 | name: i.toString(), 229 | type: typeToString(field, lookup) 230 | }) 231 | ) 232 | .concat({ name: 'length', type: fields.length.toString() }) 233 | }); 234 | 235 | const tupleToConstructor = ( 236 | name: string, 237 | fields: Type[], 238 | lookup: LookupName 239 | ): TsFileBlock => 240 | ts.ArrowFunc({ 241 | name, 242 | params: fields.map( 243 | (f, i): Code.Field => ({ 244 | name: 'p' + i.toString(), 245 | type: typeToString(f, lookup) 246 | }) 247 | ), 248 | body: `[${fields.map((_, i) => 'p' + i.toString()).join(', ')}]`, 249 | returnType: name 250 | }); 251 | 252 | const scalarToTypeString = (scalar: Scalar): string => { 253 | switch (scalar) { 254 | case Scalar.Bool: 255 | return 'boolean'; 256 | case Scalar.F32: 257 | case Scalar.F64: 258 | case Scalar.I32: 259 | case Scalar.U8: 260 | case Scalar.U16: 261 | case Scalar.U32: 262 | case Scalar.USIZE: 263 | return 'number'; 264 | case Scalar.Str: 265 | return 'string'; 266 | } 267 | }; 268 | 269 | export const typeToString = (type: Type, lookup: LookupName): string => { 270 | switch (type.tag) { 271 | case TypeTag.Option: 272 | return `(${typeToString(type.value, lookup)}) | undefined`; 273 | case TypeTag.Nullable: 274 | return `(${typeToString(type.value, lookup)}) | null`; 275 | case TypeTag.Scalar: 276 | return scalarToTypeString(type.value); 277 | case TypeTag.Vec: 278 | return `Array<${typeToString(type.value, lookup)}>`; 279 | } 280 | return lookup(type); 281 | }; 282 | 283 | const newtypeToTypeStr = ( 284 | type: Type, 285 | name: string, 286 | lookup: LookupName 287 | ): string => `${typeToString(type, lookup)} & { type: '${name}'}`; 288 | 289 | export const variantPayloadTypeName = ( 290 | unionName: string, 291 | variantName: string 292 | ): string => unionName + '_' + variantName; 293 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.build.json", 3 | 4 | "compilerOptions": { 5 | "outDir": "./dist", 6 | "rootDir": "./src", 7 | "composite": true 8 | }, 9 | 10 | "include": ["src/**/*"] 11 | } 12 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | 4 | "include": ["src/**/*"] 5 | } 6 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/types.rs: -------------------------------------------------------------------------------- 1 | 2 | #[derive(Deserialize, Serialize)] 3 | pub struct Shirt { 4 | pub size: Size, 5 | pub color: String, 6 | pub price: f32, 7 | } 8 | 9 | 10 | #[derive(Deserialize, Serialize)] 11 | pub enum Size { 12 | S, 13 | M, 14 | L, 15 | } 16 | -------------------------------------------------------------------------------- /packages/ts-rust-bridge-codegen/types.ts: -------------------------------------------------------------------------------- 1 | export interface Shirt { 2 | size: Size; 3 | color: string; 4 | price: number; 5 | } 6 | 7 | export enum Size { 8 | S = 'S', 9 | M = 'M', 10 | L = 'L' 11 | } 12 | -------------------------------------------------------------------------------- /packages/workspace.code-workspace: -------------------------------------------------------------------------------- 1 | { 2 | "folders": [ 3 | { 4 | "path": "ts-binary-types" 5 | }, 6 | { 7 | "path": "ts-binary" 8 | }, 9 | { 10 | "path": "ts-rust-bridge-codegen" 11 | }, 12 | { 13 | "path": "/Users/twop.sk/work/ts-rust-bridge" 14 | } 15 | ], 16 | "settings": { 17 | "workbench.colorCustomizations": { 18 | "activityBar.background": "#3f341a", 19 | "activityBar.activeBorder": "#2f715d", 20 | "activityBar.foreground": "#e7e7e7", 21 | "activityBar.inactiveForeground": "#e7e7e799", 22 | "activityBarBadge.background": "#2f715d", 23 | "activityBarBadge.foreground": "#e7e7e7", 24 | "titleBar.activeBackground": "#1b160b", 25 | "titleBar.inactiveBackground": "#1b160b99", 26 | "titleBar.activeForeground": "#e7e7e7", 27 | "titleBar.inactiveForeground": "#e7e7e799", 28 | "statusBar.background": "#1b160b", 29 | "statusBarItem.hoverBackground": "#3f341a", 30 | "statusBar.foreground": "#e7e7e7", 31 | "activityBar.activeBackground": "#3f341a", 32 | "statusBar.border": "#1b160b", 33 | "titleBar.border": "#1b160b" 34 | }, 35 | "peacock.color": "#1b160b" 36 | } 37 | } -------------------------------------------------------------------------------- /tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2019", 4 | "module": "esnext", 5 | "removeComments": false, 6 | "sourceMap": true, 7 | "moduleResolution": "node", 8 | "noUnusedLocals": true, 9 | "noUnusedParameters": true, 10 | "noImplicitAny": true, 11 | "noImplicitReturns": true, 12 | "declaration": true, 13 | "strictNullChecks": true, 14 | "strict": true 15 | }, 16 | "exclude": ["node_modules", "__tests__", "**/__tests__", "src/__tests__"] 17 | } 18 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.build.json", 3 | 4 | "compilerOptions": { 5 | "baseUrl": "./packages", 6 | "paths": { 7 | "ts-rust-bridge-codegen": ["ts-rust-bridge-codegen/src"], 8 | "ts-binary-types": ["ts-binary-types/src"], 9 | "ts-binary": ["ts-binary/src"] 10 | } 11 | } 12 | } 13 | --------------------------------------------------------------------------------