├── .github └── workflows │ └── push-checks.yml ├── .gitignore ├── .yarnrc.yml ├── README.md ├── eslint.config.js ├── jest.config.cjs ├── package.json ├── src ├── ascii.ts ├── benchmark │ ├── index.ts │ ├── tests │ │ ├── cbor.ts │ │ ├── common.ts │ │ ├── generated │ │ │ └── user.ts │ │ ├── json.ts │ │ ├── msgpackr.ts │ │ ├── protobuf.ts │ │ ├── sia-v1.ts │ │ ├── sia.ts │ │ └── user.proto │ └── ws │ │ ├── heavy │ │ ├── index.ts │ │ └── server.ts │ │ └── simple │ │ ├── index.ts │ │ └── server.ts ├── buffer.ts └── index.ts ├── tests └── sia.test.ts ├── tsconfig.json ├── types └── utfz.d.ts └── yarn.lock /.github/workflows/push-checks.yml: -------------------------------------------------------------------------------- 1 | name: Push Checks 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | lint-and-format: 7 | runs-on: ubuntu-latest 8 | 9 | strategy: 10 | matrix: 11 | node-version: [20.x, 18.x] 12 | 13 | steps: 14 | - uses: actions/checkout@v1 15 | 16 | - name: Use Node.js ${{ matrix.node-version }} 17 | uses: actions/setup-node@v1 18 | with: 19 | node-version: ${{ matrix.node-version }} 20 | 21 | - name: Install dependencies, lint, type-check, and test 22 | run: | 23 | corepack enable 24 | yarn install 25 | yarn lint 26 | yarn ts:check 27 | yarn test 28 | env: 29 | CI: true 30 | 31 | - name: Codecov Action 32 | uses: codecov/codecov-action@v1 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | processed.txt 3 | isolate-* 4 | .DS_Store 5 | *.cpuprofile 6 | *.ignore 7 | coverage 8 | dist 9 | .yarn 10 | -------------------------------------------------------------------------------- /.yarnrc.yml: -------------------------------------------------------------------------------- 1 | nodeLinker: node-modules 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Sia 2 | 3 | ![Build Status](https://github.com/TimeleapLabs/ts-sia/actions/workflows/push-checks.yml/badge.svg?branch=master) 4 | 5 | Sia serialization for JavaScript/TypeScript 6 | 7 | ## What is Sia? 8 | 9 | Sia is the serialization library used by [Timeleap](https://github.com/TimeleapLabs/timeleap). Check more details on [Sia's official documentation](https://timeleap.swiss/docs/products/sia). 10 | 11 | ## Installation 12 | 13 | ```bash 14 | npm install @timeleap/sia 15 | ``` 16 | 17 | ## Usage 18 | 19 | ```python 20 | import { Sia } from "@timeleap/sia"; 21 | 22 | const sia = new Sia(); 23 | 24 | sia 25 | .addString8("Hello") 26 | .addUint8(25) 27 | .addAscii("World"); 28 | 29 | console.log(sia.content); 30 | ``` 31 | -------------------------------------------------------------------------------- /eslint.config.js: -------------------------------------------------------------------------------- 1 | import js from "@eslint/js"; 2 | import typescript from "@typescript-eslint/eslint-plugin"; 3 | import tsParser from "@typescript-eslint/parser"; 4 | import prettier from "eslint-config-prettier"; 5 | import jest from "eslint-plugin-jest"; 6 | 7 | export default [ 8 | js.configs.recommended, 9 | { 10 | files: ["**/*.ts", "**/*.tsx"], 11 | languageOptions: { 12 | parser: tsParser, 13 | parserOptions: { 14 | ecmaVersion: "latest", 15 | sourceType: "module", 16 | project: "./tsconfig.json", 17 | }, 18 | globals: { 19 | console: true, 20 | Buffer: true, 21 | setTimeout: true, 22 | }, 23 | }, 24 | plugins: { 25 | "@typescript-eslint": typescript, 26 | }, 27 | rules: { 28 | ...typescript.configs.recommended.rules, 29 | "@typescript-eslint/no-explicit-any": "error", 30 | "@typescript-eslint/no-unused-vars": [ 31 | "error", 32 | { argsIgnorePattern: "^_" }, 33 | ], 34 | }, 35 | }, 36 | 37 | { 38 | files: ["**/*.test.js", "**/*.spec.js", "**/*.test.ts", "**/*.spec.ts"], 39 | plugins: { 40 | jest, 41 | }, 42 | languageOptions: { 43 | globals: jest.environments.globals.globals, 44 | }, 45 | rules: { 46 | "jest/no-disabled-tests": "warn", 47 | "jest/no-focused-tests": "error", 48 | "jest/no-identical-title": "error", 49 | "jest/prefer-to-have-length": "warn", 50 | "jest/valid-expect": "error", 51 | }, 52 | }, 53 | 54 | { 55 | ignores: [ 56 | "node_modules/**", 57 | "dist/**", 58 | "coverage/**", 59 | "*.js", 60 | "*.cjs", 61 | "*.mjs", 62 | ], 63 | }, 64 | prettier, 65 | ]; 66 | -------------------------------------------------------------------------------- /jest.config.cjs: -------------------------------------------------------------------------------- 1 | // jest.config.cjs 2 | 3 | const { tr } = require("@faker-js/faker"); 4 | 5 | /** @type {import('jest').Config} */ 6 | module.exports = { 7 | // ESM‐aware preset 8 | preset: "ts-jest/presets/default-esm", 9 | testEnvironment: "node", 10 | 11 | // will rewrite .js → .ts imports 12 | resolver: require.resolve("ts-jest-resolver"), 13 | 14 | // treat TS files as ESM ('.js' is inferred) 15 | extensionsToTreatAsEsm: [".ts", ".tsx"], 16 | 17 | // compile .ts, .tsx, and local .js via ts-jest in ESM mode 18 | transform: { 19 | "^.+\\.(ts|tsx|js)$": [ 20 | "ts-jest", 21 | { 22 | useESM: true, 23 | tsconfig: "tsconfig.json", 24 | isolatedModules: true, 25 | }, 26 | ], 27 | }, 28 | 29 | // these are the file extensions Jest will look for 30 | moduleFileExtensions: ["ts", "tsx", "js", "jsx", "json", "node"], 31 | 32 | // node_modules stay out of transforms 33 | transformIgnorePatterns: ["/node_modules/"], 34 | }; 35 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@timeleap/sia", 3 | "version": "2.2.0", 4 | "description": "", 5 | "main": "dist/index.js", 6 | "typings": "dist/index.d.ts", 7 | "scripts": { 8 | "build": "tsc", 9 | "test": "node --experimental-vm-modules node_modules/.bin/jest", 10 | "lint": "eslint .", 11 | "lint:fix": "eslint . --fix", 12 | "format": "prettier --write .", 13 | "ts:check": "tsc --noEmit", 14 | "prepublishOnly": "yarn build", 15 | "benchmark": "yarn build && yarn node dist/benchmark/index.js", 16 | "benchmark:ws:server": "yarn build && yarn node dist/benchmark/ws/simple/server.js", 17 | "benchmark:ws": "yarn build && yarn node dist/benchmark/ws/simple/index.js", 18 | "benchmark:ws:server:heavy": "yarn build && yarn node dist/benchmark/ws/heavy/server.js", 19 | "benchmark:ws:heavy": "yarn build && yarn node dist/benchmark/ws/heavy/index.js" 20 | }, 21 | "keywords": [], 22 | "author": "", 23 | "license": "ISC", 24 | "devDependencies": { 25 | "@eslint/js": "latest", 26 | "@faker-js/faker": "^9.2.0", 27 | "@types/jest": "^29.5.14", 28 | "@types/node": "^22.9.0", 29 | "@types/ws": "^8", 30 | "@typescript-eslint/eslint-plugin": "latest", 31 | "@typescript-eslint/parser": "latest", 32 | "cbor-x": "^1.6.0", 33 | "eslint": "latest", 34 | "eslint-config-prettier": "latest", 35 | "eslint-plugin-jest": "^28.11.0", 36 | "jest": "^29.7.0", 37 | "msgpackr": "^1.11.2", 38 | "prettier": "^3.4.2", 39 | "sializer": "0", 40 | "tinybench": "^3.0.6", 41 | "ts-jest": "^29.3.4", 42 | "ts-jest-resolver": "^2.0.1", 43 | "ts-proto": "^2.4.2", 44 | "typescript": "^5.4.3", 45 | "ws": "^8.18.0" 46 | }, 47 | "dependencies": { 48 | "utfz-lib": "^0.2.0" 49 | }, 50 | "packageManager": "yarn@4.4.1", 51 | "type": "module" 52 | } 53 | -------------------------------------------------------------------------------- /src/ascii.ts: -------------------------------------------------------------------------------- 1 | export const asciiToUint8Array = ( 2 | str: string, 3 | strLength: number, 4 | buffer: Uint8Array, 5 | offset: number, 6 | ) => { 7 | for (let i = 0; i < strLength; i++) { 8 | buffer[offset + i] = str.charCodeAt(i); 9 | } 10 | return strLength; 11 | }; 12 | 13 | const fns = new Array(66).fill(0).map((_, i) => { 14 | const codes = new Array(i) 15 | .fill(0) 16 | .map((_, j) => `buf[offset + ${j}]`) 17 | .join(", "); 18 | return new Function( 19 | "buf", 20 | "length", 21 | "offset", 22 | `return String.fromCharCode(${codes});`, 23 | ); 24 | }); 25 | 26 | export const uint8ArrayToAscii = ( 27 | buffer: Uint8Array, 28 | byteLength: number, 29 | offset: number, 30 | ) => { 31 | return fns[byteLength](buffer, byteLength, offset); 32 | }; 33 | -------------------------------------------------------------------------------- /src/benchmark/index.ts: -------------------------------------------------------------------------------- 1 | import { Bench } from "tinybench"; 2 | import { 3 | siaFiveThousandUsers, 4 | siaFiveThousandUsersDecode, 5 | } from "./tests/sia.js"; 6 | import { 7 | jsonFiveThousandUsers, 8 | jsonFiveThousandUsersDecode, 9 | } from "./tests/json.js"; 10 | import { 11 | cborFiveThousandUsers, 12 | cborFiveThousandUsersDecode, 13 | } from "./tests/cbor.js"; 14 | import { 15 | siaOneFiveThousandUsers, 16 | siaOneFiveThousandUsersDecode, 17 | } from "./tests/sia-v1.js"; 18 | import { 19 | msgpackrFiveThousandUsers, 20 | msgpackrFiveThousandUsersDecode, 21 | } from "./tests/msgpackr.js"; 22 | 23 | import { 24 | protobufFiveThousandUsers, 25 | protobufFiveThousandUsersDecode, 26 | } from "./tests/protobuf.js"; 27 | 28 | const bench = new Bench({ name: "serialization", time: 60 * 1000 }); 29 | 30 | bench.add("JSON", () => jsonFiveThousandUsers()); 31 | bench.add("Sializer", () => siaFiveThousandUsers()); 32 | bench.add("Sializer (v1)", () => siaOneFiveThousandUsers()); 33 | bench.add("CBOR-X", () => cborFiveThousandUsers()); 34 | bench.add("MsgPackr", () => msgpackrFiveThousandUsers()); 35 | bench.add("Protobuf", () => protobufFiveThousandUsers()); 36 | 37 | console.log(`Running ${bench.name} benchmark...`); 38 | await bench.run(); 39 | 40 | console.table(bench.table()); 41 | 42 | const deserializeBench = new Bench({ 43 | name: "deserialization", 44 | time: 60 * 1000, 45 | }); 46 | 47 | deserializeBench.add("JSON", () => jsonFiveThousandUsersDecode()); 48 | deserializeBench.add("Sializer", () => siaFiveThousandUsersDecode()); 49 | deserializeBench.add("Sializer (v1)", () => siaOneFiveThousandUsersDecode()); 50 | deserializeBench.add("CBOR-X", () => cborFiveThousandUsersDecode()); 51 | deserializeBench.add("MsgPackr", () => msgpackrFiveThousandUsersDecode()); 52 | deserializeBench.add("Protobuf", () => protobufFiveThousandUsersDecode()); 53 | 54 | console.log(`Running ${deserializeBench.name} benchmark...`); 55 | await deserializeBench.run(); 56 | 57 | console.table(deserializeBench.table()); 58 | 59 | console.log("Sia file size:", siaFiveThousandUsers().length); 60 | console.log("Sia v1 file size:", siaOneFiveThousandUsers().length); 61 | console.log("JSON file size:", jsonFiveThousandUsers().length); 62 | console.log("MsgPackr file size:", cborFiveThousandUsers().length); 63 | console.log("CBOR-X file size:", msgpackrFiveThousandUsers().length); 64 | console.log("Protobuf file size:", protobufFiveThousandUsers().length); 65 | -------------------------------------------------------------------------------- /src/benchmark/tests/cbor.ts: -------------------------------------------------------------------------------- 1 | import { fiveThousandUsers } from "./common.js"; 2 | import { encode, decode } from "cbor-x"; 3 | 4 | export const cborFiveThousandUsers = () => encode(fiveThousandUsers); 5 | 6 | const encoded = cborFiveThousandUsers(); 7 | 8 | export const cborFiveThousandUsersDecode = () => decode(encoded); 9 | -------------------------------------------------------------------------------- /src/benchmark/tests/common.ts: -------------------------------------------------------------------------------- 1 | import { faker } from "@faker-js/faker"; 2 | 3 | export function createRandomUser() { 4 | return { 5 | userId: faker.string.uuid(), 6 | username: faker.internet.username(), 7 | email: faker.internet.email(), 8 | avatar: faker.image.avatar(), 9 | password: faker.internet.password(), 10 | birthdate: faker.date.birthdate(), 11 | registeredAt: faker.date.past(), 12 | }; 13 | } 14 | 15 | export const fiveUsers = faker.helpers.multiple(createRandomUser, { 16 | count: 5, 17 | }); 18 | 19 | export const fiveHundredUsers = faker.helpers.multiple(createRandomUser, { 20 | count: 500, 21 | }); 22 | 23 | export const fiveThousandUsers = faker.helpers.multiple(createRandomUser, { 24 | count: 5_000, 25 | }); 26 | -------------------------------------------------------------------------------- /src/benchmark/tests/generated/user.ts: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-ts_proto. DO NOT EDIT. 2 | // versions: 3 | // protoc-gen-ts_proto v2.4.2 4 | // protoc v5.28.3 5 | // source: src/benchmark/tests/user.proto 6 | 7 | /* eslint-disable */ 8 | import { BinaryReader, BinaryWriter } from "@bufbuild/protobuf/wire"; 9 | 10 | export const protobufPackage = ""; 11 | 12 | export interface User { 13 | userId: string; 14 | username: string; 15 | email: string; 16 | avatar: string; 17 | password: string; 18 | birthdate: number; 19 | registeredAt: number; 20 | } 21 | 22 | export interface Users { 23 | users: User[]; 24 | } 25 | 26 | function createBaseUser(): User { 27 | return { 28 | userId: "", 29 | username: "", 30 | email: "", 31 | avatar: "", 32 | password: "", 33 | birthdate: 0, 34 | registeredAt: 0, 35 | }; 36 | } 37 | 38 | export const User: MessageFns = { 39 | encode( 40 | message: User, 41 | writer: BinaryWriter = new BinaryWriter(), 42 | ): BinaryWriter { 43 | if (message.userId !== "") { 44 | writer.uint32(10).string(message.userId); 45 | } 46 | if (message.username !== "") { 47 | writer.uint32(18).string(message.username); 48 | } 49 | if (message.email !== "") { 50 | writer.uint32(26).string(message.email); 51 | } 52 | if (message.avatar !== "") { 53 | writer.uint32(34).string(message.avatar); 54 | } 55 | if (message.password !== "") { 56 | writer.uint32(42).string(message.password); 57 | } 58 | if (message.birthdate !== 0) { 59 | writer.uint32(48).int64(message.birthdate); 60 | } 61 | if (message.registeredAt !== 0) { 62 | writer.uint32(56).int64(message.registeredAt); 63 | } 64 | return writer; 65 | }, 66 | 67 | decode(input: BinaryReader | Uint8Array, length?: number): User { 68 | const reader = 69 | input instanceof BinaryReader ? input : new BinaryReader(input); 70 | let end = length === undefined ? reader.len : reader.pos + length; 71 | const message = createBaseUser(); 72 | while (reader.pos < end) { 73 | const tag = reader.uint32(); 74 | switch (tag >>> 3) { 75 | case 1: { 76 | if (tag !== 10) { 77 | break; 78 | } 79 | 80 | message.userId = reader.string(); 81 | continue; 82 | } 83 | case 2: { 84 | if (tag !== 18) { 85 | break; 86 | } 87 | 88 | message.username = reader.string(); 89 | continue; 90 | } 91 | case 3: { 92 | if (tag !== 26) { 93 | break; 94 | } 95 | 96 | message.email = reader.string(); 97 | continue; 98 | } 99 | case 4: { 100 | if (tag !== 34) { 101 | break; 102 | } 103 | 104 | message.avatar = reader.string(); 105 | continue; 106 | } 107 | case 5: { 108 | if (tag !== 42) { 109 | break; 110 | } 111 | 112 | message.password = reader.string(); 113 | continue; 114 | } 115 | case 6: { 116 | if (tag !== 48) { 117 | break; 118 | } 119 | 120 | message.birthdate = longToNumber(reader.int64()); 121 | continue; 122 | } 123 | case 7: { 124 | if (tag !== 56) { 125 | break; 126 | } 127 | 128 | message.registeredAt = longToNumber(reader.int64()); 129 | continue; 130 | } 131 | } 132 | if ((tag & 7) === 4 || tag === 0) { 133 | break; 134 | } 135 | reader.skip(tag & 7); 136 | } 137 | return message; 138 | }, 139 | 140 | fromJSON(object: any): User { 141 | return { 142 | userId: isSet(object.userId) ? globalThis.String(object.userId) : "", 143 | username: isSet(object.username) 144 | ? globalThis.String(object.username) 145 | : "", 146 | email: isSet(object.email) ? globalThis.String(object.email) : "", 147 | avatar: isSet(object.avatar) ? globalThis.String(object.avatar) : "", 148 | password: isSet(object.password) 149 | ? globalThis.String(object.password) 150 | : "", 151 | birthdate: isSet(object.birthdate) 152 | ? globalThis.Number(object.birthdate) 153 | : 0, 154 | registeredAt: isSet(object.registeredAt) 155 | ? globalThis.Number(object.registeredAt) 156 | : 0, 157 | }; 158 | }, 159 | 160 | toJSON(message: User): unknown { 161 | const obj: any = {}; 162 | if (message.userId !== "") { 163 | obj.userId = message.userId; 164 | } 165 | if (message.username !== "") { 166 | obj.username = message.username; 167 | } 168 | if (message.email !== "") { 169 | obj.email = message.email; 170 | } 171 | if (message.avatar !== "") { 172 | obj.avatar = message.avatar; 173 | } 174 | if (message.password !== "") { 175 | obj.password = message.password; 176 | } 177 | if (message.birthdate !== 0) { 178 | obj.birthdate = Math.round(message.birthdate); 179 | } 180 | if (message.registeredAt !== 0) { 181 | obj.registeredAt = Math.round(message.registeredAt); 182 | } 183 | return obj; 184 | }, 185 | 186 | create, I>>(base?: I): User { 187 | return User.fromPartial(base ?? ({} as any)); 188 | }, 189 | fromPartial, I>>(object: I): User { 190 | const message = createBaseUser(); 191 | message.userId = object.userId ?? ""; 192 | message.username = object.username ?? ""; 193 | message.email = object.email ?? ""; 194 | message.avatar = object.avatar ?? ""; 195 | message.password = object.password ?? ""; 196 | message.birthdate = object.birthdate ?? 0; 197 | message.registeredAt = object.registeredAt ?? 0; 198 | return message; 199 | }, 200 | }; 201 | 202 | function createBaseUsers(): Users { 203 | return { users: [] }; 204 | } 205 | 206 | export const Users: MessageFns = { 207 | encode( 208 | message: Users, 209 | writer: BinaryWriter = new BinaryWriter(), 210 | ): BinaryWriter { 211 | for (const v of message.users) { 212 | User.encode(v!, writer.uint32(10).fork()).join(); 213 | } 214 | return writer; 215 | }, 216 | 217 | decode(input: BinaryReader | Uint8Array, length?: number): Users { 218 | const reader = 219 | input instanceof BinaryReader ? input : new BinaryReader(input); 220 | let end = length === undefined ? reader.len : reader.pos + length; 221 | const message = createBaseUsers(); 222 | while (reader.pos < end) { 223 | const tag = reader.uint32(); 224 | switch (tag >>> 3) { 225 | case 1: { 226 | if (tag !== 10) { 227 | break; 228 | } 229 | 230 | message.users.push(User.decode(reader, reader.uint32())); 231 | continue; 232 | } 233 | } 234 | if ((tag & 7) === 4 || tag === 0) { 235 | break; 236 | } 237 | reader.skip(tag & 7); 238 | } 239 | return message; 240 | }, 241 | 242 | fromJSON(object: any): Users { 243 | return { 244 | users: globalThis.Array.isArray(object?.users) 245 | ? object.users.map((e: any) => User.fromJSON(e)) 246 | : [], 247 | }; 248 | }, 249 | 250 | toJSON(message: Users): unknown { 251 | const obj: any = {}; 252 | if (message.users?.length) { 253 | obj.users = message.users.map((e) => User.toJSON(e)); 254 | } 255 | return obj; 256 | }, 257 | 258 | create, I>>(base?: I): Users { 259 | return Users.fromPartial(base ?? ({} as any)); 260 | }, 261 | fromPartial, I>>(object: I): Users { 262 | const message = createBaseUsers(); 263 | message.users = object.users?.map((e) => User.fromPartial(e)) || []; 264 | return message; 265 | }, 266 | }; 267 | 268 | type Builtin = 269 | | Date 270 | | Function 271 | | Uint8Array 272 | | string 273 | | number 274 | | boolean 275 | | undefined; 276 | 277 | export type DeepPartial = T extends Builtin 278 | ? T 279 | : T extends globalThis.Array 280 | ? globalThis.Array> 281 | : T extends ReadonlyArray 282 | ? ReadonlyArray> 283 | : T extends {} 284 | ? { [K in keyof T]?: DeepPartial } 285 | : Partial; 286 | 287 | type KeysOfUnion = T extends T ? keyof T : never; 288 | export type Exact = P extends Builtin 289 | ? P 290 | : P & { [K in keyof P]: Exact } & { 291 | [K in Exclude>]: never; 292 | }; 293 | 294 | function longToNumber(int64: { toString(): string }): number { 295 | const num = globalThis.Number(int64.toString()); 296 | if (num > globalThis.Number.MAX_SAFE_INTEGER) { 297 | throw new globalThis.Error("Value is larger than Number.MAX_SAFE_INTEGER"); 298 | } 299 | if (num < globalThis.Number.MIN_SAFE_INTEGER) { 300 | throw new globalThis.Error("Value is smaller than Number.MIN_SAFE_INTEGER"); 301 | } 302 | return num; 303 | } 304 | 305 | function isSet(value: any): boolean { 306 | return value !== null && value !== undefined; 307 | } 308 | 309 | export interface MessageFns { 310 | encode(message: T, writer?: BinaryWriter): BinaryWriter; 311 | decode(input: BinaryReader | Uint8Array, length?: number): T; 312 | fromJSON(object: any): T; 313 | toJSON(message: T): unknown; 314 | create, I>>(base?: I): T; 315 | fromPartial, I>>(object: I): T; 316 | } 317 | -------------------------------------------------------------------------------- /src/benchmark/tests/json.ts: -------------------------------------------------------------------------------- 1 | import { fiveThousandUsers } from "./common.js"; 2 | 3 | export const jsonFiveThousandUsers = () => 4 | Buffer.from(JSON.stringify(fiveThousandUsers)); 5 | 6 | const encoded = jsonFiveThousandUsers(); 7 | 8 | export const jsonFiveThousandUsersDecode = () => JSON.parse(encoded.toString()); 9 | -------------------------------------------------------------------------------- /src/benchmark/tests/msgpackr.ts: -------------------------------------------------------------------------------- 1 | import { fiveThousandUsers } from "./common.js"; 2 | import { pack, unpack } from "msgpackr"; 3 | 4 | export const msgpackrFiveThousandUsers = () => pack(fiveThousandUsers); 5 | 6 | const encoded = msgpackrFiveThousandUsers(); 7 | 8 | export const msgpackrFiveThousandUsersDecode = () => unpack(encoded); 9 | -------------------------------------------------------------------------------- /src/benchmark/tests/protobuf.ts: -------------------------------------------------------------------------------- 1 | import { fiveThousandUsers } from "./common.js"; 2 | import { Users } from "./generated/user.js"; 3 | 4 | export const protobufFiveThousandUsers = () => 5 | Users.encode({ 6 | users: fiveThousandUsers.map((user) => ({ 7 | ...user, 8 | birthdate: user.birthdate.getTime(), 9 | registeredAt: user.registeredAt.getTime(), 10 | })), 11 | }).finish(); 12 | 13 | const encoded = protobufFiveThousandUsers(); 14 | 15 | export const protobufFiveThousandUsersDecode = () => { 16 | const { users } = Users.decode(encoded); 17 | return users.map((user) => ({ 18 | ...user, 19 | birthdate: new Date(user.birthdate), 20 | registeredAt: new Date(user.registeredAt), 21 | })); 22 | }; 23 | -------------------------------------------------------------------------------- /src/benchmark/tests/sia-v1.ts: -------------------------------------------------------------------------------- 1 | import { fiveThousandUsers } from "./common.js"; 2 | import { sia, desia } from "sializer"; 3 | 4 | export const siaOneFiveThousandUsers = () => sia(fiveThousandUsers); 5 | 6 | const encoded = siaOneFiveThousandUsers(); 7 | 8 | export const siaOneFiveThousandUsersDecode = () => desia(encoded); 9 | -------------------------------------------------------------------------------- /src/benchmark/tests/sia.ts: -------------------------------------------------------------------------------- 1 | import { Sia } from "../../index.js"; 2 | import { fiveThousandUsers } from "./common.js"; 3 | 4 | const sia = new Sia(); 5 | 6 | export const siaFiveThousandUsers = () => 7 | sia 8 | .seek(0) 9 | .addArray16(fiveThousandUsers, (sia, user) => { 10 | sia 11 | .addAscii(user.userId) 12 | .addAscii(user.username) 13 | .addAscii(user.email) 14 | .addAscii(user.avatar) 15 | .addAscii(user.password) 16 | .addInt64(user.birthdate.valueOf()) 17 | .addInt64(user.registeredAt.valueOf()); 18 | }) 19 | .toUint8ArrayReference(); 20 | 21 | const encoded = siaFiveThousandUsers(); 22 | const desia = new Sia(encoded); 23 | 24 | const decodeUser = (sia: Sia) => ({ 25 | userId: sia.readAscii(), 26 | username: sia.readAscii(), 27 | email: sia.readAscii(), 28 | avatar: sia.readAscii(), 29 | password: sia.readAscii(), 30 | birthdate: new Date(sia.readInt64()), 31 | registeredAt: new Date(sia.readInt64()), 32 | }); 33 | 34 | export const siaFiveThousandUsersDecode = () => 35 | desia.seek(0).readArray16(decodeUser); 36 | -------------------------------------------------------------------------------- /src/benchmark/tests/user.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | message User { 4 | string userId = 1; 5 | string username = 2; 6 | string email = 3; 7 | string avatar = 4; 8 | string password = 5; 9 | int64 birthdate = 6; 10 | int64 registeredAt = 7; 11 | } 12 | 13 | message Users { 14 | repeated User users = 1; 15 | } 16 | -------------------------------------------------------------------------------- /src/benchmark/ws/heavy/index.ts: -------------------------------------------------------------------------------- 1 | import { Bench } from "tinybench"; 2 | import WebSocket from "ws"; 3 | import { Sia } from "../../../index.js"; 4 | import { pack, unpack } from "msgpackr"; 5 | import { decode, encode } from "cbor-x"; 6 | import { fiveHundredUsers } from "../../tests/common.js"; 7 | 8 | const sia = new Sia(); 9 | 10 | const rpcRequest = { 11 | method: "batchCalculateUserAges", 12 | params: fiveHundredUsers, 13 | }; 14 | 15 | export const payloads = { 16 | sia: () => 17 | sia 18 | .seek(0) 19 | .addAscii(rpcRequest.method) 20 | .addArray16(rpcRequest.params, (sia, user) => { 21 | sia 22 | .addAscii(user.userId) 23 | .addAscii(user.username) 24 | .addAscii(user.email) 25 | .addAscii(user.avatar) 26 | .addAscii(user.password) 27 | .addInt64(user.birthdate.getTime()) 28 | .addInt64(user.registeredAt.getTime()); 29 | }) 30 | .toUint8ArrayReference(), 31 | json: () => new Uint8Array(Buffer.from(JSON.stringify(rpcRequest))), 32 | cbor: () => new Uint8Array(encode(rpcRequest)), 33 | msgpack: () => new Uint8Array(pack(rpcRequest)), 34 | }; 35 | 36 | const clients = { 37 | sia: new WebSocket("ws://localhost:8080"), 38 | cbor: new WebSocket("ws://localhost:8081"), 39 | msgpack: new WebSocket("ws://localhost:8082"), 40 | json: new WebSocket("ws://localhost:8083"), 41 | }; 42 | 43 | const callbacks = { 44 | sia: (data: Buffer) => { 45 | return new Sia(new Uint8Array(data)).readArray16((sia) => { 46 | const userId = sia.readAscii(); 47 | const age = sia.readUInt8(); 48 | return { userId, age }; 49 | }); 50 | }, 51 | cbor: (data: Buffer) => decode(data), 52 | msgpack: (data: Buffer) => unpack(data), 53 | json: (data: Buffer) => JSON.parse(data.toString()), 54 | }; 55 | 56 | console.log("Waiting for connections..."); 57 | await new Promise((resolve) => setTimeout(resolve, 15 * 1000)); 58 | 59 | const bench = new Bench({ name: "RPC", time: 10 * 1000 }); 60 | 61 | const makeRpcCall = async ( 62 | ws: WebSocket, 63 | ondata: (data: Buffer) => void, 64 | payload: Uint8Array, 65 | ) => 66 | new Promise((resolve) => { 67 | ws.send(payload, { binary: true }); 68 | const done = (data: Buffer) => { 69 | ws.off("message", done); 70 | ondata(data); 71 | resolve(null); 72 | }; 73 | ws.on("message", done); 74 | }); 75 | 76 | bench 77 | .add( 78 | "JSON", 79 | async () => 80 | await makeRpcCall(clients.json, callbacks.json, payloads.json()), 81 | ) 82 | .addEventListener("complete", () => clients.json.close()); 83 | 84 | bench 85 | .add( 86 | "Sia", 87 | async () => await makeRpcCall(clients.sia, callbacks.sia, payloads.sia()), 88 | ) 89 | .addEventListener("complete", () => clients.sia.close()); 90 | 91 | bench 92 | .add( 93 | "CBOR", 94 | async () => 95 | await makeRpcCall(clients.cbor, callbacks.cbor, payloads.cbor()), 96 | ) 97 | .addEventListener("complete", () => clients.cbor.close()); 98 | 99 | bench 100 | .add( 101 | "MsgPack", 102 | async () => 103 | await makeRpcCall(clients.msgpack, callbacks.msgpack, payloads.msgpack()), 104 | ) 105 | .addEventListener("complete", () => clients.msgpack.close()); 106 | 107 | console.log(`Running ${bench.name} benchmark...`); 108 | await bench.run(); 109 | 110 | console.table(bench.table()); 111 | -------------------------------------------------------------------------------- /src/benchmark/ws/heavy/server.ts: -------------------------------------------------------------------------------- 1 | import { WebSocketServer } from "ws"; 2 | import { Sia } from "../../../index.js"; 3 | import { pack, unpack } from "msgpackr"; 4 | import { encode, decode } from "cbor-x"; 5 | 6 | const sia = new Sia(); 7 | 8 | type User = { userId: string; birthdate: Date }; 9 | 10 | const getUserAge = (user: User) => ({ 11 | userId: user.userId, 12 | age: new Date().getFullYear() - user.birthdate.getFullYear(), 13 | }); 14 | 15 | const BASE_PORT = 8080; 16 | 17 | console.log("Starting Sia WS server on port", BASE_PORT); 18 | const siaWss = new WebSocketServer({ port: BASE_PORT + 0 }); 19 | siaWss.on("connection", function connection(ws) { 20 | ws.on("error", console.error); 21 | ws.on("message", (data) => { 22 | // Read and skip method name 23 | sia.setContent(data as Buffer).readAscii(); 24 | const users = sia.readArray16((sia: Sia) => { 25 | const userId = sia.readAscii(); 26 | sia.readAscii(); // username 27 | sia.readAscii(); // email 28 | sia.readAscii(); // avatar 29 | sia.readAscii(); // password 30 | const birthdate = new Date(sia.readInt64()); 31 | sia.readInt64(); // registeredAt 32 | return { userId, birthdate }; 33 | }); 34 | const ages = users.map(getUserAge); 35 | const payload = sia 36 | .seek(0) 37 | .addArray16(ages, (sia: Sia, age) => { 38 | sia.addAscii(age.userId).addUInt8(age.age); 39 | }) 40 | .toUint8ArrayReference(); 41 | ws.send(payload, { binary: true }); 42 | }); 43 | }); 44 | 45 | console.log("Starting CBOR WS server on port", BASE_PORT + 1); 46 | const cborWss = new WebSocketServer({ port: BASE_PORT + 1 }); 47 | cborWss.on("connection", function connection(ws) { 48 | ws.on("error", console.error); 49 | ws.on("message", (data) => { 50 | const users = decode(data as Buffer).params; 51 | const ages = users.map(getUserAge); 52 | const payload = encode(ages); 53 | ws.send(payload, { binary: true }); 54 | }); 55 | }); 56 | 57 | console.log("Starting MsgPack WS server on port", BASE_PORT + 2); 58 | const msgpackWss = new WebSocketServer({ port: BASE_PORT + 2 }); 59 | msgpackWss.on("connection", function connection(ws) { 60 | ws.on("error", console.error); 61 | ws.on("message", (data) => { 62 | const users = unpack(data as Buffer).params; 63 | const ages = users.map(getUserAge); 64 | const payload = pack(ages); 65 | ws.send(payload, { binary: true }); 66 | }); 67 | }); 68 | 69 | console.log("Starting JSON WS server on port", BASE_PORT + 3); 70 | const jsonWss = new WebSocketServer({ port: BASE_PORT + 3 }); 71 | jsonWss.on("connection", function connection(ws) { 72 | ws.on("error", console.error); 73 | ws.on("message", (data) => { 74 | const users = JSON.parse(data.toString()).params; 75 | const ages = users 76 | .map((user: User) => ({ ...user, birthdate: new Date(user.birthdate) })) 77 | .map(getUserAge); 78 | const payload = new Uint8Array(Buffer.from(JSON.stringify(ages))); 79 | ws.send(payload); 80 | }); 81 | }); 82 | -------------------------------------------------------------------------------- /src/benchmark/ws/simple/index.ts: -------------------------------------------------------------------------------- 1 | import { Bench } from "tinybench"; 2 | import WebSocket from "ws"; 3 | import { Sia } from "../../../index.js"; 4 | import { pack } from "msgpackr"; 5 | import { encode } from "cbor-x"; 6 | 7 | const address = "0x1234567890123456789012345678901234567890"; 8 | const sia = new Sia(); 9 | 10 | const rpcRequest = { 11 | method: "getBalance", 12 | params: [address], 13 | }; 14 | 15 | export const payloads = { 16 | sia: () => 17 | sia 18 | .seek(0) 19 | .addAscii(rpcRequest.method) 20 | .addAscii(rpcRequest.params[0]) 21 | .toUint8ArrayReference(), 22 | json: () => new Uint8Array(Buffer.from(JSON.stringify(rpcRequest))), 23 | cbor: () => new Uint8Array(encode(rpcRequest)), 24 | msgpack: () => new Uint8Array(pack(rpcRequest)), 25 | }; 26 | 27 | const clients = { 28 | sia: new WebSocket("ws://localhost:8080"), 29 | cbor: new WebSocket("ws://localhost:8081"), 30 | msgpack: new WebSocket("ws://localhost:8082"), 31 | json: new WebSocket("ws://localhost:8083"), 32 | }; 33 | 34 | console.log("Waiting for connections..."); 35 | await new Promise((resolve) => setTimeout(resolve, 15 * 1000)); 36 | 37 | const bench = new Bench({ name: "RPC", time: 10 * 1000 }); 38 | 39 | const makeRpcCall = async (ws: WebSocket, payload: Uint8Array) => 40 | new Promise((resolve) => { 41 | ws.send(payload, { binary: true }); 42 | const done = () => { 43 | ws.off("message", done); 44 | resolve(null); 45 | }; 46 | ws.on("message", done); 47 | }); 48 | 49 | bench 50 | .add("JSON", async () => await makeRpcCall(clients.json, payloads.json())) 51 | .addEventListener("complete", () => clients.json.close()); 52 | 53 | bench 54 | .add("Sia", async () => await makeRpcCall(clients.sia, payloads.sia())) 55 | .addEventListener("complete", () => clients.sia.close()); 56 | 57 | bench 58 | .add("CBOR", async () => await makeRpcCall(clients.cbor, payloads.cbor())) 59 | .addEventListener("complete", () => clients.cbor.close()); 60 | 61 | bench 62 | .add( 63 | "MsgPack", 64 | async () => await makeRpcCall(clients.msgpack, payloads.msgpack()), 65 | ) 66 | .addEventListener("complete", () => clients.msgpack.close()); 67 | 68 | console.log(`Running ${bench.name} benchmark...`); 69 | await bench.run(); 70 | 71 | console.table(bench.table()); 72 | -------------------------------------------------------------------------------- /src/benchmark/ws/simple/server.ts: -------------------------------------------------------------------------------- 1 | import { WebSocketServer } from "ws"; 2 | import { Sia } from "../../../index.js"; 3 | import { pack, unpack } from "msgpackr"; 4 | import { encode, decode } from "cbor-x"; 5 | 6 | const address = "0x1234567890123456789012345678901234567890"; 7 | const map = new Map([[address, Number.MAX_SAFE_INTEGER]]); 8 | 9 | const getAccountBalance = (address: string) => map.get(address) || 0; 10 | const sia = new Sia(); 11 | 12 | const BASE_PORT = 8080; 13 | 14 | console.log("Starting Sia WS server on port", BASE_PORT); 15 | const siaWss = new WebSocketServer({ port: BASE_PORT + 0 }); 16 | siaWss.on("connection", function connection(ws) { 17 | ws.on("error", console.error); 18 | ws.on("message", (data) => { 19 | // Read and skip method name 20 | sia.setContent(data as Buffer).readAscii(); 21 | const address = sia.readAscii(); 22 | const balance = getAccountBalance(address); 23 | const payload = sia.seek(0).addInt64(balance).toUint8ArrayReference(); 24 | ws.send(payload, { binary: true }); 25 | }); 26 | }); 27 | 28 | console.log("Starting CBOR WS server on port", BASE_PORT + 1); 29 | const cborWss = new WebSocketServer({ port: BASE_PORT + 1 }); 30 | cborWss.on("connection", function connection(ws) { 31 | ws.on("error", console.error); 32 | ws.on("message", (data) => { 33 | const address = decode(data as Buffer).params[0]; 34 | const balance = getAccountBalance(address); 35 | const payload = encode({ balance }); 36 | ws.send(payload, { binary: true }); 37 | }); 38 | }); 39 | 40 | console.log("Starting MsgPack WS server on port", BASE_PORT + 2); 41 | const msgpackWss = new WebSocketServer({ port: BASE_PORT + 2 }); 42 | msgpackWss.on("connection", function connection(ws) { 43 | ws.on("error", console.error); 44 | ws.on("message", (data) => { 45 | const address = unpack(data as Buffer).params[0]; 46 | const balance = getAccountBalance(address); 47 | const payload = pack({ balance }); 48 | ws.send(payload, { binary: true }); 49 | }); 50 | }); 51 | 52 | console.log("Starting JSON WS server on port", BASE_PORT + 3); 53 | const jsonWss = new WebSocketServer({ port: BASE_PORT + 3 }); 54 | jsonWss.on("connection", function connection(ws) { 55 | ws.on("error", console.error); 56 | ws.on("message", (data) => { 57 | const address = JSON.parse(data.toString()).params[0]; 58 | const balance = getAccountBalance(address); 59 | const payload = new Uint8Array(Buffer.from(JSON.stringify({ balance }))); 60 | ws.send(payload); 61 | }); 62 | }); 63 | -------------------------------------------------------------------------------- /src/buffer.ts: -------------------------------------------------------------------------------- 1 | export class Buffer { 2 | public size: number; 3 | public content: Uint8Array; 4 | public offset: number; 5 | public dataView: DataView; 6 | 7 | constructor(uint8Array: Uint8Array) { 8 | this.size = uint8Array.length; 9 | this.content = uint8Array; 10 | this.offset = 0; 11 | this.dataView = new DataView(uint8Array.buffer); 12 | } 13 | 14 | static new(size: number = 32 * 1024 * 1024) { 15 | return new Buffer(new Uint8Array(size)); 16 | } 17 | 18 | seek(offset: number) { 19 | this.offset = offset; 20 | } 21 | 22 | skip(count: number) { 23 | this.offset += count; 24 | } 25 | 26 | add(data: Uint8Array) { 27 | if (this.offset + data.length > this.size) { 28 | throw new Error("Buffer overflow"); 29 | } 30 | this.content.set(data, this.offset); 31 | this.offset += data.length; 32 | } 33 | 34 | addOne(data: number) { 35 | if (this.offset + 1 > this.size) { 36 | throw new Error("Buffer overflow"); 37 | } 38 | this.content[this.offset] = data; 39 | this.offset++; 40 | } 41 | 42 | toUint8Array() { 43 | return this.content.slice(0, this.offset); 44 | } 45 | 46 | toUint8ArrayReference() { 47 | return this.content.subarray(0, this.offset); 48 | } 49 | 50 | slice(start: number, end: number) { 51 | return this.content.slice(start, end); 52 | } 53 | 54 | get(offset: number) { 55 | return this.content[offset]; 56 | } 57 | 58 | get length() { 59 | return this.size; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { Buffer } from "./buffer.js"; 2 | import { pack, unpack } from "utfz-lib"; 3 | import { asciiToUint8Array, uint8ArrayToAscii } from "./ascii.js"; 4 | 5 | const GLOBAL_SHARED_UNSAFE_BUFFER = { 6 | buffer: new Uint8Array(32 * 1024 * 1024), 7 | offset: 0, 8 | }; 9 | 10 | export class Sia extends Buffer { 11 | encoder: TextEncoder = new TextEncoder(); 12 | decoder: TextDecoder = new TextDecoder(); 13 | 14 | constructor(content?: Uint8Array) { 15 | super(content || GLOBAL_SHARED_UNSAFE_BUFFER.buffer); 16 | } 17 | 18 | static alloc(size: number): Sia { 19 | return new Sia(new Uint8Array(size)); 20 | } 21 | 22 | static allocUnsafe(size: number): Sia { 23 | const begin = 24 | GLOBAL_SHARED_UNSAFE_BUFFER.offset + size > 25 | GLOBAL_SHARED_UNSAFE_BUFFER.buffer.length 26 | ? 0 27 | : GLOBAL_SHARED_UNSAFE_BUFFER.offset; 28 | 29 | const subarray = GLOBAL_SHARED_UNSAFE_BUFFER.buffer.subarray( 30 | begin, 31 | begin + size, 32 | ); 33 | 34 | return new Sia(subarray); 35 | } 36 | 37 | seek(index: number): Sia { 38 | this.offset = index; 39 | return this; 40 | } 41 | 42 | skip(bytes: number): Sia { 43 | this.offset += bytes; 44 | return this; 45 | } 46 | 47 | setContent(uint8Array: Uint8Array): Sia { 48 | this.size = uint8Array.length; 49 | this.content = uint8Array; 50 | this.offset = 0; 51 | this.dataView = new DataView(uint8Array.buffer); 52 | return this; 53 | } 54 | 55 | embedSia(sia: Sia): Sia { 56 | // Add the content of the given Sia object to the current content 57 | this.add(sia.toUint8Array()); 58 | return this; 59 | } 60 | 61 | embedBytes(bytes: Uint8Array): Sia { 62 | // Add the given bytes to the current content 63 | this.add(bytes); 64 | return this; 65 | } 66 | 67 | addUInt8(n: number): Sia { 68 | // Add a single byte to the content 69 | this.addOne(n); 70 | return this; 71 | } 72 | 73 | readUInt8(): number { 74 | // Read a single byte from the current index 75 | if (this.offset >= this.length) { 76 | throw new Error("Not enough data to read uint8"); 77 | } 78 | return this.get(this.offset++); 79 | } 80 | 81 | addInt8(n: number): Sia { 82 | // Add a single signed byte to the content 83 | this.addOne(n); 84 | return this; 85 | } 86 | 87 | readInt8(): number { 88 | if (this.offset >= this.size) { 89 | throw new Error("Not enough data to read int8"); 90 | } 91 | const value = this.dataView.getInt8(this.offset); 92 | this.offset++; 93 | return value; 94 | } 95 | 96 | addUInt16(n: number): Sia { 97 | // Add a uint16 value to the content 98 | this.dataView.setUint16(this.offset, n, true); 99 | this.offset += 2; 100 | return this; 101 | } 102 | 103 | readUInt16(): number { 104 | // Read a uint16 value from the current index 105 | if (this.offset + 2 > this.content.length) { 106 | throw new Error("Not enough data to read uint16"); 107 | } 108 | const value = this.dataView.getUint16(this.offset, true); 109 | this.offset += 2; 110 | return value; 111 | } 112 | 113 | addInt16(n: number): Sia { 114 | // Add an int16 value to the content 115 | this.dataView.setInt16(this.offset, n, true); 116 | this.offset += 2; 117 | return this; 118 | } 119 | 120 | readInt16(): number { 121 | // Read an int16 value from the current index 122 | if (this.offset + 2 > this.content.length) { 123 | throw new Error("Not enough data to read int16"); 124 | } 125 | const value = this.dataView.getInt16(this.offset, true); 126 | this.offset += 2; 127 | return value; 128 | } 129 | 130 | addUInt32(n: number): Sia { 131 | // Add a uint32 value to the content 132 | this.dataView.setUint32(this.offset, n, true); 133 | this.offset += 4; 134 | return this; 135 | } 136 | 137 | readUInt32(): number { 138 | // Read a uint32 value from the current index 139 | if (this.offset + 4 > this.content.length) { 140 | throw new Error("Not enough data to read uint32"); 141 | } 142 | const value = this.dataView.getUint32(this.offset, true); 143 | this.offset += 4; 144 | return value; 145 | } 146 | 147 | addInt32(n: number): Sia { 148 | // Add an int32 value to the content 149 | this.dataView.setInt32(this.offset, n, true); 150 | this.offset += 4; 151 | return this; 152 | } 153 | 154 | readInt32(): number { 155 | // Read an int32 value from the current index 156 | if (this.offset + 4 > this.content.length) { 157 | throw new Error("Not enough data to read int32"); 158 | } 159 | const value = this.dataView.getInt32(this.offset, true); 160 | this.offset += 4; 161 | return value; 162 | } 163 | 164 | // Add a uint64 value to the content 165 | addUInt64(n: number): Sia { 166 | // Append the uint64 value at the end of the current content 167 | this.dataView.setUint32(this.offset, n & 0xffffffff, true); // Lower 32 bits 168 | this.dataView.setUint32(this.offset + 4, n / 0x100000000, true); // Upper 32 bits 169 | this.offset += 8; 170 | return this; 171 | } 172 | 173 | // Read a uint64 value from the current index 174 | readUInt64(): number { 175 | if (this.offset + 8 > this.content.length) { 176 | throw new Error("Not enough data to read uint64"); 177 | } 178 | // Read the uint64 value 179 | const lower = this.dataView.getUint32(this.offset, true); 180 | const upper = this.dataView.getUint32(this.offset + 4, true); 181 | this.offset += 8; 182 | // JavaScript does not support 64-bit integers natively, so this is an approximation 183 | return upper * 0x100000000 + lower; 184 | } 185 | 186 | // Add an int64 value to the content 187 | addInt64(n: number): Sia { 188 | const lower = n & 0xffffffff; // Extract lower 32 bits 189 | const upper = Math.floor(n / 0x100000000); // Extract upper 32 bits (signed) 190 | 191 | // Handle negative numbers correctly by ensuring the lower part wraps around 192 | this.dataView.setUint32(this.offset, lower >>> 0, true); // Lower as unsigned 193 | this.dataView.setInt32(this.offset + 4, upper, true); // Upper as signed 194 | this.offset += 8; 195 | return this; 196 | } 197 | 198 | // Read an int64 value from the current index 199 | readInt64(): number { 200 | if (this.offset + 8 > this.content.length) { 201 | throw new Error("Not enough data to read int64"); 202 | } 203 | // Read the int64 value 204 | const lower = this.dataView.getUint32(this.offset, true); 205 | const upper = this.dataView.getInt32(this.offset + 4, true); // Treat as signed for the upper part 206 | this.offset += 8; 207 | // Combine the upper and lower parts 208 | return upper * 0x100000000 + lower; 209 | } 210 | 211 | addUtfz(str: string): Sia { 212 | const lengthOffset = this.offset++; 213 | const length = pack(str, str.length, this.content, this.offset); 214 | this.content[lengthOffset] = length; 215 | this.offset += length; 216 | return this; 217 | } 218 | 219 | readUtfz(): string { 220 | const length = this.readUInt8(); 221 | const str = unpack(this.content, length, this.offset); 222 | this.offset += length; 223 | return str; 224 | } 225 | 226 | addAscii(str: string): Sia { 227 | const length = str.length; 228 | this.addUInt8(length); 229 | this.offset += asciiToUint8Array(str, length, this.content, this.offset); 230 | return this; 231 | } 232 | 233 | readAscii(): string { 234 | const length = this.readUInt8(); 235 | const str = uint8ArrayToAscii(this.content, length, this.offset); 236 | this.offset += length; 237 | return str; 238 | } 239 | 240 | addString8(str: string): Sia { 241 | const encodedString = this.encoder.encode(str); 242 | return this.addByteArray8(encodedString); 243 | } 244 | 245 | readString8(): string { 246 | const bytes = this.readByteArray8(true); 247 | return this.decoder.decode(bytes); 248 | } 249 | 250 | addString16(str: string): Sia { 251 | const encodedString = this.encoder.encode(str); 252 | return this.addByteArray16(encodedString); 253 | } 254 | 255 | readString16(): string { 256 | const bytes = this.readByteArray16(true); 257 | return this.decoder.decode(bytes); 258 | } 259 | 260 | addString32(str: string): Sia { 261 | const encodedString = this.encoder.encode(str); 262 | return this.addByteArray32(encodedString); 263 | } 264 | 265 | readString32(): string { 266 | const bytes = this.readByteArray32(true); 267 | return this.decoder.decode(bytes); 268 | } 269 | 270 | addString64(str: string): Sia { 271 | const encodedString = this.encoder.encode(str); 272 | return this.addByteArray64(encodedString); 273 | } 274 | 275 | readString64(): string { 276 | const bytes = this.readByteArray64(true); 277 | return this.decoder.decode(bytes); 278 | } 279 | 280 | addByteArrayN(bytes: Uint8Array): Sia { 281 | this.add(bytes); 282 | return this; 283 | } 284 | 285 | addByteArray8(bytes: Uint8Array): Sia { 286 | return this.addUInt8(bytes.length).addByteArrayN(bytes); 287 | } 288 | 289 | addByteArray16(bytes: Uint8Array): Sia { 290 | return this.addUInt16(bytes.length).addByteArrayN(bytes); 291 | } 292 | 293 | addByteArray32(bytes: Uint8Array): Sia { 294 | return this.addUInt32(bytes.length).addByteArrayN(bytes); 295 | } 296 | 297 | addByteArray64(bytes: Uint8Array): Sia { 298 | return this.addUInt64(bytes.length).addByteArrayN(bytes); 299 | } 300 | 301 | readByteArrayN(length: number, asReference = false): Uint8Array { 302 | if (this.offset + length > this.content.length) { 303 | throw new Error("Not enough data to read byte array"); 304 | } 305 | const bytes = asReference 306 | ? this.content.subarray(this.offset, this.offset + length) 307 | : this.content.slice(this.offset, this.offset + length); 308 | this.offset += length; 309 | return bytes; 310 | } 311 | 312 | readByteArray8(asReference = false): Uint8Array { 313 | const length = this.readUInt8(); 314 | return this.readByteArrayN(length, asReference); 315 | } 316 | 317 | readByteArray16(asReference = false): Uint8Array { 318 | const length = this.readUInt16(); 319 | return this.readByteArrayN(length, asReference); 320 | } 321 | 322 | readByteArray32(asReference = false): Uint8Array { 323 | const length = this.readUInt32(); 324 | return this.readByteArrayN(length, asReference); 325 | } 326 | 327 | readByteArray64(asReference = false): Uint8Array { 328 | const length = this.readUInt64(); 329 | return this.readByteArrayN(length, asReference); 330 | } 331 | 332 | addBool(b: boolean): Sia { 333 | const boolByte = b ? 1 : 0; 334 | this.addOne(boolByte); 335 | return this; 336 | } 337 | 338 | readBool(): boolean { 339 | return this.readUInt8() === 1; 340 | } 341 | 342 | addBigInt(n: bigint): Sia { 343 | // Convert BigInt to a byte array in a way that matches Go's encoding 344 | let hex = n.toString(16); 345 | // Ensure even length 346 | if (hex.length % 2 === 1) { 347 | hex = "0" + hex; 348 | } 349 | 350 | const length = hex.length / 2; 351 | const bytes = new Uint8Array(length); 352 | 353 | for (let i = 0; i < length; i++) { 354 | bytes[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16); 355 | } 356 | 357 | // Add length as a single byte (assuming the byte array is not longer than 255 for simplicity) 358 | if (length > 255) { 359 | throw new Error("BigInt too large for this simple implementation"); 360 | } 361 | 362 | return this.addByteArray8(bytes); // Reuse addByteArray to handle appending 363 | } 364 | 365 | readBigInt(): bigint { 366 | const bytes = this.readByteArray8(); // Reuse readByteArray to handle reading 367 | let hex = ""; 368 | 369 | bytes.forEach((byte) => { 370 | hex += byte.toString(16).padStart(2, "0"); 371 | }); 372 | 373 | return BigInt("0x" + hex); 374 | } 375 | 376 | private readArray(length: number, fn: (s: Sia) => T): T[] { 377 | const array = new Array(length); 378 | for (let i = 0; i < length; i++) { 379 | array[i] = fn(this); 380 | } 381 | return array; 382 | } 383 | 384 | addArray8(arr: T[], fn: (s: Sia, item: T) => void): Sia { 385 | this.addUInt8(arr.length); 386 | arr.forEach((item) => fn(this, item)); 387 | return this; 388 | } 389 | 390 | readArray8(fn: (s: Sia) => T): T[] { 391 | const length = this.readUInt8(); 392 | return this.readArray(length, fn); 393 | } 394 | 395 | addArray16(arr: T[], fn: (s: Sia, item: T) => void): Sia { 396 | this.addUInt16(arr.length); 397 | arr.forEach((item) => fn(this, item)); 398 | return this; 399 | } 400 | 401 | readArray16(fn: (s: Sia) => T): T[] { 402 | const length = this.readUInt16(); 403 | return this.readArray(length, fn); 404 | } 405 | 406 | addArray32(arr: T[], fn: (s: Sia, item: T) => void): Sia { 407 | this.addUInt32(arr.length); 408 | arr.forEach((item) => fn(this, item)); 409 | return this; 410 | } 411 | 412 | readArray32(fn: (s: Sia) => T): T[] { 413 | const length = this.readUInt32(); 414 | return this.readArray(length, fn); 415 | } 416 | 417 | addArray64(arr: T[], fn: (s: Sia, item: T) => void): Sia { 418 | this.addUInt64(arr.length); 419 | arr.forEach((item) => fn(this, item)); 420 | return this; 421 | } 422 | 423 | readArray64(fn: (s: Sia) => T): T[] { 424 | const length = this.readUInt64(); 425 | return this.readArray(length, fn); 426 | } 427 | } 428 | -------------------------------------------------------------------------------- /tests/sia.test.ts: -------------------------------------------------------------------------------- 1 | import { Sia } from "../src/index.js"; 2 | 3 | describe("Sia - Integer (Signed & Unsigned)", () => { 4 | describe("8-bit", () => { 5 | it("should correctly add and read a uint8 value", () => { 6 | const sia = new Sia(); 7 | sia.addUInt8(255); 8 | sia.seek(0); 9 | expect(sia.readUInt8()).toBe(255); 10 | }); 11 | 12 | it("should correctly add and read an int8 value", () => { 13 | const sia = new Sia(); 14 | sia.addInt8(-120); 15 | sia.seek(0); 16 | expect(sia.readInt8()).toBe(-120); 17 | }); 18 | 19 | it("should truncate values outside the int8 range (-128 to 127)", () => { 20 | const sia = new Sia(); 21 | sia.addInt8(-129); 22 | sia.seek(0); 23 | expect(sia.readInt8()).toBe(127); 24 | }); 25 | }); 26 | 27 | describe("16-bit", () => { 28 | it("should correctly add and read a uint16 value", () => { 29 | const sia = new Sia(); 30 | sia.addUInt16(65535); 31 | sia.seek(0); 32 | expect(sia.readUInt16()).toBe(65535); 33 | }); 34 | 35 | it("should correctly add and read an int16 value", () => { 36 | const sia = new Sia(); 37 | sia.addInt16(-32768); 38 | sia.seek(0); 39 | expect(sia.readInt16()).toBe(-32768); 40 | }); 41 | }); 42 | 43 | describe("32-bit", () => { 44 | it("should correctly add and read a uint32 value", () => { 45 | const sia = new Sia(); 46 | sia.addUInt32(4294967295); 47 | sia.seek(0); 48 | expect(sia.readUInt32()).toBe(4294967295); 49 | }); 50 | 51 | it("should correctly add and read an int32 value", () => { 52 | const sia = new Sia(); 53 | sia.addInt32(-2147483648); 54 | sia.seek(0); 55 | expect(sia.readInt32()).toBe(-2147483648); 56 | }); 57 | }); 58 | 59 | describe("64-bit", () => { 60 | it("should correctly add and read a uint64 value", () => { 61 | const sia = new Sia(); 62 | const value = Number.MAX_SAFE_INTEGER; // 2^53 - 1 63 | sia.addUInt64(value); 64 | sia.seek(0); 65 | expect(sia.readUInt64()).toBe(value); 66 | }); 67 | 68 | it("should correctly add and read an int64 value", () => { 69 | const sia = new Sia(); 70 | const value = Number.MIN_SAFE_INTEGER; // -(2^53 - 1) 71 | sia.addInt64(value); 72 | sia.seek(0); 73 | expect(sia.readInt64()).toBe(value); 74 | }); 75 | }); 76 | }); 77 | 78 | describe("Sia - Boolean", () => { 79 | it("should correctly add and read a true boolean value", () => { 80 | const sia = new Sia(); 81 | sia.addBool(true); 82 | sia.seek(0); 83 | expect(sia.readBool()).toBe(true); 84 | }); 85 | 86 | it("should correctly add and read a false boolean value", () => { 87 | const sia = new Sia(); 88 | sia.addBool(false); 89 | sia.seek(0); 90 | expect(sia.readBool()).toBe(false); 91 | }); 92 | }); 93 | 94 | describe("Sia - UTFZ String", () => { 95 | it("should correctly add and read a UTFZ string", () => { 96 | const sia = new Sia(); 97 | const testString = "Hello, UTFZ!"; 98 | sia.addUtfz(testString); 99 | sia.seek(0); 100 | expect(sia.readUtfz()).toBe(testString); 101 | }); 102 | }); 103 | 104 | describe("Sia - ASCII String", () => { 105 | it("should correctly add and read an ASCII string", () => { 106 | const sia = new Sia(); 107 | const testString = "Hello, ASCII!"; 108 | sia.addAscii(testString); 109 | sia.seek(0); 110 | expect(sia.readAscii()).toBe(testString); 111 | }); 112 | }); 113 | 114 | describe("Sia - String", () => { 115 | it("should correctly add and read a String8 value", () => { 116 | const sia = new Sia(); 117 | const testString = "Hello, String8!"; 118 | sia.addString8(testString); 119 | sia.seek(0); 120 | expect(sia.readString8()).toBe(testString); 121 | }); 122 | it("should correctly add and read a String16 value", () => { 123 | const sia = new Sia(); 124 | const testString = "Hello, String16!"; 125 | sia.addString16(testString); 126 | sia.seek(0); 127 | expect(sia.readString16()).toBe(testString); 128 | }); 129 | it("should correctly add and read a String32 value", () => { 130 | const sia = new Sia(); 131 | const testString = "Hello, String32!"; 132 | sia.addString32(testString); 133 | sia.seek(0); 134 | expect(sia.readString32()).toBe(testString); 135 | }); 136 | it("should correctly add and read a String64 value", () => { 137 | const sia = new Sia(); 138 | const testString = "Hello, String64!"; 139 | sia.addString64(testString); 140 | sia.seek(0); 141 | expect(sia.readString64()).toBe(testString); 142 | }); 143 | }); 144 | 145 | describe("Sia - BigInt", () => { 146 | it("should correctly add and read a BigInt value", () => { 147 | const sia = new Sia(); 148 | const bigIntValue = BigInt("123456789012345678901234567890"); 149 | sia.addBigInt(bigIntValue); 150 | sia.seek(0); 151 | expect(sia.readBigInt()).toBe(bigIntValue); 152 | }); 153 | 154 | it("should throw an error when adding a BigInt larger than 255 bytes", () => { 155 | const sia = new Sia(); 156 | const largeBigInt = BigInt("0x" + "ff".repeat(256)); 157 | expect(() => sia.addBigInt(largeBigInt)).toThrow( 158 | "BigInt too large for this simple implementation", 159 | ); 160 | }); 161 | }); 162 | 163 | describe("Sia - ByteArray", () => { 164 | it("should correctly add and read a ByteArray8 value", () => { 165 | const sia = new Sia(); 166 | const byteArray = new Uint8Array([1, 2, 3, 4, 5]); 167 | sia.addByteArray8(byteArray); 168 | sia.seek(0); 169 | const result = sia.readByteArray8(); 170 | expect(result).toEqual(byteArray); 171 | }); 172 | 173 | it("should correctly add and read a ByteArray16 value", () => { 174 | const sia = new Sia(); 175 | const byteArray = new Uint8Array([10, 20, 30, 40, 50]); 176 | sia.addByteArray16(byteArray); 177 | sia.seek(0); 178 | const result = sia.readByteArray16(); 179 | expect(result).toEqual(byteArray); 180 | }); 181 | 182 | it("should correctly add and read a ByteArray32 value", () => { 183 | const sia = new Sia(); 184 | const byteArray = new Uint8Array(100).map((_, i) => i); 185 | sia.addByteArray32(byteArray); 186 | sia.seek(0); 187 | const result = sia.readByteArray32(); 188 | expect(result).toEqual(byteArray); 189 | }); 190 | 191 | it("should correctly add and read a ByteArray64 value", () => { 192 | const sia = new Sia(); 193 | const byteArray = new Uint8Array(50).map((_, i) => (i * 2) % 256); 194 | sia.addByteArray64(byteArray); 195 | sia.seek(0); 196 | const result = sia.readByteArray64(); 197 | expect(result).toEqual(byteArray); 198 | }); 199 | }); 200 | 201 | describe("Sia - Array tests", () => { 202 | const writeNumber = (sia: Sia, n: number) => sia.addUInt8(n); 203 | const readNumber = (sia: Sia): number => sia.readUInt8(); 204 | 205 | it("should correctly add and read array with addArray8/readArray8", () => { 206 | const sia = new Sia(); 207 | const arr = [1, 2, 3, 255]; 208 | sia.addArray8(arr, writeNumber); 209 | sia.seek(0); 210 | const result = sia.readArray8(readNumber); 211 | expect(result).toEqual(arr); 212 | }); 213 | 214 | it("should correctly add and read array with addArray16/readArray16", () => { 215 | const sia = new Sia(); 216 | const arr = [1000, 2000, 3000, 65535]; 217 | const writeNum16 = (sia: Sia, n: number) => sia.addUInt16(n); 218 | const readNum16 = (sia: Sia) => sia.readUInt16(); 219 | 220 | sia.addArray16(arr, writeNum16); 221 | sia.seek(0); 222 | const result = sia.readArray16(readNum16); 223 | expect(result).toEqual(arr); 224 | }); 225 | 226 | it("should correctly add and read array with addArray32/readArray32", () => { 227 | const sia = new Sia(); 228 | const arr = [100000, 200000, 300000, 4294967295]; 229 | const writeNum32 = (sia: Sia, n: number) => sia.addUInt32(n); 230 | const readNum32 = (sia: Sia) => sia.readUInt32(); 231 | 232 | sia.addArray32(arr, writeNum32); 233 | sia.seek(0); 234 | const result = sia.readArray32(readNum32); 235 | expect(result).toEqual(arr); 236 | }); 237 | 238 | it("should correctly add and read array with addArray64/readArray64", () => { 239 | const sia = new Sia(); 240 | const arr = [9007199254740991, 1234567890123456]; 241 | const writeNum64 = (sia: Sia, n: number) => sia.addUInt64(n); 242 | const readNum64 = (sia: Sia) => sia.readUInt64(); 243 | 244 | sia.addArray64(arr, writeNum64); 245 | sia.seek(0); 246 | const result = sia.readArray64(readNum64); 247 | expect(result).toEqual(arr); 248 | }); 249 | }); 250 | 251 | describe("Sia read methods - insufficient data error tests", () => { 252 | it("should throw an error when reading int8 with insufficient data", () => { 253 | const smallBuffer = new Uint8Array(1); 254 | const sia = new Sia(smallBuffer); 255 | sia.seek(1); // Move offset to the end 256 | expect(() => sia.readInt8()).toThrow("Not enough data to read int8"); 257 | }); 258 | 259 | it("should throw an error when reading uint8 with insufficient data", () => { 260 | const smallBuffer = new Uint8Array(1); 261 | const sia = new Sia(smallBuffer); 262 | sia.seek(1); // Move offset to the end 263 | expect(() => sia.readUInt8()).toThrow("Not enough data to read uint8"); 264 | }); 265 | 266 | it("should throw an error when reading uint16 with insufficient data", () => { 267 | const smallBuffer = new Uint8Array(1); // less than 2 bytes needed 268 | const sia = new Sia(smallBuffer); 269 | sia.seek(0); 270 | expect(() => sia.readUInt16()).toThrow("Not enough data to read uint16"); 271 | }); 272 | 273 | it("should throw an error when reading int16 with insufficient data", () => { 274 | const smallBuffer = new Uint8Array(1); 275 | const sia = new Sia(smallBuffer); 276 | sia.seek(0); 277 | expect(() => sia.readInt16()).toThrow("Not enough data to read int16"); 278 | }); 279 | 280 | it("should throw an error when reading uint32 with insufficient data", () => { 281 | const smallBuffer = new Uint8Array(3); // less than 4 bytes 282 | const sia = new Sia(smallBuffer); 283 | sia.seek(0); 284 | expect(() => sia.readUInt32()).toThrow("Not enough data to read uint32"); 285 | }); 286 | 287 | it("should throw an error when reading int32 with insufficient data", () => { 288 | const smallBuffer = new Uint8Array(3); 289 | const sia = new Sia(smallBuffer); 290 | sia.seek(0); 291 | expect(() => sia.readInt32()).toThrow("Not enough data to read int32"); 292 | }); 293 | 294 | it("should throw an error when reading uint64 with insufficient data", () => { 295 | const smallBuffer = new Uint8Array(7); // less than 8 bytes 296 | const sia = new Sia(smallBuffer); 297 | sia.seek(0); 298 | expect(() => sia.readUInt64()).toThrow("Not enough data to read uint64"); 299 | }); 300 | 301 | it("should throw an error when reading int64 with insufficient data", () => { 302 | const smallBuffer = new Uint8Array(7); 303 | const sia = new Sia(smallBuffer); 304 | sia.seek(0); 305 | expect(() => sia.readInt64()).toThrow("Not enough data to read int64"); 306 | }); 307 | 308 | it("should throw an error when reading byte array with insufficient data", () => { 309 | const smallBuffer = new Uint8Array(3); 310 | const sia = new Sia(smallBuffer); 311 | sia.seek(0); 312 | expect(() => sia.readByteArrayN(4)).toThrow( 313 | "Not enough data to read byte array", 314 | ); 315 | }); 316 | }); 317 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "module": "Node16", 5 | "moduleResolution": "Node16", 6 | "strict": true, 7 | "esModuleInterop": true, 8 | "skipLibCheck": true, 9 | "forceConsistentCasingInFileNames": true, 10 | "outDir": "dist", 11 | "typeRoots": ["./node_modules/@types", "./src/types"], 12 | "declaration": true 13 | }, 14 | "$schema": "https://json.schemastore.org/tsconfig", 15 | "display": "Recommended" 16 | } 17 | -------------------------------------------------------------------------------- /types/utfz.d.ts: -------------------------------------------------------------------------------- 1 | declare module "utfz-lib" { 2 | export function pack( 3 | value: string, 4 | length: number, 5 | buffer: Uint8Array, 6 | offset: number, 7 | ): number; 8 | 9 | export function unpack( 10 | buffer: Uint8Array, 11 | byteLength: number, 12 | offset: number, 13 | ): string; 14 | } 15 | --------------------------------------------------------------------------------