├── .gitignore ├── .gitmodules ├── LICENSE ├── README.md ├── dev ├── .gitignore ├── docker-compose.yml ├── healthz.ts └── package.json ├── k_yrs_go.code-workspace ├── latencies.png ├── package-lock.json ├── package.json ├── port.sh ├── server ├── .air.toml ├── .env ├── .gitignore ├── db │ ├── .gitignore │ └── db.go ├── go.mod ├── go.sum ├── healthz.ts ├── main.go ├── package.json ├── server.sh └── setup_ffi.sh ├── system_config.png ├── test ├── .env ├── package.json ├── psql.sh └── test.ts └── turbo.json /.gitignore: -------------------------------------------------------------------------------- 1 | .turbo 2 | node_modules -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "y-crdt"] 2 | path = y-crdt 3 | url = https://github.com/y-crdt/y-crdt.git 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2025 Kapil Verma 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # k_yrs_go - Golang database server for YJS CRDT using Postgres + Redis 2 | 3 | `k_yrs_go` is a database server for [YJS](https://docs.yjs.dev/) documents. It works on top of [Postgres](http://postgresql.org/) and [Redis](https://redis.io/). 4 | `k_yrs_go` uses binary redis queues as I/O buffers for YJS document updates, and uses the following PG table to store the updates: 5 | 6 | ```sql 7 | CREATE TABLE IF NOT EXISTS k_yrs_go_yupdates_store ( 8 | id TEXT PRIMARY KEY, 9 | doc_id TEXT NOT NULL, 10 | data BYTEA NOT NULL 11 | ); 12 | 13 | CREATE INDEX IF NOT EXISTS k_yrs_go_yupdates_store_doc_id_idx ON k_yrs_go_yupdates_store (doc_id); 14 | ``` 15 | 16 | Rows in `k_yrs_go_yupdates_store` undergo compaction when fetching the state for a document if the number of 17 | rows of updates for the `doc_id` are > 100 in count. Compaction happens in a serializable transaction, and the 18 | combined-yupdate is inserted in the table only when the number of deleted yupdates is equal to what was fetched 19 | from the db. 20 | 21 | Even the Reads and Writes happen in serializable transactions. From what all I have read about databases, Reads, Writes, and Compactions in `k_yrs_go` should be consistent with each other. 22 | 23 | Max document size supported is [1 GB](https://www.postgresql.org/docs/7.4/jdbc-binary-data.html#:~:text=The%20bytea%20data%20type%20is,process%20such%20a%20large%20value.). The test suite contains a `'large doc'` test which tests persistence for 100MB docs. 24 | 25 | ## Usage: 26 | 27 | ```ts 28 | import axios from 'axios'; 29 | 30 | const api = axios.create({ baseURL: env.SERVER_URL }); // `env.SERVER_URL is where `k_yrs_go/server` is deployed 31 | 32 | const docId = uuid(); 33 | const ydoc = new Y.Doc(); 34 | 35 | // WRITE 36 | ydoc.on('update', async (update: Uint8Array, origin: any, doc: Y.Doc) => { 37 | api.post(`/docs/${docId}/updates`, update, {headers: {'Content-Type': 'application/octet-stream'}}) 38 | }); 39 | 40 | // READ 41 | const response = await api.get(`/docs/${docId}/updates`, { responseType: 'arraybuffer' }); 42 | const update = new Uint8Array(response.data); 43 | const ydoc2 = new Y.Doc(); 44 | Y.applyUpdate(ydoc2, update); 45 | 46 | ``` 47 | 48 | ### Clone 49 | 50 | ```bash 51 | git clone --recurse-submodules git@github.com:kapv89/k_yrs_go.git 52 | ``` 53 | 54 | ### Setup 55 | 56 | 1. Install [docker](https://docs.docker.com/engine/install/) and [docker-compose](https://docs.docker.com/compose/install/) 57 | 1. Make sure you can [run `docker` without `sudo`](https://docs.docker.com/engine/install/linux-postinstall/). 58 | 1. [Install go](https://go.dev/doc/install) 59 | 1. [Install rust](https://www.rust-lang.org/tools/install) 60 | 1. [Install node.js v20.10.0+](https://github.com/nvm-sh/nvm) 61 | 1. Install [tsx](https://www.npmjs.com/package/tsx) globally `npm i -g tsx` 62 | 1. Install [turbo](https://turbo.build/repo/docs/getting-started/installation) `npm i -g turbo` 63 | 1. `cd k_yrs_go` 64 | 1. `npm ci` 65 | 66 | ### Run locally 67 | 68 | ```bash 69 | turbo run dev 70 | ``` 71 | 72 | ### Run test 73 | 74 | ```bash 75 | turbo run test 76 | ``` 77 | 78 | ## 1ms +- x write latencies! 79 | 80 | [**latencies.png**](latencies.png) & [**system_config.png**](system_config.png) 81 | 82 | Seems to be very fast on my system. 83 | 84 | ### Testing 85 | 86 | Tests are written in typescript with actual YJS docs. They can be found in [`test/test.ts`](test/test.ts). 87 | 88 | To run the test on prod binary: 89 | 90 | 1. First start the dev infra: `turbo run dev#dev` 91 | 1. Run the production binary on dev infra: `turbo run server` 92 | 1. Run the test suite: `turbo run test` 93 | 94 | If you want to supply custom env-params to tests: 95 | 96 | 1. First start the dev infra: `turbo run dev#dev` 97 | 1. Run the production binary on dev infra: `turbo run server` 98 | 1. `cd test` 99 | 1. Supply env-params and run the `npm run test` command. Example: `RW_ITERS=3 COMPACTION_ITERS=0 CONSISTENCY_SIMPLE_ITERS=0 CONSISTENCY_LOAD_TEST_ITERS=0 npm run test` 100 | 101 | Available env params to tweak tests are (with default values): 102 | 103 | ``` 104 | { 105 | RW_ITERS: 1, 106 | RW_Y_OPS_WAIT_MS: 0, 107 | 108 | COMPACTION_ITERS: 1, 109 | COMPACTION_YDOC_UPDATE_INTERVAL_MS: 0, 110 | COMPACTION_YDOC_UPDATE_ITERS: 10000, 111 | COMPACTION_Y_OPS_WAIT_MS: 0, 112 | 113 | CONSISTENCY_SIMPLE_ITERS: 1, 114 | CONSISTENCY_SIMPLE_READTIMEOUT_MS: 0, 115 | CONSISTENCY_SIMPLE_YDOC_UPDATE_ITERS: 10000, 116 | 117 | CONSISTENCY_LOAD_TEST_ITERS: 1, 118 | CONSISTENCY_LOAD_YDOC_UPDATE_ITERS: 10000, 119 | CONSISTENCY_LOAD_YDOC_UPDATE_TIMEOUT_MS: 2, 120 | CONSISTENCY_LOAD_READ_PER_N_WRITES: 5, 121 | CONSISTENCY_LOAD_YDOC_READ_TIMEOUT_MS: 3, 122 | } 123 | ``` 124 | 125 | 126 | There are 5 types of tests: 127 | 128 | #### Read-Write test 129 | 130 | Read-Write test tests for persistence of 2 operations on a simple list, and ensures that reading them back is consistent. Relevant env params (with default values) are: 131 | 132 | ```ts 133 | { 134 | RW_ITERS: 1, // number of times the read-write test suite should be run 135 | RW_Y_OPS_WAIT_MS: 0, // ms of wait between (rw) operations on yjs docs 136 | } 137 | ``` 138 | 139 | #### Compaction test 140 | 141 | Compaction test writes a large number of updates to a yjs doc, the performs the following checks: 142 | 143 | 1. Checks that the number of rows in the `k_yrs_go_yupdates_store` table for the test `doc_id` are > 100 144 | after the writes. 145 | 1. Fetches 2 yjs updates for `doc_id` **within the same millisecond** (while compaction is happening), loads them in 2 other 146 | yjs docs and checks that this new yjs doc is consistent with the original yjs doc and that both responses within the same millisecond are consistent with each other. 147 | 1. Checks that the number of rows in `k_yrs_go_yupdates_store` table for the test `doc_id` are <= 100 (compaction has happened). 148 | 149 | Relevant env params (with default values) are: 150 | 151 | ```ts 152 | { 153 | COMPACTION_ITERS: 1, // number of times the compaction test-suite should be run 154 | COMPACTION_YDOC_UPDATE_INTERVAL_MS: 0, // ms of wait between performing 2 update operations to the test yjs doc 155 | COMPACTION_YDOC_UPDATE_ITERS: 10000, // number of updates to be performed on the test yjs doc 156 | COMPACTION_Y_OPS_WAIT_MS: 0, // ms of wait between different compaction stages 157 | } 158 | ``` 159 | 160 | #### Simple Consistency test 161 | 162 | In this test, the following steps happen in sequence in a loop: 163 | 164 | 1. An update is written to test yjs doc, and gets persisted to the db server 165 | 1. State of doc is read back from the db server, and applied to a new yjs doc 166 | 1. The new yjs doc is compared to be consistent with the test yjs doc 167 | 1. Go back to #1 168 | 169 | Relevant env params (with default values) are: 170 | 171 | ```ts 172 | { 173 | CONSISTENCY_SIMPLE_ITERS: 1, // number of times the simple consistency test should be run 174 | CONSISTENCY_SIMPLE_READTIMEOUT_MS: 0, // ms to wait before reading yjs doc state from db server after a write to test yjs doc 175 | CONSISTENCY_SIMPLE_YDOC_UPDATE_ITERS: 10000, // number of updates to be applied to the test yjs doc 176 | } 177 | ``` 178 | 179 | #### Load Consistency test 180 | 181 | This test tries to get to the limits of how consistent writes and reads are for a frequently updated document which is also frequently fetched. This is important for scenarios where new user can try to request a document which is being frequently updated by multiple other users and you need to ensure that they get the latest state. 182 | 183 | Relevant env params (with default values) are: 184 | 185 | ```ts 186 | { 187 | CONSISTENCY_LOAD_TEST_ITERS: 1, // number of times the load consistency test should be run 188 | CONSISTENCY_LOAD_YDOC_UPDATE_ITERS: 10000, // number of updates to be applied to the test yjs doc 189 | CONSISTENCY_LOAD_YDOC_UPDATE_TIMEOUT_MS: 2, // ms to wait before applying an update to the test yjs doc 190 | CONSISTENCY_LOAD_READ_PER_N_WRITES: 5, // number of writes after which consistency of a read from the db server should be checked 191 | CONSISTENCY_LOAD_YDOC_READ_TIMEOUT_MS: 3, // ms to wait after an update before reading yjs doc state from db server and verifying its consistency 192 | } 193 | ``` 194 | 195 | I wasn't able to reach a better (and stable) consistency under load numbers than this on my local machine. 196 | 197 | #### Large Doc test 198 | 199 | 1. This tests writes data to the test yjs doc till it becomes 100MB in size, all the while persisting updates to server. 200 | 2. Then it reads the data back from server twice - once before compaction, and once after compaction, and creates 2 new yjs docs. 201 | 3. The 3 yjs docs are compared to be consistent with each other. 202 | 203 | Relevant env params (with default values) are: 204 | 205 | ```ts 206 | { 207 | LARGE_DOC_TEST_ITERS: 1, // number of times the large doc test should be run 208 | LARGE_DOC_MAX_DOC_SIZE_MB: 100, // max size of test yjs doc. the test doc will be written to till it reaches this size 209 | LARGE_DOC_CHECK_DOC_SIZE_PER_ITER: 10000, // checking yjs doc size becomes an expensive operation quickly as it grows more than 10MB, hence it is checked per these many iters 210 | LARGE_DOC_YDOC_WRITE_INTERVAL_MS: 0, // interval between 2 write operations to the test yjs doc 211 | LARGE_DOC_YDOC_READ_TIMEOUT_MS: 0 // time to wait before reading yjs doc back after all write requests have completed 212 | } 213 | ``` 214 | 215 | ### Build 216 | 217 | If you are running the dev setup, stop it. It's gonna be useless after `build` runs because the C ffi files will get refreshed. 218 | 219 | ```bash 220 | turbo run build 221 | ``` 222 | 223 | Server binary will be available at `server/server`. You can deploy this server binary in a horizontally scalable manner 224 | like a normal API server over a Postgres DB and a Redis DB and things will work correctly. 225 | 226 | #### Run prod binary 227 | 228 | You can see an example of running in prod in [server/server.sh](server/server.sh). Tweak it however you like. 229 | You'll also need the following generated files co-located with the `server` binary in a directory named `db` 230 | 231 | 1. `server/db/libyrs.a` 232 | 1. `server/db/libyrs.h` 233 | 1. `server/db/libyrs.so` 234 | 235 | Directory structure for running prod binary using `server.sh` should look something like this: 236 | 237 | ``` 238 | deployment/ 239 | |- .env 240 | |- server 241 | |- server.sh 242 | |- db/ 243 | |- libyrs.a 244 | |- libyrs.h 245 | |- libyrs.so 246 | ``` 247 | 248 | 249 | If you want to run the prod binary with default dev infra, you can do the following: 250 | 251 | 1. Spin up dev infra: 252 | ```bash 253 | turbo run dev#dev 254 | ``` 255 | 1. Run the prod server binary 256 | ```bash 257 | turbo run server 258 | ``` 259 | 1. Optionally, run the test-suite 260 | ```bash 261 | turbo run test 262 | ``` 263 | 264 | ### Configuration 265 | See the file [`server/.env`](server/.env). You can tweak it however you want. 266 | 267 | To make sure tests run after your tweaking `server/.env`, you'd need to tweak [`test/.env`](test/.env). 268 | 269 | Relevant ones are: 270 | 271 | ```bash 272 | SERVER_PORT=3000 273 | 274 | PG_URL=postgres://dev:dev@localhost:5432/k_yrs_dev?sslmode=disable 275 | 276 | REDIS_URL=redis://localhost:6379 277 | 278 | DEBUG=true 279 | 280 | REDIS_QUEUE_MAX_SIZE=1000 281 | ``` 282 | 283 | # USE WITH [`yjs-scalable-ws-backend`](https://github.com/kapv89/yjs-scalable-ws-backend) -------------------------------------------------------------------------------- /dev/.gitignore: -------------------------------------------------------------------------------- 1 | docker-data/ -------------------------------------------------------------------------------- /dev/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | services: 3 | k_yrs_redis: 4 | container_name: k_yrs_redis 5 | image: redis:latest 6 | command: redis-server 7 | ports: 8 | - '6379:6379' 9 | k_yrs_pg: 10 | container_name: k_yrs_pg 11 | image: postgres:latest 12 | ports: 13 | - '5432:5432' 14 | environment: 15 | POSTGRES_USER: dev 16 | POSTGRES_PASSWORD: dev 17 | POSTGRES_DB: k_yrs_dev 18 | k_yrs_redis_test: 19 | container_name: k_yrs_redis_test 20 | image: redis:latest 21 | command: redis-server 22 | ports: 23 | - '6380:6379' 24 | k_yrs_pg_test: 25 | container_name: k_yrs_pg_test 26 | image: postgres:14 27 | ports: 28 | - '5433:5432' 29 | environment: 30 | POSTGRES_USER: dev 31 | POSTGRES_PASSWORD: dev 32 | POSTGRES_DB: k_yrs_test 33 | k_yrs_infra_healthcheck: 34 | container_name: k_yrs_infra_healthcheck 35 | image: vad1mo/hello-world-rest 36 | ports: 37 | - '5050:5050' 38 | depends_on: 39 | - k_yrs_redis 40 | - k_yrs_redis_test 41 | - k_yrs_pg 42 | - k_yrs_pg_test -------------------------------------------------------------------------------- /dev/healthz.ts: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | 3 | const check = async () => { 4 | try { 5 | await axios.get('http://localhost:5050') 6 | } catch (err) { 7 | await new Promise(resolve => setTimeout(resolve, 3000)); 8 | await check(); 9 | } 10 | } 11 | 12 | check(); -------------------------------------------------------------------------------- /dev/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dev", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "healthz": "tsx healthz.ts", 8 | "dev": "docker-compose up -d", 9 | "down": "docker-compose down" 10 | }, 11 | "keywords": [], 12 | "author": "", 13 | "license": "ISC", 14 | "devDependencies": { 15 | "axios": "^1.6.2", 16 | "tsx": "^4.6.2" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /k_yrs_go.code-workspace: -------------------------------------------------------------------------------- 1 | { 2 | "folders": [ 3 | { 4 | "path": "." 5 | }, 6 | { 7 | "path": "../Projects/yjs-scalable-ws-backend" 8 | } 9 | ], 10 | "settings": {} 11 | } -------------------------------------------------------------------------------- /latencies.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kapv89/k_yrs_go/7d9a9eb321880ee749fa7590d547cdfdb9058d5d/latencies.png -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "k_yrs_go", 3 | "lockfileVersion": 3, 4 | "requires": true, 5 | "packages": { 6 | "": { 7 | "name": "k_yrs_go", 8 | "workspaces": [ 9 | "server", 10 | "dev", 11 | "test" 12 | ] 13 | }, 14 | "dev": { 15 | "version": "1.0.0", 16 | "license": "ISC", 17 | "devDependencies": { 18 | "axios": "^1.6.2", 19 | "tsx": "^4.6.2" 20 | } 21 | }, 22 | "node_modules/@esbuild/aix-ppc64": { 23 | "version": "0.19.11", 24 | "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.19.11.tgz", 25 | "integrity": "sha512-FnzU0LyE3ySQk7UntJO4+qIiQgI7KoODnZg5xzXIrFJlKd2P2gwHsHY4927xj9y5PJmJSzULiUCWmv7iWnNa7g==", 26 | "cpu": [ 27 | "ppc64" 28 | ], 29 | "dev": true, 30 | "optional": true, 31 | "os": [ 32 | "aix" 33 | ], 34 | "engines": { 35 | "node": ">=12" 36 | } 37 | }, 38 | "node_modules/@esbuild/android-arm": { 39 | "version": "0.19.11", 40 | "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.19.11.tgz", 41 | "integrity": "sha512-5OVapq0ClabvKvQ58Bws8+wkLCV+Rxg7tUVbo9xu034Nm536QTII4YzhaFriQ7rMrorfnFKUsArD2lqKbFY4vw==", 42 | "cpu": [ 43 | "arm" 44 | ], 45 | "dev": true, 46 | "optional": true, 47 | "os": [ 48 | "android" 49 | ], 50 | "engines": { 51 | "node": ">=12" 52 | } 53 | }, 54 | "node_modules/@esbuild/android-arm64": { 55 | "version": "0.19.11", 56 | "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.19.11.tgz", 57 | "integrity": "sha512-aiu7K/5JnLj//KOnOfEZ0D90obUkRzDMyqd/wNAUQ34m4YUPVhRZpnqKV9uqDGxT7cToSDnIHsGooyIczu9T+Q==", 58 | "cpu": [ 59 | "arm64" 60 | ], 61 | "dev": true, 62 | "optional": true, 63 | "os": [ 64 | "android" 65 | ], 66 | "engines": { 67 | "node": ">=12" 68 | } 69 | }, 70 | "node_modules/@esbuild/android-x64": { 71 | "version": "0.19.11", 72 | "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.19.11.tgz", 73 | "integrity": "sha512-eccxjlfGw43WYoY9QgB82SgGgDbibcqyDTlk3l3C0jOVHKxrjdc9CTwDUQd0vkvYg5um0OH+GpxYvp39r+IPOg==", 74 | "cpu": [ 75 | "x64" 76 | ], 77 | "dev": true, 78 | "optional": true, 79 | "os": [ 80 | "android" 81 | ], 82 | "engines": { 83 | "node": ">=12" 84 | } 85 | }, 86 | "node_modules/@esbuild/darwin-arm64": { 87 | "version": "0.19.11", 88 | "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.19.11.tgz", 89 | "integrity": "sha512-ETp87DRWuSt9KdDVkqSoKoLFHYTrkyz2+65fj9nfXsaV3bMhTCjtQfw3y+um88vGRKRiF7erPrh/ZuIdLUIVxQ==", 90 | "cpu": [ 91 | "arm64" 92 | ], 93 | "dev": true, 94 | "optional": true, 95 | "os": [ 96 | "darwin" 97 | ], 98 | "engines": { 99 | "node": ">=12" 100 | } 101 | }, 102 | "node_modules/@esbuild/darwin-x64": { 103 | "version": "0.19.11", 104 | "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.19.11.tgz", 105 | "integrity": "sha512-fkFUiS6IUK9WYUO/+22omwetaSNl5/A8giXvQlcinLIjVkxwTLSktbF5f/kJMftM2MJp9+fXqZ5ezS7+SALp4g==", 106 | "cpu": [ 107 | "x64" 108 | ], 109 | "dev": true, 110 | "optional": true, 111 | "os": [ 112 | "darwin" 113 | ], 114 | "engines": { 115 | "node": ">=12" 116 | } 117 | }, 118 | "node_modules/@esbuild/freebsd-arm64": { 119 | "version": "0.19.11", 120 | "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.19.11.tgz", 121 | "integrity": "sha512-lhoSp5K6bxKRNdXUtHoNc5HhbXVCS8V0iZmDvyWvYq9S5WSfTIHU2UGjcGt7UeS6iEYp9eeymIl5mJBn0yiuxA==", 122 | "cpu": [ 123 | "arm64" 124 | ], 125 | "dev": true, 126 | "optional": true, 127 | "os": [ 128 | "freebsd" 129 | ], 130 | "engines": { 131 | "node": ">=12" 132 | } 133 | }, 134 | "node_modules/@esbuild/freebsd-x64": { 135 | "version": "0.19.11", 136 | "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.19.11.tgz", 137 | "integrity": "sha512-JkUqn44AffGXitVI6/AbQdoYAq0TEullFdqcMY/PCUZ36xJ9ZJRtQabzMA+Vi7r78+25ZIBosLTOKnUXBSi1Kw==", 138 | "cpu": [ 139 | "x64" 140 | ], 141 | "dev": true, 142 | "optional": true, 143 | "os": [ 144 | "freebsd" 145 | ], 146 | "engines": { 147 | "node": ">=12" 148 | } 149 | }, 150 | "node_modules/@esbuild/linux-arm": { 151 | "version": "0.19.11", 152 | "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.19.11.tgz", 153 | "integrity": "sha512-3CRkr9+vCV2XJbjwgzjPtO8T0SZUmRZla+UL1jw+XqHZPkPgZiyWvbDvl9rqAN8Zl7qJF0O/9ycMtjU67HN9/Q==", 154 | "cpu": [ 155 | "arm" 156 | ], 157 | "dev": true, 158 | "optional": true, 159 | "os": [ 160 | "linux" 161 | ], 162 | "engines": { 163 | "node": ">=12" 164 | } 165 | }, 166 | "node_modules/@esbuild/linux-arm64": { 167 | "version": "0.19.11", 168 | "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.19.11.tgz", 169 | "integrity": "sha512-LneLg3ypEeveBSMuoa0kwMpCGmpu8XQUh+mL8XXwoYZ6Be2qBnVtcDI5azSvh7vioMDhoJFZzp9GWp9IWpYoUg==", 170 | "cpu": [ 171 | "arm64" 172 | ], 173 | "dev": true, 174 | "optional": true, 175 | "os": [ 176 | "linux" 177 | ], 178 | "engines": { 179 | "node": ">=12" 180 | } 181 | }, 182 | "node_modules/@esbuild/linux-ia32": { 183 | "version": "0.19.11", 184 | "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.19.11.tgz", 185 | "integrity": "sha512-caHy++CsD8Bgq2V5CodbJjFPEiDPq8JJmBdeyZ8GWVQMjRD0sU548nNdwPNvKjVpamYYVL40AORekgfIubwHoA==", 186 | "cpu": [ 187 | "ia32" 188 | ], 189 | "dev": true, 190 | "optional": true, 191 | "os": [ 192 | "linux" 193 | ], 194 | "engines": { 195 | "node": ">=12" 196 | } 197 | }, 198 | "node_modules/@esbuild/linux-loong64": { 199 | "version": "0.19.11", 200 | "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.19.11.tgz", 201 | "integrity": "sha512-ppZSSLVpPrwHccvC6nQVZaSHlFsvCQyjnvirnVjbKSHuE5N24Yl8F3UwYUUR1UEPaFObGD2tSvVKbvR+uT1Nrg==", 202 | "cpu": [ 203 | "loong64" 204 | ], 205 | "dev": true, 206 | "optional": true, 207 | "os": [ 208 | "linux" 209 | ], 210 | "engines": { 211 | "node": ">=12" 212 | } 213 | }, 214 | "node_modules/@esbuild/linux-mips64el": { 215 | "version": "0.19.11", 216 | "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.19.11.tgz", 217 | "integrity": "sha512-B5x9j0OgjG+v1dF2DkH34lr+7Gmv0kzX6/V0afF41FkPMMqaQ77pH7CrhWeR22aEeHKaeZVtZ6yFwlxOKPVFyg==", 218 | "cpu": [ 219 | "mips64el" 220 | ], 221 | "dev": true, 222 | "optional": true, 223 | "os": [ 224 | "linux" 225 | ], 226 | "engines": { 227 | "node": ">=12" 228 | } 229 | }, 230 | "node_modules/@esbuild/linux-ppc64": { 231 | "version": "0.19.11", 232 | "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.19.11.tgz", 233 | "integrity": "sha512-MHrZYLeCG8vXblMetWyttkdVRjQlQUb/oMgBNurVEnhj4YWOr4G5lmBfZjHYQHHN0g6yDmCAQRR8MUHldvvRDA==", 234 | "cpu": [ 235 | "ppc64" 236 | ], 237 | "dev": true, 238 | "optional": true, 239 | "os": [ 240 | "linux" 241 | ], 242 | "engines": { 243 | "node": ">=12" 244 | } 245 | }, 246 | "node_modules/@esbuild/linux-riscv64": { 247 | "version": "0.19.11", 248 | "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.19.11.tgz", 249 | "integrity": "sha512-f3DY++t94uVg141dozDu4CCUkYW+09rWtaWfnb3bqe4w5NqmZd6nPVBm+qbz7WaHZCoqXqHz5p6CM6qv3qnSSQ==", 250 | "cpu": [ 251 | "riscv64" 252 | ], 253 | "dev": true, 254 | "optional": true, 255 | "os": [ 256 | "linux" 257 | ], 258 | "engines": { 259 | "node": ">=12" 260 | } 261 | }, 262 | "node_modules/@esbuild/linux-s390x": { 263 | "version": "0.19.11", 264 | "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.19.11.tgz", 265 | "integrity": "sha512-A5xdUoyWJHMMlcSMcPGVLzYzpcY8QP1RtYzX5/bS4dvjBGVxdhuiYyFwp7z74ocV7WDc0n1harxmpq2ePOjI0Q==", 266 | "cpu": [ 267 | "s390x" 268 | ], 269 | "dev": true, 270 | "optional": true, 271 | "os": [ 272 | "linux" 273 | ], 274 | "engines": { 275 | "node": ">=12" 276 | } 277 | }, 278 | "node_modules/@esbuild/linux-x64": { 279 | "version": "0.19.11", 280 | "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.19.11.tgz", 281 | "integrity": "sha512-grbyMlVCvJSfxFQUndw5mCtWs5LO1gUlwP4CDi4iJBbVpZcqLVT29FxgGuBJGSzyOxotFG4LoO5X+M1350zmPA==", 282 | "cpu": [ 283 | "x64" 284 | ], 285 | "dev": true, 286 | "optional": true, 287 | "os": [ 288 | "linux" 289 | ], 290 | "engines": { 291 | "node": ">=12" 292 | } 293 | }, 294 | "node_modules/@esbuild/netbsd-x64": { 295 | "version": "0.19.11", 296 | "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.19.11.tgz", 297 | "integrity": "sha512-13jvrQZJc3P230OhU8xgwUnDeuC/9egsjTkXN49b3GcS5BKvJqZn86aGM8W9pd14Kd+u7HuFBMVtrNGhh6fHEQ==", 298 | "cpu": [ 299 | "x64" 300 | ], 301 | "dev": true, 302 | "optional": true, 303 | "os": [ 304 | "netbsd" 305 | ], 306 | "engines": { 307 | "node": ">=12" 308 | } 309 | }, 310 | "node_modules/@esbuild/openbsd-x64": { 311 | "version": "0.19.11", 312 | "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.19.11.tgz", 313 | "integrity": "sha512-ysyOGZuTp6SNKPE11INDUeFVVQFrhcNDVUgSQVDzqsqX38DjhPEPATpid04LCoUr2WXhQTEZ8ct/EgJCUDpyNw==", 314 | "cpu": [ 315 | "x64" 316 | ], 317 | "dev": true, 318 | "optional": true, 319 | "os": [ 320 | "openbsd" 321 | ], 322 | "engines": { 323 | "node": ">=12" 324 | } 325 | }, 326 | "node_modules/@esbuild/sunos-x64": { 327 | "version": "0.19.11", 328 | "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.19.11.tgz", 329 | "integrity": "sha512-Hf+Sad9nVwvtxy4DXCZQqLpgmRTQqyFyhT3bZ4F2XlJCjxGmRFF0Shwn9rzhOYRB61w9VMXUkxlBy56dk9JJiQ==", 330 | "cpu": [ 331 | "x64" 332 | ], 333 | "dev": true, 334 | "optional": true, 335 | "os": [ 336 | "sunos" 337 | ], 338 | "engines": { 339 | "node": ">=12" 340 | } 341 | }, 342 | "node_modules/@esbuild/win32-arm64": { 343 | "version": "0.19.11", 344 | "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.19.11.tgz", 345 | "integrity": "sha512-0P58Sbi0LctOMOQbpEOvOL44Ne0sqbS0XWHMvvrg6NE5jQ1xguCSSw9jQeUk2lfrXYsKDdOe6K+oZiwKPilYPQ==", 346 | "cpu": [ 347 | "arm64" 348 | ], 349 | "dev": true, 350 | "optional": true, 351 | "os": [ 352 | "win32" 353 | ], 354 | "engines": { 355 | "node": ">=12" 356 | } 357 | }, 358 | "node_modules/@esbuild/win32-ia32": { 359 | "version": "0.19.11", 360 | "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.19.11.tgz", 361 | "integrity": "sha512-6YOrWS+sDJDmshdBIQU+Uoyh7pQKrdykdefC1avn76ss5c+RN6gut3LZA4E2cH5xUEp5/cA0+YxRaVtRAb0xBg==", 362 | "cpu": [ 363 | "ia32" 364 | ], 365 | "dev": true, 366 | "optional": true, 367 | "os": [ 368 | "win32" 369 | ], 370 | "engines": { 371 | "node": ">=12" 372 | } 373 | }, 374 | "node_modules/@esbuild/win32-x64": { 375 | "version": "0.19.11", 376 | "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.19.11.tgz", 377 | "integrity": "sha512-vfkhltrjCAb603XaFhqhAF4LGDi2M4OrCRrFusyQ+iTLQ/o60QQXxc9cZC/FFpihBI9N1Grn6SMKVJ4KP7Fuiw==", 378 | "cpu": [ 379 | "x64" 380 | ], 381 | "dev": true, 382 | "optional": true, 383 | "os": [ 384 | "win32" 385 | ], 386 | "engines": { 387 | "node": ">=12" 388 | } 389 | }, 390 | "node_modules/@ioredis/commands": { 391 | "version": "1.2.0", 392 | "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", 393 | "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" 394 | }, 395 | "node_modules/@types/chai": { 396 | "version": "4.3.11", 397 | "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.11.tgz", 398 | "integrity": "sha512-qQR1dr2rGIHYlJulmr8Ioq3De0Le9E4MJ5AiaeAETJJpndT1uUNHsGFK3L/UIu+rbkQSdj8J/w2bCsBZc/Y5fQ==", 399 | "dev": true 400 | }, 401 | "node_modules/@types/lodash": { 402 | "version": "4.17.16", 403 | "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.16.tgz", 404 | "integrity": "sha512-HX7Em5NYQAXKW+1T+FiuG27NGwzJfCX3s1GjOa7ujxZa52kjJLOr4FUxT+giF6Tgxv1e+/czV/iTtBw27WTU9g==", 405 | "dev": true 406 | }, 407 | "node_modules/@types/node": { 408 | "version": "20.10.5", 409 | "resolved": "https://registry.npmjs.org/@types/node/-/node-20.10.5.tgz", 410 | "integrity": "sha512-nNPsNE65wjMxEKI93yOP+NPGGBJz/PoN3kZsVLee0XMiJolxSekEVD8wRwBUBqkwc7UWop0edW50yrCQW4CyRw==", 411 | "dev": true, 412 | "dependencies": { 413 | "undici-types": "~5.26.4" 414 | } 415 | }, 416 | "node_modules/@types/uuid": { 417 | "version": "9.0.7", 418 | "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.7.tgz", 419 | "integrity": "sha512-WUtIVRUZ9i5dYXefDEAI7sh9/O7jGvHg7Df/5O/gtH3Yabe5odI3UWopVR1qbPXQtvOxWu3mM4XxlYeZtMWF4g==", 420 | "dev": true 421 | }, 422 | "node_modules/ansi-escapes": { 423 | "version": "7.0.0", 424 | "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-7.0.0.tgz", 425 | "integrity": "sha512-GdYO7a61mR0fOlAsvC9/rIHf7L96sBc6dEWzeOu+KAea5bZyQRPIpojrVoI4AXGJS/ycu/fBTdLrUkA4ODrvjw==", 426 | "dependencies": { 427 | "environment": "^1.0.0" 428 | }, 429 | "engines": { 430 | "node": ">=18" 431 | }, 432 | "funding": { 433 | "url": "https://github.com/sponsors/sindresorhus" 434 | } 435 | }, 436 | "node_modules/ansi-regex": { 437 | "version": "6.1.0", 438 | "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", 439 | "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", 440 | "engines": { 441 | "node": ">=12" 442 | }, 443 | "funding": { 444 | "url": "https://github.com/chalk/ansi-regex?sponsor=1" 445 | } 446 | }, 447 | "node_modules/ansi-styles": { 448 | "version": "6.2.1", 449 | "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", 450 | "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", 451 | "engines": { 452 | "node": ">=12" 453 | }, 454 | "funding": { 455 | "url": "https://github.com/chalk/ansi-styles?sponsor=1" 456 | } 457 | }, 458 | "node_modules/assertion-error": { 459 | "version": "2.0.1", 460 | "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", 461 | "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", 462 | "engines": { 463 | "node": ">=12" 464 | } 465 | }, 466 | "node_modules/asynckit": { 467 | "version": "0.4.0", 468 | "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", 469 | "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" 470 | }, 471 | "node_modules/axios": { 472 | "version": "1.6.3", 473 | "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.3.tgz", 474 | "integrity": "sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww==", 475 | "dependencies": { 476 | "follow-redirects": "^1.15.0", 477 | "form-data": "^4.0.0", 478 | "proxy-from-env": "^1.1.0" 479 | } 480 | }, 481 | "node_modules/buffer-writer": { 482 | "version": "2.0.0", 483 | "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", 484 | "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==", 485 | "engines": { 486 | "node": ">=4" 487 | } 488 | }, 489 | "node_modules/chai": { 490 | "version": "5.0.0", 491 | "resolved": "https://registry.npmjs.org/chai/-/chai-5.0.0.tgz", 492 | "integrity": "sha512-HO5p0oEKd5M6HEcwOkNAThAE3j960vIZvVcc0t2tI06Dd0ATu69cEnMB2wOhC5/ZyQ6m67w3ePjU/HzXsSsdBA==", 493 | "dependencies": { 494 | "assertion-error": "^2.0.1", 495 | "check-error": "^2.0.0", 496 | "deep-eql": "^5.0.1", 497 | "loupe": "^3.0.0", 498 | "pathval": "^2.0.0" 499 | }, 500 | "engines": { 501 | "node": ">=12" 502 | } 503 | }, 504 | "node_modules/check-error": { 505 | "version": "2.0.0", 506 | "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.0.0.tgz", 507 | "integrity": "sha512-tjLAOBHKVxtPoHe/SA7kNOMvhCRdCJ3vETdeY0RuAc9popf+hyaSV6ZEg9hr4cpWF7jmo/JSWEnLDrnijS9Tog==", 508 | "engines": { 509 | "node": ">= 16" 510 | } 511 | }, 512 | "node_modules/cli-cursor": { 513 | "version": "5.0.0", 514 | "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-5.0.0.tgz", 515 | "integrity": "sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==", 516 | "dependencies": { 517 | "restore-cursor": "^5.0.0" 518 | }, 519 | "engines": { 520 | "node": ">=18" 521 | }, 522 | "funding": { 523 | "url": "https://github.com/sponsors/sindresorhus" 524 | } 525 | }, 526 | "node_modules/cluster-key-slot": { 527 | "version": "1.1.2", 528 | "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", 529 | "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", 530 | "engines": { 531 | "node": ">=0.10.0" 532 | } 533 | }, 534 | "node_modules/colorette": { 535 | "version": "2.0.19", 536 | "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.19.tgz", 537 | "integrity": "sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ==" 538 | }, 539 | "node_modules/combined-stream": { 540 | "version": "1.0.8", 541 | "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", 542 | "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", 543 | "dependencies": { 544 | "delayed-stream": "~1.0.0" 545 | }, 546 | "engines": { 547 | "node": ">= 0.8" 548 | } 549 | }, 550 | "node_modules/commander": { 551 | "version": "10.0.1", 552 | "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", 553 | "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", 554 | "engines": { 555 | "node": ">=14" 556 | } 557 | }, 558 | "node_modules/cross-spawn": { 559 | "version": "7.0.3", 560 | "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", 561 | "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", 562 | "dev": true, 563 | "dependencies": { 564 | "path-key": "^3.1.0", 565 | "shebang-command": "^2.0.0", 566 | "which": "^2.0.1" 567 | }, 568 | "engines": { 569 | "node": ">= 8" 570 | } 571 | }, 572 | "node_modules/debug": { 573 | "version": "4.3.4", 574 | "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", 575 | "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", 576 | "dependencies": { 577 | "ms": "2.1.2" 578 | }, 579 | "engines": { 580 | "node": ">=6.0" 581 | }, 582 | "peerDependenciesMeta": { 583 | "supports-color": { 584 | "optional": true 585 | } 586 | } 587 | }, 588 | "node_modules/deep-eql": { 589 | "version": "5.0.1", 590 | "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.1.tgz", 591 | "integrity": "sha512-nwQCf6ne2gez3o1MxWifqkciwt0zhl0LO1/UwVu4uMBuPmflWM4oQ70XMqHqnBJA+nhzncaqL9HVL6KkHJ28lw==", 592 | "engines": { 593 | "node": ">=6" 594 | } 595 | }, 596 | "node_modules/delayed-stream": { 597 | "version": "1.0.0", 598 | "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", 599 | "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", 600 | "engines": { 601 | "node": ">=0.4.0" 602 | } 603 | }, 604 | "node_modules/denque": { 605 | "version": "2.1.0", 606 | "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", 607 | "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", 608 | "engines": { 609 | "node": ">=0.10" 610 | } 611 | }, 612 | "node_modules/dev": { 613 | "resolved": "dev", 614 | "link": true 615 | }, 616 | "node_modules/dotenv": { 617 | "version": "16.3.1", 618 | "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", 619 | "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==", 620 | "dev": true, 621 | "engines": { 622 | "node": ">=12" 623 | }, 624 | "funding": { 625 | "url": "https://github.com/motdotla/dotenv?sponsor=1" 626 | } 627 | }, 628 | "node_modules/dotenv-cli": { 629 | "version": "7.3.0", 630 | "resolved": "https://registry.npmjs.org/dotenv-cli/-/dotenv-cli-7.3.0.tgz", 631 | "integrity": "sha512-314CA4TyK34YEJ6ntBf80eUY+t1XaFLyem1k9P0sX1gn30qThZ5qZr/ZwE318gEnzyYP9yj9HJk6SqwE0upkfw==", 632 | "dev": true, 633 | "dependencies": { 634 | "cross-spawn": "^7.0.3", 635 | "dotenv": "^16.3.0", 636 | "dotenv-expand": "^10.0.0", 637 | "minimist": "^1.2.6" 638 | }, 639 | "bin": { 640 | "dotenv": "cli.js" 641 | } 642 | }, 643 | "node_modules/dotenv-expand": { 644 | "version": "10.0.0", 645 | "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-10.0.0.tgz", 646 | "integrity": "sha512-GopVGCpVS1UKH75VKHGuQFqS1Gusej0z4FyQkPdwjil2gNIv+LNsqBlboOzpJFZKVT95GkCyWJbBSdFEFUWI2A==", 647 | "dev": true, 648 | "engines": { 649 | "node": ">=12" 650 | } 651 | }, 652 | "node_modules/emoji-regex": { 653 | "version": "10.4.0", 654 | "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.4.0.tgz", 655 | "integrity": "sha512-EC+0oUMY1Rqm4O6LLrgjtYDvcVYTy7chDnM4Q7030tP4Kwj3u/pR6gP9ygnp2CJMK5Gq+9Q2oqmrFJAz01DXjw==" 656 | }, 657 | "node_modules/environment": { 658 | "version": "1.1.0", 659 | "resolved": "https://registry.npmjs.org/environment/-/environment-1.1.0.tgz", 660 | "integrity": "sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==", 661 | "engines": { 662 | "node": ">=18" 663 | }, 664 | "funding": { 665 | "url": "https://github.com/sponsors/sindresorhus" 666 | } 667 | }, 668 | "node_modules/esbuild": { 669 | "version": "0.19.11", 670 | "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.19.11.tgz", 671 | "integrity": "sha512-HJ96Hev2hX/6i5cDVwcqiJBBtuo9+FeIJOtZ9W1kA5M6AMJRHUZlpYZ1/SbEwtO0ioNAW8rUooVpC/WehY2SfA==", 672 | "dev": true, 673 | "hasInstallScript": true, 674 | "bin": { 675 | "esbuild": "bin/esbuild" 676 | }, 677 | "engines": { 678 | "node": ">=12" 679 | }, 680 | "optionalDependencies": { 681 | "@esbuild/aix-ppc64": "0.19.11", 682 | "@esbuild/android-arm": "0.19.11", 683 | "@esbuild/android-arm64": "0.19.11", 684 | "@esbuild/android-x64": "0.19.11", 685 | "@esbuild/darwin-arm64": "0.19.11", 686 | "@esbuild/darwin-x64": "0.19.11", 687 | "@esbuild/freebsd-arm64": "0.19.11", 688 | "@esbuild/freebsd-x64": "0.19.11", 689 | "@esbuild/linux-arm": "0.19.11", 690 | "@esbuild/linux-arm64": "0.19.11", 691 | "@esbuild/linux-ia32": "0.19.11", 692 | "@esbuild/linux-loong64": "0.19.11", 693 | "@esbuild/linux-mips64el": "0.19.11", 694 | "@esbuild/linux-ppc64": "0.19.11", 695 | "@esbuild/linux-riscv64": "0.19.11", 696 | "@esbuild/linux-s390x": "0.19.11", 697 | "@esbuild/linux-x64": "0.19.11", 698 | "@esbuild/netbsd-x64": "0.19.11", 699 | "@esbuild/openbsd-x64": "0.19.11", 700 | "@esbuild/sunos-x64": "0.19.11", 701 | "@esbuild/win32-arm64": "0.19.11", 702 | "@esbuild/win32-ia32": "0.19.11", 703 | "@esbuild/win32-x64": "0.19.11" 704 | } 705 | }, 706 | "node_modules/escalade": { 707 | "version": "3.1.1", 708 | "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", 709 | "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", 710 | "engines": { 711 | "node": ">=6" 712 | } 713 | }, 714 | "node_modules/esm": { 715 | "version": "3.2.25", 716 | "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", 717 | "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==", 718 | "engines": { 719 | "node": ">=6" 720 | } 721 | }, 722 | "node_modules/follow-redirects": { 723 | "version": "1.15.3", 724 | "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", 725 | "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==", 726 | "funding": [ 727 | { 728 | "type": "individual", 729 | "url": "https://github.com/sponsors/RubenVerborgh" 730 | } 731 | ], 732 | "engines": { 733 | "node": ">=4.0" 734 | }, 735 | "peerDependenciesMeta": { 736 | "debug": { 737 | "optional": true 738 | } 739 | } 740 | }, 741 | "node_modules/form-data": { 742 | "version": "4.0.0", 743 | "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", 744 | "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", 745 | "dependencies": { 746 | "asynckit": "^0.4.0", 747 | "combined-stream": "^1.0.8", 748 | "mime-types": "^2.1.12" 749 | }, 750 | "engines": { 751 | "node": ">= 6" 752 | } 753 | }, 754 | "node_modules/fsevents": { 755 | "version": "2.3.3", 756 | "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", 757 | "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", 758 | "dev": true, 759 | "hasInstallScript": true, 760 | "optional": true, 761 | "os": [ 762 | "darwin" 763 | ], 764 | "engines": { 765 | "node": "^8.16.0 || ^10.6.0 || >=11.0.0" 766 | } 767 | }, 768 | "node_modules/function-bind": { 769 | "version": "1.1.2", 770 | "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", 771 | "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", 772 | "funding": { 773 | "url": "https://github.com/sponsors/ljharb" 774 | } 775 | }, 776 | "node_modules/get-east-asian-width": { 777 | "version": "1.3.0", 778 | "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.3.0.tgz", 779 | "integrity": "sha512-vpeMIQKxczTD/0s2CdEWHcb0eeJe6TFjxb+J5xgX7hScxqrGuyjmv4c1D4A/gelKfyox0gJJwIHF+fLjeaM8kQ==", 780 | "engines": { 781 | "node": ">=18" 782 | }, 783 | "funding": { 784 | "url": "https://github.com/sponsors/sindresorhus" 785 | } 786 | }, 787 | "node_modules/get-func-name": { 788 | "version": "2.0.2", 789 | "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz", 790 | "integrity": "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==", 791 | "engines": { 792 | "node": "*" 793 | } 794 | }, 795 | "node_modules/get-package-type": { 796 | "version": "0.1.0", 797 | "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", 798 | "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", 799 | "engines": { 800 | "node": ">=8.0.0" 801 | } 802 | }, 803 | "node_modules/get-tsconfig": { 804 | "version": "4.7.2", 805 | "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.7.2.tgz", 806 | "integrity": "sha512-wuMsz4leaj5hbGgg4IvDU0bqJagpftG5l5cXIAvo8uZrqn0NJqwtfupTN00VnkQJPcIRrxYrm1Ue24btpCha2A==", 807 | "dev": true, 808 | "dependencies": { 809 | "resolve-pkg-maps": "^1.0.0" 810 | }, 811 | "funding": { 812 | "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" 813 | } 814 | }, 815 | "node_modules/getopts": { 816 | "version": "2.3.0", 817 | "resolved": "https://registry.npmjs.org/getopts/-/getopts-2.3.0.tgz", 818 | "integrity": "sha512-5eDf9fuSXwxBL6q5HX+dhDj+dslFGWzU5thZ9kNKUkcPtaPdatmUFKwHFrLb/uf/WpA4BHET+AX3Scl56cAjpA==" 819 | }, 820 | "node_modules/hasown": { 821 | "version": "2.0.0", 822 | "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz", 823 | "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==", 824 | "dependencies": { 825 | "function-bind": "^1.1.2" 826 | }, 827 | "engines": { 828 | "node": ">= 0.4" 829 | } 830 | }, 831 | "node_modules/interpret": { 832 | "version": "2.2.0", 833 | "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz", 834 | "integrity": "sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==", 835 | "engines": { 836 | "node": ">= 0.10" 837 | } 838 | }, 839 | "node_modules/ioredis": { 840 | "version": "5.3.2", 841 | "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", 842 | "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", 843 | "dependencies": { 844 | "@ioredis/commands": "^1.1.1", 845 | "cluster-key-slot": "^1.1.0", 846 | "debug": "^4.3.4", 847 | "denque": "^2.1.0", 848 | "lodash.defaults": "^4.2.0", 849 | "lodash.isarguments": "^3.1.0", 850 | "redis-errors": "^1.2.0", 851 | "redis-parser": "^3.0.0", 852 | "standard-as-callback": "^2.1.0" 853 | }, 854 | "engines": { 855 | "node": ">=12.22.0" 856 | }, 857 | "funding": { 858 | "type": "opencollective", 859 | "url": "https://opencollective.com/ioredis" 860 | } 861 | }, 862 | "node_modules/is-core-module": { 863 | "version": "2.13.1", 864 | "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", 865 | "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==", 866 | "dependencies": { 867 | "hasown": "^2.0.0" 868 | }, 869 | "funding": { 870 | "url": "https://github.com/sponsors/ljharb" 871 | } 872 | }, 873 | "node_modules/is-fullwidth-code-point": { 874 | "version": "5.0.0", 875 | "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-5.0.0.tgz", 876 | "integrity": "sha512-OVa3u9kkBbw7b8Xw5F9P+D/T9X+Z4+JruYVNapTjPYZYUznQ5YfWeFkOj606XYYW8yugTfC8Pj0hYqvi4ryAhA==", 877 | "dependencies": { 878 | "get-east-asian-width": "^1.0.0" 879 | }, 880 | "engines": { 881 | "node": ">=18" 882 | }, 883 | "funding": { 884 | "url": "https://github.com/sponsors/sindresorhus" 885 | } 886 | }, 887 | "node_modules/isexe": { 888 | "version": "2.0.0", 889 | "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", 890 | "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", 891 | "dev": true 892 | }, 893 | "node_modules/isomorphic.js": { 894 | "version": "0.2.5", 895 | "resolved": "https://registry.npmjs.org/isomorphic.js/-/isomorphic.js-0.2.5.tgz", 896 | "integrity": "sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==", 897 | "funding": { 898 | "type": "GitHub Sponsors ❤", 899 | "url": "https://github.com/sponsors/dmonad" 900 | } 901 | }, 902 | "node_modules/knex": { 903 | "version": "3.1.0", 904 | "resolved": "https://registry.npmjs.org/knex/-/knex-3.1.0.tgz", 905 | "integrity": "sha512-GLoII6hR0c4ti243gMs5/1Rb3B+AjwMOfjYm97pu0FOQa7JH56hgBxYf5WK2525ceSbBY1cjeZ9yk99GPMB6Kw==", 906 | "dependencies": { 907 | "colorette": "2.0.19", 908 | "commander": "^10.0.0", 909 | "debug": "4.3.4", 910 | "escalade": "^3.1.1", 911 | "esm": "^3.2.25", 912 | "get-package-type": "^0.1.0", 913 | "getopts": "2.3.0", 914 | "interpret": "^2.2.0", 915 | "lodash": "^4.17.21", 916 | "pg-connection-string": "2.6.2", 917 | "rechoir": "^0.8.0", 918 | "resolve-from": "^5.0.0", 919 | "tarn": "^3.0.2", 920 | "tildify": "2.0.0" 921 | }, 922 | "bin": { 923 | "knex": "bin/cli.js" 924 | }, 925 | "engines": { 926 | "node": ">=16" 927 | }, 928 | "peerDependenciesMeta": { 929 | "better-sqlite3": { 930 | "optional": true 931 | }, 932 | "mysql": { 933 | "optional": true 934 | }, 935 | "mysql2": { 936 | "optional": true 937 | }, 938 | "pg": { 939 | "optional": true 940 | }, 941 | "pg-native": { 942 | "optional": true 943 | }, 944 | "sqlite3": { 945 | "optional": true 946 | }, 947 | "tedious": { 948 | "optional": true 949 | } 950 | } 951 | }, 952 | "node_modules/lib0": { 953 | "version": "0.2.88", 954 | "resolved": "https://registry.npmjs.org/lib0/-/lib0-0.2.88.tgz", 955 | "integrity": "sha512-KyroiEvCeZcZEMx5Ys+b4u4eEBbA1ch7XUaBhYpwa/nPMrzTjUhI4RfcytmQfYoTBPcdyx+FX6WFNIoNuJzJfQ==", 956 | "dependencies": { 957 | "isomorphic.js": "^0.2.4" 958 | }, 959 | "bin": { 960 | "0gentesthtml": "bin/gentesthtml.js", 961 | "0serve": "bin/0serve.js" 962 | }, 963 | "engines": { 964 | "node": ">=16" 965 | }, 966 | "funding": { 967 | "type": "GitHub Sponsors ❤", 968 | "url": "https://github.com/sponsors/dmonad" 969 | } 970 | }, 971 | "node_modules/lodash": { 972 | "version": "4.17.21", 973 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", 974 | "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" 975 | }, 976 | "node_modules/lodash.defaults": { 977 | "version": "4.2.0", 978 | "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", 979 | "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" 980 | }, 981 | "node_modules/lodash.isarguments": { 982 | "version": "3.1.0", 983 | "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", 984 | "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" 985 | }, 986 | "node_modules/log-update": { 987 | "version": "6.1.0", 988 | "resolved": "https://registry.npmjs.org/log-update/-/log-update-6.1.0.tgz", 989 | "integrity": "sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w==", 990 | "dependencies": { 991 | "ansi-escapes": "^7.0.0", 992 | "cli-cursor": "^5.0.0", 993 | "slice-ansi": "^7.1.0", 994 | "strip-ansi": "^7.1.0", 995 | "wrap-ansi": "^9.0.0" 996 | }, 997 | "engines": { 998 | "node": ">=18" 999 | }, 1000 | "funding": { 1001 | "url": "https://github.com/sponsors/sindresorhus" 1002 | } 1003 | }, 1004 | "node_modules/loupe": { 1005 | "version": "3.0.2", 1006 | "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.0.2.tgz", 1007 | "integrity": "sha512-Tzlkbynv7dtqxTROe54Il+J4e/zG2iehtJGZUYpTv8WzlkW9qyEcE83UhGJCeuF3SCfzHuM5VWhBi47phV3+AQ==", 1008 | "dependencies": { 1009 | "get-func-name": "^2.0.1" 1010 | } 1011 | }, 1012 | "node_modules/mime-db": { 1013 | "version": "1.52.0", 1014 | "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", 1015 | "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", 1016 | "engines": { 1017 | "node": ">= 0.6" 1018 | } 1019 | }, 1020 | "node_modules/mime-types": { 1021 | "version": "2.1.35", 1022 | "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", 1023 | "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", 1024 | "dependencies": { 1025 | "mime-db": "1.52.0" 1026 | }, 1027 | "engines": { 1028 | "node": ">= 0.6" 1029 | } 1030 | }, 1031 | "node_modules/mimic-function": { 1032 | "version": "5.0.1", 1033 | "resolved": "https://registry.npmjs.org/mimic-function/-/mimic-function-5.0.1.tgz", 1034 | "integrity": "sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==", 1035 | "engines": { 1036 | "node": ">=18" 1037 | }, 1038 | "funding": { 1039 | "url": "https://github.com/sponsors/sindresorhus" 1040 | } 1041 | }, 1042 | "node_modules/minimist": { 1043 | "version": "1.2.8", 1044 | "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", 1045 | "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", 1046 | "dev": true, 1047 | "funding": { 1048 | "url": "https://github.com/sponsors/ljharb" 1049 | } 1050 | }, 1051 | "node_modules/ms": { 1052 | "version": "2.1.2", 1053 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", 1054 | "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" 1055 | }, 1056 | "node_modules/neon-env": { 1057 | "version": "0.2.2", 1058 | "resolved": "https://registry.npmjs.org/neon-env/-/neon-env-0.2.2.tgz", 1059 | "integrity": "sha512-Pl7qmcZE6klOwpjrOcTZn6ONYFg9S186PWGjPy3apsqLRVBIXloDrAzVKildM3oR4g7ZXaiEtqa/g3JSvWjkJw==", 1060 | "engines": { 1061 | "node": "^14.18.0 || >=16.0.0" 1062 | } 1063 | }, 1064 | "node_modules/onetime": { 1065 | "version": "7.0.0", 1066 | "resolved": "https://registry.npmjs.org/onetime/-/onetime-7.0.0.tgz", 1067 | "integrity": "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==", 1068 | "dependencies": { 1069 | "mimic-function": "^5.0.0" 1070 | }, 1071 | "engines": { 1072 | "node": ">=18" 1073 | }, 1074 | "funding": { 1075 | "url": "https://github.com/sponsors/sindresorhus" 1076 | } 1077 | }, 1078 | "node_modules/packet-reader": { 1079 | "version": "1.0.0", 1080 | "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", 1081 | "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" 1082 | }, 1083 | "node_modules/path-key": { 1084 | "version": "3.1.1", 1085 | "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", 1086 | "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", 1087 | "dev": true, 1088 | "engines": { 1089 | "node": ">=8" 1090 | } 1091 | }, 1092 | "node_modules/path-parse": { 1093 | "version": "1.0.7", 1094 | "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", 1095 | "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" 1096 | }, 1097 | "node_modules/pathval": { 1098 | "version": "2.0.0", 1099 | "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.0.tgz", 1100 | "integrity": "sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA==", 1101 | "engines": { 1102 | "node": ">= 14.16" 1103 | } 1104 | }, 1105 | "node_modules/pg": { 1106 | "version": "8.11.3", 1107 | "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.3.tgz", 1108 | "integrity": "sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==", 1109 | "dependencies": { 1110 | "buffer-writer": "2.0.0", 1111 | "packet-reader": "1.0.0", 1112 | "pg-connection-string": "^2.6.2", 1113 | "pg-pool": "^3.6.1", 1114 | "pg-protocol": "^1.6.0", 1115 | "pg-types": "^2.1.0", 1116 | "pgpass": "1.x" 1117 | }, 1118 | "engines": { 1119 | "node": ">= 8.0.0" 1120 | }, 1121 | "optionalDependencies": { 1122 | "pg-cloudflare": "^1.1.1" 1123 | }, 1124 | "peerDependencies": { 1125 | "pg-native": ">=3.0.1" 1126 | }, 1127 | "peerDependenciesMeta": { 1128 | "pg-native": { 1129 | "optional": true 1130 | } 1131 | } 1132 | }, 1133 | "node_modules/pg-cloudflare": { 1134 | "version": "1.1.1", 1135 | "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", 1136 | "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", 1137 | "optional": true 1138 | }, 1139 | "node_modules/pg-connection-string": { 1140 | "version": "2.6.2", 1141 | "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz", 1142 | "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==" 1143 | }, 1144 | "node_modules/pg-int8": { 1145 | "version": "1.0.1", 1146 | "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", 1147 | "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", 1148 | "engines": { 1149 | "node": ">=4.0.0" 1150 | } 1151 | }, 1152 | "node_modules/pg-pool": { 1153 | "version": "3.6.1", 1154 | "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.1.tgz", 1155 | "integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==", 1156 | "peerDependencies": { 1157 | "pg": ">=8.0" 1158 | } 1159 | }, 1160 | "node_modules/pg-protocol": { 1161 | "version": "1.6.0", 1162 | "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", 1163 | "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==" 1164 | }, 1165 | "node_modules/pg-types": { 1166 | "version": "2.2.0", 1167 | "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", 1168 | "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", 1169 | "dependencies": { 1170 | "pg-int8": "1.0.1", 1171 | "postgres-array": "~2.0.0", 1172 | "postgres-bytea": "~1.0.0", 1173 | "postgres-date": "~1.0.4", 1174 | "postgres-interval": "^1.1.0" 1175 | }, 1176 | "engines": { 1177 | "node": ">=4" 1178 | } 1179 | }, 1180 | "node_modules/pgpass": { 1181 | "version": "1.0.5", 1182 | "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", 1183 | "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", 1184 | "dependencies": { 1185 | "split2": "^4.1.0" 1186 | } 1187 | }, 1188 | "node_modules/postgres-array": { 1189 | "version": "2.0.0", 1190 | "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", 1191 | "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", 1192 | "engines": { 1193 | "node": ">=4" 1194 | } 1195 | }, 1196 | "node_modules/postgres-bytea": { 1197 | "version": "1.0.0", 1198 | "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", 1199 | "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", 1200 | "engines": { 1201 | "node": ">=0.10.0" 1202 | } 1203 | }, 1204 | "node_modules/postgres-date": { 1205 | "version": "1.0.7", 1206 | "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", 1207 | "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", 1208 | "engines": { 1209 | "node": ">=0.10.0" 1210 | } 1211 | }, 1212 | "node_modules/postgres-interval": { 1213 | "version": "1.2.0", 1214 | "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", 1215 | "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", 1216 | "dependencies": { 1217 | "xtend": "^4.0.0" 1218 | }, 1219 | "engines": { 1220 | "node": ">=0.10.0" 1221 | } 1222 | }, 1223 | "node_modules/proxy-from-env": { 1224 | "version": "1.1.0", 1225 | "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", 1226 | "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" 1227 | }, 1228 | "node_modules/rechoir": { 1229 | "version": "0.8.0", 1230 | "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz", 1231 | "integrity": "sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==", 1232 | "dependencies": { 1233 | "resolve": "^1.20.0" 1234 | }, 1235 | "engines": { 1236 | "node": ">= 10.13.0" 1237 | } 1238 | }, 1239 | "node_modules/redis-errors": { 1240 | "version": "1.2.0", 1241 | "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", 1242 | "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", 1243 | "engines": { 1244 | "node": ">=4" 1245 | } 1246 | }, 1247 | "node_modules/redis-parser": { 1248 | "version": "3.0.0", 1249 | "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", 1250 | "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", 1251 | "dependencies": { 1252 | "redis-errors": "^1.0.0" 1253 | }, 1254 | "engines": { 1255 | "node": ">=4" 1256 | } 1257 | }, 1258 | "node_modules/resolve": { 1259 | "version": "1.22.8", 1260 | "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", 1261 | "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", 1262 | "dependencies": { 1263 | "is-core-module": "^2.13.0", 1264 | "path-parse": "^1.0.7", 1265 | "supports-preserve-symlinks-flag": "^1.0.0" 1266 | }, 1267 | "bin": { 1268 | "resolve": "bin/resolve" 1269 | }, 1270 | "funding": { 1271 | "url": "https://github.com/sponsors/ljharb" 1272 | } 1273 | }, 1274 | "node_modules/resolve-from": { 1275 | "version": "5.0.0", 1276 | "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", 1277 | "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", 1278 | "engines": { 1279 | "node": ">=8" 1280 | } 1281 | }, 1282 | "node_modules/resolve-pkg-maps": { 1283 | "version": "1.0.0", 1284 | "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", 1285 | "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", 1286 | "dev": true, 1287 | "funding": { 1288 | "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" 1289 | } 1290 | }, 1291 | "node_modules/restore-cursor": { 1292 | "version": "5.1.0", 1293 | "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-5.1.0.tgz", 1294 | "integrity": "sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==", 1295 | "dependencies": { 1296 | "onetime": "^7.0.0", 1297 | "signal-exit": "^4.1.0" 1298 | }, 1299 | "engines": { 1300 | "node": ">=18" 1301 | }, 1302 | "funding": { 1303 | "url": "https://github.com/sponsors/sindresorhus" 1304 | } 1305 | }, 1306 | "node_modules/server": { 1307 | "resolved": "server", 1308 | "link": true 1309 | }, 1310 | "node_modules/shebang-command": { 1311 | "version": "2.0.0", 1312 | "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", 1313 | "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", 1314 | "dev": true, 1315 | "dependencies": { 1316 | "shebang-regex": "^3.0.0" 1317 | }, 1318 | "engines": { 1319 | "node": ">=8" 1320 | } 1321 | }, 1322 | "node_modules/shebang-regex": { 1323 | "version": "3.0.0", 1324 | "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", 1325 | "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", 1326 | "dev": true, 1327 | "engines": { 1328 | "node": ">=8" 1329 | } 1330 | }, 1331 | "node_modules/signal-exit": { 1332 | "version": "4.1.0", 1333 | "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", 1334 | "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", 1335 | "engines": { 1336 | "node": ">=14" 1337 | }, 1338 | "funding": { 1339 | "url": "https://github.com/sponsors/isaacs" 1340 | } 1341 | }, 1342 | "node_modules/slice-ansi": { 1343 | "version": "7.1.0", 1344 | "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-7.1.0.tgz", 1345 | "integrity": "sha512-bSiSngZ/jWeX93BqeIAbImyTbEihizcwNjFoRUIY/T1wWQsfsm2Vw1agPKylXvQTU7iASGdHhyqRlqQzfz+Htg==", 1346 | "dependencies": { 1347 | "ansi-styles": "^6.2.1", 1348 | "is-fullwidth-code-point": "^5.0.0" 1349 | }, 1350 | "engines": { 1351 | "node": ">=18" 1352 | }, 1353 | "funding": { 1354 | "url": "https://github.com/chalk/slice-ansi?sponsor=1" 1355 | } 1356 | }, 1357 | "node_modules/split2": { 1358 | "version": "4.2.0", 1359 | "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", 1360 | "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", 1361 | "engines": { 1362 | "node": ">= 10.x" 1363 | } 1364 | }, 1365 | "node_modules/standard-as-callback": { 1366 | "version": "2.1.0", 1367 | "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", 1368 | "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" 1369 | }, 1370 | "node_modules/string-width": { 1371 | "version": "7.2.0", 1372 | "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", 1373 | "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", 1374 | "dependencies": { 1375 | "emoji-regex": "^10.3.0", 1376 | "get-east-asian-width": "^1.0.0", 1377 | "strip-ansi": "^7.1.0" 1378 | }, 1379 | "engines": { 1380 | "node": ">=18" 1381 | }, 1382 | "funding": { 1383 | "url": "https://github.com/sponsors/sindresorhus" 1384 | } 1385 | }, 1386 | "node_modules/strip-ansi": { 1387 | "version": "7.1.0", 1388 | "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", 1389 | "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", 1390 | "dependencies": { 1391 | "ansi-regex": "^6.0.1" 1392 | }, 1393 | "engines": { 1394 | "node": ">=12" 1395 | }, 1396 | "funding": { 1397 | "url": "https://github.com/chalk/strip-ansi?sponsor=1" 1398 | } 1399 | }, 1400 | "node_modules/supports-preserve-symlinks-flag": { 1401 | "version": "1.0.0", 1402 | "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", 1403 | "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", 1404 | "engines": { 1405 | "node": ">= 0.4" 1406 | }, 1407 | "funding": { 1408 | "url": "https://github.com/sponsors/ljharb" 1409 | } 1410 | }, 1411 | "node_modules/tarn": { 1412 | "version": "3.0.2", 1413 | "resolved": "https://registry.npmjs.org/tarn/-/tarn-3.0.2.tgz", 1414 | "integrity": "sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==", 1415 | "engines": { 1416 | "node": ">=8.0.0" 1417 | } 1418 | }, 1419 | "node_modules/test": { 1420 | "resolved": "test", 1421 | "link": true 1422 | }, 1423 | "node_modules/tildify": { 1424 | "version": "2.0.0", 1425 | "resolved": "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz", 1426 | "integrity": "sha512-Cc+OraorugtXNfs50hU9KS369rFXCfgGLpfCfvlc+Ud5u6VWmUQsOAa9HbTvheQdYnrdJqqv1e5oIqXppMYnSw==", 1427 | "engines": { 1428 | "node": ">=8" 1429 | } 1430 | }, 1431 | "node_modules/tsx": { 1432 | "version": "4.7.0", 1433 | "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.7.0.tgz", 1434 | "integrity": "sha512-I+t79RYPlEYlHn9a+KzwrvEwhJg35h/1zHsLC2JXvhC2mdynMv6Zxzvhv5EMV6VF5qJlLlkSnMVvdZV3PSIGcg==", 1435 | "dev": true, 1436 | "dependencies": { 1437 | "esbuild": "~0.19.10", 1438 | "get-tsconfig": "^4.7.2" 1439 | }, 1440 | "bin": { 1441 | "tsx": "dist/cli.mjs" 1442 | }, 1443 | "engines": { 1444 | "node": ">=18.0.0" 1445 | }, 1446 | "optionalDependencies": { 1447 | "fsevents": "~2.3.3" 1448 | } 1449 | }, 1450 | "node_modules/undici-types": { 1451 | "version": "5.26.5", 1452 | "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", 1453 | "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", 1454 | "dev": true 1455 | }, 1456 | "node_modules/uuid": { 1457 | "version": "9.0.1", 1458 | "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", 1459 | "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", 1460 | "funding": [ 1461 | "https://github.com/sponsors/broofa", 1462 | "https://github.com/sponsors/ctavan" 1463 | ], 1464 | "bin": { 1465 | "uuid": "dist/bin/uuid" 1466 | } 1467 | }, 1468 | "node_modules/which": { 1469 | "version": "2.0.2", 1470 | "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", 1471 | "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", 1472 | "dev": true, 1473 | "dependencies": { 1474 | "isexe": "^2.0.0" 1475 | }, 1476 | "bin": { 1477 | "node-which": "bin/node-which" 1478 | }, 1479 | "engines": { 1480 | "node": ">= 8" 1481 | } 1482 | }, 1483 | "node_modules/wrap-ansi": { 1484 | "version": "9.0.0", 1485 | "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.0.tgz", 1486 | "integrity": "sha512-G8ura3S+3Z2G+mkgNRq8dqaFZAuxfsxpBB8OCTGRTCtp+l/v9nbFNmCUP1BZMts3G1142MsZfn6eeUKrr4PD1Q==", 1487 | "dependencies": { 1488 | "ansi-styles": "^6.2.1", 1489 | "string-width": "^7.0.0", 1490 | "strip-ansi": "^7.1.0" 1491 | }, 1492 | "engines": { 1493 | "node": ">=18" 1494 | }, 1495 | "funding": { 1496 | "url": "https://github.com/chalk/wrap-ansi?sponsor=1" 1497 | } 1498 | }, 1499 | "node_modules/xtend": { 1500 | "version": "4.0.2", 1501 | "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", 1502 | "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", 1503 | "engines": { 1504 | "node": ">=0.4" 1505 | } 1506 | }, 1507 | "node_modules/yjs": { 1508 | "version": "13.6.10", 1509 | "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.10.tgz", 1510 | "integrity": "sha512-1JcyQek1vaMyrDm7Fqfa+pvHg/DURSbVo4VmeN7wjnTKB/lZrfIPhdCj7d8sboK6zLfRBJXegTjc9JlaDd8/Zw==", 1511 | "dependencies": { 1512 | "lib0": "^0.2.86" 1513 | }, 1514 | "engines": { 1515 | "node": ">=16.0.0", 1516 | "npm": ">=8.0.0" 1517 | }, 1518 | "funding": { 1519 | "type": "GitHub Sponsors ❤", 1520 | "url": "https://github.com/sponsors/dmonad" 1521 | } 1522 | }, 1523 | "server": { 1524 | "version": "1.0.0", 1525 | "license": "ISC", 1526 | "devDependencies": { 1527 | "dotenv-cli": "^7.3.0" 1528 | } 1529 | }, 1530 | "test": { 1531 | "version": "1.0.0", 1532 | "license": "ISC", 1533 | "dependencies": { 1534 | "axios": "^1.6.3", 1535 | "chai": "^5.0.0", 1536 | "ioredis": "^5.3.2", 1537 | "knex": "^3.1.0", 1538 | "lodash": "^4.17.21", 1539 | "log-update": "^6.1.0", 1540 | "neon-env": "^0.2.2", 1541 | "pg": "^8.11.3", 1542 | "uuid": "^9.0.1", 1543 | "yjs": "^13.6.10" 1544 | }, 1545 | "devDependencies": { 1546 | "@types/chai": "^4.3.11", 1547 | "@types/lodash": "^4.17.16", 1548 | "@types/node": "^20.10.5", 1549 | "@types/uuid": "^9.0.7", 1550 | "dotenv-cli": "^7.3.0", 1551 | "tsx": "^4.7.0" 1552 | } 1553 | } 1554 | } 1555 | } 1556 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "workspaces": [ 3 | "server", 4 | "dev", 5 | "test" 6 | ], 7 | "packageManager": "npm@10.5.2", 8 | "name": "k_yrs_go" 9 | } 10 | -------------------------------------------------------------------------------- /port.sh: -------------------------------------------------------------------------------- 1 | lsof -n -i :$1 | grep LISTEN 2 | -------------------------------------------------------------------------------- /server/.air.toml: -------------------------------------------------------------------------------- 1 | # .air.toml 2 | [build] 3 | full_bin = "go run main.go --SERVER_HOST=$SERVER_HOST --SERVER_PORT=$SERVER_PORT --REDIS_URL=$REDIS_URL --PG_URL=$PG_URL --MODE=$MODE --DEBUG=$DEBUG --REDIS_QUEUE_MAX_SIZE=$REDIS_QUEUE_MAX_SIZE" 4 | -------------------------------------------------------------------------------- /server/.env: -------------------------------------------------------------------------------- 1 | # keeping this handy in git since this runs the dev server 2 | 3 | SERVER_HOST=localhost 4 | 5 | SERVER_PORT=3000 6 | 7 | PG_URL=postgres://dev:dev@localhost:5432/k_yrs_dev?sslmode=disable 8 | 9 | REDIS_URL=redis://localhost:6379 10 | 11 | MODE=dev 12 | 13 | DEBUG=true 14 | 15 | REDIS_QUEUE_MAX_SIZE=1000 -------------------------------------------------------------------------------- /server/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /tmp 3 | server -------------------------------------------------------------------------------- /server/db/.gitignore: -------------------------------------------------------------------------------- 1 | libyrs.a 2 | libyrs.h 3 | libyrs.so 4 | -------------------------------------------------------------------------------- /server/db/db.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | /* 4 | #cgo LDFLAGS: -L. -lyrs 5 | #include 6 | #include 7 | */ 8 | import "C" 9 | 10 | import ( 11 | "context" 12 | "database/sql" 13 | "errors" 14 | "fmt" 15 | "log" 16 | "math" 17 | "math/rand" 18 | "time" 19 | "unsafe" 20 | 21 | "github.com/oklog/ulid/v2" 22 | 23 | _ "github.com/lib/pq" 24 | 25 | "github.com/redis/go-redis/v9" 26 | ) 27 | 28 | const DEFAULT_REDIS_QUEUE_KEY = "k_yrs_go.yupdates" 29 | const DEFAULT_REDIS_QUEUE_MAX_SIZE = 1000 30 | 31 | func generateULID() (ulid.ULID, error) { 32 | t := time.Now() 33 | entropy := ulid.Monotonic(rand.New(rand.NewSource(t.UnixNano())), 0) 34 | id, err := ulid.New(ulid.Timestamp(t), entropy) 35 | if err != nil { 36 | return ulid.ULID{}, err // handle error appropriately 37 | } 38 | return id, nil 39 | } 40 | 41 | func byteSliceToCString(b []byte) *C.char { 42 | if len(b) == 0 { 43 | return (*C.char)(C.calloc(1, 1)) // Allocate 1 byte and set it to 0 (null terminator) 44 | } 45 | cstr := (*C.char)(C.malloc(C.size_t(len(b) + 1))) // Allocate memory for the string + null terminator 46 | copy((*[1 << 30]byte)(unsafe.Pointer(cstr))[:], b) // Copy the slice data 47 | (*[1 << 30]byte)(unsafe.Pointer(cstr))[len(b)] = 0 // Set the null terminator 48 | return cstr 49 | } 50 | 51 | func cStringToByteSlice(cstr *C.char) []byte { 52 | if cstr == nil { 53 | return nil 54 | } 55 | length := C.strlen(cstr) 56 | slice := make([]byte, length) 57 | copy(slice, (*[1 << 30]byte)(unsafe.Pointer(cstr))[:length:length]) 58 | return slice 59 | } 60 | 61 | func combineYUpdates(updates [][]byte) []byte { 62 | // Create a document with GC disabled. 63 | opts := C.yoptions() 64 | opts.skip_gc = 1 65 | ydoc := C.ydoc_new_with_options(opts) 66 | 67 | // Apply each update in order. 68 | for _, update := range updates { 69 | cUpdateStr := byteSliceToCString(update) 70 | cBytesLen := C.uint(len(update)) 71 | 72 | wtrx := C.ydoc_write_transaction(ydoc, cBytesLen, cUpdateStr) 73 | C.ytransaction_apply(wtrx, cUpdateStr, cBytesLen) 74 | C.ytransaction_commit(wtrx) 75 | 76 | C.free(unsafe.Pointer(cUpdateStr)) 77 | } 78 | 79 | // Open a read transaction. 80 | rtrx := C.ydoc_read_transaction(ydoc) 81 | 82 | // Obtain a snapshot descriptor of the document state. 83 | var snapshotLen C.uint = 0 84 | cSnapshot := C.ytransaction_snapshot(rtrx, &snapshotLen) 85 | 86 | // Encode the snapshot to a full update. 87 | var encodedLen C.uint = 0 88 | cCombinedUpdate := C.ytransaction_encode_state_from_snapshot_v1(rtrx, cSnapshot, snapshotLen, &encodedLen) 89 | combinedUpdate := C.GoBytes(unsafe.Pointer(cCombinedUpdate), C.int(encodedLen)) 90 | 91 | // Free allocated memory. 92 | C.ybinary_destroy(cCombinedUpdate, encodedLen) 93 | C.ystring_destroy(cSnapshot) 94 | C.ydoc_destroy(ydoc) 95 | 96 | return combinedUpdate 97 | } 98 | 99 | type RedisDB struct { 100 | DB *DB 101 | client *redis.Client 102 | queueKeyPrefix string 103 | queueMaxSize int 104 | } 105 | 106 | func NewRedisDB(url string, redisQueueMaxSize int) (*RedisDB, error) { 107 | opts, err := redis.ParseURL(url) 108 | if err != nil { 109 | return nil, fmt.Errorf("failed to parse redis url: %v", err) 110 | } 111 | 112 | client := redis.NewClient(opts) 113 | return &RedisDB{ 114 | client: client, 115 | queueKeyPrefix: DEFAULT_REDIS_QUEUE_KEY, 116 | queueMaxSize: int(math.Max(DEFAULT_REDIS_QUEUE_MAX_SIZE, float64(redisQueueMaxSize))), 117 | }, nil 118 | } 119 | 120 | func (r *RedisDB) Close() error { 121 | return r.client.Close() 122 | } 123 | 124 | func (r *RedisDB) queueKey(docID string) string { 125 | return fmt.Sprintf("%s.%s", r.queueKeyPrefix, docID) 126 | } 127 | 128 | const REDIS_QUEUE_PUSH_LUA_SCRIPT = ` 129 | local queue_key = KEYS[1] 130 | local data = ARGV[1] 131 | local queue_size = tonumber(redis.call('LLEN', queue_key)) 132 | 133 | if queue_size > tonumber(ARGV[2]) then 134 | redis.call('LPOP', queue_key) 135 | end 136 | 137 | redis.call('RPUSH', queue_key, data) 138 | ` 139 | 140 | func (r *RedisDB) PushYUpdate(ctx context.Context, docID string, update []byte) error { 141 | _, err := r.client.Eval(ctx, REDIS_QUEUE_PUSH_LUA_SCRIPT, []string{r.queueKey(docID)}, update, r.queueMaxSize).Result() 142 | if err != nil { 143 | // Check if the error is the "redis: nil" error 144 | if errors.Is(err, redis.Nil) { 145 | // Treat "redis: nil" as a success scenario 146 | return nil 147 | } 148 | 149 | return fmt.Errorf("failed to push update to Redis queue: %v", err) 150 | } 151 | return nil 152 | } 153 | 154 | func (r *RedisDB) GetYUpdates(ctx context.Context, docID string) ([][]byte, error) { 155 | strUpdates, err := r.client.LRange(ctx, r.queueKey(docID), 0, -1).Result() 156 | if err != nil { 157 | return nil, fmt.Errorf("failed to get yupdates from Redis queue: %v", err) 158 | } 159 | 160 | byteUpdates := make([][]byte, len(strUpdates)) 161 | for i, s := range strUpdates { 162 | byteUpdates[i] = []byte(s) 163 | } 164 | 165 | return byteUpdates, nil 166 | } 167 | 168 | type PGDB struct { 169 | DB *DB 170 | client *sql.DB 171 | } 172 | 173 | type PGCombinedYUpdateRes struct { 174 | CombinedUpdate []byte 175 | LastId string 176 | UpdatesCount int 177 | } 178 | 179 | func NewPGDB(url string) (*PGDB, error) { 180 | db, err := sql.Open("postgres", url) 181 | db.SetMaxOpenConns(100) 182 | if err != nil { 183 | return nil, fmt.Errorf("failed to open postgres db: %v", err) 184 | } 185 | 186 | return &PGDB{ 187 | client: db, 188 | }, nil 189 | } 190 | 191 | func (p *PGDB) Close() error { 192 | return p.client.Close() 193 | } 194 | 195 | func (p *PGDB) Debug(key string, data []byte) error { 196 | if !p.DB.Debug { 197 | return nil 198 | } 199 | 200 | _, err := p.client.Exec("INSERT INTO debug (key, data) VALUES ($1, $2)", key, data) 201 | if err != nil { 202 | return fmt.Errorf("=======> Debug failed to write update to store: %v", err) 203 | } 204 | 205 | return nil 206 | 207 | } 208 | 209 | func (p *PGDB) SetupTables(ctx context.Context) error { 210 | if _, err := p.client.ExecContext(ctx, ` 211 | CREATE TABLE IF NOT EXISTS k_yrs_go_yupdates_store ( 212 | id TEXT PRIMARY KEY, 213 | doc_id TEXT NOT NULL, 214 | data BYTEA NOT NULL 215 | ); 216 | 217 | CREATE INDEX IF NOT EXISTS k_yrs_go_yupdates_store_doc_id_idx ON k_yrs_go_yupdates_store (doc_id); 218 | `); err != nil { 219 | return fmt.Errorf("failed to create store table: %v", err) 220 | } 221 | 222 | log.Printf("created table (if not exists) k_yrs_go_yupdates_store") 223 | 224 | return nil 225 | } 226 | 227 | func (p *PGDB) WriteYUpdateToStore(ctx context.Context, docID string, update []byte) error { 228 | // Begin a new transaction with serializable isolation level. 229 | tx, err := p.client.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelSerializable}) 230 | if err != nil { 231 | return fmt.Errorf("failed to begin serializable transaction: %v", err) 232 | } 233 | 234 | // Ensure rollback if something goes wrong. 235 | defer func() { 236 | if err != nil { 237 | tx.Rollback() 238 | } 239 | }() 240 | 241 | // Generate ULID for the new record. 242 | id, err := generateULID() 243 | if err != nil { 244 | tx.Rollback() 245 | return fmt.Errorf("failed to generate ULID in WriteYUpdateToStore: %v", err) 246 | } 247 | 248 | // Execute the insert within the transaction. 249 | _, err = tx.ExecContext(ctx, "INSERT INTO k_yrs_go_yupdates_store (id, doc_id, data) VALUES ($1, $2, $3)", id.String(), docID, update) 250 | if err != nil { 251 | tx.Rollback() 252 | return fmt.Errorf("failed to write update to store: %v", err) 253 | } 254 | 255 | // Commit the transaction. 256 | if err = tx.Commit(); err != nil { 257 | return fmt.Errorf("failed to commit transaction: %v", err) 258 | } 259 | 260 | return nil 261 | } 262 | 263 | func (p *PGDB) GetCombinedYUpdate(ctx context.Context, docID string) (PGCombinedYUpdateRes, error) { 264 | // Begin a read-only transaction with serializable isolation level. 265 | tx, err := p.client.BeginTx(ctx, &sql.TxOptions{ 266 | Isolation: sql.LevelSerializable, 267 | ReadOnly: true, 268 | }) 269 | if err != nil { 270 | return PGCombinedYUpdateRes{}, fmt.Errorf("failed to begin serializable transaction: %v", err) 271 | } 272 | 273 | // Execute the query within the transaction. 274 | rows, err := tx.QueryContext(ctx, "SELECT id, data FROM k_yrs_go_yupdates_store WHERE doc_id = $1 ORDER BY id ASC", docID) 275 | if err != nil { 276 | tx.Rollback() 277 | return PGCombinedYUpdateRes{}, fmt.Errorf("failed to query updates from store: %v", err) 278 | } 279 | defer rows.Close() 280 | 281 | var lastId string 282 | var updates [][]byte 283 | for rows.Next() { 284 | var id string 285 | var update []byte 286 | if err := rows.Scan(&id, &update); err != nil { 287 | tx.Rollback() 288 | return PGCombinedYUpdateRes{}, fmt.Errorf("failed to scan update: %v", err) 289 | } 290 | lastId = id 291 | updates = append(updates, update) 292 | } 293 | if err := rows.Err(); err != nil { 294 | tx.Rollback() 295 | return PGCombinedYUpdateRes{}, fmt.Errorf("error iterating rows: %v", err) 296 | } 297 | 298 | // Commit the transaction. 299 | if err := tx.Commit(); err != nil { 300 | return PGCombinedYUpdateRes{}, fmt.Errorf("failed to commit transaction: %v", err) 301 | } 302 | 303 | combinedUpdate := combineYUpdates(updates) 304 | return PGCombinedYUpdateRes{ 305 | CombinedUpdate: combinedUpdate, 306 | LastId: lastId, 307 | UpdatesCount: len(updates), 308 | }, nil 309 | } 310 | 311 | func (p *PGDB) PerformCompaction(ctx context.Context, docID string, lastID string, combinedUpdate []byte, pgUpdatesCount int) error { 312 | // Begin a transaction with a serializable isolation level 313 | tx, err := p.client.BeginTx(ctx, &sql.TxOptions{ 314 | Isolation: sql.LevelSerializable, 315 | }) 316 | if err != nil { 317 | return fmt.Errorf("failed to begin transaction: %v", err) 318 | } 319 | 320 | // Ensure the transaction is rolled back in case of an error. 321 | defer func() { 322 | // If an error occurred, attempt to roll back the transaction. 323 | if err != nil { 324 | tx.Rollback() 325 | } 326 | }() 327 | 328 | // Delete rows from the store table within the transaction. 329 | res, err := tx.ExecContext(ctx, "DELETE FROM k_yrs_go_yupdates_store WHERE doc_id = $1 AND id <= $2", docID, lastID) 330 | if err != nil { 331 | return fmt.Errorf("failed to delete rows from store table: %v", err) 332 | } 333 | 334 | rowsDeleted, err := res.RowsAffected() 335 | if err != nil { 336 | return fmt.Errorf("failed to retrieve affected rows: %v", err) 337 | } 338 | 339 | if int(rowsDeleted) == pgUpdatesCount { 340 | // Generate a new ULID for the combined update. 341 | newID, err := generateULID() 342 | if err != nil { 343 | return fmt.Errorf("failed to generate ULID in PerformCompaction: %v", err) 344 | } 345 | 346 | // Insert the combined update into the store table within the same transaction. 347 | _, err = tx.ExecContext(ctx, "INSERT INTO k_yrs_go_yupdates_store (doc_id, id, data) VALUES ($1, $2, $3)", docID, newID.String(), combinedUpdate) 348 | if err != nil { 349 | return fmt.Errorf("failed to insert combined update into store table: %v", err) 350 | } 351 | } 352 | 353 | // Commit the transaction. 354 | if err = tx.Commit(); err != nil { 355 | return fmt.Errorf("failed to commit transaction: %v", err) 356 | } 357 | 358 | return nil 359 | } 360 | 361 | type DB struct { 362 | Redis *RedisDB 363 | PG *PGDB 364 | Debug bool 365 | } 366 | 367 | type DBConfig struct { 368 | RedisURL string 369 | PGURL string 370 | Debug bool 371 | RedisQueueMaxSize int 372 | } 373 | 374 | func NewDB(dbConfig DBConfig) (*DB, error) { 375 | redisDB, err := NewRedisDB(dbConfig.RedisURL, dbConfig.RedisQueueMaxSize) 376 | if err != nil { 377 | return nil, fmt.Errorf("failed to create redis db: %v", err) 378 | } 379 | 380 | pgDB, err := NewPGDB(dbConfig.PGURL) 381 | if err != nil { 382 | return nil, fmt.Errorf("failed to create pg db: %v", err) 383 | } 384 | 385 | db := &DB{ 386 | Redis: redisDB, 387 | PG: pgDB, 388 | Debug: dbConfig.Debug, 389 | } 390 | 391 | redisDB.DB = db 392 | pgDB.DB = db 393 | 394 | return db, nil 395 | } 396 | 397 | func (db *DB) Close() error { 398 | if err := db.Redis.Close(); err != nil { 399 | return fmt.Errorf("failed to close redis db: %v", err) 400 | } 401 | 402 | return nil 403 | } 404 | 405 | type DBCombinedYUpdateRes struct { 406 | CombinedUpdate []byte 407 | ShouldPerformCompaction bool 408 | LastId string 409 | PGUpdatesCount int 410 | } 411 | 412 | func (db *DB) GetCombinedYUpdate(ctx context.Context, docID string) (DBCombinedYUpdateRes, error) { 413 | redisCh := make(chan [][]byte) 414 | pgCh := make(chan *PGCombinedYUpdateRes) 415 | 416 | go func() { 417 | defer close(redisCh) 418 | redisUpdates, err := db.Redis.GetYUpdates(ctx, docID) 419 | if err != nil { 420 | redisCh <- nil 421 | return 422 | } 423 | redisCh <- redisUpdates 424 | }() 425 | 426 | go func() { 427 | defer close(pgCh) 428 | cuRes, err := db.PG.GetCombinedYUpdate(ctx, docID) 429 | if err != nil { 430 | pgCh <- nil 431 | return 432 | } 433 | 434 | pgCh <- &cuRes 435 | }() 436 | 437 | redisUpdates := <-redisCh 438 | pgCombinedUpdateRes := <-pgCh 439 | var res DBCombinedYUpdateRes 440 | 441 | if redisUpdates == nil && pgCombinedUpdateRes == nil { 442 | res = DBCombinedYUpdateRes{ 443 | CombinedUpdate: []byte{}, 444 | ShouldPerformCompaction: false, 445 | } 446 | } else if pgCombinedUpdateRes == nil { 447 | res = DBCombinedYUpdateRes{ 448 | CombinedUpdate: combineYUpdates(redisUpdates), 449 | ShouldPerformCompaction: false, 450 | } 451 | } else if redisUpdates == nil { 452 | res = DBCombinedYUpdateRes{ 453 | CombinedUpdate: pgCombinedUpdateRes.CombinedUpdate, 454 | ShouldPerformCompaction: pgCombinedUpdateRes.UpdatesCount > 100, 455 | LastId: pgCombinedUpdateRes.LastId, 456 | PGUpdatesCount: pgCombinedUpdateRes.UpdatesCount, 457 | } 458 | } else { 459 | combinedUpdate := combineYUpdates(append([][]byte{pgCombinedUpdateRes.CombinedUpdate}, redisUpdates...)) 460 | res = DBCombinedYUpdateRes{ 461 | CombinedUpdate: combinedUpdate, 462 | ShouldPerformCompaction: pgCombinedUpdateRes.UpdatesCount > 100, 463 | LastId: pgCombinedUpdateRes.LastId, 464 | PGUpdatesCount: pgCombinedUpdateRes.UpdatesCount, 465 | } 466 | } 467 | 468 | return res, nil 469 | } 470 | -------------------------------------------------------------------------------- /server/go.mod: -------------------------------------------------------------------------------- 1 | module server 2 | 3 | go 1.21.5 4 | 5 | require ( 6 | github.com/bytedance/sonic v1.10.2 // indirect 7 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 8 | github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect 9 | github.com/chenzhuoyu/iasm v0.9.1 // indirect 10 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 11 | github.com/gabriel-vasile/mimetype v1.4.3 // indirect 12 | github.com/gin-contrib/sse v0.1.0 // indirect 13 | github.com/gin-gonic/gin v1.9.1 // indirect 14 | github.com/go-playground/locales v0.14.1 // indirect 15 | github.com/go-playground/universal-translator v0.18.1 // indirect 16 | github.com/go-playground/validator/v10 v10.16.0 // indirect 17 | github.com/goccy/go-json v0.10.2 // indirect 18 | github.com/json-iterator/go v1.1.12 // indirect 19 | github.com/klauspost/cpuid/v2 v2.2.6 // indirect 20 | github.com/leodido/go-urn v1.2.4 // indirect 21 | github.com/lib/pq v1.10.9 // indirect 22 | github.com/mattn/go-isatty v0.0.20 // indirect 23 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 24 | github.com/modern-go/reflect2 v1.0.2 // indirect 25 | github.com/oklog/ulid/v2 v2.1.0 // indirect 26 | github.com/pelletier/go-toml/v2 v2.1.1 // indirect 27 | github.com/redis/go-redis/v9 v9.3.1 // indirect 28 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 29 | github.com/ugorji/go/codec v1.2.12 // indirect 30 | golang.org/x/arch v0.6.0 // indirect 31 | golang.org/x/crypto v0.17.0 // indirect 32 | golang.org/x/net v0.19.0 // indirect 33 | golang.org/x/sync v0.5.0 // indirect 34 | golang.org/x/sys v0.15.0 // indirect 35 | golang.org/x/text v0.14.0 // indirect 36 | google.golang.org/protobuf v1.32.0 // indirect 37 | gopkg.in/yaml.v3 v3.0.1 // indirect 38 | ) 39 | -------------------------------------------------------------------------------- /server/go.sum: -------------------------------------------------------------------------------- 1 | github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= 2 | github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= 3 | github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE= 4 | github.com/bytedance/sonic v1.10.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= 5 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 6 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 7 | github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= 8 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= 9 | github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0= 10 | github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA= 11 | github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= 12 | github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0= 13 | github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= 14 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 15 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 16 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 17 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 18 | github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= 19 | github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= 20 | github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= 21 | github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= 22 | github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= 23 | github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= 24 | github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= 25 | github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= 26 | github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= 27 | github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= 28 | github.com/go-playground/validator/v10 v10.16.0 h1:x+plE831WK4vaKHO/jpgUGsvLKIqRRkz6M78GuJAfGE= 29 | github.com/go-playground/validator/v10 v10.16.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= 30 | github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= 31 | github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= 32 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 33 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 34 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 35 | github.com/howeyc/fsnotify v0.9.0 h1:0gtV5JmOKH4A8SsFxG2BczSeXWWPvcMT0euZt5gDAxY= 36 | github.com/howeyc/fsnotify v0.9.0/go.mod h1:41HzSPxBGeFRQKEEwgh49TRw/nKBsYZ2cF1OzPjSJsA= 37 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 38 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 39 | github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 40 | github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= 41 | github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= 42 | github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= 43 | github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= 44 | github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= 45 | github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= 46 | github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 47 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 48 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 49 | github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= 50 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 51 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 52 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 53 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 54 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 55 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 56 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 57 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 58 | github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU= 59 | github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= 60 | github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= 61 | github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= 62 | github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= 63 | github.com/pilu/config v0.0.0-20131214182432-3eb99e6c0b9a h1:Tg4E4cXPZSZyd3H1tJlYo6ZreXV0ZJvE/lorNqyw1AU= 64 | github.com/pilu/config v0.0.0-20131214182432-3eb99e6c0b9a/go.mod h1:9Or9aIl95Kp43zONcHd5tLZGKXb9iLx0pZjau0uJ5zg= 65 | github.com/pilu/fresh v0.0.0-20190826141211-0fa698148017 h1:XXDLZIIt9NqdeIEva0DM+z1npM0Tsx6h5TYqwNvXfP0= 66 | github.com/pilu/fresh v0.0.0-20190826141211-0fa698148017/go.mod h1:2LLTtftTZSdAPR/iVyennXZDLZOYzyDn+T0qEKJ8eSw= 67 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 68 | github.com/redis/go-redis/v9 v9.3.1 h1:KqdY8U+3X6z+iACvumCNxnoluToB+9Me+TvyFa21Mds= 69 | github.com/redis/go-redis/v9 v9.3.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= 70 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 71 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 72 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 73 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 74 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 75 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 76 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 77 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 78 | github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 79 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 80 | github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= 81 | github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= 82 | github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= 83 | github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= 84 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= 85 | golang.org/x/arch v0.6.0 h1:S0JTfE48HbRj80+4tbvZDYsJ3tGv6BUU3XxyZ7CirAc= 86 | golang.org/x/arch v0.6.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= 87 | golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= 88 | golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= 89 | golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= 90 | golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= 91 | golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= 92 | golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 93 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= 94 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 95 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 96 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 97 | golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= 98 | golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 99 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 100 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 101 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 102 | google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= 103 | google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= 104 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 105 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 106 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 107 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 108 | nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= 109 | rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= 110 | -------------------------------------------------------------------------------- /server/healthz.ts: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | 3 | const check = async (intervalMs: number, iters=10) => { 4 | if (iters === 0) { 5 | throw new Error('healthcheck failed') 6 | } 7 | 8 | try { 9 | await axios.get(`http://127.0.0.1:3000/healthz`) 10 | } catch (err) { 11 | await new Promise(resolve => setTimeout(resolve, intervalMs)); 12 | await check(intervalMs, iters-1); 13 | } 14 | } 15 | 16 | check(3000); -------------------------------------------------------------------------------- /server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "log" 9 | "net/http" 10 | "server/db" 11 | 12 | "github.com/gin-gonic/gin" 13 | "golang.org/x/sync/errgroup" 14 | ) 15 | 16 | const DEFAULT_SERVE_PORT = 3000 17 | const DEFAULT_MODE = "prod" 18 | 19 | var ( 20 | serverHost string 21 | serverPort int 22 | redisURL string 23 | pgURL string 24 | mode string 25 | user string 26 | password string 27 | debug bool 28 | redisQueueMaxSize int 29 | ) 30 | 31 | var ( 32 | dbh *db.DB 33 | ) 34 | 35 | var ( 36 | dataConsistencyErrChan chan error 37 | ) 38 | 39 | func init() { 40 | flag.StringVar(&serverHost, "SERVER_HOST", "localhost", "Server host") 41 | flag.IntVar(&serverPort, "SERVER_PORT", 3000, "Server port") 42 | flag.StringVar(&redisURL, "REDIS_URL", "redis://localhost:6379", "Redis URL") 43 | flag.StringVar(&pgURL, "PG_URL", "", "PostgreSQL URL") 44 | flag.StringVar(&mode, "MODE", DEFAULT_MODE, "Mode") 45 | flag.StringVar(&user, "USER", "", "User") 46 | flag.StringVar(&password, "PASSWORD", "", "Password") 47 | flag.BoolVar(&debug, "DEBUG", false, "Debug") 48 | flag.IntVar(&redisQueueMaxSize, "REDIS_QUEUE_MAX_SIZE", 100, "Redis Queue Max Size") 49 | flag.Parse() 50 | } 51 | 52 | func init() { 53 | var err error 54 | dbh, err = db.NewDB(db.DBConfig{ 55 | RedisURL: redisURL, 56 | PGURL: pgURL, 57 | Debug: debug, 58 | RedisQueueMaxSize: redisQueueMaxSize, 59 | }) 60 | if err != nil { 61 | panic(err) 62 | } 63 | } 64 | 65 | func init() { 66 | dataConsistencyErrChan = make(chan error, 1) 67 | } 68 | 69 | func setupRouter() *gin.Engine { 70 | // gin.DisableConsoleColor() 71 | // gin.SetMode(gin.Mode()) 72 | r := gin.Default() 73 | 74 | r.SetTrustedProxies(nil) 75 | 76 | if user != "" && password != "" { 77 | r.Use(gin.BasicAuth(gin.Accounts{ 78 | user: password, 79 | })) 80 | } 81 | 82 | r.GET("/healthz", func(c *gin.Context) { 83 | c.String(http.StatusOK, "ok") 84 | }) 85 | 86 | r.GET("/docs/:id/updates", func(c *gin.Context) { 87 | docID := c.Params.ByName("id") 88 | res, err := dbh.GetCombinedYUpdate(c.Request.Context(), docID) 89 | if err != nil { 90 | c.String(http.StatusInternalServerError, "Error reading updates from redis") 91 | return 92 | } 93 | 94 | c.Data(http.StatusOK, "application/octet-stream", res.CombinedUpdate) 95 | 96 | if res.ShouldPerformCompaction { 97 | err = dbh.PG.PerformCompaction(c.Request.Context(), docID, res.LastId, res.CombinedUpdate, res.PGUpdatesCount) 98 | if err != nil { 99 | dataConsistencyErrChan <- fmt.Errorf("error performing compaction: %v", err) 100 | } 101 | } 102 | }) 103 | 104 | r.POST("/docs/:id/updates", func(c *gin.Context) { 105 | contentType := c.Request.Header.Get("Content-Type") 106 | if contentType != "application/octet-stream" { 107 | c.String(http.StatusBadRequest, "Invalid content type") 108 | return 109 | } 110 | 111 | docID := c.Params.ByName("id") 112 | 113 | body, err := io.ReadAll(c.Request.Body) 114 | if err != nil { 115 | c.String(http.StatusInternalServerError, "Error reading request body") 116 | return 117 | } 118 | 119 | ackErrgroup, ackEgctx := errgroup.WithContext(c.Request.Context()) 120 | ackErrgroup.SetLimit(1) 121 | 122 | ackErrgroup.Go(func() error { 123 | return dbh.Redis.PushYUpdate(ackEgctx, docID, body) 124 | }) 125 | 126 | commitErrgroup, commitEgctx := errgroup.WithContext(c.Request.Context()) 127 | commitErrgroup.SetLimit(1) 128 | 129 | commitErrgroup.Go(func() error { 130 | return dbh.PG.WriteYUpdateToStore(commitEgctx, docID, body) 131 | }) 132 | 133 | err = ackErrgroup.Wait() 134 | if err != nil { 135 | c.String(http.StatusInternalServerError, "Error writing updates: %v", err) 136 | return 137 | } 138 | 139 | c.String(http.StatusOK, "ok") 140 | 141 | err = commitErrgroup.Wait() 142 | if err != nil { 143 | dataConsistencyErrChan <- fmt.Errorf("error writing doc:%s updates to store: %v", docID, err) 144 | } 145 | }) 146 | 147 | log.Print("Router setup complete\n") 148 | 149 | return r 150 | } 151 | 152 | func main() { 153 | log.Printf("Starting server on port %d\n\n", serverPort) 154 | 155 | tablesSetupChan := make(chan struct{}) 156 | serverErrChan := make(chan error) 157 | ctx := context.Background() 158 | 159 | go func() { 160 | err := dbh.PG.SetupTables(ctx) 161 | if err != nil { 162 | serverErrChan <- fmt.Errorf("error setting up tables: %v", err) 163 | close(serverErrChan) 164 | } 165 | 166 | tablesSetupChan <- struct{}{} 167 | close(tablesSetupChan) 168 | }() 169 | 170 | go func() { 171 | r := setupRouter() 172 | serverErrChan <- r.Run(fmt.Sprintf("%s:%d", serverHost, serverPort)) 173 | close(serverErrChan) 174 | }() 175 | 176 | <-tablesSetupChan 177 | 178 | go func() { 179 | for err := range dataConsistencyErrChan { 180 | log.Printf("Data consistency error: %v", err) 181 | } 182 | }() 183 | 184 | select { 185 | case err := <-serverErrChan: 186 | log.Fatalf("Error running server: %v", err) 187 | case <-ctx.Done(): 188 | log.Println("Server stopped") 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /server/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "server", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "dev": "LD_LIBRARY_PATH=./db dotenv -- air", 8 | "server": "./server.sh", 9 | "setup_ffi": "./setup_ffi.sh", 10 | "build": "go build", 11 | "healthz": "tsx healthz.ts" 12 | }, 13 | "keywords": [], 14 | "author": "", 15 | "license": "ISC", 16 | "devDependencies": { 17 | "dotenv-cli": "^7.3.0" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /server/server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -a 4 | source .env 5 | set +a 6 | 7 | LD_LIBRARY_PATH=./db ./server --SERVER_HOST=$SERVER_HOST --SERVER_PORT=$SERVER_PORT --REDIS_URL=$REDIS_URL --PG_URL=$PG_URL --MODE=$MODE --DEBUG=$DEBUG --REDIS_QUEUE_MAX_SIZE=$REDIS_QUEUE_MAX_SIZE -------------------------------------------------------------------------------- /server/setup_ffi.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | serverdir="$(pwd)" 4 | ycrdtdir="$serverdir/../y-crdt" 5 | 6 | cd $ycrdtdir 7 | ls 8 | cargo build --release 9 | cp $ycrdtdir/target/release/{libyrs.a,libyrs.so} $serverdir/db 10 | cp $ycrdtdir/tests-ffi/include/libyrs.h $serverdir/db 11 | cd $serverdir -------------------------------------------------------------------------------- /system_config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kapv89/k_yrs_go/7d9a9eb321880ee749fa7590d547cdfdb9058d5d/system_config.png -------------------------------------------------------------------------------- /test/.env: -------------------------------------------------------------------------------- 1 | SERVER_URL=http://0.0.0.0:3000 2 | PG_URL="postgres://dev:dev@localhost:5432/k_yrs_dev?sslmode=disable" 3 | REDIS_URL=redis://localhost:6379 -------------------------------------------------------------------------------- /test/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "dotenv -- tsx --test test.ts", 8 | "test:compaction:stability": "RW_ITERS=0 CONSISTENCY_LOAD_TEST_ITERS=0 CONSISTENCY_SIMPLE_ITERS=0 LARGE_DOC_TEST_ITERS=0 COMPACTION_ITERS=100 dotenv -- tsx --test test.ts", 9 | "test:consistency": "RW_ITERS=0 COMPACTION_ITERS=0 CONSISTENCY_LOAD_TEST_ITERS=10 CONSISTENCY_SIMPLE_ITERS=0 LARGE_DOC_TEST_ITERS=0 dotenv -- tsx --test test.ts", 10 | "test:large_doc": "NODE_OPTIONS=\"--max-old-space-size=8192\" RW_ITERS=0 COMPACTION_ITERS=0 CONSISTENCY_LOAD_TEST_ITERS=0 CONSISTENCY_SIMPLE_ITERS=0 LARGE_DOC_TEST_ITERS=1 dotenv -- tsx --test test.ts" 11 | }, 12 | "author": "", 13 | "license": "ISC", 14 | "devDependencies": { 15 | "@types/chai": "^4.3.11", 16 | "@types/lodash": "^4.17.16", 17 | "@types/node": "^20.10.5", 18 | "@types/uuid": "^9.0.7", 19 | "dotenv-cli": "^7.3.0", 20 | "tsx": "^4.7.0" 21 | }, 22 | "dependencies": { 23 | "axios": "^1.6.3", 24 | "chai": "^5.0.0", 25 | "ioredis": "^5.3.2", 26 | "knex": "^3.1.0", 27 | "lodash": "^4.17.21", 28 | "log-update": "^6.1.0", 29 | "neon-env": "^0.2.2", 30 | "pg": "^8.11.3", 31 | "uuid": "^9.0.1", 32 | "yjs": "^13.6.10" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /test/psql.sh: -------------------------------------------------------------------------------- 1 | PGPASSWORD=dev psql -U dev -h localhost -d k_yrs_dev 2 | -------------------------------------------------------------------------------- /test/test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, before, after } from "node:test"; 2 | import { v4 as uuid } from "uuid"; 3 | import * as Y from 'yjs'; 4 | import axios, { AxiosResponse } from 'axios'; 5 | import Redis from 'ioredis'; 6 | import knex from 'knex'; 7 | import {expect} from "chai"; 8 | import { randomInt, randomUUID } from "crypto"; 9 | import { createEnv } from 'neon-env'; 10 | import { zip } from "lodash"; 11 | import logUpdate from 'log-update'; 12 | 13 | process.on('unhandledRejection', (err) => { 14 | console.error(err); 15 | }) 16 | 17 | const log = (key: string, value?: any) => typeof value !== "undefined" ? console.log(`======> ${key}`, value) : console.log(`======> ${key}:`); 18 | 19 | const defaults = { 20 | SERVER_URL: 'http://localhost:3000', 21 | PG_URL: 'postgres://dev:dev@localhost:5432/k_yrs_dev?sslmode=disable', 22 | REDIS_URL: 'redis://localhost:6379', 23 | 24 | RW_ITERS: 1, 25 | RW_Y_OPS_WAIT_MS: 0, 26 | 27 | COMPACTION_ITERS: 1, 28 | COMPACTION_YDOC_UPDATE_INTERVAL_MS: 0, 29 | COMPACTION_YDOC_UPDATE_ITERS: 10000, 30 | COMPACTION_Y_OPS_WAIT_MS: 0, 31 | 32 | CONSISTENCY_SIMPLE_ITERS: 1, 33 | CONSISTENCY_SIMPLE_READ_TIMEOUT_MS: 0, 34 | CONSISTENCY_SIMPLE_YDOC_UPDATE_ITERS: 10000, 35 | 36 | CONSISTENCY_LOAD_TEST_ITERS: 1, 37 | CONSISTENCY_LOAD_YDOC_UPDATE_ITERS: 10000, 38 | CONSISTENCY_LOAD_YDOC_UPDATE_TIMEOUT_MS: 2, 39 | CONSISTENCY_LOAD_READ_PER_N_WRITES: 5, 40 | CONSISTENCY_LOAD_YDOC_READ_TIMEOUT_MS: 3, 41 | 42 | LARGE_DOC_TEST_ITERS: 1, 43 | LARGE_DOC_MAX_DOC_SIZE_MB: 100, 44 | LARGE_DOC_CHECK_DOC_SIZE_PER_ITER: 10000, 45 | LARGE_DOC_YDOC_WRITE_INTERVAL_MS: 0, 46 | LARGE_DOC_YDOC_READ_TIMEOUT_MS: 0 47 | } as const; 48 | 49 | type ConfigSchema = { 50 | [K in keyof T]: { 51 | type: T[K] extends number ? 'number' : T[K] extends string ? 'string' : never; 52 | default: T[K]; 53 | }; 54 | }; 55 | 56 | function createEnvSchema(obj: T): ConfigSchema { 57 | return Object.keys(obj).reduce((acc, key) => { 58 | // Cast key to keyof T for proper type inference 59 | const typedKey = key as keyof T; 60 | const value = obj[typedKey]; 61 | let type: 'number' | 'string'; 62 | if (typeof value === 'string') { 63 | type = 'string'; 64 | } else if (typeof value === 'number') { 65 | type = 'number'; 66 | } else { 67 | throw new Error(`Unsupported type for key ${key}`); 68 | } 69 | return { 70 | ...acc, 71 | [typedKey]: { type, default: value }, 72 | }; 73 | }, {} as ConfigSchema); 74 | } 75 | 76 | 77 | const env = createEnv(createEnvSchema(defaults)); 78 | 79 | log(`env`, env); 80 | 81 | const wait = async (ms: number) => { if (ms === 0) { return; } else { await new Promise(resolve => setTimeout(resolve, ms)); } }; 82 | const api = axios.create({ baseURL: env.SERVER_URL }); 83 | let redis: Redis; 84 | let db: knex.Knex 85 | 86 | async function deleteAllRows() { 87 | try { 88 | await db.schema.createTableIfNotExists("debug", (t) => { 89 | t.bigIncrements('id').primary(); 90 | t.text('key'); 91 | t.binary('data'); 92 | }) 93 | 94 | await db('k_yrs_go_yupdates_store').truncate(); 95 | await db('debug').truncate(); 96 | console.log('k_yrs_go tables truncated successfully.'); 97 | } catch (error) { 98 | console.error('Error deleting rows:', error); 99 | } 100 | } 101 | 102 | const debugPromises: Promise[] = []; 103 | async function debug(key: string, data: Uint8Array | Buffer) { 104 | debugPromises.push((async () => { db('debug').insert({key, data}); })()); 105 | } 106 | 107 | before(async () => { 108 | redis = new Redis(env.REDIS_URL); 109 | 110 | // Clear all keys 111 | await redis.flushall(); 112 | log('redis cleared successfully.'); 113 | 114 | db = knex({ 115 | client: 'pg', 116 | connection: env.PG_URL, 117 | }); 118 | 119 | await deleteAllRows(); 120 | log('all tables truncated') 121 | }); 122 | 123 | after(async () => { 124 | // Close the Redis connection 125 | redis.disconnect(); 126 | 127 | await Promise.all(debugPromises) 128 | // Close PG connection 129 | await db.destroy(); 130 | }); 131 | 132 | new Array(env.RW_ITERS).fill(0).forEach((_, i) => { 133 | describe(`write and read iter: ${i}`, () => { 134 | const docId = uuid(); 135 | const ydoc = new Y.Doc(); 136 | 137 | before(() => { 138 | ydoc.on('update', async (update: Uint8Array, origin: any, doc: Y.Doc) => { 139 | try { 140 | await Promise.all([ 141 | (async () => { 142 | const res = await api.post(`/docs/${docId}/updates`, update, {headers: {'Content-Type': 'application/octet-stream'}}) 143 | log("update sent, response: ", res.data) 144 | })(), 145 | (async () => { 146 | log('debug table written'); 147 | })() 148 | ]) 149 | } catch (err) { 150 | if (axios.isAxiosError(err)) { 151 | log("error sending update", err.response?.data) 152 | } else { 153 | log("error sending update", err) 154 | } 155 | } 156 | }) 157 | }) 158 | 159 | it(`persists simple list`, async () => { 160 | const yarray = ydoc.getArray('simple_list'); 161 | yarray.insert(0, ['a', 'b', 'c']); 162 | 163 | await wait(env.RW_Y_OPS_WAIT_MS); 164 | 165 | yarray.insert(yarray.length, ['d', 'e', 'f']) 166 | 167 | await wait(env.RW_Y_OPS_WAIT_MS); 168 | 169 | const response = await api.get(`/docs/${docId}/updates`, { responseType: 'arraybuffer' }); 170 | const update = new Uint8Array(response.data); 171 | 172 | const ydoc2 = new Y.Doc(); 173 | Y.applyUpdate(ydoc2, update); 174 | const yarray2 = ydoc2.getArray('simple_list'); 175 | 176 | for (let i = 0; i < yarray2.length; i++) { 177 | expect(yarray2.get(i)).to.equal(yarray.get(i)); 178 | } 179 | }) 180 | 181 | after(() => { 182 | ydoc.destroy(); 183 | }) 184 | }) 185 | }) 186 | 187 | new Array(env.COMPACTION_ITERS).fill(0).forEach((_, i) => { 188 | describe(`compaction iter ${i}`, () => { 189 | const docId = uuid(); 190 | const ydoc = new Y.Doc(); 191 | 192 | log("starting compaction test suite") 193 | 194 | before(() => { 195 | ydoc.on('update', async (update: Uint8Array, origin: any, doc: Y.Doc) => { 196 | try { 197 | await api.post(`/docs/${docId}/updates`, update, {headers: {'Content-Type': 'application/octet-stream'}}) 198 | } catch (err) { 199 | if (axios.isAxiosError(err)) { 200 | log('error sending update', err.response?.data) 201 | } else { 202 | log("error sending update", err) 203 | } 204 | } 205 | }) 206 | }); 207 | 208 | it(`performs compaction in db: iter ${i}`, async () => { 209 | const yintlist = ydoc.getArray('int_list'); 210 | const ystrlist = ydoc.getArray('str_list'); 211 | 212 | const p = new Promise((resolve) => { 213 | let iter = 0; 214 | const t = setInterval(() => { 215 | logUpdate(`compaction ydoc update iter ${iter}`) 216 | ydoc.transact(() => { 217 | yintlist.insert(yintlist.length, [randomInt(10 ** 6), randomInt(10 ** 6)]) 218 | ystrlist.insert(ystrlist.length, [randomUUID().toString(), randomUUID().toString()]) 219 | }) 220 | 221 | if (iter === env.COMPACTION_YDOC_UPDATE_ITERS) { 222 | clearInterval(t); 223 | resolve(); 224 | } 225 | 226 | iter++; 227 | }, env.COMPACTION_YDOC_UPDATE_INTERVAL_MS); 228 | }); 229 | 230 | await p; 231 | 232 | await wait(env.COMPACTION_Y_OPS_WAIT_MS); 233 | 234 | let countRes = await db('k_yrs_go_yupdates_store').where('doc_id', docId).count('id'); 235 | let rowsInDB = Number(countRes[0].count) 236 | 237 | expect(rowsInDB).to.greaterThan(100); 238 | 239 | const [response, response2] = await new Promise<[AxiosResponse, AxiosResponse]>(async (resolve) => { 240 | const t1 = new Date().getTime() 241 | const promises = [ 242 | api.get(`/docs/${docId}/updates`, { responseType: 'arraybuffer' }), 243 | api.get(`/docs/${docId}/updates`, { responseType: 'arraybuffer' }) 244 | ] as const; 245 | const t2 = new Date().getTime(); 246 | 247 | log('compaction: diff between the 2 GET `/docs{$docId}/updates calls (ms)', t2-t1); 248 | 249 | resolve(await Promise.all(promises)) 250 | }); 251 | 252 | // const response = await api.get(`/docs/${docId}/updates`, { responseType: 'arraybuffer' }); 253 | const update = new Uint8Array(response.data); 254 | const ydoc2 = new Y.Doc(); 255 | Y.applyUpdate(ydoc2, update); 256 | 257 | await wait(env.COMPACTION_Y_OPS_WAIT_MS); 258 | 259 | const yintlist2 = ydoc2.getArray('int_list'); 260 | const ystrlist2 = ydoc2.getArray('str_list'); 261 | 262 | type Diff = { 263 | index: number, 264 | expected: string | number, 265 | actual: string | number 266 | } 267 | 268 | const intlistdiffs: Diff[] = [] 269 | for (let i=0; i < yintlist.length; i++) { 270 | const expected = yintlist.get(i); 271 | const actual = yintlist2.get(i); 272 | 273 | if (expected !== actual) { 274 | intlistdiffs.push({index: i, expected, actual}); 275 | } 276 | } 277 | 278 | const strlistdiffs: Diff[] = [] 279 | for (let i=0; i < ystrlist.length; i++) { 280 | const expected = ystrlist.get(i); 281 | const actual = ystrlist2.get(i); 282 | 283 | if (expected !== actual) { 284 | strlistdiffs.push({index: i, expected, actual}); 285 | } 286 | } 287 | 288 | log('intlistdiffs', intlistdiffs); 289 | log('strlistdiffs', strlistdiffs); 290 | 291 | expect(intlistdiffs.length).to.equal(0); 292 | expect(strlistdiffs.length).to.equal(0); 293 | 294 | await wait(env.COMPACTION_Y_OPS_WAIT_MS); 295 | 296 | countRes = await db('k_yrs_go_yupdates_store').where('doc_id', docId).count('id') 297 | rowsInDB = Number(countRes[0].count); 298 | 299 | expect(rowsInDB).to.lessThanOrEqual(100); 300 | 301 | // const response2 = await api.get(`/docs/${docId}/updates`, { responseType: 'arraybuffer' }); 302 | const update2 = new Uint8Array(response2.data); 303 | 304 | zip(update, update2).forEach(([u1, u2]) => { 305 | expect(u1).to.equal(u2) 306 | }) 307 | }) 308 | 309 | after(() => { 310 | ydoc.destroy(); 311 | }) 312 | }) 313 | }); 314 | 315 | new Array(env.CONSISTENCY_SIMPLE_ITERS).fill(0).forEach((_, i) => { 316 | describe(`consistency simple- iter ${i}`, () => { 317 | log("starting consistency test suite") 318 | 319 | it(`writes and reads work consistently iter:${i}`, async () => { 320 | const docId = uuid(); 321 | const ydoc = new Y.Doc(); 322 | const ymap = ydoc.getMap('ymap'); 323 | 324 | let onYDocUpdateIter = 0; 325 | const onYDocUpdatePromises: Promise[] = []; 326 | ydoc.on('update', async (update: Uint8Array, origin: any, doc: Y.Doc) => { 327 | try { 328 | ((iter: number) => { 329 | onYDocUpdatePromises.push(new Promise(async (resolve) => { 330 | await api.post(`/docs/${docId}/updates`, update, {headers: {'Content-Type': 'application/octet-stream'}}) 331 | resolve(iter) 332 | })) 333 | })(onYDocUpdateIter) 334 | } catch (err) { 335 | if (axios.isAxiosError(err)) { 336 | log('error sending update', err.response?.data) 337 | } else { 338 | log("error sending update", err) 339 | } 340 | } finally { 341 | onYDocUpdateIter++; 342 | } 343 | }) 344 | 345 | const mismatches: {want: number | undefined, got: number | undefined}[] = []; 346 | const times: number[] = [] 347 | let n = 0; 348 | const writeAndConfirm = async () => { 349 | if (n == env.CONSISTENCY_SIMPLE_YDOC_UPDATE_ITERS) { 350 | return; 351 | } 352 | 353 | const t1 = new Date().getTime(); 354 | 355 | ymap.set('n', n++); 356 | 357 | const waitForOnYDocUpdate = new Promise((resolve) => { 358 | const t = setInterval(() => { 359 | if (onYDocUpdatePromises.length > 0) { 360 | clearInterval(t); 361 | resolve(); 362 | } 363 | }, 0) 364 | }); 365 | 366 | await waitForOnYDocUpdate; 367 | 368 | // validate from db 369 | await new Promise((resolve) => { 370 | setTimeout(async () => { 371 | const p = onYDocUpdatePromises.shift(); 372 | logUpdate(`simple consistency onYDocUpdate iter: ${(await p)}`); 373 | 374 | const ydoc2 = new Y.Doc(); 375 | const res = await api.get(`/docs/${docId}/updates`, { responseType: 'arraybuffer' }); 376 | const update = new Uint8Array(res.data); 377 | Y.applyUpdate(ydoc2, update); 378 | 379 | const ymap2 = ydoc2.getMap('ymap'); 380 | 381 | if (ymap2.get('n') === undefined || ymap.get('n') === undefined || (ymap2.get('n') as number) !== (ymap.get('n') as number)){ 382 | mismatches.push({want: ymap.get('n'), got: ymap2.get('n')}); 383 | } 384 | 385 | const t2 = new Date().getTime(); 386 | times.push(t2-t1); 387 | resolve(); 388 | ydoc2.destroy(); 389 | }, env.CONSISTENCY_SIMPLE_READ_TIMEOUT_MS); 390 | }) 391 | 392 | await writeAndConfirm(); 393 | } 394 | 395 | await writeAndConfirm(); 396 | 397 | log(`mismatches`, mismatches); 398 | log(`average time per write+read+check (ms)`, (times.reduce((sum, t) => sum + t, 0) / times.length)) 399 | expect(mismatches.length).to.equal(0); 400 | }) 401 | }) 402 | }) 403 | 404 | new Array(env.CONSISTENCY_LOAD_TEST_ITERS).fill(0).forEach((_, i) => { 405 | describe(`consistency load - iter: ${i}`, () => { 406 | it(`writes and reads work consistently in load - iter: ${i}`, async () => { 407 | const docId = uuid(); 408 | const ydoc = new Y.Doc(); 409 | const yintmap = ydoc.getMap('int_map'); 410 | 411 | const onYDocUpdatePromises: Promise[] = []; 412 | ydoc.on('update', async (update: Uint8Array, origin: any, doc: Y.Doc) => { 413 | onYDocUpdatePromises.push(Promise.resolve()); 414 | try { 415 | await api.post(`/docs/${docId}/updates`, update, {headers: {'Content-Type': 'application/octet-stream'}}) 416 | } catch (err) { 417 | if (axios.isAxiosError(err)) { 418 | log('error sending update', err.response?.data) 419 | } else { 420 | log("error sending update", err) 421 | } 422 | } 423 | }) 424 | 425 | 426 | const mismatches: {want: number, got: number | undefined}[] = []; 427 | const readPromises: Promise[] = []; 428 | 429 | let n = 0; 430 | const write = async () => { 431 | if (n == env.CONSISTENCY_LOAD_YDOC_UPDATE_ITERS) { 432 | return; 433 | } 434 | 435 | yintmap.set('n', n); 436 | 437 | const waitForYDocUpdate = new Promise((resolve) => { 438 | const t = setInterval(async () => { 439 | if (onYDocUpdatePromises.length > 0) { 440 | const p = onYDocUpdatePromises.shift(); 441 | await p; 442 | resolve(); 443 | clearInterval(t); 444 | } 445 | }, 0) 446 | }) 447 | 448 | await waitForYDocUpdate; 449 | 450 | if (n > 0 && n % env.CONSISTENCY_LOAD_READ_PER_N_WRITES === 0) { 451 | ((n) => { 452 | setTimeout(() => { 453 | readPromises.push(checkPersistedYDoc(n)); 454 | }, env.CONSISTENCY_LOAD_YDOC_READ_TIMEOUT_MS); 455 | })(n); 456 | } 457 | 458 | n++; 459 | await wait(env.CONSISTENCY_LOAD_YDOC_UPDATE_TIMEOUT_MS) 460 | await write(); 461 | } 462 | 463 | const checkPersistedYDoc = async (n: number) => { 464 | logUpdate(`consistency in load checkPersistedYDoc n received: ${n}`); 465 | const ydoc2 = new Y.Doc(); 466 | const res = await api.get(`/docs/${docId}/updates`, { responseType: 'arraybuffer' }); 467 | const update = new Uint8Array(res.data); 468 | Y.applyUpdate(ydoc2, update); 469 | const yintmap2 = ydoc2.getMap('int_map'); 470 | 471 | if (yintmap2.get('n') === undefined || (yintmap2.get('n') as number) < n) { 472 | // we check for only < n because fresher data can be read 473 | mismatches.push({want: n, got: yintmap2.get('n')}) 474 | } 475 | } 476 | 477 | await write(); 478 | await Promise.all(readPromises); 479 | 480 | log("mismatches: ", mismatches); 481 | expect(mismatches.length).equal(0) 482 | }) 483 | }) 484 | }); 485 | 486 | new Array(env.LARGE_DOC_TEST_ITERS).fill(0).map((_, i) => { 487 | describe(`large docs iter: ${i}`, () => { 488 | it(`works well for ${env.LARGE_DOC_MAX_DOC_SIZE_MB}MB docs`, async () => { 489 | log(`large docs test for ${env.LARGE_DOC_MAX_DOC_SIZE_MB}MB`) 490 | 491 | const docId = uuid(); 492 | const ydoc = new Y.Doc(); 493 | const yrecordlist = ydoc.getArray>('record_list'); 494 | 495 | const onYDocUpdatePromises: Promise[] = []; 496 | ydoc.on('update', async (update: Uint8Array, origin: any, doc: Y.Doc) => { 497 | onYDocUpdatePromises.push((async () => { await api.post(`/docs/${docId}/updates`, update, {headers: {'Content-Type': 'application/octet-stream'}}) })()); 498 | }) 499 | 500 | 501 | const keys: string[] = new Array(10).fill(0).map((_, i) => `n${i}`) 502 | 503 | let docSizeInBytes: number = 0 504 | let i = 0; 505 | 506 | const writesPromise = new Promise((resolve) => { 507 | const t = setInterval(() => { 508 | const record = new Y.Map(); 509 | for (const k of keys) { 510 | record.set(k, Math.floor(Math.random() * 1000000000)) 511 | } 512 | 513 | yrecordlist.insert(yrecordlist.length, [record]); 514 | if (i % env.LARGE_DOC_CHECK_DOC_SIZE_PER_ITER === 0) { 515 | docSizeInBytes = Y.encodeStateAsUpdate(ydoc).length; 516 | } 517 | 518 | logUpdate(`inserted record: ${i++}\ndoc size in MB: ${docSizeInBytes / (1024 * 1024)}`) 519 | if (docSizeInBytes >= env.LARGE_DOC_MAX_DOC_SIZE_MB * 1024 * 1024) { 520 | clearInterval(t); 521 | resolve(); 522 | } 523 | }, env.LARGE_DOC_YDOC_WRITE_INTERVAL_MS); 524 | }) 525 | 526 | await writesPromise; 527 | await Promise.all(onYDocUpdatePromises); 528 | await wait(env.LARGE_DOC_YDOC_READ_TIMEOUT_MS); 529 | const res = await api.get(`/docs/${docId}/updates`, { responseType: 'arraybuffer' }); 530 | const update = new Uint8Array(res.data); 531 | const ydoc2 = new Y.Doc(); 532 | Y.applyUpdate(ydoc2, update); 533 | const yrecordlist2 = ydoc2.getArray>('record_list') 534 | 535 | const mismatches: {index: number, key: string, want: number | undefined, got: number | undefined}[] = []; 536 | 537 | log('matching ydocs') 538 | for (let i = 0; i < yrecordlist.length; i++) { 539 | const yrecordwant = yrecordlist.get(i); 540 | const yrecordgot = yrecordlist2.get(i); 541 | 542 | logUpdate(`matching yrecord: ${i}`) 543 | 544 | for (const k of keys) { 545 | const want = yrecordwant.get(k) 546 | const got = yrecordgot.get(k); 547 | 548 | if (want !== got) { 549 | mismatches.push({index: i, key: k, want, got}) 550 | } 551 | } 552 | } 553 | 554 | log('mismatches', mismatches) 555 | expect(mismatches.length).to.eq(0); 556 | 557 | log('comparing 2 read responses', update.length) 558 | 559 | const res2 = await api.get(`/docs/${docId}/updates`, { responseType: 'arraybuffer' }); 560 | const update2 = new Uint8Array(res2.data); 561 | const ydoc3 = new Y.Doc(); 562 | 563 | Y.applyUpdate(ydoc3, update2); 564 | const yrecordlist3 = ydoc3.getArray>('record_list') 565 | 566 | for (let i=0; i < yrecordlist2.length; i++) { 567 | const yrecordwant = yrecordlist2.get(i); 568 | const yrecordgot = yrecordlist3.get(i); 569 | 570 | logUpdate(`matching yrecord: ${i}`) 571 | 572 | for (const k of keys) { 573 | const want = yrecordwant.get(k) 574 | const got = yrecordgot.get(k); 575 | 576 | expect(want).eq(got); 577 | } 578 | } 579 | }) 580 | }) 581 | }) -------------------------------------------------------------------------------- /turbo.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://turbo.build/schema.json", 3 | "tasks": { 4 | "dev": { 5 | "cache": false, 6 | "dependsOn": [ 7 | "dev#dev", 8 | "server#dev" 9 | ] 10 | }, 11 | "build": { 12 | "cache": false, 13 | "dependsOn": [ 14 | "server#build" 15 | ] 16 | }, 17 | "test": { 18 | "cache": false, 19 | "dependsOn": [ 20 | "test#test" 21 | ] 22 | }, 23 | "server": { 24 | "cache": false, 25 | "dependsOn": [ 26 | "server#server" 27 | ] 28 | }, 29 | "dev#dev": { 30 | "cache": false, 31 | "dependsOn": [] 32 | }, 33 | "dev#healthz": { 34 | "cache": false, 35 | "dependsOn": [] 36 | }, 37 | "dev#down": { 38 | "cache": false, 39 | "dependsOn": [] 40 | }, 41 | "server#setup_ffi": { 42 | "cache": false, 43 | "dependsOn": [] 44 | }, 45 | "server#dev": { 46 | "cache": false, 47 | "dependsOn": [ 48 | "server#setup_ffi", 49 | "dev#healthz" 50 | ] 51 | }, 52 | "server#build": { 53 | "cache": false, 54 | "dependsOn": [ 55 | "server#setup_ffi" 56 | ] 57 | }, 58 | "server#server": { 59 | "cache": false, 60 | "dependsOn": [ 61 | "server#build" 62 | ] 63 | }, 64 | "test#test": { 65 | "cache": false, 66 | "dependsOn": [] 67 | }, 68 | "test#test:compaction:stability": { 69 | "cache": false, 70 | "dependsOn": [] 71 | } 72 | } 73 | } 74 | --------------------------------------------------------------------------------